Changeset 96879 in vbox for trunk/src/VBox/VMM/VMMAll

Timestamp:

Sep 26, 2022 5:43:43 PM (2 years ago)

Author:

vboxsync

Message:

VMM/PGM: Nested VMX: bugref:10092 Nested EPT shadow page-pool handling.

Location:

trunk/src/VBox/VMM/VMMAll

Files:

• PGMAll.cpp (modified) (25 diffs)
• PGMAllBth.h (modified) (15 diffs)
• PGMAllPool.cpp (modified) (34 diffs)
• PGMAllShw.h (modified) (7 diffs)

trunk/src/VBox/VMM/VMMAll/PGMAll.cpp

@@ -66 +66 @@
 static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested, PPGMPTWALK pWalk,
                           PPGMPTWALKGST pGstWalk);
-static int pgmGstSlatWalkPhys(PVMCPUCC pVCpu, PGMSLAT enmSlatMode, RTGCPHYS GCPhysNested, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk);
+static int pgmGstSlatWalkPhys(PVMCPUCC pVCpu, PGMSLAT enmSlatMode, RTGCPHYS GCPhysNested, PPGMPTWALK pWalk,
+                              PPGMPTWALKGST pGstWalk);
 static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3);
+static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT *ppPdpt, PEPTPD *ppPD,
+                                   PPGMPTWALKGST pGstWalkAll);
 #endif
 static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
@@ -74 +77 @@
 
 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
-/* Guest - EPT SLAT is identical for all guest paging mode. */
 # define PGM_SLAT_TYPE               PGM_SLAT_TYPE_EPT
-# define PGM_GST_TYPE                PGM_TYPE_EPT
-# include "PGMGstDefs.h"
+# include "PGMSlatDefs.h"
 # include "PGMAllGstSlatEpt.cpp.h"
-# undef PGM_GST_TYPE
+# undef PGM_SLAT_TYPE
 #endif
 
@@ -899 +900 @@
 #undef PGMMODEDATABTH_NULL_ENTRY
 };
+
+
+/**
+ * Gets the CR3 mask corresponding to the given paging mode.
+ *
+ * @returns The CR3 mask.
+ * @param   enmMode         The paging mode.
+ * @param   enmSlatMode     The second-level address translation mode.
+ */
+DECLINLINE(uint64_t) pgmGetCr3MaskForMode(PGMMODE enmMode, PGMSLAT enmSlatMode)
+{
+    /** @todo This work can be optimized either by storing the masks in
+     *        pVCpu->pgm.s.afGstCr3Masks[] for all PGMMODEs -or- just do this once and
+     *        store the result when entering guest mode since we currently use it only
+     *        for enmGuestMode. */
+    if (enmSlatMode == PGMSLAT_DIRECT)
+    {
+        Assert(enmMode != PGMMODE_EPT);
+        switch (enmMode)
+        {
+            case PGMMODE_PAE:
+            case PGMMODE_PAE_NX:
+                return X86_CR3_PAE_PAGE_MASK;
+            case PGMMODE_AMD64:
+            case PGMMODE_AMD64_NX:
+                return X86_CR3_AMD64_PAGE_MASK;
+            default:
+                return X86_CR3_PAGE_MASK;
+        }
+    }
+    else
+    {
+        Assert(enmSlatMode == PGMSLAT_EPT);
+        return X86_CR3_EPT_PAGE_MASK;
+    }
+}
+
+
+/**
+ * Gets the masked CR3 value according to the current guest paging mode.
+ *
+ * @returns The masked PGM CR3 value.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   uCr3    The raw guest CR3 value.
+ */
+DECLINLINE(RTGCPHYS) pgmGetGuestMaskedCr3(PVMCPUCC pVCpu, uint64_t uCr3)
+{
+    uint64_t const fCr3Mask  = pgmGetCr3MaskForMode(pVCpu->pgm.s.enmGuestMode, pVCpu->pgm.s.enmGuestSlatMode);
+    RTGCPHYS       GCPhysCR3 = (RTGCPHYS)(uCr3 & fCr3Mask);
+    PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
+    return GCPhysCR3;
+}
 
 
@@ -1670 +1723 @@
 
 
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/**
+ * Syncs the SHADOW nested-guest page directory pointer for the specified address.
+ * Allocates backing pages in case the PDPT or PML4 entry is missing.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPhysNested    The nested-guest physical address.
+ * @param   ppPdpt          Where to store the PDPT. Optional, can be NULL.
+ * @param   ppPD            Where to store the PD. Optional, can be NULL.
+ * @param   pGstWalkAll     The guest walk info.
+ */
+static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT *ppPdpt, PEPTPD *ppPD,
+                                   PPGMPTWALKGST pGstWalkAll)
+{
+    PVMCC    pVM   = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    int      rc;
+
+    PPGMPOOLPAGE pShwPage;
+    Assert(pVM->pgm.s.fNestedPaging);
+    Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * PML4 level.
+     */
+    {
+        PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+        Assert(pPml4);
+
+        /* Allocate page directory pointer table if not present. */
+        {
+            uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pml4e.u & pVCpu->pgm.s.fGstEptShadowedPml4eMask;
+            const unsigned iPml4e    = (GCPhysNested >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
+            PEPTPML4E      pPml4e    = &pPml4->a[iPml4e];
+
+            if (!(pPml4e->u & (EPT_E_PG_MASK | EPT_PRESENT_MASK)))
+            {
+                RTGCPHYS const GCPhysPdpt = pGstWalkAll->u.Ept.Pml4e.u & EPT_PML4E_PG_MASK;
+                rc = pgmPoolAlloc(pVM, GCPhysPdpt, PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT, PGMPOOLACCESS_DONTCARE,
+                                  PGM_A20_IS_ENABLED(pVCpu), pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4e, false /*fLockPage*/,
+                                  &pShwPage);
+                AssertRCReturn(rc, rc);
+
+                /* Hook up the new PDPT now. */
+                ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | fShwFlags);
+            }
+            else
+            {
+                pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
+                AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+                pgmPoolCacheUsed(pPool, pShwPage);
+
+                /* Hook up the cached PDPT if needed (probably not given 512*512 PTs to sync). */
+                if (pPml4e->u != (pShwPage->Core.Key | fShwFlags))
+                    ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | fShwFlags);
+            }
+            Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
+            Log7Func(("GstPml4e=%RX64 ShwPml4e=%RX64 iPml4e=%u\n", pGstWalkAll->u.Ept.Pml4e.u, pPml4e->u, iPml4e));
+        }
+    }
+
+    /*
+     * PDPT level.
+     */
+    {
+        AssertReturn(!(pGstWalkAll->u.Ept.Pdpte.u & EPT_E_LEAF), VERR_NOT_SUPPORTED); /* shadowing 1GB pages not supported yet. */
+
+        PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+        if (ppPdpt)
+            *ppPdpt = pPdpt;
+
+        uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pdpte.u & pVCpu->pgm.s.fGstEptShadowedPdpteMask;
+        const unsigned iPdPte    = (GCPhysNested >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
+        PEPTPDPTE      pPdpte    = &pPdpt->a[iPdPte];
+
+        if (!(pPdpte->u & (EPT_E_PG_MASK | EPT_PRESENT_MASK)))
+        {
+            RTGCPHYS const GCPhysPd = pGstWalkAll->u.Ept.Pdpte.u & EPT_PDPTE_PG_MASK;
+            rc = pgmPoolAlloc(pVM, GCPhysPd, PGMPOOLKIND_EPT_PD_FOR_EPT_PD, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                              pShwPage->idx, iPdPte, false /*fLockPage*/, &pShwPage);
+            AssertRCReturn(rc, rc);
+
+            /* Hook up the new PD now. */
+            ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key | fShwFlags);
+        }
+        else
+        {
+            pShwPage = pgmPoolGetPage(pPool, pPdpte->u & EPT_PDPTE_PG_MASK);
+            AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+            pgmPoolCacheUsed(pPool, pShwPage);
+
+            /* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
+            if (pPdpte->u != (pShwPage->Core.Key | fShwFlags))
+                ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key | fShwFlags);
+        }
+        Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
+        Log7Func(("GstPdpte=%RX64 ShwPdpte=%RX64 iPdPte=%u \n", pGstWalkAll->u.Ept.Pdpte.u, pPdpte->u, iPdPte));
+
+        *ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    }
+
+    return VINF_SUCCESS;
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
+
+
 #ifdef IN_RING0
 /**
@@ -1787 +1950 @@
     uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
     AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
-    AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
     return g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu);
 }
@@ -2135 +2298 @@
     Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd));
 
-    RTGCPHYS    GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAGE_MASK;
+    RTGCPHYS    GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
     PPGMPAGE    pPage;
     int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
@@ -2176 +2339 @@
     PGM_LOCK_VOID(pVM);
 
-    RTGCPHYS    GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAE_PAGE_MASK;
+    RTGCPHYS    GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
     PPGMPAGE    pPage;
-    /** @todo Nested VMX: convert GCPhysCR3 from nested-guest physical to
-     *        guest-physical address here. */
     int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
     if (RT_SUCCESS(rc))
@@ -2272 +2433 @@
     PGM_LOCK_VOID(pVM);
 
-    RTGCPHYS    GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_AMD64_PAGE_MASK;
+    RTGCPHYS    GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
     PPGMPAGE    pPage;
     int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
@@ -2369 +2530 @@
 
 
-/**
- * Gets the CR3 mask corresponding to the given paging mode.
- *
- * @returns The CR3 mask.
- * @param   enmMode     The paging mode.
- */
-DECLINLINE(uint64_t) pgmGetCr3MaskForMode(PGMMODE enmMode)
-{
-    /** @todo This work can be optimized either by storing the masks in
-     *        pVCpu->pgm.s.afGstCr3Masks[] for all PGMMODEs -or- just do this once and
-     *        store the result when entering guest mode since we currently use it only
-     *        for enmGuestMode. */
-    switch (enmMode)
-    {
-        case PGMMODE_PAE:
-        case PGMMODE_PAE_NX:
-            return X86_CR3_PAE_PAGE_MASK;
-        case PGMMODE_AMD64:
-        case PGMMODE_AMD64_NX:
-            return X86_CR3_AMD64_PAGE_MASK;
-        case PGMMODE_EPT:
-            return X86_CR3_EPT_PAGE_MASK;
-        default:
-            return X86_CR3_PAGE_MASK;
-    }
-}
-
-
-/**
- * Gets the masked CR3 value according to the current guest paging mode.
- *
- * @returns The masked PGM CR3 value.
- * @param   pVCpu   The cross context virtual CPU structure.
- * @param   uCr3    The raw guest CR3 value.
- */
-DECLINLINE(RTGCPHYS) pgmGetGuestMaskedCr3(PVMCPUCC pVCpu, uint64_t uCr3)
-{
-    uint64_t const fCr3Mask = pgmGetCr3MaskForMode(pVCpu->pgm.s.enmGuestMode);
-    RTGCPHYS      GCPhysCR3 = (RTGCPHYS)(uCr3 & fCr3Mask);
-    PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
-    return GCPhysCR3;
-}
-
-
 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
 /**
@@ -2429 +2546 @@
 static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3)
 {
+# if 0
     if (uCr3 != pVCpu->pgm.s.GCPhysNstGstCR3)
+# endif
     {
         PGMPTWALK    Walk;
@@ -2449 +2568 @@
     }
 
+# if 0
     /*
      * If the nested-guest CR3 has not changed, then the previously
@@ -2455 +2575 @@
     *pGCPhysCr3 = pVCpu->pgm.s.GCPhysCR3;
     return VINF_SUCCESS;
+# endif
 }
 #endif
@@ -2495 +2616 @@
         && PGMMODE_WITH_PAGING(pVCpu->pgm.s.enmGuestMode))
     {
-        LogFlowFunc(("nested_cr3=%RX64 old=%RX64\n", GCPhysCR3, pVCpu->pgm.s.GCPhysNstGstCR3));
         RTGCPHYS GCPhysOut;
         int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
@@ -2601 +2721 @@
     if (CPUMIsGuestVmxEptPagingEnabled(pVCpu))
     {
-        LogFlowFunc(("nested_cr3=%RX64 old_nested_cr3=%RX64\n", cr3, pVCpu->pgm.s.GCPhysNstGstCR3));
         RTGCPHYS GCPhysOut;
         int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
@@ -2610 +2729 @@
             /* CR3 SLAT translation failed but we try to pretend it
                succeeded for the reasons mentioned in PGMHCChangeMode(). */
-            AssertMsgFailed(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
+            Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
             int const rc2 = pgmGstUnmapCr3(pVCpu);
             pVCpu->pgm.s.GCPhysCR3       = NIL_RTGCPHYS;
             pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
             return rc2;
         }
@@ -2717 +2837 @@
                    succeeded for the reasons mentioned in PGMHCChangeMode(). */
                 AssertMsgFailed(("Failed to translate CR3 %#RX64. rc=%Rrc\n", cr3, rc2));
-                rc2 = pgmGstUnmapCr3(pVCpu);
                 pVCpu->pgm.s.GCPhysCR3       = NIL_RTGCPHYS;
                 pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
@@ -2811 +2930 @@
          * PDPE entries. Here we assume the caller has already validated or doesn't require
          * validation of the PDPEs.
+         *
+         * In the case of nested EPT (i.e. for nested-guests), the PAE PDPEs have been
+         * validated by the VMX transition.
          *
          * [1] -- See AMD spec. 15.25.10 "Legacy PAE Mode".
@@ -2844 +2966 @@
     }
 
+    /*
+     * Update CPUM with the PAE PDPEs.
+     */
+    CPUMSetGuestPaePdpes(pVCpu, paPaePdpes);
     return VINF_SUCCESS;
 }
@@ -2868 +2994 @@
 
 #ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
-    if (CPUMIsGuestVmxEptPaePagingEnabled(pVCpu))
+    if (CPUMIsGuestVmxEptPagingEnabled(pVCpu))
     {
         RTGCPHYS GCPhysOut;
@@ -2908 +3034 @@
 
             /*
-             * Update CPUM.
-             * We do this prior to mapping the PDPEs to keep the order consistent
-             * with what's used in HM. In practice, it doesn't really matter.
-             */
-            CPUMSetGuestPaePdpes(pVCpu, &aPaePdpes[0]);
-
-            /*
-             * Map the PDPEs.
+             * Map the PDPEs and update CPUM.
              */
             rc = PGMGstMapPaePdpes(pVCpu, &aPaePdpes[0]);
@@ -3320 +3439 @@
 
     /*
+     * Determine SLAT mode -before- entering the new shadow mode!
+     */
+    pVCpu->pgm.s.enmGuestSlatMode = !CPUMIsGuestVmxEptPagingEnabled(pVCpu) ? PGMSLAT_DIRECT : PGMSLAT_EPT;
+
+    /*
      * Enter new shadow mode (if changed).
      */
@@ -3380 +3504 @@
      *   - Indicate that the CR3 is nested-guest physical address.
      */
-    if (CPUMIsGuestVmxEptPagingEnabled(pVCpu))
+    if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
     {
         if (PGMMODE_WITH_PAGING(enmGuestMode))
@@ -3405 +3529 @@
                  * See Intel spec. 27.2.1 "EPT Overview".
                  */
-                AssertMsgFailed(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", GCPhysCR3, rc));
+                Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", GCPhysCR3, rc));
 
                 /* Trying to coax PGM to succeed for the time being... */
                 Assert(pVCpu->pgm.s.GCPhysCR3 == NIL_RTGCPHYS);
                 pVCpu->pgm.s.GCPhysNstGstCR3  = GCPhysCR3;
-                pVCpu->pgm.s.enmGuestSlatMode = PGMSLAT_EPT;
                 pVCpu->pgm.s.enmGuestMode     = enmGuestMode;
                 HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
@@ -3416 +3539 @@
             }
             pVCpu->pgm.s.GCPhysNstGstCR3  = GCPhysCR3;
-            GCPhysCR3 = Walk.GCPhys;
-        }
-        pVCpu->pgm.s.enmGuestSlatMode = PGMSLAT_EPT;
+            GCPhysCR3 = Walk.GCPhys & X86_CR3_EPT_PAGE_MASK;
+        }
     }
     else
-    {
         Assert(pVCpu->pgm.s.GCPhysNstGstCR3 == NIL_RTGCPHYS);
-        pVCpu->pgm.s.enmGuestSlatMode = PGMSLAT_DIRECT;
-    }
 #endif
 
@@ -3953 +4072 @@
     PGM_LOCK_VOID(pVM);
     pVCpu->pgm.s.uEptPtr = uEptPtr;
+    pVCpu->pgm.s.pGstEptPml4R3 = 0;
+    pVCpu->pgm.s.pGstEptPml4R0 = 0;
     PGM_UNLOCK(pVM);
 }

trunk/src/VBox/VMM/VMMAll/PGMAllBth.h

@@ -47 +47 @@
 #ifndef IN_RING3
 PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
-PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysNested,
-                                       bool fIsLinearAddrValid, RTGCPTR GCPtrNested, PPGMPTWALK pWalk, bool *pfLockTaken);
+PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysNestedFault,
+                                       bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk, bool *pfLockTaken);
+# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT
+static void PGM_BTH_NAME(NestedSyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPte, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage,
+                                               unsigned iPte, PPGMPTWALKGST pGstWalkAll);
+static int  PGM_BTH_NAME(NestedSyncPage)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, unsigned cPages,
+                                         uint32_t uErr, PPGMPTWALKGST pGstWalkAll);
+static int  PGM_BTH_NAME(NestedSyncPT)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, PPGMPTWALKGST pGstWalkAll);
+# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
 #endif
 PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
@@ -983 +990 @@
 
 
+# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT)
+/**
+ * Deals with a nested-guest \#PF fault for a guest-physical page with a handler.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   uErr                The error code.
+ * @param   pRegFrame           The register frame.
+ * @param   GCPhysNestedFault   The nested-guest physical address of the fault.
+ * @param   pPage               The guest page at @a GCPhysNestedFault.
+ * @param   GCPhysFault         The guest-physical address of the fault.
+ * @param   pGstWalkAll         The guest page walk result.
+ * @param   pfLockTaken         Where to store whether the PGM is still held when
+ *                              this function completes.
+ *
+ * @note    The caller has taken the PGM lock.
+ */
+static VBOXSTRICTRC PGM_BTH_NAME(NestedTrap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
+                                                                      RTGCPHYS GCPhysNestedFault, PPGMPAGE pPage,
+                                                                      RTGCPHYS GCPhysFault, PPGMPTWALKGST pGstWalkAll,
+                                                                      bool *pfLockTaken)
+{
+#  if PGM_GST_TYPE == PGM_TYPE_PROT \
+   && PGM_SHW_TYPE == PGM_TYPE_EPT
+
+    /** @todo Assert uErr isn't X86_TRAP_PF_RSVD and remove release checks. */
+    PGM_A20_ASSERT_MASKED(pVCpu, GCPhysFault);
+    AssertMsgReturn(PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage), ("%RGp %RGp uErr=%u\n", GCPhysNestedFault, GCPhysFault, uErr),
+                    VERR_PGM_HANDLER_IPE_1);
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    RTGCPHYS const GCPhysNestedPage = GCPhysNestedFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+    RTGCPHYS const GCPhysPage       = GCPhysFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+
+    /*
+     * Physical page access handler.
+     */
+    PPGMPHYSHANDLER pCur;
+    VBOXSTRICTRC rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhysPage, &pCur);
+    AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
+
+    PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+    Assert(pCurType);
+
+    /*
+     * If the region is write protected and we got a page not present fault, then sync
+     * the pages. If the fault was caused by a read, then restart the instruction.
+     * In case of write access continue to the GC write handler.
+     */
+    if (   !(uErr & X86_TRAP_PF_P)
+        &&  pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
+    {
+        Log7Func(("Syncing Monitored: GCPhysNestedPage=%RGp GCPhysPage=%RGp uErr=%#x\n", GCPhysNestedPage, GCPhysPage, uErr));
+        rcStrict = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /*cPages*/, uErr, pGstWalkAll);
+        Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE);
+        if (    RT_FAILURE(rcStrict)
+            || !(uErr & X86_TRAP_PF_RW))
+        {
+            AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
+            STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
+            return rcStrict;
+        }
+    }
+    else if (   !(uErr & X86_TRAP_PF_RSVD)
+             && pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE)
+    {
+        /*
+         * If the access was NOT through an EPT misconfig (i.e. RSVD), sync the page.
+         * This can happen for the VMX APIC-access page.
+         */
+        Log7Func(("Syncing MMIO: GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
+        rcStrict = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /*cPages*/, uErr, pGstWalkAll);
+        Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE);
+        if (RT_FAILURE(rcStrict))
+        {
+            AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
+            STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
+            return rcStrict;
+        }
+    }
+
+    AssertMsg(   pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
+              || (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
+              ("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
+               GCPhysNestedFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
+    if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysWrite);
+    else
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAll);
+        if (uErr & X86_TRAP_PF_RSVD)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAllOpt);
+    }
+
+    if (pCurType->pfnPfHandler)
+    {
+        STAM_PROFILE_START(&pCur->Stat, h);
+        uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
+                             : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
+
+        if (pCurType->fKeepPgmLock)
+        {
+            rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pRegFrame, GCPhysNestedFault, GCPhysFault, uUser);
+            STAM_PROFILE_STOP(&pCur->Stat, h);
+        }
+        else
+        {
+            PGM_UNLOCK(pVM);
+            *pfLockTaken = false;
+            rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pRegFrame, GCPhysNestedFault, GCPhysFault, uUser);
+            STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("What's going on here!? Fault falls outside handler range!?\n"));
+        rcStrict = VINF_EM_RAW_EMULATE_INSTR;
+    }
+
+    STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndPhys; });
+    return rcStrict;
+
+#  else
+    RT_NOREF8(pVCpu, uErr, pRegFrame, GCPhysNestedFault, pPage, GCPhysFault, pGstWalkAll, pfLockTaken);
+    AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
+    return VERR_PGM_NOT_USED_IN_MODE;
+#  endif
+}
+# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
+
+
 /**
  * Nested \#PF handler for nested-guest hardware-assisted execution using nested
@@ -991 +1131 @@
  * @param   uErr                The fault error (X86_TRAP_PF_*).
  * @param   pRegFrame           The register frame.
- * @param   GCPhysNested        The nested-guest physical address being accessed.
+ * @param   GCPhysNestedFault   The nested-guest physical address of the fault.
  * @param   fIsLinearAddrValid  Whether translation of a nested-guest linear address
- *                              caused this fault. If @c false, GCPtrNested must be
- *                              0.
- * @param   GCPtrNested         The nested-guest linear address that caused this
- *                              fault.
+ *                              caused this fault. If @c false, GCPtrNestedFault
+ *                              must be 0.
+ * @param   GCPtrNestedFault    The nested-guest linear address of this fault.
  * @param   pWalk               The guest page table walk result.
  * @param   pfLockTaken         Where to store whether the PGM lock is still held
  *                              when this function completes.
  */
-PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysNested,
-                                       bool fIsLinearAddrValid, RTGCPTR GCPtrNested, PPGMPTWALK pWalk, bool *pfLockTaken)
+PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysNestedFault,
+                                       bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk, bool *pfLockTaken)
 {
     *pfLockTaken = false;
 # if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) \
-    && (   PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_32BIT  \
-        || PGM_GST_TYPE == PGM_TYPE_PAE  || PGM_GST_TYPE == PGM_TYPE_AMD64) \
+    && PGM_GST_TYPE == PGM_TYPE_PROT \
     && PGM_SHW_TYPE == PGM_TYPE_EPT
 
     Assert(CPUMIsGuestVmxEptPagingEnabled(pVCpu));
+    Assert(PGM_A20_IS_ENABLED(pVCpu));
+
+    /* We don't support mode-based execute control for EPT yet. */
+    Assert(!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fVmxModeBasedExecuteEpt);
+    Assert(!(uErr & X86_TRAP_PF_US));
+
+    /* Take the big lock now. */
+    *pfLockTaken = true;
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PGM_LOCK_VOID(pVM);
 
     /*
      * Walk the guest EPT tables and check if it's an EPT violation or misconfiguration.
      */
+    Log7Func(("cs:rip=%04x:%#RX64 GCPhysNestedFault=%RGp\n", pRegFrame->cs.Sel, pRegFrame->rip, GCPhysNestedFault));
     PGMPTWALKGST GstWalkAll;
-    int rc = pgmGstSlatWalk(pVCpu, GCPhysNested, fIsLinearAddrValid, GCPtrNested, pWalk, &GstWalkAll);
+    int rc = pgmGstSlatWalk(pVCpu, GCPhysNestedFault, fIsLinearAddrValid, GCPtrNestedFault, pWalk, &GstWalkAll);
     if (RT_FAILURE(rc))
         return rc;
@@ -1022 +1171 @@
     Assert(GstWalkAll.enmType == PGMPTWALKGSTTYPE_EPT);
     Assert(pWalk->fSucceeded);
-    Assert(pWalk->fEffective & PGM_PTATTRS_R_MASK);
+    Assert(pWalk->fEffective & (PGM_PTATTRS_EPT_R_MASK | PGM_PTATTRS_EPT_W_MASK | PGM_PTATTRS_EPT_X_SUPER_MASK));
     Assert(pWalk->fIsSlat);
 
-    if (uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID))
-    {
-        if (    (   (uErr & X86_TRAP_PF_RW)
-                 && !(pWalk->fEffective & PGM_PTATTRS_W_MASK)
-                 && (   (uErr & X86_TRAP_PF_US)
-                     || CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
-            ||  ((uErr & X86_TRAP_PF_US) && !(pWalk->fEffective & PGM_PTATTRS_US_MASK))
-            ||  ((uErr & X86_TRAP_PF_ID) &&  (pWalk->fEffective & PGM_PTATTRS_NX_MASK))
-           )
-            return VERR_ACCESS_DENIED;
-    }
-
-    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
-    RTGCPHYS const GCPhysFault = PGM_A20_APPLY(pVCpu, GCPhysNested & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
-    GSTPDE const   PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
-
-    /* Take the big lock now. */
-    *pfLockTaken = true;
-    PGM_LOCK_VOID(pVM);
-
-    /*
-     * Check if this is an APIC-access page access (VMX specific).
-     */
-    RTGCPHYS const GCPhysApicAccess = CPUMGetGuestVmxApicAccessPageAddr(pVCpu);
-    if ((pWalk->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK) == GCPhysApicAccess)
-    {
-        PPGMPAGE pPage;
-        rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPhysApicAccess), &pPage);
-        if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
-        {
-            rc = VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pWalk->GCPhys, pPage,
-                                                                               pfLockTaken));
-            return rc;
-        }
-    }
-
-#  ifdef PGM_WITH_MMIO_OPTIMIZATIONS
-    /*
-    * Check if this is an MMIO access.
-    */
-    if (uErr & X86_TRAP_PF_RSVD)
-    {
-        PPGMPAGE pPage;
-        rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)GCPhysFault), &pPage);
-        if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
-            return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, GCPhysFault, pPage,
-                                                                                 pfLockTaken));
-        rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, GCPhysFault, 1, uErr);
-        AssertRC(rc);
-        HMInvalidatePhysPage(pVM, GCPhysFault);
-        return rc; /* Restart with the corrected entry. */
-    }
-#  endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
-
-    /*
-     * Fetch the guest EPT page directory pointer.
-     */
-    const unsigned  iPDDst = ((GCPhysFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
-    PEPTPD          pPDDst;
-    rc = pgmShwGetEPTPDPtr(pVCpu, GCPhysFault, NULL /* ppPdpt */, &pPDDst);
-    AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
-    Assert(pPDDst);
+    /* Paranoia: Remove later. */
+    Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_R_MASK)  ==  RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_R_MASK));
+    Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_W_MASK)  ==  RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_W_MASK));
+    Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_NX_MASK) == !RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_X_SUPER_MASK));
+
+    /*
+     * Check page-access permissions.
+     */
+    if (   ((uErr & X86_TRAP_PF_RW) && !(pWalk->fEffective & PGM_PTATTRS_W_MASK))
+        || ((uErr & X86_TRAP_PF_ID) &&  (pWalk->fEffective & PGM_PTATTRS_NX_MASK)))
+    {
+        Log7Func(("Permission failed! GCPtrNested=%RGv GCPhysNested=%RGp uErr=%#x fEffective=%#RX64\n", GCPtrNestedFault,
+                  GCPhysNestedFault, uErr, pWalk->fEffective));
+        pWalk->fFailed = PGM_WALKFAIL_EPT_VIOLATION;
+        return VERR_ACCESS_DENIED;
+    }
+
+    PGM_A20_ASSERT_MASKED(pVCpu, pWalk->GCPhys);
+    RTGCPHYS const GCPhysPage       = pWalk->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+    RTGCPHYS const GCPhysNestedPage = GCPhysNestedFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+
+    /*
+     * If we were called via an EPT misconfig, it should've already resulted in a nested-guest VM-exit.
+     */
+    AssertMsgReturn(!(uErr & X86_TRAP_PF_RSVD),
+                    ("Unexpected EPT misconfig VM-exit. GCPhysPage=%RGp GCPhysNestedPage=%RGp\n", GCPhysPage, GCPhysNestedPage),
+                    VERR_PGM_MAPPING_IPE);
+
+    /*
+     * Fetch and sync the nested-guest EPT page directory pointer.
+     */
+    PEPTPD pEptPd;
+    rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, NULL /*ppPdpt*/, &pEptPd, &GstWalkAll);
+    AssertRCReturn(rc, rc);
+    Assert(pEptPd);
 
     /*
@@ -1093 +1215 @@
      * It is IMPORTANT that we weed out any access to non-present shadow PDEs
      * here so we can safely assume that the shadow PT is present when calling
-     * SyncPage later.
+     * NestedSyncPage later.
      *
-     * On failure, we ASSUME that SyncPT is out of memory or detected some kind
-     * of mapping conflict and defer to SyncCR3 in R3.
-     * (Again, we do NOT support access handlers for non-present guest pages.)
-     *
-     */
-    if (   !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
-        && !SHW_PDE_IS_P(pPDDst->a[iPDDst]))
+     * NOTE: It's possible we will be syncing the VMX APIC-access page here.
+     * In that case, we would sync the page but will NOT go ahead with emulating
+     * the APIC-access VM-exit through IEM. However, once the page is mapped in
+     * the shadow tables, subsequent APIC-access VM-exits for the nested-guest
+     * will be triggered by hardware. Maybe calling the IEM #PF handler can be
+     * considered as an optimization later.
+     */
+    unsigned const iPde = (GCPhysNestedPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    if (   !(uErr & X86_TRAP_PF_P)
+        && !(pEptPd->a[iPde].u & EPT_PRESENT_MASK))
     {
         STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2SyncPT; });
-        LogFlow(("=>SyncPT GCPhysFault=%RGp\n", GCPhysFault));
-        rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0 /* iPDSrc */, NULL /* pPDSrc */, GCPhysFault);
+        Log7Func(("NestedSyncPT: Lazy. GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
+        rc = PGM_BTH_NAME(NestedSyncPT)(pVCpu, GCPhysNestedPage, GCPhysPage, &GstWalkAll);
         if (RT_SUCCESS(rc))
             return rc;
-        Log(("SyncPT: %RGp failed!! rc=%Rrc\n", GCPhysFault, rc));
-        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
-        return VINF_PGM_SYNC_CR3;
-    }
-
-    /*
-     * Check if this fault address is flagged for special treatment,
-     * which means we'll have to figure out the physical address and
-     * check flags associated with it.
+        AssertMsgFailedReturn(("NestedSyncPT: %RGv failed! rc=%Rrc\n", GCPhysNestedPage, rc), VERR_PGM_MAPPING_IPE);
+    }
+
+    /*
+     * Check if this fault address is flagged for special treatment.
+     * This handles faults on an MMIO or write-monitored page.
      *
-     * ASSUME that we can limit any special access handling to pages
-     * in page tables which the guest believes to be present.
+     * If this happens to be the VMX APIC-access page, we sync it in the shadow tables
+     * and emulate the APIC-access VM-exit by calling IEM's VMX APIC-access #PF handler
+     * registered for the page. Once the page is mapped in the shadow tables, subsequent
+     * APIC-access VM-exits for the nested-guest will be triggered by hardware.
      */
     PPGMPAGE pPage;
-    rc = pgmPhysGetPageEx(pVM, GCPhysFault, &pPage);
-    if (RT_FAILURE(rc))
-    {
-        /*
-         * When the guest accesses invalid physical memory (e.g. probing
-         * of RAM or accessing a remapped MMIO range), then we'll fall
-         * back to the recompiler to emulate the instruction.
-         */
-        LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhysFault, rc));
-        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersInvalid);
-        STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2InvalidPhys; });
-        return VINF_EM_RAW_EMULATE_INSTR;
-    }
-
-    /*
-     * Any handlers for this page?
-     */
+    rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
+    AssertRCReturn(rc, rc);
     if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
-        return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, GCPhysFault, pPage,
-                                                                             pfLockTaken));
-
-    /*
-     * We are here only if page is present in Guest page tables and
-     * trap is not handled by our handlers.
-     *
-     * Check it for page out-of-sync situation.
+    {
+        Log7Func(("MMIO: Calling NestedTrap0eHandlerDoAccessHandlers for GCPhys %RGp\n", GCPhysPage));
+        return VBOXSTRICTRC_TODO(PGM_BTH_NAME(NestedTrap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, GCPhysNestedFault,
+                                                                                   pPage, pWalk->GCPhys, &GstWalkAll,
+                                                                                   pfLockTaken));
+    }
+
+    /*
+     * We are here only if page is present in nested-guest page tables but the
+     * trap is not handled by our handlers. Check for page out-of-sync situation.
      */
     if (!(uErr & X86_TRAP_PF_P))
     {
-        /*
-         * Page is not present in our page tables. Try to sync it!
-         */
-        if (uErr & X86_TRAP_PF_US)
-            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser));
-        else /* supervisor */
-            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
-
-        if (PGM_PAGE_IS_BALLOONED(pPage))
-        {
-            /* Emulate reads from ballooned pages as they are not present in
-               our shadow page tables. (Required for e.g. Solaris guests; soft
-               ecc, random nr generator.) */
-            rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, GCPhysFault));
-            LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
-            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncBallloon));
-            STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Ballooned; });
-            return rc;
-        }
-
-        rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, GCPhysFault, PGM_SYNC_NR_PAGES, uErr);
+        Assert(!PGM_PAGE_IS_BALLOONED(pPage));
+        Assert(!(uErr & X86_TRAP_PF_US));   /* Mode-based execute not supported yet. */
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
+
+        Log7Func(("SyncPage: Not-Present: GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedFault, GCPhysPage));
+        rc = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, PGM_SYNC_NR_PAGES, uErr, &GstWalkAll);
         if (RT_SUCCESS(rc))
         {
-            /* The page was successfully synced, return to the guest. */
             STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSync; });
             return VINF_SUCCESS;
         }
     }
-    else
+    else if (uErr & X86_TRAP_PF_RW)
     {
         /*
          * Write protected pages are made writable when the guest makes the
-         * first write to it.  This happens for pages that are shared, write
+         * first write to it. This happens for pages that are shared, write
          * monitored or not yet allocated.
          *
@@ -1192 +1287 @@
          * Assume for now it only applies to the read/write flag.
          */
-        if (uErr & X86_TRAP_PF_RW)
-        {
-            /*
-             * Check if it is a read-only page.
-             */
-            if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
+        if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
+        {
+            /* This is a read-only page. */
+            AssertMsgFailed(("Failed\n"));
+
+            Assert(!PGM_PAGE_IS_ZERO(pPage));
+            AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhysPage));
+            STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; });
+
+            Log7Func(("Calling pgmPhysPageMakeWritable for GCPhysPage=%RGp\n", GCPhysPage));
+            rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
+            if (rc != VINF_SUCCESS)
             {
-                Assert(!PGM_PAGE_IS_ZERO(pPage));
-                AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhysFault));
-                STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; });
-
-                rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysFault);
-                if (rc != VINF_SUCCESS)
-                {
-                    AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc));
-                    return rc;
-                }
-                if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
-                    return VINF_EM_NO_MEMORY;
+                AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc));
+                return rc;
             }
-
-            if (uErr & X86_TRAP_PF_US)
-                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
-            else
-                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
-
-            /*
-             * Sync the page.
-             *
-             * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
-             *       page is not present, which is not true in this case.
-             */
-            rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, GCPhysFault, 1, uErr);
-            if (RT_SUCCESS(rc))
-            {
-               /*
-                * Page was successfully synced, return to guest but invalidate
-                * the TLB first as the page is very likely to be in it.
-                */
-                HMInvalidatePhysPage(pVM, GCPhysFault);
-                STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; });
-                return VINF_SUCCESS;
-            }
-        }
-    }
-
-    /*
-     * If we get here it is because something failed above, i.e. most like guru meditation time.
-     */
-    LogRelFunc(("returns rc=%Rrc GCPhysFault=%RGp uErr=%RX64 cs:rip=%04x:%08RX64\n", rc, GCPhysFault, (uint64_t)uErr,
-                pRegFrame->cs.Sel, pRegFrame->rip));
-    return rc;
+            if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
+                return VINF_EM_NO_MEMORY;
+        }
+
+        Assert(!(uErr & X86_TRAP_PF_US));   /* Mode-based execute not supported yet. */
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
+
+        /*
+         * Sync the write-protected page.
+         * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
+         *       page is not present, which is not true in this case.
+         */
+        Log7Func(("SyncPage: RW: cs:rip=%04x:%#RX64 GCPhysNestedPage=%RGp uErr=%#RX32 GCPhysPage=%RGp WalkGCPhys=%RGp\n",
+                  pRegFrame->cs.Sel, pRegFrame->rip, GCPhysNestedPage, (uint32_t)uErr, GCPhysPage, pWalk->GCPhys));
+        rc = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /* cPages */, uErr, &GstWalkAll);
+        if (RT_SUCCESS(rc))
+        {
+            HMInvalidatePhysPage(pVM, GCPhysPage);
+            STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; });
+            return VINF_SUCCESS;
+        }
+    }
+
+    /*
+     * If we get here it is because something failed above => guru meditation time.
+     */
+    LogRelFunc(("GCPhysNestedFault=%#RGp (%#RGp) uErr=%#RX32 cs:rip=%04x:%08RX64\n", rc, GCPhysNestedFault, GCPhysPage,
+                (uint32_t)uErr, pRegFrame->cs.Sel, pRegFrame->rip));
+    return VERR_PGM_MAPPING_IPE;
 
 # else
-    RT_NOREF7(pVCpu, uErr, pRegFrame, GCPhysNested, fIsLinearAddrValid, GCPtrNested, pWalk);
+    RT_NOREF7(pVCpu, uErr, pRegFrame, GCPhysNestedFault, fIsLinearAddrValid, GCPtrNestedFault, pWalk);
     AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
     return VERR_PGM_NOT_USED_IN_MODE;
@@ -1658 +1746 @@
  *
  * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
  * @param   pPage       The page in question.
+ * @param   GCPhysPage  The guest-physical address of the page.
  * @param   fPteSrc     The shadowed flags of the source PTE.  Must include the
  *                      A (accessed) bit so it can be emulated correctly.
@@ -1664 +1754 @@
  *                      does not need to be set atomically.
  */
-DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVMCC pVM, PCPGMPAGE pPage, uint64_t fPteSrc, PSHWPTE pPteDst)
+DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVMCC pVM, PVMCPUCC pVCpu, PCPGMPAGE pPage, RTGCPHYS GCPhysPage, uint64_t fPteSrc,
+                                              PSHWPTE pPteDst)
 {
-    NOREF(pVM); RT_NOREF_PV(fPteSrc);
+    RT_NOREF_PV(pVM); RT_NOREF_PV(fPteSrc); RT_NOREF_PV(pVCpu); RT_NOREF_PV(GCPhysPage);
 
     /** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
@@ -1695 +1786 @@
             )
     {
+#   if defined(VBOX_WITH_NESTED_HWVIRT_VMX) && PGM_SHW_TYPE == PGM_TYPE_EPT
+        /*
+         * If an "ALL" access handler has been registered for the VMX APIC-access page,
+         * we want to ensure EPT violations are triggered rather than EPT misconfigs
+         * as the former allows us to translate it to an APIC-access VM-exit. This is a
+         * weird case because this is not an MMIO page (it's regular guest RAM) but we
+         * want to treat it as an MMIO page wrt to trapping all accesses but we only
+         * want EPT violations for the reasons state above.
+         *
+         * NOTE! This is required even when the nested-hypervisor is not using EPT!
+         */
+        if (CPUMIsGuestVmxApicAccessPageAddr(pVCpu, GCPhysPage))
+        {
+            Log7Func(("SyncHandlerPte: VMX APIC-access page at %#RGp -> marking not present\n", GCPhysPage));
+            pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage);
+            return;
+        }
+#   endif
+
         LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
 #   if PGM_SHW_TYPE == PGM_TYPE_EPT
@@ -1826 +1936 @@
 # endif
             if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
-                PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, fGstShwPteFlags, &PteDst);
+                PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhysPage, fGstShwPteFlags, &PteDst);
             else
             {
@@ -2197 +2307 @@
                     SHWPTE PteDst;
                     if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
-                        PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
+                        PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhys, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
                     else
                         SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) | PGM_PAGE_GET_HCPHYS(pPage));
@@ -2409 +2519 @@
 
 #endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */
+
+
+#if !defined(IN_RING3) && defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT
+/**
+ * Sync a shadow page for a nested-guest page.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pPte            The shadow page table entry.
+ * @param   GCPhysPage      The guest-physical address of the page.
+ * @param   pShwPage        The shadow page of the page table.
+ * @param   iPte            The index of the page table entry.
+ * @param   pGstWalkAll     The guest page table walk result.
+ *
+ * @note    Not to be used for 2/4MB pages!
+ */
+static void PGM_BTH_NAME(NestedSyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPte, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage,
+                                               unsigned iPte, PPGMPTWALKGST pGstWalkAll)
+{
+    PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
+    Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
+    Assert(!pShwPage->fDirty);
+    Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT);
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    AssertMsg(GCPhysPage == (pGstWalkAll->u.Ept.Pte.u & EPT_PTE_PG_MASK),
+              ("GCPhys=%RGp Ept=%RX64\n", GCPhysPage, pGstWalkAll->u.Ept.Pte.u & EPT_PTE_PG_MASK));
+
+    /*
+     * Find the ram range.
+     */
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
+    AssertRCReturnVoid(rc);
+
+    Assert(!PGM_PAGE_IS_BALLOONED(pPage));
+
+# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
+    /* Make the page writable if necessary. */
+    if (   PGM_PAGE_GET_TYPE(pPage)  == PGMPAGETYPE_RAM
+        && PGM_PAGE_IS_ZERO(pPage)
+        && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
+#  ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
+        && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
+#  endif
+#  ifdef VBOX_WITH_PAGE_SHARING
+        && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
+#  endif
+       )
+    {
+        AssertMsgFailed(("GCPhysPage=%RGp\n", GCPhysPage)); /** @todo Shouldn't happen but if it does deal with it later. */
+    }
+# endif
+
+    /*
+     * Make page table entry.
+     */
+    SHWPTE Pte;
+    uint64_t const fGstShwPteFlags = pGstWalkAll->u.Ept.Pte.u & pVCpu->pgm.s.fGstEptShadowedPteMask;
+    if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
+    {
+        if (CPUMIsGuestVmxApicAccessPageAddr(pVCpu, GCPhysPage))
+        {
+            Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | fGstShwPteFlags;
+            Log7Func(("APIC-access page at %RGp -> shadowing nested-hypervisor %RX64 (%RGp)\n", GCPhysPage, fGstShwPteFlags, pShwPage->GCPhys));
+        }
+        else if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+        {
+            Assert(!CPUMIsGuestVmxApicAccessPageAddr(pVCpu, GCPhysPage));
+            if (fGstShwPteFlags & EPT_E_WRITE)
+            {
+                PGMHandlerPhysicalPageTempOff(pVCpu->CTX_SUFF(pVM), GCPhysPage, GCPhysPage);
+                Log7Func(("monitored page (%R[pgmpage]) at %RGp -> read-write, monitoring disabled\n", pPage, GCPhysPage));
+            }
+            Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | fGstShwPteFlags;
+            Log7Func(("monitored page (%R[pgmpage]) at %RGp -> shadowing nested-hypervisor %RX64\n", pPage, GCPhysPage, fGstShwPteFlags));
+        }
+        else
+        {
+            /** @todo Track using fVirtVmxApicAccess bit in PGMPHYSHANDLER and maybe in PGMPAGE
+             *        too? */
+            PGMHandlerPhysicalDeregister(pVCpu->CTX_SUFF(pVM), GCPhysPage);
+            Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | fGstShwPteFlags;
+            Log7Func(("MMIO at %RGp potentially former VMX APIC-access page -> unregistered\n", GCPhysPage));
+        }
+    }
+    else
+        Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | fGstShwPteFlags;
+
+    /* Make sure only allocated pages are mapped writable. */
+    Assert(!SHW_PTE_IS_P_RW(Pte) || PGM_PAGE_IS_ALLOCATED(pPage));
+
+    /*
+     * Keep user track up to date.
+     */
+    if (SHW_PTE_IS_P(Pte))
+    {
+        if (!SHW_PTE_IS_P(*pPte))
+            PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPte);
+        else if (SHW_PTE_GET_HCPHYS(*pPte) != SHW_PTE_GET_HCPHYS(Pte))
+        {
+            Log2(("SyncPageWorker: deref! *pPte=%RX64 Pte=%RX64\n", SHW_PTE_LOG64(*pPte), SHW_PTE_LOG64(Pte)));
+            PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
+            PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPte);
+        }
+    }
+    else if (SHW_PTE_IS_P(*pPte))
+    {
+        Log2(("SyncPageWorker: deref! *pPte=%RX64\n", SHW_PTE_LOG64(*pPte)));
+        PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
+    }
+
+    /*
+     * Commit the entry.
+     */
+    SHW_PTE_ATOMIC_SET2(*pPte, Pte);
+    return;
+}
+
+
+/**
+ * Syncs a nested-guest page.
+ *
+ * There are no conflicts at this point, neither is there any need for
+ * page table allocations.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   GCPhysNestedPage    The nested-guest physical address of the page being
+ *                              synced.
+ * @param   GCPhysPage          The guest-physical address of the page being synced.
+ * @param   cPages              Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
+ * @param   uErr                The page fault error (X86_TRAP_PF_XXX).
+ * @param   pGstWalkAll         The guest page table walk result.
+ */
+static int PGM_BTH_NAME(NestedSyncPage)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, unsigned cPages,
+                                        uint32_t uErr, PPGMPTWALKGST pGstWalkAll)
+{
+    PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
+    Assert(!(GCPhysNestedPage & GUEST_PAGE_OFFSET_MASK));
+    Assert(!(GCPhysPage & GUEST_PAGE_OFFSET_MASK));
+
+    PVMCC    pVM   = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
+    Log7Func(("GCPhysNestedPage=%RGv GCPhysPage=%RGp cPages=%u uErr=%#x\n", GCPhysNestedPage, GCPhysPage, cPages, uErr));
+    RT_NOREF_PV(uErr); RT_NOREF_PV(cPages);
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Get the shadow PDE, find the shadow page table in the pool.
+     */
+    unsigned const iPde = ((GCPhysNestedPage >> EPT_PD_SHIFT) & EPT_PD_MASK);
+    PEPTPD pPd;
+    int rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, NULL, &pPd, pGstWalkAll);
+    if (RT_SUCCESS(rc))
+    { /* likely */ }
+    else
+    {
+        Log(("Failed to fetch EPT PD for %RGp (%RGp) rc=%Rrc\n", GCPhysNestedPage, GCPhysPage, rc));
+        return rc;
+    }
+    Assert(pPd);
+    EPTPDE Pde = pPd->a[iPde];
+
+# if 0  /* Enable this later? */
+    /* In the guest SMP case we could have blocked while another VCPU reused this page table. */
+    if (!SHW_PDE_IS_P(Pde))
+    {
+        AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)Pde.u));
+        Log7Func(("CPU%d: SyncPage: Pde at %RGp changed behind our back!\n", pVCpu->idCpu, GCPhysNestedPage));
+        return VINF_SUCCESS;    /* force the instruction to be executed again. */
+    }
+
+    /* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
+    if (SHW_PDE_IS_BIG(Pde))
+    {
+        Assert(pVM->pgm.s.fNestedPaging);
+        Log7Func(("CPU%d: SyncPage: %RGp changed behind our back!\n", pVCpu->idCpu, GCPhysNestedPage));
+        return VINF_SUCCESS;
+    }
+# else
+    AssertMsg(SHW_PDE_IS_P(Pde), ("Pde=%RX64 iPde=%u\n", Pde.u, iPde));
+    Assert(!SHW_PDE_IS_BIG(Pde));
+# endif
+
+    PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, Pde.u & EPT_PDE_PG_MASK);
+    PEPTPT       pPt      = (PEPTPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+
+    Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
+# ifdef PGM_SYNC_N_PAGES
+    if (    cPages > 1
+        && !(uErr & X86_TRAP_PF_P)
+        && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+    {
+        /*
+         * This code path is currently only taken for non-present pages!
+         *
+         * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
+         * deal with locality.
+         */
+        unsigned       iPte    = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+        unsigned const iPteEnd = RT_MIN(iPte + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPt->a));
+        if (iPte < PGM_SYNC_NR_PAGES / 2)
+            iPte = 0;
+        else
+            iPte -= PGM_SYNC_NR_PAGES / 2;
+        for (; iPte < iPteEnd; iPte++)
+        {
+            if (!SHW_PTE_IS_P(pPt->a[iPte]))
+            {
+                PGMPTWALKGST GstWalkPt;
+                PGMPTWALK    WalkPt;
+                GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
+                GCPhysNestedPage |= (iPte << GUEST_PAGE_SHIFT);
+                rc = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/, &WalkPt,
+                                    &GstWalkPt);
+                if (RT_SUCCESS(rc))
+                    PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], WalkPt.GCPhys, pShwPage, iPte, &GstWalkPt);
+                else
+                {
+                    /*
+                     * This could be MMIO pages reserved by the nested-hypevisor or genuinely not-present pages.
+                     * Ensure the shadow tables entry is not-present.
+                     */
+                    /** @todo Potential room for optimization (explained in NestedSyncPT). */
+                    AssertMsg(!pPt->a[iPte].u, ("%RX64\n", pPt->a[iPte].u));
+                }
+                Log7Func(("Many: %RGp iPte=%u ShwPte=%RX64\n", GCPhysNestedPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
+                if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
+                    break;
+            }
+            else
+            {
+#  ifdef VBOX_STRICT
+                /* Paranoia - Verify address of the page is what it should be. */
+                PGMPTWALKGST GstWalkPt;
+                PGMPTWALK    WalkPt;
+                GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
+                GCPhysNestedPage |= (iPte << GUEST_PAGE_SHIFT);
+                rc = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/, &WalkPt, &GstWalkPt);
+                AssertRC(rc);
+                PPGMPAGE pPage;
+                rc = pgmPhysGetPageEx(pVM, WalkPt.GCPhys, &pPage);
+                AssertRC(rc);
+                AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte]),
+                          ("PGM page and shadow PTE address conflict. GCPhysNestedPage=%RGp GCPhysPage=%RGp HCPhys=%RHp Shw=%RHp\n",
+                           GCPhysNestedPage, WalkPt.GCPhys, PGM_PAGE_GET_HCPHYS(pPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
+#  endif
+                Log7Func(("Many3: %RGp iPte=%u ShwPte=%RX64\n", GCPhysNestedPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
+            }
+        }
+    }
+    else
+# endif /* PGM_SYNC_N_PAGES */
+    {
+        unsigned const iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+        PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll);
+        Log7Func(("4K: GCPhysPage=%RGp iPte=%u ShwPte=%08llx\n", GCPhysPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sync a shadow page table for a nested-guest page table.
+ *
+ * The shadow page table is not present in the shadow PDE.
+ *
+ * Handles mapping conflicts.
+ *
+ * A precondition for this method is that the shadow PDE is not present.  The
+ * caller must take the PGM lock before checking this and continue to hold it
+ * when calling this method.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   GCPhysNestedPage    The nested-guest physical page address of the page
+ *                              being synced.
+ * @param   GCPhysPage          The guest-physical address of the page being synced.
+ * @param   pGstWalkAll         The guest page table walk result.
+ */
+static int PGM_BTH_NAME(NestedSyncPT)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, PPGMPTWALKGST pGstWalkAll)
+{
+    PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
+    Assert(!(GCPhysNestedPage & GUEST_PAGE_OFFSET_MASK));
+    Assert(!(GCPhysPage & GUEST_PAGE_OFFSET_MASK));
+
+    PVMCC    pVM   = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+    Log7Func(("GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+
+    PEPTPD         pPd;
+    PEPTPDPT       pPdpt;
+    unsigned const iPde = (GCPhysNestedPage >> EPT_PD_SHIFT)   & EPT_PD_MASK;
+    int rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, &pPdpt, &pPd, pGstWalkAll);
+    if (rc != VINF_SUCCESS)
+    {
+        STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+        AssertRC(rc);
+        return rc;
+    }
+    Assert(pPd);
+    PSHWPDE pPde = &pPd->a[iPde];
+
+    unsigned const iPdpt = (GCPhysNestedPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
+    PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdpt->a[iPdpt].u & EPT_PDPTE_PG_MASK);
+    Assert(pShwPde->enmKind == PGMPOOLKIND_EPT_PD_FOR_EPT_PD);
+
+    SHWPDE Pde = *pPde;
+    Assert(!SHW_PDE_IS_P(Pde));    /* We're only supposed to call SyncPT on PDE!P and conflicts. */
+
+# ifdef PGM_WITH_LARGE_PAGES
+    if (BTH_IS_NP_ACTIVE(pVM))
+    {
+        /* Check if we allocated a big page before for this 2 MB range and disable it. */
+        PPGMPAGE pPage;
+        rc = pgmPhysGetPageEx(pVM, GCPhysPage & X86_PDE2M_PAE_PG_MASK, &pPage);
+        if (   RT_SUCCESS(rc)
+            && PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
+        {
+            Log7Func(("Disabling large page %RGp\n", GCPhysPage));
+            Assert(PGM_A20_IS_ENABLED(pVCpu));  /* Should never be in A20M mode in VMX operation. */
+            PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
+            pVM->pgm.s.cLargePagesDisabled++;
+        }
+    }
+# endif /* PGM_WITH_LARGE_PAGES */
+
+    /*
+     * Allocate & map the page table.
+     */
+    PSHWPT       pPt;
+    PPGMPOOLPAGE pShwPage;
+
+    RTGCPHYS const GCPhysPt = pGstWalkAll->u.Ept.Pde.u & EPT_PDE_PG_MASK;
+    rc = pgmPoolAlloc(pVM, GCPhysPt, PGMPOOLKIND_EPT_PT_FOR_EPT_PT, PGMPOOLACCESS_DONTCARE,
+                      PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPde, false /*fLockPage*/, &pShwPage);
+    if (   rc == VINF_SUCCESS
+        || rc == VINF_PGM_CACHED_PAGE)
+    { /* likely */ }
+    else
+    {
+       STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+       AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
+    }
+
+    pPt = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    Assert(pPt);
+    Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
+
+    if (rc == VINF_SUCCESS)
+    {
+        /* Sync the page we've already translated through SLAT. */
+        const unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+        Assert((pGstWalkAll->u.Ept.Pte.u & EPT_PTE_PG_MASK) == GCPhysPage);
+        PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll);
+        Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u\n", pGstWalkAll->u.Ept.Pte.u, pPt->a[iPte].u, iPte));
+
+        /* Sync the rest of page table (expensive but might be cheaper than nested-guest VM-exits in hardware). */
+        for (unsigned iPteCur = 0; iPteCur < RT_ELEMENTS(pPt->a); iPteCur++)
+        {
+            if (iPteCur != iPte)
+            {
+                PGMPTWALKGST GstWalkPt;
+                PGMPTWALK    WalkPt;
+                GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
+                GCPhysNestedPage |= (iPteCur << GUEST_PAGE_SHIFT);
+                int const rc2 = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/,
+                                               &WalkPt, &GstWalkPt);
+                if (RT_SUCCESS(rc2))
+                {
+                    PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPteCur], WalkPt.GCPhys, pShwPage, iPteCur, &GstWalkPt);
+                    Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u\n", GstWalkPt.u.Ept.Pte.u, pPt->a[iPteCur].u, iPteCur));
+                }
+                else
+                {
+                    /*
+                     * This could be MMIO pages reserved by the nested-hypevisor or genuinely not-present pages.
+                     * Ensure the shadow tables entry is not-present.
+                     */
+                    /** @todo We currently don't sync. them to cause EPT misconfigs and trap all of them
+                     *        using EPT violation and walk the guest EPT tables to determine EPT
+                     *        misconfigs VM-exits for the nested-guest. In the future we could optimize
+                     *        this by using a specific combination of reserved bits which we can
+                     *        immediately identify as EPT misconfigs for the nested-guest without having
+                     *        to walk its EPT tables. Tracking non-present entries might be tricky...
+                     */
+                    AssertMsg(!pPt->a[iPteCur].u, ("%RX64\n", pPt->a[iPteCur].u));
+                }
+                if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
+                    break;
+            }
+        }
+    }
+    else
+    {
+        Assert(rc == VINF_PGM_CACHED_PAGE);
+# ifdef VBOX_STRICT
+        /* Paranoia - Verify address of the page is what it should be. */
+        PPGMPAGE pPage;
+        rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
+        AssertRC(rc);
+        const unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+        AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte]),
+                  ("PGM page and shadow PTE address conflict. GCPhysNestedPage=%RGp GCPhysPage=%RGp Page=%RHp Shw=%RHp\n",
+                   GCPhysNestedPage, GCPhysPage, PGM_PAGE_GET_HCPHYS(pPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
+        Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u [cache]\n", pGstWalkAll->u.Ept.Pte.u, pPt->a[iPte].u,  iPte));
+# endif
+        rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
+    }
+
+    /* Save the new PDE. */
+    uint64_t const fShwPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedPdeMask;
+    AssertReturn(!(pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF), VERR_NOT_SUPPORTED);  /* Implement this later. */
+    Pde.u = pShwPage->Core.Key | fShwPdeFlags;
+    SHW_PDE_ATOMIC_SET2(*pPde, Pde);
+    Log7Func(("GstPde=%RGp ShwPde=%RX64 iPde=%u\n", pGstWalkAll->u.Ept.Pde.u, pPde->u, iPde));
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+    return rc;
+}
+#endif  /* !IN_RING3 && VBOX_WITH_NESTED_HWVIRT_VMX_EPT && PGM_SHW_TYPE == PGM_TYPE_EPT*/
+
+
 #if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
 
@@ -2934 +3473 @@
 
                         if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
-                            PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, SHW_PTE_GET_U(PteDstBase), &PteDst);
+                            PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhys, SHW_PTE_GET_U(PteDstBase), &PteDst);
                         else if (PGM_PAGE_IS_BALLOONED(pPage))
                             SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
@@ -4392 +4931 @@
             X86PDPE aGstPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
             memcpy(&aGstPaePdpes, HCPtrGuestCR3, sizeof(aGstPaePdpes));
-            CPUMSetGuestPaePdpes(pVCpu, &aGstPaePdpes[0]);
             PGMGstMapPaePdpes(pVCpu, &aGstPaePdpes[0]);
 
@@ -4513 +5051 @@
 #endif
 
-    /*
-     * Update second-level address translation info.
-     */
-#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
-    pVCpu->pgm.s.pGstEptPml4R3 = 0;
-    pVCpu->pgm.s.pGstEptPml4R0 = 0;
-#endif
-
+    /** @todo This should probably be moved inside \#if PGM_GST_TYPE == PGM_TYPE_PAE? */
     pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
     pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;

trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp

@@ -191 +191 @@
              (RTGCPTR)(CTXTYPE(RTGCPTR, uintptr_t, RTGCPTR))(uintptr_t)pvAddress, GCPhysFault, cbWrite));
 
+    if (PGMPOOL_PAGE_IS_NESTED(pPage))
+        Log7Func(("%RGv phys=%RGp cbWrite=%d\n",
+                  (RTGCPTR)(CTXTYPE(RTGCPTR, uintptr_t, RTGCPTR))(uintptr_t)pvAddress, GCPhysFault, cbWrite));
+
     for (;;)
     {
        union
-        {
+       {
             void           *pv;
             PX86PT          pPT;
@@ -202 +206 @@
             PX86PDPT        pPDPT;
             PX86PML4        pPML4;
-        } uShw;
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            PEPTPDPT        pEptPdpt;
+            PEPTPD          pEptPd;
+            PEPTPT          pEptPt;
+#endif
+       } uShw;
 
         LogFlow(("pgmPoolMonitorChainChanging: page idx=%d phys=%RGp (next=%d) kind=%s write=%#x\n",
@@ -563 +572 @@
             }
 
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+            {
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(EPTPML4E);
+                X86PGPAEUINT const uPml4e = uShw.pPML4->a[iShw].u;
+                if (uPml4e & EPT_PRESENT_MASK)
+                {
+                    Log7Func(("PML4 iShw=%#x: %RX64 (%RGp) -> freeing it!\n", iShw, uPml4e, pPage->GCPhys));
+                    pgmPoolFree(pVM, uPml4e & X86_PML4E_PG_MASK, pPage->idx, iShw);
+                    ASMAtomicWriteU64(&uShw.pPML4->a[iShw].u, 0);
+                }
+
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(X86PML4E))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
+                    X86PGPAEUINT const uPml4e2 = uShw.pPML4->a[iShw2].u;
+                    if (uPml4e2 & EPT_PRESENT_MASK)
+                    {
+                        Log7Func(("PML4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPml4e2));
+                        pgmPoolFree(pVM, uPml4e2 & X86_PML4E_PG_MASK, pPage->idx, iShw2);
+                        ASMAtomicWriteU64(&uShw.pPML4->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+            {
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(EPTPDPTE);
+                X86PGPAEUINT const uPdpte = uShw.pEptPdpt->a[iShw].u;
+                if (uPdpte & EPT_PRESENT_MASK)
+                {
+                    Log7Func(("EPT PDPT iShw=%#x: %RX64 (%RGp) -> freeing it!\n", iShw, uPdpte, pPage->GCPhys));
+                    pgmPoolFree(pVM, uPdpte & EPT_PDPTE_PG_MASK, pPage->idx, iShw);
+                    ASMAtomicWriteU64(&uShw.pEptPdpt->a[iShw].u, 0);
+                }
+
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(EPTPDPTE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(EPTPDPTE);
+                    X86PGPAEUINT const uPdpte2 = uShw.pEptPdpt->a[iShw2].u;
+                    if (uPdpte2 & EPT_PRESENT_MASK)
+                    {
+                        Log7Func(("EPT PDPT iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPdpte2));
+                        pgmPoolFree(pVM, uPdpte2 & EPT_PDPTE_PG_MASK, pPage->idx, iShw2);
+                        ASMAtomicWriteU64(&uShw.pEptPdpt->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+            {
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(EPTPDE);
+                X86PGPAEUINT const uPde = uShw.pEptPd->a[iShw].u;
+                if (uPde & EPT_PRESENT_MASK)
+                {
+                    Log7Func(("EPT PD iShw=%#x: %RX64 (%RGp) -> freeing it!\n", iShw, uPde, pPage->GCPhys));
+                    pgmPoolFree(pVM, uPde & EPT_PDE_PG_MASK, pPage->idx, iShw);
+                    ASMAtomicWriteU64(&uShw.pEptPd->a[iShw].u, 0);
+                }
+
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(EPTPDE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(EPTPDE);
+                    AssertBreak(iShw2 < RT_ELEMENTS(uShw.pEptPd->a));
+                    X86PGPAEUINT const uPde2 = uShw.pEptPd->a[iShw2].u;
+                    if (uPde2 & EPT_PRESENT_MASK)
+                    {
+                        Log7Func(("EPT PD (2): iShw2=%#x: %RX64 (%RGp) -> freeing it!\n", iShw2, uPde2, pPage->GCPhys));
+                        pgmPoolFree(pVM, uPde2 & EPT_PDE_PG_MASK, pPage->idx, iShw2);
+                        ASMAtomicWriteU64(&uShw.pEptPd->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+            {
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(EPTPTE);
+                X86PGPAEUINT const uPte = uShw.pEptPt->a[iShw].u;
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
+                if (uPte & EPT_PRESENT_MASK)
+                {
+                    EPTPTE GstPte;
+                    int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
+                    AssertRC(rc);
+
+                    Log7Func(("EPT PT: iShw=%#x %RX64 (%RGp)\n", iShw, uPte, pPage->GCPhys));
+                    pgmPoolTracDerefGCPhysHint(pPool, pPage,
+                                               uShw.pEptPt->a[iShw].u & EPT_PTE_PG_MASK,
+                                               GstPte.u & EPT_PTE_PG_MASK,
+                                               iShw);
+                    ASMAtomicWriteU64(&uShw.pEptPt->a[iShw].u, 0);
+                }
+
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(EPTPTE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(EPTPTE);
+                    AssertBreak(iShw2 < RT_ELEMENTS(uShw.pEptPt->a));
+                    X86PGPAEUINT const uPte2 = uShw.pEptPt->a[iShw2].u;
+                    if (uPte2 & EPT_PRESENT_MASK)
+                    {
+                        EPTPTE GstPte;
+                        int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte,
+                                                             pvAddress ? (uint8_t const *)pvAddress + sizeof(GstPte) : NULL,
+                                                             GCPhysFault + sizeof(GstPte), sizeof(GstPte));
+                        AssertRC(rc);
+                        Log7Func(("EPT PT (2): iShw=%#x %RX64 (%RGp)\n", iShw2, uPte2, pPage->GCPhys));
+                        pgmPoolTracDerefGCPhysHint(pPool, pPage,
+                                                   uShw.pEptPt->a[iShw2].u & EPT_PTE_PG_MASK,
+                                                   GstPte.u & EPT_PTE_PG_MASK,
+                                                   iShw2);
+                        ASMAtomicWriteU64(&uShw.pEptPt->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+#endif  /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
+
             default:
                 AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
@@ -959 +1100 @@
     RT_NOREF_PV(uErrorCode);
 
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+    AssertMsg(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT,
+              ("pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
+# endif
     LogFlow(("pgmRZPoolAccessPfHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
 
@@ -973 +1118 @@
     if (pPage->fDirty)
     {
+#  ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        Assert(!PGMPOOL_PAGE_IS_NESTED(pPage));
+#  endif
         Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH));
         PGM_UNLOCK(pVM);
@@ -988 +1136 @@
         PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
         PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
+    }
+# endif
+
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+    if (PGMPOOL_PAGE_IS_NESTED(pPage))
+    {
+        Assert(!CPUMIsGuestInVmxNonRootMode(CPUMQueryGuestCtxPtr(pVCpu)));
+        Log7Func(("Flushing pvFault=%RGv GCPhysFault=%RGp\n", pvFault, GCPhysFault));
+        pgmPoolMonitorChainFlush(pPool, pPage);
+        PGM_UNLOCK(pVM);
+        return VINF_SUCCESS;
     }
 # endif
@@ -1237 +1396 @@
         &&  fReused)
     {
+        Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* temporary, remove later. */
         /* Make sure that the current instruction still has shadow page backing, otherwise we'll end up in a loop. */
         if (PGMShwGetPage(pVCpu, pRegFrame->rip, NULL, NULL) == VINF_SUCCESS)
@@ -1681 +1841 @@
                                              (PCX86PTPAE)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
     else
+    {
+        Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* temporary, remove later. */
         cChanges = pgmPoolTrackFlushPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst,
                                                (PCX86PT)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
+    }
 
     PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
@@ -1728 +1891 @@
     AssertCompile(RT_ELEMENTS(pPool->aDirtyPages) == 8 || RT_ELEMENTS(pPool->aDirtyPages) == 16);
     Assert(!pPage->fDirty);
+    Assert(!PGMPOOL_PAGE_IS_NESTED(pPage));
 
     unsigned idxFree = pPool->idxFreeDirtyPage;
@@ -2104 +2268 @@
         case PGMPOOLKIND_32BIT_PD:
         case PGMPOOLKIND_PAE_PDPT:
+            Assert(!PGMPOOL_PAGE_IS_KIND_NESTED(enmKind2));
             switch (enmKind2)
             {
@@ -2133 +2298 @@
         case PGMPOOLKIND_64BIT_PML4:
         case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+            Assert(!PGMPOOL_PAGE_IS_KIND_NESTED(enmKind2));
             switch (enmKind2)
             {
@@ -2155 +2321 @@
             }
 
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+        case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+        case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+            return PGMPOOL_PAGE_IS_KIND_NESTED(enmKind2);
+
+        case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+            return false;
+#endif
+
         /*
          * These cannot be flushed, and it's common to reuse the PDs as PTs.
@@ -2368 +2544 @@
                 case PGMPOOLKIND_32BIT_PD:
                 case PGMPOOLKIND_PAE_PDPT:
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+                case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+                case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+                case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+#endif
                 {
                     /* find the head */
@@ -2394 +2575 @@
                 case PGMPOOLKIND_32BIT_PD_PHYS:
                 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+                case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+#endif
                     break;
                 default:
@@ -2459 +2643 @@
             /* Nothing to monitor here. */
             return VINF_SUCCESS;
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+        case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+        case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+            break;
+
+        case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+            /* Nothing to monitor here. */
+            return VINF_SUCCESS;
+#endif
+
         default:
             AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
@@ -2484 +2680 @@
         pPageHead->iMonitoredNext = pPage->idx;
         rc = VINF_SUCCESS;
+        if (PGMPOOL_PAGE_IS_NESTED(pPage))
+            Log7Func(("Adding to monitoring list GCPhysPage=%RGp\n", pPage->GCPhys));
     }
     else
     {
+        if (PGMPOOL_PAGE_IS_NESTED(pPage))
+            Log7Func(("Started monitoring GCPhysPage=%RGp HCPhys=%RHp enmKind=%s\n", pPage->GCPhys, pPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
+
         Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
         PVMCC pVM = pPool->CTX_SUFF(pVM);
@@ -2551 +2752 @@
             return VINF_SUCCESS;
 
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+        case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+        case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+            break;
+
+        case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+            /* Nothing to monitor here. */
+            Assert(!pPage->fMonitored);
+            return VINF_SUCCESS;
+#endif
+
         default:
             AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
@@ -2599 +2812 @@
      */
     pgmPoolMonitorModifiedRemove(pPool, pPage);
+
+    if (PGMPOOL_PAGE_IS_NESTED(pPage))
+        Log7Func(("Stopped monitoring %RGp\n", pPage->GCPhys));
 
     return rc;
@@ -3173 +3389 @@
                         break;
                     default:
-                        /* (shouldn't be here, will assert below) */
+                        /* We will end up here when called with an "ALL" access handler. */
                         STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
                         break;
@@ -3197 +3413 @@
                 if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
                     Pte.u &= ~(X86PGUINT)X86_PTE_RW; /* need to disallow writes when dirty bit tracking is still active. */
+
                 ASMAtomicWriteU32(&pPT->a[iPte].u, Pte.u);
                 PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
@@ -3220 +3437 @@
         case PGMPOOLKIND_PAE_PT_FOR_PHYS:
         case PGMPOOLKIND_EPT_PT_FOR_PHYS:   /* physical mask the same as PAE; RW bit as well; be careful! */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+#endif
         {
             const uint64_t  u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
@@ -3246 +3466 @@
 
                     default:
-                        /* (shouldn't be here, will assert below) */
+                        /* We will end up here when called with an "ALL" access handler. */
                         STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
                         break;
@@ -3594 +3814 @@
             &&  pPage->cPresent)
         {
+            Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* see if it hits */
             switch (pPage->enmKind)
             {
@@ -3790 +4011 @@
             break;
 
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+        case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+        case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+        case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+            Assert(iUserTable < EPT_PG_ENTRIES);
+            break;
+# endif
+
         default:
-            AssertMsgFailed(("enmKind=%d\n", pUserPage->enmKind));
+            AssertMsgFailed(("enmKind=%d GCPhys=%RGp\n", pUserPage->enmKind, pPage->GCPhys));
             break;
     }
@@ -3825 +4055 @@
         case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
         case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+        case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+        case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+        case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+#endif
             ASMAtomicWriteU64(&u.pau64[iUserTable], 0);
             break;
@@ -4204 +4440 @@
         }
 
-        AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp; found page has HCPhys=%RHp\n",
-                              HCPhys, GCPhys, PGM_PAGE_GET_HCPHYS(pPhysPage)));
+        AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp; found page has HCPhys=%RHp iPte=%u fIsNested=%RTbool\n",
+                              HCPhys, GCPhys, PGM_PAGE_GET_HCPHYS(pPhysPage), iPte, PGMPOOL_PAGE_IS_NESTED(pPage)));
     }
     AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
@@ -4438 +4674 @@
 }
 
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/**
+ * Clears references to shadowed pages in a SLAT EPT page table.
+ *
+ * @param   pPool   The pool.
+ * @param   pPage   The page.
+ * @param   pShwPT  The shadow page table (mapping of the page).
+ * @param   pGstPT  The guest page table.
+ */
+DECLINLINE(void) pgmPoolTrackDerefNestedPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT, PCEPTPT pGstPT)
+{
+    Assert(PGMPOOL_PAGE_IS_NESTED(pPage));
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        X86PGPAEUINT const uShwPte = pShwPT->a[i].u;
+        Assert((uShwPte & UINT64_C(0xfff0000000000f80)) == 0); /* Access, Dirty, UserX (not supported) and ignored bits 7, 11. */
+        if (uShwPte & EPT_PRESENT_MASK)
+        {
+            Log7Func(("Shw=%RX64 GstPte=%RX64\n", uShwPte, pGstPT->a[i].u));
+            pgmPoolTracDerefGCPhys(pPool, pPage, uShwPte & EPT_PTE_PG_MASK, pGstPT->a[i].u & EPT_PTE_PG_MASK, i);
+            if (!pPage->cPresent)
+                break;
+        }
+    }
+}
+
+
+# if 0
+/**
+ * Clears refernces to shadowed pages in a SLAT EPT PM4 table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPml4    The shadow PML4 table.
+ */
+DECLINLINE(void) pgmPoolTrackDerefNestedPML4EPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPML4 pShwPml4)
+{
+    /** @todo later merge this with 64-bit PML and pass the assert and present masks as
+     *        parameters. */
+    Assert(PGMPOOL_PAGE_IS_NESTED(pPage));
+    for (unsigned i = 0; i < RT_ELEMENTS(pShwPml4->a); i++)
+    {
+        X86PGPAEUINT const uPml4e = pShwPml4->a[i].u;
+        Assert((uPml4e & (EPT_PML4E_MBZ_MASK | UINT64_C(0xfff0000000000000))) == 0);
+        if (uPml4e & EPT_PRESENT_MASK)
+        {
+            PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPml4e & EPT_PML4E_PG_MASK);
+            if (pSubPage)
+                pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+            else
+                AssertFatalMsgFailed(("%RX64\n", uPml4e & EPT_PML4E_PG_MASK));
+        }
+    }
+}
+# endif
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
 
 /**
@@ -4591 +4883 @@
     {
         X86PGPAEUINT const uPde = pShwPD->a[i].u;
-        Assert((uPde & UINT64_C(0xfff0000000000f80)) == 0);
+#ifdef PGM_WITH_LARGE_PAGES
+        AssertMsg((uPde & UINT64_C(0xfff0000000000f00)) == 0, ("uPde=%RX64\n", uPde));
+#else
+        AssertMsg((uPde & UINT64_C(0xfff0000000000f80)) == 0, ("uPde=%RX64\n", uPde));
+#endif
         if (uPde & EPT_E_READ)
         {
@@ -4599 +4895 @@
                 Log4(("pgmPoolTrackDerefPDEPT: i=%d pde=%RX64 GCPhys=%RX64\n",
                       i, uPde & EPT_PDE2M_PG_MASK, pPage->GCPhys));
+                Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* We don't support large guest EPT yet. */
                 pgmPoolTracDerefGCPhys(pPool, pPage, uPde & EPT_PDE2M_PG_MASK,
                                        pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
@@ -4756 +5053 @@
             break;
 
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+        {
+            STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
+            void *pvGst;
+            int const rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
+            pgmPoolTrackDerefNestedPTEPT(pPool, pPage, (PEPTPT)pvShw, (PCEPTPT)pvGst);
+            PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
+            STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
+            break;
+        }
+
+        case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+            pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
+            break;
+
+        case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+            pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
+            break;
+
+        case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+            //pgmPoolTrackDerefNestedPML4EPT(pPool, pPage, (PEPTPML4)pvShw);
+            RT_FALL_THRU();
+#endif
+
         default:
-            AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
+            AssertFatalMsgFailed(("enmKind=%d GCPhys=%RGp\n", pPage->enmKind, pPage->GCPhys));
     }
 
@@ -4790 +5112 @@
     LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
              pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
+
+    if (PGMPOOL_PAGE_IS_NESTED(pPage))
+        Log7Func(("pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
+                  pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
 
     /*
@@ -4989 +5315 @@
      *  (TRPMR3SyncIDT) because of FF priority. Try fix that?
      *  Assert(!(pVM->pgm.s.fGlobalSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); */
+
+#if defined(VBOX_STRICT) && defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT)
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT || PGMPOOL_PAGE_IS_KIND_NESTED(enmKind));
+#endif
 
     PGM_LOCK_VOID(pVM);
@@ -5200 +5531 @@
         if (pPage->GCPhys - GCPhys < PAGE_SIZE)
         {
+            Assert(!PGMPOOL_PAGE_IS_NESTED(pPage));  /* Temporary to see if it hits. Remove later. */
             switch (pPage->enmKind)
             {
@@ -5503 +5835 @@
         case PGMPOOLKIND_ROOT_NESTED:
             return "PGMPOOLKIND_ROOT_NESTED";
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+            return "PGMPOOLKIND_EPT_PT_FOR_EPT_PT";
+        case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+            return "PGMPOOLKIND_EPT_PD_FOR_EPT_PD";
+        case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+            return "PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT";
+        case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+            return "PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4";
     }
     return "Unknown kind!";

trunk/src/VBox/VMM/VMMAll/PGMAllShw.h

@@ -238 +238 @@
        a different shadow paging root/mode in both cases. */
     RTGCPHYS     GCPhysCR3 = (fIs64BitsPagingMode) ? RT_BIT_64(63) : RT_BIT_64(62);
+    PGMPOOLKIND  enmKind   = PGMPOOLKIND_ROOT_NESTED;
+# elif defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT)
+    RTGCPHYS    GCPhysCR3;
+    PGMPOOLKIND enmKind;
+    if (pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_EPT)
+    {
+        GCPhysCR3 = RT_BIT_64(63); NOREF(fIs64BitsPagingMode);
+        enmKind   = PGMPOOLKIND_ROOT_NESTED;
+    }
+    else
+    {
+        GCPhysCR3 = pVCpu->pgm.s.uEptPtr & EPT_EPTP_PG_MASK;
+        enmKind   = PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4;
+    }
 # else
     RTGCPHYS     GCPhysCR3 = RT_BIT_64(63); NOREF(fIs64BitsPagingMode);
+    PGMPOOLKIND const enmKind = PGMPOOLKIND_ROOT_NESTED;
 # endif
     PPGMPOOLPAGE pNewShwPageCR3;
@@ -250 +265 @@
     PGM_LOCK_VOID(pVM);
 
-    int rc = pgmPoolAlloc(pVM, GCPhysCR3, PGMPOOLKIND_ROOT_NESTED, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+    int rc = pgmPoolAlloc(pVM, GCPhysCR3, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
                           NIL_PGMPOOL_IDX, UINT32_MAX, true /*fLockPage*/,
                           &pNewShwPageCR3);
@@ -283 +298 @@
 
         PGM_LOCK_VOID(pVM);
+
+# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT
+        if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
+            pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+# endif
 
         /* Do *not* unlock this page as we have two of them floating around in the 32-bit host & 64-bit guest case.
@@ -372 +392 @@
 
 # elif PGM_SHW_TYPE == PGM_TYPE_EPT
+    Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT);
     PEPTPD          pPDDst;
     int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtr, NULL, &pPDDst);
@@ -527 +548 @@
 
 # elif PGM_SHW_TYPE == PGM_TYPE_EPT
+        Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT);
         const unsigned  iPd = ((GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK);
         PEPTPD          pPDDst;
@@ -546 +568 @@
             return VERR_PAGE_TABLE_NOT_PRESENT;
 
-        AssertFatal(!SHW_PDE_IS_BIG(Pde));
+        AssertFatalMsg(!SHW_PDE_IS_BIG(Pde), ("Pde=%#RX64\n", (uint64_t)Pde.u));
 
         /*
@@ -569 +591 @@
                     /** @todo Some CSAM code path might end up here and upset
                      *  the page pool. */
-                    AssertFailed();
+                    AssertMsgFailed(("NewPte=%#RX64 OrgPte=%#RX64 GCPtr=%#RGv\n", SHW_PTE_LOG64(NewPte), SHW_PTE_LOG64(OrgPte), GCPtr));
                 }
                 else if (   SHW_PTE_IS_RW(NewPte)