Changeset 91848 in vbox for trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp

Timestamp:

Oct 19, 2021 11:18:13 PM (3 years ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

147650

Message:

VMM/NEM,PGM: First bunch changes for watered down guest memory managment for NEM. (Gets stuck in bios.) bugref:10122

File:

• trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp (modified) (56 diffs)

trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp

@@ -1158 +1158 @@
  * @param   GCPhys      The address of the first page.
  * @param   GCPhysLast  The address of the last page.
- * @param   enmType     The page type to replace then with.
- */
-static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, PGMPAGETYPE enmType)
+ * @param   pvMmio2     Pointer to the ring-3 mapping of any MMIO2 memory that
+ *                      will replace the pages we're freeing up.
+ */
+static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, void *pvMmio2)
 {
     PGM_LOCK_ASSERT_OWNER(pVM);
+
+#ifdef VBOX_WITH_PGM_NEM_MODE
+    /*
+     * In simplified memory mode we don't actually free the memory,
+     * we just unmap it and let NEM do any unlocking of it.
+     */
+    if (pVM->pgm.s.fNemMode)
+    {
+        Assert(VM_IS_NEM_ENABLED(pVM));
+        uint32_t const  fNemNotify = (pvMmio2 ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2 : 0) | NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE;
+        uint8_t         u2State    = 0; /* (We don't support UINT8_MAX here.) */
+        int rc = NEMR3NotifyPhysMmioExMapEarly(pVM, GCPhys, GCPhysLast - GCPhys + 1, fNemNotify,
+                                               pRam->pvR3 ? (uint8_t *)pRam->pvR3 + GCPhys - pRam->GCPhys : NULL,
+                                               pvMmio2, &u2State);
+        AssertLogRelRCReturn(rc, rc);
+
+        /* Iterate the pages. */
+        PPGMPAGE pPageDst   = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+        uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
+        while (cPagesLeft-- > 0)
+        {
+            rc = pgmPhysFreePage(pVM, NULL, NULL, pPageDst, GCPhys, PGMPAGETYPE_MMIO);
+            AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
+
+            PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO);
+            PGM_PAGE_SET_NEM_STATE(pPageDst, u2State);
+
+            GCPhys += PAGE_SIZE;
+            pPageDst++;
+        }
+        return rc;
+    }
+#endif /* VBOX_WITH_PGM_NEM_MODE */
+
+    /*
+     * Regular mode.
+     */
+    /* Prepare. */
     uint32_t            cPendingPages = 0;
     PGMMFREEPAGESREQ    pReq;
@@ -1168 +1207 @@
     AssertLogRelRCReturn(rc, rc);
 
+#ifdef VBOX_WITH_NATIVE_NEM
+    /* Tell NEM up-front. */
+    uint8_t u2State = UINT8_MAX;
+    if (VM_IS_NEM_ENABLED(pVM))
+    {
+        uint32_t const fNemNotify = (pvMmio2 ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2 : 0) | NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE;
+        rc = NEMR3NotifyPhysMmioExMapEarly(pVM, GCPhys, GCPhysLast - GCPhys + 1, fNemNotify, NULL, pvMmio2, &u2State);
+        AssertLogRelRCReturnStmt(rc, GMMR3FreePagesCleanup(pReq), rc);
+    }
+#endif
+
     /* Iterate the pages. */
     PPGMPAGE pPageDst   = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
@@ -1173 +1223 @@
     while (cPagesLeft-- > 0)
     {
-        rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys, enmType);
+        rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys, PGMPAGETYPE_MMIO);
         AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
 
-        PGM_PAGE_SET_TYPE(pVM, pPageDst, enmType);
+        PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO);
+#ifdef VBOX_WITH_NATIVE_NEM
+        if (u2State != UINT8_MAX)
+            PGM_PAGE_SET_NEM_STATE(pPageDst, u2State);
+#endif
 
         GCPhys += PAGE_SIZE;
@@ -1182 +1236 @@
     }
 
+    /* Finish pending and cleanup. */
     if (cPendingPages)
     {
@@ -1634 +1689 @@
  * PGMR3PhysRegisterRam worker that initializes and links a RAM range.
  *
+ * In NEM mode, this will allocate the pages backing the RAM range and this may
+ * fail.  NEM registration may also fail.  (In regular HM mode it won't fail.)
+ *
+ * @returns VBox status code.
  * @param   pVM             The cross context VM structure.
  * @param   pNew            The new RAM range.
@@ -1644 +1703 @@
  * @param   pPrev           The previous RAM range (for linking).
  */
-static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
-                                         RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
+static int pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
+                                        RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
 {
     /*
      * Initialize the range.
      */
-    pNew->pSelfR0       = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
+    pNew->pSelfR0       = R0PtrNew;
     pNew->GCPhys        = GCPhys;
     pNew->GCPhysLast    = GCPhysLast;
     pNew->cb            = GCPhysLast - GCPhys + 1;
     pNew->pszDesc       = pszDesc;
-    pNew->fFlags        = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
     pNew->pvR3          = NULL;
     pNew->paLSPages     = NULL;
+    pNew->fFlags        = 0;
+#ifndef PGM_WITHOUT_MAPPINGS
+    if (RCPtrNew != NIL_RTRCPTR)
+        pNew->fFlags   |= PGM_RAM_RANGE_FLAGS_FLOATING;
+#else
+    NOREF(RCPtrNew);
+#endif
 
     uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
-    RTGCPHYS iPage = cPages;
-    while (iPage-- > 0)
-        PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
-
-    /* Update the page count stats. */
-    pVM->pgm.s.cZeroPages += cPages;
-    pVM->pgm.s.cAllPages  += cPages;
+#ifdef VBOX_WITH_PGM_NEM_MODE
+    if (!pVM->pgm.s.fNemMode)
+#endif
+    {
+        RTGCPHYS iPage = cPages;
+        while (iPage-- > 0)
+            PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
+
+        /* Update the page count stats. */
+        pVM->pgm.s.cZeroPages += cPages;
+        pVM->pgm.s.cAllPages  += cPages;
+    }
+#ifdef VBOX_WITH_PGM_NEM_MODE
+    else
+    {
+        int rc = SUPR3PageAlloc(cPages, &pNew->pvR3);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        RTGCPHYS iPage = cPages;
+        while (iPage-- > 0)
+            PGM_PAGE_INIT(&pNew->aPages[iPage], UINT64_C(0x0000fffffffff000), NIL_GMM_PAGEID,
+                          PGMPAGETYPE_RAM, PGM_PAGE_STATE_ALLOCATED);
+
+        /* Update the page count stats. */
+        pVM->pgm.s.cPrivatePages += cPages;
+        pVM->pgm.s.cAllPages     += cPages;
+    }
+#endif
 
     /*
@@ -1672 +1759 @@
      */
     pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
+
+    /*
+     * Notify NEM now that it has been linked.
+     */
+    int rc = NEMR3NotifyPhysRamRegister(pVM, GCPhys, pNew->cb, pNew->pvR3);
+    if (RT_FAILURE(rc))
+        pgmR3PhysUnlinkRamRange2(pVM, pNew, pPrev);
+    return rc;
 }
 
@@ -1757 +1852 @@
          * We push these in below the HMA.
          */
-        RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
-#ifndef PGM_WITHOUT_MAPPINGS
+#ifdef PGM_WITHOUT_MAPPINGS
+        RTGCPTR const GCPtrChunk = NIL_RTGCPTR;
+        RT_NOREF(cbChunk);
+#else
+        RTGCPTR const GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
         rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /*fFlags*/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
         if (RT_SUCCESS(rc))
-#endif /* !PGM_WITHOUT_MAPPINGS */
         {
             pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
 
             RTGCPTR const   GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
-#ifndef PGM_WITHOUT_MAPPINGS
             RTGCPTR         GCPtrPage  = GCPtrChunk;
             for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
@@ -1776 +1872 @@
                  * Ok, init and link the range.
                  */
-                pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
-                                             (RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
-                *ppPrev = pNew;
+                rc = pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
+                                                  (RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
+                if (RT_SUCCESS(rc))
+                    *ppPrev = pNew;
             }
-        }
+#ifndef PGM_WITHOUT_MAPPINGS
+        }
+#endif
 
         if (RT_FAILURE(rc))
@@ -1897 +1996 @@
         PPGMRAMRANGE pNew;
         rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
-        AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
-
-        pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
+        AssertLogRelMsgRCReturn(rc, ("rc=%Rrc cbRamRange=%zu\n", rc, cbRamRange), rc);
+
+        rc = pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, MMHyperCCToR0(pVM, pNew), pszDesc, pPrev);
+        AssertLogRelMsgRCReturn(rc, ("rc=%Rrc cbRamRange=%zu\n", rc, cbRamRange), rc);
     }
     pgmPhysInvalidatePageMapTLB(pVM);
 
-    /*
-     * Notify NEM while holding the lock (experimental) and REM without (like always).
-     */
-    rc = NEMR3NotifyPhysRamRegister(pVM, GCPhys, cb);
     PGM_UNLOCK(pVM);
     return rc;
@@ -1926 +2022 @@
     Assert(pVM->pgm.s.fRamPreAlloc);
     Log(("pgmR3PhysRamPreAllocate: enter\n"));
+#ifdef VBOX_WITH_PGM_NEM_MODE
+    AssertLogRelReturn(!pVM->pgm.s.fNemMode, VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE);
+#endif
 
     /*
@@ -2100 +2199 @@
 
         if (   !pVM->pgm.s.fRamPreAlloc
+#ifdef VBOX_WITH_PGM_NEM_MODE
+            && !pVM->pgm.s.fNemMode
+#endif
             && pVM->pgm.s.fZeroRamPagesOnReset)
         {
@@ -2135 +2237 @@
                     case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone?  I don't think VT-x copes with this code. */
                         pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
-                                                           true /*fDoAccounting*/);
+                                                           pRam, true /*fDoAccounting*/);
                         break;
 
@@ -2188 +2290 @@
                     case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone?  I don't think VT-x copes with this code. */
                         pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
-                                                           true /*fDoAccounting*/);
+                                                           pRam, true /*fDoAccounting*/);
                         break;
 
@@ -2396 +2498 @@
          * for PCI memory, but we're doing the same thing for MMIO2 pages.
          */
-        rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
+        rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, NULL);
         AssertRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
 
@@ -2409 +2511 @@
     else
     {
-
         /*
          * No RAM range, insert an ad hoc one.
@@ -2418 +2519 @@
         Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
 
+        /* Alloc. */
         const uint32_t cPages = cb >> PAGE_SHIFT;
         const size_t cbRamRange = RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]);
         rc = MMHyperAlloc(pVM, RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
         AssertLogRelMsgRCReturnStmt(rc, ("cbRamRange=%zu\n", cbRamRange), PGM_UNLOCK(pVM), rc);
+
+#ifdef VBOX_WITH_NATIVE_NEM
+        /* Notify NEM. */
+        uint8_t u2State = 0; /* (must have valid state as there can't be anything to preserve) */
+        if (VM_IS_NEM_ENABLED(pVM))
+        {
+            rc = NEMR3NotifyPhysMmioExMapEarly(pVM, GCPhys, cPages << PAGE_SHIFT, 0 /*fFlags*/, NULL, NULL, &u2State);
+            AssertLogRelRCReturnStmt(rc, MMHyperFree(pVM, pNew), rc);
+        }
+#endif
 
         /* Initialize the range. */
@@ -2435 +2547 @@
         uint32_t iPage = cPages;
         while (iPage-- > 0)
+        {
             PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
+#ifdef VBOX_WITH_NATIVE_NEM
+            PGM_PAGE_SET_NEM_STATE(&pNew->aPages[iPage], u2State);
+#endif
+        }
         Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
 
@@ -2450 +2567 @@
      */
     rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, hType, pvUserR3, pvUserR0, pvUserRC, pszDesc);
-    if (    RT_FAILURE(rc)
-        &&  !fRamExists)
+    if (RT_SUCCESS(rc))
+    {
+#ifdef VBOX_WITH_NATIVE_NEM
+        /* Late NEM notification. */
+        if (VM_IS_NEM_ENABLED(pVM))
+        {
+            uint32_t const fNemNotify = (fRamExists ? NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE : 0);
+            rc = NEMR3NotifyPhysMmioExMapLate(pVM, GCPhys, GCPhysLast - GCPhys + 1, fNemNotify,
+                                              fRamExists ? (uint8_t *)pRam->pvR3 + (uintptr_t)(GCPhys - pRam->GCPhys) : NULL,
+                                              NULL);
+            AssertLogRelRCReturn(rc, rc);
+        }
+#endif
+    }
+    /** @todo the phys handler failure handling isn't complete, esp. wrt NEM. */
+    else if (!fRamExists)
     {
         pVM->pgm.s.cPureMmioPages -= cb >> PAGE_SHIFT;
@@ -2532 +2663 @@
                     Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
                          GCPhys, GCPhysLast, pRam->pszDesc));
+                    /** @todo check the ad-hoc flags? */
+
+#ifdef VBOX_WITH_NATIVE_NEM
+                    if (VM_IS_NEM_ENABLED(pVM)) /* Notify REM before we unlink the range. */
+                    {
+                        rc = NEMR3NotifyPhysMmioExUnmap(pVM, GCPhys, GCPhysLast - GCPhys + 1, 0 /*fFlags*/, NULL, NULL, NULL);
+                        AssertLogRelRCReturn(rc, rc);
+                    }
+#endif
 
                     pVM->pgm.s.cAllPages      -= cPages;
@@ -2568 +2708 @@
                     iPage++;
                 }
+
+#ifdef VBOX_WITH_NATIVE_NEM
+                /* Notify REM (failure will probably leave things in a non-working state). */
+                if (VM_IS_NEM_ENABLED(pVM))
+                {
+                    uint8_t u2State = UINT8_MAX;
+                    rc = NEMR3NotifyPhysMmioExUnmap(pVM, GCPhys, GCPhysLast - GCPhys + 1, NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE,
+                                                    pRam->pvR3 ? (uint8_t *)pRam->pvR3 + GCPhys - pRam->GCPhys : NULL,
+                                                    NULL, &u2State);
+                    AssertLogRelRCReturn(rc, rc);
+                    if (u2State != UINT8_MAX)
+                        pgmPhysSetNemStateForPages(&pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT],
+                                                   cb >> PAGE_SHIFT, u2State);
+                }
+#endif
                 break;
             }
@@ -3384 +3539 @@
          * for PCI memory, but we're doing the same thing for MMIO2 pages.
          *
-         * We replace this MMIO/ZERO pages with real pages in the MMIO2 case.
+         * We replace these MMIO/ZERO pages with real pages in the MMIO2 case.
          */
         Assert(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK); /* Only one chunk */
-
-        int rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
+        Assert(pFirstMmio->pvR3 == pFirstMmio->RamRange.pvR3);
+        Assert(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2
+               ? pFirstMmio->RamRange.pvR3 != NULL : pFirstMmio->RamRange.pvR3 == NULL);
+
+#ifdef VBOX_WITH_PGM_NEM_MODE
+        /* We cannot mix MMIO2 into a RAM range in simplified memory mode because pRam->pvR3 can't point
+           both at the RAM and MMIO2, so we won't ever write & read from the actual MMIO2 memory if we try. */
+        AssertLogRelMsgReturn(!pVM->pgm.s.fNemMode || !(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2),
+                              ("%s at %RGp-%RGp\n", pFirstMmio->RamRange.pszDesc, GCPhys, GCPhysLast),
+                              VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE);
+#endif
+
+        int rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, pFirstMmio->RamRange.pvR3);
         AssertRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
 
@@ -3410 +3576 @@
                 PGM_PAGE_SET_PTE_INDEX(pVM, pPageDst, 0);
                 PGM_PAGE_SET_TRACKING(pVM, pPageDst, 0);
-                /* (We tell NEM at the end of the function.) */
+                /* NEM state is set by pgmR3PhysFreePageRange. */
 
                 pVM->pgm.s.cZeroPages--;
@@ -3436 +3602 @@
         for (PPGMREGMMIO2RANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
         {
+#ifdef VBOX_WITH_NATIVE_NEM
+            /* Tell NEM and get the new NEM state for the pages. */
+            uint8_t u2NemState = 0;
+            if (VM_IS_NEM_ENABLED(pVM))
+            {
+                int rc = NEMR3NotifyPhysMmioExMapEarly(pVM, pCurMmio->RamRange.GCPhys,
+                                                       pCurMmio->RamRange.GCPhysLast - pCurMmio->RamRange.GCPhys + 1,
+                                                       pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2
+                                                       ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2 : 0,
+                                                       NULL, pCurMmio->RamRange.pvR3, &u2NemState);
+                AssertLogRelRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
+            }
+#endif
+
             /* Clear the tracking data of pages we're going to reactivate. */
             PPGMPAGE pPageSrc = &pCurMmio->RamRange.aPages[0];
@@ -3443 +3623 @@
                 PGM_PAGE_SET_TRACKING(pVM, pPageSrc, 0);
                 PGM_PAGE_SET_PTE_INDEX(pVM, pPageSrc, 0);
+#ifdef VBOX_WITH_NATIVE_NEM
+                PGM_PAGE_SET_NEM_STATE(pPageSrc, u2NemState);
+#endif
                 pPageSrc++;
             }
@@ -3487 +3670 @@
                 {
                     pCurMmio->fFlags &= ~PGMREGMMIO2RANGE_F_MAPPED;
-                    pgmHandlerPhysicalExDeregister(pVM, pCurMmio->pPhysHandlerR3, fRamExists);
+                    pgmHandlerPhysicalExDeregister(pVM, pCurMmio->pPhysHandlerR3);
                 }
 
@@ -3511 +3694 @@
             }
 
+            /** @todo NEM notification cleanup */
             PGM_UNLOCK(pVM);
             return rc;
@@ -3531 +3715 @@
     pgmPhysInvalidatePageMapTLB(pVM);
 
-    /*
-     * Notify NEM while holding the lock (experimental) and REM without (like always).
-     */
-    uint32_t const fNemNotify = (pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2       ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2   : 0)
-                              | (pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_OVERLAPPING ? NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE : 0);
-    int rc = NEMR3NotifyPhysMmioExMap(pVM, GCPhys, cbRange, fNemNotify, pFirstMmio->pvR3);
+#ifdef VBOX_WITH_NATIVE_NEM
+    /*
+     * Late NEM notification.
+     */
+    if (VM_IS_NEM_ENABLED(pVM))
+    {
+        int rc;
+        uint32_t fNemFlags = pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2 ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2 : 0;
+        if (fRamExists)
+            rc = NEMR3NotifyPhysMmioExMapLate(pVM, GCPhys, GCPhysLast - GCPhys + 1, fNemFlags | NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE,
+                                              pRam->pvR3 ? (uint8_t *)pRam->pvR3 + GCPhys - pRam->GCPhys : NULL, pFirstMmio->pvR3);
+        else
+        {
+            rc = VINF_SUCCESS;
+            for (PPGMREGMMIO2RANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
+            {
+                rc = NEMR3NotifyPhysMmioExMapLate(pVM, pCurMmio->RamRange.GCPhys, pCurMmio->RamRange.cb, fNemFlags,
+                                                  NULL, pCurMmio->RamRange.pvR3);
+                if ((pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK) || RT_FAILURE(rc))
+                    break;
+            }
+        }
+        AssertLogRelRCReturnStmt(rc, PGMR3PhysMmio2Unmap(pVM, pDevIns, hMmio2, GCPhys); PGM_UNLOCK(pVM), rc);
+    }
+#endif
 
     PGM_UNLOCK(pVM);
 
-    return rc;
+    return VINF_SUCCESS;
 }
 
@@ -3601 +3804 @@
 
         PPGMREGMMIO2RANGE pCurMmio = pFirstMmio;
-        rc = pgmHandlerPhysicalExDeregister(pVM, pFirstMmio->pPhysHandlerR3, RT_BOOL(fOldFlags & PGMREGMMIO2RANGE_F_OVERLAPPING));
+        rc = pgmHandlerPhysicalExDeregister(pVM, pFirstMmio->pPhysHandlerR3);
         AssertRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
         while (!(pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK))
         {
             pCurMmio = pCurMmio->pNextR3;
-            rc = pgmHandlerPhysicalExDeregister(pVM, pCurMmio->pPhysHandlerR3, RT_BOOL(fOldFlags & PGMREGMMIO2RANGE_F_OVERLAPPING));
+            rc = pgmHandlerPhysicalExDeregister(pVM, pCurMmio->pPhysHandlerR3);
             AssertRCReturnStmt(rc, PGM_UNLOCK(pVM), VERR_PGM_PHYS_MMIO_EX_IPE);
         }
@@ -3614 +3817 @@
      * Unmap it.
      */
-    RTGCPHYS const GCPhysRangeNotify = pFirstMmio->RamRange.GCPhys;
+    int            rcRet     = VINF_SUCCESS;
+#ifdef VBOX_WITH_NATIVE_NEM
+    uint32_t const fNemFlags = pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_MMIO2 ?  NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2 : 0;
+#endif
     if (fOldFlags & PGMREGMMIO2RANGE_F_OVERLAPPING)
     {
@@ -3632 +3838 @@
             pRam = pRam->pNextR3;
 
+        PPGMPAGE pPageDst   = &pRam->aPages[(pFirstMmio->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
         uint32_t cPagesLeft = pFirstMmio->RamRange.cb >> PAGE_SHIFT;
         if (fOldFlags & PGMREGMMIO2RANGE_F_MMIO2)
             pVM->pgm.s.cZeroPages += cPagesLeft;
 
-        PPGMPAGE pPageDst = &pRam->aPages[(pFirstMmio->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+#ifdef VBOX_WITH_NATIVE_NEM
+        if (VM_IS_NEM_ENABLED(pVM)) /* Notify NEM. Note! we cannot be here in simple memory mode, see mapping function. */
+        {
+            uint8_t u2State = UINT8_MAX;
+            rc = NEMR3NotifyPhysMmioExUnmap(pVM, pFirstMmio->RamRange.GCPhys, pFirstMmio->RamRange.cb,
+                                            fNemFlags | NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE,
+                                            pRam->pvR3
+                                            ? (uint8_t *)pRam->pvR3 + pFirstMmio->RamRange.GCPhys - pRam->GCPhys : NULL,
+                                            pFirstMmio->pvR3, &u2State);
+            AssertRCStmt(rc, rcRet = rc);
+            if (u2State != UINT8_MAX)
+                pgmPhysSetNemStateForPages(pPageDst, cPagesLeft, u2State);
+        }
+#endif
+
         while (cPagesLeft-- > 0)
         {
@@ -3658 +3879 @@
         for (PPGMREGMMIO2RANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
         {
+#ifdef VBOX_WITH_NATIVE_NEM
+            if (VM_IS_NEM_ENABLED(pVM)) /* Notify NEM. */
+            {
+                uint8_t u2State = UINT8_MAX;
+                rc = NEMR3NotifyPhysMmioExUnmap(pVM, pCurMmio->RamRange.GCPhys, pCurMmio->RamRange.cb, fNemFlags,
+                                                NULL, pCurMmio->pvR3, &u2State);
+                AssertRCStmt(rc, rcRet = rc);
+                if (u2State != UINT8_MAX)
+                    pgmPhysSetNemStateForPages(pCurMmio->RamRange.aPages, pCurMmio->RamRange.cb >> PAGE_SHIFT, u2State);
+            }
+#endif
             pgmR3PhysUnlinkRamRange(pVM, &pCurMmio->RamRange);
             pCurMmio->RamRange.GCPhys = NIL_RTGCPHYS;
@@ -3677 +3909 @@
     pgmPhysInvalidatePageMapTLB(pVM);
     pgmPhysInvalidRamRangeTlbs(pVM);
-
-    /*
-     * Notify NEM while holding the lock (experimental) and REM without (like always).
-     */
-    uint32_t const fNemFlags = (fOldFlags & PGMREGMMIO2RANGE_F_MMIO2       ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2   : 0)
-                             | (fOldFlags & PGMREGMMIO2RANGE_F_OVERLAPPING ? NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE : 0);
-    rc = NEMR3NotifyPhysMmioExUnmap(pVM, GCPhysRangeNotify, cbRange, fNemFlags);
 
     PGM_UNLOCK(pVM);
@@ -3931 +4156 @@
  */
 static int pgmR3PhysRomRegisterLocked(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
-                                      const void *pvBinary, uint32_t cbBinary, uint32_t fFlags, const char *pszDesc)
+                                      const void *pvBinary, uint32_t cbBinary, uint8_t fFlags, const char *pszDesc)
 {
     /*
@@ -3969 +4194 @@
      * Find the RAM location and check for conflicts.
      *
-     * Conflict detection is a bit different than for RAM
-     * registration since a ROM can be located within a RAM
-     * range. So, what we have to check for is other memory
-     * types (other than RAM that is) and that we don't span
-     * more than one RAM range (layz).
+     * Conflict detection is a bit different than for RAM registration since a
+     * ROM can be located within a RAM range. So, what we have to check for is
+     * other memory types (other than RAM that is) and that we don't span more
+     * than one RAM range (lazy).
      */
     bool            fRamExists = false;
@@ -4008 +4232 @@
                                    pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
                                    pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
-            Assert(PGM_PAGE_IS_ZERO(pPage));
+            Assert(PGM_PAGE_IS_ZERO(pPage) || PGM_IS_IN_NEM_MODE(pVM));
             pPage++;
         }
@@ -4026 +4250 @@
     }
 
-    /*
-     * Allocate memory for the virgin copy of the RAM.
-     */
-    PGMMALLOCATEPAGESREQ pReq;
-    int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
-    AssertRCReturn(rc, rc);
-
-    for (uint32_t iPage = 0; iPage < cPages; iPage++)
-    {
-        pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
-        pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
-        pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
-    }
-
-    rc = GMMR3AllocatePagesPerform(pVM, pReq);
-    if (RT_FAILURE(rc))
-    {
-        GMMR3AllocatePagesCleanup(pReq);
-        return rc;
+#ifdef VBOX_WITH_NATIVE_NEM
+    /*
+     * Early NEM notification before we've made any changes or anything.
+     */
+    uint32_t const fNemNotify = (fRamExists ? NEM_NOTIFY_PHYS_ROM_F_REPLACE : 0)
+                              | (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED ? NEM_NOTIFY_PHYS_ROM_F_SHADOW : 0);
+    uint8_t        u2NemState = UINT8_MAX;
+    if (VM_IS_NEM_ENABLED(pVM))
+    {
+        int rc = NEMR3NotifyPhysRomRegisterEarly(pVM, GCPhys, cPages << PAGE_SHIFT,
+                                                fRamExists ? PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhys) : NULL,
+                                                fNemNotify, &u2NemState);
+        AssertLogRelRCReturn(rc, rc);
+    }
+#endif
+
+    /*
+     * Allocate memory for the virgin copy of the RAM.  In simplified memory mode,
+     * we allocate memory for any ad-hoc RAM range and for shadow pages.
+     */
+    PGMMALLOCATEPAGESREQ pReq  = NULL;
+#ifdef VBOX_WITH_PGM_NEM_MODE
+    void                *pvRam = NULL;
+    void                *pvAlt = NULL;
+    if (pVM->pgm.s.fNemMode)
+    {
+        if (!fRamExists)
+        {
+            int rc = SUPR3PageAlloc(cPages, &pvRam);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+        if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
+        {
+            int rc = SUPR3PageAlloc(cPages, &pvAlt);
+            if (RT_FAILURE(rc))
+            {
+                if (pvRam)
+                    SUPR3PageFree(pvRam, cPages);
+                return rc;
+            }
+        }
+    }
+    else
+#endif
+    {
+        int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
+        AssertRCReturn(rc, rc);
+
+        for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        {
+            pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
+            pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
+            pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
+        }
+
+        rc = GMMR3AllocatePagesPerform(pVM, pReq);
+        if (RT_FAILURE(rc))
+        {
+            GMMR3AllocatePagesCleanup(pReq);
+            return rc;
+        }
     }
 
@@ -4051 +4318 @@
      */
     PPGMROMRANGE pRomNew;
-    rc = MMHyperAlloc(pVM, RT_UOFFSETOF_DYN(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
+    int rc = MMHyperAlloc(pVM, RT_UOFFSETOF_DYN(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
     if (RT_SUCCESS(rc))
     {
@@ -4062 +4329 @@
              * Initialize and insert the RAM range (if required).
              */
+            uint32_t const idxFirstRamPage = fRamExists ? (GCPhys - pRam->GCPhys) >> PAGE_SHIFT : 0;
             PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
             if (!fRamExists)
             {
+                /* New RAM range. */
                 pRamNew->pSelfR0       = MMHyperCCToR0(pVM, pRamNew);
                 pRamNew->GCPhys        = GCPhys;
@@ -4074 +4343 @@
                 pRamNew->paLSPages     = NULL;
 
-                PPGMPAGE pPage = &pRamNew->aPages[0];
-                for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
+                PPGMPAGE pRamPage = &pRamNew->aPages[idxFirstRamPage];
+#ifdef VBOX_WITH_PGM_NEM_MODE
+                if (pVM->pgm.s.fNemMode)
                 {
-                    PGM_PAGE_INIT(pPage,
-                                  pReq->aPages[iPage].HCPhysGCPhys,
-                                  pReq->aPages[iPage].idPage,
-                                  PGMPAGETYPE_ROM,
-                                  PGM_PAGE_STATE_ALLOCATED);
-
-                    pRomPage->Virgin = *pPage;
+                    AssertPtr(pvRam); Assert(pReq == NULL);
+                    pRamNew->pvR3 = pvRam;
+                    for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++, pRomPage++)
+                    {
+                        PGM_PAGE_INIT(pRamPage, UINT64_C(0x0000fffffffff000), NIL_GMM_PAGEID,
+                                      PGMPAGETYPE_ROM, PGM_PAGE_STATE_ALLOCATED);
+                        pRomPage->Virgin = *pRamPage;
+                    }
                 }
-
-                pVM->pgm.s.cAllPages += cPages;
+                else
+#endif
+                    for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++, pRomPage++)
+                    {
+                        PGM_PAGE_INIT(pRamPage,
+                                      pReq->aPages[iPage].HCPhysGCPhys,
+                                      pReq->aPages[iPage].idPage,
+                                      PGMPAGETYPE_ROM,
+                                      PGM_PAGE_STATE_ALLOCATED);
+
+                        pRomPage->Virgin = *pRamPage;
+                    }
+
+                pVM->pgm.s.cAllPages     += cPages;
+                pVM->pgm.s.cPrivatePages += cPages;
                 pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
             }
             else
             {
-                PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
-                for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
+                /* Existing RAM range. */
+                PPGMPAGE pRamPage = &pRam->aPages[idxFirstRamPage];
+#ifdef VBOX_WITH_PGM_NEM_MODE
+                if (pVM->pgm.s.fNemMode)
                 {
-                    PGM_PAGE_SET_TYPE(pVM, pPage,   PGMPAGETYPE_ROM);
-                    PGM_PAGE_SET_HCPHYS(pVM, pPage, pReq->aPages[iPage].HCPhysGCPhys);
-                    PGM_PAGE_SET_STATE(pVM, pPage,  PGM_PAGE_STATE_ALLOCATED);
-                    PGM_PAGE_SET_PAGEID(pVM, pPage, pReq->aPages[iPage].idPage);
-                    PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
-                    PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
-                    PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
-
-                    pRomPage->Virgin = *pPage;
+                    Assert(pvRam == NULL); Assert(pReq == NULL);
+                    for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++, pRomPage++)
+                    {
+                        Assert(PGM_PAGE_GET_HCPHYS(pRamPage) == UINT64_C(0x0000fffffffff000));
+                        Assert(PGM_PAGE_GET_PAGEID(pRamPage) == NIL_GMM_PAGEID);
+                        Assert(PGM_PAGE_GET_STATE(pRamPage) == PGM_PAGE_STATE_ALLOCATED);
+                        PGM_PAGE_SET_TYPE(pVM, pRamPage, PGMPAGETYPE_ROM);
+                        PGM_PAGE_SET_STATE(pVM, pRamPage,  PGM_PAGE_STATE_ALLOCATED);
+                        PGM_PAGE_SET_PDE_TYPE(pVM, pRamPage, PGM_PAGE_PDE_TYPE_DONTCARE);
+                        PGM_PAGE_SET_PTE_INDEX(pVM, pRamPage, 0);
+                        PGM_PAGE_SET_TRACKING(pVM, pRamPage, 0);
+
+                        pRomPage->Virgin = *pRamPage;
+                    }
                 }
-
+                else
+#endif
+                {
+                    for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++, pRomPage++)
+                    {
+                        PGM_PAGE_SET_TYPE(pVM, pRamPage,   PGMPAGETYPE_ROM);
+                        PGM_PAGE_SET_HCPHYS(pVM, pRamPage, pReq->aPages[iPage].HCPhysGCPhys);
+                        PGM_PAGE_SET_STATE(pVM, pRamPage,  PGM_PAGE_STATE_ALLOCATED);
+                        PGM_PAGE_SET_PAGEID(pVM, pRamPage, pReq->aPages[iPage].idPage);
+                        PGM_PAGE_SET_PDE_TYPE(pVM, pRamPage, PGM_PAGE_PDE_TYPE_DONTCARE);
+                        PGM_PAGE_SET_PTE_INDEX(pVM, pRamPage, 0);
+                        PGM_PAGE_SET_TRACKING(pVM, pRamPage, 0);
+
+                        pRomPage->Virgin = *pRamPage;
+                    }
+                    pVM->pgm.s.cZeroPages    -= cPages;
+                    pVM->pgm.s.cPrivatePages += cPages;
+                }
                 pRamNew = pRam;
-
-                pVM->pgm.s.cZeroPages -= cPages;
             }
-            pVM->pgm.s.cPrivatePages += cPages;
+
+            /* Set the NEM state of the pages if needed. */
+            if (u2NemState != UINT8_MAX)
+                pgmPhysSetNemStateForPages(&pRamNew->aPages[idxFirstRamPage], cPages, u2NemState);
 
             /* Flush physical page map TLB. */
             pgmPhysInvalidatePageMapTLB(pVM);
 
-
-            /* Notify NEM before we register handlers. */
-            uint32_t const fNemNotify = (fRamExists ? NEM_NOTIFY_PHYS_ROM_F_REPLACE : 0)
-                                      | (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED ? NEM_NOTIFY_PHYS_ROM_F_SHADOW : 0);
-            rc = NEMR3NotifyPhysRomRegisterEarly(pVM, GCPhys, cb, fNemNotify);
-
-            /* Register the ROM access handler. */
-            if (RT_SUCCESS(rc))
-                rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, pVM->pgm.s.hRomPhysHandlerType,
-                                                pRomNew, MMHyperCCToR0(pVM, pRomNew), MMHyperCCToRC(pVM, pRomNew),
-                                                pszDesc);
+            /*
+             * Register the ROM access handler.
+             */
+            rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, pVM->pgm.s.hRomPhysHandlerType,
+                                            pRomNew, MMHyperCCToR0(pVM, pRomNew), MMHyperCCToRC(pVM, pRomNew), pszDesc);
             if (RT_SUCCESS(rc))
             {
@@ -4132 +4435 @@
                  */
                 size_t          cbBinaryLeft = cbBinary;
-                PPGMPAGE        pRamPage     = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
+                PPGMPAGE        pRamPage     = &pRamNew->aPages[idxFirstRamPage];
                 for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
                 {
@@ -4170 +4473 @@
                     pRomNew->cbOriginal = cbBinary;
                     pRomNew->pszDesc    = pszDesc;
+#ifdef VBOX_WITH_PGM_NEM_MODE
+                    pRomNew->pbR3Alternate = (uint8_t *)pvAlt;
+#endif
                     pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
                                         ? pvBinary : RTMemDup(pvBinary, cbBinary);
@@ -4178 +4484 @@
                             PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
                             pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
-                            PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
+#ifdef VBOX_WITH_PGM_NEM_MODE
+                            if (pVM->pgm.s.fNemMode)
+                                PGM_PAGE_INIT(&pPage->Shadow, UINT64_C(0x0000fffffffff000), NIL_GMM_PAGEID,
+                                              PGMPAGETYPE_ROM_SHADOW, PGM_PAGE_STATE_ALLOCATED);
+                            else
+#endif
+                                PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
                         }
 
@@ -4184 +4496 @@
                         if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
                         {
-                            pVM->pgm.s.cZeroPages += cPages;
-                            pVM->pgm.s.cAllPages  += cPages;
+#ifdef VBOX_WITH_PGM_NEM_MODE
+                            if (pVM->pgm.s.fNemMode)
+                                pVM->pgm.s.cPrivatePages += cPages;
+                            else
+#endif
+                                pVM->pgm.s.cZeroPages += cPages;
+                            pVM->pgm.s.cAllPages += cPages;
                         }
 
@@ -4206 +4523 @@
 
                         pgmPhysInvalidatePageMapTLB(pVM);
-                        GMMR3AllocatePagesCleanup(pReq);
-
-                        /* Notify NEM again. */
-                        return NEMR3NotifyPhysRomRegisterLate(pVM, GCPhys, cb, fNemNotify);
+#ifdef VBOX_WITH_PGM_NEM_MODE
+                        if (!pVM->pgm.s.fNemMode)
+#endif
+                            GMMR3AllocatePagesCleanup(pReq);
+
+                        /*
+                         * Notify NEM again.
+                         */
+                        u2NemState = UINT8_MAX;
+                        rc = NEMR3NotifyPhysRomRegisterLate(pVM, GCPhys, cb, PGM_RAMRANGE_CALC_PAGE_R3PTR(pRamNew, GCPhys),
+                                                            fNemNotify, &u2NemState);
+                        if (u2NemState != UINT8_MAX)
+                            pgmPhysSetNemStateForPages(&pRamNew->aPages[idxFirstRamPage], cPages, u2NemState);
+                        if (RT_SUCCESS(rc))
+                            return rc;
+
+                        /*
+                         * bail out
+                         */
+                        /* unlink */
+                        if (pRomPrev)
+                        {
+                            pRomPrev->pNextR3 = pRom;
+                            pRomPrev->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
+                        }
+                        else
+                        {
+                            pVM->pgm.s.pRomRangesR3 = pRom;
+                            pVM->pgm.s.pRomRangesR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
+                        }
+
+                        if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
+                        {
+#ifdef VBOX_WITH_PGM_NEM_MODE
+                            if (pVM->pgm.s.fNemMode)
+                                pVM->pgm.s.cPrivatePages -= cPages;
+                            else
+#endif
+                                pVM->pgm.s.cZeroPages -= cPages;
+                            pVM->pgm.s.cAllPages -= cPages;
+                        }
+
                     }
-
-                    /* bail out */
-                    rc = VERR_NO_MEMORY;
+                    else
+                        rc = VERR_NO_MEMORY;
                 }
 
@@ -4225 +4579 @@
                 MMHyperFree(pVM, pRamNew);
             }
+            else
+            {
+                PPGMPAGE pRamPage = &pRam->aPages[idxFirstRamPage];
+#ifdef VBOX_WITH_PGM_NEM_MODE
+                if (pVM->pgm.s.fNemMode)
+                {
+                    Assert(pvRam == NULL); Assert(pReq == NULL);
+                    for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++, pRomPage++)
+                    {
+                        Assert(PGM_PAGE_GET_HCPHYS(pRamPage) == UINT64_C(0x0000fffffffff000));
+                        Assert(PGM_PAGE_GET_PAGEID(pRamPage) == NIL_GMM_PAGEID);
+                        Assert(PGM_PAGE_GET_STATE(pRamPage) == PGM_PAGE_STATE_ALLOCATED);
+                        PGM_PAGE_SET_TYPE(pVM, pRamPage, PGMPAGETYPE_RAM);
+                        PGM_PAGE_SET_STATE(pVM, pRamPage,  PGM_PAGE_STATE_ALLOCATED);
+                    }
+                }
+                else
+#endif
+                {
+                    for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
+                        PGM_PAGE_INIT_ZERO(pRamPage, pVM, PGMPAGETYPE_RAM);
+                    pVM->pgm.s.cZeroPages    += cPages;
+                    pVM->pgm.s.cPrivatePages -= cPages;
+                }
+            }
         }
         MMHyperFree(pVM, pRomNew);
@@ -4230 +4609 @@
 
     /** @todo Purge the mapping cache or something... */
-    GMMR3FreeAllocatedPages(pVM, pReq);
-    GMMR3AllocatePagesCleanup(pReq);
+#ifdef VBOX_WITH_PGM_NEM_MODE
+    if (pVM->pgm.s.fNemMode)
+    {
+        Assert(!pReq);
+        if (pvRam)
+            SUPR3PageFree(pvRam, cPages);
+        if (pvAlt)
+            SUPR3PageFree(pvAlt, cPages);
+    }
+    else
+#endif
+    {
+        GMMR3FreeAllocatedPages(pVM, pReq);
+        GMMR3AllocatePagesCleanup(pReq);
+    }
     return rc;
 }
@@ -4267 +4659 @@
  */
 VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
-                                    const void *pvBinary, uint32_t cbBinary, uint32_t fFlags, const char *pszDesc)
+                                    const void *pvBinary, uint32_t cbBinary, uint8_t fFlags, const char *pszDesc)
 {
     Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
@@ -4308 +4700 @@
              * out all the dirty pages and replace them by the zero page.
              */
+#ifdef VBOX_WITH_PGM_NEM_MODE
+            if (pVM->pgm.s.fNemMode)
+            {
+                /* Clear all the shadow pages (currently using alternate backing). */
+                RT_BZERO(pRom->pbR3Alternate, pRom->cb);
+            }
+            else
+#endif
             if (!pVM->pgm.s.fRamPreAlloc)
             {
@@ -4467 +4867 @@
              * Iterate the relevant pages and make necessary the changes.
              */
+#ifdef VBOX_WITH_NATIVE_NEM
+            PPGMRAMRANGE const pRam = pgmPhysGetRange(pVM, GCPhys);
+            AssertPtrReturn(pRam, VERR_INTERNAL_ERROR_3);
+#endif
             bool fChanges = false;
             uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
@@ -4481 +4885 @@
 
                     /* flush references to the page. */
-                    PPGMPAGE pRamPage = pgmPhysGetPage(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT));
-                    int rc2 = pgmPoolTrackUpdateGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage,
-                                                       true /*fFlushPTEs*/, &fFlushTLB);
+                    RTGCPHYS const GCPhysPage = pRom->GCPhys + (iPage << PAGE_SHIFT);
+                    PPGMPAGE pRamPage = pgmPhysGetPage(pVM, GCPhysPage);
+                    int rc2 = pgmPoolTrackUpdateGCPhys(pVM, GCPhysPage, pRamPage, true /*fFlushPTEs*/, &fFlushTLB);
                     if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS || RT_FAILURE(rc2)))
                         rc = rc2;
@@ -4495 +4899 @@
                     /** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
 
+#ifdef VBOX_WITH_NATIVE_NEM
+# ifdef VBOX_WITH_PGM_NEM_MODE
+                    /* In simplified mode we have to switch the page data around too. */
+                    if (pVM->pgm.s.fNemMode)
+                    {
+                        uint8_t         abPage[PAGE_SIZE];
+                        uint8_t * const pbRamPage = PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage);
+                        memcpy(abPage, &pRom->pbR3Alternate[(size_t)iPage << PAGE_SHIFT], sizeof(abPage));
+                        memcpy(&pRom->pbR3Alternate[(size_t)iPage << PAGE_SHIFT], pbRamPage, sizeof(abPage));
+                        memcpy(pbRamPage, abPage, sizeof(abPage));
+                    }
+# endif
                     /* Tell NEM about the backing and protection change. */
                     if (VM_IS_NEM_ENABLED(pVM))
@@ -4500 +4916 @@
                         PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pNew);
                         NEMHCNotifyPhysPageChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pOld), PGM_PAGE_GET_HCPHYS(pNew),
+                                                   PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage),
                                                    pgmPhysPageCalcNemProtection(pRamPage, enmType), enmType, &u2State);
                         PGM_PAGE_SET_NEM_STATE(pRamPage, u2State);
                     }
+#endif
                 }
                 pRomPage->enmProt = enmProt;
@@ -5216 +5634 @@
  *
  * @param   pVM             The cross context VM structure.
- * @param   pReq            Pointer to the request.
+ * @param   pReq            Pointer to the request.  This is NULL when doing a
+ *                          bulk free in NEM memory mode.
  * @param   pcPendingPages  Where the number of pages waiting to be freed are
- *                          kept.  This will normally be incremented.
+ *                          kept.  This will normally be incremented.  This is
+ *                          NULL when doing a bulk free in NEM memory mode.
  * @param   pPage           Pointer to the page structure.
  * @param   GCPhys          The guest physical address of the page, if applicable.
@@ -5250 +5670 @@
     const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
     Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
-    if (RT_UNLIKELY(    idPage == NIL_GMM_PAGEID
-                    ||  idPage > GMM_PAGEID_LAST
-                    ||  PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
+    if (RT_UNLIKELY(!PGM_IS_IN_NEM_MODE(pVM)
+                    ?    idPage == NIL_GMM_PAGEID
+                      ||  idPage > GMM_PAGEID_LAST
+                      ||  PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID
+                    :    idPage != NIL_GMM_PAGEID))
     {
         AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
@@ -5286 +5708 @@
     pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
 
+#ifdef VBOX_WITH_PGM_NEM_MODE
+    /*
+     * Skip the rest if we're doing a bulk free in NEM memory mode.
+     */
+    if (!pReq)
+        return VINF_SUCCESS;
+    AssertLogRelReturn(!pVM->pgm.s.fNemMode, VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE);
+#endif
+
+#ifdef VBOX_WITH_NATIVE_NEM
     /* Notify NEM. */
-    /** @todo consider doing batch NEM notifications.  */
+    /** @todo Remove this one? */
     if (VM_IS_NEM_ENABLED(pVM))
     {
         uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
-        NEMHCNotifyPhysPageChanged(pVM, GCPhys, HCPhysPrev, pVM->pgm.s.HCPhysZeroPg,
+        NEMHCNotifyPhysPageChanged(pVM, GCPhys, HCPhysPrev, pVM->pgm.s.HCPhysZeroPg, pVM->pgm.s.pvZeroPgR3,
                                    pgmPhysPageCalcNemProtection(pPage, enmNewType), enmNewType, &u2State);
         PGM_PAGE_SET_NEM_STATE(pPage, u2State);
     }
+#endif
 
     /*