Index: /trunk/include/VBox/vmm/iom.h
===================================================================
--- /trunk/include/VBox/vmm/iom.h	(revision 81332)
+++ /trunk/include/VBox/vmm/iom.h	(revision 81333)
@@ -155,6 +155,11 @@
 #define IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_WRITE      UINT32_C(0x00000200)
 
+/** Pass the absolute physical address (GC) to the callback rather than the
+ * relative one.
+ * @note New-style only, is implicit in old-style interface.  */
+#define IOMMMIO_FLAGS_ABS                               UINT32_C(0x00001000)
+
 /** Mask of valid flags. */
-#define IOMMMIO_FLAGS_VALID_MASK                        UINT32_C(0x00000373)
+#define IOMMMIO_FLAGS_VALID_MASK                        UINT32_C(0x00001373)
 /** @} */
 
@@ -399,5 +404,5 @@
  * @param   pvUser      User argument.
  * @param   off         Offset into the mapping of the read,
- *                      or the physical address if IOM_MMIO_F_ABS is active.
+ *                      or the physical address if IOMMMIO_FLAGS_ABS is active.
  * @param   pv          Where to store the result.
  * @param   cb          Number of bytes read.
@@ -416,5 +421,5 @@
  * @param   pvUser      User argument.
  * @param   off         Offset into the mapping of the write,
- *                      or the physical address if IOM_MMIO_F_ABS is active.
+ *                      or the physical address if IOMMMIO_FLAGS_ABS is active.
  * @param   pv          Where to fetch the result.
  * @param   cb          Number of bytes to write.
@@ -433,5 +438,5 @@
  * @param   pvUser      User argument.
  * @param   off         Offset into the mapping of the fill,
- *                      or the physical address if IOM_MMIO_F_ABS is active.
+ *                      or the physical address if IOMMMIO_FLAGS_ABS is active.
  * @param   u32Item     Byte/Word/Dword data to fill.
  * @param   cbItem      Size of data in u32Item parameter, restricted to 1/2/4 bytes.
@@ -456,5 +461,11 @@
 VMMDECL(int)            IOMMMIOMapMMIOHCPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint64_t fPageFlags);
 VMMDECL(int)            IOMMMIOResetRegion(PVMCC pVM, RTGCPHYS GCPhys);
-VMMDECL(bool)           IOMIsLockWriteOwner(PVM pVM);
+
+VMM_INT_DECL(VBOXSTRICTRC)  IOMMmioPhysHandler(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, RTGCPHYS GCPhysFault);
+VMM_INT_DECL(int)           IOMMmioMapMmio2Page(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, RTGCPHYS offRegion,
+                                                uint64_t hMmio2, RTGCPHYS offMmio2, uint64_t fPageFlags);
+VMM_INT_DECL(int)           IOMMmioMapMmioHCPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint64_t fPageFlags);
+VMM_INT_DECL(int)           IOMMmioResetRegion(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion);
+
 
 /** @name IOM_IOPORT_F_XXX - Flags for IOMR3IoPortCreate() and PDMDevHlpIoPortCreateEx().
@@ -464,13 +475,4 @@
 /** Valid flags for IOMR3IoPortCreate(). */
 #define IOM_IOPORT_F_VALID_MASK     UINT32_C(0x00000001)
-/** @} */
-
-/** @name IOM_MMIO_F_XXX - Flags for IOMR3MmioCreate() and PDMDevHlpMmioCreateEx().
- * @{ */
-/** Pass the absolute physical address (GC) to the callback rather than the
- * relative one. */
-#define IOM_MMIO_F_ABS              RT_BIT_32(0)
-/** Valid flags for IOMR3IoPortCreate(). */
-#define IOM_MMIO_F_VALID_MASK       UINT32_C(0x00000001)
 /** @} */
 
@@ -574,4 +576,5 @@
 VMMR0_INT_DECL(int)  IOMR0IoPortGrowRegistrationTables(PGVM pGVM, uint64_t cMinEntries);
 VMMR0_INT_DECL(int)  IOMR0IoPortGrowStatisticsTable(PGVM pGVM, uint64_t cMinEntries);
+VMMR0_INT_DECL(int)  IOMR0IoPortSyncStatisticsIndices(PGVM pGVM);
 
 VMMR0_INT_DECL(int)  IOMR0MmioSetUpContext(PGVM pGVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, PFNIOMMMIONEWWRITE pfnWrite,
@@ -579,4 +582,6 @@
 VMMR0_INT_DECL(int)  IOMR0MmioGrowRegistrationTables(PGVM pGVM, uint64_t cMinEntries);
 VMMR0_INT_DECL(int)  IOMR0MmioGrowStatisticsTable(PGVM pGVM, uint64_t cMinEntries);
+VMMR0_INT_DECL(int)  IOMR0MmioSyncStatisticsIndices(PGVM pGVM);
+
 /** @} */
 #endif /* IN_RING0 || DOXYGEN_RUNNING */
Index: /trunk/include/VBox/vmm/pdmdev.h
===================================================================
--- /trunk/include/VBox/vmm/pdmdev.h	(revision 81332)
+++ /trunk/include/VBox/vmm/pdmdev.h	(revision 81333)
@@ -2418,5 +2418,5 @@
      * @param   pDevIns     The device instance to register the ports with.
      * @param   cbRegion    The size of the region in bytes.
-     * @param   fFlags      Reserved, MBZ.
+     * @param   fFlags      Flags, IOMMMIO_FLAGS_XXX.
      * @param   pPciDev     The PCI device the range is associated with, if
      *                      applicable.
Index: /trunk/include/VBox/vmm/vmm.h
===================================================================
--- /trunk/include/VBox/vmm/vmm.h	(revision 81332)
+++ /trunk/include/VBox/vmm/vmm.h	(revision 81333)
@@ -419,4 +419,6 @@
     /** Grow the MMIO statistics tables. */
     VMMR0_DO_IOM_GROW_MMIO_STATS,
+    /** Synchronize statistics indices for I/O ports and MMIO regions. */
+    VMMR0_DO_IOM_SYNC_STATS_INDICES,
 
     /** Official call we use for testing Ring-0 APIs. */
Index: /trunk/src/VBox/VMM/Makefile.kmk
===================================================================
--- /trunk/src/VBox/VMM/Makefile.kmk	(revision 81332)
+++ /trunk/src/VBox/VMM/Makefile.kmk	(revision 81333)
@@ -181,4 +181,5 @@
 	VMMAll/IOMAll.cpp \
 	VMMAll/IOMAllMMIO.cpp \
+	VMMAll/IOMAllMmioNew.cpp \
 	VMMAll/MMAll.cpp \
 	VMMAll/MMAllHyper.cpp \
@@ -510,4 +511,5 @@
 	VMMAll/IOMAll.cpp \
 	VMMAll/IOMAllMMIO.cpp \
+	VMMAll/IOMAllMmioNew.cpp \
 	VMMAll/MMAll.cpp \
 	VMMAll/MMAllHyper.cpp \
Index: /trunk/src/VBox/VMM/VMMAll/IOMAll.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/IOMAll.cpp	(revision 81332)
+++ /trunk/src/VBox/VMM/VMMAll/IOMAll.cpp	(revision 81333)
@@ -37,21 +37,4 @@
 #include <iprt/string.h>
 #include "IOMInline.h"
-
-
-/**
- * Check if this VCPU currently owns the IOM lock exclusively.
- *
- * @returns bool owner/not owner
- * @param   pVM         The cross context VM structure.
- */
-VMMDECL(bool) IOMIsLockWriteOwner(PVM pVM)
-{
-#ifdef IOM_WITH_CRIT_SECT_RW
-    return PDMCritSectRwIsInitialized(&pVM->iom.s.CritSect)
-        && PDMCritSectRwIsWriteOwner(&pVM->iom.s.CritSect);
-#else
-    return PDMCritSectIsOwner(&pVM->iom.s.CritSect);
-#endif
-}
 
 
Index: /trunk/src/VBox/VMM/VMMAll/IOMAllMMIO.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/IOMAllMMIO.cpp	(revision 81332)
+++ /trunk/src/VBox/VMM/VMMAll/IOMAllMMIO.cpp	(revision 81333)
@@ -67,4 +67,5 @@
         AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
         pVCpu->iom.s.PendingMmioWrite.cbValue = (uint32_t)cbBuf;
+        pVCpu->iom.s.PendingMmioWrite.idxMmioRegionHint = UINT32_MAX;
         memcpy(pVCpu->iom.s.PendingMmioWrite.abValue, pvBuf, cbBuf);
     }
@@ -584,6 +585,6 @@
  * @param   pvUser      Pointer to the MMIO ring-3 range entry.
  */
-static VBOXSTRICTRC iomMmioCommonPfHandler(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, PCPUMCTXCORE pCtxCore,
-                                           RTGCPHYS GCPhysFault, void *pvUser)
+VBOXSTRICTRC iomMmioCommonPfHandlerOld(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, PCPUMCTXCORE pCtxCore,
+                                       RTGCPHYS GCPhysFault, void *pvUser)
 {
     RT_NOREF_PV(uErrorCode);
@@ -596,5 +597,5 @@
 
     STAM_PROFILE_START(&pVM->iom.s.StatRZMMIOHandler, a);
-    Log(("iomMmioCommonPfHandler: GCPhys=%RGp uErr=%#x rip=%RGv\n", GCPhysFault, uErrorCode, (RTGCPTR)pCtxCore->rip));
+    Log(("iomMmioCommonPfHandlerOld: GCPhys=%RGp uErr=%#x rip=%RGv\n", GCPhysFault, uErrorCode, (RTGCPTR)pCtxCore->rip));
 
     PIOMMMIORANGE pRange = (PIOMMMIORANGE)pvUser;
@@ -617,5 +618,4 @@
 # else
         STAM_PROFILE_STOP(&pVM->iom.s.StatRZMMIOHandler, a);
-        STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOFailures);
         return VINF_IOM_R3_MMIO_READ_WRITE;
 # endif
@@ -645,5 +645,4 @@
 
         STAM_PROFILE_STOP(&pVM->iom.s.StatRZMMIOHandler, a);
-        STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOFailures);
         iomMmioReleaseRange(pVM, pRange);
         return VINF_IOM_R3_MMIO_READ_WRITE;
@@ -694,5 +693,5 @@
     LogFlow(("iomMmioPfHandler: GCPhys=%RGp uErr=%#x pvFault=%RGv rip=%RGv\n",
              GCPhysFault, (uint32_t)uErrorCode, pvFault, (RTGCPTR)pCtxCore->rip)); NOREF(pvFault);
-    return iomMmioCommonPfHandler(pVM, pVCpu, (uint32_t)uErrorCode, pCtxCore, GCPhysFault, pvUser);
+    return iomMmioCommonPfHandlerOld(pVM, pVCpu, (uint32_t)uErrorCode, pCtxCore, GCPhysFault, pvUser);
 }
 
@@ -727,5 +726,5 @@
     IOM_UNLOCK_SHARED(pVM);
 
-    VBOXSTRICTRC rcStrict = iomMmioCommonPfHandler(pVM, pVCpu, (uint32_t)uErrorCode, pCtxCore, GCPhysFault, pRange);
+    VBOXSTRICTRC rcStrict = iomMmioCommonPfHandlerOld(pVM, pVCpu, (uint32_t)uErrorCode, pCtxCore, GCPhysFault, pRange);
 
     iomMmioReleaseRange(pVM, pRange);
Index: /trunk/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp	(revision 81333)
+++ /trunk/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp	(revision 81333)
@@ -0,0 +1,1209 @@
+/* $Id$ */
+/** @file
+ * IOM - Input / Output Monitor - Any Context, MMIO & String I/O.
+ */
+
+/*
+ * Copyright (C) 2006-2019 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_IOM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/iem.h>
+#include "IOMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/hm.h>
+#include "IOMInline.h"
+
+#include <VBox/dis.h>
+#include <VBox/disopcode.h>
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <iprt/assert.h>
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** @def IOM_MMIO_STATS_COMMA_DECL
+ * Parameter list declaration for statistics entry pointer. */
+/** @def IOM_MMIO_STATS_COMMA_ARG
+ * Statistics entry pointer argument. */
+#ifdef VBOX_WITH_STATISTICS
+# define IOM_MMIO_STATS_COMMA_DECL , PIOMMMIOSTATSENTRY pStats
+# define IOM_MMIO_STATS_COMMA_ARG  , pStats
+#else
+# define IOM_MMIO_STATS_COMMA_DECL
+# define IOM_MMIO_STATS_COMMA_ARG
+#endif
+
+
+
+#ifndef IN_RING3
+/**
+ * Defers a pending MMIO write to ring-3.
+ *
+ * @returns VINF_IOM_R3_MMIO_COMMIT_WRITE
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPhys      The write address.
+ * @param   pvBuf       The bytes being written.
+ * @param   cbBuf       How many bytes.
+ * @param   idxRegEntry The MMIO registration index (handle) if available,
+ *                      otherwise UINT32_MAX.
+ */
+static VBOXSTRICTRC iomMmioRing3WritePending(PVMCPU pVCpu, RTGCPHYS GCPhys, void const *pvBuf, size_t cbBuf,
+                                             uint32_t idxRegEntry)
+{
+    Log5(("iomMmioRing3WritePending: %RGp LB %#x (idx=%#x)\n", GCPhys, cbBuf, idxRegEntry));
+    if (pVCpu->iom.s.PendingMmioWrite.cbValue == 0)
+    {
+        pVCpu->iom.s.PendingMmioWrite.GCPhys            = GCPhys;
+        AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
+        pVCpu->iom.s.PendingMmioWrite.cbValue           = (uint32_t)cbBuf;
+        pVCpu->iom.s.PendingMmioWrite.idxMmioRegionHint = idxRegEntry;
+        memcpy(pVCpu->iom.s.PendingMmioWrite.abValue, pvBuf, cbBuf);
+    }
+    else
+    {
+        /*
+         * Join with pending if adjecent.
+         *
+         * This may happen if the stack overflows into MMIO territory and RSP/ESP/SP
+         * isn't aligned. IEM will bounce buffer the access and do one write for each
+         * page.  We get here when the 2nd page part is written.
+         */
+        uint32_t const cbOldValue = pVCpu->iom.s.PendingMmioWrite.cbValue;
+        AssertMsgReturn(GCPhys == pVCpu->iom.s.PendingMmioWrite.GCPhys + cbOldValue,
+                        ("pending %RGp LB %#x; incoming %RGp LB %#x\n",
+                         pVCpu->iom.s.PendingMmioWrite.GCPhys, cbOldValue, GCPhys, cbBuf),
+                        VERR_IOM_MMIO_IPE_1);
+        AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue) - cbOldValue, VERR_IOM_MMIO_IPE_2);
+        pVCpu->iom.s.PendingMmioWrite.cbValue = cbOldValue + (uint32_t)cbBuf;
+        memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[cbOldValue], pvBuf, cbBuf);
+    }
+
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
+    return VINF_IOM_R3_MMIO_COMMIT_WRITE;
+}
+#endif
+
+
+/**
+ * Deals with complicated MMIO writes.
+ *
+ * Complicated means unaligned or non-dword/qword sized accesses depending on
+ * the MMIO region's access mode flags.
+ *
+ * @returns Strict VBox status code. Any EM scheduling status code,
+ *          VINF_IOM_R3_MMIO_WRITE, VINF_IOM_R3_MMIO_READ_WRITE or
+ *          VINF_IOM_R3_MMIO_READ may be returned.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pRegEntry   The MMIO entry for the current context.
+ * @param   GCPhys      The physical address to start writing.
+ * @param   pvValue     Where to store the value.
+ * @param   cbValue     The size of the value to write.
+ */
+static VBOXSTRICTRC iomMmioDoComplicatedWrite(PVM pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys,
+                                              void const *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    AssertReturn(   (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) != IOMMMIO_FLAGS_WRITE_PASSTHRU
+                 && (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) <= IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING,
+                 VERR_IOM_MMIO_IPE_1);
+    AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
+    RTGCPHYS const GCPhysStart  = GCPhys; NOREF(GCPhysStart);
+    bool const     fReadMissing = (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_READ_MISSING
+                               || (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING;
+    RT_NOREF_PV(pVCpu); /* ring-3 */
+
+    /*
+     * Do debug stop if requested.
+     */
+    VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
+#ifdef VBOX_STRICT
+    if (!(pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_WRITE))
+    { /* likely */ }
+    else
+    {
+# ifdef IN_RING3
+        LogRel(("IOM: Complicated write %#x byte at %RGp to %s, initiating debugger intervention\n", cbValue, GCPhys,
+                R3STRING(pRegEntry->pszDesc)));
+        rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
+                            "Complicated write %#x byte at %RGp to %s\n", cbValue, GCPhys, pRegEntry->pszDesc);
+        if (rc == VERR_DBGF_NOT_ATTACHED)
+            rc = VINF_SUCCESS;
+# else
+        return VINF_IOM_R3_MMIO_WRITE;
+# endif
+    }
+#endif
+
+    STAM_COUNTER_INC(&pStats->ComplicatedWrites);
+
+    /*
+     * Check if we should ignore the write.
+     */
+    if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD)
+    {
+        Assert(cbValue != 4 || (GCPhys & 3));
+        return VINF_SUCCESS;
+    }
+    if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD_QWORD)
+    {
+        Assert((cbValue != 4 && cbValue != 8) || (GCPhys & (cbValue - 1)));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Split and conquer.
+     */
+    for (;;)
+    {
+        unsigned const  offAccess  = GCPhys & 3;
+        unsigned        cbThisPart = 4 - offAccess;
+        if (cbThisPart > cbValue)
+            cbThisPart = cbValue;
+
+        /*
+         * Get the missing bits (if any).
+         */
+        uint32_t u32MissingValue = 0;
+        if (fReadMissing && cbThisPart != 4)
+        {
+            VBOXSTRICTRC rc2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                          GCPhys & ~(RTGCPHYS)3, &u32MissingValue, sizeof(u32MissingValue));
+            switch (VBOXSTRICTRC_VAL(rc2))
+            {
+                case VINF_SUCCESS:
+                    break;
+                case VINF_IOM_MMIO_UNUSED_FF:
+                    STAM_COUNTER_INC(&pStats->FFor00Reads);
+                    u32MissingValue = UINT32_C(0xffffffff);
+                    break;
+                case VINF_IOM_MMIO_UNUSED_00:
+                    STAM_COUNTER_INC(&pStats->FFor00Reads);
+                    u32MissingValue = 0;
+                    break;
+#ifndef IN_RING3
+                case VINF_IOM_R3_MMIO_READ:
+                case VINF_IOM_R3_MMIO_READ_WRITE:
+                case VINF_IOM_R3_MMIO_WRITE:
+                    LogFlow(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, rc2));
+                    rc2 = iomMmioRing3WritePending(pVCpu, GCPhys, pvValue, cbValue, pRegEntry->idxSelf);
+                    if (rc == VINF_SUCCESS || rc2 < rc)
+                        rc = rc2;
+                    return rc;
+#endif
+                default:
+                    if (RT_FAILURE(rc2))
+                    {
+                        Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, rc2));
+                        return rc2;
+                    }
+                    AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", rc2), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                    if (rc == VINF_SUCCESS || rc2 < rc)
+                        rc = rc2;
+                    break;
+            }
+        }
+
+        /*
+         * Merge missing and given bits.
+         */
+        uint32_t u32GivenMask;
+        uint32_t u32GivenValue;
+        switch (cbThisPart)
+        {
+            case 1:
+                u32GivenValue = *(uint8_t  const *)pvValue;
+                u32GivenMask  = UINT32_C(0x000000ff);
+                break;
+            case 2:
+                u32GivenValue = *(uint16_t const *)pvValue;
+                u32GivenMask  = UINT32_C(0x0000ffff);
+                break;
+            case 3:
+                u32GivenValue = RT_MAKE_U32_FROM_U8(((uint8_t const *)pvValue)[0], ((uint8_t const *)pvValue)[1],
+                                                    ((uint8_t const *)pvValue)[2], 0);
+                u32GivenMask  = UINT32_C(0x00ffffff);
+                break;
+            case 4:
+                u32GivenValue = *(uint32_t const *)pvValue;
+                u32GivenMask  = UINT32_C(0xffffffff);
+                break;
+            default:
+                AssertFailedReturn(VERR_IOM_MMIO_IPE_3);
+        }
+        if (offAccess)
+        {
+            u32GivenValue <<= offAccess * 8;
+            u32GivenMask  <<= offAccess * 8;
+        }
+
+        uint32_t u32Value = (u32MissingValue & ~u32GivenMask)
+                          | (u32GivenValue & u32GivenMask);
+
+        /*
+         * Do DWORD write to the device.
+         */
+        VBOXSTRICTRC rc2 = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                       GCPhys & ~(RTGCPHYS)3, &u32Value, sizeof(u32Value));
+        switch (VBOXSTRICTRC_VAL(rc2))
+        {
+            case VINF_SUCCESS:
+                break;
+#ifndef IN_RING3
+            case VINF_IOM_R3_MMIO_READ:
+            case VINF_IOM_R3_MMIO_READ_WRITE:
+            case VINF_IOM_R3_MMIO_WRITE:
+                Log3(("iomMmioDoComplicatedWrite: deferring GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, rc2));
+                AssertReturn(pVCpu->iom.s.PendingMmioWrite.cbValue == 0, VERR_IOM_MMIO_IPE_1);
+                AssertReturn(cbValue + (GCPhys & 3) <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
+                pVCpu->iom.s.PendingMmioWrite.GCPhys  = GCPhys & ~(RTGCPHYS)3;
+                pVCpu->iom.s.PendingMmioWrite.cbValue = cbValue + (GCPhys & 3);
+                *(uint32_t *)pVCpu->iom.s.PendingMmioWrite.abValue = u32Value;
+                if (cbValue > cbThisPart)
+                    memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[4],
+                           (uint8_t const *)pvValue + cbThisPart, cbValue - cbThisPart);
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
+                if (rc == VINF_SUCCESS)
+                    rc = VINF_IOM_R3_MMIO_COMMIT_WRITE;
+                return rc;
+#endif
+            default:
+                if (RT_FAILURE(rc2))
+                {
+                    Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, rc2));
+                    return rc2;
+                }
+                AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", rc2), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                if (rc == VINF_SUCCESS || rc2 < rc)
+                    rc = rc2;
+                break;
+        }
+
+        /*
+         * Advance.
+         */
+        cbValue -= cbThisPart;
+        if (!cbValue)
+            break;
+        GCPhys += cbThisPart;
+        pvValue = (uint8_t const *)pvValue + cbThisPart;
+    }
+
+    return rc;
+}
+
+
+
+
+/**
+ * Wrapper which does the write.
+ */
+DECLINLINE(VBOXSTRICTRC) iomMmioDoWrite(PVMCC pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhysFault,
+                                        const void *pvData, uint32_t cb IOM_MMIO_STATS_COMMA_DECL)
+{
+    VBOXSTRICTRC rcStrict;
+    if (RT_LIKELY(pRegEntry->pfnWriteCallback))
+    {
+        if (   (cb == 4 && !(GCPhysFault & 3))
+            || (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_PASSTHRU
+            || (cb == 8 && !(GCPhysFault & 7) && IOMMMIO_DOES_WRITE_MODE_ALLOW_QWORD(pRegEntry->fFlags)) )
+            rcStrict = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                   GCPhysFault, pvData, cb);
+        else
+            rcStrict = iomMmioDoComplicatedWrite(pVM, pVCpu, pRegEntry, GCPhysFault, pvData, cb IOM_MMIO_STATS_COMMA_ARG);
+    }
+    else
+        rcStrict = VINF_SUCCESS;
+    return rcStrict;
+}
+
+
+/**
+ * Deals with complicated MMIO reads.
+ *
+ * Complicated means unaligned or non-dword/qword sized accesses depending on
+ * the MMIO region's access mode flags.
+ *
+ * @returns Strict VBox status code. Any EM scheduling status code,
+ *          VINF_IOM_R3_MMIO_READ, VINF_IOM_R3_MMIO_READ_WRITE or
+ *          VINF_IOM_R3_MMIO_WRITE may be returned.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pRegEntry           The MMIO entry for the current context.
+ * @param   GCPhys              The physical address to start reading.
+ * @param   pvValue             Where to store the value.
+ * @param   cbValue             The size of the value to read.
+ */
+static VBOXSTRICTRC iomMMIODoComplicatedRead(PVM pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys,
+                                             void *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    AssertReturn(   (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD
+                 || (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD,
+                 VERR_IOM_MMIO_IPE_1);
+    AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
+#ifdef LOG_ENABLED
+    RTGCPHYS const GCPhysStart = GCPhys;
+#endif
+
+    /*
+     * Do debug stop if requested.
+     */
+    VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
+#ifdef VBOX_STRICT
+    if (pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_READ)
+    {
+# ifdef IN_RING3
+        rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
+                            "Complicated read %#x byte at %RGp to %s\n", cbValue, GCPhys, R3STRING(pRegEntry->pszDesc));
+        if (rc == VERR_DBGF_NOT_ATTACHED)
+            rc = VINF_SUCCESS;
+# else
+        return VINF_IOM_R3_MMIO_READ;
+# endif
+    }
+#endif
+
+    STAM_COUNTER_INC(&pStats->ComplicatedReads);
+
+    /*
+     * Split and conquer.
+     */
+    for (;;)
+    {
+        /*
+         * Do DWORD read from the device.
+         */
+        uint32_t u32Value;
+        VBOXSTRICTRC rcStrict2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                            GCPhys & ~(RTGCPHYS)3, &u32Value, sizeof(u32Value));
+        switch (VBOXSTRICTRC_VAL(rcStrict2))
+        {
+            case VINF_SUCCESS:
+                break;
+            case VINF_IOM_MMIO_UNUSED_FF:
+                STAM_COUNTER_INC(&pStats->FFor00Reads);
+                u32Value = UINT32_C(0xffffffff);
+                break;
+            case VINF_IOM_MMIO_UNUSED_00:
+                STAM_COUNTER_INC(&pStats->FFor00Reads);
+                u32Value = 0;
+                break;
+            case VINF_IOM_R3_MMIO_READ:
+            case VINF_IOM_R3_MMIO_READ_WRITE:
+            case VINF_IOM_R3_MMIO_WRITE:
+                /** @todo What if we've split a transfer and already read
+                 * something?  Since reads can have sideeffects we could be
+                 * kind of screwed here... */
+                LogFlow(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
+                         GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
+                return rcStrict2;
+            default:
+                if (RT_FAILURE(rcStrict2))
+                {
+                    Log(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
+                         GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
+                    return rcStrict2;
+                }
+                AssertMsgReturn(rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)),
+                                VERR_IPE_UNEXPECTED_INFO_STATUS);
+                if (rc == VINF_SUCCESS || rcStrict2 < rc)
+                    rc = rcStrict2;
+                break;
+        }
+        u32Value >>= (GCPhys & 3) * 8;
+
+        /*
+         * Write what we've read.
+         */
+        unsigned cbThisPart = 4 - (GCPhys & 3);
+        if (cbThisPart > cbValue)
+            cbThisPart = cbValue;
+
+        switch (cbThisPart)
+        {
+            case 1:
+                *(uint8_t *)pvValue = (uint8_t)u32Value;
+                break;
+            case 2:
+                *(uint16_t *)pvValue = (uint16_t)u32Value;
+                break;
+            case 3:
+                ((uint8_t *)pvValue)[0] = RT_BYTE1(u32Value);
+                ((uint8_t *)pvValue)[1] = RT_BYTE2(u32Value);
+                ((uint8_t *)pvValue)[2] = RT_BYTE3(u32Value);
+                break;
+            case 4:
+                *(uint32_t *)pvValue = u32Value;
+                break;
+        }
+
+        /*
+         * Advance.
+         */
+        cbValue -= cbThisPart;
+        if (!cbValue)
+            break;
+        GCPhys += cbThisPart;
+        pvValue = (uint8_t *)pvValue + cbThisPart;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Implements VINF_IOM_MMIO_UNUSED_FF.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pvValue             Where to store the zeros.
+ * @param   cbValue             How many bytes to read.
+ */
+static int iomMMIODoReadFFs(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    switch (cbValue)
+    {
+        case 1: *(uint8_t  *)pvValue = UINT8_C(0xff); break;
+        case 2: *(uint16_t *)pvValue = UINT16_C(0xffff); break;
+        case 4: *(uint32_t *)pvValue = UINT32_C(0xffffffff); break;
+        case 8: *(uint64_t *)pvValue = UINT64_C(0xffffffffffffffff); break;
+        default:
+        {
+            uint8_t *pb = (uint8_t *)pvValue;
+            while (cbValue--)
+                *pb++ = UINT8_C(0xff);
+            break;
+        }
+    }
+    STAM_COUNTER_INC(&pStats->FFor00Reads);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements VINF_IOM_MMIO_UNUSED_00.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pvValue             Where to store the zeros.
+ * @param   cbValue             How many bytes to read.
+ */
+static int iomMMIODoRead00s(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    switch (cbValue)
+    {
+        case 1: *(uint8_t  *)pvValue = UINT8_C(0x00); break;
+        case 2: *(uint16_t *)pvValue = UINT16_C(0x0000); break;
+        case 4: *(uint32_t *)pvValue = UINT32_C(0x00000000); break;
+        case 8: *(uint64_t *)pvValue = UINT64_C(0x0000000000000000); break;
+        default:
+        {
+            uint8_t *pb = (uint8_t *)pvValue;
+            while (cbValue--)
+                *pb++ = UINT8_C(0x00);
+            break;
+        }
+    }
+    STAM_COUNTER_INC(&pStats->FFor00Reads);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Wrapper which does the read.
+ */
+DECLINLINE(VBOXSTRICTRC) iomMmioDoRead(PVMCC pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys,
+                                       void *pvValue, uint32_t cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    VBOXSTRICTRC rcStrict;
+    if (RT_LIKELY(pRegEntry->pfnReadCallback))
+    {
+        if (   (   cbValue == 4
+                && !(GCPhys & 3))
+            || (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_PASSTHRU
+            || (    cbValue == 8
+                && !(GCPhys & 7)
+                && (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD ) )
+            rcStrict = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser, GCPhys,
+                                                  pvValue, cbValue);
+        else
+            rcStrict = iomMMIODoComplicatedRead(pVM, pRegEntry, GCPhys, pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG);
+    }
+    else
+        rcStrict = VINF_IOM_MMIO_UNUSED_FF;
+    if (rcStrict != VINF_SUCCESS)
+    {
+        switch (VBOXSTRICTRC_VAL(rcStrict))
+        {
+            case VINF_IOM_MMIO_UNUSED_FF: rcStrict = iomMMIODoReadFFs(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
+            case VINF_IOM_MMIO_UNUSED_00: rcStrict = iomMMIODoRead00s(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
+        }
+    }
+    return rcStrict;
+}
+
+#ifndef IN_RING3
+/**
+ * Checks if we can handle an MMIO \#PF in R0/RC.
+ */
+DECLINLINE(bool) iomMmioCanHandlePfInRZ(PVMCC pVM, uint32_t uErrorCode, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
+{
+    if (pRegEntry->cbRegion > 0)
+    {
+        if (   pRegEntry->pfnWriteCallback
+            && pRegEntry->pfnReadCallback)
+            return true;
+
+        PIOMMMIOENTRYR3 const pRegEntryR3 = &pVM->iomr0.s.paMmioRing3Regs[pRegEntry->idxSelf];
+        if (  uErrorCode == UINT32_MAX
+            ? pRegEntryR3->pfnWriteCallback || pRegEntryR3->pfnReadCallback
+            : uErrorCode & X86_TRAP_PF_RW
+              ? !pRegEntry->pfnWriteCallback && pRegEntryR3->pfnWriteCallback
+              : !pRegEntry->pfnReadCallback  && pRegEntryR3->pfnReadCallback)
+            return false;
+
+        return true;
+    }
+    return false;
+}
+#endif
+
+/**
+ * Common worker for the \#PF handler and IOMMMIOPhysHandler (APIC+VT-x).
+ *
+ * @returns VBox status code (appropriate for GC return).
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uErrorCode  CPU Error code.  This is UINT32_MAX when we don't have
+ *                      any error code (the EPT misconfig hack).
+ * @param   pCtxCore    Trap register frame.
+ * @param   GCPhysFault The GC physical address corresponding to pvFault.
+ * @param   pRegEntry   The MMIO entry for the current context.
+ */
+DECLINLINE(VBOXSTRICTRC) iomMmioCommonPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode,
+                                                   RTGCPHYS GCPhysFault, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
+{
+    STAM_PROFILE_START(&pVM->iom.s.StatRZMMIOHandler, a);
+    Log(("iomMmioCommonPfHandler: GCPhysFault=%RGp uErr=%#x rip=%RGv\n", GCPhysFault, uErrorCode, CPUMGetGuestRIP(pVCpu) ));
+    RT_NOREF(GCPhysFault, uErrorCode);
+
+    VBOXSTRICTRC rcStrict;
+
+#ifndef IN_RING3
+    /*
+     * Should we defer the request right away?  This isn't usually the case, so
+     * do the simple test first and the try deal with uErrorCode being N/A.
+     */
+    PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
+    if (RT_LIKELY(   pDevIns
+                  && iomMmioCanHandlePfInRZ(pVM, uErrorCode, pRegEntry)))
+    {
+        /*
+         * Enter the device critsect prior to engaging IOM in case of lock contention.
+         * Note! Perhaps not a good move?
+         */
+        rcStrict = PDMCritSectEnter(pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_MMIO_READ_WRITE);
+        if (rcStrict == VINF_SUCCESS)
+        {
+#endif /* !IN_RING3 */
+
+            /*
+             * Let IEM call us back via iomMmioHandler.
+             */
+            rcStrict = IEMExecOne(pVCpu);
+
+#ifndef IN_RING3
+            PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
+#endif
+            if (RT_SUCCESS(rcStrict))
+            { /* likely */ }
+            else if (   rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
+                     || rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
+            {
+                Log(("IOM: Hit unsupported IEM feature!\n"));
+                rcStrict = VINF_EM_RAW_EMULATE_INSTR;
+            }
+#ifndef IN_RING3
+            STAM_PROFILE_STOP(&pVM->iom.s.StatRZMMIOHandler, a);
+            return rcStrict;
+        }
+        else
+            STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIODevLockContention);
+    }
+    else
+        rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
+
+# ifdef VBOX_WITH_STATISTICS
+    if (rcStrict == VINF_IOM_R3_MMIO_READ_WRITE)
+    {
+        PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry);
+        if (uErrorCode & X86_TRAP_PF_RW)
+        {
+            STAM_COUNTER_INC(&pStats->WriteRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOWritesToR3);
+        }
+        else
+        {
+            STAM_COUNTER_INC(&pStats->ReadRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOReadsToR3);
+        }
+    }
+# endif
+#else  /* IN_RING3 */
+    RT_NOREF(pRegEntry);
+#endif /* IN_RING3 */
+
+    STAM_PROFILE_STOP(&pVM->iom.s.StatRZMMIOHandler, a);
+    return rcStrict;
+}
+
+
+/**
+ * @callback_method_impl{FNPGMRZPHYSPFHANDLER,
+ *      \#PF access handler callback for MMIO pages.}
+ *
+ * @remarks The @a pvUser argument is the MMIO handle.
+ */
+DECLEXPORT(VBOXSTRICTRC) iomMmioPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pCtxCore,
+                                             RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
+{
+    LogFlow(("iomMmioPfHandler: GCPhys=%RGp uErr=%#x pvFault=%RGv rip=%RGv\n",
+             GCPhysFault, (uint32_t)uErrorCode, pvFault, (RTGCPTR)pCtxCore->rip));
+    RT_NOREF(pvFault, pCtxCore);
+
+    /* Translate the MMIO handle to a registration entry for the current context. */
+    AssertReturn((uintptr_t)pvUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+#ifdef IN_RING0
+    AssertReturn((uintptr_t)pvUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iomr0.s.paMmioRegs[(uintptr_t)pvUser];
+#else
+    CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iom.s.paMmioRegs[(uintptr_t)pvUser];
+#endif
+
+    return iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
+}
+
+
+#ifdef IN_RING0
+/**
+ * Physical access handler for MMIO ranges.
+ *
+ * This is actually only used by VT-x for APIC page accesses.
+ *
+ * @returns VBox status code (appropriate for GC return).
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uErrorCode  CPU Error code.
+ * @param   GCPhysFault The GC physical address.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IOMMmioPhysHandler(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, RTGCPHYS GCPhysFault)
+{
+    /*
+     * We don't have a range here, so look it up before calling the common function.
+     */
+    VBOXSTRICTRC rcStrict = IOM_LOCK_SHARED(pVM);
+    if (RT_SUCCESS(rcStrict))
+    {
+        uint16_t idxLastHint = UINT16_MAX; /** @todo dedicate hint for this bugger, since it will only be the APIC. */
+        RTGCPHYS offRegion;
+        CTX_SUFF(PIOMMMIOENTRY) pRegEntry = iomMmioGetEntry(pVM, GCPhysFault, &offRegion, &idxLastHint);
+        if (RT_LIKELY(pRegEntry))
+        {
+            IOM_UNLOCK_SHARED(pVM);
+            rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
+        }
+        else
+        {
+            /*
+             * Old style registrations.
+             */
+            PIOMMMIORANGE pRange = iomMmioGetRange(pVM, pVCpu, GCPhysFault);
+            if (pRange)
+            {
+                iomMmioRetainRange(pRange);
+                IOM_UNLOCK_SHARED(pVM);
+
+                rcStrict = iomMmioCommonPfHandlerOld(pVM, pVCpu, (uint32_t)uErrorCode,
+                                                     CPUMCTX2CORE(&pVCpu->cpum.GstCtx), GCPhysFault, pRange);
+
+                iomMmioReleaseRange(pVM, pRange);
+            }
+            else
+            {
+                IOM_UNLOCK_SHARED(pVM);
+                rcStrict = VERR_IOM_MMIO_RANGE_NOT_FOUND;
+            }
+        }
+    }
+    else if (rcStrict == VERR_SEM_BUSY)
+        rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
+    return rcStrict;
+}
+#endif /* IN_RING0 */
+
+
+/**
+ * @callback_method_impl{FNPGMPHYSHANDLER, MMIO page accesses}
+ *
+ * @remarks The @a pvUser argument is the MMIO handle.
+ */
+PGM_ALL_CB2_DECL(VBOXSTRICTRC) iomMmioHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, void *pvPhys, void *pvBuf,
+                                                 size_t cbBuf, PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
+{
+    STAM_PROFILE_START(UnusedMacroArg, Prf);
+    STAM_COUNTER_INC(&pVM->iom.s.CTX_SUFF(StatMmioHandler));
+
+    Assert(enmAccessType == PGMACCESSTYPE_READ || enmAccessType == PGMACCESSTYPE_WRITE);
+    AssertMsg(cbBuf >= 1, ("%zu\n", cbBuf));
+    NOREF(pvPhys); NOREF(enmOrigin);
+
+#ifdef IN_RING3
+    int const rcToRing3 = VERR_IOM_MMIO_IPE_3;
+#else
+    int const rcToRing3 = enmAccessType == PGMACCESSTYPE_READ ? VINF_IOM_R3_MMIO_READ : VINF_IOM_R3_MMIO_WRITE;
+#endif
+
+    /*
+     * Translate pvUser to an MMIO registration table entry.  We can do this
+     * without any locking as the data is static after VM creation.
+     */
+    AssertReturn((uintptr_t)pvUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+#ifdef IN_RING0
+    AssertReturn((uintptr_t)pvUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    CTX_SUFF(PIOMMMIOENTRY) const pRegEntry = &pVM->iomr0.s.paMmioRegs[(uintptr_t)pvUser];
+#else
+    CTX_SUFF(PIOMMMIOENTRY) const pRegEntry = &pVM->iom.s.paMmioRegs[(uintptr_t)pvUser];
+#endif
+#ifdef VBOX_WITH_STATISTICS
+    PIOMMMIOSTATSENTRY const      pStats    = iomMmioGetStats(pVM, pRegEntry);  /* (Works even without ring-0 setup.) */
+#endif
+    PPDMDEVINS const              pDevIns   = pRegEntry->pDevIns;
+
+#ifndef IN_RING3
+# if defined(VBOX_STRICT)
+    /*
+     * Assert the right entry in strict builds.  This may yield a false positive
+     * for SMP VMs if we're unlucky and the guest isn't well behaved.
+     */
+    IOM_LOCK_SHARED(pVM);  /** @todo Need lookup that doesn't require locking... */
+    RTGCPHYS offIgn;
+    uint16_t idxIgn = UINT16_MAX;
+    Assert(   pRegEntry == iomMmioGetEntry(pVM, GCPhysFault, &offIgn, &idxIgn)
+           || !pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser].fMapped);
+    IOM_UNLOCK_SHARED(pVM);
+# endif
+
+    /*
+     * If someone is doing FXSAVE, FXRSTOR, XSAVE, XRSTOR or other stuff dealing with
+     * large amounts of data, just go to ring-3 where we don't need to deal with partial
+     * successes.  No chance any of these will be problematic read-modify-write stuff.
+     *
+     * Also drop back if the ring-0 registration entry isn't actually used.
+     */
+    if (   cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue)
+        && pRegEntry->cbRegion != 0
+        && (  enmAccessType == PGMACCESSTYPE_READ
+            ? pRegEntry->pfnReadCallback  != NULL || pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser].pfnReadCallback == NULL
+            : pRegEntry->pfnWriteCallback != NULL || pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser].pfnWriteCallback == NULL)
+        && pDevIns )
+    { /* likely */ }
+    else
+    {
+        STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3);
+        STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatRZMMIOReadsToR3 : &pVM->iom.s.StatRZMMIOWritesToR3);
+        return rcToRing3;
+    }
+#endif /* !IN_RING3 */
+
+
+    /*
+     * Perform locking and the access.
+     *
+     * Writes requiring a return to ring-3 are buffered by IOM so IEM can
+     * commit the instruction.
+     *
+     * Note! We may end up locking the device even when the relevant callback is
+     *       NULL.  This is supposed to be an unlikely case, so not optimized yet.
+     */
+    VBOXSTRICTRC rcStrict = PDMCritSectEnter(pDevIns->CTX_SUFF(pCritSectRo), rcToRing3);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        if (enmAccessType == PGMACCESSTYPE_READ)
+        {
+            rcStrict = iomMmioDoRead(pVM, pRegEntry, GCPhysFault, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
+
+            PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
+#ifndef IN_RING3
+            if (rcStrict == VINF_IOM_R3_MMIO_READ)
+            {
+                STAM_COUNTER_INC(&pStats->ReadRZToR3);
+                STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOReadsToR3);
+            }
+            else
+#endif
+                STAM_COUNTER_INC(&pStats->Reads);
+            STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfRead), Prf);
+        }
+        else
+        {
+            rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, GCPhysFault, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
+            PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
+#ifndef IN_RING3
+            if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
+                rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
+            if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
+            {
+                STAM_COUNTER_INC(&pStats->WriteRZToR3);
+                STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOWritesToR3);
+            }
+            else if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
+            {
+                STAM_COUNTER_INC(&pStats->CommitRZToR3);
+                STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOCommitsToR3);
+            }
+            else
+#endif
+                STAM_COUNTER_INC(&pStats->Writes);
+            STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfWrite), Prf);
+        }
+
+        /*
+         * Check the return code.
+         */
+#ifdef IN_RING3
+        AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc -  Access type %d - %RGp - %s\n",
+                                             VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pRegEntry->pszDesc));
+#else
+        AssertMsg(   rcStrict == VINF_SUCCESS
+                  || rcStrict == rcToRing3
+                  || (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE && enmAccessType == PGMACCESSTYPE_WRITE)
+                  || rcStrict == VINF_EM_DBG_STOP
+                  || rcStrict == VINF_EM_DBG_EVENT
+                  || rcStrict == VINF_EM_DBG_BREAKPOINT
+                  || rcStrict == VINF_EM_OFF
+                  || rcStrict == VINF_EM_SUSPEND
+                  || rcStrict == VINF_EM_RESET
+                  //|| rcStrict == VINF_EM_HALT       /* ?? */
+                  //|| rcStrict == VINF_EM_NO_MEMORY  /* ?? */
+                  , ("%Rrc - Access type %d - %RGp - %s #%u\n",
+                     VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pDevIns->pReg->szName, pDevIns->iInstance));
+#endif
+    }
+    /*
+     * Deal with enter-critsect failures.
+     */
+#ifndef IN_RING3
+    else if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
+    {
+        Assert(enmAccessType == PGMACCESSTYPE_WRITE);
+        rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
+        if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
+        {
+            STAM_COUNTER_INC(&pStats->CommitRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOCommitsToR3);
+        }
+        else
+        {
+            STAM_COUNTER_INC(&pStats->WriteRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOWritesToR3);
+        }
+        STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIODevLockContention);
+    }
+    else if (rcStrict == VINF_IOM_R3_MMIO_READ)
+    {
+        Assert(enmAccessType == PGMACCESSTYPE_READ);
+        STAM_COUNTER_INC(&pStats->ReadRZToR3);
+        STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIODevLockContention);
+    }
+#endif
+    else
+        AssertMsg(RT_FAILURE_NP(rcStrict), ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * Mapping an MMIO2 page in place of an MMIO page for direct access.
+ *
+ * This is a special optimization used by the VGA device.  Call
+ * IOMMmioResetRegion() to undo the mapping.
+ *
+ * @returns VBox status code.  This API may return VINF_SUCCESS even if no
+ *          remapping is made,.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device instance @a hRegion and @a hMmio2 are
+ *                          associated with.
+ * @param   hRegion         The handle to the MMIO region.
+ * @param   offRegion       The offset into @a hRegion of the page to be
+ *                          remapped.
+ * @param   hMmio2          The MMIO2 handle.
+ * @param   offMmio2        Offset into @a hMmio2 of the page to be use for the
+ *                          mapping.
+ * @param   fPageFlags      Page flags to set. Must be (X86_PTE_RW | X86_PTE_P)
+ *                          for the time being.
+ */
+VMM_INT_DECL(int) IOMMmioMapMmio2Page(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, RTGCPHYS offRegion,
+                                      uint64_t hMmio2, RTGCPHYS offMmio2, uint64_t fPageFlags)
+{
+    /* Currently only called from the VGA device during MMIO. */
+    Log(("IOMMmioMapMmio2Page %#RX64/%RGp -> %#RX64/%RGp flags=%RX64\n", hRegion, offRegion, hMmio2, offMmio2, fPageFlags));
+    AssertReturn(fPageFlags == (X86_PTE_RW | X86_PTE_P), VERR_INVALID_PARAMETER);
+    AssertReturn(pDevIns, VERR_INVALID_POINTER);
+
+/** @todo Why is this restricted to protected mode???  Try it in all modes! */
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+
+    /* This currently only works in real mode, protected mode without paging or with nested paging. */
+    /** @todo NEM: MMIO page aliasing. */
+    if (    !HMIsEnabled(pVM)       /* useless without VT-x/AMD-V */
+        ||  (   CPUMIsGuestInPagedProtectedMode(pVCpu)
+             && !HMIsNestedPagingActive(pVM)))
+        return VINF_SUCCESS;    /* ignore */ /** @todo return some indicator if we fail here */
+
+    /*
+     * Translate the handle into an entry and check the region offset.
+     */
+    AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+#ifdef IN_RING0
+    AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(offRegion < pVM->iomr0.s.paMmioRegs[hRegion].cbRegion, VERR_OUT_OF_RANGE);
+    AssertReturn(   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
+                 || (   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
+                     && pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
+#else
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
+#endif
+    AssertReturn(offRegion < pRegEntry->cbRegion, VERR_OUT_OF_RANGE);
+    Assert((pRegEntry->cbRegion & PAGE_OFFSET_MASK) == 0);
+
+    /*
+     * When getting and using the mapping address, we must sit on the IOM lock
+     * to prevent remapping.  Shared suffices as we change nothing.
+     */
+    IOM_LOCK_SHARED(pVM);
+
+    RTGCPHYS const GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
+    AssertReturnStmt(GCPhys != NIL_RTGCPHYS, IOM_UNLOCK_SHARED(pVM), VERR_IOM_MMIO_REGION_NOT_MAPPED);
+    Assert(!(GCPhys & PAGE_OFFSET_MASK));
+
+    /*
+     * Do the aliasing; page align the addresses since PGM is picky.
+     */
+#if 0 /** @todo fix when DevVGA is converted to new model.  */
+    int rc = PGMHandlerPhysicalPageAlias(pVM, GCPhys, GCPhys + (offRange & ~(RTGCPHYS)PAGE_OFFSET_MASK),
+                                         pDevIns, hMmio2, offMmio2);
+#else
+    AssertFailed();
+    int rc = VERR_NOT_IMPLEMENTED;
+    RT_NOREF(offMmio2, hMmio2);
+#endif
+
+    IOM_UNLOCK_SHARED(pVM);
+
+    AssertRCReturn(rc, rc);
+
+/** @todo either ditch this or replace it with something that works in the
+ *        nested case, since we really only care about nested paging! */
+#if 0
+    /*
+     * Modify the shadow page table. Since it's an MMIO page it won't be present and we
+     * can simply prefetch it.
+     *
+     * Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page.
+     */
+# if 0 /* The assertion is wrong for the PGM_SYNC_CLEAR_PGM_POOL and VINF_PGM_HANDLER_ALREADY_ALIASED cases. */
+#  ifdef VBOX_STRICT
+    uint64_t fFlags;
+    RTHCPHYS HCPhys;
+    rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys);
+    Assert(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT);
+#  endif
+# endif
+    rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys);
+    Assert(rc == VINF_SUCCESS || rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+#ifdef IN_RING0 /* VT-x ring-0 only, move to IOMR0Mmio.cpp later.  */
+/**
+ * Mapping a HC page in place of an MMIO page for direct access.
+ *
+ * This is a special optimization used by the APIC in the VT-x case.  This VT-x
+ * code uses PGMHandlerPhysicalReset rather than IOMMmioResetRegion() to undo
+ * the effects here.
+ *
+ * @todo Make VT-x usage more consistent.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPhys          The address of the MMIO page to be changed.
+ * @param   HCPhys          The address of the host physical page.
+ * @param   fPageFlags      Page flags to set. Must be (X86_PTE_RW | X86_PTE_P)
+ *                          for the time being.
+ */
+VMM_INT_DECL(int) IOMR0MmioMapMmioHCPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint64_t fPageFlags)
+{
+    /* Currently only called from VT-x code during a page fault. */
+    Log(("IOMR0MmioMapMmioHCPage %RGp -> %RGp flags=%RX64\n", GCPhys, HCPhys, fPageFlags));
+
+    AssertReturn(fPageFlags == (X86_PTE_RW | X86_PTE_P), VERR_INVALID_PARAMETER);
+    /** @todo NEM: MMIO page aliasing?? */
+    Assert(HMIsEnabled(pVM));
+
+# ifdef VBOX_STRICT
+    /*
+     * Check input address (it's HM calling, not the device, so no region handle).
+     */
+    IOM_LOCK_SHARED(pVM);
+    RTGCPHYS offIgn;
+    uint16_t idxIgn = UINT16_MAX;
+    PIOMMMIOENTRYR0 pRegEntry = iomMmioGetEntry(pVM, GCPhys, &offIgn, &idxIgn);
+    IOM_UNLOCK_SHARED(pVM);
+    Assert(pRegEntry);
+    Assert(pRegEntry && !(pRegEntry->cbRegion & PAGE_OFFSET_MASK));
+# endif
+
+    /*
+     * Do the aliasing; page align the addresses since PGM is picky.
+     */
+    GCPhys &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
+    HCPhys &= ~(RTHCPHYS)PAGE_OFFSET_MASK;
+
+    int rc = PGMHandlerPhysicalPageAliasHC(pVM, GCPhys, GCPhys, HCPhys);
+    AssertRCReturn(rc, rc);
+
+/** @todo either ditch this or replace it with something that works in the
+ *        nested case, since we really only care about nested paging! */
+
+    /*
+     * Modify the shadow page table. Since it's an MMIO page it won't be present and we
+     * can simply prefetch it.
+     *
+     * Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page.
+     */
+    rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys);
+    Assert(rc == VINF_SUCCESS || rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT);
+    return VINF_SUCCESS;
+}
+#endif
+
+
+/**
+ * Reset a previously modified MMIO region; restore the access flags.
+ *
+ * This undoes the effects of IOMMmioMapMmio2Page() and is currently only
+ * intended for some ancient VGA hack.  However, it would be great to extend it
+ * beyond VT-x and/or nested-paging.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device instance @a hRegion is associated with.
+ * @param   hRegion         The handle to the MMIO region.
+ */
+VMM_INT_DECL(int) IOMMmioResetRegion(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion)
+{
+    Log(("IOMMMIOResetRegion %#RX64\n", hRegion));
+    AssertReturn(pDevIns, VERR_INVALID_POINTER);
+
+/** @todo Get rid of this this real/protected or nested paging restriction,
+ *        it probably shouldn't be here and would be nasty when the CPU
+ *        changes mode while we have the hack enabled... */
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+
+    /* This currently only works in real mode, protected mode without paging or with nested paging. */
+    /** @todo NEM: MMIO page aliasing. */
+    if (   !HMIsEnabled(pVM)       /* useless without VT-x/AMD-V */
+        || (   CPUMIsGuestInPagedProtectedMode(pVCpu)
+            && !HMIsNestedPagingActive(pVM)))
+        return VINF_SUCCESS;    /* ignore */
+
+    /*
+     * Translate the handle into an entry and mapping address for PGM.
+     * We have to take the lock to safely access the mapping address here.
+     */
+    AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+#ifdef IN_RING0
+    AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
+                 || (   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
+                     && pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
+#else
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
+#endif
+    Assert((pRegEntry->cbRegion & PAGE_OFFSET_MASK) == 0);
+
+    IOM_LOCK_SHARED(pVM);
+    RTGCPHYS GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
+    IOM_UNLOCK_SHARED(pVM);
+
+    Assert(!(GCPhys              & PAGE_OFFSET_MASK));
+    Assert(!(pRegEntry->cbRegion & PAGE_OFFSET_MASK));
+
+    /*
+     * Call PGM to do the job work.
+     *
+     * After the call, all the pages should be non-present, unless there is
+     * a page pool flush pending (unlikely).
+     */
+    int rc = PGMHandlerPhysicalReset(pVM, GCPhys);
+    AssertRC(rc);
+
+# ifdef VBOX_STRICT
+    if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
+    {
+        RTGCPHYS cb = pRegEntry->cbRegion;
+        while (cb)
+        {
+            uint64_t fFlags;
+            RTHCPHYS HCPhys;
+            rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys);
+            Assert(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT);
+            cb     -= PAGE_SIZE;
+            GCPhys += PAGE_SIZE;
+        }
+    }
+# endif
+    return rc;
+}
+
Index: /trunk/src/VBox/VMM/VMMR0/IOMR0IoPort.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR0/IOMR0IoPort.cpp	(revision 81332)
+++ /trunk/src/VBox/VMM/VMMR0/IOMR0IoPort.cpp	(revision 81333)
@@ -271,4 +271,5 @@
     AssertReturn(pGVM->iom.s.cIoPortStatsAllocation == cOldEntries, VERR_IOM_IOPORT_IPE_1);
     AssertReturn(pGVM->iom.s.cIoPortStats <= cOldEntries, VERR_IOM_IOPORT_IPE_2);
+    AssertReturn(!pGVM->iomr0.s.fIoPortStatsFrozen, VERR_WRONG_ORDER);
 
     /*
@@ -329,2 +330,47 @@
 }
 
+/**
+ * Called after all devices has been instantiated to copy over the statistics
+ * indices to the ring-0 I/O port registration table.
+ *
+ * This simplifies keeping statistics for I/O port ranges that are ring-3 only.
+ *
+ * After this call, IOMR0IoPortGrowStatisticsTable() will stop working.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @thread  EMT(0)
+ * @note    Only callable at VM creation time.
+ */
+VMMR0_INT_DECL(int) IOMR0IoPortSyncStatisticsIndices(PGVM pGVM)
+{
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * First, freeze the statistics array:
+     */
+    pGVM->iomr0.s.fIoPortStatsFrozen = true;
+
+    /*
+     * Second, synchronize the indices:
+     */
+    uint32_t const          cRegs        = RT_MIN(pGVM->iom.s.cIoPortRegs, pGVM->iomr0.s.cIoPortAlloc);
+    uint32_t const          cStatsAlloc  = pGVM->iomr0.s.cIoPortStatsAllocation;
+    PIOMIOPORTENTRYR0       paIoPortRegs   = pGVM->iomr0.s.paIoPortRegs;
+    IOMIOPORTENTRYR3 const *paIoPortRegsR3 = pGVM->iomr0.s.paIoPortRing3Regs;
+    AssertReturn((paIoPortRegs && paIoPortRegsR3) || cRegs == 0, VERR_IOM_IOPORT_IPE_3);
+
+    for (uint32_t i = 0 ; i < cRegs; i++)
+    {
+        uint16_t idxStats = paIoPortRegsR3[i].idxStats;
+        paIoPortRegs[i].idxStats = idxStats < cStatsAlloc ? idxStats : UINT16_MAX;
+    }
+
+#else
+    RT_NOREF(pGVM);
+#endif
+    return VINF_SUCCESS;
+}
+
Index: /trunk/src/VBox/VMM/VMMR0/IOMR0Mmio.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR0/IOMR0Mmio.cpp	(revision 81332)
+++ /trunk/src/VBox/VMM/VMMR0/IOMR0Mmio.cpp	(revision 81333)
@@ -268,4 +268,5 @@
     AssertReturn(pGVM->iom.s.cMmioStatsAllocation == cOldEntries, VERR_IOM_MMIO_IPE_1);
     AssertReturn(pGVM->iom.s.cMmioStats <= cOldEntries, VERR_IOM_MMIO_IPE_2);
+    AssertReturn(!pGVM->iomr0.s.fMmioStatsFrozen, VERR_WRONG_ORDER);
 
     /*
@@ -326,2 +327,46 @@
 }
 
+
+/**
+ * Called after all devices has been instantiated to copy over the statistics
+ * indices to the ring-0 MMIO registration table.
+ *
+ * This simplifies keeping statistics for MMIO ranges that are ring-3 only.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @thread  EMT(0)
+ * @note    Only callable at VM creation time.
+ */
+VMMR0_INT_DECL(int) IOMR0MmioSyncStatisticsIndices(PGVM pGVM)
+{
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * First, freeze the statistics array:
+     */
+    pGVM->iomr0.s.fMmioStatsFrozen = true;
+
+    /*
+     * Second, synchronize the indices:
+     */
+    uint32_t const          cRegs        = RT_MIN(pGVM->iom.s.cMmioRegs, pGVM->iomr0.s.cMmioAlloc);
+    uint32_t const          cStatsAlloc  = pGVM->iomr0.s.cMmioStatsAllocation;
+    PIOMMMIOENTRYR0         paMmioRegs   = pGVM->iomr0.s.paMmioRegs;
+    IOMMMIOENTRYR3 const   *paMmioRegsR3 = pGVM->iomr0.s.paMmioRing3Regs;
+    AssertReturn((paMmioRegs && paMmioRegsR3) || cRegs == 0, VERR_IOM_MMIO_IPE_3);
+
+    for (uint32_t i = 0 ; i < cRegs; i++)
+    {
+        uint16_t idxStats = paMmioRegsR3[i].idxStats;
+        paMmioRegs[i].idxStats = idxStats < cStatsAlloc ? idxStats : UINT16_MAX;
+    }
+
+#else
+    RT_NOREF(pGVM);
+#endif
+    return VINF_SUCCESS;
+}
+
Index: /trunk/src/VBox/VMM/VMMR0/VMMR0.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR0/VMMR0.cpp	(revision 81332)
+++ /trunk/src/VBox/VMM/VMMR0/VMMR0.cpp	(revision 81333)
@@ -2217,4 +2217,15 @@
         }
 
+        case VMMR0_DO_IOM_SYNC_STATS_INDICES:
+        {
+            if (pReqHdr || idCpu != 0)
+                return VERR_INVALID_PARAMETER;
+            rc = IOMR0IoPortSyncStatisticsIndices(pGVM);
+            if (RT_SUCCESS(rc))
+                rc = IOMR0MmioSyncStatisticsIndices(pGVM);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+        }
+
         /*
          * For profiling.
Index: /trunk/src/VBox/VMM/VMMR3/IOM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/IOM.cpp	(revision 81332)
+++ /trunk/src/VBox/VMM/VMMR3/IOM.cpp	(revision 81333)
@@ -138,5 +138,4 @@
 static DECLCALLBACK(int) iomR3RelocateMMIOCallback(PAVLROGCPHYSNODECORE pNode, void *pvUser);
 #endif
-static DECLCALLBACK(void) iomR3MMIOInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
 static FNIOMIOPORTIN        iomR3IOPortDummyIn;
 static FNIOMIOPORTOUT       iomR3IOPortDummyOut;
@@ -180,67 +179,54 @@
      */
     rc = MMHyperAlloc(pVM, sizeof(*pVM->iom.s.pTreesR3), 0, MM_TAG_IOM, (void **)&pVM->iom.s.pTreesR3);
-    if (RT_SUCCESS(rc))
-    {
-        pVM->iom.s.pTreesR0 = MMHyperR3ToR0(pVM, pVM->iom.s.pTreesR3);
-
-        /*
-         * Register the MMIO access handler type.
-         */
-        rc = PGMR3HandlerPhysicalTypeRegister(pVM, PGMPHYSHANDLERKIND_MMIO,
-                                              iomMmioHandler,
-                                              NULL, "iomMmioHandler", "iomMmioPfHandler",
-                                              NULL, "iomMmioHandler", "iomMmioPfHandler",
-                                              "MMIO", &pVM->iom.s.hMmioHandlerType);
-        AssertRC(rc);
-        if (RT_SUCCESS(rc))
-        {
-
-            /*
-             * Info.
-             */
-            DBGFR3InfoRegisterInternal(pVM, "ioport", "Dumps all IOPort ranges. No arguments.", &iomR3IoPortInfo);
-            DBGFR3InfoRegisterInternal(pVM, "mmio", "Dumps all MMIO ranges. No arguments.", &iomR3MMIOInfo);
-
-            /*
-             * Statistics.
-             */
-            STAM_REG(pVM, &pVM->iom.s.StatRZMMIOHandler,      STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler",                      STAMUNIT_TICKS_PER_CALL, "Profiling of the iomMmioPfHandler() body, only success calls.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZMMIO1Byte,        STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access1",              STAMUNIT_OCCURENCES,     "MMIO access by 1 byte counter.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZMMIO2Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access2",              STAMUNIT_OCCURENCES,     "MMIO access by 2 bytes counter.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZMMIO4Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access4",              STAMUNIT_OCCURENCES,     "MMIO access by 4 bytes counter.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZMMIO8Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access8",              STAMUNIT_OCCURENCES,     "MMIO access by 8 bytes counter.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZMMIOFailures,     STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/MMIOFailures",         STAMUNIT_OCCURENCES,     "Number of times iomMmioPfHandler() didn't service the request.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstMov,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOV",             STAMUNIT_TICKS_PER_CALL, "Profiling of the MOV instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstCmp,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/CMP",             STAMUNIT_TICKS_PER_CALL, "Profiling of the CMP instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstAnd,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/AND",             STAMUNIT_TICKS_PER_CALL, "Profiling of the AND instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstOr,           STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/OR",              STAMUNIT_TICKS_PER_CALL, "Profiling of the OR instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstXor,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/XOR",             STAMUNIT_TICKS_PER_CALL, "Profiling of the XOR instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstBt,           STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/BT",              STAMUNIT_TICKS_PER_CALL, "Profiling of the BT instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstTest,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/TEST",            STAMUNIT_TICKS_PER_CALL, "Profiling of the TEST instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstXchg,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/XCHG",            STAMUNIT_TICKS_PER_CALL, "Profiling of the XCHG instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstStos,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/STOS",            STAMUNIT_TICKS_PER_CALL, "Profiling of the STOS instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstLods,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/LODS",            STAMUNIT_TICKS_PER_CALL, "Profiling of the LODS instruction emulation.");
-#ifdef IOM_WITH_MOVS_SUPPORT
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstMovs,     STAMTYPE_PROFILE_ADV, "/IOM/RZ-MMIOHandler/Inst/MOVS",            STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstMovsToMMIO,   STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOVS/ToMMIO",     STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation - Mem2MMIO.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstMovsFromMMIO, STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOVS/FromMMIO",   STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation - MMIO2Mem.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstMovsMMIO,     STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOVS/MMIO2MMIO",  STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation - MMIO2MMIO.");
+    AssertRCReturn(rc, rc);
+    pVM->iom.s.pTreesR0 = MMHyperR3ToR0(pVM, pVM->iom.s.pTreesR3);
+
+    /*
+     * Register the MMIO access handler type.
+     */
+    rc = PGMR3HandlerPhysicalTypeRegister(pVM, PGMPHYSHANDLERKIND_MMIO,
+                                          iomMmioHandler,
+                                          NULL, "iomMmioHandler", "iomMmioPfHandler",
+                                          NULL, "iomMmioHandler", "iomMmioPfHandler",
+                                          "MMIO", &pVM->iom.s.hMmioHandlerType);
+    AssertRCReturn(rc, rc);
+
+    rc = PGMR3HandlerPhysicalTypeRegister(pVM, PGMPHYSHANDLERKIND_MMIO,
+                                          iomMmioHandlerNew,
+                                          NULL, "iomMmioHandlerNew", "iomMmioPfHandlerNew",
+                                          NULL, "iomMmioHandlerNew", "iomMmioPfHandlerNew",
+                                          "MMIO", &pVM->iom.s.hNewMmioHandlerType);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Info.
+     */
+    DBGFR3InfoRegisterInternal(pVM, "ioport", "Dumps all IOPort ranges. No arguments.", &iomR3IoPortInfo);
+    DBGFR3InfoRegisterInternal(pVM, "mmio", "Dumps all MMIO ranges. No arguments.", &iomR3MmioInfo);
+
+    /*
+     * Statistics.
+     */
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIOHandler,      STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler",                      STAMUNIT_TICKS_PER_CALL, "Profiling of the iomMmioPfHandler() body, only success calls.");
+#if 0
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIO1Byte,        STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access1",              STAMUNIT_OCCURENCES,     "MMIO access by 1 byte counter.");
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIO2Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access2",              STAMUNIT_OCCURENCES,     "MMIO access by 2 bytes counter.");
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIO4Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access4",              STAMUNIT_OCCURENCES,     "MMIO access by 4 bytes counter.");
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIO8Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access8",              STAMUNIT_OCCURENCES,     "MMIO access by 8 bytes counter.");
 #endif
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstOther,        STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Inst/Other",           STAMUNIT_OCCURENCES,     "Other instructions counter.");
-            STAM_REG(pVM, &pVM->iom.s.StatR3MMIOHandler,      STAMTYPE_COUNTER, "/IOM/R3-MMIOHandler",                      STAMUNIT_OCCURENCES,     "Number of calls to iomR3MmioHandler.");
-#if 0 /* unused */
-            STAM_REG(pVM, &pVM->iom.s.StatInstIn,             STAMTYPE_COUNTER, "/IOM/IOWork/In",                           STAMUNIT_OCCURENCES,     "Counter of any IN instructions.");
-            STAM_REG(pVM, &pVM->iom.s.StatInstOut,            STAMTYPE_COUNTER, "/IOM/IOWork/Out",                          STAMUNIT_OCCURENCES,     "Counter of any OUT instructions.");
-            STAM_REG(pVM, &pVM->iom.s.StatInstIns,            STAMTYPE_COUNTER, "/IOM/IOWork/Ins",                          STAMUNIT_OCCURENCES,     "Counter of any INS instructions.");
-            STAM_REG(pVM, &pVM->iom.s.StatInstOuts,           STAMTYPE_COUNTER, "/IOM/IOWork/Outs",                         STAMUNIT_OCCURENCES,     "Counter of any OUTS instructions.");
-#endif
-        }
-    }
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIOReadsToR3,    STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/ReadsToR3",            STAMUNIT_OCCURENCES,     "Number of read deferred to ring-3.");
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIOWritesToR3,   STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/WritesToR3",           STAMUNIT_OCCURENCES,     "Number of writes deferred to ring-3.");
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIOCommitsToR3,  STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/CommitsToR3",          STAMUNIT_OCCURENCES,     "Number of commits deferred to ring-3.");
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIODevLockContention, STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/DevLockContention", STAMUNIT_OCCURENCES,   "Number of device lock contention force return to ring-3.");
+    STAM_REG(pVM, &pVM->iom.s.StatR3MMIOHandler,      STAMTYPE_COUNTER, "/IOM/R3-MMIOHandler",                      STAMUNIT_OCCURENCES,     "Number of calls to iomMmioHandler.");
+
+    STAM_REG(pVM, &pVM->iom.s.StatMmioHandlerR3,      STAMTYPE_COUNTER, "/IOM/MmioHandlerR3",                       STAMUNIT_OCCURENCES,     "Number of calls to iomMmioHandlerNew from ring-3.");
+    STAM_REG(pVM, &pVM->iom.s.StatMmioHandlerR0,      STAMTYPE_COUNTER, "/IOM/MmioHandlerR0",                       STAMUNIT_OCCURENCES,     "Number of calls to iomMmioHandlerNew from ring-0.");
 
     /* Redundant, but just in case we change something in the future */
     iomR3FlushCache(pVM);
 
-    LogFlow(("IOMR3Init: returns %Rrc\n", rc));
-    return rc;
+    LogFlow(("IOMR3Init: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
 }
 
@@ -258,4 +244,16 @@
     if (enmWhat == VMINITCOMPLETED_RING3)
     {
+        /*
+         * Synchronize the ring-3 I/O port and MMIO statistics indices into the
+         * ring-0 tables to simplify ring-0 code.  This also make sure that any
+         * later calls to grow the statistics tables will fail.
+         */
+        int rc = VMMR3CallR0Emt(pVM, pVM->apCpusR3[0], VMMR0_DO_IOM_SYNC_STATS_INDICES, 0, NULL);
+        AssertLogRelRCReturn(rc, rc);
+
+        /*
+         * Register I/O port and MMIO stats now that we're done registering MMIO
+         * regions and won't grow the table again.
+         */
         for (uint32_t i = 0; i < pVM->iom.s.cIoPortRegs; i++)
         {
@@ -264,4 +262,12 @@
                 && pRegEntry->idxStats != UINT16_MAX)
                 iomR3IoPortRegStats(pVM, pRegEntry);
+        }
+
+        for (uint32_t i = 0; i < pVM->iom.s.cMmioRegs; i++)
+        {
+            PIOMMMIOENTRYR3 pRegEntry = &pVM->iom.s.paMmioRegs[i];
+            if (   pRegEntry->fMapped
+                && pRegEntry->idxStats != UINT16_MAX)
+                iomR3MmioRegStats(pVM, pRegEntry);
         }
     }
@@ -1305,5 +1311,5 @@
     AssertMsgReturn(GCPhysStart + (cbRange - 1) >= GCPhysStart,("Wrapped! %RGp LB %RGp\n", GCPhysStart, cbRange),
                     VERR_IOM_INVALID_MMIO_RANGE);
-    AssertMsgReturn(   !(fFlags & ~IOMMMIO_FLAGS_VALID_MASK)
+    AssertMsgReturn(   !(fFlags & ~(IOMMMIO_FLAGS_VALID_MASK & ~IOMMMIO_FLAGS_ABS))
                     && (fFlags & IOMMMIO_FLAGS_READ_MODE)  <= IOMMMIO_FLAGS_READ_DWORD_QWORD
                     && (fFlags & IOMMMIO_FLAGS_WRITE_MODE) <= IOMMMIO_FLAGS_WRITE_ONLY_DWORD_QWORD,
@@ -1952,72 +1958,4 @@
     /** @todo IOM debug events. */
     RT_NOREF3(pVM, enmEvent, fEnabled);
-}
-
-
-/**
- * Display a single MMIO range.
- *
- * @returns 0
- * @param   pNode   Pointer to MMIO R3 range.
- * @param   pvUser  Pointer to info output callback structure.
- */
-static DECLCALLBACK(int) iomR3MMIOInfoOne(PAVLROGCPHYSNODECORE pNode, void *pvUser)
-{
-    PIOMMMIORANGE pRange = (PIOMMMIORANGE)pNode;
-    PCDBGFINFOHLP pHlp = (PCDBGFINFOHLP)pvUser;
-    pHlp->pfnPrintf(pHlp,
-                    "%RGp-%RGp %RHv %RHv %RHv %RHv %RHv %s\n",
-                    pRange->Core.Key,
-                    pRange->Core.KeyLast,
-                    pRange->pDevInsR3,
-                    pRange->pfnReadCallbackR3,
-                    pRange->pfnWriteCallbackR3,
-                    pRange->pfnFillCallbackR3,
-                    pRange->pvUserR3,
-                    pRange->pszDesc);
-    pHlp->pfnPrintf(pHlp,
-                    "%*s %RHv %RHv %RHv %RHv %RHv\n",
-                    sizeof(RTGCPHYS) * 2 * 2 + 1, "R0",
-                    pRange->pDevInsR0,
-                    pRange->pfnReadCallbackR0,
-                    pRange->pfnWriteCallbackR0,
-                    pRange->pfnFillCallbackR0,
-                    pRange->pvUserR0);
-#if 0
-    pHlp->pfnPrintf(pHlp,
-                    "%*s %RRv %RRv %RRv %RRv %RRv\n",
-                    sizeof(RTGCPHYS) * 2 * 2 + 1, "RC",
-                    pRange->pDevInsRC,
-                    pRange->pfnReadCallbackRC,
-                    pRange->pfnWriteCallbackRC,
-                    pRange->pfnFillCallbackRC,
-                    pRange->pvUserRC);
-#endif
-    return 0;
-}
-
-
-/**
- * Display registered MMIO ranges to the log.
- *
- * @param   pVM         The cross context VM structure.
- * @param   pHlp        The info helpers.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) iomR3MMIOInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    NOREF(pszArgs);
-    pHlp->pfnPrintf(pHlp,
-                    "MMIO ranges (pVM=%p)\n"
-                    "%.*s %.*s %.*s %.*s %.*s %.*s %s\n",
-                    pVM,
-                    sizeof(RTGCPHYS) * 4 + 1, "GC Phys Range                    ",
-                    sizeof(RTHCPTR) * 2,      "pDevIns         ",
-                    sizeof(RTHCPTR) * 2,      "Read            ",
-                    sizeof(RTHCPTR) * 2,      "Write           ",
-                    sizeof(RTHCPTR) * 2,      "Fill            ",
-                    sizeof(RTHCPTR) * 2,      "pvUser          ",
-                                              "Description");
-    RTAvlroGCPhysDoWithAll(&pVM->iom.s.pTreesR3->MMIOTree, true, iomR3MMIOInfoOne, (void *)pHlp);
 }
 
Index: /trunk/src/VBox/VMM/VMMR3/IOMR3Mmio.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/IOMR3Mmio.cpp	(revision 81332)
+++ /trunk/src/VBox/VMM/VMMR3/IOMR3Mmio.cpp	(revision 81333)
@@ -61,20 +61,33 @@
 
     /* Register statistics: */
-    int rc = STAMR3Register(pVM, &pStats->Accesses,    STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, pszDesc); AssertRC(rc);
+    int rc = STAMR3Register(pVM, &pRegEntry->idxSelf, STAMTYPE_U16, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_NONE, pszDesc); AssertRC(rc);
     RTStrFree(pszFreeDesc);
 
 # define SET_NM_SUFFIX(a_sz) memcpy(&szName[cchPrefix], a_sz, sizeof(a_sz))
+    SET_NM_SUFFIX("/Read-Complicated");
+    rc = STAMR3Register(pVM, &pStats->ComplicatedReads, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, NULL); AssertRC(rc);
+    SET_NM_SUFFIX("/Read-FFor00");
+    rc = STAMR3Register(pVM, &pStats->FFor00Reads, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
     SET_NM_SUFFIX("/Read-R3");
     rc = STAMR3Register(pVM, &pStats->ProfReadR3,  STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, NULL); AssertRC(rc);
-    SET_NM_SUFFIX("/Write-R3");
-    rc = STAMR3Register(pVM, &pStats->ProfWriteR3, STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, NULL); AssertRC(rc);
     SET_NM_SUFFIX("/Read-RZ");
     rc = STAMR3Register(pVM, &pStats->ProfReadRZ,  STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, NULL); AssertRC(rc);
+    SET_NM_SUFFIX("/Read-RZtoR3");
+    rc = STAMR3Register(pVM, &pStats->ReadRZToR3,  STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
+    SET_NM_SUFFIX("/Read-Total");
+    rc = STAMR3Register(pVM, &pStats->Reads,       STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
+
+    SET_NM_SUFFIX("/Write-Complicated");
+    rc = STAMR3Register(pVM, &pStats->ComplicatedWrites, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, NULL); AssertRC(rc);
+    SET_NM_SUFFIX("/Write-R3");
+    rc = STAMR3Register(pVM, &pStats->ProfWriteR3,  STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, NULL); AssertRC(rc);
     SET_NM_SUFFIX("/Write-RZ");
     rc = STAMR3Register(pVM, &pStats->ProfWriteRZ, STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, NULL); AssertRC(rc);
-    SET_NM_SUFFIX("/Read-RZtoR3");
-    rc = STAMR3Register(pVM, &pStats->ReadRZToR3,  STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
     SET_NM_SUFFIX("/Write-RZtoR3");
     rc = STAMR3Register(pVM, &pStats->WriteRZToR3, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
+    SET_NM_SUFFIX("/Write-RZtoR3-Commit");
+    rc = STAMR3Register(pVM, &pStats->CommitRZToR3, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,    NULL); AssertRC(rc);
+    SET_NM_SUFFIX("/Write-Total");
+    rc = STAMR3Register(pVM, &pStats->Writes,      STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
 }
 
@@ -111,5 +124,5 @@
 
     AssertMsgReturn(cbRegion > 0 && !(cbRegion & PAGE_OFFSET_MASK), ("cbRegion=%RGp\n", cbRegion), VERR_OUT_OF_RANGE);
-    AssertReturn(!(fFlags & ~IOM_MMIO_F_VALID_MASK), VERR_INVALID_FLAGS);
+    AssertReturn(!(fFlags & ~IOMMMIO_FLAGS_VALID_MASK), VERR_INVALID_FLAGS);
 
     AssertReturn(pfnWrite || pfnRead || pfnFill, VERR_INVALID_PARAMETER);
@@ -218,4 +231,9 @@
                     else
                     {
+                        /* Register with PGM before we shuffle the array: */
+                        rc = PGMR3PhysMMIORegister(pVM, GCPhys, cbRegion, pVM->iom.s.hNewMmioHandlerType,
+                                                   (void *)(uintptr_t)hRegion, hRegion, hRegion, pRegEntry->pszDesc);
+                        AssertRCReturnStmt(rc, IOM_UNLOCK_EXCL(pVM), rc);
+
                         /* Insert after the entry we just considered: */
                         pEntry += 1;
@@ -231,4 +249,9 @@
                     else
                     {
+                        /* Register with PGM before we shuffle the array: */
+                        rc = PGMR3PhysMMIORegister(pVM, GCPhys, cbRegion, pVM->iom.s.hNewMmioHandlerType,
+                                                   (void *)(uintptr_t)hRegion, hRegion, hRegion, pRegEntry->pszDesc);
+                        AssertRCReturnStmt(rc, IOM_UNLOCK_EXCL(pVM), rc);
+
                         /* Insert at the entry we just considered: */
                         if (i < cEntries)
@@ -366,5 +389,6 @@
                 pRegEntry->GCPhysMapping = NIL_RTGCPHYS;
                 pRegEntry->fMapped       = false;
-                rc = VINF_SUCCESS;
+                rc = PGMR3PhysMMIODeregister(pVM, GCPhys, pRegEntry->cbRegion);
+                AssertRC(rc);
                 break;
             }
Index: /trunk/src/VBox/VMM/include/IOMInline.h
===================================================================
--- /trunk/src/VBox/VMM/include/IOMInline.h	(revision 81332)
+++ /trunk/src/VBox/VMM/include/IOMInline.h	(revision 81333)
@@ -37,7 +37,7 @@
  *
  * @param   pVM             The cross context VM structure.
- * @param   uPort           The I/O port lookup.
- * @param   poffPort        Where to the port offset relative to the start of
- *                          the I/O port range.
+ * @param   uPort           The I/O port to lookup.
+ * @param   poffPort        Where to return the port offset relative to the
+ *                          start of the I/O port range.
  * @param   pidxLastHint    Pointer to IOMCPU::idxIoPortLastRead or
  *                          IOMCPU::idxIoPortLastWrite.
@@ -47,4 +47,6 @@
  *          for the entry.  Use IOMIOPORTENTRYR0::idxSelf to get the ring-3
  *          entry.
+ *
+ * @note    This code is almost identical to iomMmioGetEntry, so keep in sync.
  */
 DECLINLINE(CTX_SUFF(PIOMIOPORTENTRY)) iomIoPortGetEntry(PVMCC pVM, RTIOPORT uPort, PRTIOPORT poffPort, uint16_t *pidxLastHint)
@@ -117,5 +119,5 @@
 #ifdef VBOX_WITH_STATISTICS
 /**
- * Gets the I/O port statistics entry .
+ * Gets the statistics entry for an I/O port.
  *
  * @returns Pointer to stats.  Instead of NULL, a pointer to IoPortDummyStats is
@@ -139,4 +141,115 @@
 # endif
     return &pVM->iom.s.IoPortDummyStats;
+}
+#endif
+
+
+/**
+ * Gets the MMIO region entry for the specified address in the current context.
+ *
+ * @returns Pointer to MMIO region entry.
+ * @returns NULL if no MMIO region registered for the given address.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          The address to lookup.
+ * @param   poffRegion      Where to return the byte offset into the MMIO
+ *                          region that corresponds to @a GCPhys.
+ * @param   pidxLastHint    Pointer to IOMCPU::idxMmioLastRead,
+ *                          IOMCPU::idxMmioLastWrite, or similar.
+ *
+ * @note    In ring-0 it is possible to get an uninitialized entry (pDevIns is
+ *          NULL, cbRegion is 0), in which case there should be ring-3 handlers
+ *          for the entry.  Use IOMMMIOENTRYR0::idxSelf to get the ring-3 entry.
+ *
+ * @note    This code is almost identical to iomIoPortGetEntry, so keep in sync.
+ */
+DECLINLINE(CTX_SUFF(PIOMMMIOENTRY)) iomMmioGetEntry(PVMCC pVM, RTGCPHYS GCPhys, PRTGCPHYS poffRegion, uint16_t *pidxLastHint)
+{
+    Assert(IOM_IS_SHARED_LOCK_OWNER(pVM));
+
+#ifdef IN_RING0
+    uint32_t               iEnd      = RT_MIN(pVM->iom.s.cMmioLookupEntries, pVM->iomr0.s.cMmioAlloc);
+    PCIOMMMIOLOOKUPENTRY   paLookup  = pVM->iomr0.s.paMmioLookup;
+#else
+    uint32_t               iEnd      = pVM->iom.s.cMmioLookupEntries;
+    PCIOMMMIOLOOKUPENTRY   paLookup  = pVM->iom.s.paMmioLookup;
+#endif
+    if (iEnd > 0)
+    {
+        uint32_t iFirst = 0;
+        uint32_t i      = *pidxLastHint;
+        if (i < iEnd)
+        { /* likely */ }
+        else
+            i = iEnd / 2;
+        for (;;)
+        {
+            PCIOMMMIOLOOKUPENTRY pCur = &paLookup[i];
+            if (pCur->GCPhysFirst > GCPhys)
+            {
+                if (i > iFirst)
+                    iEnd = i;
+                else
+                    break;
+            }
+            else if (pCur->GCPhysLast < GCPhys)
+            {
+                i += 1;
+                if (i < iEnd)
+                    iFirst = i;
+                else
+                    break;
+            }
+            else
+            {
+                *pidxLastHint = (uint16_t)i;
+                *poffRegion   = GCPhys - pCur->GCPhysFirst;
+
+                /*
+                 * Translate the 'idx' member into a pointer.
+                 */
+                size_t const idx = pCur->idx;
+#ifdef IN_RING0
+                AssertMsg(idx < pVM->iom.s.cMmioRegs && idx < pVM->iomr0.s.cMmioAlloc,
+                          ("%#zx vs %#x/%x (GCPhys=%RGp)\n", idx, pVM->iom.s.cMmioRegs, pVM->iomr0.s.cMmioMax, GCPhys));
+                if (idx < pVM->iomr0.s.cMmioAlloc)
+                    return &pVM->iomr0.s.paMmioRegs[idx];
+#else
+                if (idx < pVM->iom.s.cMmioRegs)
+                    return &pVM->iom.s.paMmioRegs[idx];
+                AssertMsgFailed(("%#zx vs %#x (GCPhys=%RGp)\n", idx, pVM->iom.s.cMmioRegs, GCPhys));
+#endif
+                break;
+            }
+
+            i = iFirst + (iEnd - iFirst) / 2;
+        }
+    }
+    *poffRegion = 0;
+    return NULL;
+}
+
+
+#ifdef VBOX_WITH_STATISTICS
+/**
+ * Gets the statistics entry for an MMIO region.
+ *
+ * @returns Pointer to stats.  Instead of NULL, a pointer to MmioDummyStats is
+ *          returned, so the caller does not need to check for NULL.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pRegEntry   The I/O port entry to get stats for.
+ */
+DECLINLINE(PIOMMMIOSTATSENTRY) iomMmioGetStats(PVMCC pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
+{
+    size_t idxStats = pRegEntry->idxStats;
+# ifdef IN_RING0
+    if (idxStats < pVM->iomr0.s.cMmioStatsAllocation)
+        return &pVM->iomr0.s.paMmioStats[idxStats];
+# else
+    if (idxStats < pVM->iom.s.cMmioStats)
+        return &pVM->iom.s.paMmioStats[idxStats];
+# endif
+    return &pVM->iom.s.MmioDummyStats;
 }
 #endif
Index: /trunk/src/VBox/VMM/include/IOMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/IOMInternal.h	(revision 81332)
+++ /trunk/src/VBox/VMM/include/IOMInternal.h	(revision 81333)
@@ -307,5 +307,7 @@
     /** Pointer to the fill callback function. */
     R0PTRTYPE(PFNIOMMMIONEWFILL)        pfnFillCallback;
-    /** The entry of the first statistics entry, UINT16_MAX if no stats. */
+    /** The entry of the first statistics entry, UINT16_MAX if no stats.
+     * @note For simplicity, this is always copied from ring-3 for all entries at
+     *       the end of VM creation. */
     uint16_t                            idxStats;
     /** Same as the handle index. */
@@ -371,21 +373,29 @@
 typedef struct IOMMMIOSTATSENTRY
 {
-    /** Number of accesses (subtract ReadRZToR3 and WriteRZToR3 to get the right
-     *  number). */
-    STAMCOUNTER                 Accesses;
-
+    /** Counting and profiling reads in R0/RC. */
+    STAMPROFILE                 ProfReadRZ;
+    /** Number of successful read accesses. */
+    STAMCOUNTER                 Reads;
+    /** Number of reads to this address from R0/RC which was serviced in R3. */
+    STAMCOUNTER                 ReadRZToR3;
+    /** Number of complicated reads. */
+    STAMCOUNTER                 ComplicatedReads;
+    /** Number of reads of 0xff or 0x00. */
+    STAMCOUNTER                 FFor00Reads;
     /** Profiling read handler overhead in R3. */
     STAMPROFILE                 ProfReadR3;
+
+    /** Counting and profiling writes in R0/RC. */
+    STAMPROFILE                 ProfWriteRZ;
+    /** Number of successful read accesses. */
+    STAMCOUNTER                 Writes;
+    /** Number of writes to this address from R0/RC which was serviced in R3. */
+    STAMCOUNTER                 WriteRZToR3;
+    /** Number of writes to this address from R0/RC which was committed in R3. */
+    STAMCOUNTER                 CommitRZToR3;
+    /** Number of complicated writes. */
+    STAMCOUNTER                 ComplicatedWrites;
     /** Profiling write handler overhead in R3. */
     STAMPROFILE                 ProfWriteR3;
-    /** Counting and profiling reads in R0/RC. */
-    STAMPROFILE                 ProfReadRZ;
-    /** Counting and profiling writes in R0/RC. */
-    STAMPROFILE                 ProfWriteRZ;
-
-    /** Number of reads to this address from R0/RC which was serviced in R3. */
-    STAMCOUNTER                 ReadRZToR3;
-    /** Number of writes to this address from R0/RC which was serviced in R3. */
-    STAMCOUNTER                 WriteRZToR3;
 } IOMMMIOSTATSENTRY;
 /** Pointer to MMIO statistics entry. */
@@ -602,10 +612,10 @@
         /** Guest physical MMIO address. */
         RTGCPHYS                        GCPhys;
+        /** The number of bytes to write (0 if nothing pending). */
+        uint32_t                        cbValue;
+        /** Hint. */
+        uint32_t                        idxMmioRegionHint;
         /** The value to write. */
         uint8_t                         abValue[128];
-        /** The number of bytes to write (0 if nothing pending). */
-        uint32_t                        cbValue;
-        /** Alignment padding. */
-        uint32_t                        uAlignmentPadding;
     } PendingMmioWrite;
 
@@ -654,5 +664,6 @@
     /** MMIO physical access handler type.   */
     PGMPHYSHANDLERTYPE              hMmioHandlerType;
-    uint32_t                        u32Padding;
+    /** MMIO physical access handler type, new style.   */
+    PGMPHYSHANDLERTYPE              hNewMmioHandlerType;
 
     /** @name I/O ports
@@ -729,30 +740,18 @@
      * @{ */
     STAMPROFILE                     StatRZMMIOHandler;
-    STAMCOUNTER                     StatRZMMIOFailures;
-
-    STAMPROFILE                     StatRZInstMov;
-    STAMPROFILE                     StatRZInstCmp;
-    STAMPROFILE                     StatRZInstAnd;
-    STAMPROFILE                     StatRZInstOr;
-    STAMPROFILE                     StatRZInstXor;
-    STAMPROFILE                     StatRZInstBt;
-    STAMPROFILE                     StatRZInstTest;
-    STAMPROFILE                     StatRZInstXchg;
-    STAMPROFILE                     StatRZInstStos;
-    STAMPROFILE                     StatRZInstLods;
-#ifdef IOM_WITH_MOVS_SUPPORT
-    STAMPROFILEADV                  StatRZInstMovs;
-    STAMPROFILE                     StatRZInstMovsToMMIO;
-    STAMPROFILE                     StatRZInstMovsFromMMIO;
-    STAMPROFILE                     StatRZInstMovsMMIO;
-#endif
-    STAMCOUNTER                     StatRZInstOther;
-
+    STAMCOUNTER                     StatRZMMIOReadsToR3;
+    STAMCOUNTER                     StatRZMMIOWritesToR3;
+    STAMCOUNTER                     StatRZMMIOCommitsToR3;
+    STAMCOUNTER                     StatRZMMIODevLockContention;
+#if 0
     STAMCOUNTER                     StatRZMMIO1Byte;
     STAMCOUNTER                     StatRZMMIO2Bytes;
     STAMCOUNTER                     StatRZMMIO4Bytes;
     STAMCOUNTER                     StatRZMMIO8Bytes;
-
+#endif
     STAMCOUNTER                     StatR3MMIOHandler;
+
+    STAMCOUNTER                     StatMmioHandlerR3;
+    STAMCOUNTER                     StatMmioHandlerR0;
 
     RTUINT                          cMovsMaxBytes;
@@ -791,4 +790,6 @@
     /** The size of the paIoPortStats allocation (in entries). */
     uint32_t                        cIoPortStatsAllocation;
+    /** Prevents paIoPortStats from growing, set by IOMR0IoPortSyncStatisticsIndices(). */
+    bool                            fIoPortStatsFrozen;
     /** I/O port lookup table.   */
     R0PTRTYPE(PIOMIOPORTSTATSENTRY) paIoPortStats;
@@ -822,4 +823,6 @@
     /** The size of the paMmioStats allocation (in entries). */
     uint32_t                        cMmioStatsAllocation;
+    /* Prevents paMmioStats from growing, set by IOMR0MmioSyncStatisticsIndices(). */
+    bool                            fMmioStatsFrozen;
     /** MMIO lookup table.   */
     R0PTRTYPE(PIOMMMIOSTATSENTRY)   paMmioStats;
@@ -841,4 +844,6 @@
 DECLCALLBACK(void)  iomR3IoPortInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
 void                iomR3IoPortRegStats(PVM pVM, PIOMIOPORTENTRYR3 pRegEntry);
+DECLCALLBACK(void)  iomR3MmioInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+void                iomR3MmioRegStats(PVM pVM, PIOMMMIOENTRYR3 pRegEntry);
 #endif /* IN_RING3 */
 #ifdef IN_RING0
@@ -848,9 +853,13 @@
 void                iomR0MmioInitPerVMData(PGVM pGVM);
 #endif
+VBOXSTRICTRC        iomMmioCommonPfHandlerOld(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, PCPUMCTXCORE pCtxCore,
+                                              RTGCPHYS GCPhysFault, void *pvUser);
 
 #ifndef IN_RING3
 DECLEXPORT(FNPGMRZPHYSPFHANDLER)    iomMmioPfHandler;
+DECLEXPORT(FNPGMRZPHYSPFHANDLER)    iomMmioPfHandlerNew;
 #endif
 PGM_ALL_CB2_PROTO(FNPGMPHYSHANDLER) iomMmioHandler;
+PGM_ALL_CB2_PROTO(FNPGMPHYSHANDLER) iomMmioHandlerNew;
 
 /* IOM locking helpers. */
