Changeset 81333 in vbox

Timestamp:

Oct 17, 2019 11:49:39 PM (5 years ago)

Author:

vboxsync

Message:

IOM: More MMIO stuff, almost there now... bugref:9218

Location:

trunk

Files:

• include/VBox/vmm/iom.h (modified) (8 diffs)
• include/VBox/vmm/pdmdev.h (modified) (1 diff)
• include/VBox/vmm/vmm.h (modified) (1 diff)
• src/VBox/VMM/Makefile.kmk (modified) (2 diffs)
• src/VBox/VMM/VMMAll/IOMAll.cpp (modified) (1 diff)
• src/VBox/VMM/VMMAll/IOMAllMMIO.cpp (modified) (7 diffs)
• src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp (copied) (copied from trunk/src/VBox/VMM/VMMAll/IOMAllMMIO.cpp ) (37 diffs)
• src/VBox/VMM/VMMR0/IOMR0IoPort.cpp (modified) (2 diffs)
• src/VBox/VMM/VMMR0/IOMR0Mmio.cpp (modified) (2 diffs)
• src/VBox/VMM/VMMR0/VMMR0.cpp (modified) (1 diff)
• src/VBox/VMM/VMMR3/IOM.cpp (modified) (6 diffs)
• src/VBox/VMM/VMMR3/IOMR3Mmio.cpp (modified) (5 diffs)
• src/VBox/VMM/include/IOMInline.h (modified) (4 diffs)
• src/VBox/VMM/include/IOMInternal.h (modified) (9 diffs)

trunk/include/VBox/vmm/iom.h

@@ -155 +155 @@
 #define IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_WRITE      UINT32_C(0x00000200)
 
+/** Pass the absolute physical address (GC) to the callback rather than the
+ * relative one.
+ * @note New-style only, is implicit in old-style interface.  */
+#define IOMMMIO_FLAGS_ABS                               UINT32_C(0x00001000)
+
 /** Mask of valid flags. */
-#define IOMMMIO_FLAGS_VALID_MASK                        UINT32_C(0x00000373)
+#define IOMMMIO_FLAGS_VALID_MASK                        UINT32_C(0x00001373)
 /** @} */
 
@@ -399 +404 @@
  * @param   pvUser      User argument.
  * @param   off         Offset into the mapping of the read,
- *                      or the physical address if IOM_MMIO_F_ABS is active.
+ *                      or the physical address if IOMMMIO_FLAGS_ABS is active.
  * @param   pv          Where to store the result.
  * @param   cb          Number of bytes read.
@@ -416 +421 @@
  * @param   pvUser      User argument.
  * @param   off         Offset into the mapping of the write,
- *                      or the physical address if IOM_MMIO_F_ABS is active.
+ *                      or the physical address if IOMMMIO_FLAGS_ABS is active.
  * @param   pv          Where to fetch the result.
  * @param   cb          Number of bytes to write.
@@ -433 +438 @@
  * @param   pvUser      User argument.
  * @param   off         Offset into the mapping of the fill,
- *                      or the physical address if IOM_MMIO_F_ABS is active.
+ *                      or the physical address if IOMMMIO_FLAGS_ABS is active.
  * @param   u32Item     Byte/Word/Dword data to fill.
  * @param   cbItem      Size of data in u32Item parameter, restricted to 1/2/4 bytes.
@@ -456 +461 @@
 VMMDECL(int)            IOMMMIOMapMMIOHCPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint64_t fPageFlags);
 VMMDECL(int)            IOMMMIOResetRegion(PVMCC pVM, RTGCPHYS GCPhys);
-VMMDECL(bool)           IOMIsLockWriteOwner(PVM pVM);
+
+VMM_INT_DECL(VBOXSTRICTRC)  IOMMmioPhysHandler(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, RTGCPHYS GCPhysFault);
+VMM_INT_DECL(int)           IOMMmioMapMmio2Page(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, RTGCPHYS offRegion,
+                                                uint64_t hMmio2, RTGCPHYS offMmio2, uint64_t fPageFlags);
+VMM_INT_DECL(int)           IOMMmioMapMmioHCPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint64_t fPageFlags);
+VMM_INT_DECL(int)           IOMMmioResetRegion(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion);
+
 
 /** @name IOM_IOPORT_F_XXX - Flags for IOMR3IoPortCreate() and PDMDevHlpIoPortCreateEx().
@@ -464 +475 @@
 /** Valid flags for IOMR3IoPortCreate(). */
 #define IOM_IOPORT_F_VALID_MASK     UINT32_C(0x00000001)
-/** @} */
-
-/** @name IOM_MMIO_F_XXX - Flags for IOMR3MmioCreate() and PDMDevHlpMmioCreateEx().
- * @{ */
-/** Pass the absolute physical address (GC) to the callback rather than the
- * relative one. */
-#define IOM_MMIO_F_ABS              RT_BIT_32(0)
-/** Valid flags for IOMR3IoPortCreate(). */
-#define IOM_MMIO_F_VALID_MASK       UINT32_C(0x00000001)
 /** @} */
 
@@ -574 +576 @@
 VMMR0_INT_DECL(int)  IOMR0IoPortGrowRegistrationTables(PGVM pGVM, uint64_t cMinEntries);
 VMMR0_INT_DECL(int)  IOMR0IoPortGrowStatisticsTable(PGVM pGVM, uint64_t cMinEntries);
+VMMR0_INT_DECL(int)  IOMR0IoPortSyncStatisticsIndices(PGVM pGVM);
 
 VMMR0_INT_DECL(int)  IOMR0MmioSetUpContext(PGVM pGVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, PFNIOMMMIONEWWRITE pfnWrite,
@@ -579 +582 @@
 VMMR0_INT_DECL(int)  IOMR0MmioGrowRegistrationTables(PGVM pGVM, uint64_t cMinEntries);
 VMMR0_INT_DECL(int)  IOMR0MmioGrowStatisticsTable(PGVM pGVM, uint64_t cMinEntries);
+VMMR0_INT_DECL(int)  IOMR0MmioSyncStatisticsIndices(PGVM pGVM);
+
 /** @} */
 #endif /* IN_RING0 || DOXYGEN_RUNNING */

trunk/include/VBox/vmm/pdmdev.h

@@ -2418 +2418 @@
      * @param   pDevIns     The device instance to register the ports with.
      * @param   cbRegion    The size of the region in bytes.
-     * @param   fFlags      Reserved, MBZ.
+     * @param   fFlags      Flags, IOMMMIO_FLAGS_XXX.
      * @param   pPciDev     The PCI device the range is associated with, if
      *                      applicable.

trunk/include/VBox/vmm/vmm.h

@@ -419 +419 @@
     /** Grow the MMIO statistics tables. */
     VMMR0_DO_IOM_GROW_MMIO_STATS,
+    /** Synchronize statistics indices for I/O ports and MMIO regions. */
+    VMMR0_DO_IOM_SYNC_STATS_INDICES,
 
     /** Official call we use for testing Ring-0 APIs. */

trunk/src/VBox/VMM/Makefile.kmk

@@ -181 +181 @@
         VMMAll/IOMAll.cpp \
         VMMAll/IOMAllMMIO.cpp \
+        VMMAll/IOMAllMmioNew.cpp \
         VMMAll/MMAll.cpp \
         VMMAll/MMAllHyper.cpp \
@@ -510 +511 @@
         VMMAll/IOMAll.cpp \
         VMMAll/IOMAllMMIO.cpp \
+        VMMAll/IOMAllMmioNew.cpp \
         VMMAll/MMAll.cpp \
         VMMAll/MMAllHyper.cpp \

trunk/src/VBox/VMM/VMMAll/IOMAll.cpp

@@ -37 +37 @@
 #include <iprt/string.h>
 #include "IOMInline.h"
-
-
-/**
- * Check if this VCPU currently owns the IOM lock exclusively.
- *
- * @returns bool owner/not owner
- * @param   pVM         The cross context VM structure.
- */
-VMMDECL(bool) IOMIsLockWriteOwner(PVM pVM)
-{
-#ifdef IOM_WITH_CRIT_SECT_RW
-    return PDMCritSectRwIsInitialized(&pVM->iom.s.CritSect)
-        && PDMCritSectRwIsWriteOwner(&pVM->iom.s.CritSect);
-#else
-    return PDMCritSectIsOwner(&pVM->iom.s.CritSect);
-#endif
-}
 
 

trunk/src/VBox/VMM/VMMAll/IOMAllMMIO.cpp

@@ -67 +67 @@
         AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
         pVCpu->iom.s.PendingMmioWrite.cbValue = (uint32_t)cbBuf;
+        pVCpu->iom.s.PendingMmioWrite.idxMmioRegionHint = UINT32_MAX;
         memcpy(pVCpu->iom.s.PendingMmioWrite.abValue, pvBuf, cbBuf);
     }
@@ -584 +585 @@
  * @param   pvUser      Pointer to the MMIO ring-3 range entry.
  */
-static VBOXSTRICTRC iomMmioCommonPfHandler(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, PCPUMCTXCORE pCtxCore,
-                                           RTGCPHYS GCPhysFault, void *pvUser)
+VBOXSTRICTRC iomMmioCommonPfHandlerOld(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, PCPUMCTXCORE pCtxCore,
+                                       RTGCPHYS GCPhysFault, void *pvUser)
 {
     RT_NOREF_PV(uErrorCode);
@@ -596 +597 @@
 
     STAM_PROFILE_START(&pVM->iom.s.StatRZMMIOHandler, a);
-    Log(("iomMmioCommonPfHandler: GCPhys=%RGp uErr=%#x rip=%RGv\n", GCPhysFault, uErrorCode, (RTGCPTR)pCtxCore->rip));
+    Log(("iomMmioCommonPfHandlerOld: GCPhys=%RGp uErr=%#x rip=%RGv\n", GCPhysFault, uErrorCode, (RTGCPTR)pCtxCore->rip));
 
     PIOMMMIORANGE pRange = (PIOMMMIORANGE)pvUser;
@@ -617 +618 @@
 # else
         STAM_PROFILE_STOP(&pVM->iom.s.StatRZMMIOHandler, a);
-        STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOFailures);
         return VINF_IOM_R3_MMIO_READ_WRITE;
 # endif
@@ -645 +645 @@
 
         STAM_PROFILE_STOP(&pVM->iom.s.StatRZMMIOHandler, a);
-        STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOFailures);
         iomMmioReleaseRange(pVM, pRange);
         return VINF_IOM_R3_MMIO_READ_WRITE;
@@ -694 +693 @@
     LogFlow(("iomMmioPfHandler: GCPhys=%RGp uErr=%#x pvFault=%RGv rip=%RGv\n",
              GCPhysFault, (uint32_t)uErrorCode, pvFault, (RTGCPTR)pCtxCore->rip)); NOREF(pvFault);
-    return iomMmioCommonPfHandler(pVM, pVCpu, (uint32_t)uErrorCode, pCtxCore, GCPhysFault, pvUser);
+    return iomMmioCommonPfHandlerOld(pVM, pVCpu, (uint32_t)uErrorCode, pCtxCore, GCPhysFault, pvUser);
 }
 
@@ -727 +726 @@
     IOM_UNLOCK_SHARED(pVM);
 
-    VBOXSTRICTRC rcStrict = iomMmioCommonPfHandler(pVM, pVCpu, (uint32_t)uErrorCode, pCtxCore, GCPhysFault, pRange);
+    VBOXSTRICTRC rcStrict = iomMmioCommonPfHandlerOld(pVM, pVCpu, (uint32_t)uErrorCode, pCtxCore, GCPhysFault, pRange);
 
     iomMmioReleaseRange(pVM, pRange);

trunk/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp

@@ -21 +21 @@
 *********************************************************************************************************************************/
 #define LOG_GROUP LOG_GROUP_IOM
+#define VMCPU_INCL_CPUM_GST_CTX
 #include <VBox/vmm/iom.h>
 #include <VBox/vmm/cpum.h>
@@ -47 +48 @@
 
 
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** @def IOM_MMIO_STATS_COMMA_DECL
+ * Parameter list declaration for statistics entry pointer. */
+/** @def IOM_MMIO_STATS_COMMA_ARG
+ * Statistics entry pointer argument. */
+#ifdef VBOX_WITH_STATISTICS
+# define IOM_MMIO_STATS_COMMA_DECL , PIOMMMIOSTATSENTRY pStats
+# define IOM_MMIO_STATS_COMMA_ARG  , pStats
+#else
+# define IOM_MMIO_STATS_COMMA_DECL
+# define IOM_MMIO_STATS_COMMA_ARG
+#endif
+
+
 
 #ifndef IN_RING3
@@ -57 +74 @@
  * @param   pvBuf       The bytes being written.
  * @param   cbBuf       How many bytes.
- * @param   pRange      The range, if resolved.
- */
-static VBOXSTRICTRC iomMmioRing3WritePending(PVMCPU pVCpu, RTGCPHYS GCPhys, void const *pvBuf, size_t cbBuf, PIOMMMIORANGE pRange)
-{
-    Log5(("iomMmioRing3WritePending: %RGp LB %#x\n", GCPhys, cbBuf));
+ * @param   idxRegEntry The MMIO registration index (handle) if available,
+ *                      otherwise UINT32_MAX.
+ */
+static VBOXSTRICTRC iomMmioRing3WritePending(PVMCPU pVCpu, RTGCPHYS GCPhys, void const *pvBuf, size_t cbBuf,
+                                             uint32_t idxRegEntry)
+{
+    Log5(("iomMmioRing3WritePending: %RGp LB %#x (idx=%#x)\n", GCPhys, cbBuf, idxRegEntry));
     if (pVCpu->iom.s.PendingMmioWrite.cbValue == 0)
     {
-        pVCpu->iom.s.PendingMmioWrite.GCPhys  = GCPhys;
+        pVCpu->iom.s.PendingMmioWrite.GCPhys            = GCPhys;
         AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
-        pVCpu->iom.s.PendingMmioWrite.cbValue = (uint32_t)cbBuf;
+        pVCpu->iom.s.PendingMmioWrite.cbValue           = (uint32_t)cbBuf;
+        pVCpu->iom.s.PendingMmioWrite.idxMmioRegionHint = idxRegEntry;
         memcpy(pVCpu->iom.s.PendingMmioWrite.abValue, pvBuf, cbBuf);
     }
@@ -89 +109 @@
 
     VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
-    RT_NOREF_PV(pRange);
     return VINF_IOM_R3_MMIO_COMMIT_WRITE;
 }
@@ -107 +126 @@
  * @param   pVM         The cross context VM structure.
  * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
- * @param   pRange      The range to write to.
+ * @param   pRegEntry   The MMIO entry for the current context.
  * @param   GCPhys      The physical address to start writing.
  * @param   pvValue     Where to store the value.
  * @param   cbValue     The size of the value to write.
  */
-static VBOXSTRICTRC iomMMIODoComplicatedWrite(PVM pVM, PVMCPU pVCpu, PIOMMMIORANGE pRange, RTGCPHYS GCPhys,
-                                              void const *pvValue, unsigned cbValue)
-{
-    RT_NOREF_PV(pVCpu);
-    AssertReturn(   (pRange->fFlags & IOMMMIO_FLAGS_WRITE_MODE) != IOMMMIO_FLAGS_WRITE_PASSTHRU
-                 && (pRange->fFlags & IOMMMIO_FLAGS_WRITE_MODE) <= IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING,
+static VBOXSTRICTRC iomMmioDoComplicatedWrite(PVM pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys,
+                                              void const *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    AssertReturn(   (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) != IOMMMIO_FLAGS_WRITE_PASSTHRU
+                 && (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) <= IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING,
                  VERR_IOM_MMIO_IPE_1);
     AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
     RTGCPHYS const GCPhysStart  = GCPhys; NOREF(GCPhysStart);
-    bool const     fReadMissing = (pRange->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_READ_MISSING
-                               || (pRange->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING;
+    bool const     fReadMissing = (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_READ_MISSING
+                               || (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING;
+    RT_NOREF_PV(pVCpu); /* ring-3 */
 
     /*
      * Do debug stop if requested.
      */
-    int rc = VINF_SUCCESS; NOREF(pVM);
+    VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
 #ifdef VBOX_STRICT
-    if (pRange->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_WRITE)
+    if (!(pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_WRITE))
+    { /* likely */ }
+    else
     {
 # ifdef IN_RING3
         LogRel(("IOM: Complicated write %#x byte at %RGp to %s, initiating debugger intervention\n", cbValue, GCPhys,
-                R3STRING(pRange->pszDesc)));
+                R3STRING(pRegEntry->pszDesc)));
         rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
-                            "Complicated write %#x byte at %RGp to %s\n", cbValue, GCPhys, R3STRING(pRange->pszDesc));
+                            "Complicated write %#x byte at %RGp to %s\n", cbValue, GCPhys, pRegEntry->pszDesc);
         if (rc == VERR_DBGF_NOT_ATTACHED)
             rc = VINF_SUCCESS;
@@ -144 +165 @@
 #endif
 
+    STAM_COUNTER_INC(&pStats->ComplicatedWrites);
+
     /*
      * Check if we should ignore the write.
      */
-    if ((pRange->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD)
+    if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD)
     {
         Assert(cbValue != 4 || (GCPhys & 3));
         return VINF_SUCCESS;
     }
-    if ((pRange->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD_QWORD)
+    if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD_QWORD)
     {
         Assert((cbValue != 4 && cbValue != 8) || (GCPhys & (cbValue - 1)));
@@ -174 +197 @@
         if (fReadMissing && cbThisPart != 4)
         {
-            int rc2 = pRange->CTX_SUFF(pfnReadCallback)(pRange->CTX_SUFF(pDevIns), pRange->CTX_SUFF(pvUser),
-                                                        GCPhys & ~(RTGCPHYS)3, &u32MissingValue, sizeof(u32MissingValue));
-            switch (rc2)
+            VBOXSTRICTRC rc2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                          GCPhys & ~(RTGCPHYS)3, &u32MissingValue, sizeof(u32MissingValue));
+            switch (VBOXSTRICTRC_VAL(rc2))
             {
                 case VINF_SUCCESS:
                     break;
                 case VINF_IOM_MMIO_UNUSED_FF:
+                    STAM_COUNTER_INC(&pStats->FFor00Reads);
                     u32MissingValue = UINT32_C(0xffffffff);
                     break;
                 case VINF_IOM_MMIO_UNUSED_00:
+                    STAM_COUNTER_INC(&pStats->FFor00Reads);
                     u32MissingValue = 0;
                     break;
@@ -190 +215 @@
                 case VINF_IOM_R3_MMIO_READ_WRITE:
                 case VINF_IOM_R3_MMIO_WRITE:
-                    LogFlow(("iomMMIODoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, rc2));
-                    rc2 = VBOXSTRICTRC_TODO(iomMmioRing3WritePending(pVCpu, GCPhys, pvValue, cbValue, pRange));
+                    LogFlow(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, rc2));
+                    rc2 = iomMmioRing3WritePending(pVCpu, GCPhys, pvValue, cbValue, pRegEntry->idxSelf);
                     if (rc == VINF_SUCCESS || rc2 < rc)
                         rc = rc2;
@@ -199 +224 @@
                     if (RT_FAILURE(rc2))
                     {
-                        Log(("iomMMIODoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, rc2));
+                        Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, rc2));
                         return rc2;
                     }
@@ -248 +273 @@
          * Do DWORD write to the device.
          */
-        int rc2 = pRange->CTX_SUFF(pfnWriteCallback)(pRange->CTX_SUFF(pDevIns), pRange->CTX_SUFF(pvUser),
-                                                     GCPhys & ~(RTGCPHYS)3, &u32Value, sizeof(u32Value));
-        switch (rc2)
+        VBOXSTRICTRC rc2 = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                       GCPhys & ~(RTGCPHYS)3, &u32Value, sizeof(u32Value));
+        switch (VBOXSTRICTRC_VAL(rc2))
         {
             case VINF_SUCCESS:
@@ -258 +283 @@
             case VINF_IOM_R3_MMIO_READ_WRITE:
             case VINF_IOM_R3_MMIO_WRITE:
-                Log3(("iomMMIODoComplicatedWrite: deferring GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, rc2));
+                Log3(("iomMmioDoComplicatedWrite: deferring GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, rc2));
                 AssertReturn(pVCpu->iom.s.PendingMmioWrite.cbValue == 0, VERR_IOM_MMIO_IPE_1);
                 AssertReturn(cbValue + (GCPhys & 3) <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
@@ -275 +300 @@
                 if (RT_FAILURE(rc2))
                 {
-                    Log(("iomMMIODoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, rc2));
+                    Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, rc2));
                     return rc2;
                 }
@@ -301 +326 @@
 
 /**
- * Wrapper which does the write and updates range statistics when such are enabled.
- * @warning RT_SUCCESS(rc=VINF_IOM_R3_MMIO_WRITE) is TRUE!
- */
-static VBOXSTRICTRC iomMMIODoWrite(PVM pVM, PVMCPU pVCpu, PIOMMMIORANGE pRange, RTGCPHYS GCPhysFault,
-                                   const void *pvData, unsigned cb)
-{
-#ifdef VBOX_WITH_STATISTICS
-    int rcSem = IOM_LOCK_SHARED(pVM);
-    if (rcSem == VERR_SEM_BUSY)
-        return VINF_IOM_R3_MMIO_WRITE;
-    PIOMMMIOSTATS pStats = iomMmioGetStats(pVM, pVCpu, GCPhysFault, pRange);
-    if (!pStats)
-# ifdef IN_RING3
-        return VERR_NO_MEMORY;
-# else
-        return VINF_IOM_R3_MMIO_WRITE;
-# endif
-    STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfWrite), a);
-#else
-    NOREF(pVCpu);
-#endif
-
+ * Wrapper which does the write.
+ */
+DECLINLINE(VBOXSTRICTRC) iomMmioDoWrite(PVMCC pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhysFault,
+                                        const void *pvData, uint32_t cb IOM_MMIO_STATS_COMMA_DECL)
+{
     VBOXSTRICTRC rcStrict;
-    if (RT_LIKELY(pRange->CTX_SUFF(pfnWriteCallback)))
+    if (RT_LIKELY(pRegEntry->pfnWriteCallback))
     {
         if (   (cb == 4 && !(GCPhysFault & 3))
-            || (pRange->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_PASSTHRU
-            || (cb == 8 && !(GCPhysFault & 7) && IOMMMIO_DOES_WRITE_MODE_ALLOW_QWORD(pRange->fFlags)) )
-            rcStrict = pRange->CTX_SUFF(pfnWriteCallback)(pRange->CTX_SUFF(pDevIns), pRange->CTX_SUFF(pvUser),
-                                                          GCPhysFault, (void *)pvData, cb); /** @todo fix const!! */
+            || (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_PASSTHRU
+            || (cb == 8 && !(GCPhysFault & 7) && IOMMMIO_DOES_WRITE_MODE_ALLOW_QWORD(pRegEntry->fFlags)) )
+            rcStrict = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                   GCPhysFault, pvData, cb);
         else
-            rcStrict = iomMMIODoComplicatedWrite(pVM, pVCpu, pRange, GCPhysFault, pvData, cb);
+            rcStrict = iomMmioDoComplicatedWrite(pVM, pVCpu, pRegEntry, GCPhysFault, pvData, cb IOM_MMIO_STATS_COMMA_ARG);
     }
     else
         rcStrict = VINF_SUCCESS;
-
-    STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfWrite), a);
-    STAM_COUNTER_INC(&pStats->Accesses);
     return rcStrict;
 }
@@ -354 +359 @@
  *
  * @param   pVM                 The cross context VM structure.
- * @param   pRange              The range to read from.
+ * @param   pRegEntry           The MMIO entry for the current context.
  * @param   GCPhys              The physical address to start reading.
  * @param   pvValue             Where to store the value.
  * @param   cbValue             The size of the value to read.
  */
-static VBOXSTRICTRC iomMMIODoComplicatedRead(PVM pVM, PIOMMMIORANGE pRange, RTGCPHYS GCPhys, void *pvValue, unsigned cbValue)
-{
-    AssertReturn(   (pRange->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD
-                 || (pRange->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD,
+static VBOXSTRICTRC iomMMIODoComplicatedRead(PVM pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys,
+                                             void *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    AssertReturn(   (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD
+                 || (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD,
                  VERR_IOM_MMIO_IPE_1);
     AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
-    RTGCPHYS const GCPhysStart = GCPhys; NOREF(GCPhysStart);
+#ifdef LOG_ENABLED
+    RTGCPHYS const GCPhysStart = GCPhys;
+#endif
 
     /*
      * Do debug stop if requested.
      */
-    int rc = VINF_SUCCESS; NOREF(pVM);
+    VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
 #ifdef VBOX_STRICT
-    if (pRange->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_READ)
+    if (pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_READ)
     {
 # ifdef IN_RING3
         rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
-                            "Complicated read %#x byte at %RGp to %s\n", cbValue, GCPhys, R3STRING(pRange->pszDesc));
+                            "Complicated read %#x byte at %RGp to %s\n", cbValue, GCPhys, R3STRING(pRegEntry->pszDesc));
         if (rc == VERR_DBGF_NOT_ATTACHED)
             rc = VINF_SUCCESS;
@@ -385 +393 @@
 #endif
 
+    STAM_COUNTER_INC(&pStats->ComplicatedReads);
+
     /*
      * Split and conquer.
@@ -394 +404 @@
          */
         uint32_t u32Value;
-        int rc2 = pRange->CTX_SUFF(pfnReadCallback)(pRange->CTX_SUFF(pDevIns), pRange->CTX_SUFF(pvUser),
-                                                    GCPhys & ~(RTGCPHYS)3, &u32Value, sizeof(u32Value));
-        switch (rc2)
+        VBOXSTRICTRC rcStrict2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                            GCPhys & ~(RTGCPHYS)3, &u32Value, sizeof(u32Value));
+        switch (VBOXSTRICTRC_VAL(rcStrict2))
         {
             case VINF_SUCCESS:
                 break;
             case VINF_IOM_MMIO_UNUSED_FF:
+                STAM_COUNTER_INC(&pStats->FFor00Reads);
                 u32Value = UINT32_C(0xffffffff);
                 break;
             case VINF_IOM_MMIO_UNUSED_00:
+                STAM_COUNTER_INC(&pStats->FFor00Reads);
                 u32Value = 0;
                 break;
@@ -412 +424 @@
                  * something?  Since reads can have sideeffects we could be
                  * kind of screwed here... */
-                LogFlow(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc\n", GCPhys, GCPhysStart, cbValue, rc2));
-                return rc2;
+                LogFlow(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
+                         GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
+                return rcStrict2;
             default:
-                if (RT_FAILURE(rc2))
+                if (RT_FAILURE(rcStrict2))
                 {
-                    Log(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc\n", GCPhys, GCPhysStart, cbValue, rc2));
-                    return rc2;
+                    Log(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
+                         GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
+                    return rcStrict2;
                 }
-                AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", rc2), VERR_IPE_UNEXPECTED_INFO_STATUS);
-                if (rc == VINF_SUCCESS || rc2 < rc)
-                    rc = rc2;
+                AssertMsgReturn(rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)),
+                                VERR_IPE_UNEXPECTED_INFO_STATUS);
+                if (rc == VINF_SUCCESS || rcStrict2 < rc)
+                    rc = rcStrict2;
                 break;
         }
@@ -473 +488 @@
  * @param   cbValue             How many bytes to read.
  */
-static int iomMMIODoReadFFs(void *pvValue, size_t cbValue)
+static int iomMMIODoReadFFs(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
 {
     switch (cbValue)
@@ -489 +504 @@
         }
     }
+    STAM_COUNTER_INC(&pStats->FFor00Reads);
     return VINF_SUCCESS;
 }
@@ -500 +516 @@
  * @param   cbValue             How many bytes to read.
  */
-static int iomMMIODoRead00s(void *pvValue, size_t cbValue)
+static int iomMMIODoRead00s(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
 {
     switch (cbValue)
@@ -516 +532 @@
         }
     }
+    STAM_COUNTER_INC(&pStats->FFor00Reads);
     return VINF_SUCCESS;
 }
@@ -521 +538 @@
 
 /**
- * Wrapper which does the read and updates range statistics when such are enabled.
- */
-DECLINLINE(VBOXSTRICTRC) iomMMIODoRead(PVM pVM, PVMCPU pVCpu, PIOMMMIORANGE pRange, RTGCPHYS GCPhys,
-                                       void *pvValue, unsigned cbValue)
-{
-#ifdef VBOX_WITH_STATISTICS
-    int rcSem = IOM_LOCK_SHARED(pVM);
-    if (rcSem == VERR_SEM_BUSY)
-        return VINF_IOM_R3_MMIO_READ;
-    PIOMMMIOSTATS pStats = iomMmioGetStats(pVM, pVCpu, GCPhys, pRange);
-    if (!pStats)
-# ifdef IN_RING3
-        return VERR_NO_MEMORY;
-# else
-        return VINF_IOM_R3_MMIO_READ;
-# endif
-    STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfRead), a);
-#else
-    NOREF(pVCpu);
-#endif
-
+ * Wrapper which does the read.
+ */
+DECLINLINE(VBOXSTRICTRC) iomMmioDoRead(PVMCC pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys,
+                                       void *pvValue, uint32_t cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
     VBOXSTRICTRC rcStrict;
-    if (RT_LIKELY(pRange->CTX_SUFF(pfnReadCallback)))
+    if (RT_LIKELY(pRegEntry->pfnReadCallback))
     {
         if (   (   cbValue == 4
                 && !(GCPhys & 3))
-            || (pRange->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_PASSTHRU
+            || (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_PASSTHRU
             || (    cbValue == 8
                 && !(GCPhys & 7)
-                && (pRange->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD ) )
-            rcStrict = pRange->CTX_SUFF(pfnReadCallback)(pRange->CTX_SUFF(pDevIns), pRange->CTX_SUFF(pvUser), GCPhys,
-                                                         pvValue, cbValue);
+                && (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD ) )
+            rcStrict = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser, GCPhys,
+                                                  pvValue, cbValue);
         else
-            rcStrict = iomMMIODoComplicatedRead(pVM, pRange, GCPhys, pvValue, cbValue);
+            rcStrict = iomMMIODoComplicatedRead(pVM, pRegEntry, GCPhys, pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG);
     }
     else
@@ -562 +563 @@
         switch (VBOXSTRICTRC_VAL(rcStrict))
         {
-            case VINF_IOM_MMIO_UNUSED_FF: rcStrict = iomMMIODoReadFFs(pvValue, cbValue); break;
-            case VINF_IOM_MMIO_UNUSED_00: rcStrict = iomMMIODoRead00s(pvValue, cbValue); break;
-        }
-    }
-
-    STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfRead), a);
-    STAM_COUNTER_INC(&pStats->Accesses);
+            case VINF_IOM_MMIO_UNUSED_FF: rcStrict = iomMMIODoReadFFs(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
+            case VINF_IOM_MMIO_UNUSED_00: rcStrict = iomMMIODoRead00s(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
+        }
+    }
     return rcStrict;
 }
+
+#ifndef IN_RING3
+/**
+ * Checks if we can handle an MMIO \#PF in R0/RC.
+ */
+DECLINLINE(bool) iomMmioCanHandlePfInRZ(PVMCC pVM, uint32_t uErrorCode, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
+{
+    if (pRegEntry->cbRegion > 0)
+    {
+        if (   pRegEntry->pfnWriteCallback
+            && pRegEntry->pfnReadCallback)
+            return true;
+
+        PIOMMMIOENTRYR3 const pRegEntryR3 = &pVM->iomr0.s.paMmioRing3Regs[pRegEntry->idxSelf];
+        if (  uErrorCode == UINT32_MAX
+            ? pRegEntryR3->pfnWriteCallback || pRegEntryR3->pfnReadCallback
+            : uErrorCode & X86_TRAP_PF_RW
+              ? !pRegEntry->pfnWriteCallback && pRegEntryR3->pfnWriteCallback
+              : !pRegEntry->pfnReadCallback  && pRegEntryR3->pfnReadCallback)
+            return false;
+
+        return true;
+    }
+    return false;
+}
+#endif
 
 /**
@@ -582 +606 @@
  * @param   pCtxCore    Trap register frame.
  * @param   GCPhysFault The GC physical address corresponding to pvFault.
- * @param   pvUser      Pointer to the MMIO ring-3 range entry.
- */
-static VBOXSTRICTRC iomMmioCommonPfHandler(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, PCPUMCTXCORE pCtxCore,
-                                           RTGCPHYS GCPhysFault, void *pvUser)
-{
-    RT_NOREF_PV(uErrorCode);
-    int rc = IOM_LOCK_SHARED(pVM);
-#ifndef IN_RING3
-    if (rc == VERR_SEM_BUSY)
-        return VINF_IOM_R3_MMIO_READ_WRITE;
-#endif
-    AssertRC(rc);
-
+ * @param   pRegEntry   The MMIO entry for the current context.
+ */
+DECLINLINE(VBOXSTRICTRC) iomMmioCommonPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode,
+                                                   RTGCPHYS GCPhysFault, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
+{
     STAM_PROFILE_START(&pVM->iom.s.StatRZMMIOHandler, a);
-    Log(("iomMmioCommonPfHandler: GCPhys=%RGp uErr=%#x rip=%RGv\n", GCPhysFault, uErrorCode, (RTGCPTR)pCtxCore->rip));
-
-    PIOMMMIORANGE pRange = (PIOMMMIORANGE)pvUser;
-    Assert(pRange);
-    Assert(pRange == iomMmioGetRange(pVM, pVCpu, GCPhysFault));
-    iomMmioRetainRange(pRange);
-#ifndef VBOX_WITH_STATISTICS
-    IOM_UNLOCK_SHARED(pVM);
-
-#else
-    /*
-     * Locate the statistics.
-     */
-    PIOMMMIOSTATS pStats = iomMmioGetStats(pVM, pVCpu, GCPhysFault, pRange);
-    if (!pStats)
-    {
-        iomMmioReleaseRange(pVM, pRange);
-# ifdef IN_RING3
-        return VERR_NO_MEMORY;
-# else
-        STAM_PROFILE_STOP(&pVM->iom.s.StatRZMMIOHandler, a);
-        STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOFailures);
-        return VINF_IOM_R3_MMIO_READ_WRITE;
-# endif
-    }
-#endif
+    Log(("iomMmioCommonPfHandler: GCPhysFault=%RGp uErr=%#x rip=%RGv\n", GCPhysFault, uErrorCode, CPUMGetGuestRIP(pVCpu) ));
+    RT_NOREF(GCPhysFault, uErrorCode);
+
+    VBOXSTRICTRC rcStrict;
 
 #ifndef IN_RING3
@@ -628 +622 @@
      * do the simple test first and the try deal with uErrorCode being N/A.
      */
-    if (RT_UNLIKELY(   (   !pRange->CTX_SUFF(pfnWriteCallback)
-                        || !pRange->CTX_SUFF(pfnReadCallback))
-                    && (  uErrorCode == UINT32_MAX
-                        ? pRange->pfnWriteCallbackR3 || pRange->pfnReadCallbackR3
-                        : uErrorCode & X86_TRAP_PF_RW
-                          ? !pRange->CTX_SUFF(pfnWriteCallback) && pRange->pfnWriteCallbackR3
-                          : !pRange->CTX_SUFF(pfnReadCallback)  && pRange->pfnReadCallbackR3
-                        )
-                   )
-       )
-    {
+    PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
+    if (RT_LIKELY(   pDevIns
+                  && iomMmioCanHandlePfInRZ(pVM, uErrorCode, pRegEntry)))
+    {
+        /*
+         * Enter the device critsect prior to engaging IOM in case of lock contention.
+         * Note! Perhaps not a good move?
+         */
+        rcStrict = PDMCritSectEnter(pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_MMIO_READ_WRITE);
+        if (rcStrict == VINF_SUCCESS)
+        {
+#endif /* !IN_RING3 */
+
+            /*
+             * Let IEM call us back via iomMmioHandler.
+             */
+            rcStrict = IEMExecOne(pVCpu);
+
+#ifndef IN_RING3
+            PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
+#endif
+            if (RT_SUCCESS(rcStrict))
+            { /* likely */ }
+            else if (   rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
+                     || rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
+            {
+                Log(("IOM: Hit unsupported IEM feature!\n"));
+                rcStrict = VINF_EM_RAW_EMULATE_INSTR;
+            }
+#ifndef IN_RING3
+            STAM_PROFILE_STOP(&pVM->iom.s.StatRZMMIOHandler, a);
+            return rcStrict;
+        }
+        else
+            STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIODevLockContention);
+    }
+    else
+        rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
+
+# ifdef VBOX_WITH_STATISTICS
+    if (rcStrict == VINF_IOM_R3_MMIO_READ_WRITE)
+    {
+        PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry);
         if (uErrorCode & X86_TRAP_PF_RW)
-            STAM_COUNTER_INC(&pStats->CTX_MID_Z(Write,ToR3));
+        {
+            STAM_COUNTER_INC(&pStats->WriteRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOWritesToR3);
+        }
         else
-            STAM_COUNTER_INC(&pStats->CTX_MID_Z(Read,ToR3));
-
-        STAM_PROFILE_STOP(&pVM->iom.s.StatRZMMIOHandler, a);
-        STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOFailures);
-        iomMmioReleaseRange(pVM, pRange);
-        return VINF_IOM_R3_MMIO_READ_WRITE;
-    }
-#endif /* !IN_RING3 */
-
-    /*
-     * Retain the range and do locking.
-     */
-    PPDMDEVINS pDevIns = pRange->CTX_SUFF(pDevIns);
-    rc = PDMCritSectEnter(pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_MMIO_READ_WRITE);
-    if (rc != VINF_SUCCESS)
-    {
-        iomMmioReleaseRange(pVM, pRange);
-        return rc;
-    }
-
-    /*
-     * Let IEM call us back via iomMmioHandler.
-     */
-    VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
-
-    NOREF(pCtxCore); NOREF(GCPhysFault);
+        {
+            STAM_COUNTER_INC(&pStats->ReadRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOReadsToR3);
+        }
+    }
+# endif
+#else  /* IN_RING3 */
+    RT_NOREF(pRegEntry);
+#endif /* IN_RING3 */
+
     STAM_PROFILE_STOP(&pVM->iom.s.StatRZMMIOHandler, a);
-    PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
-    iomMmioReleaseRange(pVM, pRange);
-    if (RT_SUCCESS(rcStrict))
-        return rcStrict;
-    if (   rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
-        || rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
-    {
-        Log(("IOM: Hit unsupported IEM feature!\n"));
-        rcStrict = VINF_EM_RAW_EMULATE_INSTR;
-    }
     return rcStrict;
 }
@@ -687 +690 @@
  *      \#PF access handler callback for MMIO pages.}
  *
- * @remarks The @a pvUser argument points to the IOMMMIORANGE.
- */
-DECLEXPORT(VBOXSTRICTRC) iomMmioPfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pCtxCore, RTGCPTR pvFault,
-                                          RTGCPHYS GCPhysFault, void *pvUser)
+ * @remarks The @a pvUser argument is the MMIO handle.
+ */
+DECLEXPORT(VBOXSTRICTRC) iomMmioPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pCtxCore,
+                                             RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
 {
     LogFlow(("iomMmioPfHandler: GCPhys=%RGp uErr=%#x pvFault=%RGv rip=%RGv\n",
-             GCPhysFault, (uint32_t)uErrorCode, pvFault, (RTGCPTR)pCtxCore->rip)); NOREF(pvFault);
-    return iomMmioCommonPfHandler(pVM, pVCpu, (uint32_t)uErrorCode, pCtxCore, GCPhysFault, pvUser);
-}
-
-
+             GCPhysFault, (uint32_t)uErrorCode, pvFault, (RTGCPTR)pCtxCore->rip));
+    RT_NOREF(pvFault, pCtxCore);
+
+    /* Translate the MMIO handle to a registration entry for the current context. */
+    AssertReturn((uintptr_t)pvUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+#ifdef IN_RING0
+    AssertReturn((uintptr_t)pvUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iomr0.s.paMmioRegs[(uintptr_t)pvUser];
+#else
+    CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iom.s.paMmioRegs[(uintptr_t)pvUser];
+#endif
+
+    return iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
+}
+
+
+#ifdef IN_RING0
 /**
  * Physical access handler for MMIO ranges.
+ *
+ * This is actually only used by VT-x for APIC page accesses.
  *
  * @returns VBox status code (appropriate for GC return).
@@ -705 +722 @@
  * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
  * @param   uErrorCode  CPU Error code.
- * @param   pCtxCore    Trap register frame.
  * @param   GCPhysFault The GC physical address.
  */
-VMMDECL(VBOXSTRICTRC) IOMMMIOPhysHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pCtxCore, RTGCPHYS GCPhysFault)
+VMM_INT_DECL(VBOXSTRICTRC) IOMMmioPhysHandler(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, RTGCPHYS GCPhysFault)
 {
     /*
      * We don't have a range here, so look it up before calling the common function.
      */
-    int rc2 = IOM_LOCK_SHARED(pVM); NOREF(rc2);
+    VBOXSTRICTRC rcStrict = IOM_LOCK_SHARED(pVM);
+    if (RT_SUCCESS(rcStrict))
+    {
+        uint16_t idxLastHint = UINT16_MAX; /** @todo dedicate hint for this bugger, since it will only be the APIC. */
+        RTGCPHYS offRegion;
+        CTX_SUFF(PIOMMMIOENTRY) pRegEntry = iomMmioGetEntry(pVM, GCPhysFault, &offRegion, &idxLastHint);
+        if (RT_LIKELY(pRegEntry))
+        {
+            IOM_UNLOCK_SHARED(pVM);
+            rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
+        }
+        else
+        {
+            /*
+             * Old style registrations.
+             */
+            PIOMMMIORANGE pRange = iomMmioGetRange(pVM, pVCpu, GCPhysFault);
+            if (pRange)
+            {
+                iomMmioRetainRange(pRange);
+                IOM_UNLOCK_SHARED(pVM);
+
+                rcStrict = iomMmioCommonPfHandlerOld(pVM, pVCpu, (uint32_t)uErrorCode,
+                                                     CPUMCTX2CORE(&pVCpu->cpum.GstCtx), GCPhysFault, pRange);
+
+                iomMmioReleaseRange(pVM, pRange);
+            }
+            else
+            {
+                IOM_UNLOCK_SHARED(pVM);
+                rcStrict = VERR_IOM_MMIO_RANGE_NOT_FOUND;
+            }
+        }
+    }
+    else if (rcStrict == VERR_SEM_BUSY)
+        rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
+    return rcStrict;
+}
+#endif /* IN_RING0 */
+
+
+/**
+ * @callback_method_impl{FNPGMPHYSHANDLER, MMIO page accesses}
+ *
+ * @remarks The @a pvUser argument is the MMIO handle.
+ */
+PGM_ALL_CB2_DECL(VBOXSTRICTRC) iomMmioHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, void *pvPhys, void *pvBuf,
+                                                 size_t cbBuf, PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
+{
+    STAM_PROFILE_START(UnusedMacroArg, Prf);
+    STAM_COUNTER_INC(&pVM->iom.s.CTX_SUFF(StatMmioHandler));
+
+    Assert(enmAccessType == PGMACCESSTYPE_READ || enmAccessType == PGMACCESSTYPE_WRITE);
+    AssertMsg(cbBuf >= 1, ("%zu\n", cbBuf));
+    NOREF(pvPhys); NOREF(enmOrigin);
+
+#ifdef IN_RING3
+    int const rcToRing3 = VERR_IOM_MMIO_IPE_3;
+#else
+    int const rcToRing3 = enmAccessType == PGMACCESSTYPE_READ ? VINF_IOM_R3_MMIO_READ : VINF_IOM_R3_MMIO_WRITE;
+#endif
+
+    /*
+     * Translate pvUser to an MMIO registration table entry.  We can do this
+     * without any locking as the data is static after VM creation.
+     */
+    AssertReturn((uintptr_t)pvUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+#ifdef IN_RING0
+    AssertReturn((uintptr_t)pvUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    CTX_SUFF(PIOMMMIOENTRY) const pRegEntry = &pVM->iomr0.s.paMmioRegs[(uintptr_t)pvUser];
+#else
+    CTX_SUFF(PIOMMMIOENTRY) const pRegEntry = &pVM->iom.s.paMmioRegs[(uintptr_t)pvUser];
+#endif
+#ifdef VBOX_WITH_STATISTICS
+    PIOMMMIOSTATSENTRY const      pStats    = iomMmioGetStats(pVM, pRegEntry);  /* (Works even without ring-0 setup.) */
+#endif
+    PPDMDEVINS const              pDevIns   = pRegEntry->pDevIns;
+
 #ifndef IN_RING3
-    if (rc2 == VERR_SEM_BUSY)
-        return VINF_IOM_R3_MMIO_READ_WRITE;
-#endif
-    PIOMMMIORANGE pRange = iomMmioGetRange(pVM, pVCpu, GCPhysFault);
-    if (RT_UNLIKELY(!pRange))
-    {
-        IOM_UNLOCK_SHARED(pVM);
-        return VERR_IOM_MMIO_RANGE_NOT_FOUND;
-    }
-    iomMmioRetainRange(pRange);
+# if defined(VBOX_STRICT)
+    /*
+     * Assert the right entry in strict builds.  This may yield a false positive
+     * for SMP VMs if we're unlucky and the guest isn't well behaved.
+     */
+    IOM_LOCK_SHARED(pVM);  /** @todo Need lookup that doesn't require locking... */
+    RTGCPHYS offIgn;
+    uint16_t idxIgn = UINT16_MAX;
+    Assert(   pRegEntry == iomMmioGetEntry(pVM, GCPhysFault, &offIgn, &idxIgn)
+           || !pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser].fMapped);
     IOM_UNLOCK_SHARED(pVM);
-
-    VBOXSTRICTRC rcStrict = iomMmioCommonPfHandler(pVM, pVCpu, (uint32_t)uErrorCode, pCtxCore, GCPhysFault, pRange);
-
-    iomMmioReleaseRange(pVM, pRange);
-    return VBOXSTRICTRC_VAL(rcStrict);
-}
-
-
-/**
- * @callback_method_impl{FNPGMPHYSHANDLER, MMIO page accesses}
- *
- * @remarks The @a pvUser argument points to the MMIO range entry.
- */
-PGM_ALL_CB2_DECL(VBOXSTRICTRC) iomMmioHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, void *pvPhys, void *pvBuf,
-                                              size_t cbBuf, PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
-{
-    PIOMMMIORANGE pRange = (PIOMMMIORANGE)pvUser;
-    STAM_COUNTER_INC(&pVM->iom.s.StatR3MMIOHandler);
-
-    NOREF(pvPhys); NOREF(enmOrigin);
-    AssertPtr(pRange);
-    AssertMsg(cbBuf >= 1, ("%zu\n", cbBuf));
-
-
-#ifndef IN_RING3
+# endif
+
     /*
      * If someone is doing FXSAVE, FXRSTOR, XSAVE, XRSTOR or other stuff dealing with
      * large amounts of data, just go to ring-3 where we don't need to deal with partial
      * successes.  No chance any of these will be problematic read-modify-write stuff.
-     */
-    if (cbBuf > sizeof(pVCpu->iom.s.PendingMmioWrite.abValue))
-        return enmAccessType == PGMACCESSTYPE_WRITE ? VINF_IOM_R3_MMIO_WRITE : VINF_IOM_R3_MMIO_READ;
-#endif
-
-    /*
-     * Validate the range.
-     */
-    int rc = IOM_LOCK_SHARED(pVM);
+     *
+     * Also drop back if the ring-0 registration entry isn't actually used.
+     */
+    if (   cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue)
+        && pRegEntry->cbRegion != 0
+        && (  enmAccessType == PGMACCESSTYPE_READ
+            ? pRegEntry->pfnReadCallback  != NULL || pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser].pfnReadCallback == NULL
+            : pRegEntry->pfnWriteCallback != NULL || pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser].pfnWriteCallback == NULL)
+        && pDevIns )
+    { /* likely */ }
+    else
+    {
+        STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3);
+        STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatRZMMIOReadsToR3 : &pVM->iom.s.StatRZMMIOWritesToR3);
+        return rcToRing3;
+    }
+#endif /* !IN_RING3 */
+
+
+    /*
+     * Perform locking and the access.
+     *
+     * Writes requiring a return to ring-3 are buffered by IOM so IEM can
+     * commit the instruction.
+     *
+     * Note! We may end up locking the device even when the relevant callback is
+     *       NULL.  This is supposed to be an unlikely case, so not optimized yet.
+     */
+    VBOXSTRICTRC rcStrict = PDMCritSectEnter(pDevIns->CTX_SUFF(pCritSectRo), rcToRing3);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        if (enmAccessType == PGMACCESSTYPE_READ)
+        {
+            rcStrict = iomMmioDoRead(pVM, pRegEntry, GCPhysFault, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
+
+            PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
 #ifndef IN_RING3
-    if (rc == VERR_SEM_BUSY)
-    {
-        if (enmAccessType == PGMACCESSTYPE_READ)
-            return VINF_IOM_R3_MMIO_READ;
-        Assert(enmAccessType == PGMACCESSTYPE_WRITE);
-        return iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, NULL /*pRange*/);
-    }
-#endif
-    AssertRC(rc);
-    Assert(pRange == iomMmioGetRange(pVM, pVCpu, GCPhysFault));
-
-    /*
-     * Perform locking.
-     */
-    iomMmioRetainRange(pRange);
-    PPDMDEVINS pDevIns = pRange->CTX_SUFF(pDevIns);
-    IOM_UNLOCK_SHARED(pVM);
-#ifdef IN_RING3
-    VBOXSTRICTRC rcStrict = PDMCritSectEnter(pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_MMIO_READ_WRITE);
-#else
-    VBOXSTRICTRC rcStrict = pDevIns ? PDMCritSectEnter(pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_MMIO_READ_WRITE)
-                          : VINF_IOM_R3_MMIO_READ_WRITE;
-#endif
-    if (rcStrict == VINF_SUCCESS)
-    {
-        /*
-         * Perform the access.
-         */
-        if (enmAccessType == PGMACCESSTYPE_READ)
-            rcStrict = iomMMIODoRead(pVM, pVCpu, pRange, GCPhysFault, pvBuf, (unsigned)cbBuf);
+            if (rcStrict == VINF_IOM_R3_MMIO_READ)
+            {
+                STAM_COUNTER_INC(&pStats->ReadRZToR3);
+                STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOReadsToR3);
+            }
+            else
+#endif
+                STAM_COUNTER_INC(&pStats->Reads);
+            STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfRead), Prf);
+        }
         else
         {
-            rcStrict = iomMMIODoWrite(pVM, pVCpu, pRange, GCPhysFault, pvBuf, (unsigned)cbBuf);
+            rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, GCPhysFault, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
+            PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
 #ifndef IN_RING3
             if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
-                rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRange);
-#endif
-        }
-
-        /* Check the return code. */
+                rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
+            if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
+            {
+                STAM_COUNTER_INC(&pStats->WriteRZToR3);
+                STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOWritesToR3);
+            }
+            else if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
+            {
+                STAM_COUNTER_INC(&pStats->CommitRZToR3);
+                STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOCommitsToR3);
+            }
+            else
+#endif
+                STAM_COUNTER_INC(&pStats->Writes);
+            STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfWrite), Prf);
+        }
+
+        /*
+         * Check the return code.
+         */
 #ifdef IN_RING3
         AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc -  Access type %d - %RGp - %s\n",
-                                             VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pRange->pszDesc));
+                                             VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pRegEntry->pszDesc));
 #else
         AssertMsg(   rcStrict == VINF_SUCCESS
-                  || rcStrict == (enmAccessType == PGMACCESSTYPE_READ ? VINF_IOM_R3_MMIO_READ :  VINF_IOM_R3_MMIO_WRITE)
+                  || rcStrict == rcToRing3
                   || (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE && enmAccessType == PGMACCESSTYPE_WRITE)
-                  || rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
                   || rcStrict == VINF_EM_DBG_STOP
                   || rcStrict == VINF_EM_DBG_EVENT
@@ -821 +913 @@
                   //|| rcStrict == VINF_EM_HALT       /* ?? */
                   //|| rcStrict == VINF_EM_NO_MEMORY  /* ?? */
-                  , ("%Rrc - Access type %d - %RGp - %p\n", VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pDevIns));
-#endif
-
-        iomMmioReleaseRange(pVM, pRange);
-        PDMCritSectLeave(pDevIns->CTX_SUFF(pCritSectRo));
-    }
-#ifdef IN_RING3
+                  , ("%Rrc - Access type %d - %RGp - %s #%u\n",
+                     VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pDevIns->pReg->szName, pDevIns->iInstance));
+#endif
+    }
+    /*
+     * Deal with enter-critsect failures.
+     */
+#ifndef IN_RING3
+    else if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
+    {
+        Assert(enmAccessType == PGMACCESSTYPE_WRITE);
+        rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
+        if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
+        {
+            STAM_COUNTER_INC(&pStats->CommitRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOCommitsToR3);
+        }
+        else
+        {
+            STAM_COUNTER_INC(&pStats->WriteRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIOWritesToR3);
+        }
+        STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIODevLockContention);
+    }
+    else if (rcStrict == VINF_IOM_R3_MMIO_READ)
+    {
+        Assert(enmAccessType == PGMACCESSTYPE_READ);
+        STAM_COUNTER_INC(&pStats->ReadRZToR3);
+        STAM_COUNTER_INC(&pVM->iom.s.StatRZMMIODevLockContention);
+    }
+#endif
     else
-        iomMmioReleaseRange(pVM, pRange);
-#else
-    else
-    {
-        if (rcStrict == VINF_IOM_R3_MMIO_READ_WRITE)
-        {
-            if (enmAccessType == PGMACCESSTYPE_READ)
-                rcStrict = VINF_IOM_R3_MMIO_READ;
-            else
-            {
-                Assert(enmAccessType == PGMACCESSTYPE_WRITE);
-                rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRange);
-            }
-        }
-        iomMmioReleaseRange(pVM, pRange);
-    }
-#endif
+        AssertMsg(RT_FAILURE_NP(rcStrict), ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
     return rcStrict;
 }
@@ -853 +953 @@
  * Mapping an MMIO2 page in place of an MMIO page for direct access.
  *
- * (This is a special optimization used by the VGA device.)
+ * This is a special optimization used by the VGA device.  Call
+ * IOMMmioResetRegion() to undo the mapping.
  *
  * @returns VBox status code.  This API may return VINF_SUCCESS even if no
@@ -859 +960 @@
  *
  * @param   pVM             The cross context VM structure.
- * @param   GCPhys          The address of the MMIO page to be changed.
- * @param   GCPhysRemapped  The address of the MMIO2 page.
+ * @param   pDevIns         The device instance @a hRegion and @a hMmio2 are
+ *                          associated with.
+ * @param   hRegion         The handle to the MMIO region.
+ * @param   offRegion       The offset into @a hRegion of the page to be
+ *                          remapped.
+ * @param   hMmio2          The MMIO2 handle.
+ * @param   offMmio2        Offset into @a hMmio2 of the page to be use for the
+ *                          mapping.
  * @param   fPageFlags      Page flags to set. Must be (X86_PTE_RW | X86_PTE_P)
  *                          for the time being.
  */
-VMMDECL(int) IOMMMIOMapMMIO2Page(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysRemapped, uint64_t fPageFlags)
+VMM_INT_DECL(int) IOMMmioMapMmio2Page(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, RTGCPHYS offRegion,
+                                      uint64_t hMmio2, RTGCPHYS offMmio2, uint64_t fPageFlags)
 {
     /* Currently only called from the VGA device during MMIO. */
-    Log(("IOMMMIOMapMMIO2Page %RGp -> %RGp flags=%RX64\n", GCPhys, GCPhysRemapped, fPageFlags));
+    Log(("IOMMmioMapMmio2Page %#RX64/%RGp -> %#RX64/%RGp flags=%RX64\n", hRegion, offRegion, hMmio2, offMmio2, fPageFlags));
     AssertReturn(fPageFlags == (X86_PTE_RW | X86_PTE_P), VERR_INVALID_PARAMETER);
+    AssertReturn(pDevIns, VERR_INVALID_POINTER);
+
+/** @todo Why is this restricted to protected mode???  Try it in all modes! */
     PVMCPUCC pVCpu = VMMGetCpu(pVM);
 
@@ -876 +987 @@
         ||  (   CPUMIsGuestInPagedProtectedMode(pVCpu)
              && !HMIsNestedPagingActive(pVM)))
-        return VINF_SUCCESS;    /* ignore */
-
-    int rc = IOM_LOCK_SHARED(pVM);
-    if (RT_FAILURE(rc))
-        return VINF_SUCCESS; /* better luck the next time around */
-
-    /*
-     * Lookup the context range node the page belongs to.
-     */
-    PIOMMMIORANGE pRange = iomMmioGetRange(pVM, pVCpu, GCPhys);
-    AssertMsgReturn(pRange,
-                    ("Handlers and page tables are out of sync or something! GCPhys=%RGp\n", GCPhys), VERR_IOM_MMIO_RANGE_NOT_FOUND);
-
-    Assert((pRange->GCPhys       & PAGE_OFFSET_MASK) == 0);
-    Assert((pRange->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
+        return VINF_SUCCESS;    /* ignore */ /** @todo return some indicator if we fail here */
+
+    /*
+     * Translate the handle into an entry and check the region offset.
+     */
+    AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+#ifdef IN_RING0
+    AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(offRegion < pVM->iomr0.s.paMmioRegs[hRegion].cbRegion, VERR_OUT_OF_RANGE);
+    AssertReturn(   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
+                 || (   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
+                     && pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
+#else
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
+#endif
+    AssertReturn(offRegion < pRegEntry->cbRegion, VERR_OUT_OF_RANGE);
+    Assert((pRegEntry->cbRegion & PAGE_OFFSET_MASK) == 0);
+
+    /*
+     * When getting and using the mapping address, we must sit on the IOM lock
+     * to prevent remapping.  Shared suffices as we change nothing.
+     */
+    IOM_LOCK_SHARED(pVM);
+
+    RTGCPHYS const GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
+    AssertReturnStmt(GCPhys != NIL_RTGCPHYS, IOM_UNLOCK_SHARED(pVM), VERR_IOM_MMIO_REGION_NOT_MAPPED);
+    Assert(!(GCPhys & PAGE_OFFSET_MASK));
 
     /*
      * Do the aliasing; page align the addresses since PGM is picky.
      */
-    GCPhys         &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
-    GCPhysRemapped &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
-
-    rc = PGMHandlerPhysicalPageAlias(pVM, pRange->GCPhys, GCPhys, GCPhysRemapped);
+#if 0 /** @todo fix when DevVGA is converted to new model.  */
+    int rc = PGMHandlerPhysicalPageAlias(pVM, GCPhys, GCPhys + (offRange & ~(RTGCPHYS)PAGE_OFFSET_MASK),
+                                         pDevIns, hMmio2, offMmio2);
+#else
+    AssertFailed();
+    int rc = VERR_NOT_IMPLEMENTED;
+    RT_NOREF(offMmio2, hMmio2);
+#endif
 
     IOM_UNLOCK_SHARED(pVM);
+
     AssertRCReturn(rc, rc);
 
+/** @todo either ditch this or replace it with something that works in the
+ *        nested case, since we really only care about nested paging! */
+#if 0
     /*
      * Modify the shadow page table. Since it's an MMIO page it won't be present and we
@@ -919 +1054 @@
     rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys);
     Assert(rc == VINF_SUCCESS || rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT);
+#endif
     return VINF_SUCCESS;
 }
 
 
+#ifdef IN_RING0 /* VT-x ring-0 only, move to IOMR0Mmio.cpp later.  */
 /**
  * Mapping a HC page in place of an MMIO page for direct access.
  *
- * (This is a special optimization used by the APIC in the VT-x case.)
+ * This is a special optimization used by the APIC in the VT-x case.  This VT-x
+ * code uses PGMHandlerPhysicalReset rather than IOMMmioResetRegion() to undo
+ * the effects here.
+ *
+ * @todo Make VT-x usage more consistent.
  *
  * @returns VBox status code.
@@ -937 +1078 @@
  *                          for the time being.
  */
-VMMDECL(int) IOMMMIOMapMMIOHCPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint64_t fPageFlags)
+VMM_INT_DECL(int) IOMR0MmioMapMmioHCPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint64_t fPageFlags)
 {
     /* Currently only called from VT-x code during a page fault. */
-    Log(("IOMMMIOMapMMIOHCPage %RGp -> %RGp flags=%RX64\n", GCPhys, HCPhys, fPageFlags));
+    Log(("IOMR0MmioMapMmioHCPage %RGp -> %RGp flags=%RX64\n", GCPhys, HCPhys, fPageFlags));
 
     AssertReturn(fPageFlags == (X86_PTE_RW | X86_PTE_P), VERR_INVALID_PARAMETER);
-    /** @todo NEM: MMIO page aliasing. */
+    /** @todo NEM: MMIO page aliasing?? */
     Assert(HMIsEnabled(pVM));
 
-    /*
-     * Lookup the context range node the page belongs to.
-     */
 # ifdef VBOX_STRICT
-    /* Can't lock IOM here due to potential deadlocks in the VGA device; not safe to access. */
-    PIOMMMIORANGE pRange = iomMMIOGetRangeUnsafe(pVM, pVCpu, GCPhys);
-    AssertMsgReturn(pRange,
-            ("Handlers and page tables are out of sync or something! GCPhys=%RGp\n", GCPhys), VERR_IOM_MMIO_RANGE_NOT_FOUND);
-    Assert((pRange->GCPhys       & PAGE_OFFSET_MASK) == 0);
-    Assert((pRange->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
+    /*
+     * Check input address (it's HM calling, not the device, so no region handle).
+     */
+    IOM_LOCK_SHARED(pVM);
+    RTGCPHYS offIgn;
+    uint16_t idxIgn = UINT16_MAX;
+    PIOMMMIOENTRYR0 pRegEntry = iomMmioGetEntry(pVM, GCPhys, &offIgn, &idxIgn);
+    IOM_UNLOCK_SHARED(pVM);
+    Assert(pRegEntry);
+    Assert(pRegEntry && !(pRegEntry->cbRegion & PAGE_OFFSET_MASK));
 # endif
 
@@ -966 +1108 @@
     int rc = PGMHandlerPhysicalPageAliasHC(pVM, GCPhys, GCPhys, HCPhys);
     AssertRCReturn(rc, rc);
+
+/** @todo either ditch this or replace it with something that works in the
+ *        nested case, since we really only care about nested paging! */
 
     /*
@@ -977 +1122 @@
     return VINF_SUCCESS;
 }
+#endif
 
 
@@ -982 +1128 @@
  * Reset a previously modified MMIO region; restore the access flags.
  *
+ * This undoes the effects of IOMMmioMapMmio2Page() and is currently only
+ * intended for some ancient VGA hack.  However, it would be great to extend it
+ * beyond VT-x and/or nested-paging.
+ *
  * @returns VBox status code.
  *
  * @param   pVM             The cross context VM structure.
- * @param   GCPhys          Physical address that's part of the MMIO region to be reset.
- */
-VMMDECL(int) IOMMMIOResetRegion(PVMCC pVM, RTGCPHYS GCPhys)
-{
-    Log(("IOMMMIOResetRegion %RGp\n", GCPhys));
-
+ * @param   pDevIns         The device instance @a hRegion is associated with.
+ * @param   hRegion         The handle to the MMIO region.
+ */
+VMM_INT_DECL(int) IOMMmioResetRegion(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion)
+{
+    Log(("IOMMMIOResetRegion %#RX64\n", hRegion));
+    AssertReturn(pDevIns, VERR_INVALID_POINTER);
+
+/** @todo Get rid of this this real/protected or nested paging restriction,
+ *        it probably shouldn't be here and would be nasty when the CPU
+ *        changes mode while we have the hack enabled... */
     PVMCPUCC pVCpu = VMMGetCpu(pVM);
 
     /* This currently only works in real mode, protected mode without paging or with nested paging. */
     /** @todo NEM: MMIO page aliasing. */
-    if (    !HMIsEnabled(pVM)       /* useless without VT-x/AMD-V */
-        ||  (   CPUMIsGuestInPagedProtectedMode(pVCpu)
-             && !HMIsNestedPagingActive(pVM)))
+    if (   !HMIsEnabled(pVM)       /* useless without VT-x/AMD-V */
+        || (   CPUMIsGuestInPagedProtectedMode(pVCpu)
+            && !HMIsNestedPagingActive(pVM)))
         return VINF_SUCCESS;    /* ignore */
 
     /*
-     * Lookup the context range node the page belongs to.
-     */
-# ifdef VBOX_STRICT
-    /* Can't lock IOM here due to potential deadlocks in the VGA device; not safe to access. */
-    PIOMMMIORANGE pRange = iomMMIOGetRangeUnsafe(pVM, pVCpu, GCPhys);
-    AssertMsgReturn(pRange,
-            ("Handlers and page tables are out of sync or something! GCPhys=%RGp\n", GCPhys), VERR_IOM_MMIO_RANGE_NOT_FOUND);
-    Assert((pRange->GCPhys       & PAGE_OFFSET_MASK) == 0);
-    Assert((pRange->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
-# endif
+     * Translate the handle into an entry and mapping address for PGM.
+     * We have to take the lock to safely access the mapping address here.
+     */
+    AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+#ifdef IN_RING0
+    AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
+                 || (   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
+                     && pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
+#else
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
+#endif
+    Assert((pRegEntry->cbRegion & PAGE_OFFSET_MASK) == 0);
+
+    IOM_LOCK_SHARED(pVM);
+    RTGCPHYS GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
+    IOM_UNLOCK_SHARED(pVM);
+
+    Assert(!(GCPhys              & PAGE_OFFSET_MASK));
+    Assert(!(pRegEntry->cbRegion & PAGE_OFFSET_MASK));
 
     /*
      * Call PGM to do the job work.
      *
-     * After the call, all the pages should be non-present... unless there is
+     * After the call, all the pages should be non-present, unless there is
      * a page pool flush pending (unlikely).
      */
@@ -1024 +1193 @@
     if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
     {
-        uint32_t cb = pRange->cb;
-        GCPhys = pRange->GCPhys;
+        RTGCPHYS cb = pRegEntry->cbRegion;
         while (cb)
         {

trunk/src/VBox/VMM/VMMR0/IOMR0IoPort.cpp

@@ -271 +271 @@
     AssertReturn(pGVM->iom.s.cIoPortStatsAllocation == cOldEntries, VERR_IOM_IOPORT_IPE_1);
     AssertReturn(pGVM->iom.s.cIoPortStats <= cOldEntries, VERR_IOM_IOPORT_IPE_2);
+    AssertReturn(!pGVM->iomr0.s.fIoPortStatsFrozen, VERR_WRONG_ORDER);
 
     /*
@@ -329 +330 @@
 }
 
+/**
+ * Called after all devices has been instantiated to copy over the statistics
+ * indices to the ring-0 I/O port registration table.
+ *
+ * This simplifies keeping statistics for I/O port ranges that are ring-3 only.
+ *
+ * After this call, IOMR0IoPortGrowStatisticsTable() will stop working.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @thread  EMT(0)
+ * @note    Only callable at VM creation time.
+ */
+VMMR0_INT_DECL(int) IOMR0IoPortSyncStatisticsIndices(PGVM pGVM)
+{
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * First, freeze the statistics array:
+     */
+    pGVM->iomr0.s.fIoPortStatsFrozen = true;
+
+    /*
+     * Second, synchronize the indices:
+     */
+    uint32_t const          cRegs        = RT_MIN(pGVM->iom.s.cIoPortRegs, pGVM->iomr0.s.cIoPortAlloc);
+    uint32_t const          cStatsAlloc  = pGVM->iomr0.s.cIoPortStatsAllocation;
+    PIOMIOPORTENTRYR0       paIoPortRegs   = pGVM->iomr0.s.paIoPortRegs;
+    IOMIOPORTENTRYR3 const *paIoPortRegsR3 = pGVM->iomr0.s.paIoPortRing3Regs;
+    AssertReturn((paIoPortRegs && paIoPortRegsR3) || cRegs == 0, VERR_IOM_IOPORT_IPE_3);
+
+    for (uint32_t i = 0 ; i < cRegs; i++)
+    {
+        uint16_t idxStats = paIoPortRegsR3[i].idxStats;
+        paIoPortRegs[i].idxStats = idxStats < cStatsAlloc ? idxStats : UINT16_MAX;
+    }
+
+#else
+    RT_NOREF(pGVM);
+#endif
+    return VINF_SUCCESS;
+}
+

trunk/src/VBox/VMM/VMMR0/IOMR0Mmio.cpp

@@ -268 +268 @@
     AssertReturn(pGVM->iom.s.cMmioStatsAllocation == cOldEntries, VERR_IOM_MMIO_IPE_1);
     AssertReturn(pGVM->iom.s.cMmioStats <= cOldEntries, VERR_IOM_MMIO_IPE_2);
+    AssertReturn(!pGVM->iomr0.s.fMmioStatsFrozen, VERR_WRONG_ORDER);
 
     /*
@@ -326 +327 @@
 }
 
+
+/**
+ * Called after all devices has been instantiated to copy over the statistics
+ * indices to the ring-0 MMIO registration table.
+ *
+ * This simplifies keeping statistics for MMIO ranges that are ring-3 only.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @thread  EMT(0)
+ * @note    Only callable at VM creation time.
+ */
+VMMR0_INT_DECL(int) IOMR0MmioSyncStatisticsIndices(PGVM pGVM)
+{
+    VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * First, freeze the statistics array:
+     */
+    pGVM->iomr0.s.fMmioStatsFrozen = true;
+
+    /*
+     * Second, synchronize the indices:
+     */
+    uint32_t const          cRegs        = RT_MIN(pGVM->iom.s.cMmioRegs, pGVM->iomr0.s.cMmioAlloc);
+    uint32_t const          cStatsAlloc  = pGVM->iomr0.s.cMmioStatsAllocation;
+    PIOMMMIOENTRYR0         paMmioRegs   = pGVM->iomr0.s.paMmioRegs;
+    IOMMMIOENTRYR3 const   *paMmioRegsR3 = pGVM->iomr0.s.paMmioRing3Regs;
+    AssertReturn((paMmioRegs && paMmioRegsR3) || cRegs == 0, VERR_IOM_MMIO_IPE_3);
+
+    for (uint32_t i = 0 ; i < cRegs; i++)
+    {
+        uint16_t idxStats = paMmioRegsR3[i].idxStats;
+        paMmioRegs[i].idxStats = idxStats < cStatsAlloc ? idxStats : UINT16_MAX;
+    }
+
+#else
+    RT_NOREF(pGVM);
+#endif
+    return VINF_SUCCESS;
+}
+

trunk/src/VBox/VMM/VMMR0/VMMR0.cpp

@@ -2217 +2217 @@
         }
 
+        case VMMR0_DO_IOM_SYNC_STATS_INDICES:
+        {
+            if (pReqHdr || idCpu != 0)
+                return VERR_INVALID_PARAMETER;
+            rc = IOMR0IoPortSyncStatisticsIndices(pGVM);
+            if (RT_SUCCESS(rc))
+                rc = IOMR0MmioSyncStatisticsIndices(pGVM);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+        }
+
         /*
          * For profiling.

trunk/src/VBox/VMM/VMMR3/IOM.cpp

@@ -138 +138 @@
 static DECLCALLBACK(int) iomR3RelocateMMIOCallback(PAVLROGCPHYSNODECORE pNode, void *pvUser);
 #endif
-static DECLCALLBACK(void) iomR3MMIOInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
 static FNIOMIOPORTIN        iomR3IOPortDummyIn;
 static FNIOMIOPORTOUT       iomR3IOPortDummyOut;
@@ -180 +179 @@
      */
     rc = MMHyperAlloc(pVM, sizeof(*pVM->iom.s.pTreesR3), 0, MM_TAG_IOM, (void **)&pVM->iom.s.pTreesR3);
-    if (RT_SUCCESS(rc))
-    {
-        pVM->iom.s.pTreesR0 = MMHyperR3ToR0(pVM, pVM->iom.s.pTreesR3);
-
-        /*
-         * Register the MMIO access handler type.
-         */
-        rc = PGMR3HandlerPhysicalTypeRegister(pVM, PGMPHYSHANDLERKIND_MMIO,
-                                              iomMmioHandler,
-                                              NULL, "iomMmioHandler", "iomMmioPfHandler",
-                                              NULL, "iomMmioHandler", "iomMmioPfHandler",
-                                              "MMIO", &pVM->iom.s.hMmioHandlerType);
-        AssertRC(rc);
-        if (RT_SUCCESS(rc))
-        {
-
-            /*
-             * Info.
-             */
-            DBGFR3InfoRegisterInternal(pVM, "ioport", "Dumps all IOPort ranges. No arguments.", &iomR3IoPortInfo);
-            DBGFR3InfoRegisterInternal(pVM, "mmio", "Dumps all MMIO ranges. No arguments.", &iomR3MMIOInfo);
-
-            /*
-             * Statistics.
-             */
-            STAM_REG(pVM, &pVM->iom.s.StatRZMMIOHandler,      STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler",                      STAMUNIT_TICKS_PER_CALL, "Profiling of the iomMmioPfHandler() body, only success calls.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZMMIO1Byte,        STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access1",              STAMUNIT_OCCURENCES,     "MMIO access by 1 byte counter.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZMMIO2Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access2",              STAMUNIT_OCCURENCES,     "MMIO access by 2 bytes counter.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZMMIO4Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access4",              STAMUNIT_OCCURENCES,     "MMIO access by 4 bytes counter.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZMMIO8Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access8",              STAMUNIT_OCCURENCES,     "MMIO access by 8 bytes counter.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZMMIOFailures,     STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/MMIOFailures",         STAMUNIT_OCCURENCES,     "Number of times iomMmioPfHandler() didn't service the request.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstMov,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOV",             STAMUNIT_TICKS_PER_CALL, "Profiling of the MOV instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstCmp,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/CMP",             STAMUNIT_TICKS_PER_CALL, "Profiling of the CMP instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstAnd,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/AND",             STAMUNIT_TICKS_PER_CALL, "Profiling of the AND instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstOr,           STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/OR",              STAMUNIT_TICKS_PER_CALL, "Profiling of the OR instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstXor,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/XOR",             STAMUNIT_TICKS_PER_CALL, "Profiling of the XOR instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstBt,           STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/BT",              STAMUNIT_TICKS_PER_CALL, "Profiling of the BT instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstTest,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/TEST",            STAMUNIT_TICKS_PER_CALL, "Profiling of the TEST instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstXchg,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/XCHG",            STAMUNIT_TICKS_PER_CALL, "Profiling of the XCHG instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstStos,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/STOS",            STAMUNIT_TICKS_PER_CALL, "Profiling of the STOS instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstLods,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/LODS",            STAMUNIT_TICKS_PER_CALL, "Profiling of the LODS instruction emulation.");
-#ifdef IOM_WITH_MOVS_SUPPORT
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstMovs,     STAMTYPE_PROFILE_ADV, "/IOM/RZ-MMIOHandler/Inst/MOVS",            STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstMovsToMMIO,   STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOVS/ToMMIO",     STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation - Mem2MMIO.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstMovsFromMMIO, STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOVS/FromMMIO",   STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation - MMIO2Mem.");
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstMovsMMIO,     STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOVS/MMIO2MMIO",  STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation - MMIO2MMIO.");
+    AssertRCReturn(rc, rc);
+    pVM->iom.s.pTreesR0 = MMHyperR3ToR0(pVM, pVM->iom.s.pTreesR3);
+
+    /*
+     * Register the MMIO access handler type.
+     */
+    rc = PGMR3HandlerPhysicalTypeRegister(pVM, PGMPHYSHANDLERKIND_MMIO,
+                                          iomMmioHandler,
+                                          NULL, "iomMmioHandler", "iomMmioPfHandler",
+                                          NULL, "iomMmioHandler", "iomMmioPfHandler",
+                                          "MMIO", &pVM->iom.s.hMmioHandlerType);
+    AssertRCReturn(rc, rc);
+
+    rc = PGMR3HandlerPhysicalTypeRegister(pVM, PGMPHYSHANDLERKIND_MMIO,
+                                          iomMmioHandlerNew,
+                                          NULL, "iomMmioHandlerNew", "iomMmioPfHandlerNew",
+                                          NULL, "iomMmioHandlerNew", "iomMmioPfHandlerNew",
+                                          "MMIO", &pVM->iom.s.hNewMmioHandlerType);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Info.
+     */
+    DBGFR3InfoRegisterInternal(pVM, "ioport", "Dumps all IOPort ranges. No arguments.", &iomR3IoPortInfo);
+    DBGFR3InfoRegisterInternal(pVM, "mmio", "Dumps all MMIO ranges. No arguments.", &iomR3MmioInfo);
+
+    /*
+     * Statistics.
+     */
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIOHandler,      STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler",                      STAMUNIT_TICKS_PER_CALL, "Profiling of the iomMmioPfHandler() body, only success calls.");
+#if 0
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIO1Byte,        STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access1",              STAMUNIT_OCCURENCES,     "MMIO access by 1 byte counter.");
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIO2Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access2",              STAMUNIT_OCCURENCES,     "MMIO access by 2 bytes counter.");
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIO4Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access4",              STAMUNIT_OCCURENCES,     "MMIO access by 4 bytes counter.");
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIO8Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access8",              STAMUNIT_OCCURENCES,     "MMIO access by 8 bytes counter.");
 #endif
-            STAM_REG(pVM, &pVM->iom.s.StatRZInstOther,        STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Inst/Other",           STAMUNIT_OCCURENCES,     "Other instructions counter.");
-            STAM_REG(pVM, &pVM->iom.s.StatR3MMIOHandler,      STAMTYPE_COUNTER, "/IOM/R3-MMIOHandler",                      STAMUNIT_OCCURENCES,     "Number of calls to iomR3MmioHandler.");
-#if 0 /* unused */
-            STAM_REG(pVM, &pVM->iom.s.StatInstIn,             STAMTYPE_COUNTER, "/IOM/IOWork/In",                           STAMUNIT_OCCURENCES,     "Counter of any IN instructions.");
-            STAM_REG(pVM, &pVM->iom.s.StatInstOut,            STAMTYPE_COUNTER, "/IOM/IOWork/Out",                          STAMUNIT_OCCURENCES,     "Counter of any OUT instructions.");
-            STAM_REG(pVM, &pVM->iom.s.StatInstIns,            STAMTYPE_COUNTER, "/IOM/IOWork/Ins",                          STAMUNIT_OCCURENCES,     "Counter of any INS instructions.");
-            STAM_REG(pVM, &pVM->iom.s.StatInstOuts,           STAMTYPE_COUNTER, "/IOM/IOWork/Outs",                         STAMUNIT_OCCURENCES,     "Counter of any OUTS instructions.");
-#endif
-        }
-    }
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIOReadsToR3,    STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/ReadsToR3",            STAMUNIT_OCCURENCES,     "Number of read deferred to ring-3.");
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIOWritesToR3,   STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/WritesToR3",           STAMUNIT_OCCURENCES,     "Number of writes deferred to ring-3.");
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIOCommitsToR3,  STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/CommitsToR3",          STAMUNIT_OCCURENCES,     "Number of commits deferred to ring-3.");
+    STAM_REG(pVM, &pVM->iom.s.StatRZMMIODevLockContention, STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/DevLockContention", STAMUNIT_OCCURENCES,   "Number of device lock contention force return to ring-3.");
+    STAM_REG(pVM, &pVM->iom.s.StatR3MMIOHandler,      STAMTYPE_COUNTER, "/IOM/R3-MMIOHandler",                      STAMUNIT_OCCURENCES,     "Number of calls to iomMmioHandler.");
+
+    STAM_REG(pVM, &pVM->iom.s.StatMmioHandlerR3,      STAMTYPE_COUNTER, "/IOM/MmioHandlerR3",                       STAMUNIT_OCCURENCES,     "Number of calls to iomMmioHandlerNew from ring-3.");
+    STAM_REG(pVM, &pVM->iom.s.StatMmioHandlerR0,      STAMTYPE_COUNTER, "/IOM/MmioHandlerR0",                       STAMUNIT_OCCURENCES,     "Number of calls to iomMmioHandlerNew from ring-0.");
 
     /* Redundant, but just in case we change something in the future */
     iomR3FlushCache(pVM);
 
-    LogFlow(("IOMR3Init: returns %Rrc\n", rc));
-    return rc;
+    LogFlow(("IOMR3Init: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
 }
 
@@ -258 +244 @@
     if (enmWhat == VMINITCOMPLETED_RING3)
     {
+        /*
+         * Synchronize the ring-3 I/O port and MMIO statistics indices into the
+         * ring-0 tables to simplify ring-0 code.  This also make sure that any
+         * later calls to grow the statistics tables will fail.
+         */
+        int rc = VMMR3CallR0Emt(pVM, pVM->apCpusR3[0], VMMR0_DO_IOM_SYNC_STATS_INDICES, 0, NULL);
+        AssertLogRelRCReturn(rc, rc);
+
+        /*
+         * Register I/O port and MMIO stats now that we're done registering MMIO
+         * regions and won't grow the table again.
+         */
         for (uint32_t i = 0; i < pVM->iom.s.cIoPortRegs; i++)
         {
@@ -264 +262 @@
                 && pRegEntry->idxStats != UINT16_MAX)
                 iomR3IoPortRegStats(pVM, pRegEntry);
+        }
+
+        for (uint32_t i = 0; i < pVM->iom.s.cMmioRegs; i++)
+        {
+            PIOMMMIOENTRYR3 pRegEntry = &pVM->iom.s.paMmioRegs[i];
+            if (   pRegEntry->fMapped
+                && pRegEntry->idxStats != UINT16_MAX)
+                iomR3MmioRegStats(pVM, pRegEntry);
         }
     }
@@ -1305 +1311 @@
     AssertMsgReturn(GCPhysStart + (cbRange - 1) >= GCPhysStart,("Wrapped! %RGp LB %RGp\n", GCPhysStart, cbRange),
                     VERR_IOM_INVALID_MMIO_RANGE);
-    AssertMsgReturn(   !(fFlags & ~IOMMMIO_FLAGS_VALID_MASK)
+    AssertMsgReturn(   !(fFlags & ~(IOMMMIO_FLAGS_VALID_MASK & ~IOMMMIO_FLAGS_ABS))
                     && (fFlags & IOMMMIO_FLAGS_READ_MODE)  <= IOMMMIO_FLAGS_READ_DWORD_QWORD
                     && (fFlags & IOMMMIO_FLAGS_WRITE_MODE) <= IOMMMIO_FLAGS_WRITE_ONLY_DWORD_QWORD,
@@ -1952 +1958 @@
     /** @todo IOM debug events. */
     RT_NOREF3(pVM, enmEvent, fEnabled);
-}
-
-
-/**
- * Display a single MMIO range.
- *
- * @returns 0
- * @param   pNode   Pointer to MMIO R3 range.
- * @param   pvUser  Pointer to info output callback structure.
- */
-static DECLCALLBACK(int) iomR3MMIOInfoOne(PAVLROGCPHYSNODECORE pNode, void *pvUser)
-{
-    PIOMMMIORANGE pRange = (PIOMMMIORANGE)pNode;
-    PCDBGFINFOHLP pHlp = (PCDBGFINFOHLP)pvUser;
-    pHlp->pfnPrintf(pHlp,
-                    "%RGp-%RGp %RHv %RHv %RHv %RHv %RHv %s\n",
-                    pRange->Core.Key,
-                    pRange->Core.KeyLast,
-                    pRange->pDevInsR3,
-                    pRange->pfnReadCallbackR3,
-                    pRange->pfnWriteCallbackR3,
-                    pRange->pfnFillCallbackR3,
-                    pRange->pvUserR3,
-                    pRange->pszDesc);
-    pHlp->pfnPrintf(pHlp,
-                    "%*s %RHv %RHv %RHv %RHv %RHv\n",
-                    sizeof(RTGCPHYS) * 2 * 2 + 1, "R0",
-                    pRange->pDevInsR0,
-                    pRange->pfnReadCallbackR0,
-                    pRange->pfnWriteCallbackR0,
-                    pRange->pfnFillCallbackR0,
-                    pRange->pvUserR0);
-#if 0
-    pHlp->pfnPrintf(pHlp,
-                    "%*s %RRv %RRv %RRv %RRv %RRv\n",
-                    sizeof(RTGCPHYS) * 2 * 2 + 1, "RC",
-                    pRange->pDevInsRC,
-                    pRange->pfnReadCallbackRC,
-                    pRange->pfnWriteCallbackRC,
-                    pRange->pfnFillCallbackRC,
-                    pRange->pvUserRC);
-#endif
-    return 0;
-}
-
-
-/**
- * Display registered MMIO ranges to the log.
- *
- * @param   pVM         The cross context VM structure.
- * @param   pHlp        The info helpers.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) iomR3MMIOInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    NOREF(pszArgs);
-    pHlp->pfnPrintf(pHlp,
-                    "MMIO ranges (pVM=%p)\n"
-                    "%.*s %.*s %.*s %.*s %.*s %.*s %s\n",
-                    pVM,
-                    sizeof(RTGCPHYS) * 4 + 1, "GC Phys Range                    ",
-                    sizeof(RTHCPTR) * 2,      "pDevIns         ",
-                    sizeof(RTHCPTR) * 2,      "Read            ",
-                    sizeof(RTHCPTR) * 2,      "Write           ",
-                    sizeof(RTHCPTR) * 2,      "Fill            ",
-                    sizeof(RTHCPTR) * 2,      "pvUser          ",
-                                              "Description");
-    RTAvlroGCPhysDoWithAll(&pVM->iom.s.pTreesR3->MMIOTree, true, iomR3MMIOInfoOne, (void *)pHlp);
 }
 

trunk/src/VBox/VMM/VMMR3/IOMR3Mmio.cpp

@@ -61 +61 @@
 
     /* Register statistics: */
-    int rc = STAMR3Register(pVM, &pStats->Accesses,    STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, pszDesc); AssertRC(rc);
+    int rc = STAMR3Register(pVM, &pRegEntry->idxSelf, STAMTYPE_U16, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_NONE, pszDesc); AssertRC(rc);
     RTStrFree(pszFreeDesc);
 
 # define SET_NM_SUFFIX(a_sz) memcpy(&szName[cchPrefix], a_sz, sizeof(a_sz))
+    SET_NM_SUFFIX("/Read-Complicated");
+    rc = STAMR3Register(pVM, &pStats->ComplicatedReads, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, NULL); AssertRC(rc);
+    SET_NM_SUFFIX("/Read-FFor00");
+    rc = STAMR3Register(pVM, &pStats->FFor00Reads, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
     SET_NM_SUFFIX("/Read-R3");
     rc = STAMR3Register(pVM, &pStats->ProfReadR3,  STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, NULL); AssertRC(rc);
-    SET_NM_SUFFIX("/Write-R3");
-    rc = STAMR3Register(pVM, &pStats->ProfWriteR3, STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, NULL); AssertRC(rc);
     SET_NM_SUFFIX("/Read-RZ");
     rc = STAMR3Register(pVM, &pStats->ProfReadRZ,  STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, NULL); AssertRC(rc);
+    SET_NM_SUFFIX("/Read-RZtoR3");
+    rc = STAMR3Register(pVM, &pStats->ReadRZToR3,  STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
+    SET_NM_SUFFIX("/Read-Total");
+    rc = STAMR3Register(pVM, &pStats->Reads,       STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
+
+    SET_NM_SUFFIX("/Write-Complicated");
+    rc = STAMR3Register(pVM, &pStats->ComplicatedWrites, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES, NULL); AssertRC(rc);
+    SET_NM_SUFFIX("/Write-R3");
+    rc = STAMR3Register(pVM, &pStats->ProfWriteR3,  STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, NULL); AssertRC(rc);
     SET_NM_SUFFIX("/Write-RZ");
     rc = STAMR3Register(pVM, &pStats->ProfWriteRZ, STAMTYPE_PROFILE, STAMVISIBILITY_USED, szName, STAMUNIT_TICKS_PER_CALL, NULL); AssertRC(rc);
-    SET_NM_SUFFIX("/Read-RZtoR3");
-    rc = STAMR3Register(pVM, &pStats->ReadRZToR3,  STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
     SET_NM_SUFFIX("/Write-RZtoR3");
     rc = STAMR3Register(pVM, &pStats->WriteRZToR3, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
+    SET_NM_SUFFIX("/Write-RZtoR3-Commit");
+    rc = STAMR3Register(pVM, &pStats->CommitRZToR3, STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,    NULL); AssertRC(rc);
+    SET_NM_SUFFIX("/Write-Total");
+    rc = STAMR3Register(pVM, &pStats->Writes,      STAMTYPE_COUNTER, STAMVISIBILITY_USED, szName, STAMUNIT_OCCURENCES,     NULL); AssertRC(rc);
 }
 
@@ -111 +124 @@
 
     AssertMsgReturn(cbRegion > 0 && !(cbRegion & PAGE_OFFSET_MASK), ("cbRegion=%RGp\n", cbRegion), VERR_OUT_OF_RANGE);
-    AssertReturn(!(fFlags & ~IOM_MMIO_F_VALID_MASK), VERR_INVALID_FLAGS);
+    AssertReturn(!(fFlags & ~IOMMMIO_FLAGS_VALID_MASK), VERR_INVALID_FLAGS);
 
     AssertReturn(pfnWrite || pfnRead || pfnFill, VERR_INVALID_PARAMETER);
@@ -218 +231 @@
                     else
                     {
+                        /* Register with PGM before we shuffle the array: */
+                        rc = PGMR3PhysMMIORegister(pVM, GCPhys, cbRegion, pVM->iom.s.hNewMmioHandlerType,
+                                                   (void *)(uintptr_t)hRegion, hRegion, hRegion, pRegEntry->pszDesc);
+                        AssertRCReturnStmt(rc, IOM_UNLOCK_EXCL(pVM), rc);
+
                         /* Insert after the entry we just considered: */
                         pEntry += 1;
@@ -231 +249 @@
                     else
                     {
+                        /* Register with PGM before we shuffle the array: */
+                        rc = PGMR3PhysMMIORegister(pVM, GCPhys, cbRegion, pVM->iom.s.hNewMmioHandlerType,
+                                                   (void *)(uintptr_t)hRegion, hRegion, hRegion, pRegEntry->pszDesc);
+                        AssertRCReturnStmt(rc, IOM_UNLOCK_EXCL(pVM), rc);
+
                         /* Insert at the entry we just considered: */
                         if (i < cEntries)
@@ -366 +389 @@
                 pRegEntry->GCPhysMapping = NIL_RTGCPHYS;
                 pRegEntry->fMapped       = false;
-                rc = VINF_SUCCESS;
+                rc = PGMR3PhysMMIODeregister(pVM, GCPhys, pRegEntry->cbRegion);
+                AssertRC(rc);
                 break;
             }

trunk/src/VBox/VMM/include/IOMInline.h

@@ -37 +37 @@
  *
  * @param   pVM             The cross context VM structure.
- * @param   uPort           The I/O port lookup.
- * @param   poffPort        Where to the port offset relative to the start of
- *                          the I/O port range.
+ * @param   uPort           The I/O port to lookup.
+ * @param   poffPort        Where to return the port offset relative to the
+ *                          start of the I/O port range.
  * @param   pidxLastHint    Pointer to IOMCPU::idxIoPortLastRead or
  *                          IOMCPU::idxIoPortLastWrite.
@@ -47 +47 @@
  *          for the entry.  Use IOMIOPORTENTRYR0::idxSelf to get the ring-3
  *          entry.
+ *
+ * @note    This code is almost identical to iomMmioGetEntry, so keep in sync.
  */
 DECLINLINE(CTX_SUFF(PIOMIOPORTENTRY)) iomIoPortGetEntry(PVMCC pVM, RTIOPORT uPort, PRTIOPORT poffPort, uint16_t *pidxLastHint)
@@ -117 +119 @@
 #ifdef VBOX_WITH_STATISTICS
 /**
- * Gets the I/O port statistics entry .
+ * Gets the statistics entry for an I/O port.
  *
  * @returns Pointer to stats.  Instead of NULL, a pointer to IoPortDummyStats is
@@ -139 +141 @@
 # endif
     return &pVM->iom.s.IoPortDummyStats;
+}
+#endif
+
+
+/**
+ * Gets the MMIO region entry for the specified address in the current context.
+ *
+ * @returns Pointer to MMIO region entry.
+ * @returns NULL if no MMIO region registered for the given address.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          The address to lookup.
+ * @param   poffRegion      Where to return the byte offset into the MMIO
+ *                          region that corresponds to @a GCPhys.
+ * @param   pidxLastHint    Pointer to IOMCPU::idxMmioLastRead,
+ *                          IOMCPU::idxMmioLastWrite, or similar.
+ *
+ * @note    In ring-0 it is possible to get an uninitialized entry (pDevIns is
+ *          NULL, cbRegion is 0), in which case there should be ring-3 handlers
+ *          for the entry.  Use IOMMMIOENTRYR0::idxSelf to get the ring-3 entry.
+ *
+ * @note    This code is almost identical to iomIoPortGetEntry, so keep in sync.
+ */
+DECLINLINE(CTX_SUFF(PIOMMMIOENTRY)) iomMmioGetEntry(PVMCC pVM, RTGCPHYS GCPhys, PRTGCPHYS poffRegion, uint16_t *pidxLastHint)
+{
+    Assert(IOM_IS_SHARED_LOCK_OWNER(pVM));
+
+#ifdef IN_RING0
+    uint32_t               iEnd      = RT_MIN(pVM->iom.s.cMmioLookupEntries, pVM->iomr0.s.cMmioAlloc);
+    PCIOMMMIOLOOKUPENTRY   paLookup  = pVM->iomr0.s.paMmioLookup;
+#else
+    uint32_t               iEnd      = pVM->iom.s.cMmioLookupEntries;
+    PCIOMMMIOLOOKUPENTRY   paLookup  = pVM->iom.s.paMmioLookup;
+#endif
+    if (iEnd > 0)
+    {
+        uint32_t iFirst = 0;
+        uint32_t i      = *pidxLastHint;
+        if (i < iEnd)
+        { /* likely */ }
+        else
+            i = iEnd / 2;
+        for (;;)
+        {
+            PCIOMMMIOLOOKUPENTRY pCur = &paLookup[i];
+            if (pCur->GCPhysFirst > GCPhys)
+            {
+                if (i > iFirst)
+                    iEnd = i;
+                else
+                    break;
+            }
+            else if (pCur->GCPhysLast < GCPhys)
+            {
+                i += 1;
+                if (i < iEnd)
+                    iFirst = i;
+                else
+                    break;
+            }
+            else
+            {
+                *pidxLastHint = (uint16_t)i;
+                *poffRegion   = GCPhys - pCur->GCPhysFirst;
+
+                /*
+                 * Translate the 'idx' member into a pointer.
+                 */
+                size_t const idx = pCur->idx;
+#ifdef IN_RING0
+                AssertMsg(idx < pVM->iom.s.cMmioRegs && idx < pVM->iomr0.s.cMmioAlloc,
+                          ("%#zx vs %#x/%x (GCPhys=%RGp)\n", idx, pVM->iom.s.cMmioRegs, pVM->iomr0.s.cMmioMax, GCPhys));
+                if (idx < pVM->iomr0.s.cMmioAlloc)
+                    return &pVM->iomr0.s.paMmioRegs[idx];
+#else
+                if (idx < pVM->iom.s.cMmioRegs)
+                    return &pVM->iom.s.paMmioRegs[idx];
+                AssertMsgFailed(("%#zx vs %#x (GCPhys=%RGp)\n", idx, pVM->iom.s.cMmioRegs, GCPhys));
+#endif
+                break;
+            }
+
+            i = iFirst + (iEnd - iFirst) / 2;
+        }
+    }
+    *poffRegion = 0;
+    return NULL;
+}
+
+
+#ifdef VBOX_WITH_STATISTICS
+/**
+ * Gets the statistics entry for an MMIO region.
+ *
+ * @returns Pointer to stats.  Instead of NULL, a pointer to MmioDummyStats is
+ *          returned, so the caller does not need to check for NULL.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pRegEntry   The I/O port entry to get stats for.
+ */
+DECLINLINE(PIOMMMIOSTATSENTRY) iomMmioGetStats(PVMCC pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
+{
+    size_t idxStats = pRegEntry->idxStats;
+# ifdef IN_RING0
+    if (idxStats < pVM->iomr0.s.cMmioStatsAllocation)
+        return &pVM->iomr0.s.paMmioStats[idxStats];
+# else
+    if (idxStats < pVM->iom.s.cMmioStats)
+        return &pVM->iom.s.paMmioStats[idxStats];
+# endif
+    return &pVM->iom.s.MmioDummyStats;
 }
 #endif

trunk/src/VBox/VMM/include/IOMInternal.h

@@ -307 +307 @@
     /** Pointer to the fill callback function. */
     R0PTRTYPE(PFNIOMMMIONEWFILL)        pfnFillCallback;
-    /** The entry of the first statistics entry, UINT16_MAX if no stats. */
+    /** The entry of the first statistics entry, UINT16_MAX if no stats.
+     * @note For simplicity, this is always copied from ring-3 for all entries at
+     *       the end of VM creation. */
     uint16_t                            idxStats;
     /** Same as the handle index. */
@@ -371 +373 @@
 typedef struct IOMMMIOSTATSENTRY
 {
-    /** Number of accesses (subtract ReadRZToR3 and WriteRZToR3 to get the right
-     *  number). */
-    STAMCOUNTER                 Accesses;
-
+    /** Counting and profiling reads in R0/RC. */
+    STAMPROFILE                 ProfReadRZ;
+    /** Number of successful read accesses. */
+    STAMCOUNTER                 Reads;
+    /** Number of reads to this address from R0/RC which was serviced in R3. */
+    STAMCOUNTER                 ReadRZToR3;
+    /** Number of complicated reads. */
+    STAMCOUNTER                 ComplicatedReads;
+    /** Number of reads of 0xff or 0x00. */
+    STAMCOUNTER                 FFor00Reads;
     /** Profiling read handler overhead in R3. */
     STAMPROFILE                 ProfReadR3;
+
+    /** Counting and profiling writes in R0/RC. */
+    STAMPROFILE                 ProfWriteRZ;
+    /** Number of successful read accesses. */
+    STAMCOUNTER                 Writes;
+    /** Number of writes to this address from R0/RC which was serviced in R3. */
+    STAMCOUNTER                 WriteRZToR3;
+    /** Number of writes to this address from R0/RC which was committed in R3. */
+    STAMCOUNTER                 CommitRZToR3;
+    /** Number of complicated writes. */
+    STAMCOUNTER                 ComplicatedWrites;
     /** Profiling write handler overhead in R3. */
     STAMPROFILE                 ProfWriteR3;
-    /** Counting and profiling reads in R0/RC. */
-    STAMPROFILE                 ProfReadRZ;
-    /** Counting and profiling writes in R0/RC. */
-    STAMPROFILE                 ProfWriteRZ;
-
-    /** Number of reads to this address from R0/RC which was serviced in R3. */
-    STAMCOUNTER                 ReadRZToR3;
-    /** Number of writes to this address from R0/RC which was serviced in R3. */
-    STAMCOUNTER                 WriteRZToR3;
 } IOMMMIOSTATSENTRY;
 /** Pointer to MMIO statistics entry. */
@@ -602 +612 @@
         /** Guest physical MMIO address. */
         RTGCPHYS                        GCPhys;
+        /** The number of bytes to write (0 if nothing pending). */
+        uint32_t                        cbValue;
+        /** Hint. */
+        uint32_t                        idxMmioRegionHint;
         /** The value to write. */
         uint8_t                         abValue[128];
-        /** The number of bytes to write (0 if nothing pending). */
-        uint32_t                        cbValue;
-        /** Alignment padding. */
-        uint32_t                        uAlignmentPadding;
     } PendingMmioWrite;
 
@@ -654 +664 @@
     /** MMIO physical access handler type.   */
     PGMPHYSHANDLERTYPE              hMmioHandlerType;
-    uint32_t                        u32Padding;
+    /** MMIO physical access handler type, new style.   */
+    PGMPHYSHANDLERTYPE              hNewMmioHandlerType;
 
     /** @name I/O ports
@@ -729 +740 @@
      * @{ */
     STAMPROFILE                     StatRZMMIOHandler;
-    STAMCOUNTER                     StatRZMMIOFailures;
-
-    STAMPROFILE                     StatRZInstMov;
-    STAMPROFILE                     StatRZInstCmp;
-    STAMPROFILE                     StatRZInstAnd;
-    STAMPROFILE                     StatRZInstOr;
-    STAMPROFILE                     StatRZInstXor;
-    STAMPROFILE                     StatRZInstBt;
-    STAMPROFILE                     StatRZInstTest;
-    STAMPROFILE                     StatRZInstXchg;
-    STAMPROFILE                     StatRZInstStos;
-    STAMPROFILE                     StatRZInstLods;
-#ifdef IOM_WITH_MOVS_SUPPORT
-    STAMPROFILEADV                  StatRZInstMovs;
-    STAMPROFILE                     StatRZInstMovsToMMIO;
-    STAMPROFILE                     StatRZInstMovsFromMMIO;
-    STAMPROFILE                     StatRZInstMovsMMIO;
-#endif
-    STAMCOUNTER                     StatRZInstOther;
-
+    STAMCOUNTER                     StatRZMMIOReadsToR3;
+    STAMCOUNTER                     StatRZMMIOWritesToR3;
+    STAMCOUNTER                     StatRZMMIOCommitsToR3;
+    STAMCOUNTER                     StatRZMMIODevLockContention;
+#if 0
     STAMCOUNTER                     StatRZMMIO1Byte;
     STAMCOUNTER                     StatRZMMIO2Bytes;
     STAMCOUNTER                     StatRZMMIO4Bytes;
     STAMCOUNTER                     StatRZMMIO8Bytes;
-
+#endif
     STAMCOUNTER                     StatR3MMIOHandler;
+
+    STAMCOUNTER                     StatMmioHandlerR3;
+    STAMCOUNTER                     StatMmioHandlerR0;
 
     RTUINT                          cMovsMaxBytes;
@@ -791 +790 @@
     /** The size of the paIoPortStats allocation (in entries). */
     uint32_t                        cIoPortStatsAllocation;
+    /** Prevents paIoPortStats from growing, set by IOMR0IoPortSyncStatisticsIndices(). */
+    bool                            fIoPortStatsFrozen;
     /** I/O port lookup table.   */
     R0PTRTYPE(PIOMIOPORTSTATSENTRY) paIoPortStats;
@@ -822 +823 @@
     /** The size of the paMmioStats allocation (in entries). */
     uint32_t                        cMmioStatsAllocation;
+    /* Prevents paMmioStats from growing, set by IOMR0MmioSyncStatisticsIndices(). */
+    bool                            fMmioStatsFrozen;
     /** MMIO lookup table.   */
     R0PTRTYPE(PIOMMMIOSTATSENTRY)   paMmioStats;
@@ -841 +844 @@
 DECLCALLBACK(void)  iomR3IoPortInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
 void                iomR3IoPortRegStats(PVM pVM, PIOMIOPORTENTRYR3 pRegEntry);
+DECLCALLBACK(void)  iomR3MmioInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+void                iomR3MmioRegStats(PVM pVM, PIOMMMIOENTRYR3 pRegEntry);
 #endif /* IN_RING3 */
 #ifdef IN_RING0
@@ -848 +853 @@
 void                iomR0MmioInitPerVMData(PGVM pGVM);
 #endif
+VBOXSTRICTRC        iomMmioCommonPfHandlerOld(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, PCPUMCTXCORE pCtxCore,
+                                              RTGCPHYS GCPhysFault, void *pvUser);
 
 #ifndef IN_RING3
 DECLEXPORT(FNPGMRZPHYSPFHANDLER)    iomMmioPfHandler;
+DECLEXPORT(FNPGMRZPHYSPFHANDLER)    iomMmioPfHandlerNew;
 #endif
 PGM_ALL_CB2_PROTO(FNPGMPHYSHANDLER) iomMmioHandler;
+PGM_ALL_CB2_PROTO(FNPGMPHYSHANDLER) iomMmioHandlerNew;
 
 /* IOM locking helpers. */