Changeset 104885 in vbox for trunk/src/VBox/VMM/VMMR3

Timestamp:

Jun 11, 2024 12:37:11 PM (6 months ago)

Author:

vboxsync

Message:

VMM/PGM: Lockless PGMPhysIemGCPhys2PtrNoLock. bugref:10687

Location:

trunk/src/VBox/VMM/VMMR3

Files:

• PGM-armv8.cpp (modified) (1 diff)
• PGM.cpp (modified) (5 diffs)
• PGMPhys.cpp (modified) (13 diffs)

trunk/src/VBox/VMM/VMMR3/PGM-armv8.cpp

@@ -238 +238 @@
     AssertRCReturn(rc, rc);
 
-    pgmR3PhysChunkInvalidateTLB(pVM); /* includes pgmPhysInvalidatePageMapTLB call */
+    pgmR3PhysChunkInvalidateTLB(pVM, false /*fInRendezvous*/); /* includes pgmPhysInvalidatePageMapTLB call */
 
     /*

trunk/src/VBox/VMM/VMMR3/PGM.cpp

@@ -934 +934 @@
     AssertRCReturn(rc, rc);
 
-    pgmR3PhysChunkInvalidateTLB(pVM); /* includes pgmPhysInvalidatePageMapTLB call */
+    pgmR3PhysChunkInvalidateTLB(pVM, false /*fInRendezvous*/); /* includes pgmPhysInvalidatePageMapTLB call */
 
     /*
@@ -1417 +1417 @@
         PGM_REG_COUNTER(&pPgmCpu->cA20Changes, "/PGM/CPU%u/cA20Changes",  "Number of A20 gate changes.");
 
+        PGM_REG_COUNTER(&pPgmCpu->StatRZRamRangeTlbMisses,              "/PGM/CPU%u/RZ/RamRange/TlbMisses",         "TLB misses (lockless).");
+        PGM_REG_COUNTER(&pPgmCpu->StatRZRamRangeTlbLocking,             "/PGM/CPU%u/RZ/RamRange/TlbLocking",        "Lockless TLB failed, falling back on locked lookup.");
+        PGM_REG_COUNTER(&pPgmCpu->StatRZPageMapTlbMisses,               "/PGM/CPU%u/RZ/Page/MapTlbMisses",          "Lockless page map TLB failed, falling back on locked lookup.");
+
+        PGM_REG_COUNTER(&pPgmCpu->StatR3RamRangeTlbMisses,              "/PGM/CPU%u/R3/RamRange/TlbMisses",         "TLB misses (lockless).");
+        PGM_REG_COUNTER(&pPgmCpu->StatR3RamRangeTlbLocking,             "/PGM/CPU%u/R3/RamRange/TlbLocking",        "Lockless TLB failed, falling back on locked lookup.");
+        PGM_REG_COUNTER(&pPgmCpu->StatR3PageMapTlbMisses,               "/PGM/CPU%u/R3/Page/MapTlbMisses",          "Lockless page map TLB failed, falling back on locked lookup.");
+
 #ifdef VBOX_WITH_STATISTICS
         PGMCPUSTATS *pCpuStats = &pVM->apCpusR3[idCpu]->pgm.s.Stats;
@@ -1429 +1437 @@
 # endif
         /* R0 only: */
-        PGM_REG_PROFILE(&pCpuStats->StatR0NpMiscfg,                      "/PGM/CPU%u/R0/NpMiscfg",                     "PGMR0Trap0eHandlerNPMisconfig() profiling.");
-        PGM_REG_COUNTER(&pCpuStats->StatR0NpMiscfgSyncPage,              "/PGM/CPU%u/R0/NpMiscfgSyncPage",             "SyncPage calls from PGMR0Trap0eHandlerNPMisconfig().");
+        PGM_REG_PROFILE(&pCpuStats->StatR0NpMiscfg,                     "/PGM/CPU%u/R0/NpMiscfg",                     "PGMR0Trap0eHandlerNPMisconfig() profiling.");
+        PGM_REG_COUNTER(&pCpuStats->StatR0NpMiscfgSyncPage,             "/PGM/CPU%u/R0/NpMiscfgSyncPage",             "SyncPage calls from PGMR0Trap0eHandlerNPMisconfig().");
 
         /* RZ only: */
@@ -1569 +1577 @@
         PGM_REG_COUNTER(&pCpuStats->StatRZFlushTLBSameCR3Global,       "/PGM/CPU%u/RZ/FlushTLB/SameCR3Global",     "The number of times PGMFlushTLB was called with the same CR3, global. (flush)");
         PGM_REG_PROFILE(&pCpuStats->StatRZGstModifyPage,               "/PGM/CPU%u/RZ/GstModifyPage",              "Profiling of the PGMGstModifyPage() body.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZRamRangeTlbHits,             "/PGM/CPU%u/RZ/RamRange/TlbHits",           "TLB hits (lockless).");
+        PGM_REG_COUNTER(&pCpuStats->StatRZPageMapTlbHits,              "/PGM/CPU%u/RZ/Page/MapTlbHits",            "TLB hits (lockless).");
 
         PGM_REG_PROFILE(&pCpuStats->StatR3SyncCR3,                     "/PGM/CPU%u/R3/SyncCR3",                        "Profiling of the PGMSyncCR3() body.");
@@ -1614 +1624 @@
         PGM_REG_COUNTER(&pCpuStats->StatR3FlushTLBSameCR3Global,       "/PGM/CPU%u/R3/FlushTLB/SameCR3Global",     "The number of times PGMFlushTLB was called with the same CR3, global. (flush)");
         PGM_REG_PROFILE(&pCpuStats->StatR3GstModifyPage,               "/PGM/CPU%u/R3/GstModifyPage",              "Profiling of the PGMGstModifyPage() body.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3RamRangeTlbHits,             "/PGM/CPU%u/R3/RamRange/TlbHits",           "TLB hits (lockless).");
+        PGM_REG_COUNTER(&pCpuStats->StatR3PageMapTlbHits,              "/PGM/CPU%u/R3/Page/MapTlbHits",            "TLB hits (lockless).");
 #endif /* VBOX_WITH_STATISTICS */
 
 #undef PGM_REG_PROFILE
 #undef PGM_REG_COUNTER
-
     }
 

trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp

@@ -1422 +1422 @@
     }
 
-    /* Update the RAM range entry to indicate that it is no longer mapped. */
-    pRam->GCPhys     = NIL_RTGCPHYS;
+    /* Update the RAM range entry to indicate that it is no longer mapped.
+       The GCPhys member is accessed by the lockless TLB lookup code, so update
+       it last and atomically to be on the safe side. */
     pRam->GCPhysLast = NIL_RTGCPHYS;
+    ASMAtomicWriteU64(&pRam->GCPhys, NIL_RTGCPHYS);
 
     /*
@@ -2654 +2656 @@
 #endif
             {
-                pgmPhysInvalidatePageMapTLB(pVM);
+                pgmPhysInvalidatePageMapTLB(pVM, false /*fInRendezvous*/);
                 return VINF_SUCCESS;
             }
@@ -2679 +2681 @@
     }
 
-    pgmPhysInvalidatePageMapTLB(pVM);
+    pgmPhysInvalidatePageMapTLB(pVM, false /*fInRendezvous*/);
     return rc;
 }
@@ -2851 +2853 @@
     VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
 
-    pgmPhysInvalidatePageMapTLB(pVM);
+    pgmPhysInvalidatePageMapTLB(pVM, false /*fInRendezvous*/);
     /*pgmPhysInvalidRamRangeTlbs(pVM); - not necessary */
 
@@ -3427 +3429 @@
         }
     }
-    pgmPhysInvalidatePageMapTLB(pVM);
+    pgmPhysInvalidatePageMapTLB(pVM, false /*fInRendezvous*/);
     PGM_UNLOCK(pVM);
     return !cFound && hMmio2 != NIL_PGMMMIO2HANDLE ? VERR_NOT_FOUND : rc;
@@ -3671 +3673 @@
 
     /* Flush physical page map TLB. */
-    pgmPhysInvalidatePageMapTLB(pVM);
+    pgmPhysInvalidatePageMapTLB(pVM, false /*fInRendezvous*/);
 
 #ifdef VBOX_WITH_NATIVE_NEM
@@ -3905 +3907 @@
     VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
 
-    pgmPhysInvalidatePageMapTLB(pVM);
+    pgmPhysInvalidatePageMapTLB(pVM, false /*fInRendezvous*/);
     /* pgmPhysInvalidRamRangeTlbs(pVM); - not necessary */
 
@@ -4779 +4781 @@
 
             /* Flush physical page map TLB. */
-            pgmPhysInvalidatePageMapTLB(pVM);
+            pgmPhysInvalidatePageMapTLB(pVM, false /*fInRendezvous*/);
 
             /*
@@ -4938 +4940 @@
         }
     }
-    pgmPhysInvalidatePageMapTLB(pVM);
+    pgmPhysInvalidatePageMapTLB(pVM, false /*fInRendezvous*/);
     pgmPhysInvalidRamRangeTlbs(pVM);
 
@@ -5777 +5779 @@
 #endif
 
+#if 0 /* This is too much work with the PGMCPU::PhysTlb as well.  We flush them all instead. */
     for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR3.aEntries); i++)
         if (pVM->pgm.s.PhysTlbR3.aEntries[i].pMap == pChunk)
             return 0;
+#endif
 
     pArg->pChunk = pChunk;
@@ -5901 +5905 @@
                 }
 
-                pgmR3PhysChunkInvalidateTLB(pVM); /* includes pgmPhysInvalidatePageMapTLB call */
+                pgmR3PhysChunkInvalidateTLB(pVM, true /*fInRendezvous*/); /* includes pgmPhysInvalidatePageMapTLB call */
             }
         }
@@ -6043 +6047 @@
  * Invalidates the TLB for the ring-3 mapping cache.
  *
- * @param   pVM         The cross context VM structure.
- */
-DECLHIDDEN(void) pgmR3PhysChunkInvalidateTLB(PVM pVM)
+ * @param   pVM             The cross context VM structure.
+ * @param   fInRendezvous   Set if we're in a rendezvous.
+ */
+DECLHIDDEN(void) pgmR3PhysChunkInvalidateTLB(PVM pVM, bool fInRendezvous)
 {
     PGM_LOCK_VOID(pVM);
@@ -6054 +6059 @@
     }
     /* The page map TLB references chunks, so invalidate that one too. */
-    pgmPhysInvalidatePageMapTLB(pVM);
+    pgmPhysInvalidatePageMapTLB(pVM, fInRendezvous);
     PGM_UNLOCK(pVM);
 }