Changeset 65531 in vbox for trunk/src/VBox/VMM

Timestamp:

Jan 31, 2017 10:26:35 AM (8 years ago)

Author:

vboxsync

Message:

PGM: Put back r113092 & r113134 (revered in r113137).

Location:

trunk/src/VBox/VMM

Files:

• VMMAll/PGMAllBth.h (modified) (1 diff)
• VMMAll/PGMAllGst.h (modified) (7 diffs)
• VMMAll/PGMAllPool.cpp (modified) (45 diffs)
• VMMR3/PGM.cpp (modified) (4 diffs)
• VMMR3/PGMDbg.cpp (modified) (4 diffs)
• VMMR3/PGMPool.cpp (modified) (2 diffs)
• include/PGMGstDefs.h (modified) (2 diffs)
• include/PGMInternal.h (modified) (8 diffs)
• testcase/tstVMStruct.h (modified) (1 diff)

trunk/src/VBox/VMM/VMMAll/PGMAllBth.h

@@ -1338 +1338 @@
         PdeSrc.u = 0;
 # endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
+    const bool      fWasBigPage = RT_BOOL(PdeDst.u & PGM_PDFLAGS_BIG_PAGE);
     const bool      fIsBigPage  = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
+    if (fWasBigPage != fIsBigPage)
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
 
 # ifdef IN_RING3

trunk/src/VBox/VMM/VMMAll/PGMAllGst.h

@@ -74 +74 @@
  * @param   pWalk       Where to return the walk result. This is always set.
  */
-static int PGM_GST_NAME(Walk)(PVMCPU pVCpu, RTGCPTR GCPtr, PGSTPTWALK pWalk)
+DECLINLINE(int) PGM_GST_NAME(Walk)(PVMCPU pVCpu, RTGCPTR GCPtr, PGSTPTWALK pWalk)
 {
     int rc;
@@ -93 +93 @@
 # endif
 
+    uint32_t register fEffective = X86_PTE_RW | X86_PTE_US | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_A | 1;
     {
 # if PGM_GST_TYPE == PGM_TYPE_AMD64
@@ -99 +100 @@
          */
         rc = pgmGstGetLongModePML4PtrEx(pVCpu, &pWalk->pPml4);
-        if (RT_FAILURE(rc))
-            return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 4, rc);
-
-        PX86PML4  register  pPml4 = pWalk->pPml4;
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 4, rc);
+
+        PX86PML4E register  pPml4e;
+        pWalk->pPml4e  = pPml4e  = &pWalk->pPml4->a[(GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK];
         X86PML4E  register  Pml4e;
-        PX86PML4E register  pPml4e;
-
-        pWalk->pPml4e  = pPml4e  = &pPml4->a[(GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK];
         pWalk->Pml4e.u = Pml4e.u = pPml4e->u;
-        if (!Pml4e.n.u1Present)
-            return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 4);
-        if (RT_UNLIKELY(!GST_IS_PML4E_VALID(pVCpu, Pml4e)))
-            return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 4);
+
+        if (Pml4e.n.u1Present) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 4);
+
+        if (RT_LIKELY(GST_IS_PML4E_VALID(pVCpu, Pml4e))) { /* likely */ }
+        else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 4);
+
+        pWalk->Core.fEffective = fEffective = ((uint32_t)Pml4e.u & (X86_PML4E_RW  | X86_PML4E_US | X86_PML4E_PWT | X86_PML4E_PCD | X86_PML4E_A))
+                                            | ((uint32_t)(Pml4e.u >> 63) ^ 1) /*NX */;
 
         /*
@@ -117 +121 @@
          */
         rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, Pml4e.u & X86_PML4E_PG_MASK, &pWalk->pPdpt);
-        if (RT_FAILURE(rc))
-            return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 3, rc);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 3, rc);
 
 # elif PGM_GST_TYPE == PGM_TYPE_PAE
         rc = pgmGstGetPaePDPTPtrEx(pVCpu, &pWalk->pPdpt);
-        if (RT_FAILURE(rc))
-            return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
 # endif
     }
     {
 # if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
-        PX86PDPT register   pPdpt = pWalk->pPdpt;
         PX86PDPE register   pPdpe;
+        pWalk->pPdpe  = pPdpe  = &pWalk->pPdpt->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK];
         X86PDPE  register   Pdpe;
-
-        pWalk->pPdpe  = pPdpe  = &pPdpt->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK];
         pWalk->Pdpe.u = Pdpe.u = pPdpe->u;
-        if (!Pdpe.n.u1Present)
-            return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 3);
-        if (RT_UNLIKELY(!GST_IS_PDPE_VALID(pVCpu, Pdpe)))
-            return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 3);
+
+        if (Pdpe.n.u1Present) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 3);
+
+        if (RT_LIKELY(GST_IS_PDPE_VALID(pVCpu, Pdpe))) { /* likely */ }
+        else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 3);
+
+# if PGM_GST_TYPE == PGM_TYPE_AMD64
+        pWalk->Core.fEffective = fEffective &= ((uint32_t)Pdpe.u & (X86_PDPE_RW  | X86_PDPE_US | X86_PDPE_PWT | X86_PDPE_PCD | X86_PDPE_A))
+                                             | ((uint32_t)(Pdpe.u >> 63) ^ 1) /*NX */;
+# else
+        pWalk->Core.fEffective = fEffective  = X86_PDPE_RW  | X86_PDPE_US | X86_PDPE_A
+                                             | ((uint32_t)Pdpe.u & (X86_PDPE_PWT | X86_PDPE_PCD))
+                                             | ((uint32_t)(Pdpe.u >> 63) ^ 1) /*NX */;
+# endif
 
         /*
@@ -143 +156 @@
          */
         rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, Pdpe.u & X86_PDPE_PG_MASK, &pWalk->pPd);
-        if (RT_FAILURE(rc))
-            return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 2, rc);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 2, rc);
 # elif PGM_GST_TYPE == PGM_TYPE_32BIT
         rc = pgmGstGet32bitPDPtrEx(pVCpu, &pWalk->pPd);
-        if (RT_FAILURE(rc))
-            return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
 # endif
     }
     {
-        PGSTPD  register    pPd = pWalk->pPd;
         PGSTPDE register    pPde;
+        pWalk->pPde  = pPde  = &pWalk->pPd->a[(GCPtr >> GST_PD_SHIFT) & GST_PD_MASK];
         GSTPDE              Pde;
-
-        pWalk->pPde  = pPde  = &pPd->a[(GCPtr >> GST_PD_SHIFT) & GST_PD_MASK];
         pWalk->Pde.u = Pde.u = pPde->u;
-        if (!Pde.n.u1Present)
-            return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 2);
+        if (Pde.n.u1Present) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 2);
         if (Pde.n.u1Size && GST_IS_PSE_ACTIVE(pVCpu))
         {
-            if (RT_UNLIKELY(!GST_IS_BIG_PDE_VALID(pVCpu, Pde)))
-                return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 2);
+            if (RT_LIKELY(GST_IS_BIG_PDE_VALID(pVCpu, Pde))) { /* likely */ }
+            else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 2);
+
+            /*
+             * We're done.
+             */
+# if PGM_GST_TYPE == PGM_TYPE_32BIT
+            fEffective &= Pde.u & (X86_PDE4M_RW  | X86_PDE4M_US | X86_PDE4M_PWT | X86_PDE4M_PCD | X86_PDE4M_A);
+# else
+            fEffective &= ((uint32_t)Pde.u & (X86_PDE4M_RW  | X86_PDE4M_US | X86_PDE4M_PWT | X86_PDE4M_PCD | X86_PDE4M_A))
+                        | ((uint32_t)(Pde.u >> 63) ^ 1) /*NX */;
+# endif
+            fEffective |= (uint32_t)Pde.u & (X86_PDE4M_D | X86_PDE4M_G);
+            fEffective |= (uint32_t)(Pde.u & X86_PDE4M_PAT) >> X86_PDE4M_PAT_SHIFT;
+            pWalk->Core.fEffective = fEffective;
+
+            pWalk->Core.fEffectiveRW = !!(fEffective & X86_PTE_RW);
+            pWalk->Core.fEffectiveUS = !!(fEffective & X86_PTE_US);
+# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
+            pWalk->Core.fEffectiveNX = !(fEffective & 1) && GST_IS_NX_ACTIVE(pVCpu);
+# else
+            pWalk->Core.fEffectiveNX = false;
+# endif
+            pWalk->Core.fBigPage     = true;
+            pWalk->Core.fSucceeded   = true;
 
             pWalk->Core.GCPhys       = GST_GET_BIG_PDE_GCPHYS(pVCpu->CTX_SUFF(pVM), Pde)
                                      | (GCPtr & GST_BIG_PAGE_OFFSET_MASK);
             PGM_A20_APPLY_TO_VAR(pVCpu, pWalk->Core.GCPhys);
-            uint8_t fEffectiveXX     = (uint8_t)pWalk->Pde.u
-#  if PGM_GST_TYPE == PGM_TYPE_AMD64
-                                     & (uint8_t)pWalk->Pdpe.u
-                                     & (uint8_t)pWalk->Pml4e.u
-#  endif
-                                     ;
-            pWalk->Core.fEffectiveRW = !!(fEffectiveXX & X86_PTE_RW);
-            pWalk->Core.fEffectiveUS = !!(fEffectiveXX & X86_PTE_US);
-# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
-            pWalk->Core.fEffectiveNX = (   pWalk->Pde.n.u1NoExecute
-#  if PGM_GST_TYPE == PGM_TYPE_AMD64
-                                        || pWalk->Pdpe.lm.u1NoExecute
-                                        || pWalk->Pml4e.n.u1NoExecute
-#  endif
-                                       ) && GST_IS_NX_ACTIVE(pVCpu);
-# else
-            pWalk->Core.fEffectiveNX = false;
-# endif
-            pWalk->Core.fBigPage     = true;
-            pWalk->Core.fSucceeded   = true;
             return VINF_SUCCESS;
         }
@@ -193 +207 @@
         if (RT_UNLIKELY(!GST_IS_PDE_VALID(pVCpu, Pde)))
             return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 2);
+# if PGM_GST_TYPE == PGM_TYPE_32BIT
+        pWalk->Core.fEffective = fEffective &= Pde.u & (X86_PDE_RW  | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD | X86_PDE_A);
+# else
+        pWalk->Core.fEffective = fEffective &= ((uint32_t)Pde.u & (X86_PDE_RW  | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD | X86_PDE_A))
+                                             | ((uint32_t)(Pde.u >> 63) ^ 1) /*NX */;
+# endif
 
         /*
@@ -198 +218 @@
          */
         rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GST_GET_PDE_GCPHYS(Pde), &pWalk->pPt);
-        if (RT_FAILURE(rc))
-            return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 1, rc);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 1, rc);
     }
     {
-        PGSTPT  register    pPt = pWalk->pPt;
         PGSTPTE register    pPte;
+        pWalk->pPte  = pPte  = &pWalk->pPt->a[(GCPtr >> GST_PT_SHIFT) & GST_PT_MASK];
         GSTPTE  register    Pte;
-
-        pWalk->pPte  = pPte  = &pPt->a[(GCPtr >> GST_PT_SHIFT) & GST_PT_MASK];
         pWalk->Pte.u = Pte.u = pPte->u;
-        if (!Pte.n.u1Present)
-            return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 1);
-        if (RT_UNLIKELY(!GST_IS_PTE_VALID(pVCpu, Pte)))
-            return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 1);
+
+        if (Pte.n.u1Present) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 1);
+
+        if (RT_LIKELY(GST_IS_PTE_VALID(pVCpu, Pte))) { /* likely */ }
+        else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 1);
 
         /*
          * We're done.
          */
+# if PGM_GST_TYPE == PGM_TYPE_32BIT
+        fEffective &= Pte.u & (X86_PTE_RW  | X86_PTE_US | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_A);
+# else
+        fEffective &= ((uint32_t)Pte.u & (X86_PTE_RW  | X86_PTE_US | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_A))
+                    | ((uint32_t)(Pte.u >> 63) ^ 1) /*NX */;
+# endif
+        fEffective |= (uint32_t)Pte.u & (X86_PTE_D | X86_PTE_PAT | X86_PTE_G);
+        pWalk->Core.fEffective = fEffective;
+
+        pWalk->Core.fEffectiveRW = !!(fEffective & X86_PTE_RW);
+        pWalk->Core.fEffectiveUS = !!(fEffective & X86_PTE_US);
+# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
+        pWalk->Core.fEffectiveNX = !(fEffective & 1) && GST_IS_NX_ACTIVE(pVCpu);
+# else
+        pWalk->Core.fEffectiveNX = false;
+# endif
+        pWalk->Core.fSucceeded   = true;
+
         pWalk->Core.GCPhys       = GST_GET_PDE_GCPHYS(Pte)
                                  | (GCPtr & PAGE_OFFSET_MASK);
-        uint8_t fEffectiveXX     = (uint8_t)pWalk->Pte.u
-                                 & (uint8_t)pWalk->Pde.u
-#  if PGM_GST_TYPE == PGM_TYPE_AMD64
-                                 & (uint8_t)pWalk->Pdpe.u
-                                 & (uint8_t)pWalk->Pml4e.u
-#  endif
-                                 ;
-        pWalk->Core.fEffectiveRW = !!(fEffectiveXX & X86_PTE_RW);
-        pWalk->Core.fEffectiveUS = !!(fEffectiveXX & X86_PTE_US);
-# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
-        pWalk->Core.fEffectiveNX = (   pWalk->Pte.n.u1NoExecute
-                                    || pWalk->Pde.n.u1NoExecute
-#  if PGM_GST_TYPE == PGM_TYPE_AMD64
-                                    || pWalk->Pdpe.lm.u1NoExecute
-                                    || pWalk->Pml4e.n.u1NoExecute
-#  endif
-                                   ) && GST_IS_NX_ACTIVE(pVCpu);
-# else
-        pWalk->Core.fEffectiveNX = false;
-# endif
-        pWalk->Core.fSucceeded   = true;
         return VINF_SUCCESS;
     }

trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp

@@ -685 +685 @@
 }
 
-# ifndef IN_RING3
+#ifndef IN_RING3
 
 /**
@@ -697 +697 @@
  * @param   offFault    The access offset.
  */
-DECLINLINE(bool) pgmPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pDis, unsigned offFault)
+DECLINLINE(bool) pgmRZPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pDis, unsigned offFault)
 {
     /*
@@ -720 +720 @@
             )
     {
-        STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,Fork)); RT_NOREF_PV(pPool);
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,Fork)); RT_NOREF_PV(pPool);
         return true;
     }
@@ -740 +740 @@
  * @remark  The REP prefix check is left to the caller because of STOSD/W.
  */
-DECLINLINE(bool) pgmPoolMonitorIsReused(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, PDISCPUSTATE pDis, RTGCPTR pvFault)
-{
-#ifndef IN_RC
+DECLINLINE(bool) pgmRZPoolMonitorIsReused(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, PDISCPUSTATE pDis, RTGCPTR pvFault)
+{
+# ifndef IN_RC
     /** @todo could make this general, faulting close to rsp should be a safe reuse heuristic. */
     if (   HMHasPendingIrq(pVM)
@@ -748 +748 @@
     {
         /* Fault caused by stack writes while trying to inject an interrupt event. */
-        Log(("pgmPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pRegFrame->rsp));
+        Log(("pgmRZPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pRegFrame->rsp));
         return true;
     }
-#else
+# else
     NOREF(pVM); NOREF(pvFault);
-#endif
+# endif
 
     LogFlow(("Reused instr %RGv %d at %RGv param1.fUse=%llx param1.reg=%d\n", pRegFrame->rip, pDis->pCurInstr->uOpcode, pvFault, pDis->Param1.fUse,  pDis->Param1.Base.idxGenReg));
@@ -765 +765 @@
         /* call implies the actual push of the return address faulted */
         case OP_CALL:
-            Log4(("pgmPoolMonitorIsReused: CALL\n"));
+            Log4(("pgmRZPoolMonitorIsReused: CALL\n"));
             return true;
         case OP_PUSH:
-            Log4(("pgmPoolMonitorIsReused: PUSH\n"));
+            Log4(("pgmRZPoolMonitorIsReused: PUSH\n"));
             return true;
         case OP_PUSHF:
-            Log4(("pgmPoolMonitorIsReused: PUSHF\n"));
+            Log4(("pgmRZPoolMonitorIsReused: PUSHF\n"));
             return true;
         case OP_PUSHA:
-            Log4(("pgmPoolMonitorIsReused: PUSHA\n"));
+            Log4(("pgmRZPoolMonitorIsReused: PUSHA\n"));
             return true;
         case OP_FXSAVE:
-            Log4(("pgmPoolMonitorIsReused: FXSAVE\n"));
+            Log4(("pgmRZPoolMonitorIsReused: FXSAVE\n"));
             return true;
         case OP_MOVNTI:     /* solaris - block_zero_no_xmm */
-            Log4(("pgmPoolMonitorIsReused: MOVNTI\n"));
+            Log4(("pgmRZPoolMonitorIsReused: MOVNTI\n"));
             return true;
         case OP_MOVNTDQ:    /* solaris - hwblkclr & hwblkpagecopy */
-            Log4(("pgmPoolMonitorIsReused: MOVNTDQ\n"));
+            Log4(("pgmRZPoolMonitorIsReused: MOVNTDQ\n"));
             return true;
         case OP_MOVSWD:
@@ -793 +793 @@
                 Assert(pDis->uCpuMode == DISCPUMODE_64BIT);
 
-                Log(("pgmPoolMonitorIsReused: OP_STOSQ\n"));
+                Log(("pgmRZPoolMonitorIsReused: OP_STOSQ\n"));
                 return true;
             }
@@ -806 +806 @@
                 &&  (pDis->Param1.Base.idxGenReg == DISGREG_ESP))
             {
-                Log4(("pgmPoolMonitorIsReused: ESP\n"));
+                Log4(("pgmRZPoolMonitorIsReused: ESP\n"));
                 return true;
             }
@@ -819 +819 @@
     if ( (((uintptr_t)pvFault + cbWrite) >> X86_PAGE_SHIFT) != ((uintptr_t)pvFault >> X86_PAGE_SHIFT) )
     {
-        Log4(("pgmPoolMonitorIsReused: cross page write\n"));
+        Log4(("pgmRZPoolMonitorIsReused: cross page write\n"));
         return true;
     }
@@ -828 +828 @@
     if (cbWrite >= 8 && ((uintptr_t)pvFault & 7) != 0)
     {
-        Log4(("pgmPoolMonitorIsReused: Unaligned 8+ byte write\n"));
+        Log4(("pgmRZPoolMonitorIsReused: Unaligned 8+ byte write\n"));
         return true;
     }
@@ -850 +850 @@
  * @todo VBOXSTRICTRC
  */
-static int pgmPoolAccessPfHandlerFlush(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
-                                       PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
+static int pgmRZPoolAccessPfHandlerFlush(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
+                                         PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
 {
     NOREF(pVM); NOREF(GCPhysFault);
@@ -871 +871 @@
         if (rc == VINF_SUCCESS)
             rc = VBOXSTRICTRC_VAL(rc2);
-#ifndef IN_RING3
+# ifndef IN_RING3
         VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
-#endif
+# endif
     }
     else if (rc2 == VERR_EM_INTERPRETER)
     {
-#ifdef IN_RC
+# ifdef IN_RC
         if (PATMIsPatchGCAddr(pVM, pRegFrame->eip))
         {
-            LogFlow(("pgmPoolAccessPfHandlerPTWorker: Interpretation failed for patch code %04x:%RGv, ignoring.\n",
+            LogFlow(("pgmRZPoolAccessPfHandlerFlush: Interpretation failed for patch code %04x:%RGv, ignoring.\n",
                      pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->eip));
             rc = VINF_SUCCESS;
-            STAM_COUNTER_INC(&pPool->StatMonitorRZIntrFailPatch2);
+            STAM_COUNTER_INC(&pPool->StatMonitorPfRZIntrFailPatch2);
         }
         else
-#endif
+# endif
         {
             rc = VINF_EM_RAW_EMULATE_INSTR;
-            STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,EmulateInstr));
+            STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
         }
     }
@@ -897 +897 @@
         AssertMsgFailed(("%Rrc\n", VBOXSTRICTRC_VAL(rc2))); /* ASSUMES no complicated stuff here. */
 
-    LogFlow(("pgmPoolAccessPfHandlerPT: returns %Rrc (flushed)\n", rc));
+    LogFlow(("pgmRZPoolAccessPfHandlerFlush: returns %Rrc (flushed)\n", rc));
     return rc;
 }
@@ -914 +914 @@
  * @param   pvFault     The fault address.
  */
-DECLINLINE(int) pgmPoolAccessPfHandlerSTOSD(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
-                                            PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
+DECLINLINE(int) pgmRZPoolAccessPfHandlerSTOSD(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
+                                              PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
 {
     unsigned uIncrement = pDis->Param1.cb;
@@ -923 +923 @@
     Assert(pRegFrame->rcx <= 0x20);
 
-#ifdef VBOX_STRICT
+# ifdef VBOX_STRICT
     if (pDis->uOpMode == DISCPUMODE_32BIT)
         Assert(uIncrement == 4);
     else
         Assert(uIncrement == 8);
-#endif
-
-    Log3(("pgmPoolAccessPfHandlerSTOSD\n"));
+# endif
+
+    Log3(("pgmRZPoolAccessPfHandlerSTOSD\n"));
 
     /*
@@ -949 +949 @@
     while (pRegFrame->rcx)
     {
-#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) || defined(IN_RC)
+# if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) || defined(IN_RC)
         uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
         pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, uIncrement);
         PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
-#else
+# else
         pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, uIncrement);
-#endif
-#ifdef IN_RC
+# endif
+# ifdef IN_RC
         *(uint32_t *)(uintptr_t)pu32 = pRegFrame->eax;
-#else
+# else
         PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pRegFrame->rax, uIncrement);
-#endif
+# endif
         pu32           += uIncrement;
         GCPhysFault    += uIncrement;
@@ -968 +968 @@
     pRegFrame->rip += pDis->cbInstr;
 
-    LogFlow(("pgmPoolAccessPfHandlerSTOSD: returns\n"));
+    LogFlow(("pgmRZPoolAccessPfHandlerSTOSD: returns\n"));
     return VINF_SUCCESS;
 }
@@ -987 +987 @@
  * @param   pfReused    Reused state (in/out)
  */
-DECLINLINE(int) pgmPoolAccessPfHandlerSimple(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
-                                             PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault, bool *pfReused)
-{
-    Log3(("pgmPoolAccessPfHandlerSimple\n"));
+DECLINLINE(int) pgmRZPoolAccessPfHandlerSimple(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
+                                               PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault, bool *pfReused)
+{
+    Log3(("pgmRZPoolAccessPfHandlerSimple\n"));
     NOREF(pVM);
     NOREF(pfReused); /* initialized by caller */
@@ -1004 +1004 @@
      * Clear all the pages. ASSUMES that pvFault is readable.
      */
-#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) || defined(IN_RC)
+# if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) || defined(IN_RC)
     uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
-#endif
+# endif
 
     uint32_t cbWrite = DISGetParamSize(pDis, &pDis->Param1);
@@ -1023 +1023 @@
     }
 
-#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) || defined(IN_RC)
+# if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) || defined(IN_RC)
     PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
-#endif
+# endif
 
     /*
@@ -1035 +1035 @@
     else if (rc == VERR_EM_INTERPRETER)
     {
-        LogFlow(("pgmPoolAccessPfHandlerPTWorker: Interpretation failed for %04x:%RGv - opcode=%d\n",
+        LogFlow(("pgmRZPoolAccessPfHandlerSimple: Interpretation failed for %04x:%RGv - opcode=%d\n",
                   pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->uOpcode));
         rc = VINF_EM_RAW_EMULATE_INSTR;
-        STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,EmulateInstr));
-    }
-
-#if 0 /* experimental code */
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
+    }
+
+# if 0 /* experimental code */
     if (rc == VINF_SUCCESS)
     {
@@ -1069 +1069 @@
         }
     }
-#endif
-
-    LogFlow(("pgmPoolAccessPfHandlerSimple: returns %Rrc\n", VBOXSTRICTRC_VAL(rc)));
+# endif
+
+    LogFlow(("pgmRZPoolAccessPfHandlerSimple: returns %Rrc\n", VBOXSTRICTRC_VAL(rc)));
     return VBOXSTRICTRC_VAL(rc);
 }
@@ -1082 +1082 @@
  * @remarks The @a pvUser argument points to the PGMPOOLPAGE.
  */
-DECLEXPORT(VBOXSTRICTRC) pgmPoolAccessPfHandler(PVM pVM, PVMCPU pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame,
-                                                RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
-{
-    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), a);
+DECLEXPORT(VBOXSTRICTRC) pgmRZPoolAccessPfHandler(PVM pVM, PVMCPU pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame,
+                                                  RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
+{
+    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorRZ, a);
     PPGMPOOL        pPool = pVM->pgm.s.CTX_SUFF(pPool);
     PPGMPOOLPAGE    pPage = (PPGMPOOLPAGE)pvUser;
@@ -1092 +1092 @@
     NOREF(uErrorCode);
 
-    LogFlow(("pgmPoolAccessPfHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
+    LogFlow(("pgmRZPoolAccessPfHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
 
     pgmLock(pVM);
@@ -1098 +1098 @@
     {
         /* Pool page changed while we were waiting for the lock; ignore. */
-        Log(("CPU%d: pgmPoolAccessPfHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhysFault), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
-        STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,Handled), a);
+        Log(("CPU%d: pgmRZPoolAccessPfHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhysFault), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
+        STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
         pgmUnlock(pVM);
         return VINF_SUCCESS;
     }
-#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
     if (pPage->fDirty)
     {
@@ -1110 +1110 @@
         return VINF_SUCCESS;    /* SMP guest case where we were blocking on the pgm lock while the same page was being marked dirty. */
     }
-#endif
-
-#if 0 /* test code defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) */
+# endif
+
+# if 0 /* test code defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) */
     if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
     {
@@ -1122 +1122 @@
         PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
     }
-#endif
+# endif
 
     /*
@@ -1144 +1144 @@
     Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
 
-#ifdef IN_RING0
+# ifdef IN_RING0
     /* Maximum nr of modifications depends on the page type. */
     if (    pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
@@ -1151 +1151 @@
     else
         cMaxModifications = 24;
-#else
+# else
     cMaxModifications = 48;
-#endif
+# endif
 
     /*
@@ -1172 +1172 @@
         if (pPage->cModifications >= cMaxModifications)
         {
-            STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FlushReinit));
+            STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushReinit);
             fForcedFlush = true;
         }
@@ -1188 +1188 @@
              || pgmPoolIsPageLocked(pPage)
             )
-        &&  !(fReused = pgmPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault))
-        &&  !pgmPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
+        &&  !(fReused = pgmRZPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault))
+        &&  !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
     {
         /*
@@ -1196 +1196 @@
         if (!(pDis->fPrefix & (DISPREFIX_REP | DISPREFIX_REPNE)))
         {
-            rc = pgmPoolAccessPfHandlerSimple(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault, &fReused);
+            rc = pgmRZPoolAccessPfHandlerSimple(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault, &fReused);
             if (fReused)
                 goto flushPage;
@@ -1226 +1226 @@
             }
 
-            STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,Handled), a);
+            STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
             pgmUnlock(pVM);
             return rc;
@@ -1267 +1267 @@
             if (fValidStosd)
             {
-                rc = pgmPoolAccessPfHandlerSTOSD(pVM, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
-                STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,RepStosd), a);
+                rc = pgmRZPoolAccessPfHandlerSTOSD(pVM, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
+                STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZRepStosd, a);
                 pgmUnlock(pVM);
                 return rc;
@@ -1275 +1275 @@
 
         /* REP prefix, don't bother. */
-        STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,RepPrefix));
-        Log4(("pgmPoolAccessPfHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
+        STAM_COUNTER_INC(&pPool->StatMonitorPfRZRepPrefix);
+        Log4(("pgmRZPoolAccessPfHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
               pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->uOpcode, pDis->fPrefix));
         fNotReusedNotForking = true;
     }
 
-#if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) && defined(IN_RING0)
+# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) && defined(IN_RING0)
     /* E.g. Windows 7 x64 initializes page tables and touches some pages in the table during the process. This
      * leads to pgm pool trashing and an excessive amount of write faults due to page monitoring.
@@ -1289 +1289 @@
         &&  (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT || pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
         &&  (   fNotReusedNotForking
-             || (   !pgmPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault)
-                 && !pgmPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
+             || (   !pgmRZPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault)
+                 && !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
             )
        )
@@ -1342 +1342 @@
                         ||  rc == VERR_PAGE_NOT_PRESENT,
                         ("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", pvFault, rc));
-# ifdef VBOX_STRICT
+#  ifdef VBOX_STRICT
                 pPage->GCPtrDirtyFault = pvFault;
-# endif
-
-                STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), a);
+#  endif
+
+                STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, a);
                 pgmUnlock(pVM);
                 return rc;
@@ -1352 +1352 @@
         }
     }
-#endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
-
-    STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FlushModOverflow));
+# endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
+
+    STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushModOverflow);
 flushPage:
     /*
@@ -1364 +1364 @@
      * the reuse detection must be fixed.
      */
-    rc = pgmPoolAccessPfHandlerFlush(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
+    rc = pgmRZPoolAccessPfHandlerFlush(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
     if (    rc == VINF_EM_RAW_EMULATE_INSTR
         &&  fReused)
@@ -1372 +1372 @@
             rc = VINF_SUCCESS;  /* safe to restart the instruction. */
     }
-    STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
+    STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZFlushPage, a);
     pgmUnlock(pVM);
     return rc;
 }
 
-# endif /* !IN_RING3 */
+#endif /* !IN_RING3 */
 
 /**
@@ -1390 +1390 @@
 {
     PPGMPOOL        pPool = pVM->pgm.s.CTX_SUFF(pPool);
-    STAM_PROFILE_START(&pPool->StatMonitorR3, a);
+    STAM_PROFILE_START(&pPool->CTX_SUFF_Z(StatMonitor), a);
     PPGMPOOLPAGE    pPage = (PPGMPOOLPAGE)pvUser;
     LogFlow(("PGM_ALL_CB_DECL: GCPhys=%RGp %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
@@ -1397 +1397 @@
     NOREF(pvPhys); NOREF(pvBuf); NOREF(enmAccessType);
 
+    pgmLock(pVM);
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * Collect stats on the access.
+     */
+    AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Sizes)) == 19);
+    if (cbBuf <= 16 && cbBuf > 0)
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[cbBuf - 1]);
+    else if (cbBuf >= 17 && cbBuf < 32)
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[16]);
+    else if (cbBuf >= 32 && cbBuf < 64)
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[17]);
+    else if (cbBuf >= 64)
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[18]);
+
+    uint8_t cbAlign;
+    switch (pPage->enmKind)
+    {
+        default:
+            cbAlign = 7;
+            break;
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+            cbAlign = 3;
+            break;
+    }
+    AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Misaligned)) == 7);
+    if ((uint8_t)GCPhys & cbAlign)
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Misaligned)[((uint8_t)GCPhys & cbAlign) - 1]);
+#endif
+
     /*
      * Make sure the pool page wasn't modified by a different CPU.
      */
-    pgmLock(pVM);
     if (PHYS_PAGE_ADDRESS(GCPhys) == PHYS_PAGE_ADDRESS(pPage->GCPhys))
     {
@@ -1447 +1481 @@
         }
 
-        STAM_PROFILE_STOP_EX(&pPool->StatMonitorR3, &pPool->StatMonitorR3FlushPage, a);
+        STAM_PROFILE_STOP_EX(&pPool->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
     }
     else
@@ -4577 +4611 @@
     for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
     {
-        Assert(!(pShwPD->a[i].u & RT_BIT_32(9)));
         if (    pShwPD->a[i].n.u1Present
             &&  !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
@@ -4618 +4651 @@
 #endif
             {
-                Assert((pShwPD->a[i].u & (X86_PDE_PAE_MBZ_MASK_NX | UINT64_C(0x7ff0000000000200))) == 0);
+                Assert((pShwPD->a[i].u & (X86_PDE_PAE_MBZ_MASK_NX | UINT64_C(0x7ff0000000000000))) == 0);
                 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PAE_PG_MASK);
                 if (pSubPage)

trunk/src/VBox/VMM/VMMR3/PGM.cpp

@@ -2050 +2050 @@
         PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePagePDNPs,         "/PGM/CPU%u/RZ/InvalidatePage/PDNPs",       "The number of times PGMInvalidatePage() was called for a not present page directory.");
         PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePagePDOutOfSync,   "/PGM/CPU%u/RZ/InvalidatePage/PDOutOfSync", "The number of times PGMInvalidatePage() was called for an out of sync page directory.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePageSizeChanges,   "/PGM/CPU%u/RZ/InvalidatePage/SizeChanges", "The number of times PGMInvalidatePage() was called on a page size change (4KB <-> 2/4MB).");
         PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePageSkipped,       "/PGM/CPU%u/RZ/InvalidatePage/Skipped",     "The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3.");
         PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncSupervisor,     "/PGM/CPU%u/RZ/OutOfSync/SuperVisor",       "Number of traps due to pages out of sync (P) and times VerifyAccessSyncPage calls SyncPage.");
@@ -2097 +2098 @@
         PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePagePDNPs,         "/PGM/CPU%u/R3/InvalidatePage/PDNPs",       "The number of times PGMInvalidatePage() was called for a not present page directory.");
         PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePagePDOutOfSync,   "/PGM/CPU%u/R3/InvalidatePage/PDOutOfSync", "The number of times PGMInvalidatePage() was called for an out of sync page directory.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePageSizeChanges,   "/PGM/CPU%u/R3/InvalidatePage/SizeChanges", "The number of times PGMInvalidatePage() was called on a page size change (4KB <-> 2/4MB).");
         PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePageSkipped,       "/PGM/CPU%u/R3/InvalidatePage/Skipped",     "The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3.");
         PGM_REG_COUNTER(&pCpuStats->StatR3PageOutOfSyncSupervisor,     "/PGM/CPU%u/R3/OutOfSync/SuperVisor",       "Number of traps due to pages out of sync and times VerifyAccessSyncPage calls SyncPage.");
@@ -3188 +3190 @@
     pVCpu->pgm.s.pfnR3GstExit                 = pModeData->pfnR3GstExit;
     pVCpu->pgm.s.pfnR3GstGetPage              = pModeData->pfnR3GstGetPage;
-    Assert(pVCpu->pgm.s.pfnR3GstGetPage);
     pVCpu->pgm.s.pfnR3GstModifyPage           = pModeData->pfnR3GstModifyPage;
     pVCpu->pgm.s.pfnR3GstGetPDE               = pModeData->pfnR3GstGetPDE;
@@ -3197 +3198 @@
     pVCpu->pgm.s.pfnR0GstModifyPage           = pModeData->pfnR0GstModifyPage;
     pVCpu->pgm.s.pfnR0GstGetPDE               = pModeData->pfnR0GstGetPDE;
+    Assert(pVCpu->pgm.s.pfnR3GstGetPage);
 
     /* both */

trunk/src/VBox/VMM/VMMR3/PGMDbg.cpp

@@ -1321 +1321 @@
                                         Pde.b.u1PAT         ? "AT" : "--",
                                         Pde.b.u1NoExecute   ? "NX" : "--",
-                                        Pde.u & RT_BIT_64(9)            ? '1' : '0',
+                                        Pde.u & PGM_PDFLAGS_BIG_PAGE    ? 'b' : '-',
                                         Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
                                         Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
@@ -1350 +1350 @@
                                         Pde.n.u1CacheDisable? "CD" : "--",
                                         Pde.n.u1NoExecute   ? "NX" : "--",
-                                        Pde.u & RT_BIT_64(9)            ? '1' : '0',
+                                        Pde.u & PGM_PDFLAGS_BIG_PAGE    ? 'b' : '-',
                                         Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
                                         Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
@@ -1637 +1637 @@
                                         Pde.b.u1CacheDisable? "CD" : "--",
                                         Pde.b.u1PAT         ? "AT" : "--",
-                                        Pde.u & RT_BIT_32(9)            ? '1' : '0',
+                                        Pde.u & PGM_PDFLAGS_BIG_PAGE    ? 'b' : '-',
                                         Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
                                         Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
@@ -1658 +1658 @@
                                         Pde.n.u1WriteThru   ? "WT" : "--",
                                         Pde.n.u1CacheDisable? "CD" : "--",
-                                        Pde.u & RT_BIT_32(9)            ? '1' : '0',
+                                        Pde.u & PGM_PDFLAGS_BIG_PAGE    ? 'b' : '-',
                                         Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
                                         Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',

trunk/src/VBox/VMM/VMMR3/PGMPool.cpp

@@ -283 +283 @@
     rc = PGMR3HandlerPhysicalTypeRegister(pVM, PGMPHYSHANDLERKIND_WRITE,
                                           pgmPoolAccessHandler,
-                                          NULL, "pgmPoolAccessHandler", "pgmPoolAccessPfHandler",
-                                          NULL, "pgmPoolAccessHandler", "pgmPoolAccessPfHandler",
+                                          NULL, "pgmPoolAccessHandler", "pgmRZPoolAccessPfHandler",
+                                          NULL, "pgmPoolAccessHandler", "pgmRZPoolAccessPfHandler",
                                           "Guest Paging Access Handler",
                                           &pPool->hAccessHandlerType);
@@ -347 +347 @@
     STAM_REG(pVM, &pPool->StatTrackLinearRamSearches,   STAMTYPE_COUNTER,   "/PGM/Pool/Track/LinearRamSearches",    STAMUNIT_OCCURENCES, "The number of times we had to do linear ram searches.");
     STAM_REG(pVM, &pPool->StamTrackPhysExtAllocFailures,STAMTYPE_COUNTER,   "/PGM/Pool/Track/PhysExtAllocFailures", STAMUNIT_OCCURENCES, "The number of failing pgmPoolTrackPhysExtAlloc calls.");
-    STAM_REG(pVM, &pPool->StatMonitorRZ,                STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/RZ",                 STAMUNIT_TICKS_PER_CALL, "Profiling the RC/R0 access handler.");
-    STAM_REG(pVM, &pPool->StatMonitorRZEmulateInstr,    STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/EmulateInstr",    STAMUNIT_OCCURENCES,     "Times we've failed interpreting the instruction.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFlushPage,       STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/RZ/FlushPage",       STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFlushReinit,     STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/FlushReinit",     STAMUNIT_OCCURENCES,     "Times we've detected a page table reinit.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFlushModOverflow,STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/FlushOverflow",   STAMUNIT_OCCURENCES,     "Counting flushes for pages that are modified too often.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFork,            STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/Fork",            STAMUNIT_OCCURENCES,     "Times we've detected fork().");
-    STAM_REG(pVM, &pPool->StatMonitorRZHandled,         STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/RZ/Handled",         STAMUNIT_TICKS_PER_CALL, "Profiling the RC/R0 access we've handled (except REP STOSD).");
-    STAM_REG(pVM, &pPool->StatMonitorRZIntrFailPatch1,  STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/IntrFailPatch1",  STAMUNIT_OCCURENCES,     "Times we've failed interpreting a patch code instruction.");
-    STAM_REG(pVM, &pPool->StatMonitorRZIntrFailPatch2,  STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/IntrFailPatch2",  STAMUNIT_OCCURENCES,     "Times we've failed interpreting a patch code instruction during flushing.");
-    STAM_REG(pVM, &pPool->StatMonitorRZRepPrefix,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/RepPrefix",       STAMUNIT_OCCURENCES,     "The number of times we've seen rep prefixes we can't handle.");
-    STAM_REG(pVM, &pPool->StatMonitorRZRepStosd,        STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/RZ/RepStosd",        STAMUNIT_TICKS_PER_CALL, "Profiling the REP STOSD cases we've handled.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFaultPT,         STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/Fault/PT",        STAMUNIT_OCCURENCES,     "Nr of handled PT faults.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFaultPD,         STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/Fault/PD",        STAMUNIT_OCCURENCES,     "Nr of handled PD faults.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFaultPDPT,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/Fault/PDPT",      STAMUNIT_OCCURENCES,     "Nr of handled PDPT faults.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFaultPML4,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/Fault/PML4",      STAMUNIT_OCCURENCES,     "Nr of handled PML4 faults.");
-    STAM_REG(pVM, &pPool->StatMonitorR3,                STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/R3",                 STAMUNIT_TICKS_PER_CALL, "Profiling the R3 access handler.");
-    STAM_REG(pVM, &pPool->StatMonitorR3EmulateInstr,    STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/EmulateInstr",    STAMUNIT_OCCURENCES,     "Times we've failed interpreting the instruction.");
-    STAM_REG(pVM, &pPool->StatMonitorR3FlushPage,       STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/R3/FlushPage",       STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the R3 access handler.");
-    STAM_REG(pVM, &pPool->StatMonitorR3FlushReinit,     STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/FlushReinit",     STAMUNIT_OCCURENCES,     "Times we've detected a page table reinit.");
-    STAM_REG(pVM, &pPool->StatMonitorR3FlushModOverflow,STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/FlushOverflow",   STAMUNIT_OCCURENCES,     "Counting flushes for pages that are modified too often.");
-    STAM_REG(pVM, &pPool->StatMonitorR3Fork,            STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fork",            STAMUNIT_OCCURENCES,     "Times we've detected fork().");
-    STAM_REG(pVM, &pPool->StatMonitorR3Handled,         STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/R3/Handled",         STAMUNIT_TICKS_PER_CALL, "Profiling the R3 access we've handled (except REP STOSD).");
-    STAM_REG(pVM, &pPool->StatMonitorR3RepPrefix,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/RepPrefix",       STAMUNIT_OCCURENCES,     "The number of times we've seen rep prefixes we can't handle.");
-    STAM_REG(pVM, &pPool->StatMonitorR3RepStosd,        STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/R3/RepStosd",        STAMUNIT_TICKS_PER_CALL, "Profiling the REP STOSD cases we've handled.");
+
+    STAM_REG(pVM, &pPool->StatMonitorPfRZ,                STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/#PF",                 STAMUNIT_TICKS_PER_CALL, "Profiling the RC/R0 #PF access handler.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZEmulateInstr,    STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/#PF/EmulateInstr",    STAMUNIT_OCCURENCES,     "Times we've failed interpreting the instruction.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZFlushPage,       STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/#PF/FlushPage",       STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZFlushReinit,     STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/#PF/FlushReinit",     STAMUNIT_OCCURENCES,     "Times we've detected a page table reinit.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZFlushModOverflow,STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/#PF/FlushOverflow",   STAMUNIT_OCCURENCES,     "Counting flushes for pages that are modified too often.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZFork,            STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/#PF/Fork",            STAMUNIT_OCCURENCES,     "Times we've detected fork().");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZHandled,         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/#PF/Handled",         STAMUNIT_TICKS_PER_CALL, "Profiling the RC/R0 #PF access we've handled (except REP STOSD).");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZIntrFailPatch1,  STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/#PF/IntrFailPatch1",  STAMUNIT_OCCURENCES,     "Times we've failed interpreting a patch code instruction.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZIntrFailPatch2,  STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/#PF/IntrFailPatch2",  STAMUNIT_OCCURENCES,     "Times we've failed interpreting a patch code instruction during flushing.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZRepPrefix,       STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/#PF/RepPrefix",       STAMUNIT_OCCURENCES,     "The number of times we've seen rep prefixes we can't handle.");
+    STAM_REG(pVM, &pPool->StatMonitorPfRZRepStosd,        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/#PF/RepStosd",        STAMUNIT_TICKS_PER_CALL, "Profiling the REP STOSD cases we've handled.");
+
+    STAM_REG(pVM, &pPool->StatMonitorRZ,                  STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM",                 STAMUNIT_TICKS_PER_CALL, "Profiling the regular access handler.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFlushPage,         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/FlushPage",       STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the regular access handler.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[0],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size01",          STAMUNIT_OCCURENCES,     "Number of 1 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[1],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size02",          STAMUNIT_OCCURENCES,     "Number of 2 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[2],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size03",          STAMUNIT_OCCURENCES,     "Number of 3 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[3],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size04",          STAMUNIT_OCCURENCES,     "Number of 4 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[4],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size05",          STAMUNIT_OCCURENCES,     "Number of 5 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[5],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size06",          STAMUNIT_OCCURENCES,     "Number of 6 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[6],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size07",          STAMUNIT_OCCURENCES,     "Number of 7 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[7],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size08",          STAMUNIT_OCCURENCES,     "Number of 8 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[8],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size09",          STAMUNIT_OCCURENCES,     "Number of 9 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[9],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size0a",          STAMUNIT_OCCURENCES,     "Number of 10 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[10],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size0b",          STAMUNIT_OCCURENCES,     "Number of 11 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[11],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size0c",          STAMUNIT_OCCURENCES,     "Number of 12 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[12],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size0d",          STAMUNIT_OCCURENCES,     "Number of 13 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[13],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size0e",          STAMUNIT_OCCURENCES,     "Number of 14 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[14],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size0f",          STAMUNIT_OCCURENCES,     "Number of 15 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[15],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size10",          STAMUNIT_OCCURENCES,     "Number of 16 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[16],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size11-2f",       STAMUNIT_OCCURENCES,     "Number of 17-31 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[17],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size20-3f",       STAMUNIT_OCCURENCES,     "Number of 32-63 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZSizes[18],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Size40+",         STAMUNIT_OCCURENCES,     "Number of 64+ byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZMisaligned[0],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Misaligned1",     STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 1.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZMisaligned[1],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Misaligned2",     STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 2.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZMisaligned[2],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Misaligned3",     STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 3.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZMisaligned[3],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Misaligned4",     STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 4.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZMisaligned[4],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Misaligned5",     STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 5.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZMisaligned[5],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Misaligned6",     STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 6.");
+    STAM_REG(pVM, &pPool->aStatMonitorRZMisaligned[6],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/IEM/Misaligned7",     STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 7.");
+
+    STAM_REG(pVM, &pPool->StatMonitorRZFaultPT,           STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PT",            STAMUNIT_OCCURENCES,     "Nr of handled PT faults.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFaultPD,           STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PD",            STAMUNIT_OCCURENCES,     "Nr of handled PD faults.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFaultPDPT,         STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PDPT",          STAMUNIT_OCCURENCES,     "Nr of handled PDPT faults.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFaultPML4,         STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PML4",          STAMUNIT_OCCURENCES,     "Nr of handled PML4 faults.");
+
+    STAM_REG(pVM, &pPool->StatMonitorR3,                  STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3",                     STAMUNIT_TICKS_PER_CALL, "Profiling the R3 access handler.");
+    STAM_REG(pVM, &pPool->StatMonitorR3FlushPage,         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/FlushPage",           STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the R3 access handler.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[0],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size01",              STAMUNIT_OCCURENCES,     "Number of 1 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[1],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size02",              STAMUNIT_OCCURENCES,     "Number of 2 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[2],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size03",              STAMUNIT_OCCURENCES,     "Number of 3 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[3],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size04",              STAMUNIT_OCCURENCES,     "Number of 4 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[4],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size05",              STAMUNIT_OCCURENCES,     "Number of 5 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[5],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size06",              STAMUNIT_OCCURENCES,     "Number of 6 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[6],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size07",              STAMUNIT_OCCURENCES,     "Number of 7 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[7],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size08",              STAMUNIT_OCCURENCES,     "Number of 8 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[8],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size09",              STAMUNIT_OCCURENCES,     "Number of 9 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[9],         STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size0a",              STAMUNIT_OCCURENCES,     "Number of 10 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[10],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size0b",              STAMUNIT_OCCURENCES,     "Number of 11 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[11],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size0c",              STAMUNIT_OCCURENCES,     "Number of 12 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[12],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size0d",              STAMUNIT_OCCURENCES,     "Number of 13 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[13],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size0e",              STAMUNIT_OCCURENCES,     "Number of 14 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[14],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size0f",              STAMUNIT_OCCURENCES,     "Number of 15 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[15],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size10",              STAMUNIT_OCCURENCES,     "Number of 16 byte accesses (R3).");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[16],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size11-2f",           STAMUNIT_OCCURENCES,     "Number of 17-31 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[17],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size20-3f",           STAMUNIT_OCCURENCES,     "Number of 32-63 byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Sizes[18],        STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Size40+",             STAMUNIT_OCCURENCES,     "Number of 64+ byte accesses.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Misaligned[0],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Misaligned1",         STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 1 in R3.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Misaligned[1],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Misaligned2",         STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 2 in R3.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Misaligned[2],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Misaligned3",         STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 3 in R3.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Misaligned[3],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Misaligned4",         STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 4 in R3.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Misaligned[4],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Misaligned5",         STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 5 in R3.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Misaligned[5],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Misaligned6",         STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 6 in R3.");
+    STAM_REG(pVM, &pPool->aStatMonitorR3Misaligned[6],    STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Misaligned7",         STAMUNIT_OCCURENCES,     "Number of misaligned access with offset 7 in R3.");
+
     STAM_REG(pVM, &pPool->StatMonitorR3FaultPT,         STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PT",        STAMUNIT_OCCURENCES,     "Nr of handled PT faults.");
     STAM_REG(pVM, &pPool->StatMonitorR3FaultPD,         STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PD",        STAMUNIT_OCCURENCES,     "Nr of handled PD faults.");
     STAM_REG(pVM, &pPool->StatMonitorR3FaultPDPT,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PDPT",      STAMUNIT_OCCURENCES,     "Nr of handled PDPT faults.");
     STAM_REG(pVM, &pPool->StatMonitorR3FaultPML4,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PML4",      STAMUNIT_OCCURENCES,     "Nr of handled PML4 faults.");
-    STAM_REG(pVM, &pPool->StatMonitorR3Async,           STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Async",           STAMUNIT_OCCURENCES,     "Times we're called in an async thread and need to flush.");
+
     STAM_REG(pVM, &pPool->cModifiedPages,               STAMTYPE_U16,       "/PGM/Pool/Monitor/cModifiedPages",     STAMUNIT_PAGES,          "The current cModifiedPages value.");
     STAM_REG(pVM, &pPool->cModifiedPagesHigh,           STAMTYPE_U16_RESET, "/PGM/Pool/Monitor/cModifiedPagesHigh", STAMUNIT_PAGES,          "The high watermark for cModifiedPages.");

trunk/src/VBox/VMM/include/PGMGstDefs.h

@@ -143 +143 @@
 # define GST_GET_PDE_SHW_FLAGS(pVCpu, Pde)      ((Pde).u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A))
 # define GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, Pde) \
-    ((Pde).u & (X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_A))
+    ( ((Pde).u & (X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_A)) | PGM_PDFLAGS_BIG_PAGE )
 # define GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, Pde) \
     ((Pde).u & (X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_A | X86_PDE4M_D | X86_PDE4M_G))
@@ -183 +183 @@
 # define GST_GET_PTE_SHW_FLAGS(pVCpu, Pte)      ((Pte).u & (pVCpu)->pgm.s.fGst64ShadowedPteMask )
 # define GST_GET_PDE_SHW_FLAGS(pVCpu, Pde)      ((Pde).u & (pVCpu)->pgm.s.fGst64ShadowedPdeMask )
-# define GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, Pde)  ((Pde).u & (pVCpu)->pgm.s.fGst64ShadowedBigPdeMask )
+# define GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, Pde)  ( ((Pde).u & (pVCpu)->pgm.s.fGst64ShadowedBigPdeMask ) | PGM_PDFLAGS_BIG_PAGE)
 # define GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, Pde)  ((Pde).u & (pVCpu)->pgm.s.fGst64ShadowedBigPde4PteMask )
 

trunk/src/VBox/VMM/include/PGMInternal.h

@@ -178 +178 @@
  * the page directory entries.
  * @{ */
+/** Indicates the original entry was a big page.
+ * @remarks This is currently only used for statistics and can be recycled.  */
+#define PGM_PDFLAGS_BIG_PAGE            RT_BIT_64(9)
 /** Mapping (hypervisor allocated pagetable). */
 #define PGM_PDFLAGS_MAPPING             RT_BIT_64(10)
@@ -2496 +2499 @@
     /** The number of failing pgmPoolTrackPhysExtAlloc calls. */
     STAMCOUNTER                 StamTrackPhysExtAllocFailures;
-    /** Profiling the RC/R0 access handler. */
+
+    /** Profiling the RC/R0 \#PF access handler. */
+    STAMPROFILE                 StatMonitorPfRZ;
+    /** Profiling the RC/R0 access we've handled (except REP STOSD). */
+    STAMPROFILE                 StatMonitorPfRZHandled;
+    /** Times we've failed interpreting the instruction. */
+    STAMCOUNTER                 StatMonitorPfRZEmulateInstr;
+    /** Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler. */
+    STAMPROFILE                 StatMonitorPfRZFlushPage;
+    /* Times we've detected a page table reinit. */
+    STAMCOUNTER                 StatMonitorPfRZFlushReinit;
+    /** Counting flushes for pages that are modified too often. */
+    STAMCOUNTER                 StatMonitorPfRZFlushModOverflow;
+    /** Times we've detected fork(). */
+    STAMCOUNTER                 StatMonitorPfRZFork;
+    /** Times we've failed interpreting a patch code instruction. */
+    STAMCOUNTER                 StatMonitorPfRZIntrFailPatch1;
+    /** Times we've failed interpreting a patch code instruction during flushing. */
+    STAMCOUNTER                 StatMonitorPfRZIntrFailPatch2;
+    /** The number of times we've seen rep prefixes we can't handle. */
+    STAMCOUNTER                 StatMonitorPfRZRepPrefix;
+    /** Profiling the REP STOSD cases we've handled. */
+    STAMPROFILE                 StatMonitorPfRZRepStosd;
+
+    /** Profiling the R0/RC regular access handler. */
     STAMPROFILE                 StatMonitorRZ;
-    /** Times we've failed interpreting the instruction. */
-    STAMCOUNTER                 StatMonitorRZEmulateInstr;
-    /** Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler. */
+    /** Profiling the pgmPoolFlushPage calls made from the regular access handler in R0/RC. */
     STAMPROFILE                 StatMonitorRZFlushPage;
-    /* Times we've detected a page table reinit. */
-    STAMCOUNTER                 StatMonitorRZFlushReinit;
-    /** Counting flushes for pages that are modified too often. */
-    STAMCOUNTER                 StatMonitorRZFlushModOverflow;
-    /** Times we've detected fork(). */
-    STAMCOUNTER                 StatMonitorRZFork;
-    /** Profiling the RC/R0 access we've handled (except REP STOSD). */
-    STAMPROFILE                 StatMonitorRZHandled;
-    /** Times we've failed interpreting a patch code instruction. */
-    STAMCOUNTER                 StatMonitorRZIntrFailPatch1;
-    /** Times we've failed interpreting a patch code instruction during flushing. */
-    STAMCOUNTER                 StatMonitorRZIntrFailPatch2;
-    /** The number of times we've seen rep prefixes we can't handle. */
-    STAMCOUNTER                 StatMonitorRZRepPrefix;
-    /** Profiling the REP STOSD cases we've handled. */
-    STAMPROFILE                 StatMonitorRZRepStosd;
+    /** Per access size counts indexed by size minus 1, last for larger. */
+    STAMCOUNTER                 aStatMonitorRZSizes[16+3];
+    /** Missaligned access counts indexed by offset - 1. */
+    STAMCOUNTER                 aStatMonitorRZMisaligned[7];
+
     /** Nr of handled PT faults. */
     STAMCOUNTER                 StatMonitorRZFaultPT;
@@ -2529 +2543 @@
     /** Profiling the R3 access handler. */
     STAMPROFILE                 StatMonitorR3;
-    /** Times we've failed interpreting the instruction. */
-    STAMCOUNTER                 StatMonitorR3EmulateInstr;
     /** Profiling the pgmPoolFlushPage calls made from the R3 access handler. */
     STAMPROFILE                 StatMonitorR3FlushPage;
-    /* Times we've detected a page table reinit. */
-    STAMCOUNTER                 StatMonitorR3FlushReinit;
-    /** Counting flushes for pages that are modified too often. */
-    STAMCOUNTER                 StatMonitorR3FlushModOverflow;
-    /** Times we've detected fork(). */
-    STAMCOUNTER                 StatMonitorR3Fork;
-    /** Profiling the R3 access we've handled (except REP STOSD). */
-    STAMPROFILE                 StatMonitorR3Handled;
-    /** The number of times we've seen rep prefixes we can't handle. */
-    STAMCOUNTER                 StatMonitorR3RepPrefix;
-    /** Profiling the REP STOSD cases we've handled. */
-    STAMPROFILE                 StatMonitorR3RepStosd;
+    /** Per access size counts indexed by size minus 1, last for larger. */
+    STAMCOUNTER                 aStatMonitorR3Sizes[16+3];
+    /** Missaligned access counts indexed by offset - 1. */
+    STAMCOUNTER                 aStatMonitorR3Misaligned[7];
     /** Nr of handled PT faults. */
     STAMCOUNTER                 StatMonitorR3FaultPT;
@@ -2553 +2557 @@
     /** Nr of handled PML4 faults. */
     STAMCOUNTER                 StatMonitorR3FaultPML4;
-    /** The number of times we're called in an async thread an need to flush. */
-    STAMCOUNTER                 StatMonitorR3Async;
+
     /** Times we've called pgmPoolResetDirtyPages (and there were dirty page). */
     STAMCOUNTER                 StatResetDirtyPages;
@@ -2792 +2795 @@
     /** The effective X86_PTE_NX flag for the address. */
     bool            fEffectiveNX;
+    bool            afPadding1[2];
+    /** Effective flags thus far: RW, US, PWT, PCD, A, ~NX >> 63.
+     * The NX bit is inverted and shifted down 63 places to bit 0. */
+    uint32_t        fEffective;
 } PGMPTWALKCORE;
+
+/** @name PGMPTWALKCORE::fEffective bits.
+ * @{ */
+/** Effective execute bit (!NX).   */
+#define PGMPTWALK_EFF_X     UINT32_C(1)
+/** Effective write access bit. */
+#define PGMPTWALK_EFF_RW    X86_PTE_RW
+/** Effective user-mode access bit. */
+#define PGMPTWALK_EFF_US    X86_PTE_US
+/** Effective write through cache bit. */
+#define PGMPTWALK_EFF_PWT   X86_PTE_PWT
+/** Effective cache disabled bit. */
+#define PGMPTWALK_EFF_PCD   X86_PTE_PCD
+/** Effective accessed bit. */
+#define PGMPTWALK_EFF_A     X86_PTE_A
+/** The dirty bit of the final entry. */
+#define PGMPTWALK_EFF_D     X86_PTE_D
+/** The PAT bit of the final entry. */
+#define PGMPTWALK_EFF_PAT   X86_PTE_PAT
+/** The global bit of the final entry. */
+#define PGMPTWALK_EFF_G     X86_PTE_G
+/** @} */
 
 
@@ -3766 +3795 @@
     STAMCOUNTER StatRZInvalidatePagePDNPs;          /**< RC/R0: The number of times PGMInvalidatePage() was called for a not present page directory. */
     STAMCOUNTER StatRZInvalidatePagePDOutOfSync;    /**< RC/R0: The number of times PGMInvalidatePage() was called for an out of sync page directory. */
+    STAMCOUNTER StatRZInvalidatePageSizeChanges ;   /**< RC/R0: The number of times PGMInvalidatePage() was called on a page size change (4KB <-> 2/4MB). */
     STAMCOUNTER StatRZInvalidatePageSkipped;        /**< RC/R0: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. */
     STAMCOUNTER StatRZPageOutOfSyncUser;            /**< RC/R0: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. */
@@ -3813 +3843 @@
     STAMCOUNTER StatR3InvalidatePagePDMappings;     /**< R3: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). */
     STAMCOUNTER StatR3InvalidatePagePDOutOfSync;    /**< R3: The number of times PGMInvalidatePage() was called for an out of sync page directory. */
+    STAMCOUNTER StatR3InvalidatePageSizeChanges ;   /**< R3: The number of times PGMInvalidatePage() was called on a page size change (4KB <-> 2/4MB). */
     STAMCOUNTER StatR3InvalidatePageSkipped;        /**< R3: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. */
     STAMCOUNTER StatR3PageOutOfSyncUser;            /**< R3: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. */
@@ -4261 +4292 @@
 PGM_ALL_CB2_PROTO(FNPGMPHYSHANDLER) pgmPoolAccessHandler;
 #ifndef IN_RING3
-DECLEXPORT(FNPGMRZPHYSPFHANDLER)    pgmPoolAccessPfHandler;
+DECLEXPORT(FNPGMRZPHYSPFHANDLER)    pgmRZPoolAccessPfHandler;
 #endif
 

trunk/src/VBox/VMM/testcase/tstVMStruct.h

@@ -653 +653 @@
     GEN_CHECK_OFF(PGMCPU, pfnRCGstModifyPage);
     GEN_CHECK_OFF(PGMCPU, pfnRCGstGetPDE);
+    GEN_CHECK_OFF(PGMCPU, pfnR0GstGetPage);
+    GEN_CHECK_OFF(PGMCPU, pfnR0GstModifyPage);
+    GEN_CHECK_OFF(PGMCPU, pfnR0GstGetPDE);
     GEN_CHECK_OFF(PGMCPU, pfnR3BthRelocate);
     GEN_CHECK_OFF(PGMCPU, pfnR3BthSyncCR3);