Index: /trunk/src/VBox/VMM/VMMAll/IEMAll.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/IEMAll.cpp	(revision 105615)
+++ /trunk/src/VBox/VMM/VMMAll/IEMAll.cpp	(revision 105616)
@@ -642,5 +642,5 @@
  */
 template<bool const a_fGlobal>
-DECL_FORCE_INLINE(void) iemTlbLoadedLargePage(IEMTLB *pTlb, RTGCPTR uTagNoRev, bool f2MbLargePages)
+DECL_FORCE_INLINE(void) iemTlbLoadedLargePage(PVMCPUCC pVCpu, IEMTLB *pTlb, RTGCPTR uTagNoRev, bool f2MbLargePages)
 {
     if (a_fGlobal)
@@ -648,4 +648,9 @@
     else
         pTlb->cTlbNonGlobalLargePageCurLoads++;
+
+# ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
+    RTGCPTR const idxBit = IEMTLB_TAG_TO_EVEN_INDEX(uTagNoRev) + a_fGlobal;
+    ASMBitSet(pTlb->bmLargePage, idxBit);
+# endif
 
     AssertCompile(IEMTLB_CALC_TAG_NO_REV((RTGCPTR)0x8731U << GUEST_PAGE_SHIFT) == 0x8731U);
@@ -661,4 +666,6 @@
     if (uTagNoRev > pRange->uLastTag)
         pRange->uLastTag = uTagNoRev;
+
+    RT_NOREF_PV(pVCpu);
 }
 #endif
@@ -832,4 +839,9 @@
     AssertCompile(IEMTLB_ENTRY_COUNT >= 16); /* prefetching + unroll assumption */
 
+    if (a_fGlobal)
+        pTlb->cTlbInvlPgLargeGlobal += 1;
+    if (a_fNonGlobal)
+        pTlb->cTlbInvlPgLargeNonGlobal += 1;
+
     /*
      * Set up the scan.
@@ -845,9 +857,15 @@
      * relevant host architectures.
      */
-    /** @todo benchmark this code from the guest side.   */
+    /** @todo benchmark this code from the guest side. */
     bool const      fPartialScan = IEMTLB_ENTRY_COUNT > (a_f2MbLargePage ? 512 : 1024);
+#ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
+    uintptr_t       idxBitmap    = fPartialScan ? IEMTLB_TAG_TO_EVEN_INDEX(GCPtrTag) / 64 : 0;
+    uintptr_t const idxBitmapEnd = fPartialScan ? idxBitmap + ((a_f2MbLargePage ? 512 : 1024) * 2) / 64
+                                                : IEMTLB_ENTRY_COUNT * 2 / 64;
+#else
     uintptr_t       idxEven      = fPartialScan ? IEMTLB_TAG_TO_EVEN_INDEX(GCPtrTag) : 0;
-    MY_PREFETCH_256(&pTlb->aEntries[0 + !a_fNonGlobal]);
+    MY_PREFETCH_256(&pTlb->aEntries[idxEven + !a_fNonGlobal]);
     uintptr_t const idxEvenEnd   = fPartialScan ? idxEven + ((a_f2MbLargePage ? 512 : 1024) * 2) : IEMTLB_ENTRY_COUNT * 2;
+#endif
     RTGCPTR const   GCPtrTagMask = fPartialScan ? ~(RTGCPTR)0
                                  : ~(RTGCPTR)(  (RT_BIT_32(a_f2MbLargePage ? 9 : 10) - 1U)
@@ -873,7 +891,174 @@
      * Do the scanning.
      */
+#ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
+    uint64_t const bmMask  = a_fGlobal && a_fNonGlobal ? UINT64_MAX
+                           : a_fGlobal ? UINT64_C(0xaaaaaaaaaaaaaaaa) : UINT64_C(0x5555555555555555);
+    /* Scan bitmap entries (64 bits at the time): */
+    for (;;)
+    {
+# if 1
+        uint64_t bmEntry = pTlb->bmLargePage[idxBitmap] & bmMask;
+        if (bmEntry)
+        {
+            /* Scan the non-zero 64-bit value in groups of 8 bits: */
+            uint64_t  bmToClear = 0;
+            uintptr_t idxEven   = idxBitmap * 64;
+            uint32_t  idxTag    = 0;
+            for (;;)
+            {
+                if (bmEntry & 0xff)
+                {
+#  define ONE_PAIR(a_idxTagIter, a_idxEvenIter, a_bmNonGlobal, a_bmGlobal) \
+                        if (a_fNonGlobal) \
+                        { \
+                            if (bmEntry & a_bmNonGlobal) \
+                            { \
+                                Assert(pTlb->aEntries[a_idxEvenIter].fFlagsAndPhysRev & IEMTLBE_F_PT_LARGE_PAGE); \
+                                if ((pTlb->aEntries[a_idxEvenIter].uTag & GCPtrTagMask) == (GCPtrTag + a_idxTagIter)) \
+                                { \
+                                    IEMTLBTRACE_LARGE_EVICT_SLOT(pVCpu, GCPtrTag + a_idxTagIter, \
+                                                                 pTlb->aEntries[a_idxEvenIter].GCPhys, \
+                                                                 a_idxEvenIter, a_fDataTlb); \
+                                    pTlb->aEntries[a_idxEvenIter].uTag = 0; \
+                                    bmToClearSub8 |= a_bmNonGlobal; \
+                                } \
+                            } \
+                            else \
+                                Assert(   !(pTlb->aEntries[a_idxEvenIter].fFlagsAndPhysRev & IEMTLBE_F_PT_LARGE_PAGE)\
+                                       ||    (pTlb->aEntries[a_idxEvenIter].uTag & IEMTLB_REVISION_MASK) \
+                                          != (GCPtrTag & IEMTLB_REVISION_MASK)); \
+                        } \
+                        if (a_fGlobal) \
+                        { \
+                            if (bmEntry & a_bmGlobal) \
+                            {  \
+                                Assert(pTlb->aEntries[a_idxEvenIter + 1].fFlagsAndPhysRev & IEMTLBE_F_PT_LARGE_PAGE); \
+                                if ((pTlb->aEntries[a_idxEvenIter + 1].uTag & GCPtrTagMask) == (GCPtrTagGlob + a_idxTagIter)) \
+                                { \
+                                    IEMTLBTRACE_LARGE_EVICT_SLOT(pVCpu, GCPtrTagGlob + a_idxTagIter, \
+                                                                 pTlb->aEntries[a_idxEvenIter + 1].GCPhys, \
+                                                                 a_idxEvenIter + 1, a_fDataTlb); \
+                                    pTlb->aEntries[a_idxEvenIter + 1].uTag = 0; \
+                                    bmToClearSub8 |= a_bmGlobal; \
+                                } \
+                            } \
+                            else \
+                                Assert(   !(pTlb->aEntries[a_idxEvenIter + 1].fFlagsAndPhysRev & IEMTLBE_F_PT_LARGE_PAGE)\
+                                       ||    (pTlb->aEntries[a_idxEvenIter + 1].uTag & IEMTLB_REVISION_MASK) \
+                                          != (GCPtrTagGlob & IEMTLB_REVISION_MASK)); \
+                        }
+                    uint64_t bmToClearSub8 = 0;
+                    ONE_PAIR(idxTag + 0, idxEven + 0, 0x01, 0x02)
+                    ONE_PAIR(idxTag + 1, idxEven + 2, 0x04, 0x08)
+                    ONE_PAIR(idxTag + 2, idxEven + 4, 0x10, 0x20)
+                    ONE_PAIR(idxTag + 3, idxEven + 6, 0x40, 0x80)
+                    bmToClear |= bmToClearSub8 << (idxTag * 2);
+#  undef ONE_PAIR
+                }
+
+                /* advance to the next 8 bits. */
+                bmEntry >>= 8;
+                if (!bmEntry)
+                    break;
+                idxEven  += 8;
+                idxTag   += 4;
+            }
+
+            /* Clear the large page flags we covered. */
+            pTlb->bmLargePage[idxBitmap] &= ~bmToClear;
+        }
+# else
+        uint64_t const bmEntry = pTlb->bmLargePage[idxBitmap] & bmMask;
+        if (bmEntry)
+        {
+            /* Scan the non-zero 64-bit value completely unrolled: */
+            uintptr_t const idxEven   = idxBitmap * 64;
+            uint64_t        bmToClear = 0;
+#  define ONE_PAIR(a_idxTagIter, a_idxEvenIter, a_bmNonGlobal, a_bmGlobal) \
+                if (a_fNonGlobal) \
+                { \
+                    if (bmEntry & a_bmNonGlobal) \
+                    { \
+                        Assert(pTlb->aEntries[a_idxEvenIter].fFlagsAndPhysRev & IEMTLBE_F_PT_LARGE_PAGE); \
+                        if ((pTlb->aEntries[a_idxEvenIter].uTag & GCPtrTagMask) == (GCPtrTag + a_idxTagIter)) \
+                        { \
+                            IEMTLBTRACE_LARGE_EVICT_SLOT(pVCpu, GCPtrTag + a_idxTagIter, \
+                                                         pTlb->aEntries[a_idxEvenIter].GCPhys, \
+                                                         a_idxEvenIter, a_fDataTlb); \
+                            pTlb->aEntries[a_idxEvenIter].uTag = 0; \
+                            bmToClear |= a_bmNonGlobal; \
+                        } \
+                    } \
+                    else \
+                        Assert(   !(pTlb->aEntriqes[a_idxEvenIter].fFlagsAndPhysRev & IEMTLBE_F_PT_LARGE_PAGE)\
+                               ||    (pTlb->aEntries[a_idxEvenIter].uTag & IEMTLB_REVISION_MASK) \
+                                  != (GCPtrTag & IEMTLB_REVISION_MASK)); \
+                } \
+                if (a_fGlobal) \
+                { \
+                    if (bmEntry & a_bmGlobal) \
+                    {  \
+                        Assert(pTlb->aEntries[a_idxEvenIter + 1].fFlagsAndPhysRev & IEMTLBE_F_PT_LARGE_PAGE); \
+                        if ((pTlb->aEntries[a_idxEvenIter + 1].uTag & GCPtrTagMask) == (GCPtrTagGlob + a_idxTagIter)) \
+                        { \
+                            IEMTLBTRACE_LARGE_EVICT_SLOT(pVCpu, GCPtrTagGlob + a_idxTagIter, \
+                                                         pTlb->aEntries[a_idxEvenIter + 1].GCPhys, \
+                                                         a_idxEvenIter + 1, a_fDataTlb); \
+                            pTlb->aEntries[a_idxEvenIter + 1].uTag = 0; \
+                            bmToClear |= a_bmGlobal; \
+                        } \
+                    } \
+                    else \
+                        Assert(   !(pTlb->aEntries[a_idxEvenIter + 1].fFlagsAndPhysRev & IEMTLBE_F_PT_LARGE_PAGE)\
+                               ||    (pTlb->aEntries[a_idxEvenIter + 1].uTag & IEMTLB_REVISION_MASK) \
+                                  != (GCPtrTagGlob & IEMTLB_REVISION_MASK)); \
+                } ((void)0)
+#  define FOUR_PAIRS(a_iByte, a_cShift) \
+                ONE_PAIR(0 + a_iByte * 4, idxEven + 0 + a_iByte * 8, UINT64_C(0x01) << a_cShift, UINT64_C(0x02) << a_cShift); \
+                ONE_PAIR(1 + a_iByte * 4, idxEven + 2 + a_iByte * 8, UINT64_C(0x04) << a_cShift, UINT64_C(0x08) << a_cShift); \
+                ONE_PAIR(2 + a_iByte * 4, idxEven + 4 + a_iByte * 8, UINT64_C(0x10) << a_cShift, UINT64_C(0x20) << a_cShift); \
+                ONE_PAIR(3 + a_iByte * 4, idxEven + 6 + a_iByte * 8, UINT64_C(0x40) << a_cShift, UINT64_C(0x80) << a_cShift)
+            if (bmEntry & (uint32_t)UINT16_MAX)
+            {
+                FOUR_PAIRS(0,  0);
+                FOUR_PAIRS(1,  8);
+            }
+            if (bmEntry & ((uint32_t)UINT16_MAX << 16))
+            {
+                FOUR_PAIRS(2, 16);
+                FOUR_PAIRS(3, 24);
+            }
+            if (bmEntry & ((uint64_t)UINT16_MAX << 32))
+            {
+                FOUR_PAIRS(4, 32);
+                FOUR_PAIRS(5, 40);
+            }
+            if (bmEntry & ((uint64_t)UINT16_MAX << 16))
+            {
+                FOUR_PAIRS(6, 48);
+                FOUR_PAIRS(7, 56);
+            }
+#  undef FOUR_PAIRS
+
+            /* Clear the large page flags we covered. */
+            pTlb->bmLargePage[idxBitmap] &= ~bmToClear;
+        }
+# endif
+
+        /* advance */
+        idxBitmap++;
+        if (idxBitmap >= idxBitmapEnd)
+            break;
+        if (a_fNonGlobal)
+            GCPtrTag     += 32;
+        if (a_fGlobal)
+            GCPtrTagGlob += 32;
+    }
+
+#else  /* !IEMTLB_WITH_LARGE_PAGE_BITMAP */
+
     for (; idxEven < idxEvenEnd; idxEven += 8)
     {
-#define ONE_ITERATION(a_idxEvenIter) \
+# define ONE_ITERATION(a_idxEvenIter) \
             if (a_fNonGlobal)  \
             { \
@@ -909,6 +1094,7 @@
         ONE_ITERATION(idxEven + 4)
         ONE_ITERATION(idxEven + 6)
-#undef ONE_ITERATION
-    }
+# undef ONE_ITERATION
+    }
+#endif /* !IEMTLB_WITH_LARGE_PAGE_BITMAP */
 }
 
@@ -942,4 +1128,6 @@
 DECLINLINE(void) iemTlbInvalidatePageWorker(PVMCPUCC pVCpu, IEMTLB *pTlb, RTGCPTR GCPtrTag, uintptr_t idxEven) RT_NOEXCEPT
 {
+    pTlb->cTlbInvlPg += 1;
+
     /*
      * Flush the entry pair.
@@ -1271,7 +1459,7 @@
         {
             /* likely when executing lots of code, otherwise unlikely */
-# ifdef IEM_WITH_TLB_STATISTICS
+#  ifdef IEM_WITH_TLB_STATISTICS
             pVCpu->iem.s.CodeTlb.cTlbCoreHits++;
-# endif
+#  endif
             Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_ACCESSED));
 
@@ -1321,8 +1509,8 @@
             else
             {
-#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+#  ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
                 /** @todo Nested VMX: Need to handle EPT violation/misconfig here?  OF COURSE! */
                 Assert(!(Walk.fFailed & PGM_WALKFAIL_EPT));
-#endif
+#  endif
                 Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrFirst, rc));
                 iemRaisePageFaultJmp(pVCpu, GCPtrFirst, 1, IEM_ACCESS_INSTRUCTION, rc);
@@ -1336,5 +1524,9 @@
                 pTlbe->uTag         = uTagNoRev | pVCpu->iem.s.CodeTlb.uTlbRevision;
                 if (WalkFast.fInfo & PGM_WALKINFO_BIG_PAGE)
-                    iemTlbLoadedLargePage<false>(&pVCpu->iem.s.CodeTlb, uTagNoRev, RT_BOOL(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE));
+                    iemTlbLoadedLargePage<false>(pVCpu, &pVCpu->iem.s.CodeTlb, uTagNoRev, RT_BOOL(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE));
+#  ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
+                else
+                    ASMBitClear(pVCpu->iem.s.CodeTlb.bmLargePage, IEMTLB_TAG_TO_EVEN_INDEX(uTagNoRev));
+#  endif
             }
             else
@@ -1343,5 +1535,9 @@
                 pTlbe->uTag         = uTagNoRev | pVCpu->iem.s.CodeTlb.uTlbRevisionGlobal;
                 if (WalkFast.fInfo & PGM_WALKINFO_BIG_PAGE)
-                    iemTlbLoadedLargePage<true>(&pVCpu->iem.s.CodeTlb, uTagNoRev, RT_BOOL(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE));
+                    iemTlbLoadedLargePage<true>(pVCpu, &pVCpu->iem.s.CodeTlb, uTagNoRev, RT_BOOL(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE));
+#  ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
+                else
+                    ASMBitClear(pVCpu->iem.s.CodeTlb.bmLargePage, IEMTLB_TAG_TO_EVEN_INDEX(uTagNoRev) + 1);
+#  endif
             }
             pTlbe->fFlagsAndPhysRev = (~WalkFast.fEffective & (X86_PTE_US | X86_PTE_RW | X86_PTE_D | X86_PTE_A))
@@ -6802,5 +6998,5 @@
 # ifdef IEM_WITH_TLB_STATISTICS
         pVCpu->iem.s.DataTlb.cTlbCoreHits++;
-#endif
+# endif
 
         /* If the page is either supervisor only or non-writable, we need to do
@@ -6891,5 +7087,9 @@
                 pTlbe->uTag         = uTagNoRev | pVCpu->iem.s.DataTlb.uTlbRevision;
                 if (WalkFast.fInfo & PGM_WALKINFO_BIG_PAGE)
-                    iemTlbLoadedLargePage<false>(&pVCpu->iem.s.DataTlb, uTagNoRev, RT_BOOL(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE));
+                    iemTlbLoadedLargePage<false>(pVCpu, &pVCpu->iem.s.DataTlb, uTagNoRev, RT_BOOL(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE));
+# ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
+                else
+                    ASMBitClear(pVCpu->iem.s.DataTlb.bmLargePage, IEMTLB_TAG_TO_EVEN_INDEX(uTagNoRev));
+# endif
             }
             else
@@ -6898,5 +7098,9 @@
                 pTlbe->uTag         = uTagNoRev | pVCpu->iem.s.DataTlb.uTlbRevisionGlobal;
                 if (WalkFast.fInfo & PGM_WALKINFO_BIG_PAGE)
-                    iemTlbLoadedLargePage<true>(&pVCpu->iem.s.DataTlb, uTagNoRev, RT_BOOL(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE));
+                    iemTlbLoadedLargePage<true>(pVCpu, &pVCpu->iem.s.DataTlb, uTagNoRev, RT_BOOL(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE));
+# ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
+                else
+                    ASMBitClear(pVCpu->iem.s.DataTlb.bmLargePage, IEMTLB_TAG_TO_EVEN_INDEX(uTagNoRev) + 1);
+# endif
             }
         }
@@ -7295,5 +7499,9 @@
                 pTlbe->uTag         = uTagNoRev | pVCpu->iem.s.DataTlb.uTlbRevision;
                 if (WalkFast.fInfo & PGM_WALKINFO_BIG_PAGE)
-                    iemTlbLoadedLargePage<false>(&pVCpu->iem.s.DataTlb, uTagNoRev, RT_BOOL(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE));
+                    iemTlbLoadedLargePage<false>(pVCpu, &pVCpu->iem.s.DataTlb, uTagNoRev, RT_BOOL(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE));
+# ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
+                else
+                    ASMBitClear(pVCpu->iem.s.DataTlb.bmLargePage, IEMTLB_TAG_TO_EVEN_INDEX(uTagNoRev));
+# endif
             }
             else
@@ -7305,5 +7513,9 @@
                 pTlbe->uTag         = uTagNoRev | pVCpu->iem.s.DataTlb.uTlbRevisionGlobal;
                 if (WalkFast.fInfo & PGM_WALKINFO_BIG_PAGE)
-                    iemTlbLoadedLargePage<true>(&pVCpu->iem.s.DataTlb, uTagNoRev, RT_BOOL(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE));
+                    iemTlbLoadedLargePage<true>(pVCpu, &pVCpu->iem.s.DataTlb, uTagNoRev, RT_BOOL(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE));
+# ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
+                else
+                    ASMBitClear(pVCpu->iem.s.DataTlb.bmLargePage, IEMTLB_TAG_TO_EVEN_INDEX(uTagNoRev) + 1);
+# endif
             }
         }
Index: /trunk/src/VBox/VMM/VMMR3/IEMR3.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/IEMR3.cpp	(revision 105615)
+++ /trunk/src/VBox/VMM/VMMR3/IEMR3.cpp	(revision 105616)
@@ -387,4 +387,11 @@
                         "Code TLB non-global large page range: last tag",   "/IEM/CPU%u/Tlb/Code/LargePageNonGlobalLastTag", idCpu);
 
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.cTlbInvlPg, STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Code TLB page invalidation requests",          "/IEM/CPU%u/Tlb/Code/InvlPg", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.cTlbInvlPgLargeGlobal, STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Code TLB page invlpg scanning for global large pages",     "/IEM/CPU%u/Tlb/Code/InvlPg/LargeGlobal", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.cTlbInvlPgLargeNonGlobal, STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Code TLB page invlpg scanning for non-global large pages", "/IEM/CPU%u/Tlb/Code/InvlPg/LargeNonGlobal", idCpu);
+
         STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.cTlbCoreMisses,      STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
                         "Code TLB misses",                              "/IEM/CPU%u/Tlb/Code/Misses", idCpu);
@@ -474,4 +481,11 @@
         STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.NonGlobalLargePageRange.uLastTag, STAMTYPE_X64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
                         "Data TLB non-global large page range: last tag",   "/IEM/CPU%u/Tlb/Data/LargePageNonGlobalLastTag", idCpu);
+
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.cTlbInvlPg, STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Data TLB page invalidation requests",          "/IEM/CPU%u/Tlb/Data/InvlPg", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.cTlbInvlPgLargeGlobal, STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Data TLB page invlpg scanning for global large pages",     "/IEM/CPU%u/Tlb/Data/InvlPg/LargeGlobal", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.cTlbInvlPgLargeNonGlobal, STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Data TLB page invlpg scanning for non-global large pages", "/IEM/CPU%u/Tlb/Data/InvlPg/LargeNonGlobal", idCpu);
 
         STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.cTlbCoreMisses,      STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
Index: /trunk/src/VBox/VMM/include/IEMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/IEMInternal.h	(revision 105615)
+++ /trunk/src/VBox/VMM/include/IEMInternal.h	(revision 105616)
@@ -558,4 +558,17 @@
 #endif
 
+/** Enable the large page bitmap TLB optimization.
+ *
+ * The idea here is to avoid scanning the full 32 KB (2MB pages, 2*512 TLB
+ * entries) or 64 KB (4MB pages, 2*1024 TLB entries) worth of TLB entries during
+ * invlpg when large pages are used, and instead just scan 128 or 256 bytes of
+ * the bmLargePage bitmap to determin which TLB entires that might be containing
+ * large pages and actually require checking.
+ *
+ * There is a good posibility of false positives since we currently don't clear
+ * the bitmap when flushing the TLB, but it should help reduce the workload when
+ * the large pages aren't fully loaded into the TLB in their entirity...
+ */
+#define IEMTLB_WITH_LARGE_PAGE_BITMAP
 
 /**
@@ -672,5 +685,12 @@
     uint32_t            cTlbPhysRevRollovers;
 
-    /*uint32_t            au32Padding[2];*/
+    /** Number of INVLPG (and similar) operations. */
+    uint32_t            cTlbInvlPg;
+    /** Subset of cTlbInvlPg that involved non-global large pages. */
+    uint32_t            cTlbInvlPgLargeNonGlobal;
+    /** Subset of cTlbInvlPg that involved global large pages. */
+    uint32_t            cTlbInvlPgLargeGlobal;
+
+    uint32_t            au32Padding[13];
 
     /** The TLB entries.
@@ -678,6 +698,14 @@
      * Odd  entries are for PTE.G=1 and uses uTlbRevisionGlobal. */
     IEMTLBENTRY         aEntries[IEMTLB_ENTRY_COUNT * 2];
+#ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
+    /** Bitmap tracking TLB entries for large pages.
+     * This duplicates IEMTLBE_F_PT_LARGE_PAGE for each TLB entry. */
+    uint64_t            bmLargePage[IEMTLB_ENTRY_COUNT * 2 / 64];
+#endif
 } IEMTLB;
 AssertCompileSizeAlignment(IEMTLB, 64);
+#ifdef IEMTLB_WITH_LARGE_PAGE_BITMAP
+AssertCompile(IEMTLB_ENTRY_COUNT >= 32 /* bmLargePage ASSUMPTION */);
+#endif
 /** The width (in bits) of the address portion of the TLB tag.   */
 #define IEMTLB_TAG_ADDR_WIDTH   36
@@ -862,8 +890,8 @@
     iemTlbTrace(a_pVCpu, kIemTlbTraceType_User3, a_u64Param1, a_u64Param2, a_bParam, a_u32Param)
 #else
+# define IEMTLBTRACE_USER0(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
 # define IEMTLBTRACE_USER1(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
 # define IEMTLBTRACE_USER2(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
 # define IEMTLBTRACE_USER3(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
-# define IEMTLBTRACE_USER4(a_pVCpu, a_u64Param1, a_u64Param2, a_u32Param, a_bParam) do { } while (0)
 #endif
 
