Changeset 102800 in vbox for trunk/src

Timestamp:

Jan 9, 2024 10:19:17 PM (13 months ago)

Author:

vboxsync

Message:

VMM/IEM: Stats, go straight for the safe fallback functions in the memory access helpers. bugref:10371

Location:

trunk/src/VBox/VMM

Files:

• VMMAll/IEMAllN8veRecompiler.cpp (modified) (33 diffs)
• VMMR3/IEMR3.cpp (modified) (1 diff)
• include/IEMInternal.h (modified) (2 diffs)
• include/IEMN8veRecompilerEmit.h (modified) (1 diff)

trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

@@ -108 +108 @@
 #if (defined(RT_ARCH_AMD64) && 1) || (defined(RT_ARCH_ARM64) && 1)
 # define IEMNATIVE_WITH_TLB_LOOKUP
+#endif
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP
+# define IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+#endif
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP
+# define IEMNATIVE_WITH_TLB_LOOKUP_STORE
+#endif
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP
+# define IEMNATIVE_WITH_TLB_LOOKUP_MAPPING
 #endif
 #ifdef IEMNATIVE_WITH_TLB_LOOKUP
@@ -1664 +1673 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg))
 {
-    return (uint64_t)iemMemFetchDataU8Jmp(pVCpu, iSegReg, GCPtrMem); /** @todo use iemMemFetchDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)iemMemFetchDataU8SafeJmp(pVCpu, iSegReg, GCPtrMem);
+#else
+    return (uint64_t)iemMemFetchDataU8Jmp(pVCpu, iSegReg, GCPtrMem);
+#endif
 }
 
@@ -1674 +1687 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg))
 {
-    return (uint64_t)(uint16_t)(int16_t)(int8_t)iemMemFetchDataU8Jmp(pVCpu, iSegReg, GCPtrMem); /** @todo use iemMemFetchDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)(uint16_t)(int16_t)(int8_t)iemMemFetchDataU8SafeJmp(pVCpu, iSegReg, GCPtrMem);
+#else
+    return (uint64_t)(uint16_t)(int16_t)(int8_t)iemMemFetchDataU8Jmp(pVCpu, iSegReg, GCPtrMem);
+#endif
 }
 
@@ -1684 +1701 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg))
 {
-    return (uint64_t)(uint32_t)(int32_t)(int8_t)iemMemFetchDataU8Jmp(pVCpu, iSegReg, GCPtrMem); /** @todo use iemMemFetchDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)(uint32_t)(int32_t)(int8_t)iemMemFetchDataU8SafeJmp(pVCpu, iSegReg, GCPtrMem);
+#else
+    return (uint64_t)(uint32_t)(int32_t)(int8_t)iemMemFetchDataU8Jmp(pVCpu, iSegReg, GCPtrMem);
+#endif
 }
 
@@ -1693 +1714 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg))
 {
-    return (uint64_t)(int64_t)(int8_t)iemMemFetchDataU8Jmp(pVCpu, iSegReg, GCPtrMem); /** @todo use iemMemFetchDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)(int64_t)(int8_t)iemMemFetchDataU8SafeJmp(pVCpu, iSegReg, GCPtrMem);
+#else
+    return (uint64_t)(int64_t)(int8_t)iemMemFetchDataU8Jmp(pVCpu, iSegReg, GCPtrMem);
+#endif
 }
 
@@ -1702 +1727 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg))
 {
-    return (uint64_t)iemMemFetchDataU16Jmp(pVCpu, iSegReg, GCPtrMem); /** @todo use iemMemFetchDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)iemMemFetchDataU16SafeJmp(pVCpu, iSegReg, GCPtrMem);
+#else
+    return (uint64_t)iemMemFetchDataU16Jmp(pVCpu, iSegReg, GCPtrMem);
+#endif
 }
 
@@ -1712 +1741 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg))
 {
-    return (uint64_t)(uint32_t)(int32_t)(int16_t)iemMemFetchDataU16Jmp(pVCpu, iSegReg, GCPtrMem); /** @todo use iemMemFetchDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)(uint32_t)(int32_t)(int16_t)iemMemFetchDataU16SafeJmp(pVCpu, iSegReg, GCPtrMem);
+#else
+    return (uint64_t)(uint32_t)(int32_t)(int16_t)iemMemFetchDataU16Jmp(pVCpu, iSegReg, GCPtrMem);
+#endif
 }
 
@@ -1722 +1755 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg))
 {
-    return (uint64_t)(int64_t)(int16_t)iemMemFetchDataU16Jmp(pVCpu, iSegReg, GCPtrMem); /** @todo use iemMemFetchDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)(int64_t)(int16_t)iemMemFetchDataU16SafeJmp(pVCpu, iSegReg, GCPtrMem);
+#else
+    return (uint64_t)(int64_t)(int16_t)iemMemFetchDataU16Jmp(pVCpu, iSegReg, GCPtrMem);
+#endif
 }
 
@@ -1731 +1768 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg))
 {
-    return (uint64_t)iemMemFetchDataU32Jmp(pVCpu, iSegReg, GCPtrMem); /** @todo use iemMemFetchDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)iemMemFetchDataU32SafeJmp(pVCpu, iSegReg, GCPtrMem);
+#else
+    return (uint64_t)iemMemFetchDataU32Jmp(pVCpu, iSegReg, GCPtrMem);
+#endif
 }
 
@@ -1741 +1782 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg))
 {
-    return (uint64_t)(int64_t)(int32_t)iemMemFetchDataU32Jmp(pVCpu, iSegReg, GCPtrMem); /** @todo use iemMemFetchDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)(int64_t)(int32_t)iemMemFetchDataU32SafeJmp(pVCpu, iSegReg, GCPtrMem);
+#else
+    return (uint64_t)(int64_t)(int32_t)iemMemFetchDataU32Jmp(pVCpu, iSegReg, GCPtrMem);
+#endif
 }
 
@@ -1750 +1795 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg))
 {
-    return iemMemFetchDataU64Jmp(pVCpu, iSegReg, GCPtrMem); /** @todo use iemMemFetchDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return iemMemFetchDataU64SafeJmp(pVCpu, iSegReg, GCPtrMem);
+#else
+    return iemMemFetchDataU64Jmp(pVCpu, iSegReg, GCPtrMem);
+#endif
 }
 
@@ -1759 +1808 @@
 IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value))
 {
-    iemMemStoreDataU8Jmp(pVCpu, iSegReg, GCPtrMem, u8Value); /** @todo use iemMemStoreDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_STORE
+    iemMemStoreDataU8SafeJmp(pVCpu, iSegReg, GCPtrMem, u8Value);
+#else
+    iemMemStoreDataU8Jmp(pVCpu, iSegReg, GCPtrMem, u8Value);
+#endif
 }
 
@@ -1768 +1821 @@
 IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value))
 {
-    iemMemStoreDataU16Jmp(pVCpu, iSegReg, GCPtrMem, u16Value); /** @todo use iemMemStoreDataU16SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_STORE
+    iemMemStoreDataU16SafeJmp(pVCpu, iSegReg, GCPtrMem, u16Value);
+#else
+    iemMemStoreDataU16Jmp(pVCpu, iSegReg, GCPtrMem, u16Value);
+#endif
 }
 
@@ -1777 +1834 @@
 IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value))
 {
-    iemMemStoreDataU32Jmp(pVCpu, iSegReg, GCPtrMem, u32Value); /** @todo use iemMemStoreDataU32SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_STORE
+    iemMemStoreDataU32SafeJmp(pVCpu, iSegReg, GCPtrMem, u32Value);
+#else
+    iemMemStoreDataU32Jmp(pVCpu, iSegReg, GCPtrMem, u32Value);
+#endif
 }
 
@@ -1786 +1847 @@
 IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value))
 {
-    iemMemStoreDataU64Jmp(pVCpu, iSegReg, GCPtrMem, u64Value); /** @todo use iemMemStoreDataU64SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_STORE
+    iemMemStoreDataU64SafeJmp(pVCpu, iSegReg, GCPtrMem, u64Value);
+#else
+    iemMemStoreDataU64Jmp(pVCpu, iSegReg, GCPtrMem, u64Value);
+#endif
 }
 
@@ -1895 +1960 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
 {
-    return (uint64_t)iemMemFlatFetchDataU8Jmp(pVCpu, GCPtrMem); /** @todo use iemMemFlatFetchDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)iemMemFetchDataU8SafeJmp(pVCpu, UINT8_MAX, GCPtrMem);
+#else
+    return (uint64_t)iemMemFlatFetchDataU8Jmp(pVCpu, GCPtrMem);
+#endif
 }
 
@@ -1906 +1975 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
 {
-    return (uint64_t)(uint16_t)(int16_t)(int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, GCPtrMem); /** @todo use iemMemFlatFetchDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)(uint16_t)(int16_t)(int8_t)iemMemFetchDataU8SafeJmp(pVCpu, UINT8_MAX, GCPtrMem);
+#else
+    return (uint64_t)(uint16_t)(int16_t)(int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, GCPtrMem);
+#endif
 }
 
@@ -1917 +1990 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
 {
-    return (uint64_t)(uint32_t)(int32_t)(int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, GCPtrMem); /** @todo use iemMemFlatFetchDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)(uint32_t)(int32_t)(int8_t)iemMemFetchDataU8SafeJmp(pVCpu, UINT8_MAX, GCPtrMem);
+#else
+    return (uint64_t)(uint32_t)(int32_t)(int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, GCPtrMem);
+#endif
 }
 
@@ -1927 +2004 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
 {
-    return (uint64_t)(int64_t)(int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, GCPtrMem); /** @todo use iemMemFlatFetchDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)(int64_t)(int8_t)iemMemFetchDataU8SafeJmp(pVCpu, UINT8_MAX, GCPtrMem);
+#else
+    return (uint64_t)(int64_t)(int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, GCPtrMem);
+#endif
 }
 
@@ -1937 +2018 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
 {
-    return (uint64_t)iemMemFlatFetchDataU16Jmp(pVCpu, GCPtrMem); /** @todo use iemMemFlatFetchDataU16SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)iemMemFetchDataU16SafeJmp(pVCpu, UINT8_MAX, GCPtrMem);
+#else
+    return (uint64_t)iemMemFlatFetchDataU16Jmp(pVCpu, GCPtrMem);
+#endif
 }
 
@@ -1948 +2033 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
 {
-    return (uint64_t)(uint32_t)(int32_t)(int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, GCPtrMem); /** @todo use iemMemFlatFetchDataU16SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)(uint32_t)(int32_t)(int16_t)iemMemFetchDataU16SafeJmp(pVCpu, UINT8_MAX, GCPtrMem);
+#else
+    return (uint64_t)(uint32_t)(int32_t)(int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, GCPtrMem);
+#endif
 }
 
@@ -1959 +2048 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
 {
-    return (uint64_t)(int64_t)(int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, GCPtrMem); /** @todo use iemMemFlatFetchDataU16SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)(int64_t)(int16_t)iemMemFetchDataU16SafeJmp(pVCpu, UINT8_MAX, GCPtrMem);
+#else
+    return (uint64_t)(int64_t)(int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, GCPtrMem);
+#endif
 }
 
@@ -1969 +2062 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
 {
-    return (uint64_t)iemMemFlatFetchDataU32Jmp(pVCpu, GCPtrMem); /** @todo use iemMemFlatFetchDataU32SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)iemMemFetchDataU32SafeJmp(pVCpu, UINT8_MAX, GCPtrMem);
+#else
+    return (uint64_t)iemMemFlatFetchDataU32Jmp(pVCpu, GCPtrMem);
+#endif
 }
 
@@ -1980 +2077 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
 {
-    return (uint64_t)(int64_t)(int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, GCPtrMem); /** @todo use iemMemFlatFetchDataU32SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return (uint64_t)(int64_t)(int32_t)iemMemFetchDataU32SafeJmp(pVCpu, UINT8_MAX, GCPtrMem);
+#else
+    return (uint64_t)(int64_t)(int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, GCPtrMem);
+#endif
 }
 
@@ -1989 +2090 @@
 IEM_DECL_NATIVE_HLP_DEF(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem))
 {
-    return iemMemFlatFetchDataU64Jmp(pVCpu, GCPtrMem); /** @todo use iemMemFlatFetchDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_FETCH
+    return iemMemFetchDataU64SafeJmp(pVCpu, UINT8_MAX, GCPtrMem);
+#else
+    return iemMemFlatFetchDataU64Jmp(pVCpu, GCPtrMem);
+#endif
 }
 
@@ -1998 +2103 @@
 IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value))
 {
-    iemMemFlatStoreDataU8Jmp(pVCpu, GCPtrMem, u8Value); /** @todo use iemMemStoreDataU8SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_STORE
+    iemMemStoreDataU8SafeJmp(pVCpu, UINT8_MAX, GCPtrMem, u8Value);
+#else
+    iemMemFlatStoreDataU8Jmp(pVCpu, GCPtrMem, u8Value);
+#endif
 }
 
@@ -2007 +2116 @@
 IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value))
 {
-    iemMemFlatStoreDataU16Jmp(pVCpu, GCPtrMem, u16Value); /** @todo use iemMemStoreDataU16SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_STORE
+    iemMemStoreDataU16SafeJmp(pVCpu, UINT8_MAX, GCPtrMem, u16Value);
+#else
+    iemMemFlatStoreDataU16Jmp(pVCpu, GCPtrMem, u16Value);
+#endif
 }
 
@@ -2016 +2129 @@
 IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value))
 {
-    iemMemFlatStoreDataU32Jmp(pVCpu, GCPtrMem, u32Value); /** @todo use iemMemStoreDataU32SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_STORE
+    iemMemStoreDataU32SafeJmp(pVCpu, UINT8_MAX, GCPtrMem, u32Value);
+#else
+    iemMemFlatStoreDataU32Jmp(pVCpu, GCPtrMem, u32Value);
+#endif
 }
 
@@ -2025 +2142 @@
 IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value))
 {
-    iemMemFlatStoreDataU64Jmp(pVCpu, GCPtrMem, u64Value); /** @todo use iemMemStoreDataU64SafeJmp */
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_STORE
+    iemMemStoreDataU64SafeJmp(pVCpu, UINT8_MAX, GCPtrMem, u64Value);
+#else
+    iemMemFlatStoreDataU64Jmp(pVCpu, GCPtrMem, u64Value);
+#endif
 }
 
@@ -11051 +11172 @@
          */
         PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
+# ifdef VBOX_WITH_STATISTICS
+        off = iemNativeEmitIncStamCounterInVCpuEx(pCodeBuf, off, TlbState.idxReg1, TlbState.idxReg2,
+                                                  enmOp == kIemNativeEmitMemOp_Store
+                                                  ? RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeTlbHitsForFetch)
+                                                  : RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeTlbHitsForStore));
+# endif
         switch (enmOp)
         {
@@ -11767 +11894 @@
          */
         PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
+# ifdef VBOX_WITH_STATISTICS
+        off = iemNativeEmitIncStamCounterInVCpuEx(pCodeBuf, off, TlbState.idxReg1, TlbState.idxReg2,
+                                                  RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeTlbHitsForStack));
+# endif
         if (idxRegValue != UINT8_MAX)
         {
@@ -12111 +12242 @@
          */
         PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 32);
+# ifdef VBOX_WITH_STATISTICS
+        off = iemNativeEmitIncStamCounterInVCpuEx(pCodeBuf, off, TlbState.idxReg1, TlbState.idxReg2,
+                                                  RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeTlbHitsForStack));
+# endif
         switch (cbMem)
         {
@@ -12620 +12755 @@
         off = iemNativeEmitTlbLookup(pReNative, off, &TlbState, iSegReg, cbMem, fAlignMask, fAccess,
                                      idxLabelTlbLookup, idxLabelTlbMiss, idxRegMemResult);
+# ifdef VBOX_WITH_STATISTICS
+        off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
+                                                RT_UOFFSETOF(VMCPUCC,  iem.s.StatNativeTlbHitsForMapped));
+# endif
 
         /* [idxVarUnmapInfo] = 0; */

trunk/src/VBox/VMM/VMMR3/IEMR3.cpp

@@ -297 +297 @@
         STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.cTlbHits,            STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
                         "Data TLB hits",                            "/IEM/CPU%u/DataTlb-Hits", idCpu);
+#  ifdef VBOX_WITH_IEM_RECOMPILER
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatNativeTlbHitsForStack, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Data TLB native stack access hits",        "/IEM/CPU%u/DataTlb-Hits-Native-Stack", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatNativeTlbHitsForFetch, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Data TLB native data fetch hits",          "/IEM/CPU%u/DataTlb-Hits-Native-Fetch", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatNativeTlbHitsForStore, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Data TLB native data store hits",          "/IEM/CPU%u/DataTlb-Hits-Native-Store", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatNativeTlbHitsForMapped, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Data TLB native mapped data hits",         "/IEM/CPU%u/DataTlb-Hits-Native-Mapped", idCpu);
+#  endif
 # endif
         STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.cTlbMisses,          STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,

trunk/src/VBox/VMM/include/IEMInternal.h

@@ -1710 +1710 @@
     /** Pointer to the native recompiler state for ring-3. */
     R3PTRTYPE(struct IEMRECOMPILERSTATE *)  pNativeRecompilerStateR3;
-    /** Alignment padding. */
-    uint64_t                auAlignment10[3];
+
     /** Statistics: Times TB execution was broken off before reaching the end. */
     STAMCOUNTER             StatTbExecBreaks;
@@ -1736 +1735 @@
     /** Native TB statistics: Number of threaded calls per TB that weren't recompiled. */
     STAMPROFILE             StatNativeCallsThreaded;
+    /** Native recompiled execution: TLB hits for data fetches. */
+    STAMCOUNTER             StatNativeTlbHitsForFetch;
+    /** Native recompiled execution: TLB hits for data stores. */
+    STAMCOUNTER             StatNativeTlbHitsForStore;
+    /** Native recompiled execution: TLB hits for stack accesses. */
+    STAMCOUNTER             StatNativeTlbHitsForStack;
+    /** Native recompiled execution: TLB hits for mapped accesses. */
+    STAMCOUNTER             StatNativeTlbHitsForMapped;
+    uint64_t                au64Padding[7];
     /** @} */
 

trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

@@ -881 +881 @@
 
 
+/** This is just as a typesafe alternative to RT_UOFFSETOF. */
+DECL_FORCE_INLINE(uint32_t) iemNativeVCpuOffsetFromStamCounterPtr(PVMCPU pVCpu, PSTAMCOUNTER pStamCounter)
+{
+    uintptr_t const off = (uintptr_t)pStamCounter - (uintptr_t)pVCpu;
+    Assert(off < sizeof(VMCPU));
+    return off;
+}
+
+
+/** This is just as a typesafe alternative to RT_UOFFSETOF. */
+DECL_FORCE_INLINE(uint32_t) iemNativeVCpuOffsetFromU64Ptr(PVMCPU pVCpu, uint64_t *pu64)
+{
+    uintptr_t const off = (uintptr_t)pu64 - (uintptr_t)pVCpu;
+    Assert(off < sizeof(VMCPU));
+    return off;
+}
+
+
+/**
+ * Emits code for incrementing a statistics counter (STAMCOUNTER/uint64_t) in VMCPU.
+ *
+ * @note The two temp registers are not required for AMD64.  ARM64 always
+ *       requires the first, and the 2nd is needed if the offset cannot be
+ *       encoded as an immediate.
+ */
+DECL_FORCE_INLINE(uint32_t)
+iemNativeEmitIncStamCounterInVCpuEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t idxTmp1, uint8_t idxTmp2, uint32_t offVCpu)
+{
+#ifdef RT_ARCH_AMD64
+    /* inc qword [pVCpu + off] */
+    pCodeBuf[off++] = X86_OP_REX_W;
+    pCodeBuf[off++] = 0xff;
+    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, 0, offVCpu);
+    RT_NOREF(idxTmp1, idxTmp2);
+
+#elif defined(RT_ARCH_ARM64)
+    /* Determine how we're to access pVCpu first. */
+    uint32_t const cbData = sizeof(STAMCOUNTER);
+    if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
+    {
+        /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */
+        pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxTmp1,
+                                                   IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
+        pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxTmp1, 1);
+        pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, idxTmp1,
+                                                   IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
+    }
+    else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
+    {
+        pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxTmp1, IEMNATIVE_REG_FIXED_PCPUMCTX,
+                                                   (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
+        pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxTmp1, 1);
+        pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, idxTmp1, IEMNATIVE_REG_FIXED_PCPUMCTX,
+                                                   (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
+    }
+    else
+    {
+        /* The offset is too large, so we must load it into a register and use
+           ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */
+        off = iemNativeEmitLoadGprImmEx(pReNative, off, idxTmp2, offVCpu);
+        pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_Ld_Dword, idxTmp1, IEMNATIVE_REG_FIXED_PVMCPU, idxTmp2);
+        pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxTmp1, 1);
+        pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, idxTmp1, IEMNATIVE_REG_FIXED_PVMCPU, idxTmp2);
+    }
+
+#else
+# error "port me"
+#endif
+    return off;
+}
+
+
+/**
+ * Emits code for incrementing a statistics counter (STAMCOUNTER/uint64_t) in VMCPU.
+ *
+ * @note The two temp registers are not required for AMD64.  ARM64 always
+ *       requires the first, and the 2nd is needed if the offset cannot be
+ *       encoded as an immediate.
+ */
+DECL_FORCE_INLINE(uint32_t)
+iemNativeEmitIncStamCounterInVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxTmp1, uint8_t idxTmp2, uint32_t offVCpu)
+{
+#ifdef RT_ARCH_AMD64
+    off = iemNativeEmitIncStamCounterInVCpuEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, idxTmp1, idxTmp2, offVCpu);
+#elif defined(RT_ARCH_ARM64)
+    off = iemNativeEmitIncStamCounterInVCpuEx(iemNativeInstrBufEnsure(pReNative, off, 4+3), off, idxTmp1, idxTmp2, offVCpu);
+#else
+# error "port me"
+#endif
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+
+
 /**
  * Emits a gprdst = gprsrc load.