Changeset 53278 in vbox for trunk/src/VBox/Runtime/r3

Timestamp:

Nov 9, 2014 9:01:25 PM (10 years ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

96850

Message:

IPRT: Added experimental malloc replacement feature, add RTALLOC_REPLACE_MALLOC and VBOX_WITH_EF_WRAPS to LocalConfig.kmk (only tested on 64-bit darwin).

Location:

trunk/src/VBox/Runtime/r3

Files:

• alloc-ef.cpp (modified) (6 diffs)
• alloc-ef.h (modified) (2 diffs)
• posix/rtmempage-exec-mmap-heap-posix.cpp (modified) (3 diffs)

trunk/src/VBox/Runtime/r3/alloc-ef.cpp

@@ -45 +45 @@
 #include <iprt/string.h>
 
+#ifdef RTALLOC_REPLACE_MALLOC
+# include <VBox/dis.h>
+# include <VBox/disopcode.h>
+# include <dlfcn.h>
+# ifdef RT_OS_DARWIN
+#  include <malloc/malloc.h>
+# endif
+#endif
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+#ifdef RTALLOC_REPLACE_MALLOC
+# define RTMEM_REPLACMENT_ALIGN(a_cb) ((a_cb) >= 16 ? RT_ALIGN_Z(a_cb, 16) \
+                                       : (a_cb) >= sizeof(uintptr_t) ? RT_ALIGN_Z(a_cb,  sizeof(uintptr_t)) : (a_cb))
+#endif
+
 
 /*******************************************************************************
@@ -62 +80 @@
 static volatile size_t      g_cbBlocksDelay;
 # endif /* RTALLOC_EFENCE_FREE_DELAYED */
+# ifdef RTALLOC_REPLACE_MALLOC
+/** @name For calling the real allocation API we've replaced.
+ * @{ */
+void * (*g_pfnOrgMalloc)(size_t);
+void * (*g_pfnOrgCalloc)(size_t, size_t);
+void * (*g_pfnOrgRealloc)(void *, size_t);
+void   (*g_pfnOrgFree)(void *);
+size_t (*g_pfnOrgMallocSize)(void *);
+/** @} */
+# endif
 #endif /* RTALLOC_EFENCE_TRACE */
 /** Array of pointers free watches for. */
@@ -72 +100 @@
 *   Internal Functions                                                         *
 *******************************************************************************/
+#ifdef RTALLOC_REPLACE_MALLOC
+static void rtMemReplaceMallocAndFriends(void);
+#endif
+
+
 /**
  * Complains about something.
@@ -131 +164 @@
                                          const char *pszTag, void *pvCaller, RT_SRC_POS_DECL)
 {
+# ifdef RTALLOC_REPLACE_MALLOC
+    if (!g_pfnOrgMalloc)
+        rtMemReplaceMallocAndFriends();
+    PRTMEMBLOCK pBlock = (PRTMEMBLOCK)g_pfnOrgMalloc(sizeof(*pBlock));
+# else
     PRTMEMBLOCK pBlock = (PRTMEMBLOCK)malloc(sizeof(*pBlock));
+# endif
     if (pBlock)
     {
@@ -152 +191 @@
 DECLINLINE(void) rtmemBlockFree(PRTMEMBLOCK pBlock)
 {
+# ifdef RTALLOC_REPLACE_MALLOC
+    g_pfnOrgFree(pBlock);
+# else
     free(pBlock);
+# endif
 }
 
@@ -270 +313 @@
 
 #endif /* RTALLOC_EFENCE_TRACE */
+
+
+#if defined(RTALLOC_REPLACE_MALLOC) && defined(RTALLOC_EFENCE_TRACE)
+/*
+ *
+ * Replacing malloc, calloc, realloc, & free.
+ *
+ */
+
+/** Replacement for malloc. */
+static void *rtMemReplacementMalloc(size_t cb)
+{
+    size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cb);
+    void *pv = rtR3MemAlloc("r-malloc", RTMEMTYPE_RTMEMALLOC, cb, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
+    if (!pv)
+        pv = g_pfnOrgMalloc(cb);
+    return pv;
+}
+
+/** Replacement for calloc. */
+static void *rtMemReplacementCalloc(size_t cbItem, size_t cItems)
+{
+    size_t cb = cbItem * cItems;
+    size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cb);
+    void *pv = rtR3MemAlloc("r-calloc", RTMEMTYPE_RTMEMALLOCZ, cb, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
+    if (!pv)
+        pv = g_pfnOrgCalloc(cbItem, cItems);
+    return pv;
+}
+
+/** Replacement for realloc. */
+static void *rtMemReplacementRealloc(void *pvOld, size_t cbNew)
+{
+    if (pvOld)
+    {
+        /* We're not strict about where the memory was allocated. */
+        PRTMEMBLOCK pBlock = rtmemBlockGet(pvOld);
+        if (pBlock)
+        {
+            size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cbNew);
+            return rtR3MemRealloc("r-realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
+        }
+        return g_pfnOrgRealloc(pvOld, cbNew);
+    }
+    return rtMemReplacementMalloc(cbNew);
+}
+
+/** Replacement for free(). */
+static void rtMemReplacementFree(void *pv)
+{
+    if (pv)
+    {
+        /* We're not strict about where the memory was allocated. */
+        PRTMEMBLOCK pBlock = rtmemBlockGet(pv);
+        if (pBlock)
+            rtR3MemFree("r-free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS);
+        else
+            g_pfnOrgFree(pv);
+    }
+}
+
+# ifdef RT_OS_DARWIN
+/** Replacement for malloc. */
+static size_t rtMemReplacementMallocSize(void *pv)
+{
+    size_t cb;
+    if (pv)
+    {
+        /* We're not strict about where the memory was allocated. */
+        PRTMEMBLOCK pBlock = rtmemBlockGet(pv);
+        if (pBlock)
+            cb = pBlock->cbUnaligned;
+        else
+            cb = g_pfnOrgMallocSize(pv);
+    }
+    else
+        cb = 0;
+    return cb;
+}
+# endif
+
+
+static void rtMemReplaceMallocAndFriends(void)
+{
+    struct
+    {
+        const char *pszName;
+        PFNRT       pfnReplacement;
+        PFNRT       pfnOrg;
+        PFNRT      *ppfnJumpBack;
+    } aApis[] =
+    {
+        { "free",    (PFNRT)rtMemReplacementFree,    (PFNRT)free,    (PFNRT *)&g_pfnOrgFree },
+        { "realloc", (PFNRT)rtMemReplacementRealloc, (PFNRT)realloc, (PFNRT *)&g_pfnOrgRealloc },
+        { "calloc",  (PFNRT)rtMemReplacementCalloc,  (PFNRT)calloc,  (PFNRT *)&g_pfnOrgCalloc },
+        { "malloc",  (PFNRT)rtMemReplacementMalloc,  (PFNRT)malloc,  (PFNRT *)&g_pfnOrgMalloc },
+#ifdef RT_OS_DARWIN
+        { "malloc_size", (PFNRT)rtMemReplacementMallocSize,  (PFNRT)malloc_size,  (PFNRT *)&g_pfnOrgMallocSize },
+#endif
+    };
+
+    /*
+     * Initialize the jump backs to avoid recursivly entering this function.
+     */
+    for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
+        *aApis[i].ppfnJumpBack = aApis[i].pfnOrg;
+
+    /*
+     * Give the user an option to skip replacing malloc.
+     */
+    if (getenv("IPRT_DONT_REPLACE_MALLOC"))
+        return;
+
+    /*
+     * Allocate a page for jump back code (we leak it).
+     */
+    uint8_t *pbExecPage = (uint8_t *)RTMemPageAlloc(PAGE_SIZE); AssertFatal(pbExecPage);
+    int rc = RTMemProtect(pbExecPage, PAGE_SIZE, RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC); AssertFatalRC(rc);
+
+    /*
+     * Do the ground work.
+     */
+    uint8_t *pb = pbExecPage;
+    for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
+    {
+        /* Resolve it. */
+        PFNRT pfnOrg = (PFNRT)(uintptr_t)dlsym(RTLD_DEFAULT, aApis[i].pszName);
+        if (pfnOrg)
+            aApis[i].pfnOrg = pfnOrg;
+        else
+            pfnOrg = aApis[i].pfnOrg;
+
+        /* Figure what we can replace and how much to duplicate in the jump back code. */
+# ifdef RT_ARCH_AMD64
+        uint32_t         cbNeeded   = 12;
+        DISCPUMODE const enmCpuMode = DISCPUMODE_64BIT;
+# elif defined(RT_ARCH_X86)
+        uint32_t   const cbNeeded   = 5;
+        DISCPUMODE const enmCpuMode = DISCPUMODE_32BIT;
+# else
+#  error "Port me"
+# endif
+        uint32_t offJmpBack = 0;
+        uint32_t cbCopy = 0;
+        while (offJmpBack < cbNeeded)
+        {
+            DISCPUSTATE Dis;
+            uint32_t cbInstr = 1;
+            rc = DISInstr((void *)((uintptr_t)pfnOrg + offJmpBack), enmCpuMode, &Dis, &cbInstr); AssertFatalRC(rc);
+            AssertFatal(!(Dis.pCurInstr->fOpType & (DISOPTYPE_CONTROLFLOW)));
+# ifdef RT_ARCH_AMD64
+#  ifdef RT_OS_DARWIN
+            /* Kludge for: cmp [malloc_def_zone_state], 1; jg 2; call _malloc_initialize; 2: */
+            DISQPVPARAMVAL Parm;
+            if (   Dis.ModRM.Bits.Mod == 0
+                && Dis.ModRM.Bits.Rm == 5 /* wrt RIP */
+                && (Dis.Param2.fUse & (DISUSE_IMMEDIATE16_SX8 | DISUSE_IMMEDIATE32_SX8 | DISUSE_IMMEDIATE64_SX8))
+                && Dis.Param2.uValue == 1
+                && Dis.pCurInstr->uOpcode == OP_CMP)
+            {
+                cbCopy = offJmpBack;
+
+                offJmpBack += cbInstr;
+                rc = DISInstr((void *)((uintptr_t)pfnOrg + offJmpBack), enmCpuMode, &Dis, &cbInstr); AssertFatalRC(rc);
+                if (   Dis.pCurInstr->uOpcode == OP_JNBE
+                    && Dis.Param1.uDisp.i8 == 5)
+                {
+                    offJmpBack += cbInstr + 5;
+                    AssertFatal(offJmpBack >= cbNeeded);
+                    break;
+                }
+            }
+#  endif
+            AssertFatal(!(Dis.ModRM.Bits.Mod == 0 && Dis.ModRM.Bits.Rm == 5 /* wrt RIP */));
+# endif
+            offJmpBack += cbInstr;
+        }
+        if (!cbCopy)
+            cbCopy = offJmpBack;
+
+        /* Assemble the jump back. */
+        memcpy(pb, (void *)(uintptr_t)pfnOrg, cbCopy);
+        uint32_t off = cbCopy;
+# ifdef RT_ARCH_AMD64
+        pb[off++] = 0xff; /* jmp qword [$+8 wrt RIP] */
+        pb[off++] = 0x25;
+        *(uint32_t *)&pb[off] = 0;
+        off += 4;
+        *(uint64_t *)&pb[off] = (uintptr_t)pfnOrg + offJmpBack;
+        off += 8;
+        off = RT_ALIGN_32(off, 16);
+# elif defined(RT_ARCH_X86)
+        pb[off++] = 0xe9; /* jmp rel32 */
+        *(uint32_t *)&pb[off] = (uintptr_t)pfnOrg + offJmpBack - (uintptr_t)&pb[4];
+        off += 4;
+        off = RT_ALIGN_32(off, 8);
+# else
+#  error "Port me"
+# endif
+        *aApis[i].ppfnJumpBack = (PFNRT)(uintptr_t)pb;
+        pb += off;
+    }
+
+    /*
+     * Modify the APIs.
+     */
+    for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
+    {
+        pb = (uint8_t *)(uintptr_t)aApis[i].pfnOrg;
+        rc = RTMemProtect(pb, 16, RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC); AssertFatalRC(rc);
+
+# ifdef RT_ARCH_AMD64
+        /* Assemble the LdrLoadDll patch. */
+        *pb++ = 0x48; /* mov rax, qword */
+        *pb++ = 0xb8;
+        *(uint64_t *)pb = (uintptr_t)aApis[i].pfnReplacement;
+        pb += 8;
+        *pb++ = 0xff; /* jmp rax */
+        *pb++ = 0xe0;
+# elif defined(RT_ARCH_X86)
+        *pb++ = 0xe9; /* jmp rel32 */
+        *(uint32_t *)pb = (uintptr_t)aApis[i].pfnReplacement - (uintptr_t)&pb[4];
+# else
+#  error "Port me"
+# endif
+    }
+}
+
+#endif /* RTALLOC_REPLACE_MALLOC && RTALLOC_EFENCE_TRACE */
 
 

trunk/src/VBox/Runtime/r3/alloc-ef.h

@@ -119 +119 @@
 #endif
 
+#if defined(RUNNING_DOXYGEN)
+/** @def RTALLOC_REPLACE_MALLOC
+ * Replaces the malloc, calloc, realloc, free and friends in libc (experimental).
+ * Set in LocalConfig.kmk. Requires RTALLOC_EFENCE_TRACE to work. */
+# define RTALLOC_REPLACE_MALLOC
+#endif
+#if defined(RTALLOC_REPLACE_MALLOC) && !defined(RTALLOC_EFENCE_TRACE)
+# error "RTALLOC_REPLACE_MALLOC requires RTALLOC_EFENCE_TRACE."
+#endif
 
 
@@ -192 +201 @@
 RT_C_DECLS_END
 
-#endif
-
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+#ifdef RTALLOC_REPLACE_MALLOC
+RT_C_DECLS_BEGIN
+extern void * (*g_pfnOrgMalloc)(size_t);
+extern void * (*g_pfnOrgCalloc)(size_t, size_t);
+extern void * (*g_pfnOrgRealloc)(void *, size_t);
+extern void   (*g_pfnOrgFree)(void *);
+RT_C_DECLS_END
+#endif
+
+#endif
+

trunk/src/VBox/Runtime/r3/posix/rtmempage-exec-mmap-heap-posix.cpp

@@ -41 +41 @@
 #include <iprt/string.h>
 #include "internal/mem.h"
+#include "../alloc-ef.h"
 
 #include <stdlib.h>
@@ -412 +413 @@
     }
     /** @todo Eliminate this rtMemBaseAlloc dependency! */
-    PRTHEAPPAGEBLOCK pBlock = (PRTHEAPPAGEBLOCK)rtMemBaseAlloc(sizeof(*pBlock));
+    PRTHEAPPAGEBLOCK pBlock;
+#ifdef RTALLOC_REPLACE_MALLOC
+    if (g_pfnOrgMalloc)
+        pBlock = (PRTHEAPPAGEBLOCK)g_pfnOrgMalloc(sizeof(*pBlock));
+    else
+#endif
+        pBlock = (PRTHEAPPAGEBLOCK)rtMemBaseAlloc(sizeof(*pBlock));
     if (!pBlock)
     {
@@ -583 +590 @@
                         pBlock->Core.Key = pBlock->Core.KeyLast = NULL;
                         pBlock->cFreePages = 0;
-                        rtMemBaseFree(pBlock);
+#ifdef RTALLOC_REPLACE_MALLOC
+                        if (g_pfnOrgFree)
+                            g_pfnOrgFree(pBlock);
+                        else
+#endif
+                            rtMemBaseFree(pBlock);
 
                         RTCritSectEnter(&pHeap->CritSect);