Changeset 58269 in vbox for trunk/src/VBox/Runtime/r0drv

Timestamp:

Oct 15, 2015 6:29:21 PM (9 years ago)

Author:

vboxsync

Message:

iprt/r0drv: untested electrical heap alloc functions.

Location:

trunk/src/VBox/Runtime/r0drv

Files:

• alloc-ef-r0drv.cpp (copied) (copied from trunk/src/VBox/Runtime/r3/alloc-ef.cpp ) (45 diffs)
• initterm-r0drv.cpp (modified) (4 diffs)

trunk/src/VBox/Runtime/r0drv/alloc-ef-r0drv.cpp

@@ -1 +1 @@
 /* $Id$ */
 /** @file
- * IPRT - Memory Allocation, electric fence.
+ * IPRT - Memory Allocation, electric fence for ring-0 drivers.
  */
 
@@ -29 +29 @@
 *   Header Files                                                                                                                 *
 *********************************************************************************************************************************/
-#include "alloc-ef.h"
+#include "internal/iprt.h"
 #include <iprt/mem.h>
+#include <iprt/memobj.h>
 #include <iprt/log.h>
 #include <iprt/asm.h>
@@ -36 +37 @@
 #include <VBox/sup.h>
 #include <iprt/err.h>
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
 
 #include <iprt/alloc.h>
@@ -45 +43 @@
 #include <iprt/string.h>
 
-#ifdef RTALLOC_REPLACE_MALLOC
-# include <VBox/dis.h>
-# include <VBox/disopcode.h>
-# include <dlfcn.h>
-# ifdef RT_OS_DARWIN
-#  include <malloc/malloc.h>
-# endif
-#endif
-
-
-/*********************************************************************************************************************************
-*   Defined Constants And Macros                                                                                                 *
-*********************************************************************************************************************************/
-#ifdef RTALLOC_REPLACE_MALLOC
-# define RTMEM_REPLACMENT_ALIGN(a_cb) ((a_cb) >= 16 ? RT_ALIGN_Z(a_cb, 16) \
-                                       : (a_cb) >= sizeof(uintptr_t) ? RT_ALIGN_Z(a_cb,  sizeof(uintptr_t)) : (a_cb))
-#endif
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+#if defined(DOXYGEN_RUNNING)
+# define RTR0MEM_EF_IN_FRONT
+#endif
+
+/** @def RTR0MEM_EF_SIZE
+ * The size of the fence. This must be page aligned.
+ */
+#define RTR0MEM_EF_SIZE             PAGE_SIZE
+
+/** @def RTR0MEM_EF_ALIGNMENT
+ * The allocation alignment, power of two of course.
+ *
+ * Use this for working around misaligned sizes, usually stemming from
+ * allocating a string or something after the main structure.  When you
+ * encounter this, please fix the allocation to RTMemAllocVar or RTMemAllocZVar.
+ */
+#if 0
+# define RTR0MEM_EF_ALIGNMENT       (ARCH_BITS / 8)
+#else
+# define RTR0MEM_EF_ALIGNMENT       1
+#endif
+
+/** @def RTR0MEM_EF_IN_FRONT
+ * Define this to put the fence up in front of the block.
+ * The default (when this isn't defined) is to up it up after the block.
+ */
+//# define RTR0MEM_EF_IN_FRONT
+
+/** @def RTR0MEM_EF_FREE_DELAYED
+ * This define will enable free() delay and protection of the freed data
+ * while it's being delayed. The value of RTR0MEM_EF_FREE_DELAYED defines
+ * the threshold of the delayed blocks.
+ * Delayed blocks does not consume any physical memory, only virtual address space.
+ */
+#define RTR0MEM_EF_FREE_DELAYED     (20 * _1M)
+
+/** @def RTR0MEM_EF_FREE_FILL
+ * This define will enable memset(,RTR0MEM_EF_FREE_FILL,)'ing the user memory
+ * in the block before freeing/decommitting it. This is useful in GDB since GDB
+ * appears to be able to read the content of the page even after it's been
+ * decommitted.
+ */
+#define RTR0MEM_EF_FREE_FILL        'f'
+
+/** @def RTR0MEM_EF_FILLER
+ * This define will enable memset(,RTR0MEM_EF_FILLER,)'ing the allocated
+ * memory when the API doesn't require it to be zero'd.
+ */
+#define RTR0MEM_EF_FILLER           0xef
+
+/** @def RTR0MEM_EF_NOMAN_FILLER
+ * This define will enable memset(,RTR0MEM_EF_NOMAN_FILLER,)'ing the
+ * unprotected but not allocated area of memory, the so called no man's land.
+ */
+#define RTR0MEM_EF_NOMAN_FILLER     0xaa
+
+/** @def RTR0MEM_EF_FENCE_FILLER
+ * This define will enable memset(,RTR0MEM_EF_FENCE_FILLER,)'ing the
+ * fence itself, as debuggers can usually read them.
+ */
+#define RTR0MEM_EF_FENCE_FILLER     0xcc
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#ifdef RT_OS_WINDOWS
+# include <Windows.h>
+#else
+# include <sys/mman.h>
+#endif
+#include <iprt/avl.h>
+#include <iprt/thread.h>
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/**
+ * Allocation types.
+ */
+typedef enum RTMEMTYPE
+{
+    RTMEMTYPE_RTMEMALLOC,
+    RTMEMTYPE_RTMEMALLOCZ,
+    RTMEMTYPE_RTMEMREALLOC,
+    RTMEMTYPE_RTMEMFREE,
+
+    RTMEMTYPE_NEW,
+    RTMEMTYPE_NEW_ARRAY,
+    RTMEMTYPE_DELETE,
+    RTMEMTYPE_DELETE_ARRAY
+} RTMEMTYPE;
+
+/**
+ * Node tracking a memory allocation.
+ */
+typedef struct RTR0MEMEFBLOCK
+{
+    /** Avl node code, key is the user block pointer. */
+    AVLPVNODECORE   Core;
+    /** Allocation type. */
+    RTMEMTYPE       enmType;
+    /** The memory object. */
+    RTR0MEMOBJ      hMemObj;
+    /** The unaligned size of the block. */
+    size_t          cbUnaligned;
+    /** The aligned size of the block. */
+    size_t          cbAligned;
+    /** The allocation tag (read-only string). */
+    const char     *pszTag;
+    /** The return address of the allocator function. */
+    void           *pvCaller;
+    /** Line number of the alloc call. */
+    unsigned        iLine;
+    /** File from within the allocation was made. */
+    const char     *pszFile;
+    /** Function from within the allocation was made. */
+    const char     *pszFunction;
+} RTR0MEMEFBLOCK, *PRTR0MEMEFBLOCK;
+
 
 
@@ -67 +174 @@
 *   Global Variables                                                                                                             *
 *********************************************************************************************************************************/
-#ifdef RTALLOC_EFENCE_TRACE
 /** Spinlock protecting the all the block's globals. */
-static volatile uint32_t    g_BlocksLock;
+static volatile uint32_t        g_BlocksLock;
 /** Tree tracking the allocations. */
-static AVLPVTREE            g_BlocksTree;
-# ifdef RTALLOC_EFENCE_FREE_DELAYED
+static AVLPVTREE                g_BlocksTree;
+
+#ifdef RTR0MEM_EF_FREE_DELAYED
 /** Tail of the delayed blocks. */
-static volatile PRTMEMBLOCK g_pBlocksDelayHead;
+static volatile PRTR0MEMEFBLOCK g_pBlocksDelayHead;
 /** Tail of the delayed blocks. */
-static volatile PRTMEMBLOCK g_pBlocksDelayTail;
+static volatile PRTR0MEMEFBLOCK g_pBlocksDelayTail;
 /** Number of bytes in the delay list (includes fences). */
-static volatile size_t      g_cbBlocksDelay;
-# endif /* RTALLOC_EFENCE_FREE_DELAYED */
-# ifdef RTALLOC_REPLACE_MALLOC
-/** @name For calling the real allocation API we've replaced.
- * @{ */
-void * (*g_pfnOrgMalloc)(size_t);
-void * (*g_pfnOrgCalloc)(size_t, size_t);
-void * (*g_pfnOrgRealloc)(void *, size_t);
-void   (*g_pfnOrgFree)(void *);
-size_t (*g_pfnOrgMallocSize)(void *);
-/** @} */
-# endif
-#endif /* RTALLOC_EFENCE_TRACE */
+static volatile size_t          g_cbBlocksDelay;
+#endif /* RTR0MEM_EF_FREE_DELAYED */
+
 /** Array of pointers free watches for. */
 void   *gapvRTMemFreeWatch[4] = {NULL, NULL, NULL, NULL};
@@ -100 +197 @@
 *   Internal Functions                                                                                                           *
 *********************************************************************************************************************************/
-#ifdef RTALLOC_REPLACE_MALLOC
-static void rtMemReplaceMallocAndFriends(void);
-#endif
+
+
+/**
+ * @callback_method_impl{FNRTSTROUTPUT}
+ */
+static DECLCALLBACK(size_t) rtR0MemEfWrite(void *pvArg, const char *pachChars, size_t cbChars)
+{
+    if (cbChars)
+    {
+        RTLogWriteDebugger(pachChars, cbChars);
+        RTLogWriteStdOut(pachChars, cbChars);
+        RTLogWriteUser(pachChars, cbChars);
+    }
+    return cbChars;
+}
 
 
@@ -108 +217 @@
  * Complains about something.
  */
-static void rtmemComplain(const char *pszOp, const char *pszFormat, ...)
+static void rtR0MemComplain(const char *pszOp, const char *pszFormat, ...)
 {
     va_list args;
-    fprintf(stderr, "RTMem error: %s: ", pszOp);
+    RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "RTMem error: %s: ", pszOp);
     va_start(args, pszFormat);
-    vfprintf(stderr, pszFormat, args);
+    RTStrFormatV(rtR0MemEfWrite, NULL, NULL, NULL, pszFormat, args);
     va_end(args);
     RTAssertDoPanic();
@@ -121 +230 @@
  * Log an event.
  */
-DECLINLINE(void) rtmemLog(const char *pszOp, const char *pszFormat, ...)
+DECLINLINE(void) rtR0MemLog(const char *pszOp, const char *pszFormat, ...)
 {
 #if 0
     va_list args;
-    fprintf(stderr, "RTMem info: %s: ", pszOp);
+    RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "RTMem info: %s: ", pszOp);
     va_start(args, pszFormat);
-    vfprintf(stderr, pszFormat, args);
+    RTStrFormatV(rtR0MemEfWrite, NULL, NULL, NULL, pszFormat, args);
     va_end(args);
 #else
@@ -135 +244 @@
 
 
-#ifdef RTALLOC_EFENCE_TRACE
 
 /**
  * Acquires the lock.
  */
-DECLINLINE(void) rtmemBlockLock(void)
-{
+DECLINLINE(RTCCUINTREG) rtR0MemBlockLock(void)
+{
+    RTCCUINTREG uRet;
     unsigned c = 0;
-    while (!ASMAtomicCmpXchgU32(&g_BlocksLock, 1, 0))
-        RTThreadSleepNoLog(((++c) >> 2) & 31);
+    if (RTThreadPreemptIsEnabled(NIL_RTTHREAD))
+    {
+        for (;;)
+        {
+            uRet = ASMIntDisableFlags();
+            if (ASMAtomicCmpXchgU32(&g_BlocksLock, 1, 0))
+                break;
+            ASMSetFlags(uRet);
+            RTThreadSleepNoLog(((++c) >> 2) & 31);
+        }
+    }
+    else
+    {
+        for (;;)
+        {
+            uRet = ASMIntDisableFlags();
+            if (ASMAtomicCmpXchgU32(&g_BlocksLock, 1, 0))
+                break;
+            ASMSetFlags(uRet);
+            ASMNopPause();
+            if (++c & 3)
+                ASMNopPause();
+        }
+    }
+    return uRet;
 }
 
@@ -151 +283 @@
  * Releases the lock.
  */
-DECLINLINE(void) rtmemBlockUnlock(void)
+DECLINLINE(void) rtR0MemBlockUnlock(RTCCUINTREG fSavedIntFlags)
 {
     Assert(g_BlocksLock == 1);
     ASMAtomicXchgU32(&g_BlocksLock, 0);
+    ASMSetFlags(fSavedIntFlags);
 }
 
@@ -161 +294 @@
  * Creates a block.
  */
-DECLINLINE(PRTMEMBLOCK) rtmemBlockCreate(RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
-                                         const char *pszTag, void *pvCaller, RT_SRC_POS_DECL)
-{
-# ifdef RTALLOC_REPLACE_MALLOC
-    if (!g_pfnOrgMalloc)
-        rtMemReplaceMallocAndFriends();
-    PRTMEMBLOCK pBlock = (PRTMEMBLOCK)g_pfnOrgMalloc(sizeof(*pBlock));
-# else
-    PRTMEMBLOCK pBlock = (PRTMEMBLOCK)malloc(sizeof(*pBlock));
-# endif
+DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockCreate(RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
+                                               const char *pszTag, void *pvCaller, RT_SRC_POS_DECL)
+{
+    PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)RTMemAlloc(sizeof(*pBlock));
     if (pBlock)
     {
@@ -189 +316 @@
  * Frees a block.
  */
-DECLINLINE(void) rtmemBlockFree(PRTMEMBLOCK pBlock)
-{
-# ifdef RTALLOC_REPLACE_MALLOC
-    g_pfnOrgFree(pBlock);
-# else
-    free(pBlock);
-# endif
+DECLINLINE(void) rtR0MemBlockFree(PRTR0MEMEFBLOCK pBlock)
+{
+    RTMemFree(pBlock);
 }
 
@@ -202 +325 @@
  * Insert a block from the tree.
  */
-DECLINLINE(void) rtmemBlockInsert(PRTMEMBLOCK pBlock, void *pv)
+DECLINLINE(void) rtR0MemBlockInsert(PRTR0MEMEFBLOCK pBlock, void *pv, RTR0MEMOBJ hMemObj)
 {
     pBlock->Core.Key = pv;
-    rtmemBlockLock();
+    pBlock->hMemObj  = hMemObj;
+    RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
     bool fRc = RTAvlPVInsert(&g_BlocksTree, &pBlock->Core);
-    rtmemBlockUnlock();
+    rtR0MemBlockUnlock(fSavedIntFlags);
     AssertRelease(fRc);
 }
@@ -215 +339 @@
  * Remove a block from the tree and returns it to the caller.
  */
-DECLINLINE(PRTMEMBLOCK) rtmemBlockRemove(void *pv)
-{
-    rtmemBlockLock();
-    PRTMEMBLOCK pBlock = (PRTMEMBLOCK)RTAvlPVRemove(&g_BlocksTree, pv);
-    rtmemBlockUnlock();
+DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockRemove(void *pv)
+{
+    RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
+    PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)RTAvlPVRemove(&g_BlocksTree, pv);
+    rtR0MemBlockUnlock(fSavedIntFlags);
     return pBlock;
 }
 
+
 /**
  * Gets a block.
  */
-DECLINLINE(PRTMEMBLOCK) rtmemBlockGet(void *pv)
-{
-    rtmemBlockLock();
-    PRTMEMBLOCK pBlock = (PRTMEMBLOCK)RTAvlPVGet(&g_BlocksTree, pv);
-    rtmemBlockUnlock();
+DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockGet(void *pv)
+{
+    RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
+    PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)RTAvlPVGet(&g_BlocksTree, pv);
+    rtR0MemBlockUnlock(fSavedIntFlags);
     return pBlock;
 }
 
+
 /**
  * Dumps one allocation.
@@ -239 +365 @@
 static DECLCALLBACK(int) RTMemDumpOne(PAVLPVNODECORE pNode, void *pvUser)
 {
-    PRTMEMBLOCK pBlock = (PRTMEMBLOCK)pNode;
-    fprintf(stderr, "%p %08lx(+%02lx) %p\n",
-            pBlock->Core.Key,
-            (unsigned long)pBlock->cbUnaligned,
-            (unsigned long)(pBlock->cbAligned - pBlock->cbUnaligned),
-            pBlock->pvCaller);
+    PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)pNode;
+    RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "%p %08lx(+%02lx) %p\n",
+                pBlock->Core.Key,
+                (unsigned long)pBlock->cbUnaligned,
+                (unsigned long)(pBlock->cbAligned - pBlock->cbUnaligned),
+                pBlock->pvCaller);
     NOREF(pvUser);
     return 0;
 }
 
+
 /**
  * Dumps the allocated blocks.
@@ -256 +383 @@
 void RTMemDump(void)
 {
-    fprintf(stderr, "address  size(alg)     caller\n");
+    RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "address  size(alg)     caller\n");
     RTAvlPVDoWithAll(&g_BlocksTree, true, RTMemDumpOne, NULL);
 }
 
-# ifdef RTALLOC_EFENCE_FREE_DELAYED
+#ifdef RTR0MEM_EF_FREE_DELAYED
 
 /**
  * Insert a delayed block.
  */
-DECLINLINE(void) rtmemBlockDelayInsert(PRTMEMBLOCK pBlock)
-{
-    size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
+DECLINLINE(void) rtR0MemBlockDelayInsert(PRTR0MEMEFBLOCK pBlock)
+{
+    size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
     pBlock->Core.pRight = NULL;
     pBlock->Core.pLeft = NULL;
-    rtmemBlockLock();
+    RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
     if (g_pBlocksDelayHead)
     {
@@ -283 +410 @@
     }
     g_cbBlocksDelay += cbBlock;
-    rtmemBlockUnlock();
+    rtR0MemBlockUnlock(fSavedIntFlags);
 }
 
@@ -289 +416 @@
  * Removes a delayed block.
  */
-DECLINLINE(PRTMEMBLOCK) rtmemBlockDelayRemove(void)
-{
-    PRTMEMBLOCK pBlock = NULL;
-    rtmemBlockLock();
-    if (g_cbBlocksDelay > RTALLOC_EFENCE_FREE_DELAYED)
+DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockDelayRemove(void)
+{
+    PRTR0MEMEFBLOCK pBlock = NULL;
+    RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
+    if (g_cbBlocksDelay > RTR0MEM_EF_FREE_DELAYED)
     {
         pBlock = g_pBlocksDelayTail;
         if (pBlock)
         {
-            g_pBlocksDelayTail = (PRTMEMBLOCK)pBlock->Core.pLeft;
+            g_pBlocksDelayTail = (PRTR0MEMEFBLOCK)pBlock->Core.pLeft;
             if (pBlock->Core.pLeft)
                 pBlock->Core.pLeft->pRight = NULL;
             else
                 g_pBlocksDelayHead = NULL;
-            g_cbBlocksDelay -= RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
+            g_cbBlocksDelay -= RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
         }
     }
-    rtmemBlockUnlock();
+    rtR0MemBlockUnlock(fSavedIntFlags);
     return pBlock;
 }
 
-# endif  /* RTALLOC_EFENCE_FREE_DELAYED */
-
-#endif /* RTALLOC_EFENCE_TRACE */
-
-
-#if defined(RTALLOC_REPLACE_MALLOC) && defined(RTALLOC_EFENCE_TRACE)
-/*
+#endif  /* RTR0MEM_EF_FREE_DELAYED */
+
+
+static void rtR0MemFreeBlock(PRTR0MEMEFBLOCK pBlock, const char *pszOp)
+{
+    void  *pv      = pBlock->Core.Key;
+# ifdef RTR0MEM_EF_IN_FRONT
+    void  *pvBlock = (char *)pv - RTR0MEM_EF_SIZE;
+# else
+    void  *pvBlock = (void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK);
+# endif
+    size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
+
+    int rc = RTR0MemObjProtect(pBlock->hMemObj, 0 /*offSub*/, cbBlock, RTMEM_PROT_READ | RTMEM_PROT_WRITE);
+    if (RT_FAILURE(rc))
+        rtR0MemComplain(pszOp, "RTR0MemObjProtect([%p], 0, %#x, RTMEM_PROT_READ | RTMEM_PROT_WRITE) -> %Rrc\n",
+                        pvBlock, cbBlock, rc);
+
+    rc = RTR0MemObjFree(pBlock->hMemObj, true /*fFreeMappings*/);
+    if (RT_FAILURE(rc))
+        rtR0MemComplain(pszOp, "RTR0MemObjFree([%p LB %#x]) -> %Rrc\n", pvBlock, cbBlock, rc);
+    pBlock->hMemObj = NIL_RTR0MEMOBJ;
+
+    rtR0MemBlockFree(pBlock);
+}
+
+
+/**
+ * Initialize call, we shouldn't fail here.
+ */
+void rtR0MemEfInit(void)
+{
+
+}
+
+/**
+ * @callback_method_impl{AVLPVCALLBACK}
+ */
+static DECLCALLBACK(int) rtR0MemEfDestroyBlock(PAVLPVNODECORE pNode, void *pvUser)
+{
+    PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)pNode;
+
+    /* Note! pszFile and pszFunction may be invalid at this point. */
+    rtR0MemComplain("rtR0MemEfDestroyBlock", "Leaking %zu bytes at %p (iLine=%u pvCaller=%p)\n",
+                    pBlock->cbAligned, pBlock->Core.Key, pBlock->iLine, pBlock->pvCaller);
+
+    rtR0MemFreeBlock(pBlock, "rtR0MemEfDestroyBlock");
+
+    NOREF(pvUser);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Termination call.
  *
- * Replacing malloc, calloc, realloc, & free.
- *
- */
-
-/** Replacement for malloc. */
-static void *rtMemReplacementMalloc(size_t cb)
-{
-    size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cb);
-    void *pv = rtR3MemAlloc("r-malloc", RTMEMTYPE_RTMEMALLOC, cb, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
-    if (!pv)
-        pv = g_pfnOrgMalloc(cb);
-    return pv;
-}
-
-/** Replacement for calloc. */
-static void *rtMemReplacementCalloc(size_t cbItem, size_t cItems)
-{
-    size_t cb = cbItem * cItems;
-    size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cb);
-    void *pv = rtR3MemAlloc("r-calloc", RTMEMTYPE_RTMEMALLOCZ, cb, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
-    if (!pv)
-        pv = g_pfnOrgCalloc(cbItem, cItems);
-    return pv;
-}
-
-/** Replacement for realloc. */
-static void *rtMemReplacementRealloc(void *pvOld, size_t cbNew)
-{
-    if (pvOld)
-    {
-        /* We're not strict about where the memory was allocated. */
-        PRTMEMBLOCK pBlock = rtmemBlockGet(pvOld);
-        if (pBlock)
-        {
-            size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cbNew);
-            return rtR3MemRealloc("r-realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
-        }
-        return g_pfnOrgRealloc(pvOld, cbNew);
-    }
-    return rtMemReplacementMalloc(cbNew);
-}
-
-/** Replacement for free(). */
-static void rtMemReplacementFree(void *pv)
-{
-    if (pv)
-    {
-        /* We're not strict about where the memory was allocated. */
-        PRTMEMBLOCK pBlock = rtmemBlockGet(pv);
-        if (pBlock)
-            rtR3MemFree("r-free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS);
+ * Will check and free memory.
+ */
+void rtR0MemEfTerm(void)
+{
+#ifdef RTR0MEM_EF_FREE_DELAYED
+    /*
+     * Release delayed frees.
+     */
+    RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
+    for (;;)
+    {
+        PRTR0MEMEFBLOCK pBlock = g_pBlocksDelayTail;
+        g_pBlocksDelayTail = (PRTR0MEMEFBLOCK)pBlock->Core.pLeft;
+        if (pBlock->Core.pLeft)
+            pBlock->Core.pLeft->pRight = NULL;
         else
-            g_pfnOrgFree(pv);
-    }
-}
-
-# ifdef RT_OS_DARWIN
-/** Replacement for malloc. */
-static size_t rtMemReplacementMallocSize(void *pv)
-{
-    size_t cb;
-    if (pv)
-    {
-        /* We're not strict about where the memory was allocated. */
-        PRTMEMBLOCK pBlock = rtmemBlockGet(pv);
-        if (pBlock)
-            cb = pBlock->cbUnaligned;
-        else
-            cb = g_pfnOrgMallocSize(pv);
-    }
-    else
-        cb = 0;
-    return cb;
-}
-# endif
-
-
-static void rtMemReplaceMallocAndFriends(void)
-{
-    struct
-    {
-        const char *pszName;
-        PFNRT       pfnReplacement;
-        PFNRT       pfnOrg;
-        PFNRT      *ppfnJumpBack;
-    } aApis[] =
-    {
-        { "free",    (PFNRT)rtMemReplacementFree,    (PFNRT)free,    (PFNRT *)&g_pfnOrgFree },
-        { "realloc", (PFNRT)rtMemReplacementRealloc, (PFNRT)realloc, (PFNRT *)&g_pfnOrgRealloc },
-        { "calloc",  (PFNRT)rtMemReplacementCalloc,  (PFNRT)calloc,  (PFNRT *)&g_pfnOrgCalloc },
-        { "malloc",  (PFNRT)rtMemReplacementMalloc,  (PFNRT)malloc,  (PFNRT *)&g_pfnOrgMalloc },
-#ifdef RT_OS_DARWIN
-        { "malloc_size", (PFNRT)rtMemReplacementMallocSize,  (PFNRT)malloc_size,  (PFNRT *)&g_pfnOrgMallocSize },
-#endif
-    };
+            g_pBlocksDelayHead = NULL;
+        rtR0MemBlockUnlock(fSavedIntFlags);
+
+        rtR0MemFreeBlock(pBlock, "rtR0MemEfTerm");
+
+        rtR0MemBlockLock();
+    }
+    g_cbBlocksDelay = 0;
+    rtR0MemBlockUnlock(fSavedIntFlags);
+#endif
 
     /*
-     * Initialize the jump backs to avoid recursivly entering this function.
+     * Complain about leaks. Then release them.
      */
-    for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
-        *aApis[i].ppfnJumpBack = aApis[i].pfnOrg;
-
-    /*
-     * Give the user an option to skip replacing malloc.
-     */
-    if (getenv("IPRT_DONT_REPLACE_MALLOC"))
-        return;
-
-    /*
-     * Allocate a page for jump back code (we leak it).
-     */
-    uint8_t *pbExecPage = (uint8_t *)RTMemPageAlloc(PAGE_SIZE); AssertFatal(pbExecPage);
-    int rc = RTMemProtect(pbExecPage, PAGE_SIZE, RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC); AssertFatalRC(rc);
-
-    /*
-     * Do the ground work.
-     */
-    uint8_t *pb = pbExecPage;
-    for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
-    {
-        /* Resolve it. */
-        PFNRT pfnOrg = (PFNRT)(uintptr_t)dlsym(RTLD_DEFAULT, aApis[i].pszName);
-        if (pfnOrg)
-            aApis[i].pfnOrg = pfnOrg;
-        else
-            pfnOrg = aApis[i].pfnOrg;
-
-        /* Figure what we can replace and how much to duplicate in the jump back code. */
-# ifdef RT_ARCH_AMD64
-        uint32_t         cbNeeded   = 12;
-        DISCPUMODE const enmCpuMode = DISCPUMODE_64BIT;
-# elif defined(RT_ARCH_X86)
-        uint32_t   const cbNeeded   = 5;
-        DISCPUMODE const enmCpuMode = DISCPUMODE_32BIT;
-# else
-#  error "Port me"
-# endif
-        uint32_t offJmpBack = 0;
-        uint32_t cbCopy = 0;
-        while (offJmpBack < cbNeeded)
-        {
-            DISCPUSTATE Dis;
-            uint32_t cbInstr = 1;
-            rc = DISInstr((void *)((uintptr_t)pfnOrg + offJmpBack), enmCpuMode, &Dis, &cbInstr); AssertFatalRC(rc);
-            AssertFatal(!(Dis.pCurInstr->fOpType & (DISOPTYPE_CONTROLFLOW)));
-# ifdef RT_ARCH_AMD64
-#  ifdef RT_OS_DARWIN
-            /* Kludge for: cmp [malloc_def_zone_state], 1; jg 2; call _malloc_initialize; 2: */
-            DISQPVPARAMVAL Parm;
-            if (   Dis.ModRM.Bits.Mod == 0
-                && Dis.ModRM.Bits.Rm == 5 /* wrt RIP */
-                && (Dis.Param2.fUse & (DISUSE_IMMEDIATE16_SX8 | DISUSE_IMMEDIATE32_SX8 | DISUSE_IMMEDIATE64_SX8))
-                && Dis.Param2.uValue == 1
-                && Dis.pCurInstr->uOpcode == OP_CMP)
-            {
-                cbCopy = offJmpBack;
-
-                offJmpBack += cbInstr;
-                rc = DISInstr((void *)((uintptr_t)pfnOrg + offJmpBack), enmCpuMode, &Dis, &cbInstr); AssertFatalRC(rc);
-                if (   Dis.pCurInstr->uOpcode == OP_JNBE
-                    && Dis.Param1.uDisp.i8 == 5)
-                {
-                    offJmpBack += cbInstr + 5;
-                    AssertFatal(offJmpBack >= cbNeeded);
-                    break;
-                }
-            }
-#  endif
-            AssertFatal(!(Dis.ModRM.Bits.Mod == 0 && Dis.ModRM.Bits.Rm == 5 /* wrt RIP */));
-# endif
-            offJmpBack += cbInstr;
-        }
-        if (!cbCopy)
-            cbCopy = offJmpBack;
-
-        /* Assemble the jump back. */
-        memcpy(pb, (void *)(uintptr_t)pfnOrg, cbCopy);
-        uint32_t off = cbCopy;
-# ifdef RT_ARCH_AMD64
-        pb[off++] = 0xff; /* jmp qword [$+8 wrt RIP] */
-        pb[off++] = 0x25;
-        *(uint32_t *)&pb[off] = 0;
-        off += 4;
-        *(uint64_t *)&pb[off] = (uintptr_t)pfnOrg + offJmpBack;
-        off += 8;
-        off = RT_ALIGN_32(off, 16);
-# elif defined(RT_ARCH_X86)
-        pb[off++] = 0xe9; /* jmp rel32 */
-        *(uint32_t *)&pb[off] = (uintptr_t)pfnOrg + offJmpBack - (uintptr_t)&pb[4];
-        off += 4;
-        off = RT_ALIGN_32(off, 8);
-# else
-#  error "Port me"
-# endif
-        *aApis[i].ppfnJumpBack = (PFNRT)(uintptr_t)pb;
-        pb += off;
-    }
-
-    /*
-     * Modify the APIs.
-     */
-    for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
-    {
-        pb = (uint8_t *)(uintptr_t)aApis[i].pfnOrg;
-        rc = RTMemProtect(pb, 16, RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC); AssertFatalRC(rc);
-
-# ifdef RT_ARCH_AMD64
-        /* Assemble the LdrLoadDll patch. */
-        *pb++ = 0x48; /* mov rax, qword */
-        *pb++ = 0xb8;
-        *(uint64_t *)pb = (uintptr_t)aApis[i].pfnReplacement;
-        pb += 8;
-        *pb++ = 0xff; /* jmp rax */
-        *pb++ = 0xe0;
-# elif defined(RT_ARCH_X86)
-        *pb++ = 0xe9; /* jmp rel32 */
-        *(uint32_t *)pb = (uintptr_t)aApis[i].pfnReplacement - (uintptr_t)&pb[4];
-# else
-#  error "Port me"
-# endif
-    }
-}
-
-#endif /* RTALLOC_REPLACE_MALLOC && RTALLOC_EFENCE_TRACE */
+    RTAvlPVDestroy(&g_BlocksTree, rtR0MemEfDestroyBlock, NULL);
+}
 
 
@@ -547 +530 @@
  * Internal allocator.
  */
-RTDECL(void *) rtR3MemAlloc(const char *pszOp, RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
-                            const char *pszTag, void *pvCaller, RT_SRC_POS_DECL)
+static void * rtR0MemAlloc(const char *pszOp, RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
+                           const char *pszTag, void *pvCaller, RT_SRC_POS_DECL)
 {
     /*
      * Sanity.
      */
-    if (    RT_ALIGN_Z(RTALLOC_EFENCE_SIZE, PAGE_SIZE) != RTALLOC_EFENCE_SIZE
-        &&  RTALLOC_EFENCE_SIZE <= 0)
-    {
-        rtmemComplain(pszOp, "Invalid E-fence size! %#x\n", RTALLOC_EFENCE_SIZE);
+    if (    RT_ALIGN_Z(RTR0MEM_EF_SIZE, PAGE_SIZE) != RTR0MEM_EF_SIZE
+        &&  RTR0MEM_EF_SIZE <= 0)
+    {
+        rtR0MemComplain(pszOp, "Invalid E-fence size! %#x\n", RTR0MEM_EF_SIZE);
         return NULL;
     }
     if (!cbUnaligned)
     {
-#if 0
-        rtmemComplain(pszOp, "Request of ZERO bytes allocation!\n");
+#if 1
+        rtR0MemComplain(pszOp, "Request of ZERO bytes allocation!\n");
         return NULL;
 #else
@@ -569 +552 @@
     }
 
-#ifndef RTALLOC_EFENCE_IN_FRONT
+#ifndef RTR0MEM_EF_IN_FRONT
     /* Alignment decreases fence accuracy, but this is at least partially
      * counteracted by filling and checking the alignment padding. When the
      * fence is in front then then no extra alignment is needed. */
-    cbAligned = RT_ALIGN_Z(cbAligned, RTALLOC_EFENCE_ALIGNMENT);
-#endif
-
-#ifdef RTALLOC_EFENCE_TRACE
+    cbAligned = RT_ALIGN_Z(cbAligned, RTR0MEM_EF_ALIGNMENT);
+#endif
+
     /*
      * Allocate the trace block.
      */
-    PRTMEMBLOCK pBlock = rtmemBlockCreate(enmType, cbUnaligned, cbAligned, pszTag, pvCaller, RT_SRC_POS_ARGS);
+    PRTR0MEMEFBLOCK pBlock = rtR0MemBlockCreate(enmType, cbUnaligned, cbAligned, pszTag, pvCaller, RT_SRC_POS_ARGS);
     if (!pBlock)
     {
-        rtmemComplain(pszOp, "Failed to allocate trace block!\n");
+        rtR0MemComplain(pszOp, "Failed to allocate trace block!\n");
         return NULL;
     }
-#endif
 
     /*
      * Allocate a block with page alignment space + the size of the E-fence.
      */
-    size_t  cbBlock = RT_ALIGN_Z(cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
-    void   *pvBlock = RTMemPageAlloc(cbBlock);
+    void       *pvBlock = NULL;
+    RTR0MEMOBJ  hMemObj;
+    size_t      cbBlock = RT_ALIGN_Z(cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
+    int rc = RTR0MemObjAllocPage(&hMemObj, cbBlock, false /*fExecutable*/);
+    if (RT_SUCCESS(rc))
+        pvBlock = RTR0MemObjAddress(hMemObj);
     if (pvBlock)
     {
@@ -599 +584 @@
          * and then change the page protection of the fence.
          */
-#ifdef RTALLOC_EFENCE_IN_FRONT
+#ifdef RTR0MEM_EF_IN_FRONT
         void *pvEFence = pvBlock;
-        void *pv       = (char *)pvEFence + RTALLOC_EFENCE_SIZE;
-# ifdef RTALLOC_EFENCE_NOMAN_FILLER
-        memset((char *)pv + cbUnaligned, RTALLOC_EFENCE_NOMAN_FILLER, cbBlock - RTALLOC_EFENCE_SIZE - cbUnaligned);
+        void *pv       = (char *)pvEFence + RTR0MEM_EF_SIZE;
+# ifdef RTR0MEM_EF_NOMAN_FILLER
+        memset((char *)pv + cbUnaligned, RTR0MEM_EF_NOMAN_FILLER, cbBlock - RTR0MEM_EF_SIZE - cbUnaligned);
 # endif
 #else
-        void *pvEFence = (char *)pvBlock + (cbBlock - RTALLOC_EFENCE_SIZE);
+        void *pvEFence = (char *)pvBlock + (cbBlock - RTR0MEM_EF_SIZE);
         void *pv       = (char *)pvEFence - cbAligned;
-# ifdef RTALLOC_EFENCE_NOMAN_FILLER
-        memset(pvBlock, RTALLOC_EFENCE_NOMAN_FILLER, cbBlock - RTALLOC_EFENCE_SIZE - cbAligned);
-        memset((char *)pv + cbUnaligned, RTALLOC_EFENCE_NOMAN_FILLER, cbAligned - cbUnaligned);
+# ifdef RTR0MEM_EF_NOMAN_FILLER
+        memset(pvBlock, RTR0MEM_EF_NOMAN_FILLER, cbBlock - RTR0MEM_EF_SIZE - cbAligned);
+        memset((char *)pv + cbUnaligned, RTR0MEM_EF_NOMAN_FILLER, cbAligned - cbUnaligned);
 # endif
 #endif
 
-#ifdef RTALLOC_EFENCE_FENCE_FILLER
-        memset(pvEFence, RTALLOC_EFENCE_FENCE_FILLER, RTALLOC_EFENCE_SIZE);
-#endif
-        int rc = RTMemProtect(pvEFence, RTALLOC_EFENCE_SIZE, RTMEM_PROT_NONE);
+#ifdef RTR0MEM_EF_FENCE_FILLER
+        memset(pvEFence, RTR0MEM_EF_FENCE_FILLER, RTR0MEM_EF_SIZE);
+#endif
+        rc = RTR0MemObjProtect(hMemObj, (uint8_t *)pvEFence - (uint8_t *)pvBlock, RTR0MEM_EF_SIZE, RTMEM_PROT_NONE);
         if (!rc)
         {
-#ifdef RTALLOC_EFENCE_TRACE
-            rtmemBlockInsert(pBlock, pv);
-#endif
+            rtR0MemBlockInsert(pBlock, pv, hMemObj);
             if (enmType == RTMEMTYPE_RTMEMALLOCZ)
                 memset(pv, 0, cbUnaligned);
-#ifdef RTALLOC_EFENCE_FILLER
+#ifdef RTR0MEM_EF_FILLER
             else
-                memset(pv, RTALLOC_EFENCE_FILLER, cbUnaligned);
-#endif
-
-            rtmemLog(pszOp, "returns %p (pvBlock=%p cbBlock=%#x pvEFence=%p cbUnaligned=%#x)\n", pv, pvBlock, cbBlock, pvEFence, cbUnaligned);
+                memset(pv, RTR0MEM_EF_FILLER, cbUnaligned);
+#endif
+
+            rtR0MemLog(pszOp, "returns %p (pvBlock=%p cbBlock=%#x pvEFence=%p cbUnaligned=%#x)\n", pv, pvBlock, cbBlock, pvEFence, cbUnaligned);
             return pv;
         }
-        rtmemComplain(pszOp, "RTMemProtect failed, pvEFence=%p size %d, rc=%d\n", pvEFence, RTALLOC_EFENCE_SIZE, rc);
-        RTMemPageFree(pvBlock, cbBlock);
+        rtR0MemComplain(pszOp, "RTMemProtect failed, pvEFence=%p size %d, rc=%d\n", pvEFence, RTR0MEM_EF_SIZE, rc);
+        RTR0MemObjFree(hMemObj, true /*fFreeMappings*/);
     }
     else
-        rtmemComplain(pszOp, "Failed to allocated %lu (%lu) bytes.\n", (unsigned long)cbBlock, (unsigned long)cbUnaligned);
-
-#ifdef RTALLOC_EFENCE_TRACE
-    rtmemBlockFree(pBlock);
-#endif
+    {
+        rtR0MemComplain(pszOp, "Failed to allocated %zu (%zu) bytes (rc=%Rrc).\n", cbBlock, cbUnaligned, rc);
+        if (RT_SUCCESS(rc))
+            RTR0MemObjFree(hMemObj, true /*fFreeMappings*/);
+    }
+
+    rtR0MemBlockFree(pBlock);
     return NULL;
 }
@@ -649 +634 @@
  * Internal free.
  */
-RTDECL(void) rtR3MemFree(const char *pszOp, RTMEMTYPE enmType, void *pv, void *pvCaller, RT_SRC_POS_DECL)
+static void rtR0MemFree(const char *pszOp, RTMEMTYPE enmType, void *pv, void *pvCaller, RT_SRC_POS_DECL)
 {
     NOREF(enmType); RT_SRC_POS_NOREF();
@@ -666 +651 @@
             RTAssertDoPanic();
 
-#ifdef RTALLOC_EFENCE_TRACE
     /*
      * Find the block.
      */
-    PRTMEMBLOCK pBlock = rtmemBlockRemove(pv);
+    PRTR0MEMEFBLOCK pBlock = rtR0MemBlockRemove(pv);
     if (pBlock)
     {
@@ -676 +660 @@
             RTLogPrintf("RTMem %s: pv=%p pvCaller=%p cbUnaligned=%#x\n", pszOp, pv, pvCaller, pBlock->cbUnaligned);
 
-# ifdef RTALLOC_EFENCE_NOMAN_FILLER
+#ifdef RTR0MEM_EF_NOMAN_FILLER
         /*
          * Check whether the no man's land is untouched.
          */
-#  ifdef RTALLOC_EFENCE_IN_FRONT
+# ifdef RTR0MEM_EF_IN_FRONT
         void *pvWrong = ASMMemIsAll8((char *)pv + pBlock->cbUnaligned,
                                      RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbUnaligned,
-                                     RTALLOC_EFENCE_NOMAN_FILLER);
-#  else
+                                     RTR0MEM_EF_NOMAN_FILLER);
+# else
         /* Alignment must match allocation alignment in rtMemAlloc(). */
-        void  *pvWrong   = ASMMemIsAll8((char *)pv + pBlock->cbUnaligned,
-                                        pBlock->cbAligned - pBlock->cbUnaligned,
-                                        RTALLOC_EFENCE_NOMAN_FILLER);
+        void *pvWrong = ASMMemIsAll8((char *)pv + pBlock->cbUnaligned,
+                                     pBlock->cbAligned - pBlock->cbUnaligned,
+                                     RTR0MEM_EF_NOMAN_FILLER);
         if (pvWrong)
             RTAssertDoPanic();
         pvWrong = ASMMemIsAll8((void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK),
                                RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbAligned,
-                               RTALLOC_EFENCE_NOMAN_FILLER);
-#  endif
+                               RTR0MEM_EF_NOMAN_FILLER);
+# endif
         if (pvWrong)
             RTAssertDoPanic();
-# endif
-
-# ifdef RTALLOC_EFENCE_FREE_FILL
+#endif
+
+#ifdef RTR0MEM_EF_FREE_FILL
         /*
          * Fill the user part of the block.
          */
-        memset(pv, RTALLOC_EFENCE_FREE_FILL, pBlock->cbUnaligned);
-# endif
-
-# if defined(RTALLOC_EFENCE_FREE_DELAYED) && RTALLOC_EFENCE_FREE_DELAYED > 0
+        memset(pv, RTR0MEM_EF_FREE_FILL, pBlock->cbUnaligned);
+#endif
+
+#if defined(RTR0MEM_EF_FREE_DELAYED) && RTR0MEM_EF_FREE_DELAYED > 0
         /*
          * We're doing delayed freeing.
          * That means we'll expand the E-fence to cover the entire block.
          */
-        int rc = RTMemProtect(pv, pBlock->cbAligned, RTMEM_PROT_NONE);
+# ifdef RTR0MEM_EF_IN_FRONT
+        int rc = RTR0MemObjProtect(pBlock->hMemObj, RTR0MEM_EF_SIZE /*offSub*/, pBlock->cbAligned, RTMEM_PROT_NONE);
+# else
+        int rc = RTR0MemObjProtect(pBlock->hMemObj, 0 /*offSub*/,               pBlock->cbAligned, RTMEM_PROT_NONE);
+# endif
         if (RT_SUCCESS(rc))
         {
@@ -717 +705 @@
              * Insert it into the free list and process pending frees.
              */
-            rtmemBlockDelayInsert(pBlock);
-            while ((pBlock = rtmemBlockDelayRemove()) != NULL)
-            {
-                pv = pBlock->Core.Key;
-#  ifdef RTALLOC_EFENCE_IN_FRONT
-                void  *pvBlock = (char *)pv - RTALLOC_EFENCE_SIZE;
-#  else
-                void  *pvBlock = (void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK);
-#  endif
-                size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
-                rc = RTMemProtect(pvBlock, cbBlock, RTMEM_PROT_READ | RTMEM_PROT_WRITE);
-                if (RT_SUCCESS(rc))
-                    RTMemPageFree(pvBlock, RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE);
-                else
-                    rtmemComplain(pszOp, "RTMemProtect(%p, %#x, RTMEM_PROT_READ | RTMEM_PROT_WRITE) -> %d\n", pvBlock, cbBlock, rc);
-                rtmemBlockFree(pBlock);
-            }
+            rtR0MemBlockDelayInsert(pBlock);
+            while ((pBlock = rtR0MemBlockDelayRemove()) != NULL)
+                rtR0MemFreeBlock(pBlock, pszOp);
         }
         else
-            rtmemComplain(pszOp, "Failed to expand the efence of pv=%p cb=%d, rc=%d.\n", pv, pBlock, rc);
-
-# else /* !RTALLOC_EFENCE_FREE_DELAYED */
-
-        /*
-         * Turn of the E-fence and free it.
-         */
-#  ifdef RTALLOC_EFENCE_IN_FRONT
-        void *pvBlock = (char *)pv - RTALLOC_EFENCE_SIZE;
-        void *pvEFence = pvBlock;
-#  else
-        void *pvBlock = (void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK);
-        void *pvEFence = (char *)pv + pBlock->cb;
-#  endif
-        int rc = RTMemProtect(pvEFence, RTALLOC_EFENCE_SIZE, RTMEM_PROT_READ | RTMEM_PROT_WRITE);
-        if (RT_SUCCESS(rc))
-            RTMemPageFree(pvBlock, RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE);
-        else
-            rtmemComplain(pszOp, "RTMemProtect(%p, %#x, RTMEM_PROT_READ | RTMEM_PROT_WRITE) -> %d\n", pvEFence, RTALLOC_EFENCE_SIZE, rc);
-        rtmemBlockFree(pBlock);
-
-# endif /* !RTALLOC_EFENCE_FREE_DELAYED */
+            rtR0MemComplain(pszOp, "Failed to expand the efence of pv=%p cb=%d, rc=%d.\n", pv, pBlock, rc);
+
+#else  /* !RTR0MEM_EF_FREE_DELAYED */
+        rtR0MemFreeBlock(pBlock, pszOp);
+#endif /* !RTR0MEM_EF_FREE_DELAYED */
     }
     else
-        rtmemComplain(pszOp, "pv=%p not found! Incorrect free!\n", pv);
-
-#else /* !RTALLOC_EFENCE_TRACE */
-
-    /*
-     * We have no size tracking, so we're not doing any freeing because
-     * we cannot if the E-fence is after the block.
-     * Let's just expand the E-fence to the first page of the user bit
-     * since we know that it's around.
-     */
-    int rc = RTMemProtect((void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK), PAGE_SIZE, RTMEM_PROT_NONE);
-    if (RT_FAILURE(rc))
-        rtmemComplain(pszOp, "RTMemProtect(%p, PAGE_SIZE, RTMEM_PROT_NONE) -> %d\n", (void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK), rc);
-#endif /* !RTALLOC_EFENCE_TRACE */
+        rtR0MemComplain(pszOp, "pv=%p not found! Incorrect free!\n", pv);
 }
 
@@ -780 +724 @@
  * Internal realloc.
  */
-RTDECL(void *) rtR3MemRealloc(const char *pszOp, RTMEMTYPE enmType, void *pvOld, size_t cbNew,
-                              const char *pszTag, void *pvCaller, RT_SRC_POS_DECL)
+static void *rtR0MemRealloc(const char *pszOp, RTMEMTYPE enmType, void *pvOld, size_t cbNew,
+                            const char *pszTag, void *pvCaller, RT_SRC_POS_DECL)
 {
     /*
@@ -787 +731 @@
      */
     if (!pvOld)
-        return rtR3MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
+        return rtR0MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
     if (!cbNew)
     {
-        rtR3MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, pvCaller, RT_SRC_POS_ARGS);
+        rtR0MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, pvCaller, RT_SRC_POS_ARGS);
         return NULL;
     }
-
-#ifdef RTALLOC_EFENCE_TRACE
 
     /*
      * Get the block, allocate the new, copy the data, free the old one.
      */
-    PRTMEMBLOCK pBlock = rtmemBlockGet(pvOld);
+    PRTR0MEMEFBLOCK pBlock = rtR0MemBlockGet(pvOld);
     if (pBlock)
     {
-        void *pvRet = rtR3MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
+        void *pvRet = rtR0MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
         if (pvRet)
         {
             memcpy(pvRet, pvOld, RT_MIN(cbNew, pBlock->cbUnaligned));
-            rtR3MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, pvCaller, RT_SRC_POS_ARGS);
+            rtR0MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, pvCaller, RT_SRC_POS_ARGS);
         }
         return pvRet;
     }
-    else
-        rtmemComplain(pszOp, "pvOld=%p was not found!\n", pvOld);
+    rtR0MemComplain(pszOp, "pvOld=%p was not found!\n", pvOld);
     return NULL;
-
-#else /* !RTALLOC_EFENCE_TRACE */
-
-    rtmemComplain(pszOp, "Not supported if RTALLOC_EFENCE_TRACE isn't defined!\n");
-    return NULL;
-
-#endif /* !RTALLOC_EFENCE_TRACE */
 }
 
@@ -827 +761 @@
 RTDECL(void *)  RTMemEfTmpAlloc(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
 {
-    return rtR3MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
+    return rtR0MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
 }
 
@@ -833 +767 @@
 RTDECL(void *)  RTMemEfTmpAllocZ(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
 {
-    return rtR3MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
+    return rtR0MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
 }
 
@@ -840 +774 @@
 {
     if (pv)
-        rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS_ARGS);
+        rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS_ARGS);
 }
 
@@ -846 +780 @@
 RTDECL(void *)  RTMemEfAlloc(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
 {
-    return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
+    return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
 }
 
@@ -852 +786 @@
 RTDECL(void *)  RTMemEfAllocZ(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
 {
-    return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
+    return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
 }
 
@@ -863 +797 @@
     else
         cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
-    return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
+    return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
 }
 
@@ -874 +808 @@
     else
         cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
-    return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
+    return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
 }
 
@@ -880 +814 @@
 RTDECL(void *)  RTMemEfRealloc(void *pvOld, size_t cbNew, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
 {
-    return rtR3MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
+    return rtR0MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
 }
 
@@ -887 +821 @@
 {
     if (pv)
-        rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS_ARGS);
+        rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS_ARGS);
 }
 
@@ -924 +858 @@
 RTDECL(void *)  RTMemEfTmpAllocNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF
 {
-    return rtR3MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
+    return rtR0MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
 }
 
@@ -930 +864 @@
 RTDECL(void *)  RTMemEfTmpAllocZNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF
 {
-    return rtR3MemAlloc("TmpAllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
+    return rtR0MemAlloc("TmpAllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
 }
 
@@ -937 +871 @@
 {
     if (pv)
-        rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), NULL, 0, NULL);
+        rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), NULL, 0, NULL);
 }
 
@@ -943 +877 @@
 RTDECL(void *)  RTMemEfAllocNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF
 {
-    return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
+    return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
 }
 
@@ -949 +883 @@
 RTDECL(void *)  RTMemEfAllocZNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF
 {
-    return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
+    return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
 }
 
@@ -960 +894 @@
     else
         cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
-    return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
+    return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
 }
 
@@ -971 +905 @@
     else
         cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
-    return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
+    return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
 }
 
@@ -977 +911 @@
 RTDECL(void *)  RTMemEfReallocNP(void *pvOld, size_t cbNew, const char *pszTag) RT_NO_THROW_DEF
 {
-    return rtR3MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), NULL, 0, NULL);
+    return rtR0MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), NULL, 0, NULL);
 }
 
@@ -984 +918 @@
 {
     if (pv)
-        rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), NULL, 0, NULL);
+        rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), NULL, 0, NULL);
 }
 

trunk/src/VBox/Runtime/r0drv/initterm-r0drv.cpp

@@ -43 +43 @@
 
 #include "internal/initterm.h"
+#include "internal/mem.h"
 #include "internal/thread.h"
 
@@ -88 +89 @@
     if (RT_SUCCESS(rc))
     {
+#ifdef RTR0MEM_WITH_EF_APIS
+        rtR0MemEfInit();
+#endif
         rc = rtThreadInit();
         if (RT_SUCCESS(rc))
@@ -106 +110 @@
             rtThreadTerm();
         }
+#ifdef RTR0MEM_WITH_EF_APIS
+        rtR0MemEfTerm();
+#endif
         rtR0TermNative();
     }
@@ -119 +126 @@
     rtR0PowerNotificationTerm();
     rtR0MpNotificationTerm();
+#endif
+#ifdef RTR0MEM_WITH_EF_APIS
+    rtR0MemEfTerm();
 #endif
     rtR0TermNative();