Changeset 105643 in vbox for trunk

Timestamp:

Aug 9, 2024 9:38:35 PM (5 months ago)

Author:

vboxsync

Message:

IPRT,libs/openssl-3.1.5: Added RTMEMSAFER_F_NO_SUPLIB_ALLOC and made OpenSSL use it. OpenSSL can't use the SUPLIB secure memory allocator, as SUPR3Term may be invoked while the memory is still being used. bugref:10638

Location:

trunk

Files:

• include/iprt/memsafer.h (modified) (1 diff)
• src/VBox/Runtime/r3/memsafer-r3.cpp (modified) (1 diff)
• src/libs/openssl-3.1.5/crypto/mem_sec.c (modified) (2 diffs)

trunk/include/iprt/memsafer.h

@@ -76 +76 @@
  *          suspend-to-disk. */
 #define RTMEMSAFER_F_REQUIRE_NOT_PAGABLE    RT_BIT_32(0)
+/** Do not use the SUPLib allocator.
+ * The SUPLib allocator may be terminated earlier than IPRT,
+ * automatically rendering the allocations inaccessible. */
+#define RTMEMSAFER_F_NO_SUPLIB_ALLOC        RT_BIT_32(1)
 /** Mask of valid bits.  */
-#define RTMEMSAFER_F_VALID_MASK             UINT32_C(0x00000001)
+#define RTMEMSAFER_F_VALID_MASK             UINT32_C(0x00000003)
 /** @} */
 

trunk/src/VBox/Runtime/r3/memsafer-r3.cpp

@@ -422 +422 @@
              * falling back on the less safe one.
              */
-            rc = rtMemSaferSupR3AllocPages(pThis);
+            if (!(fFlags & RTMEMSAFER_F_NO_SUPLIB_ALLOC))
+                rc = rtMemSaferSupR3AllocPages(pThis);
+            else
+                rc = VERR_NO_PAGE_MEMORY;
             if (RT_SUCCESS(rc))
                 pThis->enmAllocator = RTMEMSAFERALLOCATOR_SUPR3;

trunk/src/libs/openssl-3.1.5/crypto/mem_sec.c

@@ -175 +175 @@
 #else
     RT_NOREF(line);
-    return RTMemSaferAllocZTag(num, file);
+    void *pvRet = NULL;
+    int rc = RTMemSaferAllocZExTag(&pvRet, num, RTMEMSAFER_F_NO_SUPLIB_ALLOC, file);
+    AssertRCReturn(rc, NULL);
+    return pvRet;
 #endif /* VBOX */
 }
@@ -190 +193 @@
 #else
     RT_NOREF(line);
-    return RTMemSaferAllocZTag(num, file);
+    void *pvRet = NULL;
+    int rc = RTMemSaferAllocZExTag(&pvRet, num, RTMEMSAFER_F_NO_SUPLIB_ALLOC, file);
+    AssertRCReturn(rc, NULL);
+    return pvRet;
 #endif
 }