Changeset 36465 in vbox

Timestamp:

Mar 29, 2011 4:22:33 PM (14 years ago)

Author:

vboxsync

Message:

Runtime/r3/solaris/coredumper: replace RTFileOpen() with open() syscalls to avoid touching the heap.

File:

• trunk/src/VBox/Runtime/r3/solaris/coredumper-solaris.cpp (modified) (17 diffs)

trunk/src/VBox/Runtime/r3/solaris/coredumper-solaris.cpp

@@ -53 +53 @@
 # include <sys/systeminfo.h>
 # include <sys/mman.h>
+# include <sys/types.h>
+# include <sys/stat.h>
+# include <fcntl.h>
 # include <ucontext.h>
 #endif  /* RT_OS_SOLARIS */
@@ -212 +215 @@
 {
     uint64_t cb = 0;
-    RTFILE hFile;
-    int rc = RTFileOpen(&hFile, pszPath, RTFILE_O_OPEN | RTFILE_O_READ);
-    if (RT_SUCCESS(rc))
-    {
+    int fd = open(pszPath, O_RDONLY);
+    if (fd >= 0)
+    {
+        RTFILE hFile = fd;
         RTFileGetSize(hFile, &cb);
         RTFileClose(hFile);
     }
     else
-        CORELOGRELSYS((CORELOG_NAME "GetFileSize failed to open %s rc=%Rrc\n", pszPath, rc));
+        CORELOGRELSYS((CORELOG_NAME "GetFileSize: failed to open %s rc=%Rrc\n", pszPath, RTErrConvertFromErrno(fd)));
     return cb < ~(size_t)0 ? (size_t)cb : ~(size_t)0;
 }
@@ -350 +353 @@
  * @param pcb               Where to store size of the buffer.
  *
- * @return IPRT status code.
+ * @return IPRT status code. If the proc file is 0 bytes, VINF_SUCCESS is
+ *          returned with pointed to values of @c ppv, @c pcb set to NULL and 0
+ *          respectively.
  */
 static int ProcReadFileInto(PVBOXCORE pVBoxCore, const char *pszProcFileName, void **ppv, size_t *pcb)
@@ -358 +363 @@
     char szPath[PATH_MAX];
     RTStrPrintf(szPath, sizeof(szPath), "/proc/%d/%s", (int)pVBoxCore->VBoxProc.Process, pszProcFileName);
-    RTFILE hFile;
-    int rc = RTFileOpen(&hFile, szPath, RTFILE_O_OPEN | RTFILE_O_READ);
-    if (RT_SUCCESS(rc))
-    {
+    int rc = VINF_SUCCESS;
+    int fd = open(szPath, O_RDONLY);
+    if (fd >= 0)
+    {
+        RTFILE hFile = fd;
         uint64_t u64Size;
         RTFileGetSize(hFile, &u64Size);
@@ -377 +383 @@
             *pcb =  0;
             *ppv = NULL;
+            rc = VINF_SUCCESS;
         }
         RTFileClose(hFile);
     }
     else
+    {
+        rc = RTErrConvertFromErrno(fd);
         CORELOGRELSYS((CORELOG_NAME "ProcReadFileInto: failed to open %s. rc=%Rrc\n", szPath, rc));
+    }
     return rc;
 }
@@ -399 +409 @@
     PVBOXPROCESS pVBoxProc = &pVBoxCore->VBoxProc;
     char szPath[PATH_MAX];
-    RTFILE hFile;
+    int rc = VINF_SUCCESS;
 
     RTStrPrintf(szPath, sizeof(szPath), "/proc/%d/psinfo", (int)pVBoxProc->Process);
-    int rc = RTFileOpen(&hFile, szPath, RTFILE_O_OPEN | RTFILE_O_READ);
-    if (RT_SUCCESS(rc))
-    {
+    int fd = open(szPath, O_RDONLY);
+    if (fd >= 0)
+    {
+        RTFILE hFile = fd;
         size_t cbProcInfo = sizeof(psinfo_t);
         rc = ReadFileNoIntr(hFile, &pVBoxProc->ProcInfo, cbProcInfo);
-    }
-
-    RTFileClose(hFile);
+        RTFileClose(hFile);
+    }
+    else
+    {
+        rc = RTErrConvertFromErrno(fd);
+        CORELOGRELSYS((CORELOG_NAME "ProcReadInfo: failed to open %s. rc=%Rrc\n", szPath, rc));
+    }
+
     return rc;
 }
@@ -428 +444 @@
 
     char szPath[PATH_MAX];
-    RTFILE hFile;
+    int rc = VINF_SUCCESS;
 
     RTStrPrintf(szPath, sizeof(szPath), "/proc/%d/status", (int)pVBoxProc->Process);
-    int rc = RTFileOpen(&hFile, szPath, RTFILE_O_OPEN | RTFILE_O_READ);
-    if (RT_SUCCESS(rc))
-    {
+    int fd = open(szPath, O_RDONLY);
+    if (fd >= 0)
+    {
+        RTFILE hFile = fd;
         size_t cbRead;
         size_t cbProcStatus = sizeof(pstatus_t);
         AssertCompile(sizeof(pstatus_t) == sizeof(pVBoxProc->ProcStatus));
         rc = ReadFileNoIntr(hFile, &pVBoxProc->ProcStatus, cbProcStatus);
-    }
-    RTFileClose(hFile);
+        RTFileClose(hFile);
+    }
+    else
+    {
+        rc = RTErrConvertFromErrno(fd);
+        CORELOGRELSYS((CORELOG_NAME "ProcReadStatus: failed to open %s. rc=%Rrc\n", szPath, rc));
+    }
     return rc;
 }
@@ -518 +540 @@
     PVBOXPROCESS pVBoxProc = &pVBoxCore->VBoxProc;
     char szPath[PATH_MAX];
-    RTFILE hFile = NIL_RTFILE;
+    int rc = VINF_SUCCESS;
     RTStrPrintf(szPath, sizeof(szPath), "/proc/%d/auxv", (int)pVBoxProc->Process);
-    int rc = RTFileOpen(&hFile, szPath, RTFILE_O_OPEN | RTFILE_O_READ);
-    if (RT_FAILURE(rc))
-    {
-        CORELOGRELSYS((CORELOG_NAME "ProcReadAuxVecs: RTFileOpen %s failed rc=%Rrc\n", szPath, rc));
+    int fd = open(szPath, O_RDONLY);
+    if (fd < 0)
+    {
+        rc = RTErrConvertFromErrno(fd);
+        CORELOGRELSYS((CORELOG_NAME "ProcReadAuxVecs: failed to open %s rc=%Rrc\n", szPath, rc));
         return rc;
     }
 
+    RTFILE hFile = fd;
     uint64_t u64Size;
     RTFileGetSize(hFile, &u64Size);
@@ -568 +592 @@
     }
     else
+    {
         CORELOGRELSYS((CORELOG_NAME "ProcReadAuxVecs: aux file too small %u, expecting %u or more\n", cbAuxFile, sizeof(auxv_t)));
+        rc = VERR_READ_ERROR;
+    }
 
     RTFileClose(hFile);
@@ -608 +635 @@
     PVBOXPROCESS pVBoxProc = &pVBoxCore->VBoxProc;
     char szPath[PATH_MAX];
-    RTFILE hFile = NIL_RTFILE;
+    int rc = VINF_SUCCESS;
     RTStrPrintf(szPath, sizeof(szPath), "/proc/%d/map", (int)pVBoxProc->Process);
-    int rc = RTFileOpen(&hFile, szPath, RTFILE_O_OPEN | RTFILE_O_READ);
-    if (RT_FAILURE(rc))
+    int fd = open(szPath, O_RDONLY);
+    if (fd < 0)
+    {
+        rc = RTErrConvertFromErrno(fd);
+        CORELOGRELSYS((CORELOG_NAME "ProcReadMappings: failed to open %s. rc=%Rrc\n", szPath, rc));
         return rc;
-
+    }
+
+    RTFILE hFile = fd;
     RTStrPrintf(szPath, sizeof(szPath), "/proc/%d/as", (int)pVBoxProc->Process);
-    rc = RTFileOpen(&pVBoxProc->hAs, szPath, RTFILE_O_OPEN | RTFILE_O_READ);
-    if (RT_SUCCESS(rc))
-    {
+    fd = open(szPath, O_RDONLY);
+    if (fd >= 0)
+    {
+        pVBoxProc->hAs = fd;
+
         /*
          * Allocate and read all the prmap_t objects from proc.
@@ -1147 +1181 @@
     RTStrPrintf(szLpsInfoPath, sizeof(szLpsInfoPath), "/proc/%d/lpsinfo", (int)pVBoxProc->Process);
 
-    RTFILE hFile = NIL_RTFILE;
-    int rc = RTFileOpen(&hFile, szLpsInfoPath, RTFILE_O_READ);
-    if (RT_SUCCESS(rc))
-    {
+    int rc = VINF_SUCCESS;
+    int fd = open(szLpsInfoPath, O_RDONLY);
+    if (fd >= 0)
+    {
+        RTFILE hFile = fd;
         uint64_t u64Size;
         RTFileGetSize(hFile, &u64Size);
@@ -1177 +1212 @@
         RTFileClose(hFile);
     }
+    else
+        rc = RTErrConvertFromErrno(rc);
 
     return rc;
@@ -1752 +1789 @@
  * @param pfnWriter         Pointer to the writer function to override default writer (NULL uses default).
  *
- * @remarks Resumes all suspended threads, unless it's an invalid core.
+ * @remarks Resumes all suspended threads, unless it's an invalid core. This
+ *          function must be called only -after- rtCoreDumperCreateCore().
  * @return VBox status.
  */
@@ -1767 +1805 @@
     PVBOXPROCESS pVBoxProc = &pVBoxCore->VBoxProc;
     char szPath[PATH_MAX];
+    int rc = VINF_SUCCESS;
 
     /*
@@ -1772 +1811 @@
      */
     RTStrPrintf(szPath, sizeof(szPath), "/proc/%d/as", (int)pVBoxProc->Process);
-    int rc = RTFileOpen(&pVBoxProc->hAs, szPath, RTFILE_O_OPEN | RTFILE_O_READ);
-    if (RT_FAILURE(rc))
-    {
+    int fd = open(szPath, O_RDONLY);
+    if (fd < 0)
+    {
+        rc = RTErrConvertFromErrno(fd);
         CORELOGRELSYS((CORELOG_NAME "WriteCore: Failed to open address space, %s. rc=%Rrc\n", szPath, rc));
         goto WriteCoreDone;
     }
 
+    pVBoxProc->hAs = fd;
+
     /*
      * Create the core file.
      */
-    rc = RTFileOpen(&pVBoxCore->hCoreFile, pVBoxCore->szCorePath,
-                    RTFILE_O_OPEN_CREATE | RTFILE_O_TRUNCATE | RTFILE_O_READWRITE | RTFILE_O_DENY_ALL);
-    if (RT_FAILURE(rc))
-    {
+    fd = open(pVBoxCore->szCorePath, O_CREAT | O_TRUNC | O_RDWR, S_IRUSR);
+    if (fd < 0)
+    {
+        rc = RTErrConvertFromErrno(fd);
         CORELOGRELSYS((CORELOG_NAME "WriteCore: failed to open %s. rc=%Rrc\n", pVBoxCore->szCorePath, rc));
         goto WriteCoreDone;
     }
+
+    pVBoxCore->hCoreFile = fd;
 
     pVBoxCore->offWrite = 0;
@@ -1904 +1948 @@
 
 WriteCoreDone:
-    if (pVBoxCore->hCoreFile != NIL_RTFILE)
+    if (pVBoxCore->hCoreFile != NIL_RTFILE)     /* Initialized in rtCoreDumperCreateCore() */
     {
         RTFileClose(pVBoxCore->hCoreFile);
@@ -1910 +1954 @@
     }
 
-    if (pVBoxProc->hAs != NIL_RTFILE)
+    if (pVBoxProc->hAs != NIL_RTFILE)           /* Initialized in rtCoreDumperCreateCore() */
     {
         RTFileClose(pVBoxProc->hAs);