Changeset 33104 in vbox for trunk

Timestamp:

Oct 13, 2010 12:46:59 PM (14 years ago)

Author:

vboxsync

Message:

RTPipeCreate: Use pipe2 on linux (added in 2.6.27) to avoid racing process creation.

File:

• trunk/src/VBox/Runtime/r3/posix/pipe-posix.cpp (modified) (3 diffs)

trunk/src/VBox/Runtime/r3/posix/pipe-posix.cpp

@@ -47 +47 @@
 #include <sys/stat.h>
 #include <signal.h>
+#ifdef RT_OS_LINUX
+# include <sys/syscall.h>
+#endif
 
 
@@ -82 +85 @@
 
 
+
+/**
+ * Wrapper for calling pipe2() or pipe().
+ *
+ * When using pipe2() the returned handles are marked close-on-exec and does
+ * not risk racing process creation calls on other threads.
+ *
+ * @returns See pipe().
+ * @param   paFds               See pipe().
+ * @param   piNewPipeSyscall    Where to cache which call we should used. -1 if
+ *                              pipe(), 1 if pipe2(), 0 if not yet decided.
+ */
+static int my_pipe_wrapper(int *paFds, int *piNewPipeSyscall)
+{
+    if (*piNewPipeSyscall >= 0)
+    {
+#if defined(RT_OS_LINUX) && defined(__NR_pipe2)
+        long rc = syscall(__NR_pipe2, paFds, O_CLOEXEC);
+        if (rc >= 0)
+        {
+            if (*piNewPipeSyscall == 0)
+                *piNewPipeSyscall = 1;
+            return (int)rc;
+        }
+#endif
+        *piNewPipeSyscall = -1;
+    }
+
+    return pipe(paFds);
+}
+
+
 RTDECL(int)  RTPipeCreate(PRTPIPE phPipeRead, PRTPIPE phPipeWrite, uint32_t fFlags)
 {
@@ -89 +124 @@
 
     /*
-     * Create the pipe and set the close-on-exec flag if requested.
+     * Create the pipe and clear/set the close-on-exec flag as required.
      */
     int aFds[2] = {-1, -1};
-    if (pipe(aFds))
+    static int s_iNewPipeSyscall = 0;
+    if (my_pipe_wrapper(aFds, &s_iNewPipeSyscall))
         return RTErrConvertFromErrno(errno);
 
     int rc = VINF_SUCCESS;
-    if (!(fFlags & RTPIPE_C_INHERIT_READ))
-    {
-        if (fcntl(aFds[0], F_SETFD, FD_CLOEXEC))
-            rc = RTErrConvertFromErrno(errno);
-    }
-
-    if (!(fFlags & RTPIPE_C_INHERIT_WRITE))
-    {
-        if (fcntl(aFds[1], F_SETFD, FD_CLOEXEC))
-            rc = RTErrConvertFromErrno(errno);
+    if (s_iNewPipeSyscall > 0)
+    {
+        /* created with close-on-exec set. */
+        if (fFlags & RTPIPE_C_INHERIT_READ)
+        {
+            if (fcntl(aFds[0], F_SETFD, 0))
+                rc = RTErrConvertFromErrno(errno);
+        }
+
+        if (fFlags & RTPIPE_C_INHERIT_WRITE)
+        {
+            if (fcntl(aFds[1], F_SETFD, 0))
+                rc = RTErrConvertFromErrno(errno);
+        }
+    }
+    else
+    {
+        /* created with close-on-exec cleared. */
+        if (!(fFlags & RTPIPE_C_INHERIT_READ))
+        {
+            if (fcntl(aFds[0], F_SETFD, FD_CLOEXEC))
+                rc = RTErrConvertFromErrno(errno);
+        }
+
+        if (!(fFlags & RTPIPE_C_INHERIT_WRITE))
+        {
+            if (fcntl(aFds[1], F_SETFD, FD_CLOEXEC))
+                rc = RTErrConvertFromErrno(errno);
+        }
     }