Changeset 86412 in vbox for trunk/src/VBox/Runtime

Timestamp:

Oct 2, 2020 11:39:26 AM (4 years ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

140713

Message:

IPRT/pipe: Adding RTPipeCloseEx w/ a fLeaveOpen parameter so we can prevent leaks via RTHandleGetStandard. Adding RTPIPE_N_LEAVE_OPEN to RTPipeFromNative. bugref:9841

Location:

trunk/src/VBox/Runtime/r3

Files:

• os2/pipe-os2.cpp (modified) (6 diffs)
• posix/pipe-posix.cpp (modified) (6 diffs)
• win/pipe-win.cpp (modified) (8 diffs)

trunk/src/VBox/Runtime/r3/os2/pipe-os2.cpp

@@ -68 +68 @@
     /** Set if this is the read end, clear if it's the write end. */
     bool                fRead;
+    /** RTPipeFromNative: Leave open. */
+    bool                fLeaveOpen;
     /** Whether the pipe is in blocking or non-blocking mode. */
     bool                fBlocking;
@@ -195 +197 @@
                     pThisR->fRead           = true;
                     pThisW->fRead           = false;
+                    pThisR->fLeaveOpen      = false;
+                    pThisW->fLeaveOpen      = false;
                     pThisR->fBlocking       = false;
                     pThisW->fBlocking       = true;
@@ -227 +231 @@
 
 
-RTDECL(int)  RTPipeClose(RTPIPE hPipe)
+RTDECL(int)  RTPipeCloseEx(RTPIPE hPipe, bool fLeaveOpen)
 {
     RTPIPEINTERNAL *pThis = hPipe;
@@ -243 +247 @@
 
     /* Don't call DosDisConnectNPipe! */
-    DosClose(pThis->hPipe);
+    if (!fLeaveOpen && !pThis->fLeaveOpen)
+        DosClose(pThis->hPipe);
     pThis->hPipe = (HPIPE)-1;
 
@@ -261 +266 @@
 
 
+RTDECL(int)  RTPipeClose(RTPIPE hPipe)
+{
+    return RTPipeCloseEx(hPipe, false /*fLeaveOpen*/);
+}
+
+
 RTDECL(int)  RTPipeFromNative(PRTPIPE phPipe, RTHCINTPTR hNativePipe, uint32_t fFlags)
 {
     AssertPtrReturn(phPipe, VERR_INVALID_POINTER);
-    AssertReturn(!(fFlags & ~RTPIPE_N_VALID_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(!(fFlags & ~RTPIPE_N_VALID_MASK_FN), VERR_INVALID_PARAMETER);
     AssertReturn(!!(fFlags & RTPIPE_N_READ) != !!(fFlags & RTPIPE_N_WRITE), VERR_INVALID_PARAMETER);
 
@@ -333 +344 @@
         pThis->hPipe           = hNative;
         pThis->hev             = NULLHANDLE;
-        pThis->fRead           = !!(fFlags & RTPIPE_N_READ);
+        pThis->fRead           = RT_BOOL(fFlags & RTPIPE_N_READ);
+        pThis->fLeaveOpen      = RT_BOOL(fFlags & RTPIPE_N_LEAVE_OPEN);
         pThis->fBlocking       = !(fPipeState & NP_NOWAIT);
         //pThis->fBrokenPipe     = false;

trunk/src/VBox/Runtime/r3/posix/pipe-posix.cpp

@@ -70 +70 @@
     /** Set if this is the read end, clear if it's the write end. */
     bool                fRead;
+    /** RTPipeFromNative: Leave it open on RTPipeClose. */
+    bool                fLeaveOpen;
     /** Atomically operated state variable.
      *
@@ -181 +183 @@
             if (pThisW)
             {
-                pThisR->u32Magic = RTPIPE_MAGIC;
-                pThisW->u32Magic = RTPIPE_MAGIC;
-                pThisR->fd       = aFds[0];
-                pThisW->fd       = aFds[1];
-                pThisR->fRead    = true;
-                pThisW->fRead    = false;
-                pThisR->u32State = RTPIPE_POSIX_BLOCKING;
-                pThisW->u32State = RTPIPE_POSIX_BLOCKING;
+                pThisR->u32Magic    = RTPIPE_MAGIC;
+                pThisW->u32Magic    = RTPIPE_MAGIC;
+                pThisR->fd          = aFds[0];
+                pThisW->fd          = aFds[1];
+                pThisR->fRead       = true;
+                pThisW->fRead       = false;
+                pThisR->fLeaveOpen  = false;
+                pThisW->fLeaveOpen  = false;
+                pThisR->u32State    = RTPIPE_POSIX_BLOCKING;
+                pThisW->u32State    = RTPIPE_POSIX_BLOCKING;
 
                 *phPipeRead  = pThisR;
@@ -213 +217 @@
 
 
-RTDECL(int)  RTPipeClose(RTPIPE hPipe)
+RTDECL(int)  RTPipeCloseEx(RTPIPE hPipe, bool fLeaveOpen)
 {
     RTPIPEINTERNAL *pThis = hPipe;
@@ -228 +232 @@
     int fd = pThis->fd;
     pThis->fd = -1;
-    close(fd);
+    if (!fLeaveOpen && !pThis->fLeaveOpen)
+        close(fd);
 
     if (ASMAtomicReadU32(&pThis->u32State) & RTPIPE_POSIX_USERS_MASK)
@@ -241 +246 @@
 }
 
+
+RTDECL(int)  RTPipeClose(RTPIPE hPipe)
+{
+    return RTPipeCloseEx(hPipe, false /*fLeaveOpen*/);
+}
+
+
 RTDECL(int)  RTPipeFromNative(PRTPIPE phPipe, RTHCINTPTR hNativePipe, uint32_t fFlags)
 {
     AssertPtrReturn(phPipe, VERR_INVALID_POINTER);
-    AssertReturn(!(fFlags & ~RTPIPE_N_VALID_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(!(fFlags & ~RTPIPE_N_VALID_MASK_FN), VERR_INVALID_PARAMETER);
     AssertReturn(!!(fFlags & RTPIPE_N_READ) != !!(fFlags & RTPIPE_N_WRITE), VERR_INVALID_PARAMETER);
 
@@ -268 +280 @@
         return VERR_NO_MEMORY;
 
-    pThis->u32Magic = RTPIPE_MAGIC;
-    pThis->fd       = hNative;
-    pThis->fRead    = !!(fFlags & RTPIPE_N_READ);
-    pThis->u32State = fFd & O_NONBLOCK ? 0 : RTPIPE_POSIX_BLOCKING;
+    pThis->u32Magic   = RTPIPE_MAGIC;
+    pThis->fd         = hNative;
+    pThis->fRead      = RT_BOOL(fFlags & RTPIPE_N_READ);
+    pThis->fLeaveOpen = RT_BOOL(fFlags & RTPIPE_N_LEAVE_OPEN);
+    pThis->u32State   = fFd & O_NONBLOCK ? 0 : RTPIPE_POSIX_BLOCKING;
 
     /*

trunk/src/VBox/Runtime/r3/win/pipe-win.cpp

@@ -67 +67 @@
     /** Set if this is the read end, clear if it's the write end. */
     bool                fRead;
+    /** RTPipeFromNative: Leave native handle open on RTPipeClose. */
+    bool                fLeaveOpen;
     /** Set if there is already pending I/O. */
     bool                fIOPending;
@@ -284 +286 @@
                             pThisR->fRead               = true;
                             pThisW->fRead               = false;
+                            pThisR->fLeaveOpen          = false;
+                            pThisW->fLeaveOpen          = false;
                             //pThisR->fIOPending        = false;
                             //pThisW->fIOPending        = false;
@@ -400 +404 @@
 
 
-RTDECL(int)  RTPipeClose(RTPIPE hPipe)
+RTDECL(int)  RTPipeCloseEx(RTPIPE hPipe, bool fLeaveOpen)
 {
     RTPIPEINTERNAL *pThis = hPipe;
@@ -418 +422 @@
         rtPipeWriteCheckCompletion(pThis);
 
-    CloseHandle(pThis->hPipe);
+    if (!fLeaveOpen && !pThis->fLeaveOpen)
+        CloseHandle(pThis->hPipe);
     pThis->hPipe = INVALID_HANDLE_VALUE;
 
@@ -436 +441 @@
 
 
+RTDECL(int)  RTPipeClose(RTPIPE hPipe)
+{
+    return RTPipeCloseEx(hPipe, false /*fLeaveOpen*/);
+}
+
+
 RTDECL(int)  RTPipeFromNative(PRTPIPE phPipe, RTHCINTPTR hNativePipe, uint32_t fFlags)
 {
     AssertPtrReturn(phPipe, VERR_INVALID_POINTER);
-    AssertReturn(!(fFlags & ~RTPIPE_N_VALID_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(!(fFlags & ~RTPIPE_N_VALID_MASK_FN), VERR_INVALID_PARAMETER);
     AssertReturn(!!(fFlags & RTPIPE_N_READ) != !!(fFlags & RTPIPE_N_WRITE), VERR_INVALID_PARAMETER);
 
@@ -482 +493 @@
             pThis->u32Magic             = RTPIPE_MAGIC;
             pThis->hPipe                = hNative;
-            pThis->fRead                = !!(fFlags & RTPIPE_N_READ);
+            pThis->fRead                = RT_BOOL(fFlags & RTPIPE_N_READ);
+            pThis->fLeaveOpen           = RT_BOOL(fFlags & RTPIPE_N_LEAVE_OPEN);
             //pThis->fIOPending         = false;
             //pThis->fZeroByteRead      = false;
@@ -510 +522 @@
                     pThis->hPipe = hNative2;
                     if (rtPipeQueryNtInfo(pThis, &Info))
+                    {
+                        pThis->fLeaveOpen = false;
                         rc = VINF_SUCCESS;
+                    }
                     else
                     {
@@ -552 +567 @@
                     if (hNative2 != INVALID_HANDLE_VALUE)
                     {
-                        if (   hNative != GetStdHandle(STD_INPUT_HANDLE)
+                        if (   !(fFlags & RTPIPE_N_LEAVE_OPEN)
+                            && hNative != GetStdHandle(STD_INPUT_HANDLE)
                             && hNative != GetStdHandle(STD_OUTPUT_HANDLE)
-                            && hNative != GetStdHandle(STD_ERROR_HANDLE))
+                            && hNative != GetStdHandle(STD_ERROR_HANDLE) )
                             CloseHandle(hNative);
                     }