Changeset 96475 in vbox for trunk/src/VBox/Runtime/r3/socket.cpp

Timestamp:

Aug 25, 2022 2:27:54 AM (2 years ago)

Author:

vboxsync

Message:

IPRT: Resolve SetHandleInformation and GetHandleInformation dynamically and let the affected code deal with it specifically. Refactored the rtSocketCreate function to combine with setting inheritance, so we can use the WSA_FLAG_NO_HANDLE_INHERIT when available to avoid explicitly setting inheritance. Made RTSocketSetInheritance check the status before trying to set it, returning VERR_NET_NOT_UNSUPPORTED if tool old windows version. Also made GetVersionExW go via a function pointer since it wasn't there in NT 3.1. bugref:10261

File:

• trunk/src/VBox/Runtime/r3/socket.cpp (modified) (4 diffs)

trunk/src/VBox/Runtime/r3/socket.cpp

@@ -572 +572 @@
  * @param   iType               The socket type (SOCK_XXX).
  * @param   iProtocol           Socket parameter, usually 0.
- */
-DECLHIDDEN(int) rtSocketCreate(PRTSOCKET phSocket, int iDomain, int iType, int iProtocol)
+ * @param   fInheritable        Set to true if the socket should be inherted by
+ *                              child processes, false if not inheritable.
+ */
+DECLHIDDEN(int) rtSocketCreate(PRTSOCKET phSocket, int iDomain, int iType, int iProtocol, bool fInheritable)
 {
 #ifdef RT_OS_WINDOWS
@@ -587 +589 @@
     /*
      * Create the socket.
-     */
-#ifdef RT_OS_WINDOWS
-    RTSOCKETNATIVE hNative = g_pfnsocket(iDomain, iType, iProtocol);
+     *
+     * The RTSocketSetInheritance operation isn't necessarily reliable on windows,
+     * so try use WSA_FLAG_NO_HANDLE_INHERIT with WSASocketW when possible.
+     */
+#ifdef RT_OS_WINDOWS
+    bool           fCallSetInheritance = true;
+    RTSOCKETNATIVE hNative;
+    if (g_pfnWSASocketW)
+    {
+        DWORD fWsaFlags = WSA_FLAG_OVERLAPPED | (!fInheritable ? WSA_FLAG_NO_HANDLE_INHERIT : 0);
+        hNative = g_pfnWSASocketW(iDomain, iType, iProtocol, NULL, 0 /*Group*/, fWsaFlags);
+        if (hNative != NIL_RTSOCKETNATIVE)
+            fCallSetInheritance = false;
+        else
+        {
+            if (!fInheritable)
+                hNative = g_pfnsocket(iDomain, iType, iProtocol);
+            if (hNative == NIL_RTSOCKETNATIVE)
+                return rtSocketError();
+        }
+    }
+    else
+    {
+        hNative = g_pfnsocket(iDomain, iType, iProtocol);
+        if (hNative == NIL_RTSOCKETNATIVE)
+            return rtSocketError();
+    }
 #else
     RTSOCKETNATIVE hNative = socket(iDomain, iType, iProtocol);
-#endif
     if (hNative == NIL_RTSOCKETNATIVE)
         return rtSocketError();
+#endif
 
     /*
@@ -600 +626 @@
      */
     int rc = rtSocketCreateForNative(phSocket, hNative, false /*fLeaveOpen*/);
-    if (RT_FAILURE(rc))
+    if (RT_SUCCESS(rc))
+    {
+#ifdef RT_OS_WINDOWS
+        if (fCallSetInheritance)
+#endif
+            RTSocketSetInheritance(*phSocket, fInheritable);
+    }
+    else
     {
 #ifdef RT_OS_WINDOWS
@@ -943 +976 @@
     AssertReturn(RTMemPoolRefCount(pThis) >= (pThis->cUsers ? 2U : 1U), VERR_CALLER_NO_REFERENCE);
 
-    int rc = VINF_SUCCESS;
-#ifdef RT_OS_WINDOWS
-    if (!SetHandleInformation((HANDLE)pThis->hNative, HANDLE_FLAG_INHERIT, fInheritable ? HANDLE_FLAG_INHERIT : 0))
-        rc = RTErrConvertFromWin32(GetLastError());
+#ifndef RT_OS_WINDOWS
+    if (fcntl(pThis->hNative, F_SETFD, fInheritable ? 0 : FD_CLOEXEC) < 0)
+        return RTErrConvertFromErrno(errno);
+    return VINF_SUCCESS;
 #else
-    if (fcntl(pThis->hNative, F_SETFD, fInheritable ? 0 : FD_CLOEXEC) < 0)
-        rc = RTErrConvertFromErrno(errno);
-#endif
-
-    return rc;
+    /* Windows is more complicated as sockets are complicated wrt inheritance
+       (see stackoverflow for details).  In general, though we cannot hope to
+       make a socket really non-inheritable before vista as other layers in
+       the winsock maze may have additional handles associated with the socket. */
+    if (g_pfnGetHandleInformation)
+    {
+        /* Check if the handle is already in what seems to be the right state
+           before we try doing anything. */
+        DWORD fFlags;
+        if (g_pfnGetHandleInformation((HANDLE)pThis->hNative, &fFlags))
+        {
+            if (RT_BOOL(fFlags & HANDLE_FLAG_INHERIT) == fInheritable)
+                return VINF_SUCCESS;
+        }
+    }
+
+    if (!g_pfnSetHandleInformation)
+        return VERR_NET_NOT_UNSUPPORTED;
+
+    if (!g_pfnSetHandleInformation((HANDLE)pThis->hNative, HANDLE_FLAG_INHERIT, fInheritable ? HANDLE_FLAG_INHERIT : 0))
+        return RTErrConvertFromWin32(GetLastError());
+    /** @todo Need we do something related to WS_SIO_ASSOCIATE_HANDLE or
+     *        WS_SIO_TRANSLATE_HANDLE? Or what  other handles could be associated
+     *        with the socket? that we need to modify? */
+
+    return VINF_SUCCESS;
+#endif
 }