Changeset 5756 in vbox for trunk/src/VBox

Timestamp:

Nov 15, 2007 1:41:03 PM (17 years ago)

Author:

vboxsync

Message:

Client end of named pipe must be opened in overlapped mode, else writes from client
will be severely delayed and the client VM will get stuck on disconnect.

Also changed the type of NamedPipe variable to HANDLE to avoid casts and replaced
calls to RTFileClose with CloseHandle since all other calls operating on NamedPipe
were Windows API anyway.

File:

• trunk/src/VBox/Devices/Serial/DrvNamedPipe.cpp (modified) (14 diffs)

trunk/src/VBox/Devices/Serial/DrvNamedPipe.cpp

@@ -70 +70 @@
 #ifdef RT_OS_WINDOWS
     /* File handle of the named pipe. */
-    RTFILE              NamedPipe;
+    HANDLE              NamedPipe;
     /* Overlapped structure for writes. */
     OVERLAPPED          OverlappedWrite;
@@ -104 +104 @@
     Assert(pvBuf);
 #ifdef RT_OS_WINDOWS
-    if (pData->NamedPipe != NIL_RTFILE)
+    if (pData->NamedPipe != INVALID_HANDLE_VALUE)
     {
         DWORD cbReallyRead;
         pData->OverlappedRead.Offset     = 0;
         pData->OverlappedRead.OffsetHigh = 0;
-        if (!ReadFile((HANDLE)pData->NamedPipe, pvBuf, *cbRead, &cbReallyRead, &pData->OverlappedRead))
+        if (!ReadFile(pData->NamedPipe, pvBuf, *cbRead, &cbReallyRead, &pData->OverlappedRead))
         {
             DWORD uError = GetLastError();
@@ -129 +129 @@
 
                     /* Wait for incoming bytes. */
-                    if (GetOverlappedResult((HANDLE)pData->NamedPipe, &pData->OverlappedRead, &cbReallyRead, TRUE) == FALSE)
+                    if (GetOverlappedResult(pData->NamedPipe, &pData->OverlappedRead, &cbReallyRead, TRUE) == FALSE)
                         uError = GetLastError();
                 }
@@ -148 +148 @@
 
             {
-                RTFILE tmp = pData->NamedPipe;
-                FlushFileBuffers((HANDLE)tmp);
-                DisconnectNamedPipe((HANDLE)tmp);
+                FlushFileBuffers(pData->NamedPipe);
+                DisconnectNamedPipe(pData->NamedPipe);
                 if (!pData->fIsServer)
                 {
-                    pData->NamedPipe = NIL_RTFILE;
-                    RTFileClose(tmp);
+                    CloseHandle(pData->NamedPipe);
+                    pData->NamedPipe = INVALID_HANDLE_VALUE;
                 }
                 /* pretend success */
@@ -202 +201 @@
     Assert(pvBuf);
 #ifdef RT_OS_WINDOWS
-    if (pData->NamedPipe != NIL_RTFILE)
+    if (pData->NamedPipe != INVALID_HANDLE_VALUE)
     {
         unsigned cbWritten;
         pData->OverlappedWrite.Offset     = 0;
         pData->OverlappedWrite.OffsetHigh = 0;
-        if (!WriteFile((HANDLE)pData->NamedPipe, pvBuf, *cbWrite, NULL, &pData->OverlappedWrite))
+        if (!WriteFile(pData->NamedPipe, pvBuf, *cbWrite, NULL, &pData->OverlappedWrite))
         {
             DWORD uError = GetLastError();
@@ -226 +225 @@
             {
                 /* Wait for the write to complete. */
-                if (GetOverlappedResult((HANDLE)pData->NamedPipe, &pData->OverlappedWrite, (DWORD *)&cbWritten, TRUE) == FALSE)
+                if (GetOverlappedResult(pData->NamedPipe, &pData->OverlappedWrite, (DWORD *)&cbWritten, TRUE) == FALSE)
                     uError = GetLastError();
             }
@@ -238 +237 @@
                 ||  rc == VERR_BROKEN_PIPE)
             {
-                RTFILE tmp = pData->NamedPipe;
-                FlushFileBuffers((HANDLE)tmp);
-                DisconnectNamedPipe((HANDLE)tmp);
+                FlushFileBuffers(pData->NamedPipe);
+                DisconnectNamedPipe(pData->NamedPipe);
                 if (!pData->fIsServer)
                 {
-                    pData->NamedPipe = NIL_RTFILE;
-                    RTFileClose(tmp);
+                    CloseHandle(pData->NamedPipe);
+                    pData->NamedPipe = INVALID_HANDLE_VALUE;
                 }
                 /* pretend success */
@@ -317 +315 @@
     int             rc = VINF_SUCCESS;
 #ifdef RT_OS_WINDOWS
-    RTFILE          NamedPipe = pData->NamedPipe;
+    HANDLE          NamedPipe = pData->NamedPipe;
     HANDLE          hEvent = CreateEvent(NULL, TRUE, FALSE, 0);
 #endif
@@ -329 +327 @@
         overlapped.hEvent = hEvent;
 
-        BOOL fConnected = ConnectNamedPipe((HANDLE)NamedPipe, &overlapped);
+        BOOL fConnected = ConnectNamedPipe(NamedPipe, &overlapped);
         if (    !fConnected
             &&  !pData->fShutdown)
@@ -340 +338 @@
 
                 hrc = 0;
-                if (GetOverlappedResult((HANDLE)pData->NamedPipe, &overlapped, &dummy, TRUE) == FALSE)
+                if (GetOverlappedResult(pData->NamedPipe, &overlapped, &dummy, TRUE) == FALSE)
                     hrc = GetLastError();
 
@@ -417 +415 @@
     pData->fIsServer                    = false;
 #ifdef RT_OS_WINDOWS
-    pData->NamedPipe                    = NIL_RTFILE;
+    pData->NamedPipe                    = INVALID_HANDLE_VALUE;
 #else /* !RT_OS_WINDOWS */
     pData->LocalSocketServer            = NIL_RTSOCKET;
@@ -464 +462 @@
             return PDMDrvHlpVMSetError(pDrvIns, rc, RT_SRC_POS, N_("NamedPipe#%d failed to create named pipe %s"), pDrvIns->iInstance, pszLocation);
         }
-        pData->NamedPipe = (HFILE)hPipe;
+        pData->NamedPipe = hPipe;
 
         rc = RTThreadCreate(&pData->ListenThread, drvNamedPipeListenLoop, (void *)pData, 0, RTTHREADTYPE_IO, RTTHREADFLAGS_WAITABLE, "NamedPipe");
@@ -476 +474 @@
     {
         /* Connect to the named pipe. */
-        rc = RTFileOpen(&pData->NamedPipe, pszLocation, RTFILE_O_READWRITE);
-        if (VBOX_FAILURE(rc))
+        HANDLE hPipe = CreateFile(pData->pszLocation, GENERIC_READ | GENERIC_WRITE, 0, NULL, OPEN_EXISTING, FILE_FLAG_OVERLAPPED, NULL);
+        if (hPipe == INVALID_HANDLE_VALUE)
+        {
+            rc = RTErrConvertFromWin32(GetLastError());
+            LogRel(("NamedPipe%d: CreateFile failed rc=%Vrc\n", pData->pDrvIns->iInstance));
             return PDMDrvHlpVMSetError(pDrvIns, rc, RT_SRC_POS, N_("NamedPipe#%d failed to connect to named pipe %s"), pDrvIns->iInstance, pszLocation);
+        }
+        pData->NamedPipe = hPipe;
     }
 
@@ -571 +574 @@
 
 #ifdef RT_OS_WINDOWS
-    if (pData->NamedPipe != NIL_RTFILE)
-    {
-        FlushFileBuffers((HANDLE)pData->NamedPipe);
-        if (!pData->fIsServer)
-            DisconnectNamedPipe((HANDLE)pData->NamedPipe);
-
-        RTFileClose(pData->NamedPipe);
-        pData->NamedPipe = NIL_RTFILE;
+    if (pData->NamedPipe != INVALID_HANDLE_VALUE)
+    {
+        if (pData->fIsServer) 
+        {
+            FlushFileBuffers(pData->NamedPipe);
+            DisconnectNamedPipe(pData->NamedPipe);
+        }
+
+        CloseHandle(pData->NamedPipe);
+        pData->NamedPipe = INVALID_HANDLE_VALUE;
         CloseHandle(pData->OverlappedRead.hEvent);
         CloseHandle(pData->OverlappedWrite.hEvent);
     }
-    /* Wake up listen thread */
-    RTSemEventSignal(pData->ListenSem);
-    RTSemEventDestroy(pData->ListenSem);
+    if (pData->fIsServer)
+    {
+        /* Wake up listen thread */
+        RTSemEventSignal(pData->ListenSem);
+        RTSemEventDestroy(pData->ListenSem);
+    }
 #else /* !RT_OS_WINDOWS */
     if (pData->fIsServer)