Changeset 35941 in vbox for trunk/src/VBox

Timestamp:

Feb 11, 2011 12:02:24 PM (14 years ago)

Author:

vboxsync

Message:

crOpenGL: fix for huge texture/vertex data transmit on windows guests

File:

• trunk/src/VBox/GuestHost/OpenGL/util/vboxhgcm.c (modified) (3 diffs)

trunk/src/VBox/GuestHost/OpenGL/util/vboxhgcm.c

@@ -601 +601 @@
 
 # ifdef RT_OS_WINDOWS
-    DWORD cbReturned;
+    DWORD cbReturned, lerr;
 
     if (DeviceIoControl (g_crvboxhgcm.hGuestDrv,
@@ -612 +612 @@
         return VINF_SUCCESS;
     }
-    crDebug("vboxCall failed with %x\n", GetLastError());
-    return VERR_NOT_SUPPORTED;
+    lerr=GetLastError();
+    crDebug("vboxCall failed with %x\n", lerr);
+    /*On windows if we exceed max buffer len, we only get ERROR_GEN_FAILURE, and parms.hdr.result isn't changed.
+     *Before every call here we set it to VERR_WRONG_ORDER, so checking it here as well.
+     */
+    if (ERROR_GEN_FAILURE==lerr && VERR_WRONG_ORDER==((VBoxGuestHGCMCallInfo*)pvData)->result)
+    {
+        return VERR_OUT_OF_RANGE;
+    }
+    else
+    {
+        return VERR_NOT_SUPPORTED;
+    }
 # else
     int rc;
@@ -946 +957 @@
     rc = crVBoxHGCMCall(&parms, sizeof(parms));
 
-#ifdef RT_OS_LINUX
+#if defined(RT_OS_LINUX) || defined(RT_OS_WINDOWS)
     if (VERR_OUT_OF_RANGE==rc && CR_VBOXHGCM_USERALLOCATED==bufferKind)
     {