Changeset 69989 in vbox for trunk/src/VBox/GuestHost/OpenGL

Timestamp:

Dec 7, 2017 4:25:08 PM (7 years ago)

Author:

vboxsync

Message:

Additions/common/OpenGL: fix regressions from r117617/public r68472.
bugref:9060: vboxvideo: hanging login screen starting Ubuntu 17.10 guest with 3D
r117617/public r68472 and the follow-up switched from using duplicate code in
the X11 libGL code to access the guest driver and HGCM to using the common
code. It contained at least two problems: it failed to initialise the
runtime library before calling into the Guest library and it changed error
checking semantics slightly, causing us to fail to handle a too-small-buffer
error and sending GNOME Shell into an endless loop. This change fixes that
and also adds todo comments to other places where the error handling should
be cleaned up, but which I do not want to change and test just now.

File:

• trunk/src/VBox/GuestHost/OpenGL/util/vboxhgcm.c (modified) (10 diffs)

trunk/src/VBox/GuestHost/OpenGL/util/vboxhgcm.c

@@ -37 +37 @@
 #include "cr_process.h"
 
+#include <iprt/initterm.h>
 #include <iprt/thread.h>
 
@@ -705 +706 @@
 
         rc = crVBoxHGCMCall(conn, &parms.hdr, sizeof(parms));
-        callRes = parms.hdr.Hdr.rc;
+        callRes = parms.hdr.Hdr.rc; /** @todo now rc == parms.hdr.Hdr.rc */
     }
     else
@@ -719 +720 @@
 
         rc = crVBoxHGCMCall(conn, &parms.hdr, sizeof(parms));
-        callRes = parms.hdr.Hdr.rc;
+        callRes = parms.hdr.Hdr.rc; /** @todo now rc == parms.hdr.Hdr.rc */
     }
 
@@ -759 +760 @@
     rc = crVBoxHGCMCall(conn, &parms.hdr, sizeof(parms));
 
-    if (RT_FAILURE(rc) || RT_FAILURE(parms.hdr.Hdr.rc))
+    if (RT_FAILURE(rc) || RT_FAILURE(parms.hdr.Hdr.rc) /** @todo now rc == parms.hdr.Hdr.rc */)
     {
         crWarning("SHCRGL_GUEST_FN_READ failed with %x %x\n", rc, parms.hdr.Hdr.rc);
@@ -835 +836 @@
 
             rc = crVBoxHGCMCall(conn, &wbParms.hdr, sizeof(wbParms));
-            if (RT_FAILURE(rc) || RT_FAILURE(wbParms.hdr.Hdr.rc))
+            if (RT_FAILURE(rc) || RT_FAILURE(wbParms.hdr.Hdr.rc) /** @todo now rc == wbParms.hdr.Hdr.rc */)
             {
                 crError("SHCRGL_GUEST_FN_WRITE_BUFFER (%i) failed with %x %x\n", wbParms.pBuffer.u.Pointer.size, rc, wbParms.hdr.Hdr.rc);
@@ -866 +867 @@
 #endif
 
-    if (RT_FAILURE(rc) || RT_FAILURE(parms.hdr.Hdr.rc))
-    {
-
-        if ((VERR_BUFFER_OVERFLOW == parms.hdr.Hdr.rc) && RT_SUCCESS(rc))
+    if (RT_FAILURE(rc) || RT_FAILURE(parms.hdr.Hdr.rc) /** @todo now rc == parms.hdr.Hdr.rc */)
+    {
+
+        if ((VERR_BUFFER_OVERFLOW == parms.hdr.Hdr.rc) /* && RT_SUCCESS(rc) */)
         {
             /* reallocate buffer and retry */
@@ -991 +992 @@
     rc = crVBoxHGCMCall(conn, &parms.hdr, sizeof(parms));
 
-    if (RT_FAILURE(rc) || RT_FAILURE(parms.hdr.Hdr.rc))
+    if (RT_FAILURE(rc) || RT_FAILURE(parms.hdr.Hdr.rc) /** @todo now rc == parms.hdr.Hdr.rc */)
     {
         crDebug("SHCRGL_GUEST_FN_READ failed with %x %x\n", rc, parms.hdr.Hdr.rc);
@@ -1194 +1195 @@
     if (RT_SUCCESS(rc))
     {
-        rc =  parms.hdr.Hdr.rc;
+        rc =  parms.hdr.Hdr.rc; /** @todo now rc == parms.hdr.Hdr.rc */
         if (RT_SUCCESS(rc))
         {
@@ -1256 +1257 @@
     rc = crVBoxHGCMCall(conn, &parms.hdr, sizeof(parms));
 
-    if (RT_FAILURE(rc) || RT_FAILURE(parms.hdr.Hdr.rc))
+    if (RT_FAILURE(rc) || RT_FAILURE(parms.hdr.Hdr.rc) /** @todo now rc == parms.hdr.Hdr.rc */)
     {
         crWarning("SHCRGL_GUEST_FN_SET_PID failed!");
@@ -1275 +1276 @@
     int rc;
     VBOXCRHGSMIPROFILE_FUNC_PROLOGUE();
+    RTR3InitDll(RTR3INIT_FLAGS_UNOBTRUSIVE);
     rc = VbglR3InitUser();
     if (RT_SUCCESS(rc))