Changeset 26124 in vbox for trunk/src/VBox/Additions/common/VBoxGuestLib/HGCMInternal.cpp

Timestamp:

Feb 1, 2010 2:30:36 PM (15 years ago)

Author:

vboxsync

Message:

Additions/VBoxGuest,VBoxGuestLib: fixed HGCM 32-bit calls in 64-bit kernel driver. OpenGL works again.

File:

• trunk/src/VBox/Additions/common/VBoxGuestLib/HGCMInternal.cpp (modified) (6 diffs)

trunk/src/VBox/Additions/common/VBoxGuestLib/HGCMInternal.cpp

@@ -895 +895 @@
                 rc = vbglR0HGCMInternalCopyBackResult(pCallInfo, pHGCMCall, &ParmInfo, fIsUser, rc);
             }
+            else
+                LogRel(("VbglR0HGCMInternalCall: vbglR0HGCMInternalDoCall failed. rc=%Rrc\n", rc));
 
             if (!fLeakIt)
@@ -900 +902 @@
         }
     }
+    else
+        LogRel(("VbglR0HGCMInternalCall: vbglR0HGCMInternalPreprocessCall failed. rc=%Rrc\n", rc));
 
     /*
@@ -921 +925 @@
                                          PFNVBGLHGCMCALLBACK pfnAsyncCallback, void *pvAsyncData, uint32_t u32AsyncData)
 {
-    VBoxGuestHGCMCallInfo   *pCallInfo64;
-    HGCMFunctionParameter   *pParm64;
-    HGCMFunctionParameter32 *pParm32;
-    uint32_t                 cParms;
-    uint32_t                 iParm;
-    int rc;
+    VBoxGuestHGCMCallInfo   *pCallInfo64 = NULL;
+    HGCMFunctionParameter   *pParm64 = NULL;
+    HGCMFunctionParameter32 *pParm32 = NULL;
+    uint32_t                 cParms = 0;
+    uint32_t                 iParm = 0;
+    int                      rc = VINF_SUCCESS;
 
     /*
@@ -936 +940 @@
                     || !(fFlags & ~VBGLR0_HGCMCALL_F_MODE_MASK),
                     ("pCallInfo=%p pfnAsyncCallback=%p fFlags=%#x\n", pCallInfo, pfnAsyncCallback, fFlags),
-                    VERR_INVALID_PARAMETER);
+                    VERR_INVALID_HANDLE);
     AssertReturn(   cbCallInfo >= sizeof(VBoxGuestHGCMCallInfo)
                  || cbCallInfo >= pCallInfo->cParms * sizeof(HGCMFunctionParameter32),
-                 VERR_INVALID_PARAMETER);
-    AssertReturn((fFlags & VBGLR0_HGCMCALL_F_MODE_MASK) == VBGLR0_HGCMCALL_F_KERNEL, VERR_INVALID_PARAMETER);
+                 VERR_INVALID_MAGIC);
+
+    /* This Assert does not work on Solaris 64/32 mixed mode, not sure why, skipping for now */
+#ifndef RT_OS_SOLARIS
+    AssertReturn((fFlags & VBGLR0_HGCMCALL_F_MODE_MASK) == VBGLR0_HGCMCALL_F_KERNEL, VERR_WRONG_ORDER);
+#endif
 
     cParms = pCallInfo->cParms;
@@ -979 +987 @@
             default:
                 rc = VERR_INVALID_PARAMETER;
+                LogRel(("VbglR0HGCMInternalCall32: pParm32 type %#x invalid.\n", pParm32->type));
                 break;
         }
@@ -989 +998 @@
                                     pfnAsyncCallback, pvAsyncData, u32AsyncData);
 
-        /*
-         * Copy back.
-         */
-        for (iParm = 0; iParm < cParms; iParm++, pParm32++, pParm64++)
+        if (RT_SUCCESS(rc))
         {
-            switch (pParm32->type)
+            *pCallInfo = *pCallInfo64;
+
+            /*
+             * Copy back.
+             */
+            pParm32 = VBOXGUEST_HGCM_CALL_PARMS32(pCallInfo);
+            pParm64 = VBOXGUEST_HGCM_CALL_PARMS(pCallInfo64);
+            for (iParm = 0; iParm < cParms; iParm++, pParm32++, pParm64++)
             {
-                case VMMDevHGCMParmType_32bit:
-                    pParm32->u.value32 = pParm32->u.value32;
-                    break;
-
-                case VMMDevHGCMParmType_64bit:
-                    pParm32->u.value64 = pParm64->u.value64;
-                    break;
-
-                case VMMDevHGCMParmType_LinAddr_Out:
-                case VMMDevHGCMParmType_LinAddr:
-                case VMMDevHGCMParmType_LinAddr_In:
-                    pParm32->u.Pointer.size = pParm64->u.Pointer.size;
-                    break;
-
-                default:
-                    rc = VERR_INTERNAL_ERROR_3;
-                    break;
+                LogRel(("VbglR0HGCMInternalCall32: iParm=%d cParms=%d\n",iParm, cParms)); 
+                switch (pParm64->type)
+                {
+                    case VMMDevHGCMParmType_32bit:
+                        LogRel(("pParm32->u.value32=%d\n", pParm32->u.value32));
+                        pParm32->u.value32 = pParm64->u.value32;
+                        break;
+
+                    case VMMDevHGCMParmType_64bit:
+                        pParm32->u.value64 = pParm64->u.value64;
+                        break;
+
+                    case VMMDevHGCMParmType_LinAddr_Out:
+                    case VMMDevHGCMParmType_LinAddr:
+                    case VMMDevHGCMParmType_LinAddr_In:
+                        pParm32->u.Pointer.size = pParm64->u.Pointer.size;
+                        break;
+
+                    default:
+                        LogRel(("VbglR0HGCMInternalCall32: failed invalid pParm32 type %d\n", pParm32->type));
+                        rc = VERR_INTERNAL_ERROR_3;
+                        break;
+                }
             }
         }
-        *pCallInfo = *pCallInfo64;
-    }
+        else
+            LogRel(("VbglR0HGCMInternalCall32: VbglR0HGCMInternalCall failed. rc=%Rrc\n", rc));
+    }
+    else
+        LogRel(("VbglR0HGCMInternalCall32: failed. rc=%Rrc\n", rc));
 
     RTMemTmpFree(pCallInfo64);