Changeset 26124 in vbox for trunk/src/VBox/Additions

Timestamp:

Feb 1, 2010 2:30:36 PM (15 years ago)

Author:

vboxsync

Message:

Additions/VBoxGuest,VBoxGuestLib: fixed HGCM 32-bit calls in 64-bit kernel driver. OpenGL works again.

Location:

trunk/src/VBox/Additions/common

Files:

• VBoxGuest/VBoxGuest.cpp (modified) (7 diffs)
• VBoxGuestLib/HGCMInternal.cpp (modified) (6 diffs)

trunk/src/VBox/Additions/common/VBoxGuest/VBoxGuest.cpp

@@ -1308 +1308 @@
     if (pInfo->cParms > 4096) /* (Just make sure it doesn't overflow the next check.) */
     {
-        Log(("VBoxGuestCommonIOCtl: HGCM_CALL: cParm=%RX32 is not sane\n", pInfo->cParms));
+        LogRel(("VBoxGuestCommonIOCtl: HGCM_CALL: cParm=%RX32 is not sane\n", pInfo->cParms));
         return VERR_INVALID_PARAMETER;
     }
@@ -1320 +1320 @@
     if (cbData < cbActual)
     {
-        Log(("VBoxGuestCommonIOCtl: HGCM_CALL: cbData=%#zx (%zu) required size is %#zx (%zu)\n",
+        LogRel(("VBoxGuestCommonIOCtl: HGCM_CALL: cbData=%#zx (%zu) required size is %#zx (%zu)\n",
              cbData, cbActual));
         return VERR_INVALID_PARAMETER;
@@ -1376 +1376 @@
     }
     else
-        Log(("VBoxGuestCommonIOCtl: HGCM_CALL: Failed. rc=%Rrc.\n", rc));
+        LogRel(("VBoxGuestCommonIOCtl: HGCM_CALL: %s Failed. rc=%Rrc.\n", f32Bit ? "32" : "64", rc));
     return rc;
 }
@@ -1498 +1498 @@
         if (pSession->R0Process != NIL_RTR0PROCESS) \
         { \
-            Log(("VBoxGuestCommonIOCtl: " mnemonic ": Ring-0 only, caller is %RTproc/%p\n", \
+            LogRel(("VBoxGuestCommonIOCtl: " mnemonic ": Ring-0 only, caller is %RTproc/%p\n", \
                  pSession->Process, (uintptr_t)pSession->R0Process)); \
             return VERR_PERMISSION_DENIED; \
@@ -1507 +1507 @@
         if (cbData < (cbMin)) \
         { \
-            Log(("VBoxGuestCommonIOCtl: " mnemonic ": cbData=%#zx (%zu) min is %#zx (%zu)\n", \
+            LogRel(("VBoxGuestCommonIOCtl: " mnemonic ": cbData=%#zx (%zu) min is %#zx (%zu)\n", \
                  cbData, cbData, (size_t)(cbMin), (size_t)(cbMin))); \
             return VERR_BUFFER_OVERFLOW; \
@@ -1513 +1513 @@
         if ((cbMin) != 0 && !VALID_PTR(pvData)) \
         { \
-            Log(("VBoxGuestCommonIOCtl: " mnemonic ": Invalid pointer %p\n", pvData)); \
+            LogRel(("VBoxGuestCommonIOCtl: " mnemonic ": Invalid pointer %p\n", pvData)); \
             return VERR_INVALID_POINTER; \
         } \
@@ -1626 +1626 @@
             default:
             {
-                Log(("VBoxGuestCommonIOCtl: Unknown request iFunction=%#x Stripped size=%#x\n", iFunction,
+                LogRel(("VBoxGuestCommonIOCtl: Unknown request iFunction=%#x Stripped size=%#x\n", iFunction,
                                 VBOXGUEST_IOCTL_STRIP_SIZE(iFunction)));
                 rc = VERR_NOT_SUPPORTED;

trunk/src/VBox/Additions/common/VBoxGuestLib/HGCMInternal.cpp

@@ -895 +895 @@
                 rc = vbglR0HGCMInternalCopyBackResult(pCallInfo, pHGCMCall, &ParmInfo, fIsUser, rc);
             }
+            else
+                LogRel(("VbglR0HGCMInternalCall: vbglR0HGCMInternalDoCall failed. rc=%Rrc\n", rc));
 
             if (!fLeakIt)
@@ -900 +902 @@
         }
     }
+    else
+        LogRel(("VbglR0HGCMInternalCall: vbglR0HGCMInternalPreprocessCall failed. rc=%Rrc\n", rc));
 
     /*
@@ -921 +925 @@
                                          PFNVBGLHGCMCALLBACK pfnAsyncCallback, void *pvAsyncData, uint32_t u32AsyncData)
 {
-    VBoxGuestHGCMCallInfo   *pCallInfo64;
-    HGCMFunctionParameter   *pParm64;
-    HGCMFunctionParameter32 *pParm32;
-    uint32_t                 cParms;
-    uint32_t                 iParm;
-    int rc;
+    VBoxGuestHGCMCallInfo   *pCallInfo64 = NULL;
+    HGCMFunctionParameter   *pParm64 = NULL;
+    HGCMFunctionParameter32 *pParm32 = NULL;
+    uint32_t                 cParms = 0;
+    uint32_t                 iParm = 0;
+    int                      rc = VINF_SUCCESS;
 
     /*
@@ -936 +940 @@
                     || !(fFlags & ~VBGLR0_HGCMCALL_F_MODE_MASK),
                     ("pCallInfo=%p pfnAsyncCallback=%p fFlags=%#x\n", pCallInfo, pfnAsyncCallback, fFlags),
-                    VERR_INVALID_PARAMETER);
+                    VERR_INVALID_HANDLE);
     AssertReturn(   cbCallInfo >= sizeof(VBoxGuestHGCMCallInfo)
                  || cbCallInfo >= pCallInfo->cParms * sizeof(HGCMFunctionParameter32),
-                 VERR_INVALID_PARAMETER);
-    AssertReturn((fFlags & VBGLR0_HGCMCALL_F_MODE_MASK) == VBGLR0_HGCMCALL_F_KERNEL, VERR_INVALID_PARAMETER);
+                 VERR_INVALID_MAGIC);
+
+    /* This Assert does not work on Solaris 64/32 mixed mode, not sure why, skipping for now */
+#ifndef RT_OS_SOLARIS
+    AssertReturn((fFlags & VBGLR0_HGCMCALL_F_MODE_MASK) == VBGLR0_HGCMCALL_F_KERNEL, VERR_WRONG_ORDER);
+#endif
 
     cParms = pCallInfo->cParms;
@@ -979 +987 @@
             default:
                 rc = VERR_INVALID_PARAMETER;
+                LogRel(("VbglR0HGCMInternalCall32: pParm32 type %#x invalid.\n", pParm32->type));
                 break;
         }
@@ -989 +998 @@
                                     pfnAsyncCallback, pvAsyncData, u32AsyncData);
 
-        /*
-         * Copy back.
-         */
-        for (iParm = 0; iParm < cParms; iParm++, pParm32++, pParm64++)
+        if (RT_SUCCESS(rc))
         {
-            switch (pParm32->type)
+            *pCallInfo = *pCallInfo64;
+
+            /*
+             * Copy back.
+             */
+            pParm32 = VBOXGUEST_HGCM_CALL_PARMS32(pCallInfo);
+            pParm64 = VBOXGUEST_HGCM_CALL_PARMS(pCallInfo64);
+            for (iParm = 0; iParm < cParms; iParm++, pParm32++, pParm64++)
             {
-                case VMMDevHGCMParmType_32bit:
-                    pParm32->u.value32 = pParm32->u.value32;
-                    break;
-
-                case VMMDevHGCMParmType_64bit:
-                    pParm32->u.value64 = pParm64->u.value64;
-                    break;
-
-                case VMMDevHGCMParmType_LinAddr_Out:
-                case VMMDevHGCMParmType_LinAddr:
-                case VMMDevHGCMParmType_LinAddr_In:
-                    pParm32->u.Pointer.size = pParm64->u.Pointer.size;
-                    break;
-
-                default:
-                    rc = VERR_INTERNAL_ERROR_3;
-                    break;
+                LogRel(("VbglR0HGCMInternalCall32: iParm=%d cParms=%d\n",iParm, cParms)); 
+                switch (pParm64->type)
+                {
+                    case VMMDevHGCMParmType_32bit:
+                        LogRel(("pParm32->u.value32=%d\n", pParm32->u.value32));
+                        pParm32->u.value32 = pParm64->u.value32;
+                        break;
+
+                    case VMMDevHGCMParmType_64bit:
+                        pParm32->u.value64 = pParm64->u.value64;
+                        break;
+
+                    case VMMDevHGCMParmType_LinAddr_Out:
+                    case VMMDevHGCMParmType_LinAddr:
+                    case VMMDevHGCMParmType_LinAddr_In:
+                        pParm32->u.Pointer.size = pParm64->u.Pointer.size;
+                        break;
+
+                    default:
+                        LogRel(("VbglR0HGCMInternalCall32: failed invalid pParm32 type %d\n", pParm32->type));
+                        rc = VERR_INTERNAL_ERROR_3;
+                        break;
+                }
             }
         }
-        *pCallInfo = *pCallInfo64;
-    }
+        else
+            LogRel(("VbglR0HGCMInternalCall32: VbglR0HGCMInternalCall failed. rc=%Rrc\n", rc));
+    }
+    else
+        LogRel(("VbglR0HGCMInternalCall32: failed. rc=%Rrc\n", rc));
 
     RTMemTmpFree(pCallInfo64);