Changeset 32631 in vbox

Timestamp:

Sep 20, 2010 9:55:38 AM (14 years ago)

Author:

vboxsync

Message:

Devices/VMMDev and Additions/common/VBoxGuestLib: get the mouse pointer shape IOCTL size right on the guest side and check it better on the host side

Location:

trunk

Files:

• include/VBox/VMMDev.h (modified) (1 diff)
• src/VBox/Additions/common/VBoxGuestLib/VBoxGuestR3LibVideo.cpp (modified) (2 diffs)
• src/VBox/Devices/VMMDev/VMMDev.cpp (modified) (2 diffs)

trunk/include/VBox/VMMDev.h

@@ -342 +342 @@
 } VMMDevReqMousePointer;
 AssertCompileSize(VMMDevReqMousePointer, 24+24);
+
+/**
+ * Get the size that a VMMDevReqMousePointer request should have for a given
+ * size of cursor, including the trailing cursor image and mask data.
+ * @note an "empty" request still has the four preallocated bytes of data
+ * 
+ * @returns the size
+ * @param  width   the cursor width
+ * @param  height  the cursor height
+ */
+DECLINLINE(size_t) vmmdevGetMousePointerReqSize(uint32_t width, uint32_t height)
+{
+    size_t cbBase = RT_OFFSETOF(VMMDevReqMousePointer, pointerData);
+    size_t cbMask = (width + 7) / 8 * height;
+    size_t cbArgb = width * height * 4;
+    return RT_MAX(cbBase + ((cbMask + 3) & ~3) + cbArgb,
+                  sizeof(VMMDevReqMousePointer));
+}
 
 /** @name VMMDevReqMousePointer::fFlags

trunk/src/VBox/Additions/common/VBoxGuestLib/VBoxGuestR3LibVideo.cpp

@@ -99 +99 @@
 {
     VMMDevReqMousePointer *pReq;
-    int rc = vbglR3GRAlloc((VMMDevRequestHeader **)&pReq, RT_OFFSETOF(VMMDevReqMousePointer, pointerData) + cbImg, VMMDevReq_SetPointerShape);
+    size_t cbReq = vmmdevGetMousePointerReqSize(cx, cy);
+    AssertReturn(   !pvImg
+                 || cbReq == RT_OFFSETOF(VMMDevReqMousePointer, pointerData) + cbImg,
+                 VERR_INVALID_PARAMETER);
+    int rc = vbglR3GRAlloc((VMMDevRequestHeader **)&pReq, cbReq,
+                           VMMDevReq_SetPointerShape);
     if (RT_SUCCESS(rc))
     {
@@ -111 +116 @@
 
         rc = vbglR3GRPerform(&pReq->header);
-        vbglR3GRFree(&pReq->header);
         if (RT_SUCCESS(rc))
             rc = pReq->header.rc;
+        vbglR3GRFree(&pReq->header);
     }
     return rc;

trunk/src/VBox/Devices/VMMDev/VMMDev.cpp

@@ -751 +751 @@
             if (pRequestHeader->size < sizeof(VMMDevReqMousePointer))
             {
-                AssertMsg(pRequestHeader->size == 0x10028 && pRequestHeader->version == 10000,  /* don't bitch about legacy!!! */
+                AssertMsg(pRequestHeader->size == 0x10028 && pRequestHeader->version == 10000,  /* don't complain about legacy!!! */
                           ("VMMDev mouse shape structure has invalid size %d (%#x) version=%d!\n",
                            pRequestHeader->size, pRequestHeader->size, pRequestHeader->size, pRequestHeader->version));
@@ -767 +767 @@
                      fVisible, fAlpha, fShape, pointerShape->width, pointerShape->height));
 
-                if (pRequestHeader->size == sizeof(VMMDevReqMousePointer))
+                if (      pRequestHeader->size
+                       != vmmdevGetMousePointerReqSize(pointerShape->width,
+                                                       pointerShape->height)
+                    || !pointerShape->width
+                    || !pointerShape->height)
                 {
                     /* The guest did not provide the shape actually. */