Changeset 4093 in vbox for trunk

Timestamp:

Aug 8, 2007 3:51:58 PM (17 years ago)

Author:

vboxsync

Message:

Fixed missing black edge for Windows Vista alpha pointers over RDP.

File:

• trunk/src/VBox/Main/ConsoleVRDPServer.cpp (modified) (10 diffs)

trunk/src/VBox/Main/ConsoleVRDPServer.cpp

@@ -146 +146 @@
 #endif /* VBOX_WITH_XPCOM */
 
-static void findTopLeftBorder (uint8_t *pu8Shape, uint32_t width, uint32_t height, uint32_t *pxSkip, uint32_t *pySkip)
+#ifdef DEBUG_sunlover
+#define LOGDUMPPTR Log
+void dumpPointer (const uint8_t *pu8Shape, uint32_t width, uint32_t height, bool fXorMaskRGB32)
+{
+    unsigned i;
+
+    const uint8_t *pu8And = pu8Shape;
+
+    for (i = 0; i < height; i++)
+    {
+        unsigned j;
+        LOGDUMPPTR(("%p: ", pu8And));
+        for (j = 0; j < (width + 7) / 8; j++)
+        {
+            unsigned k;
+            for (k = 0; k < 8; k++)
+            {
+                LOGDUMPPTR(("%d", ((*pu8And) & (1 << (7 - k)))? 1: 0));
+            }
+
+            pu8And++;
+        }
+        LOGDUMPPTR(("\n"));
+    }
+
+    if (fXorMaskRGB32)
+    {
+        uint32_t *pu32Xor = (uint32_t *)(pu8Shape + ((((width + 7) / 8) * height + 3) & ~3));
+
+        for (i = 0; i < height; i++)
+        {
+            unsigned j;
+            LOGDUMPPTR(("%p: ", pu32Xor));
+            for (j = 0; j < width; j++)
+            {
+                LOGDUMPPTR(("%08X", *pu32Xor++));
+            }
+            LOGDUMPPTR(("\n"));
+        }
+    }
+    else
+    {
+        /* RDP 24 bit RGB mask. */
+        uint8_t *pu8Xor = (uint8_t *)(pu8Shape + ((((width + 7) / 8) * height + 3) & ~3));
+        for (i = 0; i < height; i++)
+        {
+            unsigned j;
+            LOGDUMPPTR(("%p: ", pu8Xor));
+            for (j = 0; j < width; j++)
+            {
+                LOGDUMPPTR(("%02X%02X%02X", pu8Xor[2], pu8Xor[1], pu8Xor[0]));
+                pu8Xor += 3;
+            }
+            LOGDUMPPTR(("\n"));
+        }
+    }
+}
+#else
+#define dumpPointer(a, b, c, d) do {} while (0)
+#endif /* DEBUG_sunlover */
+
+static void findTopLeftBorder (const uint8_t *pu8AndMask, const uint8_t *pu8XorMask, uint32_t width, uint32_t height, uint32_t *pxSkip, uint32_t *pySkip)
 {
     /*
@@ -153 +214 @@
     uint32_t ySkipAnd = ~0;
 
-    uint8_t *pu8And = pu8Shape;
+    const uint8_t *pu8And = pu8AndMask;
     const uint32_t cbAndRow = (width + 7) / 8;
     const uint8_t maskLastByte = (uint8_t)( 0xFF << (cbAndRow * 8 - width) );
@@ -197 +258 @@
     for (x = 0; x < width && xSkipAnd == ~(uint32_t)0; x++)
     {
-        pu8And = pu8Shape + x/8;       /* Currently checking byte. */
+        pu8And = pu8AndMask + x/8;       /* Currently checking byte. */
         uint8_t mask = 1 << (7 - x%8); /* Currently checking bit in the byte. */
 
@@ -220 +281 @@
     uint32_t ySkipXor = ~0;
 
-    uint32_t *pu32XorStart = (uint32_t *)( pu8Shape + ((cbAndRow * height + 3) & ~3) );
+    uint32_t *pu32XorStart = (uint32_t *)pu8XorMask;
 
     uint32_t *pu32Xor = pu32XorStart;
@@ -270 +331 @@
 }
 
+/* Generate an AND mask for alpha pointers here, because
+ * guest driver does not do that correctly for Vista pointers.
+ * Similar fix, changing the alpha threshold, could be applied
+ * for the guest driver, but then additions reinstall would be
+ * necessary, which we try to avoid.
+ */
+static void mousePointerGenerateANDMask (uint8_t *pu8DstAndMask, int cbDstAndMask, const uint8_t *pu8SrcAlpha, int w, int h)
+{
+    memset (pu8DstAndMask, 0xFF, cbDstAndMask);
+    
+    int y;
+    for (y = 0; y < h; y++)
+    {
+        uint8_t bitmask = 0x80;
+            
+        int x;
+        for (x = 0; x < w; x++, bitmask >>= 1)
+        {
+            if (bitmask == 0)
+            {
+                bitmask = 0x80;
+            }
+            
+            /* Whether alpha channel value is not transparent enough for the pixel to be seen. */
+            if (pu8SrcAlpha[x * 4 + 3] > 0x7f)
+            {
+                pu8DstAndMask[x / 8] &= ~bitmask;
+            }
+        }
+    
+        /* Point to next source and dest scans. */
+        pu8SrcAlpha += w * 4;
+        pu8DstAndMask += (w + 7) / 8;
+    }
+}
 
 STDMETHODIMP VRDPConsoleCallback::OnMousePointerShapeChange (BOOL visible, BOOL alpha, ULONG xHot, ULONG yHot,
@@ -304 +400 @@
              */
 
+            dumpPointer (shape, width, height, true);
+
+            int cbDstAndMask = (((width + 7) / 8) * height + 3) & ~3;
+
+            uint8_t *pu8AndMask = shape;
+            uint8_t *pu8XorMask = shape + cbDstAndMask;
+
+            if (alpha)
+            {
+                pu8AndMask = (uint8_t *)alloca (cbDstAndMask);
+
+                mousePointerGenerateANDMask (pu8AndMask, cbDstAndMask, pu8XorMask, width, height);
+            }
+
             /* Windows guest alpha pointers are wider than 32 pixels.
              * Try to find out the top-left border of the pointer and
@@ -313 +423 @@
             uint32_t xSkip = 0; /* How many columns to skip at the left. */
 
-            findTopLeftBorder (shape, width, height, &xSkip, &ySkip);
+            findTopLeftBorder (pu8AndMask, pu8XorMask, width, height, &xSkip, &ySkip);
 
             /* Must not skip the hot spot. */
@@ -349 +459 @@
 
                 /* Copy AND mask. */
-                uint8_t *src = shape + ySkip * srcmaskwidth;
+                uint8_t *src = pu8AndMask + ySkip * srcmaskwidth;
                 uint8_t *dst = maskarray + (dstheight - 1) * rdpmaskwidth;
 
@@ -380 +490 @@
 
                 /* Point src to XOR mask */
-                src = shape + ((srcmaskwidth * height + 3) & ~3) + ySkip * srcdatawidth;
+                src = pu8XorMask + ySkip * srcdatawidth;
                 dst = dataarray + (dstheight - 1) * rdpdatawidth;
 
@@ -402 +512 @@
                 pointer->u16MaskLen = (uint16_t)rdpmasklen;
                 pointer->u16DataLen = (uint16_t)rdpdatalen;
+
+                dumpPointer ((uint8_t *)pointer + sizeof (*pointer), dstwidth, dstheight, false);
 
                 m_server->MousePointerUpdate (pointer);