Changeset 54380 in vbox for trunk/src/VBox/ExtPacks/VNC/VBoxVNC.cpp

Timestamp:

Feb 23, 2015 11:24:25 AM (10 years ago)

Author:

vboxsync

Message:

ExtPacks/VNC: fixed a possible crash when the client connects during VM restore (thanks Yonathan Randolph)

File:

• trunk/src/VBox/ExtPacks/VNC/VBoxVNC.cpp (modified) (4 diffs)

trunk/src/VBox/ExtPacks/VNC/VBoxVNC.cpp

@@ -80 +80 @@
         if (mCursor)
             rfbFreeCursor(mCursor);
-        memset(szVNCPassword, '\0', sizeof(szVNCPassword));
+        RT_ZERO(szVNCPassword);
         if (mVNCServer)
             rfbScreenCleanup(mVNCServer);
@@ -261 +261 @@
     LogFlowFunc(("enter\n"));
 
-    // query server for the framebuffer
+    // At this point, VRDECallbackFramebufferQuery will not succeed.
+    // Initialize VNC with 640x480 and wait for VRDEResize to get actual size.
+    int dummyWidth = 640, dummyHeight = 480;
+
+    rfbScreenInfoPtr vncServer = rfbGetScreen(0, NULL, dummyWidth, dummyHeight, 8, 3, VNC_SIZEOFRGBA);
+    instance->mVNCServer = vncServer;
+
     VRDEFRAMEBUFFERINFO info;
-    int rc = instance->mCallbacks->VRDECallbackFramebufferQuery(instance->mCallback, 0, &info);
-
-    rfbScreenInfoPtr vncServer = rfbGetScreen(0, NULL, info.cWidth, info.cHeight, 8, 3, VNC_SIZEOFRGBA);
-    instance->mVNCServer = vncServer;
+    RT_ZERO(info);
+    info.cWidth = dummyWidth, info.cHeight = dummyHeight;
+    info.cBitsPerPixel = 24;
+    info.pu8Bits = NULL;
+    unsigned char *FrameBuffer = (unsigned char *)RTMemAlloc(info.cWidth * info.cHeight * VNC_SIZEOFRGBA); // RGBA
+    rfbNewFramebuffer(instance->mVNCServer, (char *)FrameBuffer, info.cWidth, info.cHeight, 8, 3, VNC_SIZEOFRGBA);
+    instance->mFrameBuffer = FrameBuffer;
+    instance->mScreenBuffer = (unsigned char *)info.pu8Bits;
+    instance->FrameInfo = info;
+
     vncServer->serverFormat.redShift = 16;
     vncServer->serverFormat.greenShift = 8;
@@ -278 +290 @@
     char szAddress[VNC_ADDRESSSIZE + 1] = {0};
     uint32_t cbOut = 0;
-    rc = instance->mCallbacks->VRDECallbackProperty(instance->mCallback,
+    int rc = instance->mCallbacks->VRDECallbackProperty(instance->mCallback,
                                                     VRDE_QP_NETWORK_ADDRESS,
                                                     &szAddress, sizeof(szAddress), &cbOut);
@@ -716 +728 @@
     VNCServerImpl *instance = (VNCServerImpl *)hServer;
     VRDEFRAMEBUFFERINFO info;
-    int rc = instance->mCallbacks->VRDECallbackFramebufferQuery(instance->mCallback, 0, &info);
-    if (!RT_SUCCESS(rc))
-    {
+    bool fAvail = instance->mCallbacks->VRDECallbackFramebufferQuery(instance->mCallback, 0, &info);
+    if (!fAvail)
         return;
-    }
 
     LogRel(("VNCServerImpl::VRDEResize to %dx%dx%dbpp\n", info.cWidth, info.cHeight, info.cBitsPerPixel));