Changeset 79339 in vbox

Timestamp:

Jun 26, 2019 7:26:50 AM (6 years ago)

Author:

vboxsync

Message:

SharedOpenGL: fixed buffer size check in crFbImgFromDimOffVramBGRA, it incorrectly rejected empty rectangles, which of course fit into guest VRAM. bugref:9407

File:

• trunk/src/VBox/HostServices/SharedOpenGL/crserverlib/presenter/server_presenter.cpp (modified) (1 diff)

trunk/src/VBox/HostServices/SharedOpenGL/crserverlib/presenter/server_presenter.cpp

@@ -146 +146 @@
 static int8_t crFbImgFromDimOffVramBGRA(VBOXCMDVBVAOFFSET offVRAM, uint32_t width, uint32_t height, CR_BLITTER_IMG *pImg)
 {
-    uint32_t cbBuff;
-
-    if (width == 0 || height == 0)
-    {
-        WARN(("invalid param"));
+    if (offVRAM >= g_cbVRam)
+    {
+        WARN(("offVRAM invalid"));
         return -1;
     }
 
-    cbBuff = width * height * 4;
-    // Check if overflow happened
-    if (cbBuff / width != height * 4)
-    {
-        WARN(("invalid param"));
-        return -1;
-    }
-
-    if (offVRAM >= g_cbVRam
-            || UINT32_MAX - cbBuff <= offVRAM
-            || offVRAM + cbBuff >= g_cbVRam)
-    {
-        WARN(("invalid param"));
+    uint32_t const cbAvailVRAM = g_cbVRam - offVRAM;
+    uint32_t const cAvailScanlines = width ? cbAvailVRAM / (width * 4) : height;
+    if (height > cAvailScanlines)
+    {
+        WARN(("Not enough space in VRAM"));
         return -1;
     }