Changeset 79782 in vbox for trunk/src/VBox/Additions/3D/win

Timestamp:

Jul 15, 2019 9:22:26 AM (5 years ago)

Author:

vboxsync

Message:

Additions/3D/win/VBoxGL: check if the surface format is supported.

File:

• trunk/src/VBox/Additions/3D/win/VBoxGL/GaDrvEnvKMT.cpp (modified) (3 diffs)

trunk/src/VBox/Additions/3D/win/VBoxGL/GaDrvEnvKMT.cpp

@@ -386 +386 @@
     }
 
+    VBoxDispMpLoggerLogF("WDDM: EnvKMT: unsupported surface format %d\n", format);
     Assert(0);
-    VBoxDispMpLoggerLogF("WDDM: EnvKMT: unsupported surface format %d\n", format);
     return D3DDDIFMT_UNKNOWN;
 }
@@ -405 +405 @@
     uint8_t                          *pu8Req;
     uint32_t                          cbReq;
+
+    /* First check if the format is supported. */
+    D3DDDIFORMAT const ddiFormat = svgaToD3DDDIFormat((SVGA3dSurfaceFormat)pCreateParms->format);
+    if (ddiFormat == D3DDDIFMT_UNKNOWN)
+        return -1;
 
     /* Size of the SVGA request data */
@@ -459 +464 @@
                 wddmAllocInfo.SurfDesc.width      = paSizes[0].cWidth;
                 wddmAllocInfo.SurfDesc.height     = paSizes[0].cHeight;
-                wddmAllocInfo.SurfDesc.format     = svgaToD3DDDIFormat((SVGA3dSurfaceFormat)pCreateParms->format);
+                wddmAllocInfo.SurfDesc.format     = ddiFormat;
                 wddmAllocInfo.SurfDesc.VidPnSourceId = 0;
                 wddmAllocInfo.SurfDesc.bpp        = vboxWddmCalcBitsPerPixel(wddmAllocInfo.SurfDesc.format);