Changeset 105644 in vbox

Timestamp:

Aug 9, 2024 11:24:24 PM (4 months ago)

Author:

vboxsync

Message:

Devices/Graphics: clamp vertex buffer stride

File:

• trunk/src/VBox/Devices/Graphics/DevVGA-SVGA3d-dx-dx11.cpp (modified) (2 diffs)

trunk/src/VBox/Devices/Graphics/DevVGA-SVGA3d-dx-dx11.cpp

@@ -6000 +6000 @@
         }
 #endif
+        PVMSVGA3DSURFACE pSurface = NULL;
         ID3D11Buffer *pBuffer = NULL;
         if (pBufferPipeline->bufferId != SVGA3D_INVALID_ID)
         {
-            PVMSVGA3DSURFACE pSurface;
             ID3D11Resource *pResource;
             int rc = dxEnsureResource(pThisCC, pBufferPipeline->bufferId, &pSurface, &pResource);
@@ -6017 +6017 @@
         {
             LogFunc(("vb[%u]: %p\n", i, pBuffer));
-            paStride[i] = pBufferPipeline->stride;
-            paOffset[i] = pBufferPipeline->offset;
+
+            /* DX11 supports stride up tp 2048. Ignore large values (> 40000) that Ubuntu guest might send. */
+            paStride[i] = pBufferPipeline->stride <= 2048 ? pBufferPipeline->stride : 0;
+
+            if (   paStride[i] <= pSurface->paMipmapLevels[0].cbSurface
+                && pBufferPipeline->offset <= pSurface->paMipmapLevels[0].cbSurface - paStride[i])
+                paOffset[i] = pBufferPipeline->offset;
+            else
+            {
+                paStride[i] = 0;
+                paOffset[i] = 0;
+            }
         }
         else