Changeset 82718 in vbox

Timestamp:

Jan 10, 2020 4:25:50 PM (5 years ago)

Author:

vboxsync

Message:

3D: Check last shader bytecode token, bugref:9613

Location:

trunk/src/VBox/Devices/Graphics

Files:

• DevVGA-SVGA3d-ogl.cpp (modified) (1 diff)
• shaderlib/shader.c (modified) (1 diff)
• shaderlib/shader_sm1.c (modified) (4 diffs)
• shaderlib/shader_sm4.c (modified) (3 diffs)
• shaderlib/shaderapi.c (modified) (4 diffs)
• shaderlib/shaderlib.h (modified) (1 diff)
• shaderlib/wined3d_private.h (modified) (1 diff)

trunk/src/VBox/Devices/Graphics/DevVGA-SVGA3d-ogl.cpp

@@ -6908 +6908 @@
     {
     case SVGA3D_SHADERTYPE_VS:
-        rc = ShaderCreateVertexShader(pContext->pShaderContext, (const uint32_t *)pShaderData, &pShader->u.pVertexShader);
+        rc = ShaderCreateVertexShader(pContext->pShaderContext, (const uint32_t *)pShaderData, cbData, &pShader->u.pVertexShader);
         AssertRC(rc);
         break;
 
     case SVGA3D_SHADERTYPE_PS:
-        rc = ShaderCreatePixelShader(pContext->pShaderContext, (const uint32_t *)pShaderData, &pShader->u.pPixelShader);
+        rc = ShaderCreatePixelShader(pContext->pShaderContext, (const uint32_t *)pShaderData, cbData, &pShader->u.pPixelShader);
         AssertRC(rc);
         break;

trunk/src/VBox/Devices/Graphics/shaderlib/shader.c

@@ -1568 +1568 @@
     }
     shader->baseShader.frontend = fe;
-    shader->baseShader.frontend_data = fe->shader_init(byte_code, output_signature);
+    shader->baseShader.frontend_data = fe->shader_init(byte_code, shader->baseShader.functionLength / 4, output_signature);
     if (!shader->baseShader.frontend_data)
     {

trunk/src/VBox/Devices/Graphics/shaderlib/shader_sm1.c

@@ -213 +213 @@
 {
     struct wined3d_shader_version shader_version;
+    const DWORD *end;
     const struct wined3d_sm1_opcode_info *opcode_table;
 };
@@ -518 +519 @@
 }
 
-static void *shader_sm1_init(const DWORD *byte_code, const struct wined3d_shader_signature *output_signature)
+static void *shader_sm1_init(const DWORD *byte_code, DWORD tokens_num, const struct wined3d_shader_signature *output_signature)
 {
     struct wined3d_sm1_data *priv;
@@ -537 +538 @@
         return NULL;
     }
+    
+    priv->end = byte_code + tokens_num;
 
     if (output_signature)
@@ -674 +677 @@
 static BOOL shader_sm1_is_end(void *data, const DWORD **ptr)
 {
+    struct wined3d_sm1_data *priv = data;
+
+    if (*ptr >= priv->end)
+    {
+        return TRUE;
+    }
+
     if (**ptr == WINED3DSP_END)
     {

trunk/src/VBox/Devices/Graphics/shaderlib/shader_sm4.c

@@ -229 +229 @@
 }
 
-static void *shader_sm4_init(const DWORD *byte_code, const struct wined3d_shader_signature *output_signature)
+static void *shader_sm4_init(const DWORD *byte_code, DWORD tokens_num, const struct wined3d_shader_signature *output_signature)
 {
     struct wined3d_sm4_data *priv = HeapAlloc(GetProcessHeap(), 0, sizeof(*priv));
@@ -238 +238 @@
     }
 
+    priv->end = byte_code + tokens_num;
     priv->output_signature = output_signature;
 
@@ -425 +426 @@
 {
     struct wined3d_sm4_data *priv = data;
-    return *ptr == priv->end;
+    return *ptr >= priv->end;
 }
 

trunk/src/VBox/Devices/Graphics/shaderlib/shaderapi.c

@@ -314 +314 @@
 }
 
-SHADERDECL(int) ShaderCreateVertexShader(void *pShaderContext, const uint32_t *pShaderData, void **pShaderObj)
+SHADERDECL(int) ShaderCreateVertexShader(void *pShaderContext, const uint32_t *pShaderData, uint32_t cbShaderData, void **pShaderObj)
 {
     IWineD3DDeviceImpl *This;
@@ -329 +329 @@
         return VERR_NO_MEMORY;
     }
+
+    object->baseShader.functionLength = cbShaderData;
 
     hr = vertexshader_init(object, This, (DWORD const *)pShaderData, NULL, NULL, NULL);
@@ -356 +358 @@
 }
 
-SHADERDECL(int) ShaderCreatePixelShader(void *pShaderContext, const uint32_t *pShaderData, void **pShaderObj)
+SHADERDECL(int) ShaderCreatePixelShader(void *pShaderContext, const uint32_t *pShaderData, uint32_t cbShaderData, void **pShaderObj)
 {
     IWineD3DDeviceImpl *This;
@@ -371 +373 @@
         return VERR_NO_MEMORY;
     }
+
+    object->baseShader.functionLength = cbShaderData;
 
     hr = pixelshader_init(object, This, (DWORD const *)pShaderData, NULL, NULL, NULL);

trunk/src/VBox/Devices/Graphics/shaderlib/shaderlib.h

@@ -72 +72 @@
 SHADERDECL(int) ShaderContextDestroy(void *pShaderContext);
 
-SHADERDECL(int) ShaderCreateVertexShader(void *pShaderContext, const uint32_t *pShaderData, void **pShaderObj);
-SHADERDECL(int) ShaderCreatePixelShader(void *pShaderContext, const uint32_t *pShaderData, void **pShaderObj);
+SHADERDECL(int) ShaderCreateVertexShader(void *pShaderContext, const uint32_t *pShaderData, uint32_t cbShaderData, void **pShaderObj);
+SHADERDECL(int) ShaderCreatePixelShader(void *pShaderContext, const uint32_t *pShaderData, uint32_t cbShaderData, void **pShaderObj);
 
 SHADERDECL(int) ShaderDestroyVertexShader(void *pShaderContext, void *pShaderObj);

trunk/src/VBox/Devices/Graphics/shaderlib/wined3d_private.h

@@ -676 +676 @@
 struct wined3d_shader_frontend
 {
-    void *(*shader_init)(const DWORD *ptr, const struct wined3d_shader_signature *output_signature);
+    void *(*shader_init)(const DWORD *ptr, DWORD tokens_num, const struct wined3d_shader_signature *output_signature);
     void (*shader_free)(void *data);
     void (*shader_read_header)(void *data, const DWORD **ptr, struct wined3d_shader_version *shader_version);