Changeset 79670 in vbox for trunk/src/VBox/Devices/Graphics/shaderlib

Timestamp:

Jul 10, 2019 11:12:57 AM (6 years ago)

Author:

vboxsync

Message:

VMSVGA: Adding some more validation of a shader bytecode, bugref:9460

File:

• trunk/src/VBox/Devices/Graphics/shaderlib/shader.c (modified) (5 diffs)

trunk/src/VBox/Devices/Graphics/shaderlib/shader.c

@@ -500 +500 @@
         {
             struct wined3d_shader_semantic semantic;
+            unsigned int reg_idx;
 
             fe->shader_read_semantic(&ptr, &semantic);
+            reg_idx = semantic.reg.reg.idx;
 
             switch (semantic.reg.reg.type)
@@ -507 +509 @@
                 /* Mark input registers used. */
                 case WINED3DSPR_INPUT:
-                    reg_maps->input_registers |= 1 << semantic.reg.reg.idx;
-                    shader_signature_from_semantic(&input_signature[semantic.reg.reg.idx], &semantic);
+                    if (reg_idx >= MAX_REG_INPUT)
+                    {
+                        ERR("Invalid input register index %d.\n", reg_idx);
+                        return E_INVALIDARG;
+                    }
+                    reg_maps->input_registers |= 1 << reg_idx;
+                    shader_signature_from_semantic(&input_signature[reg_idx], &semantic);
                     break;
 
                 /* Vertex shader: mark 3.0 output registers used, save token. */
                 case WINED3DSPR_OUTPUT:
-                    reg_maps->output_registers |= 1 << semantic.reg.reg.idx;
-                    shader_signature_from_semantic(&output_signature[semantic.reg.reg.idx], &semantic);
+                    if (reg_idx >= MAX_REG_OUTPUT)
+                    {
+                        ERR("Invalid output register index %d.\n", reg_idx);
+                        return E_INVALIDARG;
+                    }
+                    reg_maps->output_registers |= 1 << reg_idx;
+                    shader_signature_from_semantic(&output_signature[reg_idx], &semantic);
                     if (semantic.usage == WINED3DDECLUSAGE_FOG) reg_maps->fog = 1;
                     break;
@@ -520 +532 @@
                 /* Save sampler usage token. */
                 case WINED3DSPR_SAMPLER:
-                    reg_maps->sampler_type[semantic.reg.reg.idx] = semantic.sampler_type;
+                    if (reg_idx >= RT_ELEMENTS(reg_maps->sampler_type))
+                    {
+                        ERR("Invalid sampler index %d.\n", reg_idx);
+                        return E_INVALIDARG;
+                    }
+                    reg_maps->sampler_type[reg_idx] = semantic.sampler_type;
                     break;
 
@@ -654 +671 @@
                         && dst_param.reg.type == WINED3DSPR_TEXCRDOUT)
                 {
-                    reg_maps->texcoord_mask[dst_param.reg.idx] |= dst_param.write_mask;
+                    unsigned int idx = dst_param.reg.idx;
+                    
+                    if (idx >= MAX_REG_TEXCRD)
+                    {
+                        ERR("Invalid texcoord index %d.\n", idx);
+                        return E_INVALIDARG;
+                    }
+                    
+                    reg_maps->texcoord_mask[idx] |= dst_param.write_mask;
                 }
 
@@ -703 +728 @@
                     /* Fake sampler usage, only set reserved bit and type. */
                     DWORD sampler_code = dst_param.reg.idx;
+                    
+                    if (sampler_code >= RT_ELEMENTS(reg_maps->sampler_type))
+                    {
+                        ERR("Invalid sampler index %d.\n", sampler_code);
+                        return E_INVALIDARG;
+                    }
 
                     TRACE("Setting fake 2D sampler for 1.x pixelshader.\n");