Merge pull request #102792 from clayjohn/varying-crash

Validate varying count when compiling shaders

Author: Repiteo

drivers/d3d12/rendering_device_driver_d3d12.cpp

@@ -6234,6 +6234,8 @@ uint64_t RenderingDeviceDriverD3D12::limit_get(Limit p_limit) {
 		case LIMIT_VRS_MAX_FRAGMENT_WIDTH:
 		case LIMIT_VRS_MAX_FRAGMENT_HEIGHT:
 			return vrs_capabilities.ss_max_fragment_size;
+		case LIMIT_MAX_SHADER_VARYINGS:
+			return MIN(D3D12_VS_OUTPUT_REGISTER_COUNT, D3D12_PS_INPUT_REGISTER_COUNT);
 		default: {
 #ifdef DEV_ENABLED
 			WARN_PRINT("Returning maximum value for unknown limit " + itos(p_limit) + ".");

drivers/gles3/storage/config.cpp

@@ -110,6 +110,11 @@ Config::Config() {
 	glGetIntegerv(GL_MAX_TEXTURE_SIZE, &max_texture_size);
 	glGetIntegerv(GL_MAX_VIEWPORT_DIMS, max_viewport_size);
 	glGetInteger64v(GL_MAX_UNIFORM_BLOCK_SIZE, &max_uniform_buffer_size);
+	GLint max_vertex_output;
+	glGetIntegerv(GL_MAX_VERTEX_OUTPUT_COMPONENTS, &max_vertex_output);
+	GLint max_fragment_input;
+	glGetIntegerv(GL_MAX_FRAGMENT_INPUT_COMPONENTS, &max_fragment_input);
+	max_shader_varyings = (uint32_t)MIN(max_vertex_output, max_fragment_input) / 4;
 
 	// sanity clamp buffer size to 16K..1MB
 	max_uniform_buffer_size = CLAMP(max_uniform_buffer_size, 16384, 1048576);

drivers/gles3/storage/config.h

@@ -62,6 +62,7 @@ class Config {
 	GLint max_texture_size = 0;
 	GLint max_viewport_size[2] = { 0, 0 };
 	GLint64 max_uniform_buffer_size = 0;
+	uint32_t max_shader_varyings = 0;
 
 	int64_t max_renderable_elements = 0;
 	int64_t max_renderable_lights = 0;

drivers/gles3/storage/utilities.cpp

@@ -465,4 +465,16 @@ Size2i Utilities::get_maximum_viewport_size() const {
 	return Size2i(config->max_viewport_size[0], config->max_viewport_size[1]);
 }
 
+uint32_t Utilities::get_maximum_shader_varyings() const {
+	Config *config = Config::get_singleton();
+	ERR_FAIL_NULL_V(config, 31);
+	return config->max_shader_varyings;
+}
+
+uint64_t Utilities::get_maximum_uniform_buffer_size() const {
+	Config *config = Config::get_singleton();
+	ERR_FAIL_NULL_V(config, 65536);
+	return uint64_t(config->max_uniform_buffer_size);
+}
+
 #endif // GLES3_ENABLED

drivers/gles3/storage/utilities.h

@@ -226,6 +226,8 @@ class Utilities : public RendererUtilities {
 	virtual String get_video_adapter_api_version() const override;
 
 	virtual Size2i get_maximum_viewport_size() const override;
+	virtual uint32_t get_maximum_shader_varyings() const override;
+	virtual uint64_t get_maximum_uniform_buffer_size() const override;
 };
 
 } // namespace GLES3

drivers/metal/metal_device_properties.h

@@ -124,6 +124,7 @@ struct MetalLimits {
 	uint32_t maxVertexInputBindings;
 	uint32_t maxVertexInputBindingStride;
 	uint32_t maxDrawIndexedIndexValue;
+	uint32_t maxShaderVaryings;
 
 	double temporalScalerInputContentMinScale;
 	double temporalScalerInputContentMaxScale;

drivers/metal/metal_device_properties.mm

@@ -303,6 +303,7 @@
 	limits.maxVertexInputAttributes = 31;
 	limits.maxVertexInputBindings = 31;
 	limits.maxVertexInputBindingStride = (2 * KIBI);
+	limits.maxShaderVaryings = 31; // Accurate on Apple4 and above. See: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
 
 #if TARGET_OS_IOS && !TARGET_OS_MACCATALYST
 	limits.minUniformBufferOffsetAlignment = 64;

drivers/metal/rendering_device_driver_metal.mm

@@ -3999,6 +3999,8 @@ bool isArrayTexture(MTLTextureType p_type) {
 			return (uint64_t)((1.0 / limits.temporalScalerInputContentMaxScale) * 1000'000);
 		case LIMIT_METALFX_TEMPORAL_SCALER_MAX_SCALE:
 			return (uint64_t)((1.0 / limits.temporalScalerInputContentMinScale) * 1000'000);
+		case LIMIT_MAX_SHADER_VARYINGS:
+			return limits.maxShaderVaryings;
 		UNKNOWN(LIMIT_VRS_TEXEL_WIDTH);
 		UNKNOWN(LIMIT_VRS_TEXEL_HEIGHT);
 		UNKNOWN(LIMIT_VRS_MAX_FRAGMENT_WIDTH);

drivers/vulkan/rendering_device_driver_vulkan.cpp

@@ -5886,6 +5886,10 @@ uint64_t RenderingDeviceDriverVulkan::limit_get(Limit p_limit) {
 			return vrs_capabilities.max_fragment_size.x;
 		case LIMIT_VRS_MAX_FRAGMENT_HEIGHT:
 			return vrs_capabilities.max_fragment_size.y;
+		case LIMIT_MAX_SHADER_VARYINGS:
+			// The Vulkan spec states that built in varyings like gl_FragCoord should count against this, but in
+			// practice, that doesn't seem to be the case. The validation layers don't even complain.
+			return MIN(limits.maxVertexOutputComponents / 4, limits.maxFragmentInputComponents / 4);
 		default:
 			ERR_FAIL_V(0);
 	}

servers/rendering/dummy/storage/utilities.h

@@ -94,6 +94,8 @@ class Utilities : public RendererUtilities {
 	virtual String get_video_adapter_api_version() const override { return String(); }
 
 	virtual Size2i get_maximum_viewport_size() const override { return Size2i(); }
+	virtual uint32_t get_maximum_shader_varyings() const override { return 31; } // Fair assumption for everything except old OpenGL-only phones.
+	virtual uint64_t get_maximum_uniform_buffer_size() const override { return 65536; } // Fair assumption for all devices.
 };
 
 } // namespace RendererDummy

servers/rendering/renderer_rd/storage_rd/utilities.cpp

@@ -329,3 +329,11 @@ Size2i Utilities::get_maximum_viewport_size() const {
 	int max_y = device->limit_get(RenderingDevice::LIMIT_MAX_VIEWPORT_DIMENSIONS_Y);
 	return Size2i(max_x, max_y);
 }
+
+uint32_t Utilities::get_maximum_shader_varyings() const {
+	return RenderingDevice::get_singleton()->limit_get(RenderingDevice::LIMIT_MAX_SHADER_VARYINGS);
+}
+
+uint64_t Utilities::get_maximum_uniform_buffer_size() const {
+	return RenderingDevice::get_singleton()->limit_get(RenderingDevice::LIMIT_MAX_UNIFORM_BUFFER_SIZE);
+}

servers/rendering/renderer_rd/storage_rd/utilities.h

@@ -117,6 +117,8 @@ class Utilities : public RendererUtilities {
 	virtual String get_video_adapter_api_version() const override;
 
 	virtual Size2i get_maximum_viewport_size() const override;
+	virtual uint32_t get_maximum_shader_varyings() const override;
+	virtual uint64_t get_maximum_uniform_buffer_size() const override;
 };
 
 } // namespace RendererRD

servers/rendering/rendering_device_commons.h

@@ -875,6 +875,7 @@ class RenderingDeviceCommons : public Object {
 		LIMIT_VRS_MAX_FRAGMENT_HEIGHT,
 		LIMIT_METALFX_TEMPORAL_SCALER_MIN_SCALE,
 		LIMIT_METALFX_TEMPORAL_SCALER_MAX_SCALE,
+		LIMIT_MAX_SHADER_VARYINGS,
 	};
 
 	enum Features {

servers/rendering/shader_compiler.cpp

@@ -686,30 +686,14 @@ String ShaderCompiler::_dump_node_code(const SL::Node *p_node, int p_level, Gene
 				vcode += _prestr(varying.precision, ShaderLanguage::is_float_type(varying.type));
 				vcode += _typestr(varying.type);
 				vcode += " " + _mkid(varying_name);
-				uint32_t inc = 1U;
+				uint32_t inc = varying.get_size();
 
 				if (varying.array_size > 0) {
-					inc = (uint32_t)varying.array_size;
-
 					vcode += "[";
 					vcode += itos(varying.array_size);
 					vcode += "]";
 				}
 
-				switch (varying.type) {
-					case SL::TYPE_MAT2:
-						inc *= 2U;
-						break;
-					case SL::TYPE_MAT3:
-						inc *= 3U;
-						break;
-					case SL::TYPE_MAT4:
-						inc *= 4U;
-						break;
-					default:
-						break;
-				}
-
 				vcode += ";\n";
 				// GLSL ES 3.0 does not allow layout qualifiers for varyings
 				if (!RS::get_singleton()->is_low_end()) {
@@ -1481,6 +1465,7 @@ Error ShaderCompiler::compile(RS::ShaderMode p_mode, const String &p_code, Ident
 	info.render_modes = ShaderTypes::get_singleton()->get_modes(p_mode);
 	info.shader_types = ShaderTypes::get_singleton()->get_types();
 	info.global_shader_uniform_type_func = _get_global_shader_uniform_type;
+	info.base_varying_index = actions.base_varying_index;
 
 	Error err = parser.compile(p_code, info);
 

servers/rendering/shader_language.cpp

@@ -33,6 +33,7 @@
 #include "core/os/os.h"
 #include "core/templates/local_vector.h"
 #include "servers/rendering/renderer_compositor.h"
+#include "servers/rendering/rendering_server_globals.h"
 #include "servers/rendering_server.h"
 #include "shader_types.h"
 
@@ -9111,17 +9112,12 @@ Error ShaderLanguage::_parse_shader(const HashMap<StringName, FunctionInfo> &p_f
 	int prop_index = 0;
 #ifdef DEBUG_ENABLED
 	uint64_t uniform_buffer_size = 0;
-	uint64_t max_uniform_buffer_size = 0;
+	uint64_t max_uniform_buffer_size = 65536;
 	int uniform_buffer_exceeded_line = -1;
-
-	bool check_device_limit_warnings = false;
-	{
-		RenderingDevice *device = RenderingDevice::get_singleton();
-		if (device != nullptr) {
-			check_device_limit_warnings = check_warnings && HAS_WARNING(ShaderWarning::DEVICE_LIMIT_EXCEEDED_FLAG);
-
-			max_uniform_buffer_size = device->limit_get(RenderingDevice::LIMIT_MAX_UNIFORM_BUFFER_SIZE);
-		}
+	bool check_device_limit_warnings = check_warnings && HAS_WARNING(ShaderWarning::DEVICE_LIMIT_EXCEEDED_FLAG);
+	// Can be false for internal shaders created in the process of initializing the engine.
+	if (RSG::utilities) {
+		max_uniform_buffer_size = RSG::utilities->get_maximum_uniform_buffer_size();
 	}
 #endif // DEBUG_ENABLED
 	ShaderNode::Uniform::Scope uniform_scope = ShaderNode::Uniform::SCOPE_LOCAL;
@@ -10959,13 +10955,27 @@ Error ShaderLanguage::_parse_shader(const HashMap<StringName, FunctionInfo> &p_f
 
 		tk = _get_token();
 	}
+	uint32_t varying_index = base_varying_index;
+	uint32_t max_varyings = 31;
+	// Can be false for internal shaders created in the process of initializing the engine.
+	if (RSG::utilities) {
+		max_varyings = RSG::utilities->get_maximum_shader_varyings();
+	}
 
 	for (const KeyValue<StringName, ShaderNode::Varying> &kv : shader->varyings) {
 		if (kv.value.stage != ShaderNode::Varying::STAGE_FRAGMENT && (kv.value.type > TYPE_BVEC4 && kv.value.type < TYPE_FLOAT) && kv.value.interpolation != INTERPOLATION_FLAT) {
 			_set_tkpos(kv.value.tkpos);
 			_set_error(vformat(RTR("Varying with integer data type must be declared with `%s` interpolation qualifier."), "flat"));
 			return ERR_PARSE_ERROR;
 		}
+
+		if (varying_index + kv.value.get_size() > max_varyings) {
+			_set_tkpos(kv.value.tkpos);
+			_set_error(vformat(RTR("Too many varyings used in shader (%d used, maximum supported is %d)."), varying_index + kv.value.get_size(), max_varyings));
+			return ERR_PARSE_ERROR;
+		}
+
+		varying_index += kv.value.get_size();
 	}
 
 #ifdef DEBUG_ENABLED
@@ -11183,6 +11193,7 @@ Error ShaderLanguage::compile(const String &p_code, const ShaderCompileInfo &p_i
 	global_shader_uniform_get_type_func = p_info.global_shader_uniform_type_func;
 
 	varying_function_names = p_info.varying_function_names;
+	base_varying_index = p_info.base_varying_index;
 
 	nodes = nullptr;
 

servers/rendering/shader_language.h

@@ -631,6 +631,28 @@ class ShaderLanguage {
 			int array_size = 0;
 			TkPos tkpos;
 
+			uint32_t get_size() const {
+				uint32_t size = 1;
+				if (array_size > 0) {
+					size = (uint32_t)array_size;
+				}
+
+				switch (type) {
+					case TYPE_MAT2:
+						size *= 2;
+						break;
+					case TYPE_MAT3:
+						size *= 3;
+						break;
+					case TYPE_MAT4:
+						size *= 4;
+						break;
+					default:
+						break;
+				}
+				return size;
+			}
+
 			Varying() {}
 		};
 
@@ -1022,6 +1044,7 @@ class ShaderLanguage {
 	String current_uniform_subgroup_name;
 
 	VaryingFunctionNames varying_function_names;
+	uint32_t base_varying_index = 0;
 
 	TkPos _get_tkpos() {
 		TkPos tkp;
@@ -1217,6 +1240,7 @@ class ShaderLanguage {
 		HashSet<String> shader_types;
 		GlobalShaderUniformGetTypeFunc global_shader_uniform_type_func = nullptr;
 		bool is_include = false;
+		uint32_t base_varying_index = 0;
 	};
 
 	Error compile(const String &p_code, const ShaderCompileInfo &p_info);

servers/rendering/storage/utilities.h

@@ -184,6 +184,8 @@ class RendererUtilities {
 	virtual String get_video_adapter_api_version() const = 0;
 
 	virtual Size2i get_maximum_viewport_size() const = 0;
+	virtual uint32_t get_maximum_shader_varyings() const = 0;
+	virtual uint64_t get_maximum_uniform_buffer_size() const = 0;
 };
 
 #endif // RENDERER_UTILITIES_H