From 7ba8fccad3b226082601368cee25161d4f1dd320 Mon Sep 17 00:00:00 2001 From: Konstantin Seurer Date: Sun, 3 Mar 2024 15:36:54 +0100 Subject: [PATCH] radv/rt: Track ray_launch_id reads We can expect the z-component to be unused most of the times. Avoid preserving it in those cases. Totals from 94 (24.80% of 379) affected shaders: MaxWaves: 916 -> 935 (+2.07%) Instrs: 3316697 -> 3318357 (+0.05%); split: -0.06%, +0.11% CodeSize: 17618704 -> 17616680 (-0.01%); split: -0.09%, +0.08% VGPRs: 11632 -> 11520 (-0.96%) SpillSGPRs: 1139 -> 1205 (+5.79%); split: -0.35%, +6.15% Latency: 22595907 -> 22598225 (+0.01%); split: -0.15%, +0.16% InvThroughput: 7036479 -> 6923740 (-1.60%); split: -1.74%, +0.14% VClause: 104325 -> 104361 (+0.03%); split: -0.16%, +0.19% SClause: 83920 -> 83925 (+0.01%); split: -0.08%, +0.08% Copies: 328140 -> 330687 (+0.78%); split: -0.27%, +1.05% Branches: 134521 -> 134541 (+0.01%); split: -0.01%, +0.02% PreSGPRs: 8753 -> 8806 (+0.61%) PreVGPRs: 10984 -> 10937 (-0.43%) VALU: 2149880 -> 2151318 (+0.07%); split: -0.08%, +0.15% SALU: 499107 -> 499128 (+0.00%); split: -0.08%, +0.09% Reviewed-by: Friedrich Vock Part-of: --- src/amd/common/ac_shader_args.h | 2 +- .../compiler/aco_instruction_selection.cpp | 8 +-- .../aco_instruction_selection_setup.cpp | 1 - src/amd/vulkan/nir/radv_nir_rt_shader.c | 60 +++++++++++++++---- src/amd/vulkan/radv_shader_args.c | 4 +- 5 files changed, 55 insertions(+), 20 deletions(-) diff --git a/src/amd/common/ac_shader_args.h b/src/amd/common/ac_shader_args.h index 1622d3d3813..794b17c9b4f 100644 --- a/src/amd/common/ac_shader_args.h +++ b/src/amd/common/ac_shader_args.h @@ -183,7 +183,7 @@ struct ac_shader_args { struct ac_arg sbt_descriptors; struct ac_arg launch_size; struct ac_arg launch_size_addr; - struct ac_arg launch_id; + struct ac_arg launch_ids[3]; struct ac_arg dynamic_callable_stack_base; struct ac_arg traversal_shader_addr; struct ac_arg shader_addr; diff --git a/src/amd/compiler/aco_instruction_selection.cpp b/src/amd/compiler/aco_instruction_selection.cpp index 4b3c5b366bb..dd05836a7b6 100644 --- a/src/amd/compiler/aco_instruction_selection.cpp +++ b/src/amd/compiler/aco_instruction_selection.cpp @@ -8305,12 +8305,6 @@ visit_intrinsic(isel_context* ctx, nir_intrinsic_instr* instr) emit_split_vector(ctx, dst, 3); break; } - case nir_intrinsic_load_ray_launch_id: { - Temp dst = get_ssa_temp(ctx, &instr->def); - bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->rt.launch_id))); - emit_split_vector(ctx, dst, 3); - break; - } case nir_intrinsic_load_local_invocation_id: { Temp dst = get_ssa_temp(ctx, &instr->def); if (ctx->options->gfx_level >= GFX11) { @@ -12358,7 +12352,7 @@ select_rt_prolog(Program* program, ac_shader_config* config, PhysReg out_launch_size_z = out_launch_size_y.advance(4); PhysReg out_launch_ids[3]; for (unsigned i = 0; i < 3; i++) - out_launch_ids[i] = get_arg_reg(out_args, out_args->rt.launch_id).advance(i * 4); + out_launch_ids[i] = get_arg_reg(out_args, out_args->rt.launch_ids[i]); PhysReg out_stack_ptr = get_arg_reg(out_args, out_args->rt.dynamic_callable_stack_base); PhysReg out_record_ptr = get_arg_reg(out_args, out_args->rt.shader_record); diff --git a/src/amd/compiler/aco_instruction_selection_setup.cpp b/src/amd/compiler/aco_instruction_selection_setup.cpp index 37ad50ff113..753ce0ffde9 100644 --- a/src/amd/compiler/aco_instruction_selection_setup.cpp +++ b/src/amd/compiler/aco_instruction_selection_setup.cpp @@ -481,7 +481,6 @@ init_context(isel_context* ctx, nir_shader* shader) case nir_intrinsic_load_local_invocation_id: case nir_intrinsic_load_local_invocation_index: case nir_intrinsic_load_subgroup_invocation: - case nir_intrinsic_load_ray_launch_id: case nir_intrinsic_load_tess_coord: case nir_intrinsic_write_invocation_amd: case nir_intrinsic_mbcnt_amd: diff --git a/src/amd/vulkan/nir/radv_nir_rt_shader.c b/src/amd/vulkan/nir/radv_nir_rt_shader.c index 547a201bf78..05c77af1126 100644 --- a/src/amd/vulkan/nir/radv_nir_rt_shader.c +++ b/src/amd/vulkan/nir/radv_nir_rt_shader.c @@ -189,6 +189,8 @@ struct rt_variables { nir_variable *ahit_isec_count; + nir_variable *launch_ids[3]; + /* global address of the SBT entry used for the shader */ nir_variable *shader_record_ptr; @@ -233,6 +235,10 @@ create_rt_variables(nir_shader *shader, struct radv_device *device, const VkPipe vars.stack_ptr = nir_variable_create(shader, nir_var_shader_temp, glsl_uint_type(), "stack_ptr"); vars.shader_record_ptr = nir_variable_create(shader, nir_var_shader_temp, glsl_uint64_t_type(), "shader_record_ptr"); + vars.launch_ids[0] = nir_variable_create(shader, nir_var_shader_temp, glsl_uint_type(), "launch_id_x"); + vars.launch_ids[1] = nir_variable_create(shader, nir_var_shader_temp, glsl_uint_type(), "launch_id_y"); + vars.launch_ids[2] = nir_variable_create(shader, nir_var_shader_temp, glsl_uint_type(), "launch_id_z"); + if (device->rra_trace.ray_history_addr) vars.ahit_isec_count = nir_variable_create(shader, nir_var_shader_temp, glsl_uint_type(), "ahit_isec_count"); @@ -273,6 +279,9 @@ map_rt_variables(struct hash_table *var_remap, struct rt_variables *src, const s _mesa_hash_table_insert(var_remap, src->stack_ptr, dst->stack_ptr); _mesa_hash_table_insert(var_remap, src->shader_record_ptr, dst->shader_record_ptr); + for (uint32_t i = 0; i < ARRAY_SIZE(src->launch_ids); i++) + _mesa_hash_table_insert(var_remap, src->launch_ids[i], dst->launch_ids[i]); + if (dst->ahit_isec_count) _mesa_hash_table_insert(var_remap, src->ahit_isec_count, dst->ahit_isec_count); @@ -366,9 +375,15 @@ load_sbt_entry(nir_builder *b, const struct rt_variables *vars, nir_def *idx, en nir_store_var(b, vars->shader_record_ptr, record_addr, 1); } +struct radv_rt_shader_info { + bool uses_launch_id; +}; + struct radv_lower_rt_instruction_data { struct rt_variables *vars; - bool apply_stack_ptr; + bool late_lowering; + + struct radv_rt_shader_info *out_info; }; static bool @@ -389,7 +404,6 @@ radv_lower_rt_instruction(nir_builder *b, nir_instr *instr, void *_data) struct radv_lower_rt_instruction_data *data = _data; struct rt_variables *vars = data->vars; - bool apply_stack_ptr = data->apply_stack_ptr; b->cursor = nir_before_instr(&intr->instr); @@ -454,12 +468,12 @@ radv_lower_rt_instruction(nir_builder *b, nir_instr *instr, void *_data) break; } case nir_intrinsic_load_scratch: { - if (apply_stack_ptr) + if (data->late_lowering) nir_src_rewrite(&intr->src[0], nir_iadd_nuw(b, nir_load_var(b, vars->stack_ptr), intr->src[0].ssa)); return true; } case nir_intrinsic_store_scratch: { - if (apply_stack_ptr) + if (data->late_lowering) nir_src_rewrite(&intr->src[1], nir_iadd_nuw(b, nir_load_var(b, vars->stack_ptr), intr->src[1].ssa)); return true; } @@ -471,6 +485,17 @@ radv_lower_rt_instruction(nir_builder *b, nir_instr *instr, void *_data) ret = nir_load_var(b, vars->shader_record_ptr); break; } + case nir_intrinsic_load_ray_launch_id: { + if (data->out_info) + data->out_info->uses_launch_id = true; + + if (!data->late_lowering) + return false; + + ret = nir_vec3(b, nir_load_var(b, vars->launch_ids[0]), nir_load_var(b, vars->launch_ids[1]), + nir_load_var(b, vars->launch_ids[2])); + break; + } case nir_intrinsic_load_ray_t_min: { ret = nir_load_var(b, vars->tmin); break; @@ -671,11 +696,13 @@ radv_lower_rt_instruction(nir_builder *b, nir_instr *instr, void *_data) /* This lowers all the RT instructions that we do not want to pass on to the combined shader and * that we can implement using the variables from the shader we are going to inline into. */ static void -lower_rt_instructions(nir_shader *shader, struct rt_variables *vars, bool apply_stack_ptr) +lower_rt_instructions(nir_shader *shader, struct rt_variables *vars, bool late_lowering, + struct radv_rt_shader_info *out_info) { struct radv_lower_rt_instruction_data data = { .vars = vars, - .apply_stack_ptr = apply_stack_ptr, + .late_lowering = late_lowering, + .out_info = out_info, }; nir_shader_instructions_pass(shader, radv_lower_rt_instruction, nir_metadata_none, &data); } @@ -782,7 +809,7 @@ insert_rt_case(nir_builder *b, nir_shader *shader, struct rt_variables *vars, ni struct rt_variables src_vars = create_rt_variables(shader, vars->device, vars->flags, vars->monolithic); map_rt_variables(var_remap, &src_vars, vars); - NIR_PASS_V(shader, lower_rt_instructions, &src_vars, false); + NIR_PASS_V(shader, lower_rt_instructions, &src_vars, false, NULL); NIR_PASS(_, shader, nir_lower_returns); NIR_PASS(_, shader, nir_opt_dce); @@ -1907,7 +1934,9 @@ radv_nir_lower_rt_abi(nir_shader *shader, const VkRayTracingPipelineCreateInfoKH if (monolithic) lower_rt_instructions_monolithic(shader, device, pipeline, pCreateInfo, &vars); - lower_rt_instructions(shader, &vars, true); + struct radv_rt_shader_info rt_info = {0}; + + lower_rt_instructions(shader, &vars, true, &rt_info); if (stack_size) { vars.stack_size = MAX2(vars.stack_size, shader->scratch_size); @@ -1933,7 +1962,12 @@ radv_nir_lower_rt_abi(nir_shader *shader, const VkRayTracingPipelineCreateInfoKH nir_def *ring_offsets = NULL; if (args->ac.ring_offsets.used) ring_offsets = ac_nir_load_arg(&b, &args->ac, args->ac.ring_offsets); - nir_def *launch_id = ac_nir_load_arg(&b, &args->ac, args->ac.rt.launch_id); + + nir_def *launch_ids[3]; + for (uint32_t i = 0; i < ARRAY_SIZE(launch_ids); i++) { + launch_ids[i] = ac_nir_load_arg(&b, &args->ac, args->ac.rt.launch_ids[i]); + nir_store_var(&b, vars.launch_ids[i], launch_ids[i], 1); + } nir_def *traversal_addr = ac_nir_load_arg(&b, &args->ac, args->ac.rt.traversal_shader_addr); nir_store_var(&b, vars.traversal_addr, nir_pack_64_2x32(&b, traversal_addr), 1); @@ -2003,7 +2037,13 @@ radv_nir_lower_rt_abi(nir_shader *shader, const VkRayTracingPipelineCreateInfoKH ac_nir_store_arg(&b, &args->ac, args->ac.scratch_offset, scratch_offset); if (ring_offsets) ac_nir_store_arg(&b, &args->ac, args->ac.ring_offsets, ring_offsets); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.launch_id, launch_id); + + for (uint32_t i = 0; i < ARRAY_SIZE(launch_ids); i++) { + if (rt_info.uses_launch_id) + ac_nir_store_arg(&b, &args->ac, args->ac.rt.launch_ids[i], launch_ids[i]); + else + radv_store_arg(&b, args, traversal_info, args->ac.rt.launch_ids[i], launch_ids[i]); + } /* store back all variables to registers */ ac_nir_store_arg(&b, &args->ac, args->ac.rt.dynamic_callable_stack_base, nir_load_var(&b, vars.stack_ptr)); diff --git a/src/amd/vulkan/radv_shader_args.c b/src/amd/vulkan/radv_shader_args.c index a2cd741c4e8..684ee395689 100644 --- a/src/amd/vulkan/radv_shader_args.c +++ b/src/amd/vulkan/radv_shader_args.c @@ -323,7 +323,9 @@ radv_declare_rt_shader_args(enum amd_gfx_level gfx_level, struct radv_shader_arg ac_add_arg(&args->ac, AC_ARG_SGPR, 2, AC_ARG_CONST_DESC_PTR, &args->ac.ring_offsets); } - ac_add_arg(&args->ac, AC_ARG_VGPR, 3, AC_ARG_INT, &args->ac.rt.launch_id); + for (uint32_t i = 0; i < ARRAY_SIZE(args->ac.rt.launch_ids); i++) + ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.rt.launch_ids[i]); + ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.rt.dynamic_callable_stack_base); ac_add_arg(&args->ac, AC_ARG_VGPR, 2, AC_ARG_CONST_PTR, &args->ac.rt.shader_addr); ac_add_arg(&args->ac, AC_ARG_VGPR, 2, AC_ARG_CONST_PTR, &args->ac.rt.shader_record);