From 843d19a995a3d50ab3acfb576ed9face32b3c189 Mon Sep 17 00:00:00 2001 From: Konstantin Seurer Date: Tue, 16 Jan 2024 11:07:58 +0100 Subject: [PATCH] radv/rt: Avoid passing unused data to the next stage We can gather the args used by all other stages and use that to optimize the traversal shader. Totals from 4 (1.06% of 379) affected shaders: Instrs: 2912 -> 2864 (-1.65%) CodeSize: 15424 -> 15232 (-1.24%) Latency: 69342 -> 69074 (-0.39%) InvThroughput: 11558 -> 11512 (-0.40%) Copies: 712 -> 676 (-5.06%) Branches: 152 -> 148 (-2.63%) PreVGPRs: 324 -> 316 (-2.47%) Reviewed-by: Friedrich Vock Part-of: --- src/amd/vulkan/nir/radv_nir_rt_shader.c | 90 ++++++++++++++++++++----- src/amd/vulkan/radv_pipeline_rt.c | 27 +++++++- src/amd/vulkan/radv_private.h | 2 + src/amd/vulkan/radv_shader.h | 7 +- 4 files changed, 106 insertions(+), 20 deletions(-) diff --git a/src/amd/vulkan/nir/radv_nir_rt_shader.c b/src/amd/vulkan/nir/radv_nir_rt_shader.c index 7956c452bdb..2e4601955f2 100644 --- a/src/amd/vulkan/nir/radv_nir_rt_shader.c +++ b/src/amd/vulkan/nir/radv_nir_rt_shader.c @@ -1884,11 +1884,20 @@ select_next_shader(nir_builder *b, nir_def *shader_addr, unsigned wave_size) return nir_iand_imm(b, next, ~radv_rt_priority_mask); } +static void +radv_store_arg(nir_builder *b, const struct radv_shader_args *args, const struct radv_ray_tracing_stage_info *info, + struct ac_arg arg, nir_def *value) +{ + /* Do not pass unused data to the next stage. */ + if (!info || !BITSET_TEST(info->unused_args, arg.arg_index)) + ac_nir_store_arg(b, &args->ac, arg, value); +} + void radv_nir_lower_rt_abi(nir_shader *shader, const VkRayTracingPipelineCreateInfoKHR *pCreateInfo, const struct radv_shader_args *args, const struct radv_shader_info *info, uint32_t *stack_size, bool resume_shader, struct radv_device *device, struct radv_ray_tracing_pipeline *pipeline, - bool monolithic) + bool monolithic, const struct radv_ray_tracing_stage_info *traversal_info) { nir_function_impl *impl = nir_shader_get_entrypoint(shader); @@ -1996,22 +2005,24 @@ radv_nir_lower_rt_abi(nir_shader *shader, const VkRayTracingPipelineCreateInfoKH /* store back all variables to registers */ ac_nir_store_arg(&b, &args->ac, args->ac.rt.dynamic_callable_stack_base, nir_load_var(&b, vars.stack_ptr)); ac_nir_store_arg(&b, &args->ac, args->ac.rt.shader_addr, shader_addr); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.shader_record, nir_load_var(&b, vars.shader_record_ptr)); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.payload_offset, nir_load_var(&b, vars.arg)); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.accel_struct, nir_load_var(&b, vars.accel_struct)); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.cull_mask_and_flags, nir_load_var(&b, vars.cull_mask_and_flags)); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.sbt_offset, nir_load_var(&b, vars.sbt_offset)); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.sbt_stride, nir_load_var(&b, vars.sbt_stride)); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.miss_index, nir_load_var(&b, vars.miss_index)); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.ray_origin, nir_load_var(&b, vars.origin)); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.ray_tmin, nir_load_var(&b, vars.tmin)); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.ray_direction, nir_load_var(&b, vars.direction)); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.ray_tmax, nir_load_var(&b, vars.tmax)); + radv_store_arg(&b, args, traversal_info, args->ac.rt.shader_record, nir_load_var(&b, vars.shader_record_ptr)); + radv_store_arg(&b, args, traversal_info, args->ac.rt.payload_offset, nir_load_var(&b, vars.arg)); + radv_store_arg(&b, args, traversal_info, args->ac.rt.accel_struct, nir_load_var(&b, vars.accel_struct)); + radv_store_arg(&b, args, traversal_info, args->ac.rt.cull_mask_and_flags, + nir_load_var(&b, vars.cull_mask_and_flags)); + radv_store_arg(&b, args, traversal_info, args->ac.rt.sbt_offset, nir_load_var(&b, vars.sbt_offset)); + radv_store_arg(&b, args, traversal_info, args->ac.rt.sbt_stride, nir_load_var(&b, vars.sbt_stride)); + radv_store_arg(&b, args, traversal_info, args->ac.rt.miss_index, nir_load_var(&b, vars.miss_index)); + radv_store_arg(&b, args, traversal_info, args->ac.rt.ray_origin, nir_load_var(&b, vars.origin)); + radv_store_arg(&b, args, traversal_info, args->ac.rt.ray_tmin, nir_load_var(&b, vars.tmin)); + radv_store_arg(&b, args, traversal_info, args->ac.rt.ray_direction, nir_load_var(&b, vars.direction)); + radv_store_arg(&b, args, traversal_info, args->ac.rt.ray_tmax, nir_load_var(&b, vars.tmax)); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.primitive_id, nir_load_var(&b, vars.primitive_id)); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.instance_addr, nir_load_var(&b, vars.instance_addr)); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.geometry_id_and_flags, nir_load_var(&b, vars.geometry_id_and_flags)); - ac_nir_store_arg(&b, &args->ac, args->ac.rt.hit_kind, nir_load_var(&b, vars.hit_kind)); + radv_store_arg(&b, args, traversal_info, args->ac.rt.primitive_id, nir_load_var(&b, vars.primitive_id)); + radv_store_arg(&b, args, traversal_info, args->ac.rt.instance_addr, nir_load_var(&b, vars.instance_addr)); + radv_store_arg(&b, args, traversal_info, args->ac.rt.geometry_id_and_flags, + nir_load_var(&b, vars.geometry_id_and_flags)); + radv_store_arg(&b, args, traversal_info, args->ac.rt.hit_kind, nir_load_var(&b, vars.hit_kind)); } nir_metadata_preserve(impl, nir_metadata_none); @@ -2022,3 +2033,50 @@ radv_nir_lower_rt_abi(nir_shader *shader, const VkRayTracingPipelineCreateInfoKH if (shader->info.stage == MESA_SHADER_CLOSEST_HIT || shader->info.stage == MESA_SHADER_INTERSECTION) NIR_PASS_V(shader, lower_hit_attribs, NULL, info->wave_size); } + +static bool +radv_arg_def_is_unused(nir_def *def) +{ + nir_foreach_use (use, def) { + nir_instr *use_instr = nir_src_parent_instr(use); + if (use_instr->type == nir_instr_type_intrinsic) { + nir_intrinsic_instr *use_intr = nir_instr_as_intrinsic(use_instr); + if (use_intr->intrinsic == nir_intrinsic_store_scalar_arg_amd || + use_intr->intrinsic == nir_intrinsic_store_vector_arg_amd) + continue; + } else if (use_instr->type == nir_instr_type_phi) { + nir_cf_node *prev_node = nir_cf_node_prev(&use_instr->block->cf_node); + if (!prev_node) + return false; + + nir_phi_instr *phi = nir_instr_as_phi(use_instr); + if (radv_arg_def_is_unused(&phi->def)) + continue; + } + + return false; + } + + return true; +} + +static bool +radv_gather_unused_args_instr(nir_builder *b, nir_intrinsic_instr *instr, void *data) +{ + if (instr->intrinsic != nir_intrinsic_load_scalar_arg_amd && instr->intrinsic != nir_intrinsic_load_vector_arg_amd) + return false; + + if (!radv_arg_def_is_unused(&instr->def)) { + /* This arg is used for more than passing data to the next stage. */ + struct radv_ray_tracing_stage_info *info = data; + BITSET_CLEAR(info->unused_args, nir_intrinsic_base(instr)); + } + + return false; +} + +void +radv_gather_unused_args(struct radv_ray_tracing_stage_info *info, nir_shader *nir) +{ + nir_shader_intrinsics_pass(nir, radv_gather_unused_args_instr, nir_metadata_all, info); +} diff --git a/src/amd/vulkan/radv_pipeline_rt.c b/src/amd/vulkan/radv_pipeline_rt.c index ca8c4c988e1..4ee4afb75c3 100644 --- a/src/amd/vulkan/radv_pipeline_rt.c +++ b/src/amd/vulkan/radv_pipeline_rt.c @@ -366,6 +366,8 @@ static VkResult radv_rt_nir_to_asm(struct radv_device *device, struct vk_pipeline_cache *cache, const VkRayTracingPipelineCreateInfoKHR *pCreateInfo, struct radv_ray_tracing_pipeline *pipeline, bool monolithic, struct radv_shader_stage *stage, uint32_t *stack_size, + struct radv_ray_tracing_stage_info *stage_info, + const struct radv_ray_tracing_stage_info *traversal_stage_info, struct radv_serialized_shader_arena_block *replay_block, struct radv_shader **out_shader) { struct radv_shader_binary *binary; @@ -417,12 +419,15 @@ radv_rt_nir_to_asm(struct radv_device *device, struct vk_pipeline_cache *cache, for (uint32_t i = 0; i < num_resume_shaders; i++) shaders[i + 1] = resume_shaders[i]; + if (stage_info) + memset(stage_info->unused_args, 0xFF, sizeof(stage_info->unused_args)); + /* Postprocess shader parts. */ for (uint32_t i = 0; i < num_shaders; i++) { struct radv_shader_stage temp_stage = *stage; temp_stage.nir = shaders[i]; radv_nir_lower_rt_abi(temp_stage.nir, pCreateInfo, &temp_stage.args, &stage->info, stack_size, i > 0, device, - pipeline, monolithic); + pipeline, monolithic, traversal_stage_info); /* Info might be out-of-date after inlining in radv_nir_lower_rt_abi(). */ nir_shader_gather_info(temp_stage.nir, nir_shader_get_entrypoint(temp_stage.nir)); @@ -430,6 +435,9 @@ radv_rt_nir_to_asm(struct radv_device *device, struct vk_pipeline_cache *cache, radv_optimize_nir(temp_stage.nir, stage->key.optimisations_disabled); radv_postprocess_nir(device, NULL, &temp_stage); + if (stage_info) + radv_gather_unused_args(stage_info, shaders[i]); + if (radv_can_dump_shader(device, temp_stage.nir, false)) nir_print_shader(temp_stage.nir, stderr); } @@ -610,7 +618,7 @@ radv_rt_compile_shaders(struct radv_device *device, struct vk_pipeline_cache *ca bool monolithic_raygen = monolithic && stage->stage == MESA_SHADER_RAYGEN; result = radv_rt_nir_to_asm(device, cache, pCreateInfo, pipeline, monolithic_raygen, stage, &stack_size, - replay_block, &rt_stages[idx].shader); + &rt_stages[idx].info, NULL, replay_block, &rt_stages[idx].shader); if (result != VK_SUCCESS) goto cleanup; @@ -632,6 +640,19 @@ radv_rt_compile_shaders(struct radv_device *device, struct vk_pipeline_cache *ca if (!traversal_needed) return VK_SUCCESS; + struct radv_ray_tracing_stage_info traversal_info = {0}; + + memset(traversal_info.unused_args, 0xFF, sizeof(traversal_info.unused_args)); + + for (uint32_t i = 0; i < pipeline->stage_count; i++) { + if (!pipeline->stages[i].shader) + continue; + + struct radv_ray_tracing_stage_info *info = &pipeline->stages[i].info; + + BITSET_AND(traversal_info.unused_args, traversal_info.unused_args, info->unused_args); + } + /* create traversal shader */ struct vk_shader_module traversal_module = { .base.type = VK_OBJECT_TYPE_SHADER_MODULE, @@ -651,7 +672,7 @@ radv_rt_compile_shaders(struct radv_device *device, struct vk_pipeline_cache *ca vk_pipeline_hash_shader_stage(&pStage, NULL, traversal_stage.shader_sha1); radv_shader_layout_init(pipeline_layout, MESA_SHADER_INTERSECTION, &traversal_stage.layout); result = radv_rt_nir_to_asm(device, cache, pCreateInfo, pipeline, false, &traversal_stage, NULL, NULL, - &pipeline->base.base.shaders[MESA_SHADER_INTERSECTION]); + &traversal_info, NULL, &pipeline->base.base.shaders[MESA_SHADER_INTERSECTION]); ralloc_free(traversal_module.nir); cleanup: diff --git a/src/amd/vulkan/radv_private.h b/src/amd/vulkan/radv_private.h index fa392be65a0..c689bac7119 100644 --- a/src/amd/vulkan/radv_private.h +++ b/src/amd/vulkan/radv_private.h @@ -2366,6 +2366,8 @@ struct radv_ray_tracing_group { struct radv_ray_tracing_stage_info { bool can_inline; + + BITSET_DECLARE(unused_args, AC_MAX_ARGS); }; struct radv_ray_tracing_stage { diff --git a/src/amd/vulkan/radv_shader.h b/src/amd/vulkan/radv_shader.h index de15904e70a..516e0cd39af 100644 --- a/src/amd/vulkan/radv_shader.h +++ b/src/amd/vulkan/radv_shader.h @@ -778,10 +778,15 @@ bool radv_shader_should_clear_lds(const struct radv_device *device, const nir_sh void radv_nir_lower_rt_io(nir_shader *shader, bool monolithic, uint32_t payload_offset); +struct radv_ray_tracing_stage_info; + void radv_nir_lower_rt_abi(nir_shader *shader, const VkRayTracingPipelineCreateInfoKHR *pCreateInfo, const struct radv_shader_args *args, const struct radv_shader_info *info, uint32_t *stack_size, bool resume_shader, struct radv_device *device, - struct radv_ray_tracing_pipeline *pipeline, bool monolithic); + struct radv_ray_tracing_pipeline *pipeline, bool monolithic, + const struct radv_ray_tracing_stage_info *traversal_info); + +void radv_gather_unused_args(struct radv_ray_tracing_stage_info *info, nir_shader *nir); struct radv_shader_stage;