From c5e40a60f88aad9ff4075f7ce456be9ea2ce0609 Mon Sep 17 00:00:00 2001 From: Konstantin Seurer Date: Sat, 3 Aug 2024 11:43:58 +0200 Subject: [PATCH] radv: Lower non-uniform access after vectorization Scalar access can make nir_lower_non_uniform_access emit a lot of waterfall loops. Totals from 83 (0.10% of 84770) affected shaders: Instrs: 2747926 -> 2745959 (-0.07%); split: -0.07%, +0.00% CodeSize: 15022460 -> 14998240 (-0.16%); split: -0.16%, +0.00% Latency: 18602932 -> 18404976 (-1.06%); split: -1.18%, +0.12% InvThroughput: 4500730 -> 4450364 (-1.12%); split: -1.18%, +0.06% VClause: 93651 -> 91848 (-1.93%); split: -1.93%, +0.00% SClause: 63672 -> 63595 (-0.12%); split: -0.13%, +0.00% Copies: 229377 -> 229896 (+0.23%); split: -0.04%, +0.27% Branches: 107630 -> 107627 (-0.00%); split: -0.01%, +0.00% PreSGPRs: 5247 -> 5253 (+0.11%) PreVGPRs: 5911 -> 5903 (-0.14%); split: -0.29%, +0.15% VALU: 1761158 -> 1761540 (+0.02%); split: -0.01%, +0.03% SALU: 419743 -> 419783 (+0.01%); split: -0.01%, +0.02% VMEM: 152142 -> 150208 (-1.27%) SMEM: 80251 -> 80244 (-0.01%) Reviewed-by: Rhys Perry Part-of: --- src/amd/vulkan/radv_pipeline.c | 47 +++++++++++++++++----------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c index a9df9b6b8ae..6ac6c2e5c9b 100644 --- a/src/amd/vulkan/radv_pipeline.c +++ b/src/amd/vulkan/radv_pipeline.c @@ -375,29 +375,6 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_graphics_stat }; NIR_PASS(_, stage->nir, radv_nir_opt_tid_function, &tid_options); - enum nir_lower_non_uniform_access_type lower_non_uniform_access_types = - nir_lower_non_uniform_ubo_access | nir_lower_non_uniform_ssbo_access | nir_lower_non_uniform_texture_access | - nir_lower_non_uniform_image_access; - - /* In practice, most shaders do not have non-uniform-qualified - * accesses (see - * https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17558#note_1475069) - * thus a cheaper and likely to fail check is run first. - */ - if (nir_has_non_uniform_access(stage->nir, lower_non_uniform_access_types)) { - if (!stage->key.optimisations_disabled) { - NIR_PASS(_, stage->nir, nir_opt_non_uniform_access); - } - - if (!radv_use_llvm_for_stage(pdev, stage->stage)) { - nir_lower_non_uniform_access_options options = { - .types = lower_non_uniform_access_types, - .callback = &non_uniform_access_callback, - .callback_data = NULL, - }; - NIR_PASS(_, stage->nir, nir_lower_non_uniform_access, &options); - } - } NIR_PASS(_, stage->nir, nir_lower_memory_model); nir_load_store_vectorize_options vectorize_opts = { @@ -435,6 +412,30 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_graphics_stat } } + enum nir_lower_non_uniform_access_type lower_non_uniform_access_types = + nir_lower_non_uniform_ubo_access | nir_lower_non_uniform_ssbo_access | nir_lower_non_uniform_texture_access | + nir_lower_non_uniform_image_access; + + /* In practice, most shaders do not have non-uniform-qualified + * accesses (see + * https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17558#note_1475069) + * thus a cheaper and likely to fail check is run first. + */ + if (nir_has_non_uniform_access(stage->nir, lower_non_uniform_access_types)) { + if (!stage->key.optimisations_disabled) { + NIR_PASS(_, stage->nir, nir_opt_non_uniform_access); + } + + if (!radv_use_llvm_for_stage(pdev, stage->stage)) { + nir_lower_non_uniform_access_options options = { + .types = lower_non_uniform_access_types, + .callback = &non_uniform_access_callback, + .callback_data = NULL, + }; + NIR_PASS(_, stage->nir, nir_lower_non_uniform_access, &options); + } + } + NIR_PASS( _, stage->nir, ac_nir_lower_subdword_loads, (ac_nir_lower_subdword_options){.modes_1_comp = nir_var_mem_ubo | nir_var_mem_push_const,