diff --git a/src/amd/common/ac_nir.c b/src/amd/common/ac_nir.c index 91ec355cbda..23c29c9889d 100644 --- a/src/amd/common/ac_nir.c +++ b/src/amd/common/ac_nir.c @@ -1746,6 +1746,59 @@ use_smem_for_load(nir_builder *b, nir_intrinsic_instr *intrin, void *cb_data_) return true; } +static nir_mem_access_size_align +lower_mem_access_cb(nir_intrinsic_op intrin, uint8_t bytes, uint8_t bit_size, uint32_t align_mul, uint32_t align_offset, + bool offset_is_const, enum gl_access_qualifier access, const void *cb_data_) +{ + const mem_access_cb_data *cb_data = (mem_access_cb_data *)cb_data_; + const bool is_load = nir_intrinsic_infos[intrin].has_dest; + const uint32_t combined_align = nir_combined_align(align_mul, align_offset); + + /* Make 8-bit accesses 16-bit if possible */ + if (is_load && bit_size == 8 && combined_align >= 2 && bytes % 2 == 0) + bit_size = 16; + + nir_mem_access_size_align res; + res.num_components = MIN2(bytes / (bit_size / 8), 4); + res.bit_size = bit_size; + res.align = MIN2(bit_size / 8, 4); /* 64-bit access only requires 4 byte alignment. */ + res.shift = nir_mem_access_shift_method_shift64; + + if (!is_load) + return res; + + /* Lower 8/16-bit loads to 32-bit, unless it's a VMEM scalar load. */ + + const bool is_smem = intrin == nir_intrinsic_load_push_constant || (access & ACCESS_SMEM_AMD); + const bool support_subdword = res.num_components == 1 && !is_smem && + (!cb_data->use_llvm || intrin != nir_intrinsic_load_ubo); + + if (res.bit_size >= 32 || support_subdword) + return res; + + const uint32_t max_pad = 4 - MIN2(combined_align, 4); + + /* Global loads don't have bounds checking, so increasing the size might not be safe. */ + if (intrin == nir_intrinsic_load_global || intrin == nir_intrinsic_load_global_constant) { + if (align_mul < 4) { + /* If we split the load, only lower it to 32-bit if this is a SMEM load. */ + const unsigned chunk_bytes = align(bytes, 4) - max_pad; + if (!is_smem && chunk_bytes < bytes) + return res; + } + + res.num_components = DIV_ROUND_UP(bytes, 4); + } else { + res.num_components = DIV_ROUND_UP(bytes + max_pad, 4); + } + res.num_components = MIN2(res.num_components, 4); + res.bit_size = 32; + res.align = 4; + res.shift = is_smem ? res.shift : nir_mem_access_shift_method_bytealign_amd; + + return res; +} + bool ac_nir_flag_smem_for_loads(nir_shader *shader, enum amd_gfx_level gfx_level, bool use_llvm, bool after_lowering) { @@ -1756,3 +1809,19 @@ ac_nir_flag_smem_for_loads(nir_shader *shader, enum amd_gfx_level gfx_level, boo }; return nir_shader_intrinsics_pass(shader, &use_smem_for_load, nir_metadata_all, &cb_data); } + +bool +ac_nir_lower_mem_access_bit_sizes(nir_shader *shader, enum amd_gfx_level gfx_level, bool use_llvm) +{ + mem_access_cb_data cb_data = { + .gfx_level = gfx_level, + .use_llvm = use_llvm, + }; + nir_lower_mem_access_bit_sizes_options lower_mem_access_options = { + .callback = &lower_mem_access_cb, + .modes = nir_var_mem_ubo | nir_var_mem_push_const | nir_var_mem_ssbo | nir_var_mem_global | nir_var_mem_constant, + .may_lower_unaligned_stores_to_atomics = false, + .cb_data = &cb_data, + }; + return nir_lower_mem_access_bit_sizes(shader, &lower_mem_access_options); +} diff --git a/src/amd/common/ac_nir.h b/src/amd/common/ac_nir.h index 166da2f378a..6aab1f3e315 100644 --- a/src/amd/common/ac_nir.h +++ b/src/amd/common/ac_nir.h @@ -326,6 +326,9 @@ ac_nir_opt_shared_append(nir_shader *shader); bool ac_nir_flag_smem_for_loads(nir_shader *shader, enum amd_gfx_level gfx_level, bool use_llvm, bool after_lowering); +bool +ac_nir_lower_mem_access_bit_sizes(nir_shader *shader, enum amd_gfx_level gfx_level, bool use_llvm); + #ifdef __cplusplus } #endif diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c index 60f41bb8096..f493dc7c403 100644 --- a/src/amd/vulkan/radv_pipeline.c +++ b/src/amd/vulkan/radv_pipeline.c @@ -375,6 +375,9 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_graphics_stat }; NIR_PASS(_, stage->nir, radv_nir_opt_tid_function, &tid_options); + nir_divergence_analysis(stage->nir); + NIR_PASS(_, stage->nir, ac_nir_flag_smem_for_loads, gfx_level, use_llvm, false); + NIR_PASS(_, stage->nir, nir_lower_memory_model); nir_load_store_vectorize_options vectorize_opts = { @@ -436,10 +439,7 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_graphics_stat } } - NIR_PASS( - _, stage->nir, ac_nir_lower_subdword_loads, - (ac_nir_lower_subdword_options){.modes_1_comp = nir_var_mem_ubo | nir_var_mem_push_const, - .modes_N_comps = nir_var_mem_ubo | nir_var_mem_push_const | nir_var_mem_ssbo}); + NIR_PASS(_, stage->nir, ac_nir_lower_mem_access_bit_sizes, gfx_level, use_llvm); progress = false; NIR_PASS(progress, stage->nir, nir_vk_lower_ycbcr_tex, ycbcr_conversion_lookup, &stage->layout);