diff --git a/src/amd/common/nir/ac_nir_lower_mem_access_bit_sizes.c b/src/amd/common/nir/ac_nir_lower_mem_access_bit_sizes.c index 3f2931f0a7b..bfee117288a 100644 --- a/src/amd/common/nir/ac_nir_lower_mem_access_bit_sizes.c +++ b/src/amd/common/nir/ac_nir_lower_mem_access_bit_sizes.c @@ -46,7 +46,6 @@ set_smem_access_flags(nir_builder *b, nir_intrinsic_instr *intrin, void *cb_data case nir_intrinsic_load_ssbo: case nir_intrinsic_load_global: case nir_intrinsic_load_global_constant: - case nir_intrinsic_load_global_amd: case nir_intrinsic_load_constant: if (cb_data->use_llvm) return false; @@ -82,6 +81,15 @@ set_smem_access_flags(nir_builder *b, nir_intrinsic_instr *intrin, void *cb_data bool ac_nir_flag_smem_for_loads(nir_shader *shader, enum amd_gfx_level gfx_level, bool use_llvm) { + /* Only use the 'ignore_undef' divergence option for ACO where we can guarantee that + * uniform phis with undef src are residing in SGPRs, and hence, indeed uniform. + */ + uint32_t options = + shader->options->divergence_analysis_options | (use_llvm ? 0 : nir_divergence_ignore_undef_if_phi_srcs); + nir_foreach_function_impl(impl, shader) { + nir_divergence_analysis_impl(impl, (nir_divergence_options)options); + } + mem_access_cb_data cb_data = { .gfx_level = gfx_level, .use_llvm = use_llvm, diff --git a/src/amd/compiler/instruction_selection/aco_isel_setup.cpp b/src/amd/compiler/instruction_selection/aco_isel_setup.cpp index 0d090a02d2f..2a44002102d 100644 --- a/src/amd/compiler/instruction_selection/aco_isel_setup.cpp +++ b/src/amd/compiler/instruction_selection/aco_isel_setup.cpp @@ -381,7 +381,6 @@ init_context(isel_context* ctx, nir_shader* shader) nir_divergence_analysis_impl(impl, (nir_divergence_options)options); apply_nuw_to_offsets(ctx, impl); - ac_nir_flag_smem_for_loads(shader, ctx->program->gfx_level, false); if (shader->info.stage == MESA_SHADER_FRAGMENT) { nir_opt_load_skip_helpers_options skip_helper_options = {}; diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c index 312e8e57328..6402e46f0c0 100644 --- a/src/amd/vulkan/radv_pipeline.c +++ b/src/amd/vulkan/radv_pipeline.c @@ -353,7 +353,6 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_graphics_stat }; NIR_PASS(_, stage->nir, radv_nir_opt_tid_function, &tid_options); - nir_divergence_analysis(stage->nir); NIR_PASS(_, stage->nir, ac_nir_flag_smem_for_loads, gfx_level, use_llvm); NIR_PASS(_, stage->nir, nir_lower_memory_model); @@ -573,6 +572,8 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_graphics_stat NIR_PASS(_, stage->nir, nir_opt_constant_folding); NIR_PASS(_, stage->nir, nir_opt_cse); NIR_PASS(_, stage->nir, nir_opt_shrink_vectors, true); + + NIR_PASS(_, stage->nir, ac_nir_flag_smem_for_loads, gfx_level, use_llvm); NIR_PASS(_, stage->nir, ac_nir_lower_mem_access_bit_sizes, gfx_level, use_llvm); nir_load_store_vectorize_options late_vectorize_opts = { diff --git a/src/gallium/drivers/radeonsi/si_shader.c b/src/gallium/drivers/radeonsi/si_shader.c index d3a23d95d66..74b47d70e03 100644 --- a/src/gallium/drivers/radeonsi/si_shader.c +++ b/src/gallium/drivers/radeonsi/si_shader.c @@ -1624,7 +1624,6 @@ static void run_late_optimization_and_lowering_passes(struct si_nir_shader_ctx * NIR_PASS(_, nir, nir_clear_shared_memory, shared_size, chunk_size); } - nir_divergence_analysis(nir); /* required by ac_nir_flag_smem_for_loads */ /* This is required by ac_nir_scalarize_overfetching_loads_callback. */ NIR_PASS(progress, nir, ac_nir_flag_smem_for_loads, sel->screen->info.gfx_level, !sel->info.base.use_aco_amd);