diff --git a/src/amd/common/nir/ac_nir.c b/src/amd/common/nir/ac_nir.c index 2b2a461f63e..8dc022c884a 100644 --- a/src/amd/common/nir/ac_nir.c +++ b/src/amd/common/nir/ac_nir.c @@ -90,6 +90,7 @@ void ac_nir_set_options(struct radeon_info *info, bool use_llvm, options->has_shfr32 = true; options->has_mul24_relaxed = true; options->has_f2e4m3fn_satfn = !use_llvm && info->gfx_level >= GFX12; + options->has_atomic_isub = true; options->lower_int64_options = nir_lower_imul64 | nir_lower_imul_high64 | nir_lower_imul_2x32_64 | nir_lower_divmod64 | nir_lower_minmax64 | nir_lower_iabs64 | nir_lower_iadd_sat64 | nir_lower_conv64 | nir_lower_bitfield_extract64; diff --git a/src/compiler/nir/nir_opt_intrinsics.c b/src/compiler/nir/nir_opt_intrinsics.c index bfdb5ce50f4..142de3bee99 100644 --- a/src/compiler/nir/nir_opt_intrinsics.c +++ b/src/compiler/nir/nir_opt_intrinsics.c @@ -324,6 +324,26 @@ try_opt_exclusive_scan_to_inclusive(nir_builder *b, nir_intrinsic_instr *intrin) return true; } +static bool +try_opt_atomic_isub(nir_builder *b, nir_intrinsic_instr *intrin, + const struct nir_shader_compiler_options *options, + unsigned data_idx) +{ + if (nir_intrinsic_atomic_op(intrin) != nir_atomic_op_iadd || !options->has_atomic_isub) + return false; + + nir_scalar data = nir_scalar_resolved(intrin->src[data_idx].ssa, 0); + + if (!nir_scalar_is_alu(data) || nir_scalar_alu_op(data) != nir_op_ineg) + return false; + + data = nir_scalar_chase_alu_src(data, 0); + + nir_src_rewrite(&intrin->src[data_idx], nir_mov_scalar(b, data)); + nir_intrinsic_set_atomic_op(intrin, nir_atomic_op_isub); + return true; +} + static bool opt_intrinsics_intrin(nir_builder *b, nir_intrinsic_instr *intrin, const struct nir_shader_compiler_options *options) @@ -379,6 +399,17 @@ opt_intrinsics_intrin(nir_builder *b, nir_intrinsic_instr *intrin, } case nir_intrinsic_exclusive_scan: return try_opt_exclusive_scan_to_inclusive(b, intrin); + case nir_intrinsic_shared_atomic: + case nir_intrinsic_global_atomic: + case nir_intrinsic_global_atomic_amd: + case nir_intrinsic_deref_atomic: + return try_opt_atomic_isub(b, intrin, options, 1); + case nir_intrinsic_ssbo_atomic: + return try_opt_atomic_isub(b, intrin, options, 2); + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_atomic: + case nir_intrinsic_bindless_image_atomic: + return try_opt_atomic_isub(b, intrin, options, 3); default: return false; } diff --git a/src/compiler/nir/nir_shader_compiler_options.h b/src/compiler/nir/nir_shader_compiler_options.h index 5a884f36b7b..6b07aad61db 100644 --- a/src/compiler/nir/nir_shader_compiler_options.h +++ b/src/compiler/nir/nir_shader_compiler_options.h @@ -649,6 +649,9 @@ typedef struct nir_shader_compiler_options { /** Backend supports f2i32_rtne opcode. */ bool has_f2i32_rtne; + /** Backend supports atomic isub. */ + bool has_atomic_isub; + /** * Is this the Intel vec4 backend? *