diff --git a/src/compiler/nir/nir_opt_algebraic.py b/src/compiler/nir/nir_opt_algebraic.py index bfb96d5e525..e7287e20db7 100644 --- a/src/compiler/nir/nir_opt_algebraic.py +++ b/src/compiler/nir/nir_opt_algebraic.py @@ -3143,7 +3143,10 @@ late_optimizations = [ # nir_lower_to_source_mods will collapse this, but its existence during the # optimization loop can prevent other optimizations. - (('fneg', ('fneg', a)), a) + (('fneg', ('fneg', a)), a), + + # combine imul and iadd to imad + (('iadd@32', ('imul(is_only_used_by_iadd)', a, b), c), ('imad', a, b, c), 'options->has_imad32'), ] # re-combine inexact mul+add to ffma. Do this before fsub so that a * b - c diff --git a/src/compiler/nir/nir_search_helpers.h b/src/compiler/nir/nir_search_helpers.h index 44ede8b3ee6..cb9165f10c4 100644 --- a/src/compiler/nir/nir_search_helpers.h +++ b/src/compiler/nir/nir_search_helpers.h @@ -583,6 +583,24 @@ is_only_used_by_fadd(const nir_alu_instr *instr) return true; } +static inline bool +is_only_used_by_iadd(const nir_alu_instr *instr) +{ + nir_foreach_use(src, &instr->def) { + const nir_instr *const user_instr = nir_src_parent_instr(src); + if (user_instr->type != nir_instr_type_alu) + return false; + + const nir_alu_instr *const user_alu = nir_instr_as_alu(user_instr); + assert(instr != user_alu); + + if (user_alu->op != nir_op_iadd) + return false; + } + + return true; +} + static inline bool only_lower_8_bits_used(const nir_alu_instr *instr) { diff --git a/src/nouveau/compiler/nak_nir_algebraic.py b/src/nouveau/compiler/nak_nir_algebraic.py index e5ecb53c20e..a10245aefa7 100644 --- a/src/nouveau/compiler/nak_nir_algebraic.py +++ b/src/nouveau/compiler/nak_nir_algebraic.py @@ -39,11 +39,6 @@ algebraic_lowering = [ (('umax', 'a', 'b'), ('bcsel', ('ult', a, b), b, a), volta), ] -late_optimizations = [ - (('iadd@32', ('imul(nak_is_only_used_by_iadd)', a, b), c), - ('imad', a, b, c), 'nak->sm >= 70'), -] - def main(): parser = argparse.ArgumentParser() parser.add_argument('--out', required=True, help='Output file.') @@ -58,7 +53,7 @@ def main(): f.write('#include "nak_private.h"') f.write(nir_algebraic.AlgebraicPass( "nak_nir_lower_algebraic_late", - algebraic_lowering + late_optimizations, + algebraic_lowering, [ ("const struct nak_compiler *", "nak"), ]).render()) diff --git a/src/nouveau/compiler/nak_private.h b/src/nouveau/compiler/nak_private.h index b15346cfbe4..f6d034b1b0b 100644 --- a/src/nouveau/compiler/nak_private.h +++ b/src/nouveau/compiler/nak_private.h @@ -222,21 +222,6 @@ bool nak_nir_lower_non_uniform_ldcx(nir_shader *nir); bool nak_nir_add_barriers(nir_shader *nir, const struct nak_compiler *nak); bool nak_nir_lower_cf(nir_shader *nir); -static inline bool -nak_is_only_used_by_iadd(const nir_alu_instr *instr) -{ - nir_foreach_use(src, &instr->def) { - nir_instr *use = nir_src_parent_instr(src); - if (use->type != nir_instr_type_alu) - return false; - - if (nir_instr_as_alu(use)->op != nir_op_iadd) - return false; - } - - return true; -} - void nak_optimize_nir(nir_shader *nir, const struct nak_compiler *nak); struct nak_memstream {