diff --git a/src/intel/compiler/brw_nir.c b/src/intel/compiler/brw_nir.c index eb048cfd6fc..fd41543b688 100644 --- a/src/intel/compiler/brw_nir.c +++ b/src/intel/compiler/brw_nir.c @@ -1745,6 +1745,7 @@ brw_postprocess_nir(nir_shader *nir, const struct brw_compiler *compiler, do { progress = false; + OPT(brw_nir_opt_fsat); OPT(nir_opt_algebraic_late); OPT(brw_nir_lower_fsign); diff --git a/src/intel/compiler/brw_nir.h b/src/intel/compiler/brw_nir.h index 22cc9f830a9..4481c079945 100644 --- a/src/intel/compiler/brw_nir.h +++ b/src/intel/compiler/brw_nir.h @@ -193,6 +193,8 @@ bool brw_nir_limit_trig_input_range_workaround(nir_shader *nir); bool brw_nir_lower_fsign(nir_shader *nir); +bool brw_nir_opt_fsat(nir_shader *); + void brw_nir_apply_key(nir_shader *nir, const struct brw_compiler *compiler, const struct brw_base_prog_key *key, diff --git a/src/intel/compiler/brw_nir_opt_fsat.c b/src/intel/compiler/brw_nir_opt_fsat.c new file mode 100644 index 00000000000..add4d3dc339 --- /dev/null +++ b/src/intel/compiler/brw_nir_opt_fsat.c @@ -0,0 +1,259 @@ +/* + * Copyright 2024 Intel Corporation + * SPDX-License-Identifier: MIT + */ + +/** + * \file + * Move fsat instructions closer to the source when it is likely to be + * profitable. + * + * Intel GPUs have a saturate destination modifier, and + * brw_fs_opt_saturate_propagation tries to replace explicit saturate + * operations with this destination modifier. That pass is limited in several + * ways. If the source of the explicit saturate is in a different block or if + * the source of the explicit saturate is live after the explicit saturate, + * brw_fs_opt_saturate_propagation will be unable to make progress. + * + * This optimization exists to help brw_fs_opt_saturate_propagation make more + * progress. It tries to move NIR fsat instructions to the same block that + * contains the definition of its source. It does this only in cases where it + * will not create additional live values. It also attempts to do this only in + * cases where the explicit saturate will ultimiately be converted to a + * destination modifier. + * + * The optimization scans all instructions. For each fsat instruction found, + * the optimization operates 4 main steps: + * + * 1. Find the source of the fsat instruction. If the source is an ALU + * instruction, add it a worklist called Sources. This occurs in the + * function \c collect_reaching_defs. + * + * 2. Process the Sources worklist. Iterate the uses of each instruction on + * the worklist. If a use is a fsat instruction or a phi node, add the + * instruction to a set of instructions to be "fixed up" in step 3, below. + * If a use is a phi node, add its uses to the worklist. If a use is + * neither a fsat instruction nor a phi node, return failure. This + * indicates that there is some path from one of the definitions to a use + * that is not fsat. This occurs in the function \c verify_users. + * + * 3. For each instruction in the "fix up" set created in step 2 that is not + * an fsat, insert a new fsat instruction immediately following it. + * Replace all uses of instruction with the new fsat. + * + * 4. Convert the old fsat instruction to a simple move instruction. This can + * be eliminated by other optimizations. + * + * If there are many fsat users of a particular instruction, the algorithm + * will only perform step 3 for the first encountered fsat. Each fsat + * encountered later will detect that it is an fsat of an fsat (with the + * latter in a different basic block). Step 4 ensures that each fsat + * encountered later will still be eliminated. + * + * \note This optimization could find even more fsat instructions to move by + * walking "up" the phi web in step 1. If the source of the fsat is a phi + * node, repeatedly iterate through the phi web to find all of the reaching + * definitions. + * + * This has already been implemented. Unfortunately, moving some fsat + * instructions in some large ray tracing shaders in fossil-db causes the + * scheduler and register allocator to make bad choices. This results in + * additional spills and fills. + */ +#include "brw_nir.h" +#include "nir_worklist.h" + +static nir_instr_worklist * +nir_instr_worklist_create_or_clear(nir_instr_worklist * wl) +{ + if (wl == NULL) { + return nir_instr_worklist_create(); + } else { + /* Clear any old cruft in the worklist. */ + nir_foreach_instr_in_worklist(_, wl) + ; + + return wl; + } +} + +static struct set * +_mesa_pointer_set_create_or_clear(void *mem_ctx, struct set *s, + void (*delete_function)(struct set_entry *entry)) +{ + if (s == NULL) { + return _mesa_pointer_set_create(mem_ctx); + } else { + _mesa_set_clear(s, delete_function); + return s; + } +} + +static void +collect_reaching_defs(nir_alu_instr *fsat, nir_instr_worklist *sources) +{ + nir_def *def = fsat->src[0].src.ssa; + + /* If the source of the fsat is in the same block, + * brw_fs_opt_saturate_propagation will already have enough information to + * do its job. Adding another fsat will not help. + */ + if (def->parent_instr->type == nir_instr_type_alu && + def->parent_instr->block != fsat->instr.block) { + nir_instr_worklist_push_tail(sources, def->parent_instr); + } +} + +static bool +verify_users(nir_instr_worklist *sources, struct set *verified_phis, + struct set *fixup) +{ + bool progress = false; + + /* For each source in the set, check that each possible user is an fsat. If + * the source itself is an fsat, the users don't matter. + */ + nir_foreach_instr_in_worklist(src, sources) { + if (src->type == nir_instr_type_phi) { + /* The phi web graph may have cycles. Don't revisit phi nodes to + * prevent infinite loops. + */ + if (_mesa_set_search(verified_phis, src) != NULL) + continue; + } else if (src->type == nir_instr_type_alu) { + /* If a reachable definition is already an fsat, there is no more + * work to be done for that instruction. + * + * FINISHME: This could be made slightly better. Range analysis could + * be used to determine that src is a number (not NaN) and that + * number is already [0, 1]. This would detect cases like 'b2f(a)' or + * 'bcsel(a, fsat(b), 0.0)'. + */ + if (nir_instr_as_alu(src)->op == nir_op_fsat) { + progress = true; + continue; + } + } + + nir_def *src_def = nir_instr_def(src); + + /* It should not be possible for an instruction to get added to the + * worklist that does not have a def. + */ + assert(src_def != NULL); + + if (nir_def_used_by_if(src_def)) + return false; + + nir_foreach_use(use, src_def) { + nir_instr *user_instr = nir_src_parent_instr(use); + + if (user_instr->type == nir_instr_type_phi) { + nir_instr_worklist_push_tail(sources, user_instr); + } else if (user_instr->type != nir_instr_type_alu || + nir_instr_as_alu(user_instr)->op != nir_op_fsat) { + return false; + } + } + + if (src->type == nir_instr_type_phi) { + /* Now that the phi is verified, add it to the cache. */ + _mesa_set_add(verified_phis, src); + } else { + /* Add this source to the set of instructions that need to be + * modified. + */ + _mesa_set_search_or_add(fixup, src, NULL); + progress = true; + } + } + + return progress; +} + +static void +fixup_defs(struct set *fixup) +{ + /* For each instruction in the fixup set, add an fsat user of it, and + * replace all of its old uses with the new fsat. + */ + set_foreach_remove(fixup, entry) { + nir_instr *src = (nir_instr *) entry->key; + nir_def *src_def = nir_instr_def(src); + + /* It should not be possible for an instruction to get added to the + * fixup set that does not have a def. + */ + assert(src_def != NULL); + + nir_builder b = nir_builder_at(nir_after_instr(src)); + + nir_def *new_fsat = nir_fsat(&b, src_def); + + nir_def_rewrite_uses_after(src_def, new_fsat, new_fsat->parent_instr); + } +} + +bool +brw_nir_opt_fsat(nir_shader *shader) +{ + bool progress = false; + void *mem_ctx = ralloc_context(NULL); + nir_instr_worklist *sources = NULL; + struct set *fixup = NULL; + struct set *verified_phis = NULL; + + nir_foreach_function_impl(impl, shader) { + bool progress_impl = false; + + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type != nir_instr_type_alu) + continue; + + nir_alu_instr *alu = nir_instr_as_alu(instr); + if (alu->op != nir_op_fsat) + continue; + + sources = nir_instr_worklist_create_or_clear(sources); + fixup = _mesa_pointer_set_create_or_clear(mem_ctx, fixup, NULL); + + collect_reaching_defs(alu, sources); + + /* verified_phis is a cache of phi nodes where all users of the + * phi node are (eventually) fsat. Once a phi node is verified, it + * will always be valid. It is not necessary to clear this set + * between passes. + */ + if (verified_phis == NULL) + verified_phis = _mesa_pointer_set_create(mem_ctx); + + if (verify_users(sources, verified_phis, fixup)) { + fixup_defs(fixup); + + /* All defs that can reach the old fsat instruction must + * already be saturated. For simplicity, convert the old fsat + * to a simple move. Other optimization passes can eliminate + * the move. + */ + alu->op = nir_op_mov; + progress_impl = true; + } + } + } + + if (progress_impl) { + nir_metadata_preserve(impl, nir_metadata_control_flow); + progress = true; + } else { + nir_metadata_preserve(impl, nir_metadata_all); + } + } + + if (sources != NULL) + nir_instr_worklist_destroy(sources); + + ralloc_free(mem_ctx); + + return progress; +} diff --git a/src/intel/compiler/meson.build b/src/intel/compiler/meson.build index ffb801ba5d5..7a4e30c8827 100644 --- a/src/intel/compiler/meson.build +++ b/src/intel/compiler/meson.build @@ -94,6 +94,7 @@ libintel_compiler_brw_files = files( 'brw_nir_lower_rt_intrinsics.c', 'brw_nir_lower_shader_calls.c', 'brw_nir_lower_storage_image.c', + 'brw_nir_opt_fsat.c', 'brw_nir_rt.h', 'brw_nir_rt.c', 'brw_nir_rt_builder.h',