diff --git a/src/compiler/nir/meson.build b/src/compiler/nir/meson.build index a5c2eeaa5f2..9a6c14ded8f 100644 --- a/src/compiler/nir/meson.build +++ b/src/compiler/nir/meson.build @@ -122,6 +122,8 @@ files_libnir = files( 'nir_inline_uniforms.c', 'nir_instr_set.c', 'nir_instr_set.h', + 'nir_legacy.c', + 'nir_legacy.h', 'nir_linking_helpers.c', 'nir_liveness.c', 'nir_loop_analyze.c', diff --git a/src/compiler/nir/nir_legacy.c b/src/compiler/nir/nir_legacy.c new file mode 100644 index 00000000000..10bf323f423 --- /dev/null +++ b/src/compiler/nir/nir_legacy.c @@ -0,0 +1,355 @@ +/* + * Copyright 2023 Valve Corporation + * Copyright 2020 Collabora, Ltd. + * SPDX-License-Identifier: MIT + */ + +#include "nir.h" +#include "nir_builder.h" +#include "nir_legacy.h" + +static inline bool +chase_source_mod(nir_ssa_def **ssa, nir_op op, uint8_t *swizzle) +{ + if ((*ssa)->parent_instr->type != nir_instr_type_alu) + return false; + + nir_alu_instr *alu = nir_instr_as_alu((*ssa)->parent_instr); + if (alu->op != op) + return false; + + /* This only works for unary ops */ + assert(nir_op_infos[op].num_inputs == 1); + + /* To fuse the source mod in, we need to compose the swizzles and string + * through the source. + */ + for (unsigned i = 0; i < NIR_MAX_VEC_COMPONENTS; ++i) + swizzle[i] = alu->src[0].swizzle[swizzle[i]]; + + assert(alu->src[0].src.is_ssa && "registers lowered to intrinsics"); + *ssa = alu->src[0].src.ssa; + return true; +} + +static inline bool +accepts_source_mod(const nir_src *src) +{ + /* No legacy user supports fp64 modifiers */ + if (nir_src_bit_size(*src) == 64) + return false; + + if (src->is_if) + return false; + + nir_instr *parent = src->parent_instr; + if (parent->type != nir_instr_type_alu) + return false; + + nir_alu_instr *alu = nir_instr_as_alu(parent); + nir_alu_src *alu_src = list_entry(src, nir_alu_src, src); + unsigned src_index = alu_src - alu->src; + + assert(src_index < nir_op_infos[alu->op].num_inputs); + nir_alu_type src_type = nir_op_infos[alu->op].input_types[src_index]; + + return nir_alu_type_get_base_type(src_type) == nir_type_float; +} + +bool +nir_legacy_float_mod_folds(nir_alu_instr *mod) +{ + assert(mod->op == nir_op_fabs || mod->op == nir_op_fneg); + assert(mod->dest.dest.is_ssa); + + /* No legacy user supports fp64 modifiers */ + if (mod->dest.dest.ssa.bit_size == 64) + return false; + + nir_foreach_use_including_if(use, &mod->dest.dest.ssa) { + if (!accepts_source_mod(use)) + return false; + } + + return true; +} + +static nir_legacy_alu_src +chase_alu_src_helper(const nir_src *src) +{ + assert(src->is_ssa && "registers lowered to intrinsics"); + nir_intrinsic_instr *load = nir_load_reg_for_def(src->ssa); + + if (load) { + bool indirect = (load->intrinsic == nir_intrinsic_load_reg_indirect); + + return (nir_legacy_alu_src) { + .src.is_ssa = false, + .src.reg = { + .handle = load->src[0].ssa, + .base_offset = nir_intrinsic_base(load), + .indirect = indirect ? load->src[1].ssa : NULL + }, + .fabs = nir_intrinsic_legacy_fabs(load), + .fneg = nir_intrinsic_legacy_fneg(load), + }; + } else { + return (nir_legacy_alu_src) { + .src.is_ssa = true, + .src.ssa = src->ssa, + }; + } +} + +nir_legacy_alu_src +nir_legacy_chase_alu_src(const nir_alu_src *src, bool fuse_fabs) +{ + assert(!src->abs && "source modifiers must be ALU"); + assert(!src->negate && "source modifiers must be ALU"); + assert(src->src.is_ssa && "registers lowered to intrinsics"); + + if (src->src.ssa->parent_instr->type == nir_instr_type_alu) { + nir_legacy_alu_src out = { + .src.is_ssa = true, + .src.ssa = src->src.ssa, + }; + STATIC_ASSERT(sizeof(src->swizzle) == sizeof(out.swizzle)); + memcpy(out.swizzle, src->swizzle, sizeof(src->swizzle)); + + /* To properly handle foo(fneg(fabs(x))), we first chase fneg and then + * fabs, since we chase from bottom-up. We don't handle fabs(fneg(x)) + * since nir_opt_algebraic should have eliminated that. + */ + if (accepts_source_mod(&src->src)) { + out.fneg = chase_source_mod(&out.src.ssa, nir_op_fneg, out.swizzle); + out.fabs = fuse_fabs && chase_source_mod(&out.src.ssa, nir_op_fabs, + out.swizzle); + } + + return out; + } else { + nir_legacy_alu_src out = chase_alu_src_helper(&src->src); + memcpy(out.swizzle, src->swizzle, sizeof(src->swizzle)); + return out; + } +} + +static nir_legacy_alu_dest +chase_alu_dest_helper(nir_dest *dest) +{ + assert(dest->is_ssa && "registers lowered to intrinsics"); + nir_intrinsic_instr *store = nir_store_reg_for_def(&dest->ssa); + + if (store) { + bool indirect = (store->intrinsic == nir_intrinsic_store_reg_indirect); + + return (nir_legacy_alu_dest) { + .dest.is_ssa = false, + .dest.reg = { + .handle = store->src[1].ssa, + .base_offset = nir_intrinsic_base(store), + .indirect = indirect ? store->src[2].ssa : NULL + }, + .fsat = nir_intrinsic_legacy_fsat(store), + .write_mask = nir_intrinsic_write_mask(store), + }; + } else { + return (nir_legacy_alu_dest) { + .dest.is_ssa = true, + .dest.ssa = &dest->ssa, + .write_mask = nir_component_mask(dest->ssa.num_components), + }; + } +} + +bool +nir_legacy_fsat_folds(nir_alu_instr *fsat) +{ + assert(fsat->op == nir_op_fsat); + nir_ssa_def *def = fsat->src[0].src.ssa; + + /* No legacy user supports fp64 modifiers */ + if (def->bit_size == 64) + return false; + + /* Must be the only use */ + if (!list_is_singular(&def->uses)) + return false; + + assert(&fsat->src[0].src == + list_first_entry(&def->uses, nir_src, use_link)); + + nir_instr *generate = def->parent_instr; + if (generate->type != nir_instr_type_alu) + return false; + + nir_alu_instr *generate_alu = nir_instr_as_alu(generate); + nir_alu_type dest_type = nir_op_infos[generate_alu->op].output_type; + if (dest_type != nir_type_float) + return false; + + /* We can't do expansions without a move in the middle */ + unsigned nr_components = nir_dest_num_components(generate_alu->dest.dest); + if (fsat->dest.dest.ssa.num_components != nr_components) + return false; + + /* We don't handle swizzles here, so check for the identity */ + for (unsigned i = 0; i < nr_components; ++i) { + if (fsat->src[0].swizzle[i] != i) + return false; + } + + return true; +} + +static inline bool +chase_fsat(nir_ssa_def **def) +{ + /* No legacy user supports fp64 modifiers */ + if ((*def)->bit_size == 64) + return false; + + if (!list_is_singular(&(*def)->uses)) + return false; + + nir_src *use = list_first_entry(&(*def)->uses, nir_src, use_link); + if (use->is_if || use->parent_instr->type != nir_instr_type_alu) + return false; + + nir_alu_instr *fsat = nir_instr_as_alu(use->parent_instr); + if (fsat->op != nir_op_fsat || !nir_legacy_fsat_folds(fsat)) + return false; + + /* Otherwise, we're good */ + nir_alu_instr *alu = nir_instr_as_alu(use->parent_instr); + assert(alu->dest.dest.is_ssa); + *def = &alu->dest.dest.ssa; + return true; +} + +nir_legacy_alu_dest +nir_legacy_chase_alu_dest(nir_dest *dest) +{ + assert(dest->is_ssa && "registers lowered to intrinsics"); + nir_ssa_def *def = &dest->ssa; + + /* Try SSA fsat. No users support 64-bit modifiers. */ + if (chase_fsat(&def)) { + return (nir_legacy_alu_dest) { + .dest.is_ssa = true, + .dest.ssa = def, + .fsat = true, + .write_mask = nir_component_mask(def->num_components), + }; + } else { + return chase_alu_dest_helper(dest); + } +} + +nir_legacy_src +nir_legacy_chase_src(const nir_src *src) +{ + nir_legacy_alu_src alu_src = chase_alu_src_helper(src); + assert(!alu_src.fabs && !alu_src.fneg); + return alu_src.src; +} + +nir_legacy_dest +nir_legacy_chase_dest(nir_dest *dest) +{ + nir_legacy_alu_dest alu_dest = chase_alu_dest_helper(dest); + assert(!alu_dest.fsat); + assert(alu_dest.write_mask == + nir_component_mask(nir_dest_num_components(*dest))); + + return alu_dest.dest; +} + +static bool +fuse_mods_with_registers(nir_builder *b, nir_instr *instr, void *fuse_fabs_) +{ + bool *fuse_fabs = fuse_fabs_; + if (instr->type != nir_instr_type_alu) + return false; + + nir_alu_instr *alu = nir_instr_as_alu(instr); + if ((alu->op == nir_op_fneg || (*fuse_fabs && alu->op == nir_op_fabs)) && + nir_legacy_float_mod_folds(alu)) { + /* Try to fold this instruction into the load, if possible. We only do + * this for loads in the same block as the use because uses of loads + * which cross block boundaries aren't trivial anyway. + */ + assert(alu->src[0].src.is_ssa); + nir_intrinsic_instr *load = nir_load_reg_for_def(alu->src[0].src.ssa); + if (load != NULL) { + /* Duplicate the load before changing it in case there are other + * users. We assume someone has run CSE so there should be at most + * four load instructions generated (one for each possible modifier + * combination), but likely only one or two. + */ + b->cursor = nir_before_instr(&load->instr); + load = nir_instr_as_intrinsic(nir_instr_clone(b->shader, &load->instr)); + nir_builder_instr_insert(b, &load->instr); + + if (alu->op == nir_op_fabs) { + nir_intrinsic_set_legacy_fabs(load, true); + nir_intrinsic_set_legacy_fneg(load, false); + } else { + assert(alu->op == nir_op_fneg); + bool old_fneg = nir_intrinsic_legacy_fneg(load); + nir_intrinsic_set_legacy_fneg(load, !old_fneg); + } + + /* Rewrite all the users to use the modified load instruction. We + * already know that they're all float ALU instructions because + * nir_legacy_float_mod_folds() returned true. + */ + nir_foreach_use_including_if_safe(use, &alu->dest.dest.ssa) { + assert(!use->is_if); + assert(use->parent_instr->type == nir_instr_type_alu); + nir_alu_src *alu_use = list_entry(use, nir_alu_src, src); + nir_src_rewrite_ssa(&alu_use->src, &load->dest.ssa); + for (unsigned i = 0; i < NIR_MAX_VEC_COMPONENTS; ++i) + alu_use->swizzle[i] = alu->src[0].swizzle[alu_use->swizzle[i]]; + } + + nir_instr_remove(&alu->instr); + + return true; + } else { + /* We don't want to attempt to add saturate to foldable mod ops */ + return false; + } + } + + nir_legacy_alu_dest dest = nir_legacy_chase_alu_dest(&alu->dest.dest); + if (dest.fsat) { + assert(dest.dest.is_ssa && "not fully chased"); + nir_intrinsic_instr *store = nir_store_reg_for_def(dest.dest.ssa); + + if (store) { + nir_intrinsic_set_legacy_fsat(store, true); + nir_src_rewrite_ssa(&store->src[0], &alu->dest.dest.ssa); + return true; + } + } + + return false; +} + +void +nir_legacy_trivialize(nir_shader *s, bool fuse_fabs) +{ + /* First, fuse modifiers with registers. This ensures that the helpers do not + * chase registers recursively, allowing registers to be trivialized easier. + */ + if (nir_shader_instructions_pass(s, fuse_mods_with_registers, + nir_metadata_block_index | + nir_metadata_dominance, &fuse_fabs)) { + /* If we made progress, we likely left dead loads. Clean them up. */ + NIR_PASS_V(s, nir_opt_dce); + } + + /* Now that modifiers are dealt with, we can trivialize the regular way. */ + NIR_PASS_V(s, nir_trivialize_registers); +} diff --git a/src/compiler/nir/nir_legacy.h b/src/compiler/nir/nir_legacy.h new file mode 100644 index 00000000000..d42cd1b4688 --- /dev/null +++ b/src/compiler/nir/nir_legacy.h @@ -0,0 +1,95 @@ +/* + * Copyright 2023 Valve Corporation + * Copyright 2014 Connor Abbott + * SPDX-License-Identifier: MIT + */ + +#ifndef NIR_LEGACY_H +#define NIR_LEGACY_H + +#include "nir.h" + +typedef struct { + nir_ssa_def *handle; + nir_ssa_def *indirect; /** < NULL for no indirect offset */ + unsigned base_offset; +} nir_reg_src; + +typedef struct { + nir_ssa_def *handle; + nir_ssa_def *indirect; /** < NULL for no indirect offset */ + unsigned base_offset; +} nir_reg_dest; + +typedef struct { + bool is_ssa; + + union { + nir_reg_src reg; + nir_ssa_def *ssa; + }; +} nir_legacy_src; + +typedef struct { + bool is_ssa; + + union { + nir_reg_dest reg; + nir_ssa_def *ssa; + }; +} nir_legacy_dest; + +typedef struct { + /* Base source */ + nir_legacy_src src; + + /* For inputs interpreted as floating point, flips the sign bit. */ + bool fneg; + + /* Clears the sign bit for floating point values, Note that the negate + * modifier acts after the absolute value modifier, therefore if both are set + * then all inputs will become negative. + */ + bool fabs; + + uint8_t swizzle[NIR_MAX_VEC_COMPONENTS]; +} nir_legacy_alu_src; + +typedef struct { + /* Base destination */ + nir_legacy_dest dest; + + nir_component_mask_t write_mask; + + /** + * Saturate output modifier + * + * Only valid for opcodes that output floating-point numbers. Clamps the + * output to between 0.0 and 1.0 inclusive. + */ + bool fsat; +} nir_legacy_alu_dest; + +/* Prepare shader for use with legacy helpers. Must call on final NIR. */ +void nir_legacy_trivialize(nir_shader *s, bool fuse_fabs); + +/* Reconstruct a legacy source/destination (including registers) */ +nir_legacy_src nir_legacy_chase_src(const nir_src *src); +nir_legacy_dest nir_legacy_chase_dest(nir_dest *dest); + +/* Reconstruct a legacy ALU source/destination (including float modifiers) */ +nir_legacy_alu_src nir_legacy_chase_alu_src(const nir_alu_src *src, + bool fuse_fabs); +nir_legacy_alu_dest nir_legacy_chase_alu_dest(nir_dest *dest); + +/* Check if a source modifier folds. If so, it may be skipped during instruction + * selection, avoiding the need for backend dead code elimination. + */ +bool nir_legacy_float_mod_folds(nir_alu_instr *mod); + +/* Check if an fsat destination modifier folds. If so, it must be skipped to + * avoid multiple conflicting writes to the same definition. + */ +bool nir_legacy_fsat_folds(nir_alu_instr *fsat); + +#endif