diff --git a/src/amd/compiler/aco_lower_phis.cpp b/src/amd/compiler/aco_lower_phis.cpp index cd314b31c3b..6b8f611ecc7 100644 --- a/src/amd/compiler/aco_lower_phis.cpp +++ b/src/amd/compiler/aco_lower_phis.cpp @@ -36,6 +36,7 @@ enum class pred_defined : uint8_t { const_1 = 1, const_0 = 2, temp = 3, + zero = 4, /* all disabled lanes are zero'd out */ }; MESA_DEFINE_CPP_ENUM_BITFIELD_OPERATORS(pred_defined); @@ -71,7 +72,8 @@ get_ssa(Program* program, unsigned block_idx, ssa_state* state, bool input) size_t pred = block.linear_preds.size(); Operand op; if (block.loop_nest_depth < state->loop_nest_depth) { - op = Operand(program->lane_mask); + /* loop-carried value for loop exit phis */ + op = Operand::zero(program->lane_mask.bytes()); } else if (block.loop_nest_depth > state->loop_nest_depth || pred == 1 || block.kind & block_kind_loop_exit) { op = get_ssa(program, block.linear_preds[0], state, false); @@ -160,6 +162,22 @@ build_merge_code(Program* program, ssa_state* state, Block* block, Operand cur) } assert(prev.isTemp()); + /* simpler sequence in case prev has only zeros in disabled lanes */ + if ((defined & pred_defined::zero) == pred_defined::zero) { + if (cur.isConstant()) { + if (!cur.constantValue()) { + bld.copy(dst, prev); + return; + } + cur = Operand(exec, bld.lm); + } else { + cur = + bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cur, Operand(exec, bld.lm)); + } + bld.sop2(Builder::s_or, dst, bld.def(s1, scc), prev, cur); + return; + } + if (cur.isConstant()) { if (cur.constantValue()) bld.sop2(Builder::s_or, dst, bld.def(s1, scc), prev, Operand(exec, bld.lm)); @@ -208,6 +226,18 @@ init_any_pred_defined(Program* program, ssa_state* state, Block* block, aco_ptr< state->any_pred_defined[block->index] = pred_defined::undef; } + /* add dominating zero: this allows to emit simpler merge sequences + * if we can ensure that all disabled lanes are always zero on incoming values */ + // TODO: find more occasions where pred_defined::zero is beneficial (e.g. with 2+ temp merges) + if (block->kind & block_kind_loop_exit) { + /* zero the loop-carried variable */ + if (program->blocks[start].linear_preds.size() > 1) { + state->any_pred_defined[start] |= pred_defined::zero; + // TODO: emit this zero explicitly + state->any_pred_defined[start - 1] = pred_defined::const_0; + } + } + for (unsigned j = start; j < end; j++) { if (state->any_pred_defined[j] == pred_defined::undef) continue;