From 005cc4110cd1284c89c26b07dc183232635ae94a Mon Sep 17 00:00:00 2001 From: Georg Lehmann Date: Fri, 5 Dec 2025 12:01:49 +0100 Subject: [PATCH] nir/peephole_select: allow ballot We can allow collapsing control flow around ballot if we update the ballot condition like we do for discards. ballot_relaxed needs no condition update, as the result bits are undefined for inactive invocations. Foz-DB Navi21: Totals from 27 (0.03% of 97591) affected shaders: Instrs: 2554506 -> 2554469 (-0.00%); split: -0.00%, +0.00% CodeSize: 13765636 -> 13765684 (+0.00%); split: -0.00%, +0.00% Latency: 14186667 -> 14186861 (+0.00%); split: -0.00%, +0.00% InvThroughput: 3542516 -> 3542595 (+0.00%); split: -0.00%, +0.00% SClause: 52038 -> 52030 (-0.02%) Copies: 209410 -> 208763 (-0.31%) Branches: 83716 -> 83399 (-0.38%) PreSGPRs: 2372 -> 2386 (+0.59%); split: -0.17%, +0.76% VALU: 1701458 -> 1701482 (+0.00%) SALU: 369884 -> 370107 (+0.06%); split: -0.00%, +0.07% SMEM: 67643 -> 67634 (-0.01%) Reviewed-by: Alyssa Rosenzweig Part-of: --- src/compiler/nir/nir_opt_peephole_select.c | 31 +++++++++++++++------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/compiler/nir/nir_opt_peephole_select.c b/src/compiler/nir/nir_opt_peephole_select.c index 896860a6123..c989e86956f 100644 --- a/src/compiler/nir/nir_opt_peephole_select.c +++ b/src/compiler/nir/nir_opt_peephole_select.c @@ -200,6 +200,8 @@ block_check_for_allowed_instrs(nir_block *block, unsigned *count, case nir_intrinsic_ddy_fine: case nir_intrinsic_ddy_coarse: case nir_intrinsic_load_const_ir3: + case nir_intrinsic_ballot: + case nir_intrinsic_ballot_relaxed: if (!alu_ok) return false; break; @@ -316,9 +318,11 @@ get_options_for_if(nir_if *if_stmt, return if_options; } -/* If we're moving discards out of the if we need to add the if's condition to it */ +/* If we're moving discards or other conditional intrinsics + * out of the if we need to add the if's condition to it + */ static void -rewrite_discard_conds(nir_instr *instr, nir_def *if_cond, bool is_else) +rewrite_intrinsic_conds(nir_instr *instr, nir_def *if_cond, bool is_else) { if (instr->type != nir_instr_type_intrinsic) return; @@ -327,18 +331,25 @@ rewrite_discard_conds(nir_instr *instr, nir_def *if_cond, bool is_else) if (intr->intrinsic != nir_intrinsic_terminate_if && intr->intrinsic != nir_intrinsic_terminate && intr->intrinsic != nir_intrinsic_demote_if && - intr->intrinsic != nir_intrinsic_demote) + intr->intrinsic != nir_intrinsic_demote && + intr->intrinsic != nir_intrinsic_ballot) return; nir_builder b = nir_builder_at(nir_before_instr(instr)); - if (is_else) - if_cond = nir_inot(&b, if_cond); - if (intr->intrinsic == nir_intrinsic_terminate_if || - intr->intrinsic == nir_intrinsic_demote_if) { + intr->intrinsic == nir_intrinsic_demote_if || + intr->intrinsic == nir_intrinsic_ballot) { + if_cond = nir_b2bN(&b, if_cond, intr->src[0].ssa->bit_size); + + if (is_else) + if_cond = nir_inot(&b, if_cond); + nir_src_rewrite(&intr->src[0], nir_iand(&b, intr->src[0].ssa, if_cond)); } else { + if (is_else) + if_cond = nir_inot(&b, if_cond); + if (intr->intrinsic == nir_intrinsic_terminate) nir_terminate_if(&b, if_cond); else @@ -447,7 +458,7 @@ nir_opt_collapse_if(nir_if *if_stmt, nir_shader *shader, /* combine condition with potential demote/terminate */ nir_foreach_instr_safe(instr, first) - rewrite_discard_conds(instr, parent_if->condition.ssa, false); + rewrite_intrinsic_conds(instr, parent_if->condition.ssa, false); /* combine the if conditions */ struct nir_builder b = nir_builder_at(nir_before_cf_node(&if_stmt->cf_node)); @@ -526,14 +537,14 @@ nir_opt_peephole_select_block(nir_block *block, nir_shader *shader, exec_node_remove(&instr->node); instr->block = prev_block; exec_list_push_tail(&prev_block->instr_list, &instr->node); - rewrite_discard_conds(instr, if_stmt->condition.ssa, false); + rewrite_intrinsic_conds(instr, if_stmt->condition.ssa, false); } nir_foreach_instr_safe(instr, else_block) { exec_node_remove(&instr->node); instr->block = prev_block; exec_list_push_tail(&prev_block->instr_list, &instr->node); - rewrite_discard_conds(instr, if_stmt->condition.ssa, true); + rewrite_intrinsic_conds(instr, if_stmt->condition.ssa, true); } nir_foreach_phi_safe(phi, block) {