diff --git a/src/amd/compiler/aco_insert_exec_mask.cpp b/src/amd/compiler/aco_insert_exec_mask.cpp index 3982ed63ee1..aa1e6fe8f92 100644 --- a/src/amd/compiler/aco_insert_exec_mask.cpp +++ b/src/amd/compiler/aco_insert_exec_mask.cpp @@ -672,7 +672,7 @@ add_branch_code(exec_ctx& ctx, Block* block) assert(block->linear_succs.size() == 2); assert(block->instructions.back()->opcode == aco_opcode::p_cbranch_z); Temp cond = block->instructions.back()->operands[0].getTemp(); - const bool sel_ctrl = block->instructions.back()->branch().selection_control_remove; + aco_ptr branch = std::move(block->instructions.back()); block->instructions.pop_back(); uint8_t mask_type = ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact); @@ -690,14 +690,15 @@ add_branch_code(exec_ctx& ctx, Block* block) Builder::Result r = bld.branch(aco_opcode::p_cbranch_z, bld.def(s2), Operand(exec, bld.lm), block->linear_succs[1], block->linear_succs[0]); - r->branch().selection_control_remove = sel_ctrl; + r->branch().rarely_taken = branch->branch().rarely_taken; + r->branch().never_taken = branch->branch().never_taken; return; } if (block->kind & block_kind_invert) { // exec = s_andn2_b64 (original_exec, exec) assert(block->instructions.back()->opcode == aco_opcode::p_branch); - const bool sel_ctrl = block->instructions.back()->branch().selection_control_remove; + aco_ptr branch = std::move(block->instructions.back()); block->instructions.pop_back(); assert(ctx.info[idx].exec.size() >= 2); Operand orig_exec = ctx.info[idx].exec[ctx.info[idx].exec.size() - 2].first; @@ -706,7 +707,8 @@ add_branch_code(exec_ctx& ctx, Block* block) Builder::Result r = bld.branch(aco_opcode::p_cbranch_z, bld.def(s2), Operand(exec, bld.lm), block->linear_succs[1], block->linear_succs[0]); - r->branch().selection_control_remove = sel_ctrl; + r->branch().rarely_taken = branch->branch().rarely_taken; + r->branch().never_taken = branch->branch().never_taken; return; } diff --git a/src/amd/compiler/aco_instruction_selection.cpp b/src/amd/compiler/aco_instruction_selection.cpp index c788dcb1269..537e0646e99 100644 --- a/src/amd/compiler/aco_instruction_selection.cpp +++ b/src/amd/compiler/aco_instruction_selection.cpp @@ -10615,9 +10615,12 @@ begin_divergent_if_then(isel_context* ctx, if_context* ic, Temp cond, branch.reset(create_instruction(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 1)); branch->definitions[0] = Definition(ctx->program->allocateTmp(s2)); branch->operands[0] = Operand(cond); - branch->branch().selection_control_remove = - sel_ctrl == nir_selection_control_flatten || - sel_ctrl == nir_selection_control_divergent_always_taken; + bool never_taken = + sel_ctrl == nir_selection_control_divergent_always_taken && + !(ctx->cf_info.exec.potentially_empty_discard || ctx->cf_info.exec.potentially_empty_break || + ctx->cf_info.exec.potentially_empty_continue); + branch->branch().rarely_taken = sel_ctrl == nir_selection_control_flatten || never_taken; + branch->branch().never_taken = never_taken; ctx->block->instructions.push_back(std::move(branch)); ic->BB_if_idx = ctx->block->index; @@ -10680,9 +10683,12 @@ begin_divergent_if_else(isel_context* ctx, if_context* ic, /* branch to linear else block (skip else) */ branch.reset(create_instruction(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 1)); branch->definitions[0] = Definition(ctx->program->allocateTmp(s2)); - branch->branch().selection_control_remove = - sel_ctrl == nir_selection_control_flatten || - sel_ctrl == nir_selection_control_divergent_always_taken; + bool never_taken = + sel_ctrl == nir_selection_control_divergent_always_taken && + !(ctx->cf_info.exec.potentially_empty_discard || ctx->cf_info.exec.potentially_empty_break || + ctx->cf_info.exec.potentially_empty_continue); + branch->branch().rarely_taken = sel_ctrl == nir_selection_control_flatten || never_taken; + branch->branch().never_taken = never_taken; ctx->block->instructions.push_back(std::move(branch)); ic->exec_old.combine(ctx->cf_info.exec); diff --git a/src/amd/compiler/aco_ir.h b/src/amd/compiler/aco_ir.h index 24da1c01d3c..df7232b0d46 100644 --- a/src/amd/compiler/aco_ir.h +++ b/src/amd/compiler/aco_ir.h @@ -1674,10 +1674,11 @@ struct Pseudo_branch_instruction : public Instruction { */ uint32_t target[2]; - /* Indicates that selection control prefers to remove this instruction if possible. - * This is set when the branch is divergent and always taken, or flattened. - */ - bool selection_control_remove; + /* Indicates that this rarely or never jumps to target[0]. */ + bool rarely_taken; + bool never_taken; + + uint16_t padding; }; static_assert(sizeof(Pseudo_branch_instruction) == sizeof(Instruction) + 12, "Unexpected padding"); diff --git a/src/amd/compiler/aco_lower_to_hw_instr.cpp b/src/amd/compiler/aco_lower_to_hw_instr.cpp index c95ad0d9ec4..ddc9daa1bb3 100644 --- a/src/amd/compiler/aco_lower_to_hw_instr.cpp +++ b/src/amd/compiler/aco_lower_to_hw_instr.cpp @@ -2832,8 +2832,7 @@ lower_to_hw_instr(Program* program) * - The application prefers to remove control flow * - The compiler stack knows that it's a divergent branch always taken */ - const bool prefer_remove = - branch->selection_control_remove && ctx.program->gfx_level >= GFX10; + const bool prefer_remove = branch->rarely_taken && ctx.program->gfx_level >= GFX10; bool can_remove = block->index < target; unsigned num_scalar = 0; unsigned num_vector = 0; diff --git a/src/amd/compiler/aco_print_ir.cpp b/src/amd/compiler/aco_print_ir.cpp index c620cf4c541..51677648e5b 100644 --- a/src/amd/compiler/aco_print_ir.cpp +++ b/src/amd/compiler/aco_print_ir.cpp @@ -601,6 +601,10 @@ print_instr_format_specific(enum amd_gfx_level gfx_level, const Instruction* ins fprintf(output, " BB%d", branch.target[0]); if (branch.target[1] != 0) fprintf(output, ", BB%d", branch.target[1]); + if (branch.rarely_taken) + fprintf(output, " rarely_taken"); + if (branch.never_taken) + fprintf(output, " never_taken"); break; } case Format::PSEUDO_REDUCTION: {