diff --git a/src/amd/compiler/aco_assembler.cpp b/src/amd/compiler/aco_assembler.cpp index bc2d9e3923b..7590ceeef60 100644 --- a/src/amd/compiler/aco_assembler.cpp +++ b/src/amd/compiler/aco_assembler.cpp @@ -13,6 +13,7 @@ #include "ac_shader_util.h" #include +#include #include #include @@ -23,10 +24,15 @@ struct constaddr_info { unsigned add_literal; }; +struct branch_info { + unsigned pos; + unsigned target; +}; + struct asm_context { Program* program; enum amd_gfx_level gfx_level; - std::vector> branches; + std::vector branches; std::map constaddrs; std::map resumeaddrs; std::vector* symbols; @@ -202,18 +208,17 @@ emit_sopc_instruction(asm_context& ctx, std::vector& out, const Instru } void -emit_sopp_instruction(asm_context& ctx, std::vector& out, Instruction* instr, +emit_sopp_instruction(asm_context& ctx, std::vector& out, const Instruction* instr, bool force_imm = false) { uint32_t opcode = ctx.opcode[(int)instr->opcode]; - SALU_instruction& sopp = instr->salu(); + const SALU_instruction& sopp = instr->salu(); uint32_t encoding = (0b101111111 << 23); encoding |= opcode << 16; if (!force_imm && instr_info.classes[(int)instr->opcode] == instr_class::branch) { - sopp.pass_flags = 0; - ctx.branches.emplace_back(out.size(), &sopp); + ctx.branches.push_back({(unsigned)out.size(), sopp.imm}); } else { assert(sopp.imm <= UINT16_MAX); encoding |= (uint16_t)sopp.imm; @@ -1465,14 +1470,11 @@ insert_code(asm_context& ctx, std::vector& out, unsigned insert_before block.offset += insert_count; } - /* Find first branch after the inserted code */ - auto branch_it = std::find_if(ctx.branches.begin(), ctx.branches.end(), - [insert_before](const auto& branch) -> bool - { return (unsigned)branch.first >= insert_before; }); - /* Update the locations of branches */ - for (; branch_it != ctx.branches.end(); ++branch_it) - branch_it->first += insert_count; + for (branch_info& info : ctx.branches) { + if (info.pos >= insert_before) + info.pos += insert_count; + } /* Update the locations of p_constaddr instructions */ for (auto& constaddr : ctx.constaddrs) { @@ -1508,101 +1510,120 @@ fix_branches_gfx10(asm_context& ctx, std::vector& out) do { auto buggy_branch_it = std::find_if( - ctx.branches.begin(), ctx.branches.end(), - [&ctx](const auto& branch) -> bool { - return ((int)ctx.program->blocks[branch.second->imm].offset - branch.first - 1) == 0x3f; - }); - + ctx.branches.begin(), ctx.branches.end(), [&](const branch_info& branch) -> bool + { return ((int)ctx.program->blocks[branch.target].offset - branch.pos - 1) == 0x3f; }); gfx10_3f_bug = buggy_branch_it != ctx.branches.end(); if (gfx10_3f_bug) { /* Insert an s_nop after the branch */ constexpr uint32_t s_nop_0 = 0xbf800000u; - insert_code(ctx, out, buggy_branch_it->first + 1, 1, &s_nop_0); + insert_code(ctx, out, buggy_branch_it->pos + 1, 1, &s_nop_0); } } while (gfx10_3f_bug); } void -emit_long_jump(asm_context& ctx, SALU_instruction* branch, bool backwards, - std::vector& out) +chain_branches(asm_context& ctx, std::vector& out, branch_info& branch) { + /* Create an empty block in order to remember the offset of the chained branch instruction. + * The new branch instructions are inserted into the program in source code order. + */ + Block* new_block = ctx.program->create_and_insert_block(); Builder bld(ctx.program); + std::vector code; + Instruction* branch_instr; - auto emit = [&](Instruction *instr_ptr, uint32_t *size=NULL) { - aco_ptr instr(instr_ptr); - emit_instruction(ctx, out, instr.get()); + /* Re-direct original branch to new block (offset). */ + unsigned target = branch.target; + branch.target = new_block->index; - if (size) - *size = out.size(); + /* Find suitable insertion point: + * We define two offset ranges within our new branch instruction should be placed. + * Then we try to maximize the distance from either the previous branch or the target. + */ + const int half_dist = (INT16_MAX - 31) / 2; + const unsigned upper_start = MIN2(ctx.program->blocks[target].offset, branch.pos) + half_dist; + const unsigned upper_end = upper_start + half_dist; + const unsigned lower_end = MAX2(ctx.program->blocks[target].offset, branch.pos) - half_dist; + const unsigned lower_start = lower_end - half_dist; + unsigned insert_at = 0; + for (unsigned i = 0; i < ctx.program->blocks.size() - 1; i++) { + Block& block = ctx.program->blocks[i]; + Block& next = ctx.program->blocks[i + 1]; + if (next.offset >= lower_end) + break; + if (next.offset < upper_start || (next.offset > upper_end && next.offset < lower_start)) + continue; - /* VALUMaskWriteHazard and VALUReadSGPRHazard needs sa_sdst=0 wait */ - if (ctx.gfx_level >= GFX11 && !instr_ptr->definitions.empty() && - instr_ptr->definitions[0].physReg() != scc) { - instr.reset(bld.sopp(aco_opcode::s_waitcnt_depctr, 0xfffe)); - emit_instruction(ctx, out, instr.get()); + /* If this block ends in an unconditional branch, we can insert + * another branch right after it without additional cost for the + * existing code. + */ + if (!block.instructions.empty() && + block.instructions.back()->opcode == aco_opcode::s_branch) { + insert_at = next.offset; + bld.reset(&block.instructions); + if (next.offset >= lower_start) + break; } - }; - - Definition def; - if (branch->definitions.empty()) { - assert(ctx.program->blocks[branch->imm].kind & block_kind_discard_early_exit); - def = Definition(PhysReg(0), s2); /* The discard early exit block doesn't use SGPRs. */ - } else { - def = branch->definitions[0]; } - Definition def_tmp_lo(def.physReg(), s1); - Operand op_tmp_lo(def.physReg(), s1); - Definition def_tmp_hi(def.physReg().advance(4), s1); - Operand op_tmp_hi(def.physReg().advance(4), s1); + /* If we didn't find a suitable insertion point, split the existing code. */ + if (insert_at == 0) { + /* Find the last block that is still within reach. */ + unsigned insertion_block_idx = 0; + while (ctx.program->blocks[insertion_block_idx + 1].offset < upper_end) + insertion_block_idx++; - size_t conditional_br_imm = 0; - if (branch->opcode != aco_opcode::s_branch) { - /* for conditional branches, skip the long jump if the condition is false */ - aco_opcode inv; - switch (branch->opcode) { - case aco_opcode::s_cbranch_scc0: inv = aco_opcode::s_cbranch_scc1; break; - case aco_opcode::s_cbranch_scc1: inv = aco_opcode::s_cbranch_scc0; break; - case aco_opcode::s_cbranch_vccz: inv = aco_opcode::s_cbranch_vccnz; break; - case aco_opcode::s_cbranch_vccnz: inv = aco_opcode::s_cbranch_vccz; break; - case aco_opcode::s_cbranch_execz: inv = aco_opcode::s_cbranch_execnz; break; - case aco_opcode::s_cbranch_execnz: inv = aco_opcode::s_cbranch_execz; break; - default: unreachable("Unhandled long jump."); + insert_at = ctx.program->blocks[insertion_block_idx].offset; + auto it = ctx.program->blocks[insertion_block_idx].instructions.begin(); + int skip = 0; + if (insert_at < upper_start) { + /* Ensure some forward progress by splitting the block if necessary. */ + while (skip-- > 0 || insert_at < upper_start) { + Instruction* instr = (it++)->get(); + if (instr->isSOPP()) { + if (instr->opcode == aco_opcode::s_clause) + skip = instr->salu().imm + 1; + else if (instr->opcode == aco_opcode::s_delay_alu) + skip = ((instr->salu().imm >> 4) & 0x7) + 1; + else if (instr->opcode == aco_opcode::s_branch) + skip = 1; + insert_at++; + continue; + } + emit_instruction(ctx, code, instr); + assert(out[insert_at] == code[0]); + insert_at += code.size(); + code.clear(); + } + + /* If the insertion point is in the middle of the block, insert the branch instructions + * into that block instead. */ + bld.reset(&ctx.program->blocks[insertion_block_idx].instructions, it); + } else { + bld.reset(&ctx.program->blocks[insertion_block_idx - 1].instructions); } - aco_ptr instr; - instr.reset(bld.sopp(inv, 0)); - emit_sopp_instruction(ctx, out, instr.get(), true); - conditional_br_imm = out.size() - 1; + + /* Since we insert a branch into existing code, mitigate LdsBranchVmemWARHazard on GFX10. */ + if (ctx.program->gfx_level == GFX10) { + emit_sopk_instruction( + ctx, code, bld.sopk(aco_opcode::s_waitcnt_vscnt, Operand(sgpr_null, s1), 0).instr); + } + + /* For the existing code, create a short jump over the new branch. */ + branch_instr = bld.sopp(aco_opcode::s_branch, 1).instr; + emit_sopp_instruction(ctx, code, branch_instr, true); } + const unsigned block_offset = insert_at + code.size(); - if (ctx.gfx_level == GFX10) /* VMEMtoScalarWriteHazard needs vm_vsrc=0 wait */ - emit(bld.sopp(aco_opcode::s_waitcnt_depctr, 0xffe3)); + branch_instr = bld.sopp(aco_opcode::s_branch, target); + emit_sopp_instruction(ctx, code, branch_instr, true); + insert_code(ctx, out, insert_at, code.size(), code.data()); - /* create the new PC and stash SCC in the LSB */ - uint32_t getpc_loc, add_literal_loc; - emit(bld.sop1(aco_opcode::s_getpc_b64, def), &getpc_loc); - if (ctx.gfx_level >= GFX12) - emit(bld.sop1(aco_opcode::s_sext_i32_i16, def_tmp_hi, op_tmp_hi)); - emit(bld.sop2(aco_opcode::s_addc_u32, def_tmp_lo, op_tmp_lo, Operand::literal32(0)), - &add_literal_loc); - - branch->pass_flags = getpc_loc | (add_literal_loc << 16); - - /* s_addc_u32 for high 32 bits not needed because the program is in a 32-bit VA range */ - - /* restore SCC and clear the LSB of the new PC */ - emit(bld.sopc(aco_opcode::s_bitcmp1_b32, Definition(scc, s1), op_tmp_lo, Operand::zero())); - emit(bld.sop1(aco_opcode::s_bitset0_b32, def_tmp_lo, Operand::zero())); - - /* create the s_setpc_b64 to jump */ - emit(bld.sop1(aco_opcode::s_setpc_b64, Operand(def.physReg(), s2))); - - if (branch->opcode != aco_opcode::s_branch) { - uint32_t imm = out.size() - conditional_br_imm - 1; - assert(imm != 0x3f || ctx.gfx_level != GFX10); - out[conditional_br_imm] |= imm; - } + new_block->offset = block_offset; + ctx.branches.push_back({block_offset, target}); + assert(out[ctx.branches.back().pos] == code.back()); } void @@ -1615,30 +1636,16 @@ fix_branches(asm_context& ctx, std::vector& out) if (ctx.gfx_level == GFX10) fix_branches_gfx10(ctx, out); - for (std::pair& branch : ctx.branches) { - int offset = (int)ctx.program->blocks[branch.second->imm].offset - branch.first - 1; - if ((offset < INT16_MIN || offset > INT16_MAX) && !branch.second->pass_flags) { - std::vector long_jump; - bool backwards = - ctx.program->blocks[branch.second->imm].offset < (unsigned)branch.first; - emit_long_jump(ctx, branch.second, backwards, long_jump); - - out[branch.first] = long_jump[0]; - insert_code(ctx, out, branch.first + 1, long_jump.size() - 1, long_jump.data() + 1); - + for (branch_info& branch : ctx.branches) { + int offset = (int)ctx.program->blocks[branch.target].offset - branch.pos - 1; + if (offset >= INT16_MIN && offset <= INT16_MAX) { + out[branch.pos] &= 0xffff0000u; + out[branch.pos] |= (uint16_t)offset; + } else { + chain_branches(ctx, out, branch); repeat = true; break; } - - if (branch.second->pass_flags) { - uint32_t after_getpc = branch.first + (branch.second->pass_flags & 0xffff); - uint32_t add_literal = branch.first + (branch.second->pass_flags >> 16) - 1; - offset = (int)ctx.program->blocks[branch.second->imm].offset - after_getpc; - out[add_literal] = offset * 4; - } else { - out[branch.first] &= 0xffff0000u; - out[branch.first] |= (uint16_t)offset; - } } } while (repeat); } diff --git a/src/amd/compiler/tests/test_assembler.cpp b/src/amd/compiler/tests/test_assembler.cpp index a11316a6a75..2ea7f21e88d 100644 --- a/src/amd/compiler/tests/test_assembler.cpp +++ b/src/amd/compiler/tests/test_assembler.cpp @@ -60,21 +60,21 @@ BEGIN_TEST(assembler.long_jump.unconditional_forwards) return; //!BB0: - //! s_waitcnt_depctr 0xffe3 ; bfa3ffe3 - //! s_getpc_b64 s[0:1] ; be801f00 - //! s_addc_u32 s0, s0, 0x20014 ; 8200ff00 00020014 - //! s_bitcmp1_b32 s0, 0 ; bf0d8000 - //! s_bitset0_b32 s0, 0 ; be801b80 - //! s_setpc_b64 s[0:1] ; be802000 + //! s_branch 16369 ; bf823ff1 bld.sopp(aco_opcode::s_branch, Definition(PhysReg(0), s2), 2); bld.reset(program->create_and_insert_block()); //! s_nop 0 ; bf800000 - //!(then repeated 32767 times) + //!(then repeated 16366 times) for (unsigned i = 0; i < INT16_MAX + 1; i++) bld.sopp(aco_opcode::s_nop, 0); + //! s_waitcnt_vscnt null, 0x0 ; bbfd0000 + //! s_branch 1 ; bf820001 + //! s_branch BB2 ; bf824011 + //! s_nop 0 ; bf800000 + //! (then repeated 16400 times) //! BB2: //! s_endpgm ; bf810000 bld.reset(program->create_and_insert_block()); @@ -91,26 +91,20 @@ BEGIN_TEST(assembler.long_jump.conditional_forwards) continue; //! BB0: - //! s_cbranch_scc1 BB1 ; $_ - //~gfx10! s_waitcnt_depctr 0xffe3 ; $_ - //! s_getpc_b64 s[0:1] ; $_ - //~gfx12! s_wait_alu 0xfffe ; $_ - //~gfx12! s_sext_i32_i16 s1, s1 ; $_ - //~gfx12! s_wait_alu 0xfffe ; $_ - //~gfx10! s_addc_u32 s0, s0, 0x20014 ; $_ $_ - //~gfx12! s_add_co_ci_u32 s0, s0, 0x20028 ; $_ $_ - //~gfx12! s_wait_alu 0xfffe ; $_ - //! s_bitcmp1_b32 s0, 0 ; $_ - //! s_bitset0_b32 s0, 0 ; $_ - //~gfx12! s_wait_alu 0xfffe ; $_ - //! s_setpc_b64 s[0:1] ; $_ + //~gfx10! s_cbranch_scc0 16369 ; bf843ff1 + //~gfx12! s_cbranch_scc0 16368 ; bfa13ff0 bld.sopp(aco_opcode::s_cbranch_scc0, Definition(PhysReg(0), s2), 2); bld.reset(program->create_and_insert_block()); //! BB1: - //! s_nop 0 ; bf800000 - //!(then repeated 32767 times) + //! s_nop 0 ; bf800000 + //!(then repeated 16366 times) + //~gfx10! s_waitcnt_vscnt null, 0x0 ; bbfd0000 + //! s_branch 1 ; $_ + //! s_branch BB2 ; $_ + //! s_nop 0 ; bf800000 + //!(then repeated 16400 times) for (unsigned i = 0; i < INT16_MAX + 1; i++) bld.sopp(aco_opcode::s_nop, 0); @@ -132,16 +126,16 @@ BEGIN_TEST(assembler.long_jump.unconditional_backwards) //!BB0: //! s_nop 0 ; bf800000 - //!(then repeated 32767 times) + //!(then repeated 16367 times) for (unsigned i = 0; i < INT16_MAX + 1; i++) bld.sopp(aco_opcode::s_nop, 0); - //! s_waitcnt_depctr 0xffe3 ; bfa3ffe3 - //! s_getpc_b64 s[0:1] ; be801f00 - //! s_addc_u32 s0, s0, 0xfffdfff8 ; 8200ff00 fffdfff8 - //! s_bitcmp1_b32 s0, 0 ; bf0d8000 - //! s_bitset0_b32 s0, 0 ; be801b80 - //! s_setpc_b64 s[0:1] ; be802000 + //! s_waitcnt_vscnt null, 0x0 ; bbfd0000 + //! s_branch 1 ; bf820001 + //! s_branch BB0 ; bf82c00d + //! s_nop 0 ; bf800000 + //! (then repeated 16399 times) + //! s_branch 49134 ; bf82bfee bld.sopp(aco_opcode::s_branch, Definition(PhysReg(0), s2), 0); //! BB1: @@ -160,17 +154,16 @@ BEGIN_TEST(assembler.long_jump.conditional_backwards) //!BB0: //! s_nop 0 ; bf800000 - //!(then repeated 32767 times) + //!(then repeated 16367 times) for (unsigned i = 0; i < INT16_MAX + 1; i++) bld.sopp(aco_opcode::s_nop, 0); - //! s_cbranch_execz BB1 ; bf880007 - //! s_waitcnt_depctr 0xffe3 ; bfa3ffe3 - //! s_getpc_b64 s[0:1] ; be801f00 - //! s_addc_u32 s0, s0, 0xfffdfff4 ; 8200ff00 fffdfff4 - //! s_bitcmp1_b32 s0, 0 ; bf0d8000 - //! s_bitset0_b32 s0, 0 ; be801b80 - //! s_setpc_b64 s[0:1] ; be802000 + //! s_waitcnt_vscnt null, 0x0 ; bbfd0000 + //! s_branch 1 ; bf820001 + //! s_branch BB0 ; bf82c00d + //! s_nop 0 ; bf800000 + //!(then repeated 16399 times) + //! s_cbranch_execnz 49134 ; bf89bfee bld.sopp(aco_opcode::s_cbranch_execnz, Definition(PhysReg(0), s2), 0); //! BB1: @@ -183,36 +176,11 @@ BEGIN_TEST(assembler.long_jump.conditional_backwards) finish_assembler_test(); END_TEST -BEGIN_TEST(assembler.long_jump._3f) - if (!setup_cs(NULL, (amd_gfx_level)GFX10)) - return; - - //! BB0: - //! s_branch BB1 ; bf820040 - //! s_nop 0 ; bf800000 - bld.sopp(aco_opcode::s_branch, Definition(PhysReg(0), s2), 1); - - for (unsigned i = 0; i < 0x3f - 7; i++) // an unconditional long jump is 7 dwords - bld.vop1(aco_opcode::v_nop); - bld.sopp(aco_opcode::s_branch, Definition(PhysReg(0), s2), 2); - - bld.reset(program->create_and_insert_block()); - for (unsigned i = 0; i < INT16_MAX + 1; i++) - bld.vop1(aco_opcode::v_nop); - bld.reset(program->create_and_insert_block()); - - program->blocks[1].linear_preds.push_back(0u); - program->blocks[2].linear_preds.push_back(0u); - program->blocks[2].linear_preds.push_back(1u); - - finish_assembler_test(); -END_TEST - BEGIN_TEST(assembler.long_jump.constaddr) if (!setup_cs(NULL, (amd_gfx_level)GFX10)) return; - //>> s_getpc_b64 s[0:1] ; be801f00 + //>> s_branch 16369 ; bf823ff1 bld.sopp(aco_opcode::s_branch, Definition(PhysReg(0), s2), 2); bld.reset(program->create_and_insert_block()); @@ -239,20 +207,19 @@ BEGIN_TEST(assembler.long_jump.discard_early_exit) return; //! BB0: - //! s_cbranch_scc1 BB1 ; bf850007 - //! s_waitcnt_depctr 0xffe3 ; bfa3ffe3 - //! s_getpc_b64 s[0:1] ; be801f00 - //! s_addc_u32 s0, s0, 0x20014 ; 8200ff00 00020014 - //! s_bitcmp1_b32 s0, 0 ; bf0d8000 - //! s_bitset0_b32 s0, 0 ; be801b80 - //! s_setpc_b64 s[0:1] ; be802000 + //! s_cbranch_scc0 16369 ; bf843ff1 bld.sopp(aco_opcode::s_cbranch_scc0, 2); bld.reset(program->create_and_insert_block()); //! BB1: //! s_nop 1 ; bf800001 - //!(then repeated 32766 times) + //! (then repeated 16366 times) + //! s_waitcnt_vscnt null, 0x0 ; bbfd0000 + //! s_branch 1 ; bf820001 + //! s_branch BB2 ; bf824011 + //! s_nop 1 ; bf800001 + //! (then repeated 16399 times) //! s_endpgm ; bf810000 for (unsigned i = 0; i < INT16_MAX; i++) bld.sopp(aco_opcode::s_nop, 1);