AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

aco/assembler: chain branches instead of emitting long jumps

Browse Source 
As regular branch instructions cannot jump further than
32768 dwords, previously we used long jumps as fallback
solution. The disadvantage of that is that an extra SGPR
pair must be provided in order to temporarily store the PC.

This patch changes that to chained branch instructions by
inserting an artificial extra block into the code to be
targeted by the original branch. This block contains a
single branch instruction jumping to the original target.
Before the block, if necessary, we insert a <branch 1>
instruction for the existing code in order to jump over
the newly inserted block.

Only a few RT shaders are affected.

Totals from 29 (0.04% of 79395) affected shaders: (Navi31)

CodeSize: 17281176 -> 17276332 (-0.03%)
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/32037>
This commit is contained in:
Daniel Schürmann
2024-11-06 14:56:53 +01:00
committed by Marge Bot
parent c3d777d8ac
commit cab5639a09
2 changed files with 150 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/amd/compiler/aco_assembler.cpp
+112 -105
View File
@@ -13,6 +13,7 @@
#include "ac_shader_util.h"
#include <algorithm>
#include <cstdint>
#include <map>
#include <vector>
@@ -23,10 +24,15 @@ struct constaddr_info {
unsigned add_literal;
};
struct branch_info {
unsigned pos;
unsigned target;
};
struct asm_context {
Program* program;
enum amd_gfx_level gfx_level;
std::vector<std::pair<int, SALU_instruction*>> branches;
std::vector<branch_info> branches;
std::map<unsigned, constaddr_info> constaddrs;
std::map<unsigned, constaddr_info> resumeaddrs;
std::vector<struct aco_symbol>* symbols;
@@ -202,18 +208,17 @@ emit_sopc_instruction(asm_context& ctx, std::vector<uint32_t>& out, const Instru
}
void
emit_sopp_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction* instr,
emit_sopp_instruction(asm_context& ctx, std::vector<uint32_t>& out, const Instruction* instr,
bool force_imm = false)
{
uint32_t opcode = ctx.opcode[(int)instr->opcode];
SALU_instruction& sopp = instr->salu();
const SALU_instruction& sopp = instr->salu();
uint32_t encoding = (0b101111111 << 23);
encoding |= opcode << 16;
if (!force_imm && instr_info.classes[(int)instr->opcode] == instr_class::branch) {
sopp.pass_flags = 0;
ctx.branches.emplace_back(out.size(), &sopp);
ctx.branches.push_back({(unsigned)out.size(), sopp.imm});
} else {
assert(sopp.imm <= UINT16_MAX);
encoding |= (uint16_t)sopp.imm;
@@ -1465,14 +1470,11 @@ insert_code(asm_context& ctx, std::vector<uint32_t>& out, unsigned insert_before
block.offset += insert_count;
}
/* Find first branch after the inserted code */
auto branch_it = std::find_if(ctx.branches.begin(), ctx.branches.end(),
[insert_before](const auto& branch) -> bool
{ return (unsigned)branch.first >= insert_before; });
/* Update the locations of branches */
for (; branch_it != ctx.branches.end(); ++branch_it)
branch_it->first += insert_count;
for (branch_info& info : ctx.branches) {
if (info.pos >= insert_before)
info.pos += insert_count;
}
/* Update the locations of p_constaddr instructions */
for (auto& constaddr : ctx.constaddrs) {
@@ -1508,101 +1510,120 @@ fix_branches_gfx10(asm_context& ctx, std::vector<uint32_t>& out)
do {
auto buggy_branch_it = std::find_if(
ctx.branches.begin(), ctx.branches.end(),
[&ctx](const auto& branch) -> bool {
return ((int)ctx.program->blocks[branch.second->imm].offset - branch.first - 1) == 0x3f;
});
ctx.branches.begin(), ctx.branches.end(), [&](const branch_info& branch) -> bool
{ return ((int)ctx.program->blocks[branch.target].offset - branch.pos - 1) == 0x3f; });
gfx10_3f_bug = buggy_branch_it != ctx.branches.end();
if (gfx10_3f_bug) {
/* Insert an s_nop after the branch */
constexpr uint32_t s_nop_0 = 0xbf800000u;
insert_code(ctx, out, buggy_branch_it->first + 1, 1, &s_nop_0);
insert_code(ctx, out, buggy_branch_it->pos + 1, 1, &s_nop_0);
}
} while (gfx10_3f_bug);
}
void
emit_long_jump(asm_context& ctx, SALU_instruction* branch, bool backwards,
std::vector<uint32_t>& out)
chain_branches(asm_context& ctx, std::vector<uint32_t>& out, branch_info& branch)
{
/* Create an empty block in order to remember the offset of the chained branch instruction.
* The new branch instructions are inserted into the program in source code order.
*/
Block* new_block = ctx.program->create_and_insert_block();
Builder bld(ctx.program);
std::vector<uint32_t> code;
Instruction* branch_instr;
auto emit = [&](Instruction *instr_ptr, uint32_t *size=NULL) {
aco_ptr<Instruction> instr(instr_ptr);
emit_instruction(ctx, out, instr.get());
/* Re-direct original branch to new block (offset). */
unsigned target = branch.target;
branch.target = new_block->index;
if (size)
*size = out.size();
/* Find suitable insertion point:
* We define two offset ranges within our new branch instruction should be placed.
* Then we try to maximize the distance from either the previous branch or the target.
*/
const int half_dist = (INT16_MAX - 31) / 2;
const unsigned upper_start = MIN2(ctx.program->blocks[target].offset, branch.pos) + half_dist;
const unsigned upper_end = upper_start + half_dist;
const unsigned lower_end = MAX2(ctx.program->blocks[target].offset, branch.pos) - half_dist;
const unsigned lower_start = lower_end - half_dist;
unsigned insert_at = 0;
for (unsigned i = 0; i < ctx.program->blocks.size() - 1; i++) {
Block& block = ctx.program->blocks[i];
Block& next = ctx.program->blocks[i + 1];
if (next.offset >= lower_end)
break;
if (next.offset < upper_start || (next.offset > upper_end && next.offset < lower_start))
continue;
/* VALUMaskWriteHazard and VALUReadSGPRHazard needs sa_sdst=0 wait */
if (ctx.gfx_level >= GFX11 && !instr_ptr->definitions.empty() &&
instr_ptr->definitions[0].physReg() != scc) {
instr.reset(bld.sopp(aco_opcode::s_waitcnt_depctr, 0xfffe));
emit_instruction(ctx, out, instr.get());
/* If this block ends in an unconditional branch, we can insert
* another branch right after it without additional cost for the
* existing code.
*/
if (!block.instructions.empty() &&
block.instructions.back()->opcode == aco_opcode::s_branch) {
insert_at = next.offset;
bld.reset(&block.instructions);
if (next.offset >= lower_start)
break;
}
};
Definition def;
if (branch->definitions.empty()) {
assert(ctx.program->blocks[branch->imm].kind & block_kind_discard_early_exit);
def = Definition(PhysReg(0), s2); /* The discard early exit block doesn't use SGPRs. */
} else {
def = branch->definitions[0];
}
Definition def_tmp_lo(def.physReg(), s1);
Operand op_tmp_lo(def.physReg(), s1);
Definition def_tmp_hi(def.physReg().advance(4), s1);
Operand op_tmp_hi(def.physReg().advance(4), s1);
/* If we didn't find a suitable insertion point, split the existing code. */
if (insert_at == 0) {
/* Find the last block that is still within reach. */
unsigned insertion_block_idx = 0;
while (ctx.program->blocks[insertion_block_idx + 1].offset < upper_end)
insertion_block_idx++;
size_t conditional_br_imm = 0;
if (branch->opcode != aco_opcode::s_branch) {
/* for conditional branches, skip the long jump if the condition is false */
aco_opcode inv;
switch (branch->opcode) {
case aco_opcode::s_cbranch_scc0: inv = aco_opcode::s_cbranch_scc1; break;
case aco_opcode::s_cbranch_scc1: inv = aco_opcode::s_cbranch_scc0; break;
case aco_opcode::s_cbranch_vccz: inv = aco_opcode::s_cbranch_vccnz; break;
case aco_opcode::s_cbranch_vccnz: inv = aco_opcode::s_cbranch_vccz; break;
case aco_opcode::s_cbranch_execz: inv = aco_opcode::s_cbranch_execnz; break;
case aco_opcode::s_cbranch_execnz: inv = aco_opcode::s_cbranch_execz; break;
default: unreachable("Unhandled long jump.");
insert_at = ctx.program->blocks[insertion_block_idx].offset;
auto it = ctx.program->blocks[insertion_block_idx].instructions.begin();
int skip = 0;
if (insert_at < upper_start) {
/* Ensure some forward progress by splitting the block if necessary. */
while (skip-- > 0 || insert_at < upper_start) {
Instruction* instr = (it++)->get();
if (instr->isSOPP()) {
if (instr->opcode == aco_opcode::s_clause)
skip = instr->salu().imm + 1;
else if (instr->opcode == aco_opcode::s_delay_alu)
skip = ((instr->salu().imm >> 4) & 0x7) + 1;
else if (instr->opcode == aco_opcode::s_branch)
skip = 1;
insert_at++;
continue;
}
emit_instruction(ctx, code, instr);
assert(out[insert_at] == code[0]);
insert_at += code.size();
code.clear();
}
/* If the insertion point is in the middle of the block, insert the branch instructions
* into that block instead. */
bld.reset(&ctx.program->blocks[insertion_block_idx].instructions, it);
} else {
bld.reset(&ctx.program->blocks[insertion_block_idx - 1].instructions);
}
aco_ptr<Instruction> instr;
instr.reset(bld.sopp(inv, 0));
emit_sopp_instruction(ctx, out, instr.get(), true);
conditional_br_imm = out.size() - 1;
/* Since we insert a branch into existing code, mitigate LdsBranchVmemWARHazard on GFX10. */
if (ctx.program->gfx_level == GFX10) {
emit_sopk_instruction(
ctx, code, bld.sopk(aco_opcode::s_waitcnt_vscnt, Operand(sgpr_null, s1), 0).instr);
}
/* For the existing code, create a short jump over the new branch. */
branch_instr = bld.sopp(aco_opcode::s_branch, 1).instr;
emit_sopp_instruction(ctx, code, branch_instr, true);
}
const unsigned block_offset = insert_at + code.size();
if (ctx.gfx_level == GFX10) /* VMEMtoScalarWriteHazard needs vm_vsrc=0 wait */
emit(bld.sopp(aco_opcode::s_waitcnt_depctr, 0xffe3));
branch_instr = bld.sopp(aco_opcode::s_branch, target);
emit_sopp_instruction(ctx, code, branch_instr, true);
insert_code(ctx, out, insert_at, code.size(), code.data());
/* create the new PC and stash SCC in the LSB */
uint32_t getpc_loc, add_literal_loc;
emit(bld.sop1(aco_opcode::s_getpc_b64, def), &getpc_loc);
if (ctx.gfx_level >= GFX12)
emit(bld.sop1(aco_opcode::s_sext_i32_i16, def_tmp_hi, op_tmp_hi));
emit(bld.sop2(aco_opcode::s_addc_u32, def_tmp_lo, op_tmp_lo, Operand::literal32(0)),
&add_literal_loc);
branch->pass_flags = getpc_loc | (add_literal_loc << 16);
/* s_addc_u32 for high 32 bits not needed because the program is in a 32-bit VA range */
/* restore SCC and clear the LSB of the new PC */
emit(bld.sopc(aco_opcode::s_bitcmp1_b32, Definition(scc, s1), op_tmp_lo, Operand::zero()));
emit(bld.sop1(aco_opcode::s_bitset0_b32, def_tmp_lo, Operand::zero()));
/* create the s_setpc_b64 to jump */
emit(bld.sop1(aco_opcode::s_setpc_b64, Operand(def.physReg(), s2)));
if (branch->opcode != aco_opcode::s_branch) {
uint32_t imm = out.size() - conditional_br_imm - 1;
assert(imm != 0x3f || ctx.gfx_level != GFX10);
out[conditional_br_imm] |= imm;
}
new_block->offset = block_offset;
ctx.branches.push_back({block_offset, target});
assert(out[ctx.branches.back().pos] == code.back());
}
void
@@ -1615,30 +1636,16 @@ fix_branches(asm_context& ctx, std::vector<uint32_t>& out)
if (ctx.gfx_level == GFX10)
fix_branches_gfx10(ctx, out);
for (std::pair<int, SALU_instruction*>& branch : ctx.branches) {
int offset = (int)ctx.program->blocks[branch.second->imm].offset - branch.first - 1;
if ((offset < INT16_MIN || offset > INT16_MAX) && !branch.second->pass_flags) {
std::vector<uint32_t> long_jump;
bool backwards =
ctx.program->blocks[branch.second->imm].offset < (unsigned)branch.first;
emit_long_jump(ctx, branch.second, backwards, long_jump);
out[branch.first] = long_jump[0];
insert_code(ctx, out, branch.first + 1, long_jump.size() - 1, long_jump.data() + 1);
for (branch_info& branch : ctx.branches) {
int offset = (int)ctx.program->blocks[branch.target].offset - branch.pos - 1;
if (offset >= INT16_MIN && offset <= INT16_MAX) {
out[branch.pos] &= 0xffff0000u;
out[branch.pos] |= (uint16_t)offset;
} else {
chain_branches(ctx, out, branch);
repeat = true;
break;
}
if (branch.second->pass_flags) {
uint32_t after_getpc = branch.first + (branch.second->pass_flags & 0xffff);
uint32_t add_literal = branch.first + (branch.second->pass_flags >> 16) - 1;
offset = (int)ctx.program->blocks[branch.second->imm].offset - after_getpc;
out[add_literal] = offset * 4;
} else {
out[branch.first] &= 0xffff0000u;
out[branch.first] |= (uint16_t)offset;
}
}
} while (repeat);
}
src/amd/compiler/tests/test_assembler.cpp
+38 -71
View File
@@ -60,21 +60,21 @@ BEGIN_TEST(assembler.long_jump.unconditional_forwards)
return;
//!BB0:
//! s_waitcnt_depctr 0xffe3 ; bfa3ffe3
//! s_getpc_b64 s[0:1] ; be801f00
//! s_addc_u32 s0, s0, 0x20014 ; 8200ff00 00020014
//! s_bitcmp1_b32 s0, 0 ; bf0d8000
//! s_bitset0_b32 s0, 0 ; be801b80
//! s_setpc_b64 s[0:1] ; be802000
//! s_branch 16369 ; bf823ff1
bld.sopp(aco_opcode::s_branch, Definition(PhysReg(0), s2), 2);
bld.reset(program->create_and_insert_block());
//! s_nop 0 ; bf800000
//!(then repeated 32767 times)
//!(then repeated 16366 times)
for (unsigned i = 0; i < INT16_MAX + 1; i++)
bld.sopp(aco_opcode::s_nop, 0);
//! s_waitcnt_vscnt null, 0x0 ; bbfd0000
//! s_branch 1 ; bf820001
//! s_branch BB2 ; bf824011
//! s_nop 0 ; bf800000
//! (then repeated 16400 times)
//! BB2:
//! s_endpgm ; bf810000
bld.reset(program->create_and_insert_block());
@@ -91,26 +91,20 @@ BEGIN_TEST(assembler.long_jump.conditional_forwards)
continue;
//! BB0:
//! s_cbranch_scc1 BB1 ; $_
//~gfx10! s_waitcnt_depctr 0xffe3 ; $_
//! s_getpc_b64 s[0:1] ; $_
//~gfx12! s_wait_alu 0xfffe ; $_
//~gfx12! s_sext_i32_i16 s1, s1 ; $_
//~gfx12! s_wait_alu 0xfffe ; $_
//~gfx10! s_addc_u32 s0, s0, 0x20014 ; $_ $_
//~gfx12! s_add_co_ci_u32 s0, s0, 0x20028 ; $_ $_
//~gfx12! s_wait_alu 0xfffe ; $_
//! s_bitcmp1_b32 s0, 0 ; $_
//! s_bitset0_b32 s0, 0 ; $_
//~gfx12! s_wait_alu 0xfffe ; $_
//! s_setpc_b64 s[0:1] ; $_
//~gfx10! s_cbranch_scc0 16369 ; bf843ff1
//~gfx12! s_cbranch_scc0 16368 ; bfa13ff0
bld.sopp(aco_opcode::s_cbranch_scc0, Definition(PhysReg(0), s2), 2);
bld.reset(program->create_and_insert_block());
//! BB1:
//! s_nop 0 ; bf800000
//!(then repeated 32767 times)
//! s_nop 0 ; bf800000
//!(then repeated 16366 times)
//~gfx10! s_waitcnt_vscnt null, 0x0 ; bbfd0000
//! s_branch 1 ; $_
//! s_branch BB2 ; $_
//! s_nop 0 ; bf800000
//!(then repeated 16400 times)
for (unsigned i = 0; i < INT16_MAX + 1; i++)
bld.sopp(aco_opcode::s_nop, 0);
@@ -132,16 +126,16 @@ BEGIN_TEST(assembler.long_jump.unconditional_backwards)
//!BB0:
//! s_nop 0 ; bf800000
//!(then repeated 32767 times)
//!(then repeated 16367 times)
for (unsigned i = 0; i < INT16_MAX + 1; i++)
bld.sopp(aco_opcode::s_nop, 0);
//! s_waitcnt_depctr 0xffe3 ; bfa3ffe3
//! s_getpc_b64 s[0:1] ; be801f00
//! s_addc_u32 s0, s0, 0xfffdfff8 ; 8200ff00 fffdfff8
//! s_bitcmp1_b32 s0, 0 ; bf0d8000
//! s_bitset0_b32 s0, 0 ; be801b80
//! s_setpc_b64 s[0:1] ; be802000
//! s_waitcnt_vscnt null, 0x0 ; bbfd0000
//! s_branch 1 ; bf820001
//! s_branch BB0 ; bf82c00d
//! s_nop 0 ; bf800000
//! (then repeated 16399 times)
//! s_branch 49134 ; bf82bfee
bld.sopp(aco_opcode::s_branch, Definition(PhysReg(0), s2), 0);
//! BB1:
@@ -160,17 +154,16 @@ BEGIN_TEST(assembler.long_jump.conditional_backwards)
//!BB0:
//! s_nop 0 ; bf800000
//!(then repeated 32767 times)
//!(then repeated 16367 times)
for (unsigned i = 0; i < INT16_MAX + 1; i++)
bld.sopp(aco_opcode::s_nop, 0);
//! s_cbranch_execz BB1 ; bf880007
//! s_waitcnt_depctr 0xffe3 ; bfa3ffe3
//! s_getpc_b64 s[0:1] ; be801f00
//! s_addc_u32 s0, s0, 0xfffdfff4 ; 8200ff00 fffdfff4
//! s_bitcmp1_b32 s0, 0 ; bf0d8000
//! s_bitset0_b32 s0, 0 ; be801b80
//! s_setpc_b64 s[0:1] ; be802000
//! s_waitcnt_vscnt null, 0x0 ; bbfd0000
//! s_branch 1 ; bf820001
//! s_branch BB0 ; bf82c00d
//! s_nop 0 ; bf800000
//!(then repeated 16399 times)
//! s_cbranch_execnz 49134 ; bf89bfee
bld.sopp(aco_opcode::s_cbranch_execnz, Definition(PhysReg(0), s2), 0);
//! BB1:
@@ -183,36 +176,11 @@ BEGIN_TEST(assembler.long_jump.conditional_backwards)
finish_assembler_test();
END_TEST
BEGIN_TEST(assembler.long_jump._3f)
if (!setup_cs(NULL, (amd_gfx_level)GFX10))
return;
//! BB0:
//! s_branch BB1 ; bf820040
//! s_nop 0 ; bf800000
bld.sopp(aco_opcode::s_branch, Definition(PhysReg(0), s2), 1);
for (unsigned i = 0; i < 0x3f - 7; i++) // an unconditional long jump is 7 dwords
bld.vop1(aco_opcode::v_nop);
bld.sopp(aco_opcode::s_branch, Definition(PhysReg(0), s2), 2);
bld.reset(program->create_and_insert_block());
for (unsigned i = 0; i < INT16_MAX + 1; i++)
bld.vop1(aco_opcode::v_nop);
bld.reset(program->create_and_insert_block());
program->blocks[1].linear_preds.push_back(0u);
program->blocks[2].linear_preds.push_back(0u);
program->blocks[2].linear_preds.push_back(1u);
finish_assembler_test();
END_TEST
BEGIN_TEST(assembler.long_jump.constaddr)
if (!setup_cs(NULL, (amd_gfx_level)GFX10))
return;
//>> s_getpc_b64 s[0:1] ; be801f00
//>> s_branch 16369 ; bf823ff1
bld.sopp(aco_opcode::s_branch, Definition(PhysReg(0), s2), 2);
bld.reset(program->create_and_insert_block());
@@ -239,20 +207,19 @@ BEGIN_TEST(assembler.long_jump.discard_early_exit)
return;
//! BB0:
//! s_cbranch_scc1 BB1 ; bf850007
//! s_waitcnt_depctr 0xffe3 ; bfa3ffe3
//! s_getpc_b64 s[0:1] ; be801f00
//! s_addc_u32 s0, s0, 0x20014 ; 8200ff00 00020014
//! s_bitcmp1_b32 s0, 0 ; bf0d8000
//! s_bitset0_b32 s0, 0 ; be801b80
//! s_setpc_b64 s[0:1] ; be802000
//! s_cbranch_scc0 16369 ; bf843ff1
bld.sopp(aco_opcode::s_cbranch_scc0, 2);
bld.reset(program->create_and_insert_block());
//! BB1:
//! s_nop 1 ; bf800001
//!(then repeated 32766 times)
//! (then repeated 16366 times)
//! s_waitcnt_vscnt null, 0x0 ; bbfd0000
//! s_branch 1 ; bf820001
//! s_branch BB2 ; bf824011
//! s_nop 1 ; bf800001
//! (then repeated 16399 times)
//! s_endpgm ; bf810000
for (unsigned i = 0; i < INT16_MAX; i++)
bld.sopp(aco_opcode::s_nop, 1);