AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

aco: Format.

Browse Source 
Manually adjusted some comments for more intuitive line breaks.

Reviewed-by: Tony Wasserka <tony.wasserka@gmx.de>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11258>
This commit is contained in:
Daniel Schürmann
2021-06-09 10:14:54 +02:00
committed by Marge Bot
parent 97ec360dc4
commit 1e2639026f
32 changed files with 7231 additions and 6574 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/amd/compiler/aco_insert_exec_mask.cpp
+129 -98
View File
@@ -24,6 +24,7 @@
#include "aco_builder.h"
#include "aco_ir.h"
#include "util/u_math.h"
#include <set>
@@ -55,10 +56,9 @@ struct wqm_ctx {
std::vector<uint16_t> defined_in;
std::vector<bool> needs_wqm;
std::vector<bool> branch_wqm; /* true if the branch condition in this block should be in wqm */
wqm_ctx(Program* program_) : program(program_),
defined_in(program->peekAllocationId(), 0xFFFF),
needs_wqm(program->peekAllocationId()),
branch_wqm(program->blocks.size())
wqm_ctx(Program* program_)
: program(program_), defined_in(program->peekAllocationId(), 0xFFFF),
needs_wqm(program->peekAllocationId()), branch_wqm(program->blocks.size())
{
for (unsigned i = 0; i < program->blocks.size(); i++)
worklist.insert(i);
@@ -72,13 +72,15 @@ struct loop_info {
bool has_divergent_break;
bool has_divergent_continue;
bool has_discard; /* has a discard or demote */
loop_info(Block* b, uint16_t num, uint8_t needs_, bool breaks, bool cont, bool discard) :
loop_header(b), num_exec_masks(num), needs(needs_), has_divergent_break(breaks),
has_divergent_continue(cont), has_discard(discard) {}
loop_info(Block* b, uint16_t num, uint8_t needs_, bool breaks, bool cont, bool discard)
: loop_header(b), num_exec_masks(num), needs(needs_), has_divergent_break(breaks),
has_divergent_continue(cont), has_discard(discard)
{}
};
struct block_info {
std::vector<std::pair<Operand, uint8_t>> exec; /* Vector of exec masks. Either a temporary or const -1. */
std::vector<std::pair<Operand, uint8_t>>
exec; /* Vector of exec masks. Either a temporary or const -1. */
std::vector<WQMState> instr_needs;
uint8_t block_needs;
uint8_t ever_again_needs;
@@ -87,14 +89,16 @@ struct block_info {
};
struct exec_ctx {
Program *program;
Program* program;
std::vector<block_info> info;
std::vector<loop_info> loop;
bool handle_wqm = false;
exec_ctx(Program *program_) : program(program_), info(program->blocks.size()) {}
exec_ctx(Program* program_) : program(program_), info(program->blocks.size()) {}
};
bool needs_exact(aco_ptr<Instruction>& instr) {
bool
needs_exact(aco_ptr<Instruction>& instr)
{
if (instr->isMUBUF()) {
return instr->mubuf().disable_wqm;
} else if (instr->isMTBUF()) {
@@ -108,7 +112,8 @@ bool needs_exact(aco_ptr<Instruction>& instr) {
}
}
void set_needs_wqm(wqm_ctx &ctx, Temp tmp)
void
set_needs_wqm(wqm_ctx& ctx, Temp tmp)
{
if (!ctx.needs_wqm[tmp.id()]) {
ctx.needs_wqm[tmp.id()] = true;
@@ -117,7 +122,8 @@ void set_needs_wqm(wqm_ctx &ctx, Temp tmp)
}
}
void mark_block_wqm(wqm_ctx &ctx, unsigned block_idx)
void
mark_block_wqm(wqm_ctx& ctx, unsigned block_idx)
{
if (ctx.branch_wqm[block_idx])
return;
@@ -136,7 +142,8 @@ void mark_block_wqm(wqm_ctx &ctx, unsigned block_idx)
mark_block_wqm(ctx, pred_idx);
}
void get_block_needs(wqm_ctx &ctx, exec_ctx &exec_ctx, Block* block)
void
get_block_needs(wqm_ctx& ctx, exec_ctx& exec_ctx, Block* block)
{
block_info& info = exec_ctx.info[block->index];
@@ -146,8 +153,8 @@ void get_block_needs(wqm_ctx &ctx, exec_ctx &exec_ctx, Block* block)
aco_ptr<Instruction>& instr = block->instructions[i];
WQMState needs = needs_exact(instr) ? Exact : Unspecified;
bool propagate_wqm = instr->opcode == aco_opcode::p_wqm ||
instr->opcode == aco_opcode::p_as_uniform;
bool propagate_wqm =
instr->opcode == aco_opcode::p_wqm || instr->opcode == aco_opcode::p_as_uniform;
bool preserve_wqm = instr->opcode == aco_opcode::p_discard_if;
bool pred_by_exec = needs_exec_mask(instr.get());
for (const Definition& definition : instr->definitions) {
@@ -214,7 +221,8 @@ void get_block_needs(wqm_ctx &ctx, exec_ctx &exec_ctx, Block* block)
* breaks, which might benefit from being in exact) by adding Exact_Branch to a
* divergent branch surrounding the nested loop, if such a branch exists.
*/
void handle_wqm_loops(wqm_ctx& ctx, exec_ctx& exec_ctx, unsigned preheader)
void
handle_wqm_loops(wqm_ctx& ctx, exec_ctx& exec_ctx, unsigned preheader)
{
for (unsigned idx = preheader + 1; idx < exec_ctx.program->blocks.size(); idx++) {
Block& block = exec_ctx.program->blocks[idx];
@@ -231,7 +239,8 @@ void handle_wqm_loops(wqm_ctx& ctx, exec_ctx& exec_ctx, unsigned preheader)
* ensure that the exact exec mask is not empty by adding Exact_Branch to
* the outer divergent branch.
*/
void handle_exact_loops(wqm_ctx& ctx, exec_ctx& exec_ctx, unsigned preheader)
void
handle_exact_loops(wqm_ctx& ctx, exec_ctx& exec_ctx, unsigned preheader)
{
assert(exec_ctx.program->blocks[preheader + 1].kind & block_kind_loop_header);
@@ -265,7 +274,8 @@ void handle_exact_loops(wqm_ctx& ctx, exec_ctx& exec_ctx, unsigned preheader)
}
}
void calculate_wqm_needs(exec_ctx& exec_ctx)
void
calculate_wqm_needs(exec_ctx& exec_ctx)
{
wqm_ctx ctx(exec_ctx.program);
@@ -307,14 +317,12 @@ void calculate_wqm_needs(exec_ctx& exec_ctx)
exec_ctx.info[i].block_needs |= Exact;
/* if discard is used somewhere in nested CF, we need to preserve the WQM mask */
if ((block.kind & block_kind_discard ||
block.kind & block_kind_uses_discard_if) &&
if ((block.kind & block_kind_discard || block.kind & block_kind_uses_discard_if) &&
ever_again_needs & WQM)
exec_ctx.info[i].block_needs |= Preserve_WQM;
ever_again_needs |= exec_ctx.info[i].block_needs & ~Exact_Branch;
if (block.kind & block_kind_discard ||
block.kind & block_kind_uses_discard_if ||
if (block.kind & block_kind_discard || block.kind & block_kind_uses_discard_if ||
block.kind & block_kind_uses_demote)
ever_again_needs |= Exact;
@@ -327,7 +335,8 @@ void calculate_wqm_needs(exec_ctx& exec_ctx)
exec_ctx.handle_wqm = true;
}
Operand get_exec_op(Operand t)
Operand
get_exec_op(Operand t)
{
if (t.isUndefined())
return Operand(exec, t.regClass());
@@ -335,7 +344,8 @@ Operand get_exec_op(Operand t)
return t;
}
void transition_to_WQM(exec_ctx& ctx, Builder bld, unsigned idx)
void
transition_to_WQM(exec_ctx& ctx, Builder bld, unsigned idx)
{
if (ctx.info[idx].exec.back().second & mask_type_wqm)
return;
@@ -346,7 +356,8 @@ void transition_to_WQM(exec_ctx& ctx, Builder bld, unsigned idx)
ctx.info[idx].exec.back().first = exec_mask;
}
exec_mask = bld.sop1(Builder::s_wqm, Definition(exec, bld.lm), bld.def(s1, scc), get_exec_op(exec_mask));
exec_mask = bld.sop1(Builder::s_wqm, Definition(exec, bld.lm), bld.def(s1, scc),
get_exec_op(exec_mask));
ctx.info[idx].exec.emplace_back(exec_mask, mask_type_global | mask_type_wqm);
return;
}
@@ -355,11 +366,12 @@ void transition_to_WQM(exec_ctx& ctx, Builder bld, unsigned idx)
assert(ctx.info[idx].exec.back().second & mask_type_wqm);
assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
assert(ctx.info[idx].exec.back().first.isTemp());
ctx.info[idx].exec.back().first = bld.pseudo(aco_opcode::p_parallelcopy, Definition(exec, bld.lm),
ctx.info[idx].exec.back().first);
ctx.info[idx].exec.back().first = bld.pseudo(
aco_opcode::p_parallelcopy, Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
}
void transition_to_Exact(exec_ctx& ctx, Builder bld, unsigned idx)
void
transition_to_Exact(exec_ctx& ctx, Builder bld, unsigned idx)
{
if (ctx.info[idx].exec.back().second & mask_type_exact)
return;
@@ -372,8 +384,8 @@ void transition_to_Exact(exec_ctx& ctx, Builder bld, unsigned idx)
assert(ctx.info[idx].exec.back().second & mask_type_exact);
assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
assert(ctx.info[idx].exec.back().first.isTemp());
ctx.info[idx].exec.back().first = bld.pseudo(aco_opcode::p_parallelcopy, Definition(exec, bld.lm),
ctx.info[idx].exec.back().first);
ctx.info[idx].exec.back().first = bld.pseudo(
aco_opcode::p_parallelcopy, Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
return;
}
/* otherwise, we create an exact mask and push to the stack */
@@ -382,14 +394,15 @@ void transition_to_Exact(exec_ctx& ctx, Builder bld, unsigned idx)
wqm = bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.def(s1, scc),
Definition(exec, bld.lm), ctx.info[idx].exec[0].first, Operand(exec, bld.lm));
} else {
bld.sop2(Builder::s_and, Definition(exec, bld.lm), bld.def(s1, scc), ctx.info[idx].exec[0].first, wqm);
bld.sop2(Builder::s_and, Definition(exec, bld.lm), bld.def(s1, scc),
ctx.info[idx].exec[0].first, wqm);
}
ctx.info[idx].exec.back().first = Operand(wqm);
ctx.info[idx].exec.emplace_back(Operand(bld.lm), mask_type_exact);
}
unsigned add_coupling_code(exec_ctx& ctx, Block* block,
std::vector<aco_ptr<Instruction>>& instructions)
unsigned
add_coupling_code(exec_ctx& ctx, Block* block, std::vector<aco_ptr<Instruction>>& instructions)
{
unsigned idx = block->index;
Builder bld(ctx.program, &instructions);
@@ -417,7 +430,8 @@ unsigned add_coupling_code(exec_ctx& ctx, Block* block,
} else {
uint8_t mask = mask_type_global;
if (ctx.program->needs_wqm) {
bld.sop1(Builder::s_wqm, Definition(exec, bld.lm), bld.def(s1, scc), Operand(exec, bld.lm));
bld.sop1(Builder::s_wqm, Definition(exec, bld.lm), bld.def(s1, scc),
Operand(exec, bld.lm));
mask |= mask_type_wqm;
} else {
mask |= mask_type_exact;
@@ -440,7 +454,8 @@ unsigned add_coupling_code(exec_ctx& ctx, Block* block,
if (info.has_discard) {
aco_ptr<Pseudo_instruction> phi;
for (int i = 0; i < info.num_exec_masks - 1; i++) {
phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1));
phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi,
Format::PSEUDO, preds.size(), 1));
phi->definitions[0] = bld.def(bld.lm);
phi->operands[0] = get_exec_op(ctx.info[preds[0]].exec[i].first);
ctx.info[idx].exec[i].first = bld.insert(std::move(phi));
@@ -450,14 +465,16 @@ unsigned add_coupling_code(exec_ctx& ctx, Block* block,
/* create ssa name for restore mask */
if (info.has_divergent_break) {
/* this phi might be trivial but ensures a parallelcopy on the loop header */
aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(
aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
phi->definitions[0] = bld.def(bld.lm);
phi->operands[0] = get_exec_op(ctx.info[preds[0]].exec[info.num_exec_masks - 1].first);
ctx.info[idx].exec.back().first = bld.insert(std::move(phi));
}
/* create ssa name for loop active mask */
aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(
aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
if (info.has_divergent_continue)
phi->definitions[0] = bld.def(bld.lm);
else
@@ -466,7 +483,8 @@ unsigned add_coupling_code(exec_ctx& ctx, Block* block,
Temp loop_active = bld.insert(std::move(phi));
if (info.has_divergent_break) {
uint8_t mask_type = (ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact)) | mask_type_loop;
uint8_t mask_type =
(ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact)) | mask_type_loop;
ctx.info[idx].exec.emplace_back(loop_active, mask_type);
} else {
ctx.info[idx].exec.back().first = Operand(loop_active);
@@ -482,8 +500,10 @@ unsigned add_coupling_code(exec_ctx& ctx, Block* block,
}
uint8_t mask_type = ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact);
assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
ctx.info[idx].exec.emplace_back(bld.pseudo(aco_opcode::p_parallelcopy, Definition(exec, bld.lm),
ctx.info[idx].exec.back().first), mask_type);
ctx.info[idx].exec.emplace_back(
bld.pseudo(aco_opcode::p_parallelcopy, Definition(exec, bld.lm),
ctx.info[idx].exec.back().first),
mask_type);
}
return i;
@@ -514,14 +534,16 @@ unsigned add_coupling_code(exec_ctx& ctx, Block* block,
aco_ptr<Instruction>& phi = header->instructions[instr_idx++];
assert(phi->opcode == aco_opcode::p_linear_phi);
for (unsigned i = 1; i < phi->operands.size(); i++)
phi->operands[i] = get_exec_op(ctx.info[header_preds[i]].exec[info.num_exec_masks - 1].first);
phi->operands[i] =
get_exec_op(ctx.info[header_preds[i]].exec[info.num_exec_masks - 1].first);
}
if (info.has_divergent_break) {
aco_ptr<Instruction>& phi = header->instructions[instr_idx];
assert(phi->opcode == aco_opcode::p_linear_phi);
for (unsigned i = 1; i < phi->operands.size(); i++)
phi->operands[i] = get_exec_op(ctx.info[header_preds[i]].exec[info.num_exec_masks].first);
phi->operands[i] =
get_exec_op(ctx.info[header_preds[i]].exec[info.num_exec_masks].first);
}
assert(!(block->kind & block_kind_top_level) || info.num_exec_masks <= 2);
@@ -541,7 +563,8 @@ unsigned add_coupling_code(exec_ctx& ctx, Block* block,
ctx.info[idx].exec.emplace_back(same, type);
} else {
/* create phi for loop footer */
aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(
aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
phi->definitions[0] = bld.def(bld.lm);
if (exec_idx == info.num_exec_masks - 1u) {
phi->definitions[0] = Definition(exec, bld.lm);
@@ -578,8 +601,8 @@ unsigned add_coupling_code(exec_ctx& ctx, Block* block,
assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
if (get_exec_op(ctx.info[idx].exec.back().first).isTemp()) {
/* move current exec mask into exec register */
ctx.info[idx].exec.back().first = bld.pseudo(aco_opcode::p_parallelcopy, Definition(exec, bld.lm),
ctx.info[idx].exec.back().first);
ctx.info[idx].exec.back().first = bld.pseudo(
aco_opcode::p_parallelcopy, Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
}
ctx.loop.pop_back();
@@ -591,8 +614,8 @@ unsigned add_coupling_code(exec_ctx& ctx, Block* block,
} else {
assert(preds.size() == 2);
/* if one of the predecessors ends in exact mask, we pop it from stack */
unsigned num_exec_masks = std::min(ctx.info[preds[0]].exec.size(),
ctx.info[preds[1]].exec.size());
unsigned num_exec_masks =
std::min(ctx.info[preds[0]].exec.size(), ctx.info[preds[1]].exec.size());
if (block->kind & block_kind_merge)
num_exec_masks--;
@@ -605,14 +628,16 @@ unsigned add_coupling_code(exec_ctx& ctx, Block* block,
if (ctx.info[preds[0]].exec[i].first == ctx.info[preds[1]].exec[i].first) {
Operand t = ctx.info[preds[0]].exec[i].first;
/* discard/demote can change the state of the current exec mask */
assert(!t.isTemp() || ctx.info[preds[0]].exec[i].second == ctx.info[preds[1]].exec[i].second);
assert(!t.isTemp() ||
ctx.info[preds[0]].exec[i].second == ctx.info[preds[1]].exec[i].second);
uint8_t mask = ctx.info[preds[0]].exec[i].second & ctx.info[preds[1]].exec[i].second;
ctx.info[idx].exec.emplace_back(t, mask);
continue;
}
bool in_exec = i == num_exec_masks - 1 && !(block->kind & block_kind_merge);
Temp phi = bld.pseudo(aco_opcode::p_linear_phi, in_exec ? Definition(exec, bld.lm) : bld.def(bld.lm),
Temp phi = bld.pseudo(aco_opcode::p_linear_phi,
in_exec ? Definition(exec, bld.lm) : bld.def(bld.lm),
get_exec_op(ctx.info[preds[0]].exec[i].first),
get_exec_op(ctx.info[preds[1]].exec[i].first));
uint8_t mask_type = ctx.info[preds[0]].exec[i].second & ctx.info[preds[1]].exec[i].second;
@@ -654,9 +679,9 @@ unsigned add_coupling_code(exec_ctx& ctx, Block* block,
return i;
}
void process_instructions(exec_ctx& ctx, Block* block,
std::vector<aco_ptr<Instruction>>& instructions,
unsigned idx)
void
process_instructions(exec_ctx& ctx, Block* block, std::vector<aco_ptr<Instruction>>& instructions,
unsigned idx)
{
WQMState state;
if (ctx.info[block->index].exec.back().second & mask_type_wqm)
@@ -667,17 +692,16 @@ void process_instructions(exec_ctx& ctx, Block* block,
}
/* if the block doesn't need both, WQM and Exact, we can skip processing the instructions */
bool process = (ctx.handle_wqm &&
(ctx.info[block->index].block_needs & state) !=
(ctx.info[block->index].block_needs & (WQM | Exact))) ||
bool process = (ctx.handle_wqm && (ctx.info[block->index].block_needs & state) !=
(ctx.info[block->index].block_needs & (WQM | Exact))) ||
block->kind & block_kind_uses_discard_if ||
block->kind & block_kind_uses_demote ||
block->kind & block_kind_needs_lowering;
block->kind & block_kind_uses_demote || block->kind & block_kind_needs_lowering;
if (!process) {
std::vector<aco_ptr<Instruction>>::iterator it = std::next(block->instructions.begin(), idx);
instructions.insert(instructions.end(),
std::move_iterator<std::vector<aco_ptr<Instruction>>::iterator>(it),
std::move_iterator<std::vector<aco_ptr<Instruction>>::iterator>(block->instructions.end()));
std::move_iterator<std::vector<aco_ptr<Instruction>>::iterator>(
block->instructions.end()));
return;
}
@@ -700,11 +724,13 @@ void process_instructions(exec_ctx& ctx, Block* block,
/* discard from current exec */
const Operand cond = instr->operands[0];
Temp exit_cond = bld.sop2(Builder::s_andn2, Definition(exec, bld.lm), bld.def(s1, scc),
Operand(exec, bld.lm), cond).def(1).getTemp();
Operand(exec, bld.lm), cond)
.def(1)
.getTemp();
/* discard from inner to outer exec mask on stack */
for (int i = num - 2; i >= 0; i--) {
Instruction *andn2 = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc),
Instruction* andn2 = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc),
ctx.info[block->index].exec[i].first, cond);
ctx.info[block->index].exec[i].first = Operand(andn2->definitions[0].getTemp());
exit_cond = andn2->definitions[1].getTemp();
@@ -726,14 +752,16 @@ void process_instructions(exec_ctx& ctx, Block* block,
Definition dst = instr->definitions[0];
assert(dst.size() == bld.lm.size());
if (state == Exact) {
instr.reset(create_instruction<SOP1_instruction>(bld.w64or32(Builder::s_mov), Format::SOP1, 1, 1));
instr.reset(create_instruction<SOP1_instruction>(bld.w64or32(Builder::s_mov),
Format::SOP1, 1, 1));
instr->operands[0] = Operand(0u);
instr->definitions[0] = dst;
} else {
std::pair<Operand, uint8_t>& exact_mask = ctx.info[block->index].exec[0];
assert(exact_mask.second & mask_type_exact);
instr.reset(create_instruction<SOP2_instruction>(bld.w64or32(Builder::s_andn2), Format::SOP2, 2, 2));
instr.reset(create_instruction<SOP2_instruction>(bld.w64or32(Builder::s_andn2),
Format::SOP2, 2, 2));
instr->operands[0] = Operand(exec, bld.lm); /* current exec */
instr->operands[1] = Operand(exact_mask.first);
instr->definitions[0] = dst;
@@ -741,7 +769,8 @@ void process_instructions(exec_ctx& ctx, Block* block,
}
} else if (instr->opcode == aco_opcode::p_demote_to_helper) {
/* turn demote into discard_if with only exact masks */
assert((ctx.info[block->index].exec[0].second & (mask_type_exact | mask_type_global)) == (mask_type_exact | mask_type_global));
assert((ctx.info[block->index].exec[0].second & (mask_type_exact | mask_type_global)) ==
(mask_type_exact | mask_type_global));
int num;
Temp cond, exit_cond;
@@ -749,8 +778,9 @@ void process_instructions(exec_ctx& ctx, Block* block,
assert(instr->operands[0].constantValue() == -1u);
/* transition to exact and set exec to zero */
exit_cond = bld.tmp(s1);
cond = bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.scc(Definition(exit_cond)),
Definition(exec, bld.lm), Operand(0u), Operand(exec, bld.lm));
cond =
bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.scc(Definition(exit_cond)),
Definition(exec, bld.lm), Operand(0u), Operand(exec, bld.lm));
num = ctx.info[block->index].exec.size() - 2;
if (!(ctx.info[block->index].exec.back().second & mask_type_exact)) {
@@ -767,7 +797,7 @@ void process_instructions(exec_ctx& ctx, Block* block,
for (int i = num; i >= 0; i--) {
if (ctx.info[block->index].exec[i].second & mask_type_exact) {
Instruction *andn2 = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc),
Instruction* andn2 = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc),
ctx.info[block->index].exec[i].first, cond);
if (i == (int)ctx.info[block->index].exec.size() - 1) {
andn2->operands[0] = Operand(exec, bld.lm);
@@ -783,14 +813,14 @@ void process_instructions(exec_ctx& ctx, Block* block,
instr->opcode = aco_opcode::p_exit_early_if;
instr->operands[0] = bld.scc(exit_cond);
state = Exact;
}
bld.insert(std::move(instr));
}
}
void add_branch_code(exec_ctx& ctx, Block* block)
void
add_branch_code(exec_ctx& ctx, Block* block)
{
unsigned idx = block->index;
Builder bld(ctx.program, block);
@@ -806,8 +836,7 @@ void add_branch_code(exec_ctx& ctx, Block* block)
}
assert(ctx.info[idx].exec.size() <= 2);
if (ctx.info[idx].ever_again_needs == 0 ||
ctx.info[idx].ever_again_needs == Exact) {
if (ctx.info[idx].ever_again_needs == 0 || ctx.info[idx].ever_again_needs == Exact) {
/* transition to Exact */
aco_ptr<Instruction> branch = std::move(block->instructions.back());
block->instructions.pop_back();
@@ -838,8 +867,7 @@ void add_branch_code(exec_ctx& ctx, Block* block)
Block& loop_block = ctx.program->blocks[i];
needs |= ctx.info[i].block_needs;
if (loop_block.kind & block_kind_uses_discard_if ||
loop_block.kind & block_kind_discard ||
if (loop_block.kind & block_kind_uses_discard_if || loop_block.kind & block_kind_discard ||
loop_block.kind & block_kind_uses_demote)
has_discard = true;
if (loop_block.loop_nest_depth != loop_nest_depth)
@@ -871,12 +899,8 @@ void add_branch_code(exec_ctx& ctx, Block* block)
if (block->kind & block_kind_top_level)
num_exec_masks = std::min(num_exec_masks, 2u);
ctx.loop.emplace_back(&ctx.program->blocks[block->linear_succs[0]],
num_exec_masks,
needs,
has_divergent_break,
has_divergent_continue,
has_discard);
ctx.loop.emplace_back(&ctx.program->blocks[block->linear_succs[0]], num_exec_masks, needs,
has_divergent_break, has_divergent_continue, has_discard);
}
/* For normal breaks, this is the exec mask. For discard+break, it's the
@@ -903,7 +927,7 @@ void add_branch_code(exec_ctx& ctx, Block* block)
Definition(exec, bld.lm), Operand(0u), Operand(exec, bld.lm));
for (int i = num - 1; i >= 0; i--) {
Instruction *andn2 = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc),
Instruction* andn2 = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc),
get_exec_op(ctx.info[block->index].exec[i].first), cond);
if (i == (int)ctx.info[idx].exec.size() - 1)
andn2->definitions[0] = Definition(exec, bld.lm);
@@ -919,8 +943,10 @@ void add_branch_code(exec_ctx& ctx, Block* block)
}
if (block->kind & block_kind_continue_or_break) {
assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[1]].linear_succs[0]].kind & block_kind_loop_header);
assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[0]].linear_succs[0]].kind & block_kind_loop_exit);
assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[1]].linear_succs[0]].kind &
block_kind_loop_header);
assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[0]].linear_succs[0]].kind &
block_kind_loop_exit);
assert(block->instructions.back()->opcode == aco_opcode::p_branch);
block->instructions.pop_back();
@@ -931,8 +957,10 @@ void add_branch_code(exec_ctx& ctx, Block* block)
}
if (need_parallelcopy)
ctx.info[idx].exec.back().first = bld.pseudo(aco_opcode::p_parallelcopy, Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
bld.branch(aco_opcode::p_cbranch_nz, bld.hint_vcc(bld.def(s2)), Operand(exec, bld.lm), block->linear_succs[1], block->linear_succs[0]);
ctx.info[idx].exec.back().first = bld.pseudo(
aco_opcode::p_parallelcopy, Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
bld.branch(aco_opcode::p_cbranch_nz, bld.hint_vcc(bld.def(s2)), Operand(exec, bld.lm),
block->linear_succs[1], block->linear_succs[0]);
return;
}
@@ -949,8 +977,7 @@ void add_branch_code(exec_ctx& ctx, Block* block)
if (block->kind & block_kind_branch) {
if (ctx.handle_wqm &&
ctx.info[idx].exec.size() >= 2 &&
if (ctx.handle_wqm && ctx.info[idx].exec.size() >= 2 &&
ctx.info[idx].exec.back().second == mask_type_exact &&
!(ctx.info[idx].block_needs & Exact_Branch) &&
ctx.info[idx].exec[ctx.info[idx].exec.size() - 2].second & mask_type_wqm) {
@@ -972,7 +999,7 @@ void add_branch_code(exec_ctx& ctx, Block* block)
bld.pseudo(aco_opcode::p_parallelcopy, Definition(exec, bld.lm), cond);
} else {
Temp old_exec = bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.def(s1, scc),
Definition(exec, bld.lm), cond, Operand(exec, bld.lm));
Definition(exec, bld.lm), cond, Operand(exec, bld.lm));
ctx.info[idx].exec.back().first = Operand(old_exec);
}
@@ -980,7 +1007,8 @@ void add_branch_code(exec_ctx& ctx, Block* block)
/* add next current exec to the stack */
ctx.info[idx].exec.emplace_back(Operand(bld.lm), mask_type);
bld.branch(aco_opcode::p_cbranch_z, bld.hint_vcc(bld.def(s2)), Operand(exec, bld.lm), block->linear_succs[1], block->linear_succs[0]);
bld.branch(aco_opcode::p_cbranch_z, bld.hint_vcc(bld.def(s2)), Operand(exec, bld.lm),
block->linear_succs[1], block->linear_succs[0]);
return;
}
@@ -990,9 +1018,11 @@ void add_branch_code(exec_ctx& ctx, Block* block)
block->instructions.pop_back();
assert(ctx.info[idx].exec.size() >= 2);
Operand orig_exec = ctx.info[idx].exec[ctx.info[idx].exec.size() - 2].first;
bld.sop2(Builder::s_andn2, Definition(exec, bld.lm), bld.def(s1, scc), orig_exec, Operand(exec, bld.lm));
bld.sop2(Builder::s_andn2, Definition(exec, bld.lm), bld.def(s1, scc), orig_exec,
Operand(exec, bld.lm));
bld.branch(aco_opcode::p_cbranch_z, bld.hint_vcc(bld.def(s2)), Operand(exec, bld.lm), block->linear_succs[1], block->linear_succs[0]);
bld.branch(aco_opcode::p_cbranch_z, bld.hint_vcc(bld.def(s2)), Operand(exec, bld.lm),
block->linear_succs[1], block->linear_succs[0]);
return;
}
@@ -1020,7 +1050,8 @@ void add_branch_code(exec_ctx& ctx, Block* block)
bld.copy(Definition(exec, bld.lm), Operand(0u, bld.lm == s2));
}
bld.branch(aco_opcode::p_cbranch_nz, bld.hint_vcc(bld.def(s2)), bld.scc(cond), block->linear_succs[1], block->linear_succs[0]);
bld.branch(aco_opcode::p_cbranch_nz, bld.hint_vcc(bld.def(s2)), bld.scc(cond),
block->linear_succs[1], block->linear_succs[0]);
return;
}
@@ -1048,12 +1079,14 @@ void add_branch_code(exec_ctx& ctx, Block* block)
bld.copy(Definition(exec, bld.lm), Operand(0u, bld.lm == s2));
}
bld.branch(aco_opcode::p_cbranch_nz, bld.hint_vcc(bld.def(s2)), bld.scc(cond), block->linear_succs[1], block->linear_succs[0]);
bld.branch(aco_opcode::p_cbranch_nz, bld.hint_vcc(bld.def(s2)), bld.scc(cond),
block->linear_succs[1], block->linear_succs[0]);
return;
}
}
void process_block(exec_ctx& ctx, Block* block)
void
process_block(exec_ctx& ctx, Block* block)
{
std::vector<aco_ptr<Instruction>> instructions;
instructions.reserve(block->instructions.size());
@@ -1072,8 +1105,8 @@ void process_block(exec_ctx& ctx, Block* block)
} /* end namespace */
void insert_exec_mask(Program *program)
void
insert_exec_mask(Program* program)
{
exec_ctx ctx(program);
@@ -1082,8 +1115,6 @@ void insert_exec_mask(Program *program)
for (Block& block : program->blocks)
process_block(ctx, &block);
}
}
} // namespace aco