diff --git a/src/panfrost/bifrost/valhall/test/test-packing.cpp b/src/panfrost/bifrost/valhall/test/test-packing.cpp index ba597ddf7ea..5cb20932f48 100644 --- a/src/panfrost/bifrost/valhall/test/test-packing.cpp +++ b/src/panfrost/bifrost/valhall/test/test-packing.cpp @@ -28,7 +28,7 @@ #include #define CASE(instr, expected) do { \ - uint64_t _value = va_pack_instr(instr, 0); \ + uint64_t _value = va_pack_instr(instr); \ if (_value != expected) { \ fprintf(stderr, "Got %" PRIx64 ", expected %" PRIx64 "\n", _value, (uint64_t) expected); \ bi_print_instr(instr, stderr); \ diff --git a/src/panfrost/bifrost/valhall/va_compiler.h b/src/panfrost/bifrost/valhall/va_compiler.h index b346561b875..7605a9da9eb 100644 --- a/src/panfrost/bifrost/valhall/va_compiler.h +++ b/src/panfrost/bifrost/valhall/va_compiler.h @@ -40,7 +40,7 @@ void va_repair_fau(bi_builder *b, bi_instr *I); void va_fuse_add_imm(bi_instr *I); void va_lower_constants(bi_context *ctx, bi_instr *I); void va_lower_isel(bi_instr *I); -uint64_t va_pack_instr(const bi_instr *I, unsigned flow); +uint64_t va_pack_instr(const bi_instr *I); static inline unsigned va_fau_page(enum bir_fau value) diff --git a/src/panfrost/bifrost/valhall/va_pack.c b/src/panfrost/bifrost/valhall/va_pack.c index 553bff7d318..3df70295190 100644 --- a/src/panfrost/bifrost/valhall/va_pack.c +++ b/src/panfrost/bifrost/valhall/va_pack.c @@ -609,11 +609,11 @@ va_pack_register_format(const bi_instr *I) } uint64_t -va_pack_instr(const bi_instr *I, unsigned flow) +va_pack_instr(const bi_instr *I) { struct va_opcode_info info = valhall_opcodes[I->op]; - uint64_t hex = info.exact | (((uint64_t) flow) << 59); + uint64_t hex = info.exact | (((uint64_t) I->flow) << 59); hex |= ((uint64_t) va_select_fau_page(I)) << 57; if (info.slot) { @@ -816,28 +816,6 @@ va_should_return(bi_block *block, bi_instr *I) return true; } -static enum va_flow -va_pack_flow(bi_block *block, bi_instr *I) -{ - if (va_should_return(block, I)) - return VA_FLOW_END; - - if (va_last_in_block(block, I) && bi_reconverge_branches(block)) - return VA_FLOW_RECONVERGE; - - if (I->op == BI_OPCODE_BARRIER) - return VA_FLOW_WAIT; - - if (I->flow) - return I->flow; - - /* TODO: Generalize waits */ - if (valhall_opcodes[I->op].nr_staging_dests > 0 || I->op == BI_OPCODE_BLEND) - return VA_FLOW_WAIT0; - - return VA_FLOW_NONE; -} - static unsigned va_instructions_in_block(bi_block *block) { @@ -958,6 +936,59 @@ va_lower_blend(bi_context *ctx) } } +/* + * Add a flow control modifier to an instruction. There may be an existing flow + * control modifier; if so, we need to add a NOP with the extra flow control + * _after_ this instruction + */ +static void +va_add_flow(bi_context *ctx, bi_instr *I, enum va_flow flow) +{ + if (I->flow != VA_FLOW_NONE) { + bi_builder b = bi_init_builder(ctx, bi_after_instr(I)); + I = bi_nop(&b); + } + + I->flow = flow; +} + +/* + * Add flow control modifiers to the program. This is a stop gap until we have a + * proper scheduler. For now, this should be conformant while doing little + * optimization of message waits. + */ +static void +va_lower_flow_control(bi_context *ctx) +{ + bi_foreach_block(ctx, block) { + bool block_reconverges = bi_reconverge_branches(block); + + bi_foreach_instr_in_block_safe(block, I) { + /* If this instruction returns, there is nothing left to do. */ + if (va_should_return(block, I)) { + I->flow = VA_FLOW_END; + continue; + } + + /* We may need to wait */ + if (I->op == BI_OPCODE_BARRIER) + va_add_flow(ctx, I, VA_FLOW_WAIT); + else if (valhall_opcodes[I->op].nr_staging_dests > 0 || I->op == BI_OPCODE_BLEND) + va_add_flow(ctx, I, VA_FLOW_WAIT0); + + /* Lastly, we may need to reconverge. If we need reconvergence, it + * has to be on the last instruction of the block. If we have to + * generate a NOP for that reconverge, we need that to be last. So + * this ordering is careful. + */ + if (va_last_in_block(block, I) && block_reconverges) + va_add_flow(ctx, I, VA_FLOW_RECONVERGE); + + + } + } +} + void bi_pack_valhall(bi_context *ctx, struct util_dynarray *emission) { @@ -969,13 +1000,14 @@ bi_pack_valhall(bi_context *ctx, struct util_dynarray *emission) if (ctx->stage == MESA_SHADER_FRAGMENT && !ctx->inputs->is_blend) va_lower_blend(ctx); + va_lower_flow_control(ctx); + bi_foreach_block(ctx, block) { bi_foreach_instr_in_block(block, I) { if (I->op == BI_OPCODE_BRANCHZ_I16) va_lower_branch_target(ctx, block, I); - unsigned flow = va_pack_flow(block, I); - uint64_t hex = va_pack_instr(I, flow); + uint64_t hex = va_pack_instr(I); util_dynarray_append(emission, uint64_t, hex); } }