diff --git a/src/freedreno/ir3/ir3.h b/src/freedreno/ir3/ir3.h index 5cdf85d2970..0c3b54d7ba9 100644 --- a/src/freedreno/ir3/ir3.h +++ b/src/freedreno/ir3/ir3.h @@ -1632,8 +1632,6 @@ void ir3_print_instr_stream(struct log_stream *stream, struct ir3_instruction *i /* delay calculation: */ int ir3_delayslots(struct ir3_instruction *assigner, struct ir3_instruction *consumer, unsigned n, bool soft); -unsigned ir3_delay_calc_prera(struct ir3_block *block, - struct ir3_instruction *instr); unsigned ir3_delay_calc_postra(struct ir3_block *block, struct ir3_instruction *instr, bool soft, bool mergedregs); diff --git a/src/freedreno/ir3/ir3_delay.c b/src/freedreno/ir3/ir3_delay.c index d06347f85c6..8a5a57dc756 100644 --- a/src/freedreno/ir3/ir3_delay.c +++ b/src/freedreno/ir3/ir3_delay.c @@ -119,74 +119,6 @@ count_instruction(struct ir3_instruction *n) (is_flow(n) && (n->opc != OPC_JUMP) && (n->opc != OPC_B)); } -static unsigned -distance(struct ir3_block *block, struct ir3_instruction *instr, unsigned maxd) -{ - unsigned d = 0; - - /* Note that this relies on incrementally building up the block's - * instruction list.. but this is how scheduling and nopsched - * work. - */ - foreach_instr_rev (n, &block->instr_list) { - if ((n == instr) || (d >= maxd)) - return MIN2(maxd, d + n->nop); - if (count_instruction(n)) - d = MIN2(maxd, d + 1 + n->repeat + n->nop); - } - - return maxd; -} - -static unsigned -delay_calc_srcn_prera(struct ir3_block *block, struct ir3_instruction *assigner, - struct ir3_instruction *consumer, unsigned srcn) -{ - unsigned delay = 0; - - if (assigner->opc == OPC_META_PHI) - return 0; - - if (is_meta(assigner)) { - foreach_src_n (src, n, assigner) { - unsigned d; - - if (!src->def) - continue; - - d = delay_calc_srcn_prera(block, src->def->instr, consumer, srcn); - delay = MAX2(delay, d); - } - } else { - delay = ir3_delayslots(assigner, consumer, srcn, false); - delay -= distance(block, assigner, delay); - } - - return delay; -} - -/** - * Calculate delay for instruction before register allocation, using SSA - * source pointers. This can't handle inter-block dependencies. - */ -unsigned -ir3_delay_calc_prera(struct ir3_block *block, struct ir3_instruction *instr) -{ - unsigned delay = 0; - - foreach_src_n (src, i, instr) { - unsigned d = 0; - - if (src->def && src->def->instr->block == block) { - d = delay_calc_srcn_prera(block, src->def->instr, instr, i); - } - - delay = MAX2(delay, d); - } - - return delay; -} - /* Post-RA, we don't have arrays any more, so we have to be a bit careful here * and have to handle relative accesses specially. */ diff --git a/src/freedreno/ir3/ir3_sched.c b/src/freedreno/ir3/ir3_sched.c index 6be356d17b0..a3ad5b93b7a 100644 --- a/src/freedreno/ir3/ir3_sched.c +++ b/src/freedreno/ir3/ir3_sched.c @@ -106,6 +106,8 @@ struct ir3_sched_ctx { bool error; + unsigned ip; + int sfu_delay; int tex_delay; @@ -128,6 +130,11 @@ struct ir3_sched_node { unsigned tex_index; unsigned sfu_index; + /* For ready instructions, the earliest possible ip that it could be + * scheduled. + */ + unsigned earliest_ip; + /* For instructions that are a meta:collect src, once we schedule * the first src of the collect, the entire vecN is live (at least * from the PoV of the first RA pass.. the 2nd scalar pass can fill @@ -302,6 +309,23 @@ schedule(struct ir3_sched_ctx *ctx, struct ir3_instruction *instr) } } + bool counts_for_delay = is_alu(instr) || is_flow(instr); + + /* TODO: switch to "cycles". For now try to match ir3_delay. */ + unsigned delay_cycles = counts_for_delay ? 1 + instr->repeat : 0; + + /* We insert any nop's needed to get to earliest_ip, then advance + * delay_cycles by scheduling the instruction. + */ + ctx->ip = MAX2(ctx->ip, n->earliest_ip) + delay_cycles; + + util_dynarray_foreach (&n->dag.edges, struct dag_edge, edge) { + unsigned delay = (unsigned)(uintptr_t)edge->data; + struct ir3_sched_node *child = + container_of(edge->child, struct ir3_sched_node, dag); + child->earliest_ip = MAX2(child->earliest_ip, ctx->ip + delay); + } + dag_prune_head(ctx->dag, &n->dag); unsigned cycles = cycle_count(instr); @@ -335,6 +359,7 @@ schedule(struct ir3_sched_ctx *ctx, struct ir3_instruction *instr) } else if (ctx->tex_delay > 0) { ctx->tex_delay -= MIN2(cycles, ctx->tex_delay); } + } struct ir3_sched_notes { @@ -619,6 +644,12 @@ dec_rank_name(enum choose_instr_dec_rank rank) } } +static unsigned +node_delay(struct ir3_sched_ctx *ctx, struct ir3_sched_node *n) +{ + return MAX2(n->earliest_ip, ctx->ip) - ctx->ip; +} + /** * Chooses an instruction to schedule using the Goodman/Hsu (1988) CSR (Code * Scheduling for Register pressure) heuristic. @@ -638,8 +669,7 @@ choose_instr_dec(struct ir3_sched_ctx *ctx, struct ir3_sched_notes *notes, if (defer && should_defer(ctx, n->instr)) continue; - /* Note: mergedregs is only used post-RA, just set it to false */ - unsigned d = ir3_delay_calc_prera(ctx->block, n->instr); + unsigned d = node_delay(ctx, n); int live = live_effect(n->instr); if (live > 0) @@ -737,7 +767,7 @@ choose_instr_inc(struct ir3_sched_ctx *ctx, struct ir3_sched_notes *notes, if (!check_instr(ctx, notes, n->instr)) continue; - unsigned d = ir3_delay_calc_prera(ctx->block, n->instr); + unsigned d = node_delay(ctx, n); enum choose_instr_inc_rank rank; if (d == 0) @@ -804,7 +834,7 @@ dump_state(struct ir3_sched_ctx *ctx) foreach_sched_node (n, &ctx->dag->heads) { di(n->instr, "maxdel=%3d le=%d del=%u ", n->max_delay, - live_effect(n->instr), ir3_delay_calc_prera(ctx->block, n->instr)); + live_effect(n->instr), node_delay(ctx, n)); util_dynarray_foreach (&n->dag.edges, struct dag_edge, edge) { struct ir3_sched_node *child = (struct ir3_sched_node *)edge->child; @@ -994,11 +1024,17 @@ sched_node_add_dep(struct ir3_instruction *instr, struct ir3_instruction *src, if (instr->opc == OPC_META_COLLECT) sn->collect = instr; - dag_add_edge(&sn->dag, &n->dag, 0); + unsigned d_soft = ir3_delayslots(src, instr, i, true); + unsigned d = ir3_delayslots(src, instr, i, false); - unsigned d = ir3_delayslots(src, instr, i, true); + /* delays from (ss) and (sy) are considered separately and more accurately in + * the scheduling heuristic, so ignore it when calculating the ip of + * instructions, but do consider it when prioritizing which instructions to + * schedule. + */ + dag_add_edge_max_data(&sn->dag, &n->dag, (uintptr_t)d); - n->delay = MAX2(n->delay, d); + n->delay = MAX2(n->delay, d_soft); } static void @@ -1172,7 +1208,7 @@ sched_block(struct ir3_sched_ctx *ctx, struct ir3_block *block) instr = choose_instr(ctx, ¬es); if (instr) { - unsigned delay = ir3_delay_calc_prera(ctx->block, instr); + unsigned delay = node_delay(ctx, instr->data); d("delay=%u", delay); /* and if we run out of instructions that can be scheduled,