diff --git a/src/.clang-format b/src/.clang-format index 142700a493c..63eb3802324 100644 --- a/src/.clang-format +++ b/src/.clang-format @@ -275,6 +275,9 @@ ForEachMacros: - perf_time_ctx - foreach_submit - foreach_submit_safe + - foreach_instr_rpt + - foreach_instr_rpt_excl + - foreach_instr_rpt_excl_safe # panfrost - foreach_batch diff --git a/src/freedreno/ir3/ir3.c b/src/freedreno/ir3/ir3.c index 0e59c341a35..a832234480f 100644 --- a/src/freedreno/ir3/ir3.c +++ b/src/freedreno/ir3/ir3.c @@ -646,6 +646,7 @@ instr_create(struct ir3_block *block, opc_t opc, int ndst, int nsrc) instr->srcs_max = nsrc; #endif + list_inithead(&instr->rpt_node); return instr; } @@ -725,6 +726,7 @@ ir3_instr_clone(struct ir3_instruction *instr) *new_instr = *instr; new_instr->dsts = dsts; new_instr->srcs = srcs; + list_inithead(&new_instr->rpt_node); insert_instr(ir3_before_terminator(instr->block), new_instr); @@ -765,6 +767,74 @@ ir3_instr_add_dep(struct ir3_instruction *instr, struct ir3_instruction *dep) array_insert(instr, instr->deps, dep); } +void +ir3_instr_remove(struct ir3_instruction *instr) +{ + list_delinit(&instr->node); + list_delinit(&instr->rpt_node); +} + +void +ir3_instr_create_rpt(struct ir3_instruction **instrs, unsigned n) +{ + assert(n > 0 && !ir3_instr_is_rpt(instrs[0])); + + for (unsigned i = 1; i < n; ++i) { + assert(!ir3_instr_is_rpt(instrs[i])); + assert(instrs[i]->serialno > instrs[i - 1]->serialno); + + list_addtail(&instrs[i]->rpt_node, &instrs[0]->rpt_node); + } +} + +bool +ir3_instr_is_rpt(const struct ir3_instruction *instr) +{ + return !list_is_empty(&instr->rpt_node); +} + +bool +ir3_instr_is_first_rpt(const struct ir3_instruction *instr) +{ + if (!ir3_instr_is_rpt(instr)) + return false; + + struct ir3_instruction *prev_rpt = + list_entry(instr->rpt_node.prev, struct ir3_instruction, rpt_node); + return prev_rpt->serialno > instr->serialno; +} + +struct ir3_instruction * +ir3_instr_prev_rpt(const struct ir3_instruction *instr) +{ + assert(ir3_instr_is_rpt(instr)); + + if (ir3_instr_is_first_rpt(instr)) + return NULL; + return list_entry(instr->rpt_node.prev, struct ir3_instruction, rpt_node); +} + +struct ir3_instruction * +ir3_instr_first_rpt(struct ir3_instruction *instr) +{ + assert(ir3_instr_is_rpt(instr)); + + while (!ir3_instr_is_first_rpt(instr)) { + instr = ir3_instr_prev_rpt(instr); + assert(instr); + } + + return instr; +} + +unsigned +ir3_instr_rpt_length(const struct ir3_instruction *instr) +{ + assert(ir3_instr_is_first_rpt(instr)); + + return list_length(&instr->rpt_node) + 1; +} + struct ir3_register * ir3_src_create(struct ir3_instruction *instr, int num, int flags) { @@ -1439,3 +1509,22 @@ ir3_get_cond_for_nonzero_compare(struct ir3_instruction *instr) return instr; } + +bool +ir3_supports_rpt(unsigned opc) +{ + switch (opc_cat(opc)) { + case 0: + return opc == OPC_NOP; + case 1: + return opc == OPC_MOV || opc == OPC_SWZ || opc == OPC_MOVMSK; + case 2: + return true; + case 3: + return opc != OPC_DP2ACC && opc != OPC_DP4ACC; + case 4: + return opc != OPC_RCP; + default: + return false; + } +} diff --git a/src/freedreno/ir3/ir3.h b/src/freedreno/ir3/ir3.h index 16e63d70e0d..7959894d539 100644 --- a/src/freedreno/ir3/ir3.h +++ b/src/freedreno/ir3/ir3.h @@ -554,6 +554,25 @@ struct ir3_instruction { /* Entry in ir3_block's instruction list: */ struct list_head node; + /* List of this instruction's repeat group. Vectorized NIR instructions are + * emitted as multiple scalar instructions that are linked together using + * this field. After RA, the ir3_combine_rpt pass iterates these groups and, + * if the register assignment allows it, merges them into a (rptN) + * instruction. + * + * NOTE: this is not a typical list as there is no empty list head. The list + * head is stored in the first instruction of the repeat group so also refers + * to a list entry. In order to distinguish the list's first entry, we use + * serialno: instructions in a repeat group are always emitted consecutively + * so the first will have the lowest serialno. + * + * As this is not a typical list, we have to be careful with using the + * existing list helper. For example, using list_length on the first + * instruction will yield one less than the number of instructions in its + * group. + */ + struct list_head rpt_node; + uint32_t serialno; // TODO only computerator/assembler: @@ -807,6 +826,14 @@ struct ir3_instruction *ir3_instr_clone(struct ir3_instruction *instr); void ir3_instr_add_dep(struct ir3_instruction *instr, struct ir3_instruction *dep); const char *ir3_instr_name(struct ir3_instruction *instr); +void ir3_instr_remove(struct ir3_instruction *instr); + +void ir3_instr_create_rpt(struct ir3_instruction **instrs, unsigned n); +bool ir3_instr_is_rpt(const struct ir3_instruction *instr); +bool ir3_instr_is_first_rpt(const struct ir3_instruction *instr); +struct ir3_instruction *ir3_instr_prev_rpt(const struct ir3_instruction *instr); +struct ir3_instruction *ir3_instr_first_rpt(struct ir3_instruction *instr); +unsigned ir3_instr_rpt_length(const struct ir3_instruction *instr); struct ir3_register *ir3_src_create(struct ir3_instruction *instr, int num, int flags); @@ -901,6 +928,8 @@ bool ir3_valid_immediate(struct ir3_instruction *instr, int32_t immed); struct ir3_instruction * ir3_get_cond_for_nonzero_compare(struct ir3_instruction *instr); +bool ir3_supports_rpt(unsigned opc); + #include "util/set.h" #define foreach_ssa_use(__use, __instr) \ for (struct ir3_instruction *__use = (void *)~0; __use && (__instr)->uses; \ @@ -1938,6 +1967,26 @@ __ssa_srcp_n(struct ir3_instruction *instr, unsigned n) list_for_each_entry_from_safe(struct ir3_instruction, __instr, __start, \ __list, node) +/* Iterate over all instructions in a repeat group. */ +#define foreach_instr_rpt(__rpt, __instr) \ + if (assert(ir3_instr_is_first_rpt(__instr)), true) \ + for (struct ir3_instruction *__rpt = __instr, *__first = __instr; \ + __first || __rpt != __instr; \ + __first = NULL, __rpt = \ + list_entry(__rpt->rpt_node.next, \ + struct ir3_instruction, rpt_node)) + +/* Iterate over all instructions except the first one in a repeat group. */ +#define foreach_instr_rpt_excl(__rpt, __instr) \ + if (assert(ir3_instr_is_first_rpt(__instr)), true) \ + list_for_each_entry (struct ir3_instruction, __rpt, &__instr->rpt_node, \ + rpt_node) + +#define foreach_instr_rpt_excl_safe(__rpt, __instr) \ + if (assert(ir3_instr_is_first_rpt(__instr)), true) \ + list_for_each_entry_safe (struct ir3_instruction, __rpt, \ + &__instr->rpt_node, rpt_node) + /* iterators for blocks: */ #define foreach_block(__block, __list) \ list_for_each_entry (struct ir3_block, __block, __list, node) diff --git a/src/freedreno/ir3/ir3_dce.c b/src/freedreno/ir3/ir3_dce.c index 456a1c805c4..0dac1d61c85 100644 --- a/src/freedreno/ir3/ir3_dce.c +++ b/src/freedreno/ir3/ir3_dce.c @@ -92,7 +92,7 @@ remove_unused_by_block(struct ir3_block *block) if (*srcp == instr) *srcp = NULL; - list_delinit(&instr->node); + ir3_instr_remove(instr); progress = true; } } diff --git a/src/freedreno/ir3/ir3_print.c b/src/freedreno/ir3/ir3_print.c index b01a7508e46..f9cd81f8eb0 100644 --- a/src/freedreno/ir3/ir3_print.c +++ b/src/freedreno/ir3/ir3_print.c @@ -460,6 +460,17 @@ print_instr(struct log_stream *stream, struct ir3_instruction *instr, int lvl) } } + if (ir3_instr_is_rpt(instr)) { + mesa_log_stream_printf(stream, ", rpt: "); + + if (ir3_instr_is_first_rpt(instr)) { + mesa_log_stream_printf(stream, "first"); + } else { + mesa_log_stream_printf(stream, "%u", + ir3_instr_prev_rpt(instr)->serialno); + } + } + mesa_log_stream_printf(stream, "\n"); } diff --git a/src/freedreno/ir3/ir3_validate.c b/src/freedreno/ir3/ir3_validate.c index 53cce7477b2..c05d9e83ec4 100644 --- a/src/freedreno/ir3/ir3_validate.c +++ b/src/freedreno/ir3/ir3_validate.c @@ -182,11 +182,46 @@ validate_dst(struct ir3_validate_ctx *ctx, struct ir3_instruction *instr, validate_assert( \ ctx, (type_size(type) <= 16) == !!((reg)->flags & IR3_REG_HALF)) +static bool +block_contains(struct ir3_block *block, struct ir3_instruction *instr) +{ + foreach_instr (block_instr, &block->instr_list) { + if (block_instr == instr) + return true; + } + + return false; +} + +static void +validate_rpt(struct ir3_validate_ctx *ctx, struct ir3_instruction *instr) +{ + if (ir3_instr_is_first_rpt(instr)) { + /* All instructions in a repeat group should be in the same block as the + * first one. + */ + foreach_instr_rpt (rpt, instr) { + validate_assert(ctx, rpt->block == instr->block); + + /* Validate that the block actually contains the repeat. This would + * fail if, for example, list_delinit is called instead of + * ir3_instr_remove. + */ + validate_assert(ctx, block_contains(instr->block, rpt)); + } + } else if (instr->repeat) { + validate_assert(ctx, ir3_supports_rpt(instr->opc)); + validate_assert(ctx, !instr->nop); + } +} + static void validate_instr(struct ir3_validate_ctx *ctx, struct ir3_instruction *instr) { struct ir3_register *last_reg = NULL; + validate_rpt(ctx, instr); + foreach_src_n (reg, n, instr) { if (reg->flags & IR3_REG_RELATIV) validate_assert(ctx, instr->address);