ir3: add backend support for repeated instructions

In order to represent repeated instructions (rptN) in ir3, this patch
introduces the concept of "repeat groups". A repeat group is a group of
instructions that were produced from a vectorized NIR operation and
linked together. They are, however, still separate scalar instructions.

Repeat groups are created by linking together multiple instructions
using a new rpt_node list. This patch adds this list as well as a number
of helper functions the can be used to create and manipulate repeat
groups.

Signed-off-by: Job Noorman <jnoorman@igalia.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/28341>

src/.clang-format

@@ -275,6 +275,9 @@ ForEachMacros:
   - perf_time_ctx
   - foreach_submit
   - foreach_submit_safe
+  - foreach_instr_rpt
+  - foreach_instr_rpt_excl
+  - foreach_instr_rpt_excl_safe
 
 # panfrost
   - foreach_batch

src/freedreno/ir3/ir3.c

@@ -646,6 +646,7 @@ instr_create(struct ir3_block *block, opc_t opc, int ndst, int nsrc)
    instr->srcs_max = nsrc;
 #endif
 
+   list_inithead(&instr->rpt_node);
    return instr;
 }
 
@@ -725,6 +726,7 @@ ir3_instr_clone(struct ir3_instruction *instr)
    *new_instr = *instr;
    new_instr->dsts = dsts;
    new_instr->srcs = srcs;
+   list_inithead(&new_instr->rpt_node);
 
    insert_instr(ir3_before_terminator(instr->block), new_instr);
 
@@ -765,6 +767,74 @@ ir3_instr_add_dep(struct ir3_instruction *instr, struct ir3_instruction *dep)
    array_insert(instr, instr->deps, dep);
 }
 
+void
+ir3_instr_remove(struct ir3_instruction *instr)
+{
+   list_delinit(&instr->node);
+   list_delinit(&instr->rpt_node);
+}
+
+void
+ir3_instr_create_rpt(struct ir3_instruction **instrs, unsigned n)
+{
+   assert(n > 0 && !ir3_instr_is_rpt(instrs[0]));
+
+   for (unsigned i = 1; i < n; ++i) {
+      assert(!ir3_instr_is_rpt(instrs[i]));
+      assert(instrs[i]->serialno > instrs[i - 1]->serialno);
+
+      list_addtail(&instrs[i]->rpt_node, &instrs[0]->rpt_node);
+   }
+}
+
+bool
+ir3_instr_is_rpt(const struct ir3_instruction *instr)
+{
+   return !list_is_empty(&instr->rpt_node);
+}
+
+bool
+ir3_instr_is_first_rpt(const struct ir3_instruction *instr)
+{
+   if (!ir3_instr_is_rpt(instr))
+      return false;
+
+   struct ir3_instruction *prev_rpt =
+      list_entry(instr->rpt_node.prev, struct ir3_instruction, rpt_node);
+   return prev_rpt->serialno > instr->serialno;
+}
+
+struct ir3_instruction *
+ir3_instr_prev_rpt(const struct ir3_instruction *instr)
+{
+   assert(ir3_instr_is_rpt(instr));
+
+   if (ir3_instr_is_first_rpt(instr))
+      return NULL;
+   return list_entry(instr->rpt_node.prev, struct ir3_instruction, rpt_node);
+}
+
+struct ir3_instruction *
+ir3_instr_first_rpt(struct ir3_instruction *instr)
+{
+   assert(ir3_instr_is_rpt(instr));
+
+   while (!ir3_instr_is_first_rpt(instr)) {
+      instr = ir3_instr_prev_rpt(instr);
+      assert(instr);
+   }
+
+   return instr;
+}
+
+unsigned
+ir3_instr_rpt_length(const struct ir3_instruction *instr)
+{
+   assert(ir3_instr_is_first_rpt(instr));
+
+   return list_length(&instr->rpt_node) + 1;
+}
+
 struct ir3_register *
 ir3_src_create(struct ir3_instruction *instr, int num, int flags)
 {
@@ -1439,3 +1509,22 @@ ir3_get_cond_for_nonzero_compare(struct ir3_instruction *instr)
 
    return instr;
 }
+
+bool
+ir3_supports_rpt(unsigned opc)
+{
+   switch (opc_cat(opc)) {
+   case 0:
+      return opc == OPC_NOP;
+   case 1:
+      return opc == OPC_MOV || opc == OPC_SWZ || opc == OPC_MOVMSK;
+   case 2:
+      return true;
+   case 3:
+      return opc != OPC_DP2ACC && opc != OPC_DP4ACC;
+   case 4:
+      return opc != OPC_RCP;
+   default:
+      return false;
+   }
+}

src/freedreno/ir3/ir3.h

@@ -554,6 +554,25 @@ struct ir3_instruction {
    /* Entry in ir3_block's instruction list: */
    struct list_head node;
 
+   /* List of this instruction's repeat group. Vectorized NIR instructions are
+    * emitted as multiple scalar instructions that are linked together using
+    * this field. After RA, the ir3_combine_rpt pass iterates these groups and,
+    * if the register assignment allows it, merges them into a (rptN)
+    * instruction.
+    *
+    * NOTE: this is not a typical list as there is no empty list head. The list
+    * head is stored in the first instruction of the repeat group so also refers
+    * to a list entry. In order to distinguish the list's first entry, we use
+    * serialno: instructions in a repeat group are always emitted consecutively
+    * so the first will have the lowest serialno.
+    *
+    * As this is not a typical list, we have to be careful with using the
+    * existing list helper. For example, using list_length on the first
+    * instruction will yield one less than the number of instructions in its
+    * group.
+    */
+   struct list_head rpt_node;
+
    uint32_t serialno;
 
    // TODO only computerator/assembler:
@@ -807,6 +826,14 @@ struct ir3_instruction *ir3_instr_clone(struct ir3_instruction *instr);
 void ir3_instr_add_dep(struct ir3_instruction *instr,
                        struct ir3_instruction *dep);
 const char *ir3_instr_name(struct ir3_instruction *instr);
+void ir3_instr_remove(struct ir3_instruction *instr);
+
+void ir3_instr_create_rpt(struct ir3_instruction **instrs, unsigned n);
+bool ir3_instr_is_rpt(const struct ir3_instruction *instr);
+bool ir3_instr_is_first_rpt(const struct ir3_instruction *instr);
+struct ir3_instruction *ir3_instr_prev_rpt(const struct ir3_instruction *instr);
+struct ir3_instruction *ir3_instr_first_rpt(struct ir3_instruction *instr);
+unsigned ir3_instr_rpt_length(const struct ir3_instruction *instr);
 
 struct ir3_register *ir3_src_create(struct ir3_instruction *instr, int num,
                                     int flags);
@@ -901,6 +928,8 @@ bool ir3_valid_immediate(struct ir3_instruction *instr, int32_t immed);
 struct ir3_instruction *
 ir3_get_cond_for_nonzero_compare(struct ir3_instruction *instr);
 
+bool ir3_supports_rpt(unsigned opc);
+
 #include "util/set.h"
 #define foreach_ssa_use(__use, __instr)                                        \
    for (struct ir3_instruction *__use = (void *)~0; __use && (__instr)->uses;  \
@@ -1938,6 +1967,26 @@ __ssa_srcp_n(struct ir3_instruction *instr, unsigned n)
    list_for_each_entry_from_safe(struct ir3_instruction, __instr, __start,     \
                                  __list, node)
 
+/* Iterate over all instructions in a repeat group. */
+#define foreach_instr_rpt(__rpt, __instr)                                      \
+   if (assert(ir3_instr_is_first_rpt(__instr)), true)                          \
+      for (struct ir3_instruction *__rpt = __instr, *__first = __instr;        \
+           __first || __rpt != __instr;                                        \
+           __first = NULL, __rpt =                                             \
+                              list_entry(__rpt->rpt_node.next,                 \
+                                         struct ir3_instruction, rpt_node))
+
+/* Iterate over all instructions except the first one in a repeat group. */
+#define foreach_instr_rpt_excl(__rpt, __instr)                                 \
+   if (assert(ir3_instr_is_first_rpt(__instr)), true)                          \
+      list_for_each_entry (struct ir3_instruction, __rpt, &__instr->rpt_node,  \
+                           rpt_node)
+
+#define foreach_instr_rpt_excl_safe(__rpt, __instr)                            \
+   if (assert(ir3_instr_is_first_rpt(__instr)), true)                          \
+      list_for_each_entry_safe (struct ir3_instruction, __rpt,                 \
+                                &__instr->rpt_node, rpt_node)
+
 /* iterators for blocks: */
 #define foreach_block(__block, __list)                                         \
    list_for_each_entry (struct ir3_block, __block, __list, node)

src/freedreno/ir3/ir3_dce.c

@@ -92,7 +92,7 @@ remove_unused_by_block(struct ir3_block *block)
                if (*srcp == instr)
                   *srcp = NULL;
 
-         list_delinit(&instr->node);
+         ir3_instr_remove(instr);
          progress = true;
       }
    }

src/freedreno/ir3/ir3_print.c

@@ -460,6 +460,17 @@ print_instr(struct log_stream *stream, struct ir3_instruction *instr, int lvl)
       }
    }
 
+   if (ir3_instr_is_rpt(instr)) {
+      mesa_log_stream_printf(stream, ", rpt: ");
+
+      if (ir3_instr_is_first_rpt(instr)) {
+         mesa_log_stream_printf(stream, "first");
+      } else {
+         mesa_log_stream_printf(stream, "%u",
+                                ir3_instr_prev_rpt(instr)->serialno);
+      }
+   }
+
    mesa_log_stream_printf(stream, "\n");
 }
 

src/freedreno/ir3/ir3_validate.c

@@ -182,11 +182,46 @@ validate_dst(struct ir3_validate_ctx *ctx, struct ir3_instruction *instr,
    validate_assert(                                                            \
       ctx, (type_size(type) <= 16) == !!((reg)->flags & IR3_REG_HALF))
 
+static bool
+block_contains(struct ir3_block *block, struct ir3_instruction *instr)
+{
+   foreach_instr (block_instr, &block->instr_list) {
+      if (block_instr == instr)
+         return true;
+   }
+
+   return false;
+}
+
+static void
+validate_rpt(struct ir3_validate_ctx *ctx, struct ir3_instruction *instr)
+{
+   if (ir3_instr_is_first_rpt(instr)) {
+      /* All instructions in a repeat group should be in the same block as the
+       * first one.
+       */
+      foreach_instr_rpt (rpt, instr) {
+         validate_assert(ctx, rpt->block == instr->block);
+
+         /* Validate that the block actually contains the repeat. This would
+          * fail if, for example, list_delinit is called instead of
+          * ir3_instr_remove.
+          */
+         validate_assert(ctx, block_contains(instr->block, rpt));
+      }
+   } else if (instr->repeat) {
+      validate_assert(ctx, ir3_supports_rpt(instr->opc));
+      validate_assert(ctx, !instr->nop);
+   }
+}
+
 static void
 validate_instr(struct ir3_validate_ctx *ctx, struct ir3_instruction *instr)
 {
    struct ir3_register *last_reg = NULL;
 
+   validate_rpt(ctx, instr);
+
    foreach_src_n (reg, n, instr) {
       if (reg->flags & IR3_REG_RELATIV)
          validate_assert(ctx, instr->address);