pan/cs: Add helpers for if/else blocks

The logic is a bit more involved than with other blocks to keep
cs_if/else() declaration consistent with the rest. This forces us to
keep track of the last end_if() so we can flush it if the next CS
[pseudo-]instruction is not a cs_else().

These changes require quite a bit of code motion to avoid forward
declarations.

Signed-off-by: Boris Brezillon <boris.brezillon@collabora.com>
Reviewed-by: Lars-Ivar Hesselberg Simonsen <lars-ivar.simonsen@arm.com>
Reviewed-by: Louis-Francis Ratté-Boulianne <lfrb@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/31205>

src/panfrost/.clang-format

@@ -40,7 +40,9 @@ ForEachMacros: [
   'cs_case',
   'cs_continue',
   'cs_default',
+  'cs_else',
   'cs_emit',
+  'cs_if',
   'cs_match',
   'cs_update_compute_ctx',
   'cs_update_frag_ctx',

src/panfrost/lib/genxml/cs_builder.h

@@ -142,6 +142,12 @@ struct cs_label {
    uint32_t target;
 };
 
+/* CS if/else block. */
+struct cs_if_else {
+   struct cs_block block;
+   struct cs_label end_label;
+};
+
 struct cs_builder {
    /* CS builder configuration */
    struct cs_builder_conf conf;
@@ -162,6 +168,7 @@ struct cs_builder {
    struct {
       struct cs_block *stack;
       struct util_dynarray instrs;
+      struct cs_if_else pending_if;
    } blocks;
 
    /* Move immediate instruction at the end of the last CS chunk that needs to
@@ -248,23 +255,6 @@ cs_wrap_chunk(struct cs_builder *b)
       b->root_chunk.size = b->cur_chunk.size;
 }
 
-/* Call this when you are done building a command stream and want to prepare
- * it for submission.
- */
-static inline void
-cs_finish(struct cs_builder *b)
-{
-   if (!cs_is_valid(b))
-      return;
-
-   cs_wrap_chunk(b);
-
-   /* This prevents adding instructions after that point. */
-   memset(&b->cur_chunk, 0, sizeof(b->cur_chunk));
-
-   util_dynarray_fini(&b->blocks.instrs);
-}
-
 enum cs_index_type {
    CS_INDEX_REGISTER = 0,
    CS_INDEX_UNDEF,
@@ -434,7 +424,7 @@ cs_cur_block(struct cs_builder *b)
 #define JUMP_SEQ_INSTR_COUNT 4
 
 static inline void *
-cs_alloc_ins(struct cs_builder *b, uint32_t num_instrs)
+cs_alloc_ins_block(struct cs_builder *b, uint32_t num_instrs)
 {
    /* Don't call this function with num_instrs=0. */
    assert(num_instrs > 0);
@@ -514,12 +504,116 @@ cs_alloc_ins(struct cs_builder *b, uint32_t num_instrs)
    return b->cur_chunk.buffer.cpu + pos;
 }
 
+static inline void
+cs_flush_block_instrs(struct cs_builder *b)
+{
+   if (cs_cur_block(b) != NULL)
+      return;
+
+   uint32_t num_instrs =
+      util_dynarray_num_elements(&b->blocks.instrs, uint64_t);
+   if (!num_instrs)
+      return;
+
+   void *buffer = cs_alloc_ins_block(b, num_instrs);
+
+   if (likely(cs_is_valid(b)))
+      memcpy(buffer, b->blocks.instrs.data, b->blocks.instrs.size);
+
+   util_dynarray_clear(&b->blocks.instrs);
+}
+
+static inline uint32_t
+cs_block_next_pos(struct cs_builder *b)
+{
+   assert(cs_cur_block(b) != NULL);
+
+   return util_dynarray_num_elements(&b->blocks.instrs, uint64_t);
+}
+
+static inline void
+cs_label_init(struct cs_label *label)
+{
+   label->last_forward_ref = CS_LABEL_INVALID_POS;
+   label->target = CS_LABEL_INVALID_POS;
+}
+
+static inline void
+cs_set_label(struct cs_builder *b, struct cs_label *label)
+{
+   assert(label->target == CS_LABEL_INVALID_POS);
+   label->target = cs_block_next_pos(b);
+
+   for (uint32_t next_forward_ref, forward_ref = label->last_forward_ref;
+        forward_ref != CS_LABEL_INVALID_POS; forward_ref = next_forward_ref) {
+      uint64_t *ins =
+         util_dynarray_element(&b->blocks.instrs, uint64_t, forward_ref);
+
+      assert(forward_ref < label->target);
+      assert(label->target - forward_ref <= INT16_MAX);
+
+      /* Save the next forward reference to this target before overwritting
+       * it with the final offset.
+       */
+      int16_t offset = *ins & BITFIELD64_MASK(16);
+
+      next_forward_ref =
+         offset > 0 ? forward_ref - offset : CS_LABEL_INVALID_POS;
+
+      assert(next_forward_ref == CS_LABEL_INVALID_POS ||
+             next_forward_ref < forward_ref);
+
+      *ins &= ~BITFIELD64_MASK(16);
+      *ins |= label->target - forward_ref - 1;
+   }
+}
+
+static inline void
+cs_flush_pending_if(struct cs_builder *b)
+{
+   if (likely(cs_cur_block(b) != &b->blocks.pending_if.block))
+      return;
+
+   cs_set_label(b, &b->blocks.pending_if.end_label);
+   b->blocks.stack = b->blocks.pending_if.block.next;
+}
+
+static inline void *
+cs_alloc_ins(struct cs_builder *b)
+{
+   /* If an instruction is emitted after an if_end(), it flushes the pending if,
+    * causing further cs_else_start() instructions to be invalid. */
+   cs_flush_pending_if(b);
+   cs_flush_block_instrs(b);
+
+   return cs_alloc_ins_block(b, 1);
+}
+
+/* Call this when you are done building a command stream and want to prepare
+ * it for submission.
+ */
+static inline void
+cs_finish(struct cs_builder *b)
+{
+   if (!cs_is_valid(b))
+      return;
+
+   cs_flush_pending_if(b);
+   cs_flush_block_instrs(b);
+   cs_wrap_chunk(b);
+
+   /* This prevents adding instructions after that point. */
+   memset(&b->cur_chunk, 0, sizeof(b->cur_chunk));
+
+   util_dynarray_fini(&b->blocks.instrs);
+}
+
 /*
  * Helper to emit a new instruction into the command queue. The allocation needs
  * to be separated out being pan_pack can evaluate its argument multiple times,
  * yet cs_alloc has side effects.
  */
-#define cs_emit(b, T, cfg) pan_pack(cs_alloc_ins(b, 1), CS_##T, cfg)
+#define cs_emit(b, T, cfg) pan_pack(cs_alloc_ins(b), CS_##T, cfg)
 
 /* Asynchronous operations take a mask of scoreboard slots to wait on
  * before executing the instruction, and signal a scoreboard slot when
@@ -619,6 +713,7 @@ cs_move48_to(struct cs_builder *b, struct cs_index dest, uint64_t imm)
 static inline void
 cs_block_start(struct cs_builder *b, struct cs_block *block)
 {
+   cs_flush_pending_if(b);
    block->next = b->blocks.stack;
    b->blocks.stack = block;
 }
@@ -626,29 +721,13 @@ cs_block_start(struct cs_builder *b, struct cs_block *block)
 static inline void
 cs_block_end(struct cs_builder *b, struct cs_block *block)
 {
+   cs_flush_pending_if(b);
+
    assert(cs_cur_block(b) == block);
 
    b->blocks.stack = block->next;
 
-   if (cs_cur_block(b) != NULL)
-      return;
-
-   uint32_t num_instrs =
-      util_dynarray_num_elements(&b->blocks.instrs, uint64_t);
-   void *buffer = cs_alloc_ins(b, num_instrs);
-
-   if (likely(cs_is_valid(b)))
-      memcpy(buffer, b->blocks.instrs.data, b->blocks.instrs.size);
-
-   util_dynarray_clear(&b->blocks.instrs);
-}
-
-static inline uint32_t
-cs_block_next_pos(struct cs_builder *b)
-{
-   assert(cs_cur_block(b) != NULL);
-
-   return util_dynarray_num_elements(&b->blocks.instrs, uint64_t);
+   cs_flush_block_instrs(b);
 }
 
 static inline void
@@ -708,52 +787,6 @@ cs_branch_label(struct cs_builder *b, struct cs_label *label,
    }
 }
 
-static inline void
-cs_label_init(struct cs_label *label)
-{
-   label->last_forward_ref = CS_LABEL_INVALID_POS;
-   label->target = CS_LABEL_INVALID_POS;
-}
-
-static inline void
-cs_set_label(struct cs_builder *b, struct cs_label *label)
-{
-   assert(label->target == CS_LABEL_INVALID_POS);
-   label->target = cs_block_next_pos(b);
-
-   for (uint32_t next_forward_ref, forward_ref = label->last_forward_ref;
-        forward_ref != CS_LABEL_INVALID_POS; forward_ref = next_forward_ref) {
-      uint64_t *ins =
-         util_dynarray_element(&b->blocks.instrs, uint64_t, forward_ref);
-
-      assert(forward_ref < label->target);
-      assert(label->target - forward_ref <= INT16_MAX);
-
-      /* Save the next forward reference to this target before overwritting
-       * it with the final offset.
-       */
-      int16_t offset = *ins & BITFIELD64_MASK(16);
-
-      next_forward_ref =
-         offset > 0 ? forward_ref - offset : CS_LABEL_INVALID_POS;
-
-      assert(next_forward_ref == CS_LABEL_INVALID_POS ||
-             next_forward_ref < forward_ref);
-
-      *ins &= ~BITFIELD64_MASK(16);
-      *ins |= label->target - forward_ref - 1;
-   }
-}
-
-struct cs_loop {
-   struct cs_label start, end;
-   struct cs_block block;
-   enum mali_cs_condition cond;
-   struct cs_index val;
-   struct cs_load_store_tracker *orig_ls_state;
-   struct cs_load_store_tracker ls_state;
-};
-
 static inline enum mali_cs_condition
 cs_invert_cond(enum mali_cs_condition cond)
 {
@@ -777,6 +810,68 @@ cs_invert_cond(enum mali_cs_condition cond)
    }
 }
 
+static inline struct cs_if_else *
+cs_if_start(struct cs_builder *b, struct cs_if_else *if_else,
+            enum mali_cs_condition cond, struct cs_index val)
+{
+   cs_block_start(b, &if_else->block);
+   cs_label_init(&if_else->end_label);
+   cs_branch_label(b, &if_else->end_label, cs_invert_cond(cond), val);
+   return if_else;
+}
+
+static inline void
+cs_if_end(struct cs_builder *b, struct cs_if_else *if_else)
+{
+   assert(cs_cur_block(b) == &if_else->block);
+
+   b->blocks.pending_if.block.next = if_else->block.next;
+   b->blocks.stack = &b->blocks.pending_if.block;
+   b->blocks.pending_if.end_label = if_else->end_label;
+}
+
+static inline struct cs_if_else *
+cs_else_start(struct cs_builder *b, struct cs_if_else *if_else)
+{
+   assert(cs_cur_block(b) == &b->blocks.pending_if.block);
+
+   if_else->block.next = b->blocks.pending_if.block.next;
+   b->blocks.stack = &if_else->block;
+   cs_label_init(&if_else->end_label);
+   cs_branch_label(b, &if_else->end_label, MALI_CS_CONDITION_ALWAYS,
+                   cs_undef());
+   cs_set_label(b, &b->blocks.pending_if.end_label);
+   cs_label_init(&b->blocks.pending_if.end_label);
+
+   return if_else;
+}
+
+static inline void
+cs_else_end(struct cs_builder *b, struct cs_if_else *if_else)
+{
+   cs_set_label(b, &if_else->end_label);
+   cs_block_end(b, &if_else->block);
+}
+
+#define cs_if(__b, __cond, __val)                                              \
+   for (struct cs_if_else __storage,                                           \
+        *__if_else = cs_if_start(__b, &__storage, __cond, __val);              \
+        __if_else != NULL; cs_if_end(__b, __if_else), __if_else = NULL)
+
+#define cs_else(__b)                                                           \
+   for (struct cs_if_else __storage,                                           \
+        *__if_else = cs_else_start(__b, &__storage);                           \
+        __if_else != NULL; cs_else_end(__b, __if_else), __if_else = NULL)
+
+struct cs_loop {
+   struct cs_label start, end;
+   struct cs_block block;
+   enum mali_cs_condition cond;
+   struct cs_index val;
+   struct cs_load_store_tracker *orig_ls_state;
+   struct cs_load_store_tracker ls_state;
+};
+
 static inline void
 cs_loop_diverge_ls_update(struct cs_builder *b, struct cs_loop *loop)
 {
@@ -835,6 +930,7 @@ static inline void
 cs_loop_conditional_continue(struct cs_builder *b, struct cs_loop *loop,
                              enum mali_cs_condition cond, struct cs_index val)
 {
+   cs_flush_pending_if(b);
    assert(cs_cur_block(b) == &loop->block);
    cs_branch_label(b, &loop->start, cond, val);
    cs_loop_diverge_ls_update(b, loop);
@@ -844,6 +940,7 @@ static inline void
 cs_loop_conditional_break(struct cs_builder *b, struct cs_loop *loop,
                           enum mali_cs_condition cond, struct cs_index val)
 {
+   cs_flush_pending_if(b);
    assert(cs_cur_block(b) == &loop->block);
    cs_branch_label(b, &loop->end, cond, val);
    cs_loop_diverge_ls_update(b, loop);
@@ -852,6 +949,7 @@ cs_loop_conditional_break(struct cs_builder *b, struct cs_loop *loop,
 static inline void
 cs_while_end(struct cs_builder *b, struct cs_loop *loop)
 {
+   cs_flush_pending_if(b);
    cs_branch_label(b, &loop->start, loop->cond, loop->val);
    cs_set_label(b, &loop->end);
    cs_block_end(b, &loop->block);