From 741f6b9e5a92ee8a47f3d70b4817c9d2908381c5 Mon Sep 17 00:00:00 2001 From: Boris Brezillon Date: Tue, 17 Sep 2024 13:38:09 +0200 Subject: [PATCH] pan/cs: Add helpers for if/else blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The logic is a bit more involved than with other blocks to keep cs_if/else() declaration consistent with the rest. This forces us to keep track of the last end_if() so we can flush it if the next CS [pseudo-]instruction is not a cs_else(). These changes require quite a bit of code motion to avoid forward declarations. Signed-off-by: Boris Brezillon Reviewed-by: Lars-Ivar Hesselberg Simonsen Reviewed-by: Louis-Francis Ratté-Boulianne Part-of: --- src/panfrost/.clang-format | 2 + src/panfrost/lib/genxml/cs_builder.h | 266 ++++++++++++++++++--------- 2 files changed, 184 insertions(+), 84 deletions(-) diff --git a/src/panfrost/.clang-format b/src/panfrost/.clang-format index 1a69d5be020..038d295b733 100644 --- a/src/panfrost/.clang-format +++ b/src/panfrost/.clang-format @@ -40,7 +40,9 @@ ForEachMacros: [ 'cs_case', 'cs_continue', 'cs_default', + 'cs_else', 'cs_emit', + 'cs_if', 'cs_match', 'cs_update_compute_ctx', 'cs_update_frag_ctx', diff --git a/src/panfrost/lib/genxml/cs_builder.h b/src/panfrost/lib/genxml/cs_builder.h index e84a56dcf48..63e16c990d2 100644 --- a/src/panfrost/lib/genxml/cs_builder.h +++ b/src/panfrost/lib/genxml/cs_builder.h @@ -142,6 +142,12 @@ struct cs_label { uint32_t target; }; +/* CS if/else block. */ +struct cs_if_else { + struct cs_block block; + struct cs_label end_label; +}; + struct cs_builder { /* CS builder configuration */ struct cs_builder_conf conf; @@ -162,6 +168,7 @@ struct cs_builder { struct { struct cs_block *stack; struct util_dynarray instrs; + struct cs_if_else pending_if; } blocks; /* Move immediate instruction at the end of the last CS chunk that needs to @@ -248,23 +255,6 @@ cs_wrap_chunk(struct cs_builder *b) b->root_chunk.size = b->cur_chunk.size; } -/* Call this when you are done building a command stream and want to prepare - * it for submission. - */ -static inline void -cs_finish(struct cs_builder *b) -{ - if (!cs_is_valid(b)) - return; - - cs_wrap_chunk(b); - - /* This prevents adding instructions after that point. */ - memset(&b->cur_chunk, 0, sizeof(b->cur_chunk)); - - util_dynarray_fini(&b->blocks.instrs); -} - enum cs_index_type { CS_INDEX_REGISTER = 0, CS_INDEX_UNDEF, @@ -434,7 +424,7 @@ cs_cur_block(struct cs_builder *b) #define JUMP_SEQ_INSTR_COUNT 4 static inline void * -cs_alloc_ins(struct cs_builder *b, uint32_t num_instrs) +cs_alloc_ins_block(struct cs_builder *b, uint32_t num_instrs) { /* Don't call this function with num_instrs=0. */ assert(num_instrs > 0); @@ -514,12 +504,116 @@ cs_alloc_ins(struct cs_builder *b, uint32_t num_instrs) return b->cur_chunk.buffer.cpu + pos; } +static inline void +cs_flush_block_instrs(struct cs_builder *b) +{ + if (cs_cur_block(b) != NULL) + return; + + uint32_t num_instrs = + util_dynarray_num_elements(&b->blocks.instrs, uint64_t); + if (!num_instrs) + return; + + void *buffer = cs_alloc_ins_block(b, num_instrs); + + if (likely(cs_is_valid(b))) + memcpy(buffer, b->blocks.instrs.data, b->blocks.instrs.size); + + util_dynarray_clear(&b->blocks.instrs); +} + +static inline uint32_t +cs_block_next_pos(struct cs_builder *b) +{ + assert(cs_cur_block(b) != NULL); + + return util_dynarray_num_elements(&b->blocks.instrs, uint64_t); +} + +static inline void +cs_label_init(struct cs_label *label) +{ + label->last_forward_ref = CS_LABEL_INVALID_POS; + label->target = CS_LABEL_INVALID_POS; +} + +static inline void +cs_set_label(struct cs_builder *b, struct cs_label *label) +{ + assert(label->target == CS_LABEL_INVALID_POS); + label->target = cs_block_next_pos(b); + + for (uint32_t next_forward_ref, forward_ref = label->last_forward_ref; + forward_ref != CS_LABEL_INVALID_POS; forward_ref = next_forward_ref) { + uint64_t *ins = + util_dynarray_element(&b->blocks.instrs, uint64_t, forward_ref); + + assert(forward_ref < label->target); + assert(label->target - forward_ref <= INT16_MAX); + + /* Save the next forward reference to this target before overwritting + * it with the final offset. + */ + int16_t offset = *ins & BITFIELD64_MASK(16); + + next_forward_ref = + offset > 0 ? forward_ref - offset : CS_LABEL_INVALID_POS; + + assert(next_forward_ref == CS_LABEL_INVALID_POS || + next_forward_ref < forward_ref); + + *ins &= ~BITFIELD64_MASK(16); + *ins |= label->target - forward_ref - 1; + } +} + +static inline void +cs_flush_pending_if(struct cs_builder *b) +{ + if (likely(cs_cur_block(b) != &b->blocks.pending_if.block)) + return; + + cs_set_label(b, &b->blocks.pending_if.end_label); + b->blocks.stack = b->blocks.pending_if.block.next; +} + +static inline void * +cs_alloc_ins(struct cs_builder *b) +{ + /* If an instruction is emitted after an if_end(), it flushes the pending if, + * causing further cs_else_start() instructions to be invalid. */ + cs_flush_pending_if(b); + cs_flush_block_instrs(b); + + return cs_alloc_ins_block(b, 1); +} + +/* Call this when you are done building a command stream and want to prepare + * it for submission. + */ +static inline void +cs_finish(struct cs_builder *b) +{ + if (!cs_is_valid(b)) + return; + + cs_flush_pending_if(b); + cs_flush_block_instrs(b); + cs_wrap_chunk(b); + + /* This prevents adding instructions after that point. */ + memset(&b->cur_chunk, 0, sizeof(b->cur_chunk)); + + util_dynarray_fini(&b->blocks.instrs); +} + /* * Helper to emit a new instruction into the command queue. The allocation needs * to be separated out being pan_pack can evaluate its argument multiple times, * yet cs_alloc has side effects. */ -#define cs_emit(b, T, cfg) pan_pack(cs_alloc_ins(b, 1), CS_##T, cfg) +#define cs_emit(b, T, cfg) pan_pack(cs_alloc_ins(b), CS_##T, cfg) /* Asynchronous operations take a mask of scoreboard slots to wait on * before executing the instruction, and signal a scoreboard slot when @@ -619,6 +713,7 @@ cs_move48_to(struct cs_builder *b, struct cs_index dest, uint64_t imm) static inline void cs_block_start(struct cs_builder *b, struct cs_block *block) { + cs_flush_pending_if(b); block->next = b->blocks.stack; b->blocks.stack = block; } @@ -626,29 +721,13 @@ cs_block_start(struct cs_builder *b, struct cs_block *block) static inline void cs_block_end(struct cs_builder *b, struct cs_block *block) { + cs_flush_pending_if(b); + assert(cs_cur_block(b) == block); b->blocks.stack = block->next; - if (cs_cur_block(b) != NULL) - return; - - uint32_t num_instrs = - util_dynarray_num_elements(&b->blocks.instrs, uint64_t); - void *buffer = cs_alloc_ins(b, num_instrs); - - if (likely(cs_is_valid(b))) - memcpy(buffer, b->blocks.instrs.data, b->blocks.instrs.size); - - util_dynarray_clear(&b->blocks.instrs); -} - -static inline uint32_t -cs_block_next_pos(struct cs_builder *b) -{ - assert(cs_cur_block(b) != NULL); - - return util_dynarray_num_elements(&b->blocks.instrs, uint64_t); + cs_flush_block_instrs(b); } static inline void @@ -708,52 +787,6 @@ cs_branch_label(struct cs_builder *b, struct cs_label *label, } } -static inline void -cs_label_init(struct cs_label *label) -{ - label->last_forward_ref = CS_LABEL_INVALID_POS; - label->target = CS_LABEL_INVALID_POS; -} - -static inline void -cs_set_label(struct cs_builder *b, struct cs_label *label) -{ - assert(label->target == CS_LABEL_INVALID_POS); - label->target = cs_block_next_pos(b); - - for (uint32_t next_forward_ref, forward_ref = label->last_forward_ref; - forward_ref != CS_LABEL_INVALID_POS; forward_ref = next_forward_ref) { - uint64_t *ins = - util_dynarray_element(&b->blocks.instrs, uint64_t, forward_ref); - - assert(forward_ref < label->target); - assert(label->target - forward_ref <= INT16_MAX); - - /* Save the next forward reference to this target before overwritting - * it with the final offset. - */ - int16_t offset = *ins & BITFIELD64_MASK(16); - - next_forward_ref = - offset > 0 ? forward_ref - offset : CS_LABEL_INVALID_POS; - - assert(next_forward_ref == CS_LABEL_INVALID_POS || - next_forward_ref < forward_ref); - - *ins &= ~BITFIELD64_MASK(16); - *ins |= label->target - forward_ref - 1; - } -} - -struct cs_loop { - struct cs_label start, end; - struct cs_block block; - enum mali_cs_condition cond; - struct cs_index val; - struct cs_load_store_tracker *orig_ls_state; - struct cs_load_store_tracker ls_state; -}; - static inline enum mali_cs_condition cs_invert_cond(enum mali_cs_condition cond) { @@ -777,6 +810,68 @@ cs_invert_cond(enum mali_cs_condition cond) } } +static inline struct cs_if_else * +cs_if_start(struct cs_builder *b, struct cs_if_else *if_else, + enum mali_cs_condition cond, struct cs_index val) +{ + cs_block_start(b, &if_else->block); + cs_label_init(&if_else->end_label); + cs_branch_label(b, &if_else->end_label, cs_invert_cond(cond), val); + return if_else; +} + +static inline void +cs_if_end(struct cs_builder *b, struct cs_if_else *if_else) +{ + assert(cs_cur_block(b) == &if_else->block); + + b->blocks.pending_if.block.next = if_else->block.next; + b->blocks.stack = &b->blocks.pending_if.block; + b->blocks.pending_if.end_label = if_else->end_label; +} + +static inline struct cs_if_else * +cs_else_start(struct cs_builder *b, struct cs_if_else *if_else) +{ + assert(cs_cur_block(b) == &b->blocks.pending_if.block); + + if_else->block.next = b->blocks.pending_if.block.next; + b->blocks.stack = &if_else->block; + cs_label_init(&if_else->end_label); + cs_branch_label(b, &if_else->end_label, MALI_CS_CONDITION_ALWAYS, + cs_undef()); + cs_set_label(b, &b->blocks.pending_if.end_label); + cs_label_init(&b->blocks.pending_if.end_label); + + return if_else; +} + +static inline void +cs_else_end(struct cs_builder *b, struct cs_if_else *if_else) +{ + cs_set_label(b, &if_else->end_label); + cs_block_end(b, &if_else->block); +} + +#define cs_if(__b, __cond, __val) \ + for (struct cs_if_else __storage, \ + *__if_else = cs_if_start(__b, &__storage, __cond, __val); \ + __if_else != NULL; cs_if_end(__b, __if_else), __if_else = NULL) + +#define cs_else(__b) \ + for (struct cs_if_else __storage, \ + *__if_else = cs_else_start(__b, &__storage); \ + __if_else != NULL; cs_else_end(__b, __if_else), __if_else = NULL) + +struct cs_loop { + struct cs_label start, end; + struct cs_block block; + enum mali_cs_condition cond; + struct cs_index val; + struct cs_load_store_tracker *orig_ls_state; + struct cs_load_store_tracker ls_state; +}; + static inline void cs_loop_diverge_ls_update(struct cs_builder *b, struct cs_loop *loop) { @@ -835,6 +930,7 @@ static inline void cs_loop_conditional_continue(struct cs_builder *b, struct cs_loop *loop, enum mali_cs_condition cond, struct cs_index val) { + cs_flush_pending_if(b); assert(cs_cur_block(b) == &loop->block); cs_branch_label(b, &loop->start, cond, val); cs_loop_diverge_ls_update(b, loop); @@ -844,6 +940,7 @@ static inline void cs_loop_conditional_break(struct cs_builder *b, struct cs_loop *loop, enum mali_cs_condition cond, struct cs_index val) { + cs_flush_pending_if(b); assert(cs_cur_block(b) == &loop->block); cs_branch_label(b, &loop->end, cond, val); cs_loop_diverge_ls_update(b, loop); @@ -852,6 +949,7 @@ cs_loop_conditional_break(struct cs_builder *b, struct cs_loop *loop, static inline void cs_while_end(struct cs_builder *b, struct cs_loop *loop) { + cs_flush_pending_if(b); cs_branch_label(b, &loop->start, loop->cond, loop->val); cs_set_label(b, &loop->end); cs_block_end(b, &loop->block);