diff --git a/src/panfrost/vulkan/csf/panvk_cmd_buffer.h b/src/panfrost/vulkan/csf/panvk_cmd_buffer.h index 3f7d9337790..6c3432e464e 100644 --- a/src/panfrost/vulkan/csf/panvk_cmd_buffer.h +++ b/src/panfrost/vulkan/csf/panvk_cmd_buffer.h @@ -386,15 +386,6 @@ struct panvk_cmd_buffer { VK_DEFINE_HANDLE_CASTS(panvk_cmd_buffer, vk.base, VkCommandBuffer, VK_OBJECT_TYPE_COMMAND_BUFFER) -static inline uint32_t -next_iter_sb(struct panvk_cmd_buffer *cmdbuf, uint32_t iter_sb_idx) -{ - struct panvk_device *dev = to_panvk_device(cmdbuf->vk.base.device); - - return iter_sb_idx + 1 < dev->csf.sb.iter_count ? SB_ITER(iter_sb_idx + 1) - : SB_ITER(0); -} - static bool inherits_render_ctx(struct panvk_cmd_buffer *cmdbuf) { @@ -431,8 +422,9 @@ extern const struct vk_command_buffer_ops panvk_per_arch(cmd_buffer_ops); void panvk_per_arch(cmd_flush_draws)(struct panvk_cmd_buffer *cmdbuf); -void panvk_per_arch(cs_pick_iter_sb)(struct panvk_cmd_buffer *cmdbuf, - enum panvk_subqueue_id subqueue); +void panvk_per_arch(cs_next_iter_sb)(struct panvk_cmd_buffer *cmdbuf, + enum panvk_subqueue_id subqueue, + struct cs_index scratch_regs); void panvk_per_arch(get_cs_deps)(struct panvk_cmd_buffer *cmdbuf, const VkDependencyInfo *in, diff --git a/src/panfrost/vulkan/csf/panvk_vX_cmd_buffer.c b/src/panfrost/vulkan/csf/panvk_vX_cmd_buffer.c index 5579d200339..154b705a3aa 100644 --- a/src/panfrost/vulkan/csf/panvk_vX_cmd_buffer.c +++ b/src/panfrost/vulkan/csf/panvk_vX_cmd_buffer.c @@ -618,16 +618,25 @@ panvk_per_arch(CmdPipelineBarrier2)(VkCommandBuffer commandBuffer, } void -panvk_per_arch(cs_pick_iter_sb)(struct panvk_cmd_buffer *cmdbuf, - enum panvk_subqueue_id subqueue) +panvk_per_arch(cs_next_iter_sb)(struct panvk_cmd_buffer *cmdbuf, + enum panvk_subqueue_id subqueue, + struct cs_index scratch_regs) { + struct panvk_device *dev = to_panvk_device(cmdbuf->vk.base.device); struct cs_builder *b = panvk_get_cs_builder(cmdbuf, subqueue); - struct cs_index iter_sb = cs_scratch_reg32(b, 0); - struct cs_index cmp_scratch = cs_scratch_reg32(b, 1); + struct cs_index iter_sb = cs_extract32(b, scratch_regs, 0); + struct cs_index cmp_scratch = cs_extract32(b, scratch_regs, 1); cs_load32_to(b, iter_sb, cs_subqueue_ctx_reg(b), offsetof(struct panvk_cs_subqueue_context, iter_sb)); + /* Select next scoreboard entry and wrap around if we get past the limit */ + cs_add32(b, iter_sb, iter_sb, 1); + cs_add32(b, cmp_scratch, iter_sb, -SB_ITER(dev->csf.sb.iter_count)); + cs_if(b, MALI_CS_CONDITION_GEQUAL, cmp_scratch) { + cs_move32_to(b, iter_sb, SB_ITER(0)); + } + cs_match(b, iter_sb, cmp_scratch) { #define CASE(x) \ cs_case(b, SB_ITER(x)) { \ @@ -642,6 +651,10 @@ panvk_per_arch(cs_pick_iter_sb)(struct panvk_cmd_buffer *cmdbuf, CASE(4) #undef CASE } + + cs_store32(b, iter_sb, cs_subqueue_ctx_reg(b), + offsetof(struct panvk_cs_subqueue_context, iter_sb)); + cs_flush_stores(b); } static struct cs_buffer diff --git a/src/panfrost/vulkan/csf/panvk_vX_cmd_dispatch.c b/src/panfrost/vulkan/csf/panvk_vX_cmd_dispatch.c index 6ae2fdb9ffc..2f0acec0150 100644 --- a/src/panfrost/vulkan/csf/panvk_vX_cmd_dispatch.c +++ b/src/panfrost/vulkan/csf/panvk_vX_cmd_dispatch.c @@ -272,7 +272,9 @@ cmd_dispatch(struct panvk_cmd_buffer *cmdbuf, struct panvk_dispatch_info *info) } } - panvk_per_arch(cs_pick_iter_sb)(cmdbuf, PANVK_SUBQUEUE_COMPUTE); + struct cs_index next_iter_sb_scratch = cs_scratch_reg_tuple(b, 0, 2); + panvk_per_arch(cs_next_iter_sb)(cmdbuf, PANVK_SUBQUEUE_COMPUTE, + next_iter_sb_scratch); if (indirect) { /* Use run_compute with a set task axis instead of run_compute_indirect as @@ -314,7 +316,6 @@ cmd_dispatch(struct panvk_cmd_buffer *cmdbuf, struct panvk_dispatch_info *info) cs_case(b, SB_ITER(x)) { \ cs_sync64_add(b, true, MALI_CS_SYNC_SCOPE_CSG, add_val, sync_addr, \ cs_defer(SB_WAIT_ITER(x), SB_ID(DEFERRED_SYNC))); \ - cs_move32_to(b, iter_sb, next_iter_sb(cmdbuf, x)); \ } CASE(0) @@ -325,10 +326,6 @@ cmd_dispatch(struct panvk_cmd_buffer *cmdbuf, struct panvk_dispatch_info *info) #undef CASE } - cs_store32(b, iter_sb, cs_subqueue_ctx_reg(b), - offsetof(struct panvk_cs_subqueue_context, iter_sb)); - cs_flush_stores(b); - ++cmdbuf->state.cs[PANVK_SUBQUEUE_COMPUTE].relative_sync_point; clear_dirty_after_dispatch(cmdbuf); } diff --git a/src/panfrost/vulkan/csf/panvk_vX_cmd_draw.c b/src/panfrost/vulkan/csf/panvk_vX_cmd_draw.c index e07a88222dd..c3b73c9a48b 100644 --- a/src/panfrost/vulkan/csf/panvk_vX_cmd_draw.c +++ b/src/panfrost/vulkan/csf/panvk_vX_cmd_draw.c @@ -1169,8 +1169,9 @@ get_tiler_desc(struct panvk_cmd_buffer *cmdbuf) /* Flush all stores to tiler_ctx_addr. */ cs_flush_stores(b); - /* Then we change the scoreboard slot used for iterators. */ - panvk_per_arch(cs_pick_iter_sb)(cmdbuf, PANVK_SUBQUEUE_VERTEX_TILER); + struct cs_index next_iter_sb_scratch = cs_scratch_reg_tuple(b, 0, 2); + panvk_per_arch(cs_next_iter_sb)(cmdbuf, PANVK_SUBQUEUE_VERTEX_TILER, + next_iter_sb_scratch); cs_heap_operation(b, MALI_CS_HEAP_OPERATION_VERTEX_TILER_STARTED, cs_now()); return VK_SUCCESS; @@ -2875,7 +2876,6 @@ flush_tiling(struct panvk_cmd_buffer *cmdbuf) cs_defer(SB_WAIT_ITER(x), SB_ID(DEFERRED_SYNC))); \ cs_sync64_add(b, true, MALI_CS_SYNC_SCOPE_CSG, add_val, sync_addr, \ cs_defer(SB_WAIT_ITER(x), SB_ID(DEFERRED_SYNC))); \ - cs_move32_to(b, iter_sb, next_iter_sb(cmdbuf, x)); \ } CASE(0) @@ -2886,10 +2886,6 @@ flush_tiling(struct panvk_cmd_buffer *cmdbuf) #undef CASE } - cs_store32(b, iter_sb, cs_subqueue_ctx_reg(b), - offsetof(struct panvk_cs_subqueue_context, iter_sb)); - cs_flush_stores(b); - /* Update the vertex seqno. */ ++cmdbuf->state.cs[PANVK_SUBQUEUE_VERTEX_TILER].relative_sync_point; } @@ -2978,8 +2974,9 @@ issue_fragment_jobs(struct panvk_cmd_buffer *cmdbuf) struct cs_builder *b = panvk_get_cs_builder(cmdbuf, PANVK_SUBQUEUE_FRAGMENT); bool has_oq_chain = cmdbuf->state.gfx.render.oq.chain != 0; - /* Reserve a scoreboard for the fragment job. */ - panvk_per_arch(cs_pick_iter_sb)(cmdbuf, PANVK_SUBQUEUE_FRAGMENT); + struct cs_index next_iter_sb_scratch = cs_scratch_reg_tuple(b, 0, 2); + panvk_per_arch(cs_next_iter_sb)(cmdbuf, PANVK_SUBQUEUE_FRAGMENT, + next_iter_sb_scratch); /* Now initialize the fragment bits. */ cs_update_frag_ctx(b) { @@ -3178,7 +3175,6 @@ issue_fragment_jobs(struct panvk_cmd_buffer *cmdbuf) } \ cs_sync64_add(b, true, MALI_CS_SYNC_SCOPE_CSG, add_val, sync_addr, \ async); \ - cs_move32_to(b, iter_sb, next_iter_sb(cmdbuf, x)); \ } CASE(0) @@ -3189,10 +3185,6 @@ issue_fragment_jobs(struct panvk_cmd_buffer *cmdbuf) #undef CASE } - cs_store32(b, iter_sb, cs_subqueue_ctx_reg(b), - offsetof(struct panvk_cs_subqueue_context, iter_sb)); - cs_flush_stores(b); - /* Update the ring buffer position. */ if (free_render_descs) { cs_render_desc_ringbuf_move_ptr(b, calc_render_descs_size(cmdbuf), diff --git a/src/panfrost/vulkan/csf/panvk_vX_cmd_precomp.c b/src/panfrost/vulkan/csf/panvk_vX_cmd_precomp.c index 537d943bd8e..bf253c85d85 100644 --- a/src/panfrost/vulkan/csf/panvk_vX_cmd_precomp.c +++ b/src/panfrost/vulkan/csf/panvk_vX_cmd_precomp.c @@ -104,7 +104,9 @@ panvk_per_arch(dispatch_precomp)(struct panvk_precomp_ctx *ctx, cs_move32_to(b, cs_sr_reg32(b, COMPUTE, JOB_SIZE_Z), grid.count[2]); } - panvk_per_arch(cs_pick_iter_sb)(cmdbuf, PANVK_SUBQUEUE_COMPUTE); + struct cs_index next_iter_sb_scratch = cs_scratch_reg_tuple(b, 0, 2); + panvk_per_arch(cs_next_iter_sb)(cmdbuf, PANVK_SUBQUEUE_COMPUTE, + next_iter_sb_scratch); unsigned task_axis = MALI_TASK_AXIS_X; unsigned task_increment = 0; @@ -132,7 +134,6 @@ panvk_per_arch(dispatch_precomp)(struct panvk_precomp_ctx *ctx, cs_case(b, SB_ITER(x)) { \ cs_sync64_add(b, true, MALI_CS_SYNC_SCOPE_CSG, add_val, sync_addr, \ cs_defer(SB_WAIT_ITER(x), SB_ID(DEFERRED_SYNC))); \ - cs_move32_to(b, iter_sb, next_iter_sb(cmdbuf, x)); \ } CASE(0) @@ -143,10 +144,6 @@ panvk_per_arch(dispatch_precomp)(struct panvk_precomp_ctx *ctx, #undef CASE } - cs_store32(b, iter_sb, cs_subqueue_ctx_reg(b), - offsetof(struct panvk_cs_subqueue_context, iter_sb)); - cs_flush_stores(b); - ++cmdbuf->state.cs[PANVK_SUBQUEUE_COMPUTE].relative_sync_point; /* XXX: clobbers the registers instead to avoid recreating them when calling diff --git a/src/panfrost/vulkan/csf/panvk_vX_queue.c b/src/panfrost/vulkan/csf/panvk_vX_queue.c index 35a46d99d7a..05e5052bf0d 100644 --- a/src/panfrost/vulkan/csf/panvk_vX_queue.c +++ b/src/panfrost/vulkan/csf/panvk_vX_queue.c @@ -395,7 +395,10 @@ init_subqueue(struct panvk_queue *queue, enum panvk_subqueue_id subqueue) .syncobjs = panvk_priv_mem_dev_addr(queue->syncobjs), .debug.syncobjs = panvk_priv_mem_dev_addr(queue->debug_syncobjs), .debug.tracebuf.cs = subq->tracebuf.addr.dev, - .iter_sb = SB_ITER(0), + /* Iterator scoreboard will be picked in CS and wrap back to SB_ITER(0) on + first RUN_* so we ensure an invalid value here that is handled by our + partial modulo implementation */ + .iter_sb = SB_ITER(dev->csf.sb.iter_count), .reg_dump_addr = panvk_priv_mem_dev_addr(subq->regs_save), };