diff --git a/src/panfrost/vulkan/csf/panvk_instr.h b/src/panfrost/vulkan/csf/panvk_instr.h index b2f6faad694..68bd8c6505c 100644 --- a/src/panfrost/vulkan/csf/panvk_instr.h +++ b/src/panfrost/vulkan/csf/panvk_instr.h @@ -14,7 +14,8 @@ enum panvk_instr_work_type { PANVK_INSTR_WORK_TYPE_DISPATCH, PANVK_INSTR_WORK_TYPE_DISPATCH_INDIRECT, PANVK_INSTR_WORK_TYPE_BARRIER, - PANVK_INSTR_WORK_TYPE_SYNC_WAIT, + PANVK_INSTR_WORK_TYPE_SYNC32_WAIT, + PANVK_INSTR_WORK_TYPE_SYNC64_WAIT, }; struct panvk_instr_end_args { @@ -52,6 +53,12 @@ struct panvk_instr_end_args { struct { uint64_t buffer_gpu; } dispatch_indirect; + + struct { + struct cs_index addr_regs; + struct cs_index val_regs; + enum mali_cs_condition cond; + } sync; }; }; @@ -77,3 +84,24 @@ void panvk_per_arch(panvk_instr_end_work_async)( enum panvk_instr_work_type work_type, const struct panvk_instr_end_args *const args, struct cs_async_op ts_async_op); + +#define PANVK_INSTR_SYNC_OPS(__cnt_width) \ + static inline void panvk_instr_sync##__cnt_width##_wait( \ + struct panvk_cmd_buffer *cmdbuf, enum panvk_subqueue_id id, \ + bool reject_error, enum mali_cs_condition cond, struct cs_index ref, \ + struct cs_index addr) \ + { \ + struct cs_builder *b = panvk_get_cs_builder(cmdbuf, id); \ + panvk_per_arch(panvk_instr_begin_work)( \ + id, cmdbuf, PANVK_INSTR_WORK_TYPE_SYNC##__cnt_width##_WAIT); \ + cs_sync##__cnt_width##_wait(b, reject_error, cond, ref, addr); \ + struct panvk_instr_end_args instr_info = { \ + .sync = {.addr_regs = addr, .val_regs = ref, .cond = cond}, \ + }; \ + panvk_per_arch(panvk_instr_end_work)( \ + id, cmdbuf, PANVK_INSTR_WORK_TYPE_SYNC##__cnt_width##_WAIT, \ + &instr_info); \ + } + +PANVK_INSTR_SYNC_OPS(32) +PANVK_INSTR_SYNC_OPS(64) diff --git a/src/panfrost/vulkan/csf/panvk_vX_cmd_buffer.c b/src/panfrost/vulkan/csf/panvk_vX_cmd_buffer.c index 6bdfed03116..90115c3fcc7 100644 --- a/src/panfrost/vulkan/csf/panvk_vX_cmd_buffer.c +++ b/src/panfrost/vulkan/csf/panvk_vX_cmd_buffer.c @@ -581,7 +581,9 @@ emit_barrier_insert_waits(struct cs_builder *b, struct panvk_cmd_buffer *cmdbuf, cs_add64(b, wait_val, cs_progress_seqno_reg(b, j), cs_state->relative_sync_point); - cs_sync64_wait(b, false, MALI_CS_CONDITION_GREATER, wait_val, sync_addr); + + panvk_instr_sync64_wait(cmdbuf, i, false, MALI_CS_CONDITION_GREATER, + wait_val, sync_addr); } } diff --git a/src/panfrost/vulkan/csf/panvk_vX_cmd_draw.c b/src/panfrost/vulkan/csf/panvk_vX_cmd_draw.c index 555be62f273..68c024add2c 100644 --- a/src/panfrost/vulkan/csf/panvk_vX_cmd_draw.c +++ b/src/panfrost/vulkan/csf/panvk_vX_cmd_draw.c @@ -3001,8 +3001,10 @@ wait_finish_tiling(struct panvk_cmd_buffer *cmdbuf) cs_add64(b, vt_sync_point, cs_progress_seqno_reg(b, PANVK_SUBQUEUE_VERTEX_TILER), rel_vt_sync_point); - cs_sync64_wait(b, false, MALI_CS_CONDITION_GREATER, vt_sync_point, - vt_sync_addr); + + panvk_instr_sync64_wait(cmdbuf, PANVK_SUBQUEUE_FRAGMENT, false, + MALI_CS_CONDITION_GREATER, vt_sync_point, + vt_sync_addr); } static uint32_t @@ -3117,11 +3119,7 @@ issue_fragment_jobs(struct panvk_cmd_buffer *cmdbuf) length_reg); /* Wait for the tiling to be done before submitting the fragment job. */ - panvk_per_arch(panvk_instr_begin_work)(PANVK_SUBQUEUE_FRAGMENT, cmdbuf, - PANVK_INSTR_WORK_TYPE_SYNC_WAIT); wait_finish_tiling(cmdbuf); - panvk_per_arch(panvk_instr_end_work)(PANVK_SUBQUEUE_FRAGMENT, cmdbuf, - PANVK_INSTR_WORK_TYPE_SYNC_WAIT, NULL); /* Disable the oom handler once the vertex/tiler work has finished. * We need to disable the handler at this point as the vertex/tiler subqueue diff --git a/src/panfrost/vulkan/csf/panvk_vX_cmd_event.c b/src/panfrost/vulkan/csf/panvk_vX_cmd_event.c index e42f0a7b377..e2d48eb39eb 100644 --- a/src/panfrost/vulkan/csf/panvk_vX_cmd_event.c +++ b/src/panfrost/vulkan/csf/panvk_vX_cmd_event.c @@ -6,6 +6,7 @@ #include "panvk_cmd_buffer.h" #include "panvk_entrypoints.h" #include "panvk_event.h" +#include "panvk_instr.h" #include "util/bitscan.h" @@ -124,7 +125,8 @@ cmd_wait_event(struct panvk_cmd_buffer *cmdbuf, struct panvk_event *event, (j * sizeof(struct panvk_cs_sync32))); cs_move32_to(b, seqno, 0); - cs_sync32_wait(b, false, MALI_CS_CONDITION_GREATER, seqno, sync_addr); + panvk_instr_sync32_wait(cmdbuf, i, false, MALI_CS_CONDITION_GREATER, + seqno, sync_addr); } } } diff --git a/src/panfrost/vulkan/csf/panvk_vX_instr.c b/src/panfrost/vulkan/csf/panvk_vX_instr.c index 4a6e7e53a6a..051c31a274e 100644 --- a/src/panfrost/vulkan/csf/panvk_vX_instr.c +++ b/src/panfrost/vulkan/csf/panvk_vX_instr.c @@ -63,6 +63,42 @@ panvk_instr_end_dispatch_indirect(enum panvk_subqueue_id id, }); } +static void +panvk_instr_end_sync32_wait(enum panvk_subqueue_id id, + struct panvk_utrace_cs_info *cs_info, + const struct panvk_instr_end_args *const args) +{ + assert(args->sync.val_regs.size == 1); + trace_end_sync32_wait(&cs_info->cmdbuf->utrace.uts[id], cs_info, + (struct u_trace_address){ + .bo = (void *)PANVK_UTRACE_CAPTURE_REGISTERS, + .offset = args->sync.addr_regs.reg, + }, + (struct u_trace_address){ + .bo = (void *)PANVK_UTRACE_CAPTURE_REGISTERS, + .offset = args->sync.val_regs.reg, + }, + args->sync.cond); +} + +static void +panvk_instr_end_sync64_wait(enum panvk_subqueue_id id, + struct panvk_utrace_cs_info *cs_info, + const struct panvk_instr_end_args *const args) +{ + assert(args->sync.val_regs.size == 2); + trace_end_sync64_wait(&cs_info->cmdbuf->utrace.uts[id], cs_info, + (struct u_trace_address){ + .bo = (void *)PANVK_UTRACE_CAPTURE_REGISTERS, + .offset = args->sync.addr_regs.reg, + }, + (struct u_trace_address){ + .bo = (void *)PANVK_UTRACE_CAPTURE_REGISTERS, + .offset = args->sync.val_regs.reg, + }, + args->sync.cond); +} + void panvk_per_arch(panvk_instr_begin_work)(enum panvk_subqueue_id id, struct panvk_cmd_buffer *cmdbuf, @@ -95,8 +131,11 @@ panvk_per_arch(panvk_instr_begin_work)(enum panvk_subqueue_id id, case PANVK_INSTR_WORK_TYPE_BARRIER: trace_begin_barrier(&cmdbuf->utrace.uts[id], &cs_info); break; - case PANVK_INSTR_WORK_TYPE_SYNC_WAIT: - trace_begin_sync_wait(&cmdbuf->utrace.uts[id], &cs_info); + case PANVK_INSTR_WORK_TYPE_SYNC32_WAIT: + trace_begin_sync32_wait(&cmdbuf->utrace.uts[id], &cs_info); + break; + case PANVK_INSTR_WORK_TYPE_SYNC64_WAIT: + trace_begin_sync64_wait(&cmdbuf->utrace.uts[id], &cs_info); break; default: UNREACHABLE("unsupported panvk_instr_work_type"); @@ -144,8 +183,11 @@ panvk_per_arch(panvk_instr_end_work_async)( case PANVK_INSTR_WORK_TYPE_BARRIER: panvk_instr_end_barrier(id, &cs_info, args); break; - case PANVK_INSTR_WORK_TYPE_SYNC_WAIT: - trace_end_sync_wait(&cmdbuf->utrace.uts[id], &cs_info); + case PANVK_INSTR_WORK_TYPE_SYNC32_WAIT: + panvk_instr_end_sync32_wait(id, &cs_info, args); + break; + case PANVK_INSTR_WORK_TYPE_SYNC64_WAIT: + panvk_instr_end_sync64_wait(id, &cs_info, args); break; default: UNREACHABLE("unsupported panvk_instr_work_type"); diff --git a/src/panfrost/vulkan/panvk_tracepoints.py b/src/panfrost/vulkan/panvk_tracepoints.py index 079f320078d..c59ea61670b 100644 --- a/src/panfrost/vulkan/panvk_tracepoints.py +++ b/src/panfrost/vulkan/panvk_tracepoints.py @@ -222,8 +222,51 @@ def define_tracepoints(): # low-level tracepoints for CS commands - begin_end_tp('sync_wait') + begin_end_tp( + 'sync32_wait', + args=[ + Arg( + type='uint64_t', + var='sync_addr', + is_indirect=True, + c_format='0x%" PRIx64 "' + ), + Arg( + type='uint32_t', + var='sync_val', + is_indirect=True, + c_format='%u', + ), + Arg( + type='unsigned', + var='cond', + c_format='%u', + ), + ], + ) + begin_end_tp( + 'sync64_wait', + args=[ + Arg( + type='uint64_t', + var='sync_addr', + is_indirect=True, + c_format='0x%" PRIx64 "' + ), + Arg( + type='uint64_t', + var='sync_val', + is_indirect=True, + c_format='%" PRIu64 "', + ), + Arg( + type='unsigned', + var='cond', + c_format='%u', + ), + ], + ) def generate_code(): utrace_generate( diff --git a/src/panfrost/vulkan/panvk_utrace_perfetto.cc b/src/panfrost/vulkan/panvk_utrace_perfetto.cc index ab069d203ae..d5c52452ce9 100644 --- a/src/panfrost/vulkan/panvk_utrace_perfetto.cc +++ b/src/panfrost/vulkan/panvk_utrace_perfetto.cc @@ -249,7 +249,8 @@ PANVK_UTRACE_PERFETTO_PROCESS_EVENT(render, RENDER) PANVK_UTRACE_PERFETTO_PROCESS_EVENT(dispatch, DISPATCH) PANVK_UTRACE_PERFETTO_PROCESS_EVENT(dispatch_indirect, DISPATCH) PANVK_UTRACE_PERFETTO_PROCESS_EVENT(barrier, BARRIER) -PANVK_UTRACE_PERFETTO_PROCESS_EVENT(sync_wait, SYNC_WAIT) +PANVK_UTRACE_PERFETTO_PROCESS_EVENT(sync32_wait, SYNC_WAIT) +PANVK_UTRACE_PERFETTO_PROCESS_EVENT(sync64_wait, SYNC_WAIT) static uint32_t get_gpu_clock_id(void)