From 0e358cec52d13901eeb1c23994eb41c54b7459ae Mon Sep 17 00:00:00 2001 From: Samuel Pitoiset Date: Fri, 10 Oct 2025 10:56:46 +0200 Subject: [PATCH] amd,radv,radeonsi: add ac_emit_cp_release_mem_pws() Signed-off-by: Samuel Pitoiset Part-of: --- src/amd/common/ac_cmdbuf.c | 56 ++++++++++++++++++++++ src/amd/common/ac_cmdbuf.h | 5 ++ src/amd/vulkan/radv_cs.c | 32 +------------ src/gallium/drivers/radeonsi/si_cp_utils.c | 53 ++------------------ 4 files changed, 66 insertions(+), 80 deletions(-) diff --git a/src/amd/common/ac_cmdbuf.c b/src/amd/common/ac_cmdbuf.c index 6f07262f43f..23a8f6a0e10 100644 --- a/src/amd/common/ac_cmdbuf.c +++ b/src/amd/common/ac_cmdbuf.c @@ -979,3 +979,59 @@ ac_emit_cp_acquire_mem_pws(struct ac_cmdbuf *cs, ASSERTED enum amd_gfx_level gfx ac_cmdbuf_emit(gcr_cntl); /* GCR_CNTL (this has no effect if PWS_STAGE_SEL isn't PFP or ME) */ ac_cmdbuf_end(); } + +/* Insert CS_DONE, PS_DONE, or a *_TS event into the pipeline, which will + * signal after the work indicated by the event is complete, which optionally + * includes flushing caches using "gcr_cntl" after the completion of the work. + * *_TS events are always signaled at the end of the pipeline, while CS_DONE + * and PS_DONE are signaled when those shaders finish. This call only inserts + * the event into the pipeline. It doesn't wait for anything and it doesn't + * execute anything immediately. The only way to wait for the event completion + * is to call si_cp_acquire_mem_pws with the same "event_type". + */ +void +ac_emit_cp_release_mem_pws(struct ac_cmdbuf *cs, ASSERTED enum amd_gfx_level gfx_level, + ASSERTED enum amd_ip_type ip_type, uint32_t event_type, + uint32_t gcr_cntl) +{ + assert(gfx_level >= GFX11 && ip_type == AMD_IP_GFX); + + /* Extract GCR_CNTL fields because the encoding is different in RELEASE_MEM. */ + assert(G_586_GLI_INV(gcr_cntl) == 0); + assert(G_586_GL1_RANGE(gcr_cntl) == 0); + const uint32_t glm_wb = G_586_GLM_WB(gcr_cntl); + const uint32_t glm_inv = G_586_GLM_INV(gcr_cntl); + const uint32_t glk_wb = G_586_GLK_WB(gcr_cntl); + const uint32_t glk_inv = G_586_GLK_INV(gcr_cntl); + const uint32_t glv_inv = G_586_GLV_INV(gcr_cntl); + const uint32_t gl1_inv = G_586_GL1_INV(gcr_cntl); + assert(G_586_GL2_US(gcr_cntl) == 0); + assert(G_586_GL2_RANGE(gcr_cntl) == 0); + assert(G_586_GL2_DISCARD(gcr_cntl) == 0); + const uint32_t gl2_inv = G_586_GL2_INV(gcr_cntl); + const uint32_t gl2_wb = G_586_GL2_WB(gcr_cntl); + const uint32_t gcr_seq = G_586_SEQ(gcr_cntl); + const bool ts = is_ts_event(event_type); + + ac_cmdbuf_begin(cs); + ac_cmdbuf_emit(PKT3(PKT3_RELEASE_MEM, 6, 0)); + ac_cmdbuf_emit(S_490_EVENT_TYPE(event_type) | + S_490_EVENT_INDEX(ts ? 5 : 6) | + S_490_GLM_WB(glm_wb) | + S_490_GLM_INV(glm_inv) | + S_490_GLV_INV(glv_inv) | + S_490_GL1_INV(gl1_inv) | + S_490_GL2_INV(gl2_inv) | + S_490_GL2_WB(gl2_wb) | + S_490_SEQ(gcr_seq) | + S_490_GLK_WB(glk_wb) | + S_490_GLK_INV(glk_inv) | + S_490_PWS_ENABLE(1)); + ac_cmdbuf_emit(0); /* DST_SEL, INT_SEL, DATA_SEL */ + ac_cmdbuf_emit(0); /* ADDRESS_LO */ + ac_cmdbuf_emit(0); /* ADDRESS_HI */ + ac_cmdbuf_emit(0); /* DATA_LO */ + ac_cmdbuf_emit(0); /* DATA_HI */ + ac_cmdbuf_emit(0); /* INT_CTXID */ + ac_cmdbuf_end(); +} diff --git a/src/amd/common/ac_cmdbuf.h b/src/amd/common/ac_cmdbuf.h index 31cf94174db..a4b8f4df0af 100644 --- a/src/amd/common/ac_cmdbuf.h +++ b/src/amd/common/ac_cmdbuf.h @@ -115,6 +115,11 @@ ac_emit_cp_acquire_mem_pws(struct ac_cmdbuf *cs, ASSERTED enum amd_gfx_level gfx uint32_t stage_sel, uint32_t count, uint32_t gcr_cntl); +void +ac_emit_cp_release_mem_pws(struct ac_cmdbuf *cs, ASSERTED enum amd_gfx_level gfx_level, + ASSERTED enum amd_ip_type ip_type, uint32_t event_type, + uint32_t gcr_cntl); + #ifdef __cplusplus } #endif diff --git a/src/amd/vulkan/radv_cs.c b/src/amd/vulkan/radv_cs.c index 74afd70c36a..6cf7936a7c3 100644 --- a/src/amd/vulkan/radv_cs.c +++ b/src/amd/vulkan/radv_cs.c @@ -242,40 +242,12 @@ gfx10_cs_emit_cache_flush(struct radv_cmd_stream *cs, enum amd_gfx_level gfx_lev if (cb_db_event) { if (gfx_level >= GFX11) { - /* Get GCR_CNTL fields, because the encoding is different in RELEASE_MEM. */ - unsigned glm_wb = G_586_GLM_WB(gcr_cntl); - unsigned glm_inv = G_586_GLM_INV(gcr_cntl); - unsigned glk_wb = G_586_GLK_WB(gcr_cntl); - unsigned glk_inv = G_586_GLK_INV(gcr_cntl); - unsigned glv_inv = G_586_GLV_INV(gcr_cntl); - unsigned gl1_inv = G_586_GL1_INV(gcr_cntl); - assert(G_586_GL2_US(gcr_cntl) == 0); - assert(G_586_GL2_RANGE(gcr_cntl) == 0); - assert(G_586_GL2_DISCARD(gcr_cntl) == 0); - unsigned gl2_inv = G_586_GL2_INV(gcr_cntl); - unsigned gl2_wb = G_586_GL2_WB(gcr_cntl); - unsigned gcr_seq = G_586_SEQ(gcr_cntl); + /* Send an event that flushes caches. */ + ac_emit_cp_release_mem_pws(cs->b, gfx_level, cs->hw_ip, cb_db_event, gcr_cntl); gcr_cntl &= C_586_GLM_WB & C_586_GLM_INV & C_586_GLK_WB & C_586_GLK_INV & C_586_GLV_INV & C_586_GL1_INV & C_586_GL2_INV & C_586_GL2_WB; /* keep SEQ */ - radeon_begin(cs); - - /* Send an event that flushes caches. */ - radeon_emit(PKT3(PKT3_RELEASE_MEM, 6, 0)); - radeon_emit(S_490_EVENT_TYPE(cb_db_event) | S_490_EVENT_INDEX(5) | S_490_GLM_WB(glm_wb) | - S_490_GLM_INV(glm_inv) | S_490_GLV_INV(glv_inv) | S_490_GL1_INV(gl1_inv) | S_490_GL2_INV(gl2_inv) | - S_490_GL2_WB(gl2_wb) | S_490_SEQ(gcr_seq) | S_490_GLK_WB(glk_wb) | S_490_GLK_INV(glk_inv) | - S_490_PWS_ENABLE(1)); - radeon_emit(0); /* DST_SEL, INT_SEL, DATA_SEL */ - radeon_emit(0); /* ADDRESS_LO */ - radeon_emit(0); /* ADDRESS_HI */ - radeon_emit(0); /* DATA_LO */ - radeon_emit(0); /* DATA_HI */ - radeon_emit(0); /* INT_CTXID */ - - radeon_end(); - /* Wait for the event and invalidate remaining caches if needed. */ ac_emit_cp_acquire_mem_pws(cs->b, gfx_level, cs->hw_ip, cb_db_event, V_580_CP_PFP, 0, gcr_cntl); diff --git a/src/gallium/drivers/radeonsi/si_cp_utils.c b/src/gallium/drivers/radeonsi/si_cp_utils.c index dc5d920f186..425c12242b6 100644 --- a/src/gallium/drivers/radeonsi/si_cp_utils.c +++ b/src/gallium/drivers/radeonsi/si_cp_utils.c @@ -6,59 +6,12 @@ #include "si_build_pm4.h" -static bool is_ts_event(unsigned event_type) -{ - return event_type == V_028A90_CACHE_FLUSH_TS || - event_type == V_028A90_CACHE_FLUSH_AND_INV_TS_EVENT || - event_type == V_028A90_BOTTOM_OF_PIPE_TS || - event_type == V_028A90_FLUSH_AND_INV_DB_DATA_TS || - event_type == V_028A90_FLUSH_AND_INV_CB_DATA_TS; -} - -/* Insert CS_DONE, PS_DONE, or a *_TS event into the pipeline, which will signal after the work - * indicated by the event is complete, which optionally includes flushing caches using "gcr_cntl" - * after the completion of the work. *_TS events are always signaled at the end of the pipeline, - * while CS_DONE and PS_DONE are signaled when those shaders finish. This call only inserts - * the event into the pipeline. It doesn't wait for anything and it doesn't execute anything - * immediately. The only way to wait for the event completion is to call si_cp_acquire_mem_pws - * with the same "event_type". - */ void si_cp_release_mem_pws(struct si_context *sctx, struct radeon_cmdbuf *cs, unsigned event_type, unsigned gcr_cntl) { - assert(sctx->gfx_level >= GFX11 && sctx->is_gfx_queue); - bool ts = is_ts_event(event_type); - /* Extract GCR_CNTL fields because the encoding is different in RELEASE_MEM. */ - assert(G_586_GLI_INV(gcr_cntl) == 0); - assert(G_586_GL1_RANGE(gcr_cntl) == 0); - unsigned glm_wb = G_586_GLM_WB(gcr_cntl); - unsigned glm_inv = G_586_GLM_INV(gcr_cntl); - unsigned glk_wb = G_586_GLK_WB(gcr_cntl); - unsigned glk_inv = G_586_GLK_INV(gcr_cntl); - unsigned glv_inv = G_586_GLV_INV(gcr_cntl); - unsigned gl1_inv = G_586_GL1_INV(gcr_cntl); - assert(G_586_GL2_US(gcr_cntl) == 0); - assert(G_586_GL2_RANGE(gcr_cntl) == 0); - assert(G_586_GL2_DISCARD(gcr_cntl) == 0); - unsigned gl2_inv = G_586_GL2_INV(gcr_cntl); - unsigned gl2_wb = G_586_GL2_WB(gcr_cntl); - unsigned gcr_seq = G_586_SEQ(gcr_cntl); - - radeon_begin(cs); - radeon_emit(PKT3(PKT3_RELEASE_MEM, 6, 0)); - radeon_emit(S_490_EVENT_TYPE(event_type) | - S_490_EVENT_INDEX(ts ? 5 : 6) | - S_490_GLM_WB(glm_wb) | S_490_GLM_INV(glm_inv) | S_490_GLV_INV(glv_inv) | - S_490_GL1_INV(gl1_inv) | S_490_GL2_INV(gl2_inv) | S_490_GL2_WB(gl2_wb) | - S_490_SEQ(gcr_seq) | S_490_GLK_WB(glk_wb) | S_490_GLK_INV(glk_inv) | - S_490_PWS_ENABLE(1)); - radeon_emit(0); /* DST_SEL, INT_SEL, DATA_SEL */ - radeon_emit(0); /* ADDRESS_LO */ - radeon_emit(0); /* ADDRESS_HI */ - radeon_emit(0); /* DATA_LO */ - radeon_emit(0); /* DATA_HI */ - radeon_emit(0); /* INT_CTXID */ - radeon_end(); + ac_emit_cp_release_mem_pws(&cs->current, sctx->gfx_level, + sctx->is_gfx_queue ? AMD_IP_GFX : AMD_IP_COMPUTE, + event_type, gcr_cntl); } void si_cp_acquire_mem_pws(struct si_context *sctx, struct radeon_cmdbuf *cs,