AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

radv: rework command buffer emission with begin/end sequences

Browse Source 
A begin/end sequence is something like (it's all macros based):

   radeon_begin(cs);
   radeon_emit(PKT3(PKT3_DRAW_INDEX_AUTO, 1, cmd_buffer->state.predicating));
   radeon_emit(vertex_count);
   radeon_emit(V_0287F0_DI_SRC_SEL_AUTO_INDEX | use_opaque);
   radeon_end();

This is loosely based on RadeonSI (see !8653 (a0978fff)) and it seems
indeed faster overall.

The main goal of this rework is to re-use the same logic as RadeonSI
for paired packets on GFX12 (also GFX11 dGPUs) because it's supposed
to be way faster, especially on GFX12 where the CP is slow. The other
goal is to share more cmdbuf emission between both drivers in the near
future.

Signed-off-by: Samuel Pitoiset <samuel.pitoiset@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/34229>
This commit is contained in:
Samuel Pitoiset
2025-03-24 14:07:01 +01:00
committed by Marge Bot
parent d3ec467031
commit f0b3a6f9d4
17 changed files with 2013 additions and 1409 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/amd/vulkan/layers/radv_sqtt_layer.c
+32 -28
View File
@@ -30,23 +30,25 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv
radv_cs_add_buffer(device->ws, cs, reloc->bo);
radeon_begin(cs);
/* VS */
if (pipeline->base.shaders[MESA_SHADER_VERTEX]) {
struct radv_shader *vs = pipeline->base.shaders[MESA_SHADER_VERTEX];
va = reloc->va[MESA_SHADER_VERTEX];
if (vs->info.vs.as_ls) {
radeon_set_sh_reg(cs, vs->info.regs.pgm_lo, va >> 8);
radeon_set_sh_reg(vs->info.regs.pgm_lo, va >> 8);
} else if (vs->info.vs.as_es) {
radeon_set_sh_reg_seq(cs, vs->info.regs.pgm_lo, 2);
radeon_emit(cs, va >> 8);
radeon_emit(cs, S_00B324_MEM_BASE(va >> 40));
radeon_set_sh_reg_seq(vs->info.regs.pgm_lo, 2);
radeon_emit(va >> 8);
radeon_emit(S_00B324_MEM_BASE(va >> 40));
} else if (vs->info.is_ngg) {
radeon_set_sh_reg(cs, vs->info.regs.pgm_lo, va >> 8);
radeon_set_sh_reg(vs->info.regs.pgm_lo, va >> 8);
} else {
radeon_set_sh_reg_seq(cs, vs->info.regs.pgm_lo, 2);
radeon_emit(cs, va >> 8);
radeon_emit(cs, S_00B124_MEM_BASE(va >> 40));
radeon_set_sh_reg_seq(vs->info.regs.pgm_lo, 2);
radeon_emit(va >> 8);
radeon_emit(S_00B124_MEM_BASE(va >> 40));
}
}
@@ -57,11 +59,11 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv
va = reloc->va[MESA_SHADER_TESS_CTRL];
if (gfx_level >= GFX9) {
radeon_set_sh_reg(cs, tcs->info.regs.pgm_lo, va >> 8);
radeon_set_sh_reg(tcs->info.regs.pgm_lo, va >> 8);
} else {
radeon_set_sh_reg_seq(cs, tcs->info.regs.pgm_lo, 2);
radeon_emit(cs, va >> 8);
radeon_emit(cs, S_00B424_MEM_BASE(va >> 40));
radeon_set_sh_reg_seq(tcs->info.regs.pgm_lo, 2);
radeon_emit(va >> 8);
radeon_emit(S_00B424_MEM_BASE(va >> 40));
}
}
@@ -71,15 +73,15 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv
va = reloc->va[MESA_SHADER_TESS_EVAL];
if (tes->info.is_ngg) {
radeon_set_sh_reg(cs, tes->info.regs.pgm_lo, va >> 8);
radeon_set_sh_reg(tes->info.regs.pgm_lo, va >> 8);
} else if (tes->info.tes.as_es) {
radeon_set_sh_reg_seq(cs, tes->info.regs.pgm_lo, 2);
radeon_emit(cs, va >> 8);
radeon_emit(cs, S_00B324_MEM_BASE(va >> 40));
radeon_set_sh_reg_seq(tes->info.regs.pgm_lo, 2);
radeon_emit(va >> 8);
radeon_emit(S_00B324_MEM_BASE(va >> 40));
} else {
radeon_set_sh_reg_seq(cs, tes->info.regs.pgm_lo, 2);
radeon_emit(cs, va >> 8);
radeon_emit(cs, S_00B124_MEM_BASE(va >> 40));
radeon_set_sh_reg_seq(tes->info.regs.pgm_lo, 2);
radeon_emit(va >> 8);
radeon_emit(S_00B124_MEM_BASE(va >> 40));
}
}
@@ -89,14 +91,14 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv
va = reloc->va[MESA_SHADER_GEOMETRY];
if (gs->info.is_ngg) {
radeon_set_sh_reg(cs, gs->info.regs.pgm_lo, va >> 8);
radeon_set_sh_reg(gs->info.regs.pgm_lo, va >> 8);
} else {
if (gfx_level >= GFX9) {
radeon_set_sh_reg(cs, gs->info.regs.pgm_lo, va >> 8);
radeon_set_sh_reg(gs->info.regs.pgm_lo, va >> 8);
} else {
radeon_set_sh_reg_seq(cs, gs->info.regs.pgm_lo, 2);
radeon_emit(cs, va >> 8);
radeon_emit(cs, S_00B224_MEM_BASE(va >> 40));
radeon_set_sh_reg_seq(gs->info.regs.pgm_lo, 2);
radeon_emit(va >> 8);
radeon_emit(S_00B224_MEM_BASE(va >> 40));
}
}
}
@@ -107,9 +109,9 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv
va = reloc->va[MESA_SHADER_FRAGMENT];
radeon_set_sh_reg_seq(cs, ps->info.regs.pgm_lo, 2);
radeon_emit(cs, va >> 8);
radeon_emit(cs, S_00B024_MEM_BASE(va >> 40));
radeon_set_sh_reg_seq(ps->info.regs.pgm_lo, 2);
radeon_emit(va >> 8);
radeon_emit(S_00B024_MEM_BASE(va >> 40));
}
/* MS */
@@ -118,8 +120,10 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv
va = reloc->va[MESA_SHADER_MESH];
radeon_set_sh_reg(cs, ms->info.regs.pgm_lo, va >> 8);
radeon_set_sh_reg(ms->info.regs.pgm_lo, va >> 8);
}
radeon_end();
}
static uint64_t
src/amd/vulkan/meta/radv_meta_buffer.c
+7 -6
View File
@@ -305,12 +305,13 @@ radv_update_buffer_cp(struct radv_cmd_buffer *cmd_buffer, uint64_t va, const voi
radv_emit_cache_flush(cmd_buffer);
radeon_check_space(device->ws, cmd_buffer->cs, words + 4);
radeon_emit(cmd_buffer->cs, PKT3(PKT3_WRITE_DATA, 2 + words, 0));
radeon_emit(cmd_buffer->cs,
S_370_DST_SEL(mec ? V_370_MEM : V_370_MEM_GRBM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME));
radeon_emit(cmd_buffer->cs, va);
radeon_emit(cmd_buffer->cs, va >> 32);
radeon_emit_array(cmd_buffer->cs, data, words);
radeon_begin(cmd_buffer->cs);
radeon_emit(PKT3(PKT3_WRITE_DATA, 2 + words, 0));
radeon_emit(S_370_DST_SEL(mec ? V_370_MEM : V_370_MEM_GRBM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME));
radeon_emit(va);
radeon_emit(va >> 32);
radeon_emit_array(data, words);
radeon_end();
if (radv_device_fault_detection_enabled(device))
radv_cmd_buffer_trace_emit(cmd_buffer);
src/amd/vulkan/radv_cmd_buffer.c
+1147 -794
View File
File diff suppressed because it is too large Load Diff
src/amd/vulkan/radv_cp_dma.c
+28 -22
View File
@@ -80,24 +80,26 @@ radv_cs_emit_cp_dma(struct radv_device *device, struct radeon_cmdbuf *cs, bool p
else if (cp_dma_use_L2)
header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2);
radeon_begin(cs);
if (pdev->info.gfx_level >= GFX7) {
radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, predicating));
radeon_emit(cs, header);
radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */
radeon_emit(cs, src_va >> 32); /* SRC_ADDR_HI [31:0] */
radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */
radeon_emit(cs, dst_va >> 32); /* DST_ADDR_HI [31:0] */
radeon_emit(cs, command);
radeon_emit(PKT3(PKT3_DMA_DATA, 5, predicating));
radeon_emit(header);
radeon_emit(src_va); /* SRC_ADDR_LO [31:0] */
radeon_emit(src_va >> 32); /* SRC_ADDR_HI [31:0] */
radeon_emit(dst_va); /* DST_ADDR_LO [31:0] */
radeon_emit(dst_va >> 32); /* DST_ADDR_HI [31:0] */
radeon_emit(command);
} else {
assert(!cp_dma_use_L2);
header |= S_411_SRC_ADDR_HI(src_va >> 32);
radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, predicating));
radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */
radeon_emit(cs, header); /* SRC_ADDR_HI [15:0] + flags. */
radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */
radeon_emit(cs, (dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */
radeon_emit(cs, command);
radeon_emit(PKT3(PKT3_CP_DMA, 4, predicating));
radeon_emit(src_va); /* SRC_ADDR_LO [31:0] */
radeon_emit(header); /* SRC_ADDR_HI [15:0] + flags. */
radeon_emit(dst_va); /* DST_ADDR_LO [31:0] */
radeon_emit((dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */
radeon_emit(command);
}
radeon_end();
}
static void
@@ -116,8 +118,10 @@ radv_emit_cp_dma(struct radv_cmd_buffer *cmd_buffer, uint64_t dst_va, uint64_t s
*/
if (flags & CP_DMA_SYNC) {
if (cmd_buffer->qf == RADV_QUEUE_GENERAL) {
radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating));
radeon_emit(cs, 0);
radeon_begin(cs);
radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating));
radeon_emit(0);
radeon_end();
}
/* CP will see the sync flag and wait for all DMAs to complete. */
@@ -157,13 +161,15 @@ radv_cs_cp_dma_prefetch(const struct radv_device *device, struct radeon_cmdbuf *
header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2);
radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, predicating));
radeon_emit(cs, header);
radeon_emit(cs, aligned_va); /* SRC_ADDR_LO [31:0] */
radeon_emit(cs, aligned_va >> 32); /* SRC_ADDR_HI [31:0] */
radeon_emit(cs, aligned_va); /* DST_ADDR_LO [31:0] */
radeon_emit(cs, aligned_va >> 32); /* DST_ADDR_HI [31:0] */
radeon_emit(cs, command);
radeon_begin(cs);
radeon_emit(PKT3(PKT3_DMA_DATA, 5, predicating));
radeon_emit(header);
radeon_emit(aligned_va); /* SRC_ADDR_LO [31:0] */
radeon_emit(aligned_va >> 32); /* SRC_ADDR_HI [31:0] */
radeon_emit(aligned_va); /* DST_ADDR_LO [31:0] */
radeon_emit(aligned_va >> 32); /* DST_ADDR_HI [31:0] */
radeon_emit(command);
radeon_end();
}
void
src/amd/vulkan/radv_cp_reg_shadowing.c
+8 -2
View File
@@ -39,7 +39,10 @@ radv_create_shadow_regs_preamble(struct radv_device *device, struct radv_queue_s
if (!pm4)
goto fail_create;
radeon_emit_array(cs, pm4->pm4, pm4->ndw);
radeon_begin(cs);
radeon_emit_array(pm4->pm4, pm4->ndw);
radeon_end();
ws->cs_pad(cs, 0);
result = radv_bo_create(
@@ -126,7 +129,10 @@ radv_init_shadowed_regs_buffer_state(const struct radv_device *device, struct ra
goto fail;
}
radeon_emit_array(cs, pm4->pm4, pm4->ndw);
radeon_begin(cs);
radeon_emit_array(pm4->pm4, pm4->ndw);
radeon_end();
ac_pm4_free_state(pm4);
}
src/amd/vulkan/radv_cs.c
+167 -115
View File
@@ -37,27 +37,29 @@ radv_cs_emit_write_event_eop(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_le
if (data_sel != EOP_DATA_SEL_DISCARD)
sel |= EOP_INT_SEL(EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM);
radeon_begin(cs);
if (gfx_level >= GFX9 || is_gfx8_mec) {
/* A ZPASS_DONE or PIXEL_STAT_DUMP_EVENT (of the DB occlusion
* counters) must immediately precede every timestamp event to
* prevent a GPU hang on GFX9.
*/
if (gfx_level == GFX9 && !is_mec) {
radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
radeon_emit(cs, EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1));
radeon_emit(cs, gfx9_eop_bug_va);
radeon_emit(cs, gfx9_eop_bug_va >> 32);
radeon_emit(PKT3(PKT3_EVENT_WRITE, 2, 0));
radeon_emit(EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1));
radeon_emit(gfx9_eop_bug_va);
radeon_emit(gfx9_eop_bug_va >> 32);
}
radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, is_gfx8_mec ? 5 : 6, false));
radeon_emit(cs, op);
radeon_emit(cs, sel);
radeon_emit(cs, va); /* address lo */
radeon_emit(cs, va >> 32); /* address hi */
radeon_emit(cs, new_fence); /* immediate data lo */
radeon_emit(cs, 0); /* immediate data hi */
radeon_emit(PKT3(PKT3_RELEASE_MEM, is_gfx8_mec ? 5 : 6, false));
radeon_emit(op);
radeon_emit(sel);
radeon_emit(va); /* address lo */
radeon_emit(va >> 32); /* address hi */
radeon_emit(new_fence); /* immediate data lo */
radeon_emit(0); /* immediate data hi */
if (!is_gfx8_mec)
radeon_emit(cs, 0); /* unused */
radeon_emit(0); /* unused */
} else {
/* On GFX6, EOS events are always emitted with EVENT_WRITE_EOS.
* On GFX7+, EOS events are emitted with EVENT_WRITE_EOS on
@@ -68,19 +70,19 @@ radv_cs_emit_write_event_eop(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_le
assert(event_flags == 0 && dst_sel == EOP_DST_SEL_MEM && data_sel == EOP_DATA_SEL_VALUE_32BIT);
if (is_mec) {
radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, 5, false));
radeon_emit(cs, op);
radeon_emit(cs, sel);
radeon_emit(cs, va); /* address lo */
radeon_emit(cs, va >> 32); /* address hi */
radeon_emit(cs, new_fence); /* immediate data lo */
radeon_emit(cs, 0); /* immediate data hi */
radeon_emit(PKT3(PKT3_RELEASE_MEM, 5, false));
radeon_emit(op);
radeon_emit(sel);
radeon_emit(va); /* address lo */
radeon_emit(va >> 32); /* address hi */
radeon_emit(new_fence); /* immediate data lo */
radeon_emit(0); /* immediate data hi */
} else {
radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOS, 3, false));
radeon_emit(cs, op);
radeon_emit(cs, va);
radeon_emit(cs, ((va >> 32) & 0xffff) | EOS_DATA_SEL(EOS_DATA_SEL_VALUE_32BIT));
radeon_emit(cs, new_fence);
radeon_emit(PKT3(PKT3_EVENT_WRITE_EOS, 3, false));
radeon_emit(op);
radeon_emit(va);
radeon_emit(((va >> 32) & 0xffff) | EOS_DATA_SEL(EOS_DATA_SEL_VALUE_32BIT));
radeon_emit(new_fence);
}
} else {
if (gfx_level == GFX7 || gfx_level == GFX8) {
@@ -88,44 +90,50 @@ radv_cs_emit_write_event_eop(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_le
* engines go idle (and optional cache flushes
* executed) before the timestamp is written.
*/
radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, false));
radeon_emit(cs, op);
radeon_emit(cs, va);
radeon_emit(cs, ((va >> 32) & 0xffff) | sel);
radeon_emit(cs, 0); /* immediate data */
radeon_emit(cs, 0); /* unused */
radeon_emit(PKT3(PKT3_EVENT_WRITE_EOP, 4, false));
radeon_emit(op);
radeon_emit(va);
radeon_emit(((va >> 32) & 0xffff) | sel);
radeon_emit(0); /* immediate data */
radeon_emit(0); /* unused */
}
radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, false));
radeon_emit(cs, op);
radeon_emit(cs, va);
radeon_emit(cs, ((va >> 32) & 0xffff) | sel);
radeon_emit(cs, new_fence); /* immediate data */
radeon_emit(cs, 0); /* unused */
radeon_emit(PKT3(PKT3_EVENT_WRITE_EOP, 4, false));
radeon_emit(op);
radeon_emit(va);
radeon_emit(((va >> 32) & 0xffff) | sel);
radeon_emit(new_fence); /* immediate data */
radeon_emit(0); /* unused */
}
}
radeon_end();
}
static void
radv_emit_acquire_mem(struct radeon_cmdbuf *cs, bool is_mec, bool is_gfx9, unsigned cp_coher_cntl)
{
radeon_begin(cs);
if (is_mec || is_gfx9) {
uint32_t hi_val = is_gfx9 ? 0xffffff : 0xff;
radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 5, false) | PKT3_SHADER_TYPE_S(is_mec));
radeon_emit(cs, cp_coher_cntl); /* CP_COHER_CNTL */
radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */
radeon_emit(cs, hi_val); /* CP_COHER_SIZE_HI */
radeon_emit(cs, 0); /* CP_COHER_BASE */
radeon_emit(cs, 0); /* CP_COHER_BASE_HI */
radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */
radeon_emit(PKT3(PKT3_ACQUIRE_MEM, 5, false) | PKT3_SHADER_TYPE_S(is_mec));
radeon_emit(cp_coher_cntl); /* CP_COHER_CNTL */
radeon_emit(0xffffffff); /* CP_COHER_SIZE */
radeon_emit(hi_val); /* CP_COHER_SIZE_HI */
radeon_emit(0); /* CP_COHER_BASE */
radeon_emit(0); /* CP_COHER_BASE_HI */
radeon_emit(0x0000000A); /* POLL_INTERVAL */
} else {
/* ACQUIRE_MEM is only required on a compute ring. */
radeon_emit(cs, PKT3(PKT3_SURFACE_SYNC, 3, false));
radeon_emit(cs, cp_coher_cntl); /* CP_COHER_CNTL */
radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */
radeon_emit(cs, 0); /* CP_COHER_BASE */
radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */
radeon_emit(PKT3(PKT3_SURFACE_SYNC, 3, false));
radeon_emit(cp_coher_cntl); /* CP_COHER_CNTL */
radeon_emit(0xffffffff); /* CP_COHER_SIZE */
radeon_emit(0); /* CP_COHER_BASE */
radeon_emit(0x0000000A); /* POLL_INTERVAL */
}
radeon_end();
}
static void
@@ -179,7 +187,9 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level
/* TODO: trigger on RADV_CMD_FLAG_FLUSH_AND_INV_CB_META */
if (gfx_level < GFX12 && flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_CB) {
/* Flush CMASK/FMASK/DCC. Will wait for idle later. */
radeon_event_write(cs, V_028A90_FLUSH_AND_INV_CB_META);
radeon_begin(cs);
radeon_event_write(V_028A90_FLUSH_AND_INV_CB_META);
radeon_end();
*sqtt_flush_bits |= RGP_FLUSH_FLUSH_CB | RGP_FLUSH_INVAL_CB;
}
@@ -188,7 +198,9 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level
/* TODO: trigger on RADV_CMD_FLAG_FLUSH_AND_INV_DB_META ? */
if (gfx_level < GFX12 && gfx_level != GFX11 && (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_DB)) {
/* Flush HTILE. Will wait for idle later. */
radeon_event_write(cs, V_028A90_FLUSH_AND_INV_DB_META);
radeon_begin(cs);
radeon_event_write(V_028A90_FLUSH_AND_INV_DB_META);
radeon_end();
*sqtt_flush_bits |= RGP_FLUSH_FLUSH_DB | RGP_FLUSH_INVAL_DB;
}
@@ -213,18 +225,24 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level
} else {
/* Wait for graphics shaders to go idle if requested. */
if (flush_bits & RADV_CMD_FLAG_PS_PARTIAL_FLUSH) {
radeon_event_write(cs, V_028A90_PS_PARTIAL_FLUSH);
radeon_begin(cs);
radeon_event_write(V_028A90_PS_PARTIAL_FLUSH);
radeon_end();
*sqtt_flush_bits |= RGP_FLUSH_PS_PARTIAL_FLUSH;
} else if (flush_bits & RADV_CMD_FLAG_VS_PARTIAL_FLUSH) {
radeon_event_write(cs, V_028A90_VS_PARTIAL_FLUSH);
radeon_begin(cs);
radeon_event_write(V_028A90_VS_PARTIAL_FLUSH);
radeon_end();
*sqtt_flush_bits |= RGP_FLUSH_VS_PARTIAL_FLUSH;
}
}
if (flush_bits & RADV_CMD_FLAG_CS_PARTIAL_FLUSH) {
radeon_event_write(cs, V_028A90_CS_PARTIAL_FLUSH);
radeon_begin(cs);
radeon_event_write(V_028A90_CS_PARTIAL_FLUSH);
radeon_end();
*sqtt_flush_bits |= RGP_FLUSH_CS_PARTIAL_FLUSH;
}
@@ -248,29 +266,33 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level
gcr_cntl &= C_586_GLM_WB & C_586_GLM_INV & C_586_GLK_WB & C_586_GLK_INV & C_586_GLV_INV & C_586_GL1_INV &
C_586_GL2_INV & C_586_GL2_WB; /* keep SEQ */
radeon_begin(cs);
/* Send an event that flushes caches. */
radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, 6, 0));
radeon_emit(cs, S_490_EVENT_TYPE(cb_db_event) | S_490_EVENT_INDEX(5) | S_490_GLM_WB(glm_wb) |
S_490_GLM_INV(glm_inv) | S_490_GLV_INV(glv_inv) | S_490_GL1_INV(gl1_inv) |
S_490_GL2_INV(gl2_inv) | S_490_GL2_WB(gl2_wb) | S_490_SEQ(gcr_seq) | S_490_GLK_WB(glk_wb) |
S_490_GLK_INV(glk_inv) | S_490_PWS_ENABLE(1));
radeon_emit(cs, 0); /* DST_SEL, INT_SEL, DATA_SEL */
radeon_emit(cs, 0); /* ADDRESS_LO */
radeon_emit(cs, 0); /* ADDRESS_HI */
radeon_emit(cs, 0); /* DATA_LO */
radeon_emit(cs, 0); /* DATA_HI */
radeon_emit(cs, 0); /* INT_CTXID */
radeon_emit(PKT3(PKT3_RELEASE_MEM, 6, 0));
radeon_emit(S_490_EVENT_TYPE(cb_db_event) | S_490_EVENT_INDEX(5) | S_490_GLM_WB(glm_wb) |
S_490_GLM_INV(glm_inv) | S_490_GLV_INV(glv_inv) | S_490_GL1_INV(gl1_inv) | S_490_GL2_INV(gl2_inv) |
S_490_GL2_WB(gl2_wb) | S_490_SEQ(gcr_seq) | S_490_GLK_WB(glk_wb) | S_490_GLK_INV(glk_inv) |
S_490_PWS_ENABLE(1));
radeon_emit(0); /* DST_SEL, INT_SEL, DATA_SEL */
radeon_emit(0); /* ADDRESS_LO */
radeon_emit(0); /* ADDRESS_HI */
radeon_emit(0); /* DATA_LO */
radeon_emit(0); /* DATA_HI */
radeon_emit(0); /* INT_CTXID */
/* Wait for the event and invalidate remaining caches if needed. */
radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 6, 0));
radeon_emit(cs, S_580_PWS_STAGE_SEL(V_580_CP_PFP) | S_580_PWS_COUNTER_SEL(V_580_TS_SELECT) |
S_580_PWS_ENA2(1) | S_580_PWS_COUNT(0));
radeon_emit(cs, 0xffffffff); /* GCR_SIZE */
radeon_emit(cs, 0x01ffffff); /* GCR_SIZE_HI */
radeon_emit(cs, 0); /* GCR_BASE_LO */
radeon_emit(cs, 0); /* GCR_BASE_HI */
radeon_emit(cs, S_585_PWS_ENA(1));
radeon_emit(cs, gcr_cntl); /* GCR_CNTL */
radeon_emit(PKT3(PKT3_ACQUIRE_MEM, 6, 0));
radeon_emit(S_580_PWS_STAGE_SEL(V_580_CP_PFP) | S_580_PWS_COUNTER_SEL(V_580_TS_SELECT) | S_580_PWS_ENA2(1) |
S_580_PWS_COUNT(0));
radeon_emit(0xffffffff); /* GCR_SIZE */
radeon_emit(0x01ffffff); /* GCR_SIZE_HI */
radeon_emit(0); /* GCR_BASE_LO */
radeon_emit(0); /* GCR_BASE_HI */
radeon_emit(S_585_PWS_ENA(1));
radeon_emit(gcr_cntl); /* GCR_CNTL */
radeon_end();
gcr_cntl = 0; /* all done */
} else {
@@ -310,9 +332,11 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level
}
}
radeon_begin(cs);
/* VGT state sync */
if (flush_bits & RADV_CMD_FLAG_VGT_FLUSH) {
radeon_event_write(cs, V_028A90_VGT_FLUSH);
radeon_event_write(V_028A90_VGT_FLUSH);
}
/* Ignore fields that only modify the behavior of other fields. */
@@ -321,37 +345,39 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level
* The cache flush is executed in the ME, but the PFP waits
* for completion.
*/
radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 6, 0));
radeon_emit(cs, 0); /* CP_COHER_CNTL */
radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */
radeon_emit(cs, 0xffffff); /* CP_COHER_SIZE_HI */
radeon_emit(cs, 0); /* CP_COHER_BASE */
radeon_emit(cs, 0); /* CP_COHER_BASE_HI */
radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */
radeon_emit(cs, gcr_cntl); /* GCR_CNTL */
radeon_emit(PKT3(PKT3_ACQUIRE_MEM, 6, 0));
radeon_emit(0); /* CP_COHER_CNTL */
radeon_emit(0xffffffff); /* CP_COHER_SIZE */
radeon_emit(0xffffff); /* CP_COHER_SIZE_HI */
radeon_emit(0); /* CP_COHER_BASE */
radeon_emit(0); /* CP_COHER_BASE_HI */
radeon_emit(0x0000000A); /* POLL_INTERVAL */
radeon_emit(gcr_cntl); /* GCR_CNTL */
} else if ((cb_db_event || (flush_bits & (RADV_CMD_FLAG_VS_PARTIAL_FLUSH | RADV_CMD_FLAG_PS_PARTIAL_FLUSH |
RADV_CMD_FLAG_CS_PARTIAL_FLUSH))) &&
!is_mec) {
/* We need to ensure that PFP waits as well. */
radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0));
radeon_emit(cs, 0);
radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0));
radeon_emit(0);
*sqtt_flush_bits |= RGP_FLUSH_PFP_SYNC_ME;
}
if (flush_bits & RADV_CMD_FLAG_START_PIPELINE_STATS) {
if (qf == RADV_QUEUE_GENERAL) {
radeon_event_write(cs, V_028A90_PIPELINESTAT_START);
radeon_event_write(V_028A90_PIPELINESTAT_START);
} else if (qf == RADV_QUEUE_COMPUTE) {
radeon_set_sh_reg(cs, R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(1));
radeon_set_sh_reg(R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(1));
}
} else if (flush_bits & RADV_CMD_FLAG_STOP_PIPELINE_STATS) {
if (qf == RADV_QUEUE_GENERAL) {
radeon_event_write(cs, V_028A90_PIPELINESTAT_STOP);
radeon_event_write(V_028A90_PIPELINESTAT_STOP);
} else if (qf == RADV_QUEUE_COMPUTE) {
radeon_set_sh_reg(cs, R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(0));
radeon_set_sh_reg(R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(0));
}
}
radeon_end();
}
void
@@ -405,29 +431,39 @@ radv_cs_emit_cache_flush(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, enu
}
if (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_CB_META) {
radeon_event_write(cs, V_028A90_FLUSH_AND_INV_CB_META);
radeon_begin(cs);
radeon_event_write(V_028A90_FLUSH_AND_INV_CB_META);
radeon_end();
*sqtt_flush_bits |= RGP_FLUSH_FLUSH_CB | RGP_FLUSH_INVAL_CB;
}
if (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_DB_META) {
radeon_event_write(cs, V_028A90_FLUSH_AND_INV_DB_META);
radeon_begin(cs);
radeon_event_write(V_028A90_FLUSH_AND_INV_DB_META);
radeon_end();
*sqtt_flush_bits |= RGP_FLUSH_FLUSH_DB | RGP_FLUSH_INVAL_DB;
}
if (flush_bits & RADV_CMD_FLAG_PS_PARTIAL_FLUSH) {
radeon_event_write(cs, V_028A90_PS_PARTIAL_FLUSH);
radeon_begin(cs);
radeon_event_write(V_028A90_PS_PARTIAL_FLUSH);
radeon_end();
*sqtt_flush_bits |= RGP_FLUSH_PS_PARTIAL_FLUSH;
} else if (flush_bits & RADV_CMD_FLAG_VS_PARTIAL_FLUSH) {
radeon_event_write(cs, V_028A90_VS_PARTIAL_FLUSH);
radeon_begin(cs);
radeon_event_write(V_028A90_VS_PARTIAL_FLUSH);
radeon_end();
*sqtt_flush_bits |= RGP_FLUSH_VS_PARTIAL_FLUSH;
}
if (flush_bits & RADV_CMD_FLAG_CS_PARTIAL_FLUSH) {
radeon_event_write(cs, V_028A90_CS_PARTIAL_FLUSH);
radeon_begin(cs);
radeon_event_write(V_028A90_CS_PARTIAL_FLUSH);
radeon_end();
*sqtt_flush_bits |= RGP_FLUSH_CS_PARTIAL_FLUSH;
}
@@ -475,12 +511,16 @@ radv_cs_emit_cache_flush(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, enu
/* VGT state sync */
if (flush_bits & RADV_CMD_FLAG_VGT_FLUSH) {
radeon_event_write(cs, V_028A90_VGT_FLUSH);
radeon_begin(cs);
radeon_event_write(V_028A90_VGT_FLUSH);
radeon_end();
}
/* VGT streamout state sync */
if (flush_bits & RADV_CMD_FLAG_VGT_STREAMOUT_SYNC) {
radeon_event_write(cs, V_028A90_VGT_STREAMOUT_SYNC);
radeon_begin(cs);
radeon_event_write(V_028A90_VGT_STREAMOUT_SYNC);
radeon_end();
}
/* Make sure ME is idle (it executes most packets) before continuing.
@@ -489,8 +529,10 @@ radv_cs_emit_cache_flush(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, enu
if ((cp_coher_cntl || (flush_bits & (RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE |
RADV_CMD_FLAG_INV_L2 | RADV_CMD_FLAG_WB_L2))) &&
!is_mec) {
radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0));
radeon_emit(cs, 0);
radeon_begin(cs);
radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0));
radeon_emit(0);
radeon_end();
*sqtt_flush_bits |= RGP_FLUSH_PFP_SYNC_ME;
}
@@ -530,19 +572,23 @@ radv_cs_emit_cache_flush(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, enu
if (cp_coher_cntl)
radv_emit_acquire_mem(cs, is_mec, gfx_level == GFX9, cp_coher_cntl);
radeon_begin(cs);
if (flush_bits & RADV_CMD_FLAG_START_PIPELINE_STATS) {
if (qf == RADV_QUEUE_GENERAL) {
radeon_event_write(cs, V_028A90_PIPELINESTAT_START);
radeon_event_write(V_028A90_PIPELINESTAT_START);
} else if (qf == RADV_QUEUE_COMPUTE) {
radeon_set_sh_reg(cs, R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(1));
radeon_set_sh_reg(R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(1));
}
} else if (flush_bits & RADV_CMD_FLAG_STOP_PIPELINE_STATS) {
if (qf == RADV_QUEUE_GENERAL) {
radeon_event_write(cs, V_028A90_PIPELINESTAT_STOP);
radeon_event_write(V_028A90_PIPELINESTAT_STOP);
} else if (qf == RADV_QUEUE_COMPUTE) {
radeon_set_sh_reg(cs, R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(0));
radeon_set_sh_reg(R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(0));
}
}
radeon_end();
}
void
@@ -551,26 +597,32 @@ radv_emit_cond_exec(const struct radv_device *device, struct radeon_cmdbuf *cs,
const struct radv_physical_device *pdev = radv_device_physical(device);
const enum amd_gfx_level gfx_level = pdev->info.gfx_level;
radeon_begin(cs);
if (gfx_level >= GFX7) {
radeon_emit(cs, PKT3(PKT3_COND_EXEC, 3, 0));
radeon_emit(cs, va);
radeon_emit(cs, va >> 32);
radeon_emit(cs, 0);
radeon_emit(cs, count);
radeon_emit(PKT3(PKT3_COND_EXEC, 3, 0));
radeon_emit(va);
radeon_emit(va >> 32);
radeon_emit(0);
radeon_emit(count);
} else {
radeon_emit(cs, PKT3(PKT3_COND_EXEC, 2, 0));
radeon_emit(cs, va);
radeon_emit(cs, va >> 32);
radeon_emit(cs, count);
radeon_emit(PKT3(PKT3_COND_EXEC, 2, 0));
radeon_emit(va);
radeon_emit(va >> 32);
radeon_emit(count);
}
radeon_end();
}
void
radv_cs_write_data_imm(struct radeon_cmdbuf *cs, unsigned engine_sel, uint64_t va, uint32_t imm)
{
radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0));
radeon_emit(cs, S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(engine_sel));
radeon_emit(cs, va);
radeon_emit(cs, va >> 32);
radeon_emit(cs, imm);
radeon_begin(cs);
radeon_emit(PKT3(PKT3_WRITE_DATA, 3, 0));
radeon_emit(S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(engine_sel));
radeon_emit(va);
radeon_emit(va >> 32);
radeon_emit(imm);
radeon_end();
}
src/amd/vulkan/radv_cs.h
+112 -81
View File
@@ -27,34 +27,58 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned
return cs->cdw + needed;
}
/* Packet building helpers. Don't use directly. */
#define radeon_set_reg_seq(cs, reg, num, idx, prefix_name, packet, reset_filter_cam) \
#define radeon_begin(cs) \
struct radeon_cmdbuf *__cs = (cs); \
uint64_t __cs_num = __cs->cdw; \
UNUSED uint64_t __cs_reserved_dw = __cs->reserved_dw; \
uint32_t *__cs_buf = __cs->buf
#define radeon_end() \
do { \
assert((reg) >= prefix_name##_REG_OFFSET && (reg) < prefix_name##_REG_END); \
assert(cs->cdw + 2 + num <= cs->reserved_dw); \
radeon_emit(cs, PKT3(packet, num, 0) | PKT3_RESET_FILTER_CAM_S(reset_filter_cam)); \
radeon_emit(cs, (((reg) - prefix_name##_REG_OFFSET) >> 2) | ((idx) << 28)); \
__cs->cdw = __cs_num; \
assert(__cs->cdw <= __cs->max_dw); \
__cs = NULL; \
} while (0)
#define radeon_set_reg(cs, reg, idx, value, prefix_name, packet) \
#define radeon_emit(value) \
do { \
radeon_set_reg_seq(cs, reg, 1, idx, prefix_name, packet, 0); \
radeon_emit(cs, value); \
assert(__cs_num < __cs_reserved_dw); \
__cs_buf[__cs_num++] = (value); \
} while (0)
#define radeon_emit_array(values, num) \
do { \
unsigned __n = (num); \
assert(__cs_num + __n <= __cs_reserved_dw); \
memcpy(__cs_buf + __cs_num, (values), __n * 4); \
__cs_num += __n; \
} while (0)
/* Packet building helpers. Don't use directly. */
#define __radeon_set_reg_seq(reg, num, idx, prefix_name, packet, reset_filter_cam) \
do { \
assert((reg) >= prefix_name##_REG_OFFSET && (reg) < prefix_name##_REG_END); \
radeon_emit(PKT3(packet, num, 0) | PKT3_RESET_FILTER_CAM_S(reset_filter_cam)); \
radeon_emit((((reg) - prefix_name##_REG_OFFSET) >> 2) | ((idx) << 28)); \
} while (0)
#define __radeon_set_reg(reg, idx, value, prefix_name, packet) \
do { \
__radeon_set_reg_seq(reg, 1, idx, prefix_name, packet, 0); \
radeon_emit(value); \
} while (0)
/* Packet building helpers for CONFIG registers. */
#define radeon_set_config_reg_seq(cs, reg, num) radeon_set_reg_seq(cs, reg, num, 0, SI_CONFIG, PKT3_SET_CONFIG_REG, 0)
#define radeon_set_config_reg_seq(reg, num) __radeon_set_reg_seq(reg, num, 0, SI_CONFIG, PKT3_SET_CONFIG_REG, 0)
#define radeon_set_config_reg(cs, reg, value) radeon_set_reg(cs, reg, 0, value, SI_CONFIG, PKT3_SET_CONFIG_REG)
#define radeon_set_config_reg(reg, value) __radeon_set_reg(reg, 0, value, SI_CONFIG, PKT3_SET_CONFIG_REG)
/* Packet building helpers for CONTEXT registers. */
#define radeon_set_context_reg_seq(cs, reg, num) \
radeon_set_reg_seq(cs, reg, num, 0, SI_CONTEXT, PKT3_SET_CONTEXT_REG, 0)
#define radeon_set_context_reg_seq(reg, num) __radeon_set_reg_seq(reg, num, 0, SI_CONTEXT, PKT3_SET_CONTEXT_REG, 0)
#define radeon_set_context_reg(cs, reg, value) radeon_set_reg(cs, reg, 0, value, SI_CONTEXT, PKT3_SET_CONTEXT_REG)
#define radeon_set_context_reg(reg, value) __radeon_set_reg(reg, 0, value, SI_CONTEXT, PKT3_SET_CONTEXT_REG)
#define radeon_set_context_reg_idx(cs, reg, idx, value) \
radeon_set_reg(cs, reg, idx, value, SI_CONTEXT, PKT3_SET_CONTEXT_REG)
#define radeon_set_context_reg_idx(reg, idx, value) __radeon_set_reg(reg, idx, value, SI_CONTEXT, PKT3_SET_CONTEXT_REG)
#define radeon_opt_set_context_reg(cmdbuf, reg, reg_enum, value) \
do { \
@@ -63,7 +87,7 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned
const uint32_t __value = (value); \
if (!BITSET_TEST(__tracked_regs->reg_saved_mask, (reg_enum)) || \
__tracked_regs->reg_value[(reg_enum)] != __value) { \
radeon_set_context_reg(__cmdbuf->cs, reg, __value); \
radeon_set_context_reg(reg, __value); \
BITSET_SET(__tracked_regs->reg_saved_mask, (reg_enum)); \
__tracked_regs->reg_value[(reg_enum)] = __value; \
__cmdbuf->state.context_roll_without_scissor_emitted = true; \
@@ -77,9 +101,9 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned
const uint32_t __v1 = (v1), __v2 = (v2); \
if (!BITSET_TEST_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 1, 0x3) || \
__tracked_regs->reg_value[(reg_enum)] != __v1 || __tracked_regs->reg_value[(reg_enum) + 1] != __v2) { \
radeon_set_context_reg_seq(cmdbuf->cs, reg, 2); \
radeon_emit(cmdbuf->cs, __v1); \
radeon_emit(cmdbuf->cs, __v2); \
radeon_set_context_reg_seq(reg, 2); \
radeon_emit(__v1); \
radeon_emit(__v2); \
BITSET_SET_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 1); \
__tracked_regs->reg_value[(reg_enum)] = __v1; \
__tracked_regs->reg_value[(reg_enum) + 1] = __v2; \
@@ -95,10 +119,10 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned
if (!BITSET_TEST_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 2, 0x7) || \
__tracked_regs->reg_value[(reg_enum)] != __v1 || __tracked_regs->reg_value[(reg_enum) + 1] != __v2 || \
__tracked_regs->reg_value[(reg_enum) + 2] != __v3) { \
radeon_set_context_reg_seq(cmdbuf->cs, reg, 3); \
radeon_emit(cmdbuf->cs, __v1); \
radeon_emit(cmdbuf->cs, __v2); \
radeon_emit(cmdbuf->cs, __v3); \
radeon_set_context_reg_seq(reg, 3); \
radeon_emit(__v1); \
radeon_emit(__v2); \
radeon_emit(__v3); \
BITSET_SET_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 2); \
__tracked_regs->reg_value[(reg_enum)] = __v1; \
__tracked_regs->reg_value[(reg_enum) + 1] = __v2; \
@@ -115,11 +139,11 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned
if (!BITSET_TEST_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 3, 0xf) || \
__tracked_regs->reg_value[(reg_enum)] != __v1 || __tracked_regs->reg_value[(reg_enum) + 1] != __v2 || \
__tracked_regs->reg_value[(reg_enum) + 2] != __v3 || __tracked_regs->reg_value[(reg_enum) + 3] != __v4) { \
radeon_set_context_reg_seq(cmdbuf->cs, reg, 4); \
radeon_emit(cmdbuf->cs, __v1); \
radeon_emit(cmdbuf->cs, __v2); \
radeon_emit(cmdbuf->cs, __v3); \
radeon_emit(cmdbuf->cs, __v4); \
radeon_set_context_reg_seq(reg, 4); \
radeon_emit(__v1); \
radeon_emit(__v2); \
radeon_emit(__v3); \
radeon_emit(__v4); \
BITSET_SET_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 3); \
__tracked_regs->reg_value[(reg_enum)] = __v1; \
__tracked_regs->reg_value[(reg_enum) + 1] = __v2; \
@@ -133,40 +157,39 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned
do { \
struct radv_cmd_buffer *__cmdbuf = (cmdbuf); \
if (memcmp(values, saved_values, sizeof(uint32_t) * (num))) { \
radeon_set_context_reg_seq(cmdbuf->cs, reg, num); \
radeon_emit_array(cmdbuf->cs, values, num); \
radeon_set_context_reg_seq(reg, num); \
radeon_emit_array(values, num); \
memcpy(saved_values, values, sizeof(uint32_t) * (num)); \
__cmdbuf->state.context_roll_without_scissor_emitted = true; \
} \
} while (0)
/* Packet building helpers for SH registers. */
#define radeon_set_sh_reg_seq(cs, reg, num) radeon_set_reg_seq(cs, reg, num, 0, SI_SH, PKT3_SET_SH_REG, 0)
#define radeon_set_sh_reg_seq(reg, num) __radeon_set_reg_seq(reg, num, 0, SI_SH, PKT3_SET_SH_REG, 0)
#define radeon_set_sh_reg(cs, reg, value) radeon_set_reg(cs, reg, 0, value, SI_SH, PKT3_SET_SH_REG)
#define radeon_set_sh_reg(reg, value) __radeon_set_reg(reg, 0, value, SI_SH, PKT3_SET_SH_REG)
#define radeon_set_sh_reg_idx(info, cs, reg, idx, value) \
#define radeon_set_sh_reg_idx(info, reg, idx, value) \
do { \
assert((idx)); \
unsigned __opcode = PKT3_SET_SH_REG_INDEX; \
if ((info)->gfx_level < GFX10) \
__opcode = PKT3_SET_SH_REG; \
radeon_set_reg(cs, reg, idx, value, SI_SH, __opcode); \
__radeon_set_reg(reg, idx, value, SI_SH, __opcode); \
} while (0)
/* Packet building helpers for UCONFIG registers. */
#define radeon_set_uconfig_reg_seq(cs, reg, num) \
radeon_set_reg_seq(cs, reg, num, 0, CIK_UCONFIG, PKT3_SET_UCONFIG_REG, 0)
#define radeon_set_uconfig_reg_seq(reg, num) __radeon_set_reg_seq(reg, num, 0, CIK_UCONFIG, PKT3_SET_UCONFIG_REG, 0)
#define radeon_set_uconfig_reg(cs, reg, value) radeon_set_reg(cs, reg, 0, value, CIK_UCONFIG, PKT3_SET_UCONFIG_REG)
#define radeon_set_uconfig_reg(reg, value) __radeon_set_reg(reg, 0, value, CIK_UCONFIG, PKT3_SET_UCONFIG_REG)
#define radeon_set_uconfig_reg_idx(info, cs, reg, idx, value) \
#define radeon_set_uconfig_reg_idx(info, reg, idx, value) \
do { \
assert((idx)); \
unsigned __opcode = PKT3_SET_UCONFIG_REG_INDEX; \
if ((info)->gfx_level < GFX9 || ((info)->gfx_level == GFX9 && (info)->me_fw_version < 26)) \
__opcode = PKT3_SET_UCONFIG_REG; \
radeon_set_reg(cs, reg, idx, value, CIK_UCONFIG, __opcode); \
__radeon_set_reg(reg, idx, value, CIK_UCONFIG, __opcode); \
} while (0)
/*
@@ -174,55 +197,54 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned
* that means that it can skip register writes due to not taking correctly into account the
* fields from the GRBM_GFX_INDEX. With this bit we can force the write.
*/
#define radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, reg, num) \
#define radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, reg, num) \
do { \
const bool __filter_cam_workaround = (gfx_level) >= GFX10 && (ring) == AMD_IP_GFX; \
radeon_set_reg_seq(cs, reg, num, 0, CIK_UCONFIG, PKT3_SET_UCONFIG_REG, __filter_cam_workaround); \
__radeon_set_reg_seq(reg, num, 0, CIK_UCONFIG, PKT3_SET_UCONFIG_REG, __filter_cam_workaround); \
} while (0)
#define radeon_set_uconfig_perfctr_reg(gfx_level, ring, cs, reg, value) \
#define radeon_set_uconfig_perfctr_reg(gfx_level, ring, reg, value) \
do { \
radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, reg, 1); \
radeon_emit(cs, value); \
radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, reg, 1); \
radeon_emit(value); \
} while (0)
#define radeon_set_privileged_config_reg(cs, reg, value) \
#define radeon_set_privileged_config_reg(reg, value) \
do { \
assert((reg) < CIK_UCONFIG_REG_OFFSET); \
assert((cs)->cdw + 6 <= (cs)->reserved_dw); \
radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); \
radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_PERF)); \
radeon_emit(cs, value); \
radeon_emit(cs, 0); /* unused */ \
radeon_emit(cs, (reg) >> 2); \
radeon_emit(cs, 0); /* unused */ \
radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); \
radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_PERF)); \
radeon_emit(value); \
radeon_emit(0); /* unused */ \
radeon_emit((reg) >> 2); \
radeon_emit(0); /* unused */ \
} while (0)
#define radeon_event_write_predicate(cs, event_type, predicate) \
#define radeon_event_write_predicate(event_type, predicate) \
do { \
unsigned __event_type = (event_type); \
radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, predicate)); \
radeon_emit(cs, EVENT_TYPE(__event_type) | EVENT_INDEX(__event_type == V_028A90_VS_PARTIAL_FLUSH || \
__event_type == V_028A90_PS_PARTIAL_FLUSH || \
__event_type == V_028A90_CS_PARTIAL_FLUSH \
? 4 \
: 0)); \
radeon_emit(PKT3(PKT3_EVENT_WRITE, 0, predicate)); \
radeon_emit(EVENT_TYPE(__event_type) | EVENT_INDEX(__event_type == V_028A90_VS_PARTIAL_FLUSH || \
__event_type == V_028A90_PS_PARTIAL_FLUSH || \
__event_type == V_028A90_CS_PARTIAL_FLUSH \
? 4 \
: 0)); \
} while (0)
#define radeon_event_write(cs, event_type) radeon_event_write_predicate(cs, event_type, false)
#define radeon_event_write(event_type) radeon_event_write_predicate(event_type, false)
#define radeon_emit_32bit_pointer(cs, sh_offset, va, info) \
#define radeon_emit_32bit_pointer(sh_offset, va, info) \
do { \
assert((va) == 0 || ((va) >> 32) == (info)->address32_hi); \
radeon_set_sh_reg_seq(cs, sh_offset, 1); \
radeon_emit(cs, va); \
radeon_set_sh_reg_seq(sh_offset, 1); \
radeon_emit(va); \
} while (0)
#define radeon_emit_64bit_pointer(cs, sh_offset, va) \
#define radeon_emit_64bit_pointer(sh_offset, va) \
do { \
radeon_set_sh_reg_seq(cs, sh_offset, 2); \
radeon_emit(cs, va); \
radeon_emit(cs, va >> 32); \
radeon_set_sh_reg_seq(sh_offset, 2); \
radeon_emit(va); \
radeon_emit(va >> 32); \
} while (0)
ALWAYS_INLINE static void
@@ -232,13 +254,15 @@ radv_cp_wait_mem(struct radeon_cmdbuf *cs, const enum radv_queue_family qf, cons
assert(op == WAIT_REG_MEM_EQUAL || op == WAIT_REG_MEM_NOT_EQUAL || op == WAIT_REG_MEM_GREATER_OR_EQUAL);
if (qf == RADV_QUEUE_GENERAL || qf == RADV_QUEUE_COMPUTE) {
radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, false));
radeon_emit(cs, op | WAIT_REG_MEM_MEM_SPACE(1));
radeon_emit(cs, va);
radeon_emit(cs, va >> 32);
radeon_emit(cs, ref); /* reference value */
radeon_emit(cs, mask); /* mask */
radeon_emit(cs, 4); /* poll interval */
radeon_begin(cs);
radeon_emit(PKT3(PKT3_WAIT_REG_MEM, 5, false));
radeon_emit(op | WAIT_REG_MEM_MEM_SPACE(1));
radeon_emit(va);
radeon_emit(va >> 32);
radeon_emit(ref); /* reference value */
radeon_emit(mask); /* mask */
radeon_emit(4); /* poll interval */
radeon_end();
} else if (qf == RADV_QUEUE_TRANSFER) {
radv_sdma_emit_wait_mem(cs, op, va, ref, mask);
} else {
@@ -254,10 +278,12 @@ radv_cs_write_data_head(const struct radv_device *device, struct radeon_cmdbuf *
const unsigned cdw_end = radeon_check_space(device->ws, cs, 4 + count);
if (qf == RADV_QUEUE_GENERAL || qf == RADV_QUEUE_COMPUTE) {
radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + count, predicating));
radeon_emit(cs, S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(engine_sel));
radeon_emit(cs, va);
radeon_emit(cs, va >> 32);
radeon_begin(cs);
radeon_emit(PKT3(PKT3_WRITE_DATA, 2 + count, predicating));
radeon_emit(S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(engine_sel));
radeon_emit(va);
radeon_emit(va >> 32);
radeon_end();
} else if (qf == RADV_QUEUE_TRANSFER) {
radv_sdma_emit_write_data_head(cs, va, count);
} else {
@@ -273,7 +299,10 @@ radv_cs_write_data(const struct radv_device *device, struct radeon_cmdbuf *cs, c
const bool predicating)
{
ASSERTED const unsigned cdw_end = radv_cs_write_data_head(device, cs, qf, engine_sel, va, count, predicating);
radeon_emit_array(cs, dwords, count);
radeon_begin(cs);
radeon_emit_array(dwords, count);
radeon_end();
assert(cs->cdw == cdw_end);
}
@@ -293,7 +322,9 @@ void radv_cs_write_data_imm(struct radeon_cmdbuf *cs, unsigned engine_sel, uint6
static inline void
radv_emit_pm4_commands(struct radeon_cmdbuf *cs, const struct ac_pm4_state *pm4)
{
radeon_emit_array(cs, pm4->pm4, pm4->ndw);
radeon_begin(cs);
radeon_emit_array(pm4->pm4, pm4->ndw);
radeon_end();
}
#endif /* RADV_CS_H */
src/amd/vulkan/radv_device.c
+26 -22
View File
@@ -972,40 +972,42 @@ radv_emit_default_sample_locations(const struct radv_physical_device *pdev, stru
{
uint64_t centroid_priority;
radeon_begin(cs);
switch (nr_samples) {
default:
case 1:
centroid_priority = centroid_priority_1x;
radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_1x);
radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_1x);
radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_1x);
radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_1x);
radeon_set_context_reg(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_1x);
radeon_set_context_reg(R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_1x);
radeon_set_context_reg(R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_1x);
radeon_set_context_reg(R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_1x);
break;
case 2:
centroid_priority = centroid_priority_2x;
radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_2x);
radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_2x);
radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_2x);
radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_2x);
radeon_set_context_reg(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_2x);
radeon_set_context_reg(R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_2x);
radeon_set_context_reg(R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_2x);
radeon_set_context_reg(R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_2x);
break;
case 4:
centroid_priority = centroid_priority_4x;
radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_4x);
radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_4x);
radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_4x);
radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_4x);
radeon_set_context_reg(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_4x);
radeon_set_context_reg(R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_4x);
radeon_set_context_reg(R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_4x);
radeon_set_context_reg(R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_4x);
break;
case 8:
centroid_priority = centroid_priority_8x;
radeon_set_context_reg_seq(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, 14);
radeon_emit_array(cs, sample_locs_8x, 4);
radeon_emit_array(cs, sample_locs_8x, 4);
radeon_emit_array(cs, sample_locs_8x, 4);
radeon_emit_array(cs, sample_locs_8x, 2);
radeon_set_context_reg_seq(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, 14);
radeon_emit_array(sample_locs_8x, 4);
radeon_emit_array(sample_locs_8x, 4);
radeon_emit_array(sample_locs_8x, 4);
radeon_emit_array(sample_locs_8x, 2);
break;
}
@@ -1014,17 +1016,19 @@ radv_emit_default_sample_locations(const struct radv_physical_device *pdev, stru
* support 16 samples.
*/
if (pdev->info.gfx_level >= GFX7) {
radeon_set_context_reg(cs, R_02882C_PA_SU_PRIM_FILTER_CNTL,
radeon_set_context_reg(R_02882C_PA_SU_PRIM_FILTER_CNTL,
S_02882C_XMAX_RIGHT_EXCLUSION(1) | S_02882C_YMAX_BOTTOM_EXCLUSION(1));
}
if (pdev->info.gfx_level >= GFX12) {
radeon_set_context_reg_seq(cs, R_028BF0_PA_SC_CENTROID_PRIORITY_0, 2);
radeon_set_context_reg_seq(R_028BF0_PA_SC_CENTROID_PRIORITY_0, 2);
} else {
radeon_set_context_reg_seq(cs, R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2);
radeon_set_context_reg_seq(R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2);
}
radeon_emit(cs, centroid_priority);
radeon_emit(cs, centroid_priority >> 32);
radeon_emit(centroid_priority);
radeon_emit(centroid_priority >> 32);
radeon_end();
}
static void
src/amd/vulkan/radv_perfcounter.c
+75 -44
View File
@@ -19,25 +19,33 @@ radv_perfcounter_emit_shaders(struct radv_device *device, struct radeon_cmdbuf *
{
const struct radv_physical_device *pdev = radv_device_physical(device);
radeon_begin(cs);
if (pdev->info.gfx_level >= GFX10) {
radeon_set_uconfig_reg(cs, R_036780_SQ_PERFCOUNTER_CTRL, shaders & 0x7f);
radeon_set_uconfig_reg(R_036780_SQ_PERFCOUNTER_CTRL, shaders & 0x7f);
if (pdev->info.gfx_level >= GFX11)
radeon_set_uconfig_reg(cs, R_036760_SQG_PERFCOUNTER_CTRL, shaders & 0x7f);
radeon_set_uconfig_reg(R_036760_SQG_PERFCOUNTER_CTRL, shaders & 0x7f);
} else {
radeon_set_uconfig_reg_seq(cs, R_036780_SQ_PERFCOUNTER_CTRL, 2);
radeon_emit(cs, shaders & 0x7f);
radeon_emit(cs, 0xffffffff);
radeon_set_uconfig_reg_seq(R_036780_SQ_PERFCOUNTER_CTRL, 2);
radeon_emit(shaders & 0x7f);
radeon_emit(0xffffffff);
}
radeon_end();
}
static void
radv_emit_windowed_counters(struct radv_device *device, struct radeon_cmdbuf *cs, int family, bool enable)
{
radeon_begin(cs);
if (family == RADV_QUEUE_GENERAL) {
radeon_event_write(cs, enable ? V_028A90_PERFCOUNTER_START : V_028A90_PERFCOUNTER_STOP);
radeon_event_write(enable ? V_028A90_PERFCOUNTER_START : V_028A90_PERFCOUNTER_STOP);
}
radeon_set_sh_reg(cs, R_00B82C_COMPUTE_PERFCOUNT_ENABLE, S_00B82C_PERFCOUNT_ENABLE(enable));
radeon_set_sh_reg(R_00B82C_COMPUTE_PERFCOUNT_ENABLE, S_00B82C_PERFCOUNT_ENABLE(enable));
radeon_end();
}
void
@@ -52,7 +60,9 @@ radv_perfcounter_emit_reset(struct radeon_cmdbuf *cs, bool is_spm)
cp_perfmon_cntl = S_036020_PERFMON_STATE(V_036020_CP_PERFMON_STATE_DISABLE_AND_RESET);
}
radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl);
radeon_begin(cs);
radeon_set_uconfig_reg(R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl);
radeon_end();
}
static void
@@ -67,7 +77,9 @@ radv_perfcounter_emit_start(struct radeon_cmdbuf *cs, bool is_spm)
cp_perfmon_cntl = S_036020_PERFMON_STATE(V_036020_CP_PERFMON_STATE_START_COUNTING);
}
radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl);
radeon_begin(cs);
radeon_set_uconfig_reg(R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl);
radeon_end();
}
static void
@@ -83,7 +95,9 @@ radv_perfcounter_emit_stop(struct radeon_cmdbuf *cs, bool is_spm)
S_036020_PERFMON_STATE(V_036020_CP_PERFMON_STATE_STOP_COUNTING) | S_036020_PERFMON_SAMPLE_ENABLE(1);
}
radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl);
radeon_begin(cs);
radeon_set_uconfig_reg(R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl);
radeon_end();
}
void
@@ -113,7 +127,9 @@ radv_perfcounter_emit_spm_stop(struct radv_device *device, struct radeon_cmdbuf
static void
radv_perfcounter_emit_sample(struct radeon_cmdbuf *cs)
{
radeon_event_write(cs, V_028A90_PERFCOUNTER_SAMPLE);
radeon_begin(cs);
radeon_event_write(V_028A90_PERFCOUNTER_SAMPLE);
radeon_end();
}
enum radv_perfcounter_op {
@@ -492,7 +508,9 @@ radv_emit_instance(struct radv_cmd_buffer *cmd_buffer, int se, int instance)
value |= S_030800_INSTANCE_BROADCAST_WRITES(1);
}
radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, value);
radeon_begin(cs);
radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, value);
radeon_end();
}
static void
@@ -512,15 +530,18 @@ radv_emit_select(struct radv_cmd_buffer *cmd_buffer, struct ac_pc_block *block,
if (!regs->select0)
return;
radeon_begin(cs);
for (idx = 0; idx < count; ++idx) {
radeon_set_uconfig_perfctr_reg(gfx_level, ring, cs, regs->select0[idx],
G_REG_SEL(selectors[idx]) | regs->select_or);
radeon_set_uconfig_perfctr_reg(gfx_level, ring, regs->select0[idx], G_REG_SEL(selectors[idx]) | regs->select_or);
}
for (idx = 0; idx < regs->num_spm_counters; idx++) {
radeon_set_uconfig_reg_seq(cs, regs->select1[idx], 1);
radeon_emit(cs, 0);
radeon_set_uconfig_reg_seq(regs->select1[idx], 1);
radeon_emit(0);
}
radeon_end();
}
static void
@@ -539,13 +560,15 @@ radv_pc_emit_block_instance_read(struct radv_cmd_buffer *cmd_buffer, struct ac_p
if (regs->counters)
reg = regs->counters[idx];
radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) | COPY_DATA_DST_SEL(COPY_DATA_TC_L2) | COPY_DATA_WR_CONFIRM |
COPY_DATA_COUNT_SEL); /* 64 bits */
radeon_emit(cs, reg >> 2);
radeon_emit(cs, 0); /* unused */
radeon_emit(cs, va);
radeon_emit(cs, va >> 32);
radeon_begin(cs);
radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0));
radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_PERF) | COPY_DATA_DST_SEL(COPY_DATA_TC_L2) | COPY_DATA_WR_CONFIRM |
COPY_DATA_COUNT_SEL); /* 64 bits */
radeon_emit(reg >> 2);
radeon_emit(0); /* unused */
radeon_emit(va);
radeon_emit(va >> 32);
radeon_end();
va += sizeof(uint64_t) * 2 * radv_pc_get_num_instances(pdev, block);
reg += reg_delta;
@@ -575,19 +598,23 @@ radv_pc_wait_idle(struct radv_cmd_buffer *cmd_buffer)
{
struct radeon_cmdbuf *cs = cmd_buffer->cs;
radeon_event_write(cs, V_028A90_CS_PARTIAL_FLUSH);
radeon_begin(cs);
radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 6, 0));
radeon_emit(cs, 0); /* CP_COHER_CNTL */
radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */
radeon_emit(cs, 0xffffff); /* CP_COHER_SIZE_HI */
radeon_emit(cs, 0); /* CP_COHER_BASE */
radeon_emit(cs, 0); /* CP_COHER_BASE_HI */
radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */
radeon_emit(cs, 0); /* GCR_CNTL */
radeon_event_write(V_028A90_CS_PARTIAL_FLUSH);
radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0));
radeon_emit(cs, 0);
radeon_emit(PKT3(PKT3_ACQUIRE_MEM, 6, 0));
radeon_emit(0); /* CP_COHER_CNTL */
radeon_emit(0xffffffff); /* CP_COHER_SIZE */
radeon_emit(0xffffff); /* CP_COHER_SIZE_HI */
radeon_emit(0); /* CP_COHER_BASE */
radeon_emit(0); /* CP_COHER_BASE_HI */
radeon_emit(0x0000000A); /* POLL_INTERVAL */
radeon_emit(0); /* GCR_CNTL */
radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0));
radeon_emit(0);
radeon_end();
}
static void
@@ -607,11 +634,13 @@ radv_pc_stop_and_sample(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query
uint64_t pred_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_PASS_OFFSET + 8 * pass;
uint64_t reg_va = va + (end ? 8 : 0);
radeon_emit(cs, PKT3(PKT3_COND_EXEC, 3, 0));
radeon_emit(cs, pred_va);
radeon_emit(cs, pred_va >> 32);
radeon_emit(cs, 0); /* Cache policy */
radeon_emit(cs, 0);
radeon_begin(cs);
radeon_emit(PKT3(PKT3_COND_EXEC, 3, 0));
radeon_emit(pred_va);
radeon_emit(pred_va >> 32);
radeon_emit(0); /* Cache policy */
radeon_emit(0);
radeon_end();
uint32_t *skip_dwords = cs->buf + (cs->cdw - 1);
@@ -677,11 +706,13 @@ radv_pc_begin_query(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query_poo
for (unsigned pass = 0; pass < pool->num_passes; ++pass) {
uint64_t pred_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_PASS_OFFSET + 8 * pass;
radeon_emit(cs, PKT3(PKT3_COND_EXEC, 3, 0));
radeon_emit(cs, pred_va);
radeon_emit(cs, pred_va >> 32);
radeon_emit(cs, 0); /* Cache policy */
radeon_emit(cs, 0);
radeon_begin(cs);
radeon_emit(PKT3(PKT3_COND_EXEC, 3, 0));
radeon_emit(pred_va);
radeon_emit(pred_va >> 32);
radeon_emit(0); /* Cache policy */
radeon_emit(0);
radeon_end();
uint32_t *skip_dwords = cs->buf + (cs->cdw - 1);
src/amd/vulkan/radv_query.c
+41 -30
View File
@@ -38,12 +38,14 @@ static void radv_query_shader(struct radv_cmd_buffer *cmd_buffer, VkQueryType qu
static void
gfx10_copy_shader_query(struct radeon_cmdbuf *cs, uint32_t src_sel, uint64_t src_va, uint64_t dst_va)
{
radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
radeon_emit(cs, COPY_DATA_SRC_SEL(src_sel) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM);
radeon_emit(cs, src_va);
radeon_emit(cs, src_va >> 32);
radeon_emit(cs, dst_va);
radeon_emit(cs, dst_va >> 32);
radeon_begin(cs);
radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0));
radeon_emit(COPY_DATA_SRC_SEL(src_sel) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM);
radeon_emit(src_va);
radeon_emit(src_va >> 32);
radeon_emit(dst_va);
radeon_emit(dst_va >> 32);
radeon_end();
}
static void
@@ -95,18 +97,20 @@ static void
radv_emit_event_write(const struct radeon_info *info, struct radeon_cmdbuf *cs, enum radv_event_write event,
uint64_t va)
{
radeon_begin(cs);
if (event == RADV_EVENT_WRITE_PIPELINE_STAT) {
radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
radeon_emit(cs, EVENT_TYPE(V_028A90_SAMPLE_PIPELINESTAT) | EVENT_INDEX(2));
radeon_emit(PKT3(PKT3_EVENT_WRITE, 2, 0));
radeon_emit(EVENT_TYPE(V_028A90_SAMPLE_PIPELINESTAT) | EVENT_INDEX(2));
} else if (event == RADV_EVENT_WRITE_OCCLUSION_QUERY) {
if (info->gfx_level >= GFX11 && info->pfp_fw_version >= EVENT_WRITE_ZPASS_PFP_VERSION) {
radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_ZPASS, 1, 0));
radeon_emit(PKT3(PKT3_EVENT_WRITE_ZPASS, 1, 0));
} else {
radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
radeon_emit(PKT3(PKT3_EVENT_WRITE, 2, 0));
if (info->gfx_level >= GFX11) {
radeon_emit(cs, EVENT_TYPE(V_028A90_PIXEL_PIPE_STAT_DUMP) | EVENT_INDEX(1));
radeon_emit(EVENT_TYPE(V_028A90_PIXEL_PIPE_STAT_DUMP) | EVENT_INDEX(1));
} else {
radeon_emit(cs, EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1));
radeon_emit(EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1));
}
}
} else {
@@ -119,12 +123,14 @@ radv_emit_event_write(const struct radeon_info *info, struct radeon_cmdbuf *cs,
V_028A90_SAMPLE_STREAMOUTSTATS3,
};
radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
radeon_emit(cs, EVENT_TYPE(streamout_events[event]) | EVENT_INDEX(3));
radeon_emit(PKT3(PKT3_EVENT_WRITE, 2, 0));
radeon_emit(EVENT_TYPE(streamout_events[event]) | EVENT_INDEX(3));
}
radeon_emit(cs, va);
radeon_emit(cs, va >> 32);
radeon_emit(va);
radeon_emit(va >> 32);
radeon_end();
}
static void
@@ -2706,13 +2712,15 @@ radv_write_timestamp(struct radv_cmd_buffer *cmd_buffer, uint64_t va, VkPipeline
struct radeon_cmdbuf *cs = cmd_buffer->cs;
if (stage == VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT) {
radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
radeon_emit(cs, COPY_DATA_COUNT_SEL | COPY_DATA_WR_CONFIRM | COPY_DATA_SRC_SEL(COPY_DATA_TIMESTAMP) |
COPY_DATA_DST_SEL(V_370_MEM));
radeon_emit(cs, 0);
radeon_emit(cs, 0);
radeon_emit(cs, va);
radeon_emit(cs, va >> 32);
radeon_begin(cs);
radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0));
radeon_emit(COPY_DATA_COUNT_SEL | COPY_DATA_WR_CONFIRM | COPY_DATA_SRC_SEL(COPY_DATA_TIMESTAMP) |
COPY_DATA_DST_SEL(V_370_MEM));
radeon_emit(0);
radeon_emit(0);
radeon_emit(va);
radeon_emit(va >> 32);
radeon_end();
} else {
radv_cs_emit_write_event_eop(cs, pdev->info.gfx_level, cmd_buffer->qf, V_028A90_BOTTOM_OF_PIPE_TS, 0,
EOP_DST_SEL_MEM, EOP_DATA_SEL_TIMESTAMP, va, 0, cmd_buffer->gfx9_eop_bug_va);
@@ -2791,6 +2799,8 @@ radv_CmdWriteAccelerationStructuresPropertiesKHR(VkCommandBuffer commandBuffer,
ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cs, 6 * accelerationStructureCount);
radeon_begin(cs);
for (uint32_t i = 0; i < accelerationStructureCount; ++i) {
VK_FROM_HANDLE(vk_acceleration_structure, accel_struct, pAccelerationStructures[i]);
uint64_t va = vk_acceleration_structure_get_va(accel_struct);
@@ -2812,16 +2822,17 @@ radv_CmdWriteAccelerationStructuresPropertiesKHR(VkCommandBuffer commandBuffer,
unreachable("Unhandle accel struct query type.");
}
radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) |
COPY_DATA_COUNT_SEL | COPY_DATA_WR_CONFIRM);
radeon_emit(cs, va);
radeon_emit(cs, va >> 32);
radeon_emit(cs, query_va);
radeon_emit(cs, query_va >> 32);
radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0));
radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL |
COPY_DATA_WR_CONFIRM);
radeon_emit(va);
radeon_emit(va >> 32);
radeon_emit(query_va);
radeon_emit(query_va >> 32);
query_va += pool->stride;
}
radeon_end();
assert(cmd_buffer->cs->cdw <= cdw_max);
}
src/amd/vulkan/radv_queue.c
+138 -102
View File
@@ -373,15 +373,19 @@ radv_emit_gs_ring_sizes(struct radv_device *device, struct radeon_cmdbuf *cs, st
if (gsvs_ring_bo)
radv_cs_add_buffer(device->ws, cs, gsvs_ring_bo);
radeon_begin(cs);
if (pdev->info.gfx_level >= GFX7) {
radeon_set_uconfig_reg_seq(cs, R_030900_VGT_ESGS_RING_SIZE, 2);
radeon_emit(cs, esgs_ring_size >> 8);
radeon_emit(cs, gsvs_ring_size >> 8);
radeon_set_uconfig_reg_seq(R_030900_VGT_ESGS_RING_SIZE, 2);
radeon_emit(esgs_ring_size >> 8);
radeon_emit(gsvs_ring_size >> 8);
} else {
radeon_set_config_reg_seq(cs, R_0088C8_VGT_ESGS_RING_SIZE, 2);
radeon_emit(cs, esgs_ring_size >> 8);
radeon_emit(cs, gsvs_ring_size >> 8);
radeon_set_config_reg_seq(R_0088C8_VGT_ESGS_RING_SIZE, 2);
radeon_emit(esgs_ring_size >> 8);
radeon_emit(gsvs_ring_size >> 8);
}
radeon_end();
}
static void
@@ -398,29 +402,33 @@ radv_emit_tess_factor_ring(struct radv_device *device, struct radeon_cmdbuf *cs,
radv_cs_add_buffer(device->ws, cs, tess_rings_bo);
radeon_begin(cs);
if (pdev->info.gfx_level >= GFX7) {
if (pdev->info.gfx_level >= GFX11) {
/* TF_RING_SIZE is per SE on GFX11. */
tf_ring_size /= pdev->info.max_se;
}
radeon_set_uconfig_reg(cs, R_030938_VGT_TF_RING_SIZE, S_030938_SIZE(tf_ring_size));
radeon_set_uconfig_reg(cs, R_030940_VGT_TF_MEMORY_BASE, tf_va >> 8);
radeon_set_uconfig_reg(R_030938_VGT_TF_RING_SIZE, S_030938_SIZE(tf_ring_size));
radeon_set_uconfig_reg(R_030940_VGT_TF_MEMORY_BASE, tf_va >> 8);
if (pdev->info.gfx_level >= GFX12) {
radeon_set_uconfig_reg(cs, R_03099C_VGT_TF_MEMORY_BASE_HI, S_03099C_BASE_HI(tf_va >> 40));
radeon_set_uconfig_reg(R_03099C_VGT_TF_MEMORY_BASE_HI, S_03099C_BASE_HI(tf_va >> 40));
} else if (pdev->info.gfx_level >= GFX10) {
radeon_set_uconfig_reg(cs, R_030984_VGT_TF_MEMORY_BASE_HI, S_030984_BASE_HI(tf_va >> 40));
radeon_set_uconfig_reg(R_030984_VGT_TF_MEMORY_BASE_HI, S_030984_BASE_HI(tf_va >> 40));
} else if (pdev->info.gfx_level == GFX9) {
radeon_set_uconfig_reg(cs, R_030944_VGT_TF_MEMORY_BASE_HI, S_030944_BASE_HI(tf_va >> 40));
radeon_set_uconfig_reg(R_030944_VGT_TF_MEMORY_BASE_HI, S_030944_BASE_HI(tf_va >> 40));
}
radeon_set_uconfig_reg(cs, R_03093C_VGT_HS_OFFCHIP_PARAM, pdev->hs.hs_offchip_param);
radeon_set_uconfig_reg(R_03093C_VGT_HS_OFFCHIP_PARAM, pdev->hs.hs_offchip_param);
} else {
radeon_set_config_reg(cs, R_008988_VGT_TF_RING_SIZE, S_008988_SIZE(tf_ring_size));
radeon_set_config_reg(cs, R_0089B8_VGT_TF_MEMORY_BASE, tf_va >> 8);
radeon_set_config_reg(cs, R_0089B0_VGT_HS_OFFCHIP_PARAM, pdev->hs.hs_offchip_param);
radeon_set_config_reg(R_008988_VGT_TF_RING_SIZE, S_008988_SIZE(tf_ring_size));
radeon_set_config_reg(R_0089B8_VGT_TF_MEMORY_BASE, tf_va >> 8);
radeon_set_config_reg(R_0089B0_VGT_HS_OFFCHIP_PARAM, pdev->hs.hs_offchip_param);
}
radeon_end();
}
static VkResult
@@ -466,12 +474,16 @@ radv_emit_task_rings(struct radv_device *device, struct radeon_cmdbuf *cs, struc
assert(util_is_aligned(task_ctrlbuf_va, 256));
radv_cs_add_buffer(device->ws, cs, task_rings_bo);
radeon_begin(cs);
/* Tell the GPU where the task control buffer is. */
radeon_emit(cs, PKT3(PKT3_DISPATCH_TASK_STATE_INIT, 1, 0) | PKT3_SHADER_TYPE_S(!!compute));
radeon_emit(PKT3(PKT3_DISPATCH_TASK_STATE_INIT, 1, 0) | PKT3_SHADER_TYPE_S(!!compute));
/* bits [31:8]: control buffer address lo, bits[7:0]: reserved (set to zero) */
radeon_emit(cs, task_ctrlbuf_va & 0xFFFFFF00);
radeon_emit(task_ctrlbuf_va & 0xFFFFFF00);
/* bits [31:0]: control buffer address hi */
radeon_emit(cs, task_ctrlbuf_va >> 32);
radeon_emit(task_ctrlbuf_va >> 32);
radeon_end();
}
static void
@@ -486,20 +498,24 @@ radv_emit_graphics_scratch(struct radv_device *device, struct radeon_cmdbuf *cs,
radv_cs_add_buffer(device->ws, cs, scratch_bo);
radeon_begin(cs);
if (gpu_info->gfx_level >= GFX11) {
uint64_t va = radv_buffer_get_va(scratch_bo);
/* WAVES is per SE for SPI_TMPRING_SIZE. */
waves /= gpu_info->max_se;
radeon_set_context_reg_seq(cs, R_0286E8_SPI_TMPRING_SIZE, 3);
radeon_emit(cs, S_0286E8_WAVES(waves) | S_0286E8_WAVESIZE(DIV_ROUND_UP(size_per_wave, 256)));
radeon_emit(cs, va >> 8); /* SPI_GFX_SCRATCH_BASE_LO */
radeon_emit(cs, va >> 40); /* SPI_GFX_SCRATCH_BASE_HI */
radeon_set_context_reg_seq(R_0286E8_SPI_TMPRING_SIZE, 3);
radeon_emit(S_0286E8_WAVES(waves) | S_0286E8_WAVESIZE(DIV_ROUND_UP(size_per_wave, 256)));
radeon_emit(va >> 8); /* SPI_GFX_SCRATCH_BASE_LO */
radeon_emit(va >> 40); /* SPI_GFX_SCRATCH_BASE_HI */
} else {
radeon_set_context_reg(cs, R_0286E8_SPI_TMPRING_SIZE,
radeon_set_context_reg(R_0286E8_SPI_TMPRING_SIZE,
S_0286E8_WAVES(waves) | S_0286E8_WAVESIZE(DIV_ROUND_UP(size_per_wave, 1024)));
}
radeon_end();
}
static void
@@ -524,21 +540,25 @@ radv_emit_compute_scratch(struct radv_device *device, struct radeon_cmdbuf *cs,
radv_cs_add_buffer(device->ws, cs, compute_scratch_bo);
radeon_begin(cs);
if (gpu_info->gfx_level >= GFX11) {
radeon_set_sh_reg_seq(cs, R_00B840_COMPUTE_DISPATCH_SCRATCH_BASE_LO, 2);
radeon_emit(cs, scratch_va >> 8);
radeon_emit(cs, scratch_va >> 40);
radeon_set_sh_reg_seq(R_00B840_COMPUTE_DISPATCH_SCRATCH_BASE_LO, 2);
radeon_emit(scratch_va >> 8);
radeon_emit(scratch_va >> 40);
waves /= gpu_info->max_se;
}
radeon_set_sh_reg_seq(cs, R_00B900_COMPUTE_USER_DATA_0, 2);
radeon_emit(cs, scratch_va);
radeon_emit(cs, rsrc1);
radeon_set_sh_reg_seq(R_00B900_COMPUTE_USER_DATA_0, 2);
radeon_emit(scratch_va);
radeon_emit(rsrc1);
radeon_set_sh_reg(cs, R_00B860_COMPUTE_TMPRING_SIZE,
radeon_set_sh_reg(R_00B860_COMPUTE_TMPRING_SIZE,
S_00B860_WAVES(waves) |
S_00B860_WAVESIZE(DIV_ROUND_UP(size_per_wave, gpu_info->gfx_level >= GFX11 ? 256 : 1024)));
radeon_end();
}
static void
@@ -554,7 +574,9 @@ radv_emit_compute_shader_pointers(struct radv_device *device, struct radeon_cmdb
/* Compute shader user data 0-1 have the scratch pointer (unlike GFX shaders),
* so emit the descriptor pointer to user data 2-3 instead (task_ring_offsets arg).
*/
radeon_emit_64bit_pointer(cs, R_00B908_COMPUTE_USER_DATA_2, va);
radeon_begin(cs);
radeon_emit_64bit_pointer(R_00B908_COMPUTE_USER_DATA_2, va);
radeon_end();
}
static void
@@ -571,33 +593,35 @@ radv_emit_graphics_shader_pointers(struct radv_device *device, struct radeon_cmd
radv_cs_add_buffer(device->ws, cs, descriptor_bo);
radeon_begin(cs);
if (pdev->info.gfx_level >= GFX12) {
uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B410_SPI_SHADER_PGM_LO_HS,
R_00B210_SPI_SHADER_PGM_LO_GS};
for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
radeon_emit_64bit_pointer(cs, regs[i], va);
radeon_emit_64bit_pointer(regs[i], va);
}
} else if (pdev->info.gfx_level >= GFX11) {
uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B420_SPI_SHADER_PGM_LO_HS,
R_00B220_SPI_SHADER_PGM_LO_GS};
for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
radeon_emit_64bit_pointer(cs, regs[i], va);
radeon_emit_64bit_pointer(regs[i], va);
}
} else if (pdev->info.gfx_level >= GFX10) {
uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B130_SPI_SHADER_USER_DATA_VS_0,
R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS, R_00B408_SPI_SHADER_USER_DATA_ADDR_LO_HS};
for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
radeon_emit_64bit_pointer(cs, regs[i], va);
radeon_emit_64bit_pointer(regs[i], va);
}
} else if (pdev->info.gfx_level == GFX9) {
uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B130_SPI_SHADER_USER_DATA_VS_0,
R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS, R_00B408_SPI_SHADER_USER_DATA_ADDR_LO_HS};
for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
radeon_emit_64bit_pointer(cs, regs[i], va);
radeon_emit_64bit_pointer(regs[i], va);
}
} else {
uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B130_SPI_SHADER_USER_DATA_VS_0,
@@ -605,9 +629,11 @@ radv_emit_graphics_shader_pointers(struct radv_device *device, struct radeon_cmd
R_00B430_SPI_SHADER_USER_DATA_HS_0, R_00B530_SPI_SHADER_USER_DATA_LS_0};
for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
radeon_emit_64bit_pointer(cs, regs[i], va);
radeon_emit_64bit_pointer(regs[i], va);
}
}
radeon_end();
}
static void
@@ -626,53 +652,56 @@ radv_emit_ge_rings(struct radv_device *device, struct radeon_cmdbuf *cs, struct
radv_cs_add_buffer(device->ws, cs, ge_rings_bo);
radeon_begin(cs);
/* We must wait for idle using an EOP event before changing the attribute ring registers. Use the
* bottom-of-pipe EOP event, but increment the PWS counter instead of writing memory.
*/
radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, 6, 0));
radeon_emit(cs, S_490_EVENT_TYPE(V_028A90_BOTTOM_OF_PIPE_TS) | S_490_EVENT_INDEX(5) | S_490_PWS_ENABLE(1));
radeon_emit(cs, 0); /* DST_SEL, INT_SEL, DATA_SEL */
radeon_emit(cs, 0); /* ADDRESS_LO */
radeon_emit(cs, 0); /* ADDRESS_HI */
radeon_emit(cs, 0); /* DATA_LO */
radeon_emit(cs, 0); /* DATA_HI */
radeon_emit(cs, 0); /* INT_CTXID */
radeon_emit(PKT3(PKT3_RELEASE_MEM, 6, 0));
radeon_emit(S_490_EVENT_TYPE(V_028A90_BOTTOM_OF_PIPE_TS) | S_490_EVENT_INDEX(5) | S_490_PWS_ENABLE(1));
radeon_emit(0); /* DST_SEL, INT_SEL, DATA_SEL */
radeon_emit(0); /* ADDRESS_LO */
radeon_emit(0); /* ADDRESS_HI */
radeon_emit(0); /* DATA_LO */
radeon_emit(0); /* DATA_HI */
radeon_emit(0); /* INT_CTXID */
/* Wait for the PWS counter. */
radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 6, 0));
radeon_emit(cs, S_580_PWS_STAGE_SEL(V_580_CP_ME) | S_580_PWS_COUNTER_SEL(V_580_TS_SELECT) | S_580_PWS_ENA2(1) |
S_580_PWS_COUNT(0));
radeon_emit(cs, 0xffffffff); /* GCR_SIZE */
radeon_emit(cs, 0x01ffffff); /* GCR_SIZE_HI */
radeon_emit(cs, 0); /* GCR_BASE_LO */
radeon_emit(cs, 0); /* GCR_BASE_HI */
radeon_emit(cs, S_585_PWS_ENA(1));
radeon_emit(cs, 0); /* GCR_CNTL */
radeon_emit(PKT3(PKT3_ACQUIRE_MEM, 6, 0));
radeon_emit(S_580_PWS_STAGE_SEL(V_580_CP_ME) | S_580_PWS_COUNTER_SEL(V_580_TS_SELECT) | S_580_PWS_ENA2(1) |
S_580_PWS_COUNT(0));
radeon_emit(0xffffffff); /* GCR_SIZE */
radeon_emit(0x01ffffff); /* GCR_SIZE_HI */
radeon_emit(0); /* GCR_BASE_LO */
radeon_emit(0); /* GCR_BASE_HI */
radeon_emit(S_585_PWS_ENA(1));
radeon_emit(0); /* GCR_CNTL */
/* The PS will read inputs from this address. */
radeon_set_uconfig_reg_seq(cs, R_031110_SPI_GS_THROTTLE_CNTL1, 4);
radeon_emit(cs, 0x12355123); /* SPI_GS_THROTTLE_CNTL1 */
radeon_emit(cs, 0x1544D); /* SPI_GS_THROTTLE_CNTL2 */
radeon_emit(cs, va >> 16); /* SPI_ATTRIBUTE_RING_BASE */
radeon_emit(cs, S_03111C_MEM_SIZE((pdev->info.attribute_ring_size_per_se >> 16) - 1) |
S_03111C_BIG_PAGE(pdev->info.discardable_allows_big_page) |
S_03111C_L1_POLICY(1)); /* SPI_ATTRIBUTE_RING_SIZE */
radeon_set_uconfig_reg_seq(R_031110_SPI_GS_THROTTLE_CNTL1, 4);
radeon_emit(0x12355123); /* SPI_GS_THROTTLE_CNTL1 */
radeon_emit(0x1544D); /* SPI_GS_THROTTLE_CNTL2 */
radeon_emit(va >> 16); /* SPI_ATTRIBUTE_RING_BASE */
radeon_emit(S_03111C_MEM_SIZE((pdev->info.attribute_ring_size_per_se >> 16) - 1) |
S_03111C_BIG_PAGE(pdev->info.discardable_allows_big_page) |
S_03111C_L1_POLICY(1)); /* SPI_ATTRIBUTE_RING_SIZE */
if (pdev->info.gfx_level >= GFX12) {
const uint64_t pos_address = va + pdev->info.pos_ring_offset;
const uint64_t prim_address = va + pdev->info.prim_ring_offset;
/* When one of these 4 registers is updated, all 4 must be updated. */
radeon_set_uconfig_reg_seq(cs, R_0309A0_GE_POS_RING_BASE, 4);
radeon_emit(cs, pos_address >> 16); /* R_0309A0_GE_POS_RING_BASE */
radeon_emit(cs, S_0309A4_MEM_SIZE(pdev->info.pos_ring_size_per_se >> 5)); /* R_0309A4_GE_POS_RING_SIZE */
radeon_emit(cs, prim_address >> 16); /* R_0309A8_GE_PRIM_RING_BASE */
radeon_emit(cs, S_0309AC_MEM_SIZE(pdev->info.prim_ring_size_per_se >> 5) | S_0309AC_SCOPE(gfx12_scope_device) |
S_0309AC_PAF_TEMPORAL(gfx12_store_high_temporal_stay_dirty) |
S_0309AC_PAB_TEMPORAL(gfx12_load_last_use_discard) |
S_0309AC_SPEC_DATA_READ(gfx12_spec_read_auto) | S_0309AC_FORCE_SE_SCOPE(1) |
S_0309AC_PAB_NOFILL(1)); /* R_0309AC_GE_PRIM_RING_SIZE */
radeon_set_uconfig_reg_seq(R_0309A0_GE_POS_RING_BASE, 4);
radeon_emit(pos_address >> 16); /* R_0309A0_GE_POS_RING_BASE */
radeon_emit(S_0309A4_MEM_SIZE(pdev->info.pos_ring_size_per_se >> 5)); /* R_0309A4_GE_POS_RING_SIZE */
radeon_emit(prim_address >> 16); /* R_0309A8_GE_PRIM_RING_BASE */
radeon_emit(S_0309AC_MEM_SIZE(pdev->info.prim_ring_size_per_se >> 5) | S_0309AC_SCOPE(gfx12_scope_device) |
S_0309AC_PAF_TEMPORAL(gfx12_store_high_temporal_stay_dirty) |
S_0309AC_PAB_TEMPORAL(gfx12_load_last_use_discard) | S_0309AC_SPEC_DATA_READ(gfx12_spec_read_auto) |
S_0309AC_FORCE_SE_SCOPE(1) | S_0309AC_PAB_NOFILL(1)); /* R_0309AC_GE_PRIM_RING_SIZE */
}
radeon_end();
}
static void
@@ -1109,8 +1138,10 @@ radv_update_preamble_cs(struct radv_queue_state *queue, struct radv_device *devi
radv_init_graphics_state(cs, device);
if (esgs_ring_bo || gsvs_ring_bo || tess_rings_bo || task_rings_bo) {
radeon_event_write(cs, V_028A90_VS_PARTIAL_FLUSH);
radeon_event_write(cs, V_028A90_VGT_FLUSH);
radeon_begin(cs);
radeon_event_write(V_028A90_VS_PARTIAL_FLUSH);
radeon_event_write(V_028A90_VGT_FLUSH);
radeon_end();
}
radv_emit_gs_ring_sizes(device, cs, esgs_ring_bo, needs->esgs_ring_size, gsvs_ring_bo, needs->gsvs_ring_size);
@@ -1126,7 +1157,9 @@ radv_update_preamble_cs(struct radv_queue_state *queue, struct radv_device *devi
radv_emit_compute(device, cs, true);
if (task_rings_bo) {
radeon_event_write(cs, V_028A90_CS_PARTIAL_FLUSH);
radeon_begin(cs);
radeon_event_write(V_028A90_CS_PARTIAL_FLUSH);
radeon_end();
}
radv_emit_task_rings(device, cs, task_rings_bo, true);
@@ -1548,51 +1581,54 @@ radv_create_perf_counter_lock_cs(struct radv_device *device, unsigned pass, bool
radv_cs_add_buffer(device->ws, cs, device->perf_counter_bo);
radeon_begin(cs);
if (!unlock) {
uint64_t mutex_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_LOCK_OFFSET;
radeon_emit(cs, PKT3(PKT3_ATOMIC_MEM, 7, 0));
radeon_emit(cs, ATOMIC_OP(TC_OP_ATOMIC_CMPSWAP_32) | ATOMIC_COMMAND(ATOMIC_COMMAND_LOOP));
radeon_emit(cs, mutex_va); /* addr lo */
radeon_emit(cs, mutex_va >> 32); /* addr hi */
radeon_emit(cs, 1); /* data lo */
radeon_emit(cs, 0); /* data hi */
radeon_emit(cs, 0); /* compare data lo */
radeon_emit(cs, 0); /* compare data hi */
radeon_emit(cs, 10); /* loop interval */
radeon_emit(PKT3(PKT3_ATOMIC_MEM, 7, 0));
radeon_emit(ATOMIC_OP(TC_OP_ATOMIC_CMPSWAP_32) | ATOMIC_COMMAND(ATOMIC_COMMAND_LOOP));
radeon_emit(mutex_va); /* addr lo */
radeon_emit(mutex_va >> 32); /* addr hi */
radeon_emit(1); /* data lo */
radeon_emit(0); /* data hi */
radeon_emit(0); /* compare data lo */
radeon_emit(0); /* compare data hi */
radeon_emit(10); /* loop interval */
}
uint64_t va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_PASS_OFFSET;
uint64_t unset_va = va + (unlock ? 8 * pass : 0);
uint64_t set_va = va + (unlock ? 0 : 8 * pass);
radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL |
COPY_DATA_WR_CONFIRM);
radeon_emit(cs, 0); /* immediate */
radeon_emit(cs, 0);
radeon_emit(cs, unset_va);
radeon_emit(cs, unset_va >> 32);
radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0));
radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL |
COPY_DATA_WR_CONFIRM);
radeon_emit(0); /* immediate */
radeon_emit(0);
radeon_emit(unset_va);
radeon_emit(unset_va >> 32);
radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL |
COPY_DATA_WR_CONFIRM);
radeon_emit(cs, 1); /* immediate */
radeon_emit(cs, 0);
radeon_emit(cs, set_va);
radeon_emit(cs, set_va >> 32);
radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0));
radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL |
COPY_DATA_WR_CONFIRM);
radeon_emit(1); /* immediate */
radeon_emit(0);
radeon_emit(set_va);
radeon_emit(set_va >> 32);
if (unlock) {
uint64_t mutex_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_LOCK_OFFSET;
radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL |
COPY_DATA_WR_CONFIRM);
radeon_emit(cs, 0); /* immediate */
radeon_emit(cs, 0);
radeon_emit(cs, mutex_va);
radeon_emit(cs, mutex_va >> 32);
radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0));
radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL |
COPY_DATA_WR_CONFIRM);
radeon_emit(0); /* immediate */
radeon_emit(0);
radeon_emit(mutex_va);
radeon_emit(mutex_va >> 32);
}
radeon_end();
assert(cs->cdw <= cdw);
VkResult result = device->ws->cs_finalize(cs);
src/amd/vulkan/radv_radeon_winsys.h
-15
View File
@@ -321,21 +321,6 @@ struct radeon_winsys {
const struct vk_sync_type *const *(*get_sync_types)(struct radeon_winsys *ws);
};
static inline void
radeon_emit(struct radeon_cmdbuf *cs, uint32_t value)
{
assert(cs->cdw < cs->reserved_dw);
cs->buf[cs->cdw++] = value;
}
static inline void
radeon_emit_array(struct radeon_cmdbuf *cs, const uint32_t *values, unsigned count)
{
assert(cs->cdw + count <= cs->reserved_dw);
memcpy(cs->buf + cs->cdw, values, count * 4);
cs->cdw += count;
}
static inline uint64_t
radv_buffer_get_va(const struct radeon_winsys_bo *bo)
{
src/amd/vulkan/radv_sdma.c
+107 -84
View File
@@ -323,44 +323,54 @@ radv_sdma_emit_nop(const struct radv_device *device, struct radeon_cmdbuf *cs)
{
/* SDMA NOP acts as a fence command and causes the SDMA engine to wait for pending copy operations. */
radeon_check_space(device->ws, cs, 1);
radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
radeon_begin(cs);
radeon_emit(SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
radeon_end();
}
void
radv_sdma_emit_write_timestamp(struct radeon_cmdbuf *cs, uint64_t va)
{
radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_TIMESTAMP, SDMA_TS_SUB_OPCODE_GET_GLOBAL_TIMESTAMP, 0));
radeon_emit(cs, va);
radeon_emit(cs, va >> 32);
radeon_begin(cs);
radeon_emit(SDMA_PACKET(SDMA_OPCODE_TIMESTAMP, SDMA_TS_SUB_OPCODE_GET_GLOBAL_TIMESTAMP, 0));
radeon_emit(va);
radeon_emit(va >> 32);
radeon_end();
}
void
radv_sdma_emit_fence(struct radeon_cmdbuf *cs, uint64_t va, uint32_t fence)
{
radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, SDMA_FENCE_MTYPE_UC));
radeon_emit(cs, va);
radeon_emit(cs, va >> 32);
radeon_emit(cs, fence);
radeon_begin(cs);
radeon_emit(SDMA_PACKET(SDMA_OPCODE_FENCE, 0, SDMA_FENCE_MTYPE_UC));
radeon_emit(va);
radeon_emit(va >> 32);
radeon_emit(fence);
radeon_end();
}
void
radv_sdma_emit_wait_mem(struct radeon_cmdbuf *cs, uint32_t op, uint64_t va, uint32_t ref, uint32_t mask)
{
radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_POLL_REGMEM, 0, 0) | op << 28 | SDMA_POLL_MEM);
radeon_emit(cs, va);
radeon_emit(cs, va >> 32);
radeon_emit(cs, ref);
radeon_emit(cs, mask);
radeon_emit(cs, SDMA_POLL_INTERVAL_160_CLK | SDMA_POLL_RETRY_INDEFINITELY << 16);
radeon_begin(cs);
radeon_emit(SDMA_PACKET(SDMA_OPCODE_POLL_REGMEM, 0, 0) | op << 28 | SDMA_POLL_MEM);
radeon_emit(va);
radeon_emit(va >> 32);
radeon_emit(ref);
radeon_emit(mask);
radeon_emit(SDMA_POLL_INTERVAL_160_CLK | SDMA_POLL_RETRY_INDEFINITELY << 16);
radeon_end();
}
void
radv_sdma_emit_write_data_head(struct radeon_cmdbuf *cs, uint64_t va, uint32_t count)
{
radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
radeon_emit(cs, va);
radeon_emit(cs, va >> 32);
radeon_emit(cs, count - 1);
radeon_begin(cs);
radeon_emit(SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
radeon_emit(va);
radeon_emit(va >> 32);
radeon_emit(count - 1);
radeon_end();
}
void
@@ -393,19 +403,23 @@ radv_sdma_copy_memory(const struct radv_device *device, struct radeon_cmdbuf *cs
radeon_check_space(device->ws, cs, ncopy * 7);
radeon_begin(cs);
for (unsigned i = 0; i < ncopy; i++) {
unsigned csize = size >= 4 ? MIN2(size & align, max_size_per_packet) : size;
radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0));
radeon_emit(cs, ver >= SDMA_4_0 ? csize - 1 : csize);
radeon_emit(cs, 0); /* src/dst endian swap */
radeon_emit(cs, src_va);
radeon_emit(cs, src_va >> 32);
radeon_emit(cs, dst_va);
radeon_emit(cs, dst_va >> 32);
radeon_emit(SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0));
radeon_emit(ver >= SDMA_4_0 ? csize - 1 : csize);
radeon_emit(0); /* src/dst endian swap */
radeon_emit(src_va);
radeon_emit(src_va >> 32);
radeon_emit(dst_va);
radeon_emit(dst_va >> 32);
dst_va += csize;
src_va += csize;
size -= csize;
}
radeon_end();
}
void
@@ -428,18 +442,21 @@ radv_sdma_fill_memory(const struct radv_device *device, struct radeon_cmdbuf *cs
const unsigned num_packets = DIV_ROUND_UP(size, max_fill_bytes);
ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cs, num_packets * 5);
radeon_begin(cs);
for (unsigned i = 0; i < num_packets; ++i) {
const uint64_t offset = i * max_fill_bytes;
const uint64_t fill_bytes = MIN2(size - offset, max_fill_bytes);
const uint64_t fill_va = va + offset;
radeon_emit(cs, constant_fill_header);
radeon_emit(cs, fill_va);
radeon_emit(cs, fill_va >> 32);
radeon_emit(cs, value);
radeon_emit(cs, fill_bytes - 1); /* Must be programmed in bytes, even if the fill is done in dwords. */
radeon_emit(constant_fill_header);
radeon_emit(fill_va);
radeon_emit(fill_va >> 32);
radeon_emit(value);
radeon_emit(fill_bytes - 1); /* Must be programmed in bytes, even if the fill is done in dwords. */
}
radeon_end();
assert(cs->cdw <= cdw_max);
}
@@ -475,20 +492,22 @@ radv_sdma_emit_copy_linear_sub_window(const struct radv_device *device, struct r
ASSERTED unsigned cdw_end = radeon_check_space(device->ws, cs, 13);
radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR_SUB_WINDOW, 0) | util_logbase2(src->bpp)
<< 29);
radeon_emit(cs, src->va);
radeon_emit(cs, src->va >> 32);
radeon_emit(cs, src_off.x | src_off.y << 16);
radeon_emit(cs, src_off.z | (src_pitch - 1) << (ver >= SDMA_7_0 ? 16 : 13));
radeon_emit(cs, src_slice_pitch - 1);
radeon_emit(cs, dst->va);
radeon_emit(cs, dst->va >> 32);
radeon_emit(cs, dst_off.x | dst_off.y << 16);
radeon_emit(cs, dst_off.z | (dst_pitch - 1) << (ver >= SDMA_7_0 ? 16 : 13));
radeon_emit(cs, dst_slice_pitch - 1);
radeon_emit(cs, (ext.width - 1) | (ext.height - 1) << 16);
radeon_emit(cs, (ext.depth - 1));
radeon_begin(cs);
radeon_emit(SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR_SUB_WINDOW, 0) | util_logbase2(src->bpp)
<< 29);
radeon_emit(src->va);
radeon_emit(src->va >> 32);
radeon_emit(src_off.x | src_off.y << 16);
radeon_emit(src_off.z | (src_pitch - 1) << (ver >= SDMA_7_0 ? 16 : 13));
radeon_emit(src_slice_pitch - 1);
radeon_emit(dst->va);
radeon_emit(dst->va >> 32);
radeon_emit(dst_off.x | dst_off.y << 16);
radeon_emit(dst_off.z | (dst_pitch - 1) << (ver >= SDMA_7_0 ? 16 : 13));
radeon_emit(dst_slice_pitch - 1);
radeon_emit((ext.width - 1) | (ext.height - 1) << 16);
radeon_emit((ext.depth - 1));
radeon_end();
assert(cs->cdw == cdw_end);
}
@@ -519,29 +538,31 @@ radv_sdma_emit_copy_tiled_sub_window(const struct radv_device *device, struct ra
ASSERTED unsigned cdw_end = radeon_check_space(device->ws, cs, 14 + (dcc ? 3 : 0));
radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_TILED_SUB_WINDOW, 0) | dcc << 19 | detile << 31 |
tiled->header_dword);
radeon_emit(cs, tiled->va);
radeon_emit(cs, tiled->va >> 32);
radeon_emit(cs, tiled_off.x | tiled_off.y << 16);
radeon_emit(cs, tiled_off.z | (tiled_ext.width - 1) << 16);
radeon_emit(cs, (tiled_ext.height - 1) | (tiled_ext.depth - 1) << 16);
radeon_emit(cs, tiled->info_dword);
radeon_emit(cs, linear->va);
radeon_emit(cs, linear->va >> 32);
radeon_emit(cs, linear_off.x | linear_off.y << 16);
radeon_emit(cs, linear_off.z | (linear_pitch - 1) << 16);
radeon_emit(cs, linear_slice_pitch - 1);
radeon_emit(cs, (ext.width - 1) | (ext.height - 1) << 16);
radeon_emit(cs, (ext.depth - 1));
radeon_begin(cs);
radeon_emit(SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_TILED_SUB_WINDOW, 0) | dcc << 19 | detile << 31 |
tiled->header_dword);
radeon_emit(tiled->va);
radeon_emit(tiled->va >> 32);
radeon_emit(tiled_off.x | tiled_off.y << 16);
radeon_emit(tiled_off.z | (tiled_ext.width - 1) << 16);
radeon_emit((tiled_ext.height - 1) | (tiled_ext.depth - 1) << 16);
radeon_emit(tiled->info_dword);
radeon_emit(linear->va);
radeon_emit(linear->va >> 32);
radeon_emit(linear_off.x | linear_off.y << 16);
radeon_emit(linear_off.z | (linear_pitch - 1) << 16);
radeon_emit(linear_slice_pitch - 1);
radeon_emit((ext.width - 1) | (ext.height - 1) << 16);
radeon_emit((ext.depth - 1));
if (tiled->meta_va) {
const unsigned write_compress_enable = !detile;
radeon_emit(cs, tiled->meta_va);
radeon_emit(cs, tiled->meta_va >> 32);
radeon_emit(cs, tiled->meta_config | write_compress_enable << 28);
radeon_emit(tiled->meta_va);
radeon_emit(tiled->meta_va >> 32);
radeon_emit(tiled->meta_config | write_compress_enable << 28);
}
radeon_end();
assert(cs->cdw == cdw_end);
}
@@ -583,34 +604,36 @@ radv_sdma_emit_copy_t2t_sub_window(const struct radv_device *device, struct rade
ASSERTED unsigned cdw_end = radeon_check_space(device->ws, cs, 15 + (dcc ? 3 : 0));
radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_T2T_SUB_WINDOW, 0) | dcc << 19 | dcc_dir << 31 |
src->header_dword);
radeon_emit(cs, src->va);
radeon_emit(cs, src->va >> 32);
radeon_emit(cs, src_off.x | src_off.y << 16);
radeon_emit(cs, src_off.z | (src_ext.width - 1) << 16);
radeon_emit(cs, (src_ext.height - 1) | (src_ext.depth - 1) << 16);
radeon_emit(cs, src->info_dword);
radeon_emit(cs, dst->va);
radeon_emit(cs, dst->va >> 32);
radeon_emit(cs, dst_off.x | dst_off.y << 16);
radeon_emit(cs, dst_off.z | (dst_ext.width - 1) << 16);
radeon_emit(cs, (dst_ext.height - 1) | (dst_ext.depth - 1) << 16);
radeon_emit(cs, dst->info_dword);
radeon_emit(cs, (ext.width - 1) | (ext.height - 1) << 16);
radeon_emit(cs, (ext.depth - 1));
radeon_begin(cs);
radeon_emit(SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_T2T_SUB_WINDOW, 0) | dcc << 19 | dcc_dir << 31 |
src->header_dword);
radeon_emit(src->va);
radeon_emit(src->va >> 32);
radeon_emit(src_off.x | src_off.y << 16);
radeon_emit(src_off.z | (src_ext.width - 1) << 16);
radeon_emit((src_ext.height - 1) | (src_ext.depth - 1) << 16);
radeon_emit(src->info_dword);
radeon_emit(dst->va);
radeon_emit(dst->va >> 32);
radeon_emit(dst_off.x | dst_off.y << 16);
radeon_emit(dst_off.z | (dst_ext.width - 1) << 16);
radeon_emit((dst_ext.height - 1) | (dst_ext.depth - 1) << 16);
radeon_emit(dst->info_dword);
radeon_emit((ext.width - 1) | (ext.height - 1) << 16);
radeon_emit((ext.depth - 1));
if (dst->meta_va) {
const uint32_t write_compress_enable = 1;
radeon_emit(cs, dst->meta_va);
radeon_emit(cs, dst->meta_va >> 32);
radeon_emit(cs, dst->meta_config | write_compress_enable << 28);
radeon_emit(dst->meta_va);
radeon_emit(dst->meta_va >> 32);
radeon_emit(dst->meta_config | write_compress_enable << 28);
} else if (src->meta_va) {
radeon_emit(cs, src->meta_va);
radeon_emit(cs, src->meta_va >> 32);
radeon_emit(cs, src->meta_config);
radeon_emit(src->meta_va);
radeon_emit(src->meta_va >> 32);
radeon_emit(src->meta_config);
}
radeon_end();
assert(cs->cdw == cdw_end);
}
src/amd/vulkan/radv_spm.c
+50 -34
View File
@@ -83,16 +83,19 @@ radv_emit_spm_counters(struct radv_device *device, struct radeon_cmdbuf *cs, enu
continue;
radeon_check_space(device->ws, cs, 3 + num_counters * 3);
radeon_begin(cs);
radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, spm->sq_wgp[instance].grbm_gfx_index);
radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, spm->sq_wgp[instance].grbm_gfx_index);
for (uint32_t b = 0; b < num_counters; b++) {
const struct ac_spm_counter_select *cntr_sel = &spm->sq_wgp[instance].counters[b];
uint32_t reg_base = R_036700_SQ_PERFCOUNTER0_SELECT;
radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, reg_base + b * 4, 1);
radeon_emit(cs, cntr_sel->sel0);
radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, reg_base + b * 4, 1);
radeon_emit(cntr_sel->sel0);
}
radeon_end();
}
}
@@ -103,18 +106,21 @@ radv_emit_spm_counters(struct radv_device *device, struct radeon_cmdbuf *cs, enu
continue;
radeon_check_space(device->ws, cs, 3 + num_counters * 3);
radeon_begin(cs);
radeon_set_uconfig_reg(
cs, R_030800_GRBM_GFX_INDEX,
S_030800_SH_BROADCAST_WRITES(1) | S_030800_INSTANCE_BROADCAST_WRITES(1) | S_030800_SE_INDEX(instance));
radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, S_030800_SH_BROADCAST_WRITES(1) |
S_030800_INSTANCE_BROADCAST_WRITES(1) |
S_030800_SE_INDEX(instance));
for (uint32_t b = 0; b < num_counters; b++) {
const struct ac_spm_counter_select *cntr_sel = &spm->sqg[instance].counters[b];
uint32_t reg_base = R_036700_SQ_PERFCOUNTER0_SELECT;
radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, reg_base + b * 4, 1);
radeon_emit(cs, cntr_sel->sel0 | S_036700_SQC_BANK_MASK(0xf)); /* SQC_BANK_MASK only gfx10 */
radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, reg_base + b * 4, 1);
radeon_emit(cntr_sel->sel0 | S_036700_SQC_BANK_MASK(0xf)); /* SQC_BANK_MASK only gfx10 */
}
radeon_end();
}
for (uint32_t b = 0; b < spm->num_block_sel; b++) {
@@ -125,8 +131,9 @@ radv_emit_spm_counters(struct radv_device *device, struct radeon_cmdbuf *cs, enu
struct ac_spm_block_instance *block_instance = &block_sel->instances[i];
radeon_check_space(device->ws, cs, 3 + (AC_SPM_MAX_COUNTER_PER_BLOCK * 6));
radeon_begin(cs);
radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, block_instance->grbm_gfx_index);
radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, block_instance->grbm_gfx_index);
for (unsigned c = 0; c < block_instance->num_counters; c++) {
const struct ac_spm_counter_select *cntr_sel = &block_instance->counters[c];
@@ -134,19 +141,22 @@ radv_emit_spm_counters(struct radv_device *device, struct radeon_cmdbuf *cs, enu
if (!cntr_sel->active)
continue;
radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, regs->select0[c], 1);
radeon_emit(cs, cntr_sel->sel0);
radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, regs->select0[c], 1);
radeon_emit(cntr_sel->sel0);
radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, regs->select1[c], 1);
radeon_emit(cs, cntr_sel->sel1);
radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, regs->select1[c], 1);
radeon_emit(cntr_sel->sel1);
}
radeon_end();
}
}
/* Restore global broadcasting. */
radeon_set_uconfig_reg(
cs, R_030800_GRBM_GFX_INDEX,
S_030800_SE_BROADCAST_WRITES(1) | S_030800_SH_BROADCAST_WRITES(1) | S_030800_INSTANCE_BROADCAST_WRITES(1));
radeon_begin(cs);
radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, S_030800_SE_BROADCAST_WRITES(1) | S_030800_SH_BROADCAST_WRITES(1) |
S_030800_INSTANCE_BROADCAST_WRITES(1));
radeon_end();
}
static void
@@ -181,23 +191,26 @@ radv_emit_spm_muxsel(struct radv_device *device, struct radeon_cmdbuf *cs, enum
}
radeon_check_space(device->ws, cs, 3 + spm->num_muxsel_lines[s] * (7 + AC_SPM_MUXSEL_LINE_SIZE));
radeon_begin(cs);
radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, grbm_gfx_index);
radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, grbm_gfx_index);
for (unsigned l = 0; l < spm->num_muxsel_lines[s]; l++) {
uint32_t *data = (uint32_t *)spm->muxsel_lines[s][l].muxsel;
/* Select MUXSEL_ADDR to point to the next muxsel. */
radeon_set_uconfig_perfctr_reg(pdev->info.gfx_level, ring, cs, rlc_muxsel_addr, l * AC_SPM_MUXSEL_LINE_SIZE);
radeon_set_uconfig_perfctr_reg(pdev->info.gfx_level, ring, rlc_muxsel_addr, l * AC_SPM_MUXSEL_LINE_SIZE);
/* Write the muxsel line configuration with MUXSEL_DATA. */
radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + AC_SPM_MUXSEL_LINE_SIZE, 0));
radeon_emit(cs, S_370_DST_SEL(V_370_MEM_MAPPED_REGISTER) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME) |
S_370_WR_ONE_ADDR(1));
radeon_emit(cs, rlc_muxsel_data >> 2);
radeon_emit(cs, 0);
radeon_emit_array(cs, data, AC_SPM_MUXSEL_LINE_SIZE);
radeon_emit(PKT3(PKT3_WRITE_DATA, 2 + AC_SPM_MUXSEL_LINE_SIZE, 0));
radeon_emit(S_370_DST_SEL(V_370_MEM_MAPPED_REGISTER) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME) |
S_370_WR_ONE_ADDR(1));
radeon_emit(rlc_muxsel_data >> 2);
radeon_emit(0);
radeon_emit_array(data, AC_SPM_MUXSEL_LINE_SIZE);
}
radeon_end();
}
}
@@ -215,14 +228,15 @@ radv_emit_spm_setup(struct radv_device *device, struct radeon_cmdbuf *cs, enum r
assert(spm->sample_interval >= 32);
radeon_check_space(device->ws, cs, 27);
radeon_begin(cs);
/* Configure the SPM ring buffer. */
radeon_set_uconfig_reg(cs, R_037200_RLC_SPM_PERFMON_CNTL,
radeon_set_uconfig_reg(R_037200_RLC_SPM_PERFMON_CNTL,
S_037200_PERFMON_RING_MODE(0) | /* no stall and no interrupt on overflow */
S_037200_PERFMON_SAMPLE_INTERVAL(spm->sample_interval)); /* in sclk */
radeon_set_uconfig_reg(cs, R_037204_RLC_SPM_PERFMON_RING_BASE_LO, va);
radeon_set_uconfig_reg(cs, R_037208_RLC_SPM_PERFMON_RING_BASE_HI, S_037208_RING_BASE_HI(va >> 32));
radeon_set_uconfig_reg(cs, R_03720C_RLC_SPM_PERFMON_RING_SIZE, ring_size);
radeon_set_uconfig_reg(R_037204_RLC_SPM_PERFMON_RING_BASE_LO, va);
radeon_set_uconfig_reg(R_037208_RLC_SPM_PERFMON_RING_BASE_HI, S_037208_RING_BASE_HI(va >> 32));
radeon_set_uconfig_reg(R_03720C_RLC_SPM_PERFMON_RING_SIZE, ring_size);
/* Configure the muxsel. */
uint32_t total_muxsel_lines = 0;
@@ -230,27 +244,29 @@ radv_emit_spm_setup(struct radv_device *device, struct radeon_cmdbuf *cs, enum r
total_muxsel_lines += spm->num_muxsel_lines[s];
}
radeon_set_uconfig_reg(cs, R_03726C_RLC_SPM_ACCUM_MODE, 0);
radeon_set_uconfig_reg(R_03726C_RLC_SPM_ACCUM_MODE, 0);
if (pdev->info.gfx_level >= GFX11) {
radeon_set_uconfig_reg(cs, R_03721C_RLC_SPM_PERFMON_SEGMENT_SIZE,
radeon_set_uconfig_reg(R_03721C_RLC_SPM_PERFMON_SEGMENT_SIZE,
S_03721C_TOTAL_NUM_SEGMENT(total_muxsel_lines) |
S_03721C_GLOBAL_NUM_SEGMENT(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_GLOBAL]) |
S_03721C_SE_NUM_SEGMENT(spm->max_se_muxsel_lines));
radeon_set_uconfig_reg(cs, R_037210_RLC_SPM_RING_WRPTR, 0);
radeon_set_uconfig_reg(R_037210_RLC_SPM_RING_WRPTR, 0);
} else {
radeon_set_uconfig_reg(cs, R_037210_RLC_SPM_PERFMON_SEGMENT_SIZE, 0);
radeon_set_uconfig_reg(cs, R_03727C_RLC_SPM_PERFMON_SE3TO0_SEGMENT_SIZE,
radeon_set_uconfig_reg(R_037210_RLC_SPM_PERFMON_SEGMENT_SIZE, 0);
radeon_set_uconfig_reg(R_03727C_RLC_SPM_PERFMON_SE3TO0_SEGMENT_SIZE,
S_03727C_SE0_NUM_LINE(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_SE0]) |
S_03727C_SE1_NUM_LINE(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_SE1]) |
S_03727C_SE2_NUM_LINE(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_SE2]) |
S_03727C_SE3_NUM_LINE(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_SE3]));
radeon_set_uconfig_reg(cs, R_037280_RLC_SPM_PERFMON_GLB_SEGMENT_SIZE,
radeon_set_uconfig_reg(R_037280_RLC_SPM_PERFMON_GLB_SEGMENT_SIZE,
S_037280_PERFMON_SEGMENT_SIZE(total_muxsel_lines) |
S_037280_GLOBAL_NUM_LINE(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_GLOBAL]));
}
radeon_end();
/* Upload each muxsel ram to the RLC. */
radv_emit_spm_muxsel(device, cs, qf);
src/amd/vulkan/radv_sqtt.c
+37 -18
View File
@@ -132,14 +132,17 @@ radv_emit_sqtt_userdata(const struct radv_cmd_buffer *cmd_buffer, const void *da
uint32_t count = MIN2(num_dwords, 2);
radeon_check_space(device->ws, cs, 2 + count);
radeon_begin(cs);
/* Without the perfctr bit the CP might not always pass the
* write on correctly. */
if (pdev->info.gfx_level >= GFX10)
radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, R_030D08_SQ_THREAD_TRACE_USERDATA_2, count);
radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, R_030D08_SQ_THREAD_TRACE_USERDATA_2, count);
else
radeon_set_uconfig_reg_seq(cs, R_030D08_SQ_THREAD_TRACE_USERDATA_2, count);
radeon_emit_array(cs, dwords, count);
radeon_set_uconfig_reg_seq(R_030D08_SQ_THREAD_TRACE_USERDATA_2, count);
radeon_emit_array(dwords, count);
radeon_end();
dwords += count;
num_dwords -= count;
@@ -151,8 +154,10 @@ radv_emit_spi_config_cntl(const struct radv_device *device, struct radeon_cmdbuf
{
const struct radv_physical_device *pdev = radv_device_physical(device);
radeon_begin(cs);
if (pdev->info.gfx_level >= GFX12) {
radeon_set_uconfig_reg(cs, R_031120_SPI_SQG_EVENT_CTL,
radeon_set_uconfig_reg(R_031120_SPI_SQG_EVENT_CTL,
S_031120_ENABLE_SQG_TOP_EVENTS(enable) | S_031120_ENABLE_SQG_BOP_EVENTS(enable));
} else if (pdev->info.gfx_level >= GFX9) {
uint32_t spi_config_cntl = S_031100_GPR_WRITE_PRIORITY(0x2c688) | S_031100_EXP_PRIORITY_ORDER(3) |
@@ -161,12 +166,14 @@ radv_emit_spi_config_cntl(const struct radv_device *device, struct radeon_cmdbuf
if (pdev->info.gfx_level >= GFX10)
spi_config_cntl |= S_031100_PS_PKR_PRIORITY_CNTL(3);
radeon_set_uconfig_reg(cs, R_031100_SPI_CONFIG_CNTL, spi_config_cntl);
radeon_set_uconfig_reg(R_031100_SPI_CONFIG_CNTL, spi_config_cntl);
} else {
/* SPI_CONFIG_CNTL is a protected register on GFX6-GFX8. */
radeon_set_privileged_config_reg(cs, R_009100_SPI_CONFIG_CNTL,
radeon_set_privileged_config_reg(R_009100_SPI_CONFIG_CNTL,
S_009100_ENABLE_SQG_TOP_EVENTS(enable) | S_009100_ENABLE_SQG_BOP_EVENTS(enable));
}
radeon_end();
}
void
@@ -177,11 +184,15 @@ radv_emit_inhibit_clockgating(const struct radv_device *device, struct radeon_cm
if (pdev->info.gfx_level >= GFX11)
return; /* not needed */
radeon_begin(cs);
if (pdev->info.gfx_level >= GFX10) {
radeon_set_uconfig_reg(cs, R_037390_RLC_PERFMON_CLK_CNTL, S_037390_PERFMON_CLOCK_STATE(inhibit));
radeon_set_uconfig_reg(R_037390_RLC_PERFMON_CLK_CNTL, S_037390_PERFMON_CLOCK_STATE(inhibit));
} else if (pdev->info.gfx_level >= GFX8) {
radeon_set_uconfig_reg(cs, R_0372FC_RLC_PERFMON_CLK_CNTL, S_0372FC_PERFMON_CLOCK_STATE(inhibit));
radeon_set_uconfig_reg(R_0372FC_RLC_PERFMON_CLK_CNTL, S_0372FC_PERFMON_CLOCK_STATE(inhibit));
}
radeon_end();
}
VkResult
@@ -520,21 +531,25 @@ radv_begin_sqtt(struct radv_queue *queue)
radeon_check_space(ws, cs, 512);
radeon_begin(cs);
switch (family) {
case RADV_QUEUE_GENERAL:
radeon_emit(cs, PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
radeon_emit(cs, CC0_UPDATE_LOAD_ENABLES(1));
radeon_emit(cs, CC1_UPDATE_SHADOW_ENABLES(1));
radeon_emit(PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
radeon_emit(CC0_UPDATE_LOAD_ENABLES(1));
radeon_emit(CC1_UPDATE_SHADOW_ENABLES(1));
break;
case RADV_QUEUE_COMPUTE:
radeon_emit(cs, PKT3(PKT3_NOP, 0, 0));
radeon_emit(cs, 0);
radeon_emit(PKT3(PKT3_NOP, 0, 0));
radeon_emit(0);
break;
default:
unreachable("Incorrect queue family");
break;
}
radeon_end();
/* Make sure to wait-for-idle before starting SQTT. */
radv_emit_wait_for_idle(device, cs, family);
@@ -593,21 +608,25 @@ radv_end_sqtt(struct radv_queue *queue)
radeon_check_space(ws, cs, 512);
radeon_begin(cs);
switch (family) {
case RADV_QUEUE_GENERAL:
radeon_emit(cs, PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
radeon_emit(cs, CC0_UPDATE_LOAD_ENABLES(1));
radeon_emit(cs, CC1_UPDATE_SHADOW_ENABLES(1));
radeon_emit(PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
radeon_emit(CC0_UPDATE_LOAD_ENABLES(1));
radeon_emit(CC1_UPDATE_SHADOW_ENABLES(1));
break;
case RADV_QUEUE_COMPUTE:
radeon_emit(cs, PKT3(PKT3_NOP, 0, 0));
radeon_emit(cs, 0);
radeon_emit(PKT3(PKT3_NOP, 0, 0));
radeon_emit(0);
break;
default:
unreachable("Incorrect queue family");
break;
}
radeon_end();
/* Make sure to wait-for-idle before stopping SQTT. */
radv_emit_wait_for_idle(device, cs, family);
src/amd/vulkan/radv_video.c
+23 -12
View File
@@ -67,10 +67,12 @@ radv_vcn_sq_header(struct radeon_cmdbuf *cs, struct rvcn_sq_var *sq, unsigned ty
{
if (!skip_signature) {
/* vcn ib signature */
radeon_emit(cs, RADEON_VCN_SIGNATURE_SIZE);
radeon_emit(cs, RADEON_VCN_SIGNATURE);
radeon_emit(cs, 0);
radeon_emit(cs, 0);
radeon_begin(cs);
radeon_emit(RADEON_VCN_SIGNATURE_SIZE);
radeon_emit(RADEON_VCN_SIGNATURE);
radeon_emit(0);
radeon_emit(0);
radeon_end();
sq->signature_ib_checksum = &cs->buf[cs->cdw - 2];
sq->signature_ib_total_size_in_dw = &cs->buf[cs->cdw - 1];
@@ -80,10 +82,12 @@ radv_vcn_sq_header(struct radeon_cmdbuf *cs, struct rvcn_sq_var *sq, unsigned ty
}
/* vcn ib engine info */
radeon_emit(cs, RADEON_VCN_ENGINE_INFO_SIZE);
radeon_emit(cs, RADEON_VCN_ENGINE_INFO);
radeon_emit(cs, type);
radeon_emit(cs, 0);
radeon_begin(cs);
radeon_emit(RADEON_VCN_ENGINE_INFO_SIZE);
radeon_emit(RADEON_VCN_ENGINE_INFO);
radeon_emit(type);
radeon_emit(0);
radeon_end();
sq->engine_ib_size_of_packages = &cs->buf[cs->cdw - 1];
}
@@ -1072,8 +1076,11 @@ static void
set_reg(struct radv_cmd_buffer *cmd_buffer, unsigned reg, uint32_t val)
{
struct radeon_cmdbuf *cs = cmd_buffer->cs;
radeon_emit(cs, RDECODE_PKT0(reg >> 2, 0));
radeon_emit(cs, val);
radeon_begin(cs);
radeon_emit(RDECODE_PKT0(reg >> 2, 0));
radeon_emit(val);
radeon_end();
}
static void
@@ -2584,8 +2591,10 @@ radv_vcn_cmd_reset(struct radv_cmd_buffer *cmd_buffer)
if (pdev->vid_decode_ip != AMD_IP_VCN_UNIFIED) {
radeon_check_space(device->ws, cmd_buffer->cs, 8);
radeon_begin(cmd_buffer->cs);
for (unsigned i = 0; i < 8; i++)
radeon_emit(cmd_buffer->cs, 0x81ff);
radeon_emit(0x81ff);
radeon_end();
} else
radv_vcn_sq_tail(cmd_buffer->cs, &cmd_buffer->video.sq);
}
@@ -2610,8 +2619,10 @@ radv_uvd_cmd_reset(struct radv_cmd_buffer *cmd_buffer)
/* pad out the IB to the 16 dword boundary - otherwise the fw seems to be unhappy */
int padsize = vid->sessionctx.mem ? 4 : 6;
radeon_check_space(device->ws, cmd_buffer->cs, padsize);
radeon_begin(cmd_buffer->cs);
for (unsigned i = 0; i < padsize; i++)
radeon_emit(cmd_buffer->cs, PKT2_NOP_PAD);
radeon_emit(PKT2_NOP_PAD);
radeon_end();
}
VKAPI_ATTR void VKAPI_CALL
src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c
+15
View File
@@ -101,6 +101,21 @@ struct radv_winsys_sem_info {
struct radv_winsys_sem_counts signal;
};
static void
radeon_emit(struct radeon_cmdbuf *cs, uint32_t value)
{
assert(cs->cdw < cs->reserved_dw);
cs->buf[cs->cdw++] = value;
}
static void
radeon_emit_array(struct radeon_cmdbuf *cs, const uint32_t *values, unsigned count)
{
assert(cs->cdw + count <= cs->reserved_dw);
memcpy(cs->buf + cs->cdw, values, count * 4);
cs->cdw += count;
}
static void
radeon_emit_unchecked(struct radeon_cmdbuf *cs, uint32_t value)
{