From f0b3a6f9d42955ddc1130de955d33d9e3ac4e32b Mon Sep 17 00:00:00 2001 From: Samuel Pitoiset Date: Mon, 24 Mar 2025 14:07:01 +0100 Subject: [PATCH] radv: rework command buffer emission with begin/end sequences A begin/end sequence is something like (it's all macros based): radeon_begin(cs); radeon_emit(PKT3(PKT3_DRAW_INDEX_AUTO, 1, cmd_buffer->state.predicating)); radeon_emit(vertex_count); radeon_emit(V_0287F0_DI_SRC_SEL_AUTO_INDEX | use_opaque); radeon_end(); This is loosely based on RadeonSI (see !8653 (a0978fff)) and it seems indeed faster overall. The main goal of this rework is to re-use the same logic as RadeonSI for paired packets on GFX12 (also GFX11 dGPUs) because it's supposed to be way faster, especially on GFX12 where the CP is slow. The other goal is to share more cmdbuf emission between both drivers in the near future. Signed-off-by: Samuel Pitoiset Part-of: --- src/amd/vulkan/layers/radv_sqtt_layer.c | 60 +- src/amd/vulkan/meta/radv_meta_buffer.c | 13 +- src/amd/vulkan/radv_cmd_buffer.c | 1941 ++++++++++------- src/amd/vulkan/radv_cp_dma.c | 50 +- src/amd/vulkan/radv_cp_reg_shadowing.c | 10 +- src/amd/vulkan/radv_cs.c | 282 ++- src/amd/vulkan/radv_cs.h | 193 +- src/amd/vulkan/radv_device.c | 48 +- src/amd/vulkan/radv_perfcounter.c | 119 +- src/amd/vulkan/radv_query.c | 71 +- src/amd/vulkan/radv_queue.c | 240 +- src/amd/vulkan/radv_radeon_winsys.h | 15 - src/amd/vulkan/radv_sdma.c | 191 +- src/amd/vulkan/radv_spm.c | 84 +- src/amd/vulkan/radv_sqtt.c | 55 +- src/amd/vulkan/radv_video.c | 35 +- src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c | 15 + 17 files changed, 2013 insertions(+), 1409 deletions(-) diff --git a/src/amd/vulkan/layers/radv_sqtt_layer.c b/src/amd/vulkan/layers/radv_sqtt_layer.c index 0351bff7ae0..e0cd3e66c0f 100644 --- a/src/amd/vulkan/layers/radv_sqtt_layer.c +++ b/src/amd/vulkan/layers/radv_sqtt_layer.c @@ -30,23 +30,25 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv radv_cs_add_buffer(device->ws, cs, reloc->bo); + radeon_begin(cs); + /* VS */ if (pipeline->base.shaders[MESA_SHADER_VERTEX]) { struct radv_shader *vs = pipeline->base.shaders[MESA_SHADER_VERTEX]; va = reloc->va[MESA_SHADER_VERTEX]; if (vs->info.vs.as_ls) { - radeon_set_sh_reg(cs, vs->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(vs->info.regs.pgm_lo, va >> 8); } else if (vs->info.vs.as_es) { - radeon_set_sh_reg_seq(cs, vs->info.regs.pgm_lo, 2); - radeon_emit(cs, va >> 8); - radeon_emit(cs, S_00B324_MEM_BASE(va >> 40)); + radeon_set_sh_reg_seq(vs->info.regs.pgm_lo, 2); + radeon_emit(va >> 8); + radeon_emit(S_00B324_MEM_BASE(va >> 40)); } else if (vs->info.is_ngg) { - radeon_set_sh_reg(cs, vs->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(vs->info.regs.pgm_lo, va >> 8); } else { - radeon_set_sh_reg_seq(cs, vs->info.regs.pgm_lo, 2); - radeon_emit(cs, va >> 8); - radeon_emit(cs, S_00B124_MEM_BASE(va >> 40)); + radeon_set_sh_reg_seq(vs->info.regs.pgm_lo, 2); + radeon_emit(va >> 8); + radeon_emit(S_00B124_MEM_BASE(va >> 40)); } } @@ -57,11 +59,11 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv va = reloc->va[MESA_SHADER_TESS_CTRL]; if (gfx_level >= GFX9) { - radeon_set_sh_reg(cs, tcs->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(tcs->info.regs.pgm_lo, va >> 8); } else { - radeon_set_sh_reg_seq(cs, tcs->info.regs.pgm_lo, 2); - radeon_emit(cs, va >> 8); - radeon_emit(cs, S_00B424_MEM_BASE(va >> 40)); + radeon_set_sh_reg_seq(tcs->info.regs.pgm_lo, 2); + radeon_emit(va >> 8); + radeon_emit(S_00B424_MEM_BASE(va >> 40)); } } @@ -71,15 +73,15 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv va = reloc->va[MESA_SHADER_TESS_EVAL]; if (tes->info.is_ngg) { - radeon_set_sh_reg(cs, tes->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(tes->info.regs.pgm_lo, va >> 8); } else if (tes->info.tes.as_es) { - radeon_set_sh_reg_seq(cs, tes->info.regs.pgm_lo, 2); - radeon_emit(cs, va >> 8); - radeon_emit(cs, S_00B324_MEM_BASE(va >> 40)); + radeon_set_sh_reg_seq(tes->info.regs.pgm_lo, 2); + radeon_emit(va >> 8); + radeon_emit(S_00B324_MEM_BASE(va >> 40)); } else { - radeon_set_sh_reg_seq(cs, tes->info.regs.pgm_lo, 2); - radeon_emit(cs, va >> 8); - radeon_emit(cs, S_00B124_MEM_BASE(va >> 40)); + radeon_set_sh_reg_seq(tes->info.regs.pgm_lo, 2); + radeon_emit(va >> 8); + radeon_emit(S_00B124_MEM_BASE(va >> 40)); } } @@ -89,14 +91,14 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv va = reloc->va[MESA_SHADER_GEOMETRY]; if (gs->info.is_ngg) { - radeon_set_sh_reg(cs, gs->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(gs->info.regs.pgm_lo, va >> 8); } else { if (gfx_level >= GFX9) { - radeon_set_sh_reg(cs, gs->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(gs->info.regs.pgm_lo, va >> 8); } else { - radeon_set_sh_reg_seq(cs, gs->info.regs.pgm_lo, 2); - radeon_emit(cs, va >> 8); - radeon_emit(cs, S_00B224_MEM_BASE(va >> 40)); + radeon_set_sh_reg_seq(gs->info.regs.pgm_lo, 2); + radeon_emit(va >> 8); + radeon_emit(S_00B224_MEM_BASE(va >> 40)); } } } @@ -107,9 +109,9 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv va = reloc->va[MESA_SHADER_FRAGMENT]; - radeon_set_sh_reg_seq(cs, ps->info.regs.pgm_lo, 2); - radeon_emit(cs, va >> 8); - radeon_emit(cs, S_00B024_MEM_BASE(va >> 40)); + radeon_set_sh_reg_seq(ps->info.regs.pgm_lo, 2); + radeon_emit(va >> 8); + radeon_emit(S_00B024_MEM_BASE(va >> 40)); } /* MS */ @@ -118,8 +120,10 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv va = reloc->va[MESA_SHADER_MESH]; - radeon_set_sh_reg(cs, ms->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(ms->info.regs.pgm_lo, va >> 8); } + + radeon_end(); } static uint64_t diff --git a/src/amd/vulkan/meta/radv_meta_buffer.c b/src/amd/vulkan/meta/radv_meta_buffer.c index 563043b3673..4b4bfe39038 100644 --- a/src/amd/vulkan/meta/radv_meta_buffer.c +++ b/src/amd/vulkan/meta/radv_meta_buffer.c @@ -305,12 +305,13 @@ radv_update_buffer_cp(struct radv_cmd_buffer *cmd_buffer, uint64_t va, const voi radv_emit_cache_flush(cmd_buffer); radeon_check_space(device->ws, cmd_buffer->cs, words + 4); - radeon_emit(cmd_buffer->cs, PKT3(PKT3_WRITE_DATA, 2 + words, 0)); - radeon_emit(cmd_buffer->cs, - S_370_DST_SEL(mec ? V_370_MEM : V_370_MEM_GRBM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME)); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, va >> 32); - radeon_emit_array(cmd_buffer->cs, data, words); + radeon_begin(cmd_buffer->cs); + radeon_emit(PKT3(PKT3_WRITE_DATA, 2 + words, 0)); + radeon_emit(S_370_DST_SEL(mec ? V_370_MEM : V_370_MEM_GRBM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit_array(data, words); + radeon_end(); if (radv_device_fault_detection_enabled(device)) radv_cmd_buffer_trace_emit(cmd_buffer); diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c index 8a1c59ea6c9..14adda707af 100644 --- a/src/amd/vulkan/radv_cmd_buffer.c +++ b/src/amd/vulkan/radv_cmd_buffer.c @@ -619,8 +619,10 @@ radv_cmd_buffer_trace_emit(struct radv_cmd_buffer *cmd_buffer) radeon_check_space(device->ws, cs, 2); - radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); - radeon_emit(cs, AC_ENCODE_TRACE_POINT(cmd_buffer->state.trace_id)); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_NOP, 0, 0)); + radeon_emit(AC_ENCODE_TRACE_POINT(cmd_buffer->state.trace_id)); + radeon_end(); } void @@ -836,8 +838,9 @@ radv_cmd_buffer_after_draw(struct radv_cmd_buffer *cmd_buffer, enum radv_cmd_flu if (unlikely(device->sqtt.bo) && !dgc) { radeon_check_space(device->ws, cmd_buffer->cs, 2); - - radeon_event_write_predicate(cmd_buffer->cs, V_028A90_THREAD_TRACE_MARKER, cmd_buffer->state.predicating); + radeon_begin(cmd_buffer->cs); + radeon_event_write_predicate(V_028A90_THREAD_TRACE_MARKER, cmd_buffer->state.predicating); + radeon_end(); } if (instance->debug_flags & RADV_DEBUG_SYNC_SHADERS) { @@ -962,7 +965,9 @@ radv_emit_userdata_address(const struct radv_device *device, struct radeon_cmdbu if (!offset) return; - radeon_emit_32bit_pointer(cs, offset, va, &pdev->info); + radeon_begin(cs); + radeon_emit_32bit_pointer(offset, va, &pdev->info); + radeon_end(); } uint64_t @@ -988,8 +993,10 @@ radv_emit_descriptors_per_stage(const struct radv_device *device, struct radeon_ const uint32_t indirect_descriptor_sets_offset = radv_get_user_sgpr_loc(shader, AC_UD_INDIRECT_DESCRIPTOR_SETS); if (indirect_descriptor_sets_offset) { - radeon_emit_32bit_pointer(cs, indirect_descriptor_sets_offset, descriptors_state->indirect_descriptor_sets_va, + radeon_begin(cs); + radeon_emit_32bit_pointer(indirect_descriptor_sets_offset, descriptors_state->indirect_descriptor_sets_va, &pdev->info); + radeon_end(); } else { const struct radv_userdata_locations *locs = &shader->info.user_sgprs_locs; const uint32_t sh_base = shader->info.user_data_0; @@ -1005,13 +1012,16 @@ radv_emit_descriptors_per_stage(const struct radv_device *device, struct radeon_ const struct radv_userdata_info *loc = &locs->descriptor_sets[start]; const unsigned sh_offset = sh_base + loc->sgpr_idx * 4; - radeon_set_sh_reg_seq(cs, sh_offset, count); + radeon_begin(cs); + radeon_set_sh_reg_seq(sh_offset, count); for (int i = 0; i < count; i++) { uint64_t va = radv_descriptor_get_va(descriptors_state, start + i); - radeon_emit(cs, va); + radeon_emit(va); } + + radeon_end(); } } } @@ -1212,36 +1222,38 @@ radv_emit_sample_locations(struct radv_cmd_buffer *cmd_buffer) /* Compute the PA_SC_CENTROID_PRIORITY_* mask. */ centroid_priority = radv_compute_centroid_priority(cmd_buffer, sample_locs[0], num_samples); + radeon_begin(cs); + /* Emit the specified user sample locations. */ switch (num_samples) { case 2: case 4: - radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_pixel[0][0]); - radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_pixel[1][0]); - radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_pixel[2][0]); - radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_pixel[3][0]); + radeon_set_context_reg(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_pixel[0][0]); + radeon_set_context_reg(R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_pixel[1][0]); + radeon_set_context_reg(R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_pixel[2][0]); + radeon_set_context_reg(R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_pixel[3][0]); break; case 8: - radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_pixel[0][0]); - radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_pixel[1][0]); - radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_pixel[2][0]); - radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_pixel[3][0]); - radeon_set_context_reg(cs, R_028BFC_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1, sample_locs_pixel[0][1]); - radeon_set_context_reg(cs, R_028C0C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1, sample_locs_pixel[1][1]); - radeon_set_context_reg(cs, R_028C1C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1, sample_locs_pixel[2][1]); - radeon_set_context_reg(cs, R_028C2C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1, sample_locs_pixel[3][1]); + radeon_set_context_reg(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_pixel[0][0]); + radeon_set_context_reg(R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_pixel[1][0]); + radeon_set_context_reg(R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_pixel[2][0]); + radeon_set_context_reg(R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_pixel[3][0]); + radeon_set_context_reg(R_028BFC_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1, sample_locs_pixel[0][1]); + radeon_set_context_reg(R_028C0C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1, sample_locs_pixel[1][1]); + radeon_set_context_reg(R_028C1C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1, sample_locs_pixel[2][1]); + radeon_set_context_reg(R_028C2C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1, sample_locs_pixel[3][1]); break; default: unreachable("invalid number of samples"); } if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg_seq(cs, R_028BF0_PA_SC_CENTROID_PRIORITY_0, 2); + radeon_set_context_reg_seq(R_028BF0_PA_SC_CENTROID_PRIORITY_0, 2); } else { - radeon_set_context_reg_seq(cs, R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2); + radeon_set_context_reg_seq(R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2); } - radeon_emit(cs, centroid_priority); - radeon_emit(cs, centroid_priority >> 32); + radeon_emit(centroid_priority); + radeon_emit(centroid_priority >> 32); if (pdev->info.gfx_level >= GFX7) { /* The exclusion bits can be set to improve rasterization efficiency if no sample lies on the pixel boundary @@ -1257,8 +1269,10 @@ radv_emit_sample_locations(struct radv_cmd_buffer *cmd_buffer) } } - radeon_set_context_reg(cs, R_02882C_PA_SU_PRIM_FILTER_CNTL, pa_su_prim_filter_cntl); + radeon_set_context_reg(R_02882C_PA_SU_PRIM_FILTER_CNTL, pa_su_prim_filter_cntl); } + + radeon_end(); } static void @@ -1272,9 +1286,10 @@ radv_emit_inline_push_consts(const struct radv_device *device, struct radeon_cmd return; radeon_check_space(device->ws, cs, 2 + loc->num_sgprs); - - radeon_set_sh_reg_seq(cs, base_reg + loc->sgpr_idx * 4, loc->num_sgprs); - radeon_emit_array(cs, values, loc->num_sgprs); + radeon_begin(cs); + radeon_set_sh_reg_seq(base_reg + loc->sgpr_idx * 4, loc->num_sgprs); + radeon_emit_array(values, loc->num_sgprs); + radeon_end(); } struct radv_bin_size_entry { @@ -1731,8 +1746,10 @@ radv_emit_binning_state(struct radv_cmd_buffer *cmd_buffer) pa_sc_binner_cntl_0 = radv_get_binning_state(cmd_buffer); + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028C44_PA_SC_BINNER_CNTL_0, RADV_TRACKED_PA_SC_BINNER_CNTL_0, pa_sc_binner_cntl_0); + radeon_end(); } static void @@ -1926,8 +1943,10 @@ radv_emit_rbplus_state(struct radv_cmd_buffer *cmd_buffer) * breaks dual source blending in SkQP and does not seem to improve * performance. */ + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg3(cmd_buffer, R_028754_SX_PS_DOWNCONVERT, RADV_TRACKED_SX_PS_DOWNCONVERT, sx_ps_downconvert, sx_blend_opt_epsilon, sx_blend_opt_control); + radeon_end(); cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_RBPLUS; } @@ -1946,7 +1965,10 @@ radv_emit_ps_epilog_state(struct radv_cmd_buffer *cmd_buffer, struct radv_shader if (G_00B848_VGPRS(ps_epilog->rsrc1) > G_00B848_VGPRS(ps_shader->config.rsrc1)) { uint32_t rsrc1 = ps_shader->config.rsrc1; rsrc1 = (rsrc1 & C_00B848_VGPRS) | (ps_epilog->rsrc1 & ~C_00B848_VGPRS); - radeon_set_sh_reg(cmd_buffer->cs, ps_shader->info.regs.pgm_rsrc1, rsrc1); + + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(ps_shader->info.regs.pgm_rsrc1, rsrc1); + radeon_end(); } radv_cs_add_buffer(device->ws, cmd_buffer->cs, ps_epilog->bo); @@ -1954,7 +1976,9 @@ radv_emit_ps_epilog_state(struct radv_cmd_buffer *cmd_buffer, struct radv_shader assert((ps_epilog->va >> 32) == pdev->info.address32_hi); const uint32_t epilog_pc_offset = radv_get_user_sgpr_loc(ps_shader, AC_UD_EPILOG_PC); - radeon_emit_32bit_pointer(cmd_buffer->cs, epilog_pc_offset, ps_epilog->va, &pdev->info); + radeon_begin(cmd_buffer->cs); + radeon_emit_32bit_pointer(epilog_pc_offset, ps_epilog->va, &pdev->info); + radeon_end(); cmd_buffer->shader_upload_seq = MAX2(cmd_buffer->shader_upload_seq, ps_epilog->upload_seq); @@ -1967,20 +1991,22 @@ radv_emit_compute_shader(const struct radv_physical_device *pdev, struct radeon_ { uint64_t va = radv_shader_get_va(shader); - radeon_set_sh_reg(cs, shader->info.regs.pgm_lo, va >> 8); + radeon_begin(cs); + radeon_set_sh_reg(shader->info.regs.pgm_lo, va >> 8); - radeon_set_sh_reg_seq(cs, shader->info.regs.pgm_rsrc1, 2); - radeon_emit(cs, shader->config.rsrc1); - radeon_emit(cs, shader->config.rsrc2); + radeon_set_sh_reg_seq(shader->info.regs.pgm_rsrc1, 2); + radeon_emit(shader->config.rsrc1); + radeon_emit(shader->config.rsrc2); if (pdev->info.gfx_level >= GFX10) { - radeon_set_sh_reg(cs, shader->info.regs.pgm_rsrc3, shader->config.rsrc3); + radeon_set_sh_reg(shader->info.regs.pgm_rsrc3, shader->config.rsrc3); } - radeon_set_sh_reg(cs, R_00B854_COMPUTE_RESOURCE_LIMITS, shader->info.regs.cs.compute_resource_limits); - radeon_set_sh_reg_seq(cs, R_00B81C_COMPUTE_NUM_THREAD_X, 3); - radeon_emit(cs, shader->info.regs.cs.compute_num_thread_x); - radeon_emit(cs, shader->info.regs.cs.compute_num_thread_y); - radeon_emit(cs, shader->info.regs.cs.compute_num_thread_z); + radeon_set_sh_reg(R_00B854_COMPUTE_RESOURCE_LIMITS, shader->info.regs.cs.compute_resource_limits); + radeon_set_sh_reg_seq(R_00B81C_COMPUTE_NUM_THREAD_X, 3); + radeon_emit(shader->info.regs.cs.compute_num_thread_x); + radeon_emit(shader->info.regs.cs.compute_num_thread_y); + radeon_emit(shader->info.regs.cs.compute_num_thread_z); + radeon_end(); } static void @@ -2002,9 +2028,11 @@ radv_emit_vgt_gs_mode(struct radv_cmd_buffer *cmd_buffer) vgt_primitiveid_en |= S_028A84_PRIMITIVEID_EN(1); } + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028A84_VGT_PRIMITIVEID_EN, RADV_TRACKED_VGT_PRIMITIVEID_EN, vgt_primitiveid_en); radeon_opt_set_context_reg(cmd_buffer, R_028A40_VGT_GS_MODE, RADV_TRACKED_VGT_GS_MODE, vgt_gs_mode); + radeon_end(); } static void @@ -2014,11 +2042,12 @@ radv_emit_hw_vs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *sh const struct radv_physical_device *pdev = radv_device_physical(device); const uint64_t va = radv_shader_get_va(shader); - radeon_set_sh_reg_seq(cmd_buffer->cs, shader->info.regs.pgm_lo, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, S_00B124_MEM_BASE(va >> 40)); - radeon_emit(cmd_buffer->cs, shader->config.rsrc1); - radeon_emit(cmd_buffer->cs, shader->config.rsrc2); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg_seq(shader->info.regs.pgm_lo, 4); + radeon_emit(va >> 8); + radeon_emit(S_00B124_MEM_BASE(va >> 40)); + radeon_emit(shader->config.rsrc1); + radeon_emit(shader->config.rsrc2); radeon_opt_set_context_reg(cmd_buffer, R_0286C4_SPI_VS_OUT_CONFIG, RADV_TRACKED_SPI_VS_OUT_CONFIG, shader->info.regs.spi_vs_out_config); @@ -2032,13 +2061,12 @@ radv_emit_hw_vs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *sh shader->info.regs.vs.vgt_reuse_off); if (pdev->info.gfx_level >= GFX7) { - radeon_set_sh_reg_idx(&pdev->info, cmd_buffer->cs, R_00B118_SPI_SHADER_PGM_RSRC3_VS, 3, + radeon_set_sh_reg_idx(&pdev->info, R_00B118_SPI_SHADER_PGM_RSRC3_VS, 3, shader->info.regs.vs.spi_shader_pgm_rsrc3_vs); - radeon_set_sh_reg(cmd_buffer->cs, R_00B11C_SPI_SHADER_LATE_ALLOC_VS, - shader->info.regs.vs.spi_shader_late_alloc_vs); + radeon_set_sh_reg(R_00B11C_SPI_SHADER_LATE_ALLOC_VS, shader->info.regs.vs.spi_shader_late_alloc_vs); if (pdev->info.gfx_level >= GFX10) { - radeon_set_uconfig_reg(cmd_buffer->cs, R_030980_GE_PC_ALLOC, shader->info.regs.ge_pc_alloc); + radeon_set_uconfig_reg(R_030980_GE_PC_ALLOC, shader->info.regs.ge_pc_alloc); if (shader->info.stage == MESA_SHADER_TESS_EVAL) { radeon_opt_set_context_reg(cmd_buffer, R_028A44_VGT_GS_ONCHIP_CNTL, RADV_TRACKED_VGT_GS_ONCHIP_CNTL, @@ -2046,6 +2074,8 @@ radv_emit_hw_vs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *sh } } } + + radeon_end(); } static void @@ -2053,11 +2083,13 @@ radv_emit_hw_es(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *sh { const uint64_t va = radv_shader_get_va(shader); - radeon_set_sh_reg_seq(cmd_buffer->cs, shader->info.regs.pgm_lo, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, S_00B324_MEM_BASE(va >> 40)); - radeon_emit(cmd_buffer->cs, shader->config.rsrc1); - radeon_emit(cmd_buffer->cs, shader->config.rsrc2); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg_seq(shader->info.regs.pgm_lo, 4); + radeon_emit(va >> 8); + radeon_emit(S_00B324_MEM_BASE(va >> 40)); + radeon_emit(shader->config.rsrc1); + radeon_emit(shader->config.rsrc2); + radeon_end(); } static void @@ -2065,9 +2097,10 @@ radv_emit_hw_ls(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *sh { const uint64_t va = radv_shader_get_va(shader); - radeon_set_sh_reg(cmd_buffer->cs, shader->info.regs.pgm_lo, va >> 8); - - radeon_set_sh_reg(cmd_buffer->cs, shader->info.regs.pgm_rsrc1, shader->config.rsrc1); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(shader->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(shader->info.regs.pgm_rsrc1, shader->config.rsrc1); + radeon_end(); } static void @@ -2089,12 +2122,14 @@ radv_emit_hw_ngg(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *e es_type = shader->info.stage; } - if (!shader->info.merged_shader_compiled_separately) { - radeon_set_sh_reg(cmd_buffer->cs, shader->info.regs.pgm_lo, va >> 8); + radeon_begin(cmd_buffer->cs); - radeon_set_sh_reg_seq(cmd_buffer->cs, shader->info.regs.pgm_rsrc1, 2); - radeon_emit(cmd_buffer->cs, shader->config.rsrc1); - radeon_emit(cmd_buffer->cs, shader->config.rsrc2); + if (!shader->info.merged_shader_compiled_separately) { + radeon_set_sh_reg(shader->info.regs.pgm_lo, va >> 8); + + radeon_set_sh_reg_seq(shader->info.regs.pgm_rsrc1, 2); + radeon_emit(shader->config.rsrc1); + radeon_emit(shader->config.rsrc2); } const struct radv_vs_output_info *outinfo = &shader->info.outinfo; @@ -2114,7 +2149,7 @@ radv_emit_hw_ngg(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *e radeon_opt_set_context_reg(cmd_buffer, R_028B3C_VGT_GS_INSTANCE_CNT, RADV_TRACKED_VGT_GS_INSTANCE_CNT, shader->info.regs.vgt_gs_instance_cnt); - radeon_set_uconfig_reg(cmd_buffer->cs, R_030988_VGT_PRIMITIVEID_EN, shader->info.regs.ngg.vgt_primitiveid_en); + radeon_set_uconfig_reg(R_030988_VGT_PRIMITIVEID_EN, shader->info.regs.ngg.vgt_primitiveid_en); radeon_opt_set_context_reg2(cmd_buffer, R_028648_SPI_SHADER_IDX_FORMAT, RADV_TRACKED_SPI_SHADER_IDX_FORMAT, shader->info.regs.ngg.spi_shader_idx_format, shader->info.regs.spi_shader_pos_format); @@ -2164,21 +2199,23 @@ radv_emit_hw_ngg(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *e shader->info.regs.vgt_gs_onchip_cntl); } - radeon_set_uconfig_reg(cmd_buffer->cs, R_03096C_GE_CNTL, ge_cntl); + radeon_set_uconfig_reg(R_03096C_GE_CNTL, ge_cntl); if (pdev->info.gfx_level >= GFX12) { - radeon_set_sh_reg(cmd_buffer->cs, R_00B220_SPI_SHADER_PGM_RSRC4_GS, shader->info.regs.spi_shader_pgm_rsrc4_gs); + radeon_set_sh_reg(R_00B220_SPI_SHADER_PGM_RSRC4_GS, shader->info.regs.spi_shader_pgm_rsrc4_gs); } else { if (pdev->info.gfx_level >= GFX7) { - radeon_set_sh_reg_idx(&pdev->info, cmd_buffer->cs, R_00B21C_SPI_SHADER_PGM_RSRC3_GS, 3, + radeon_set_sh_reg_idx(&pdev->info, R_00B21C_SPI_SHADER_PGM_RSRC3_GS, 3, shader->info.regs.spi_shader_pgm_rsrc3_gs); } - radeon_set_sh_reg_idx(&pdev->info, cmd_buffer->cs, R_00B204_SPI_SHADER_PGM_RSRC4_GS, 3, + radeon_set_sh_reg_idx(&pdev->info, R_00B204_SPI_SHADER_PGM_RSRC4_GS, 3, shader->info.regs.spi_shader_pgm_rsrc4_gs); - radeon_set_uconfig_reg(cmd_buffer->cs, R_030980_GE_PC_ALLOC, shader->info.regs.ge_pc_alloc); + radeon_set_uconfig_reg(R_030980_GE_PC_ALLOC, shader->info.regs.ge_pc_alloc); } + + radeon_end(); } static void @@ -2188,16 +2225,18 @@ radv_emit_hw_hs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *sh const struct radv_physical_device *pdev = radv_device_physical(device); const uint64_t va = radv_shader_get_va(shader); + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX9) { - radeon_set_sh_reg(cmd_buffer->cs, shader->info.regs.pgm_lo, va >> 8); - radeon_set_sh_reg(cmd_buffer->cs, shader->info.regs.pgm_rsrc1, shader->config.rsrc1); + radeon_set_sh_reg(shader->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(shader->info.regs.pgm_rsrc1, shader->config.rsrc1); } else { - radeon_set_sh_reg_seq(cmd_buffer->cs, shader->info.regs.pgm_lo, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, S_00B424_MEM_BASE(va >> 40)); - radeon_emit(cmd_buffer->cs, shader->config.rsrc1); - radeon_emit(cmd_buffer->cs, shader->config.rsrc2); + radeon_set_sh_reg_seq(shader->info.regs.pgm_lo, 4); + radeon_emit(va >> 8); + radeon_emit(S_00B424_MEM_BASE(va >> 40)); + radeon_emit(shader->config.rsrc1); + radeon_emit(shader->config.rsrc2); } + radeon_end(); } static void @@ -2215,12 +2254,14 @@ radv_emit_vertex_shader(struct radv_cmd_buffer *cmd_buffer) if (!vs->info.vs.has_prolog) { uint32_t rsrc1, rsrc2; - radeon_set_sh_reg(cmd_buffer->cs, vs->info.regs.pgm_lo, vs->va >> 8); + radeon_begin(cmd_buffer->cs); + + radeon_set_sh_reg(vs->info.regs.pgm_lo, vs->va >> 8); if (vs->info.next_stage == MESA_SHADER_TESS_CTRL) { radv_shader_combine_cfg_vs_tcs(vs, next_stage, &rsrc1, NULL); - radeon_set_sh_reg(cmd_buffer->cs, vs->info.regs.pgm_rsrc1, rsrc1); + radeon_set_sh_reg(vs->info.regs.pgm_rsrc1, rsrc1); } else { radv_shader_combine_cfg_vs_gs(vs, next_stage, &rsrc1, &rsrc2); @@ -2231,14 +2272,18 @@ radv_emit_vertex_shader(struct radv_cmd_buffer *cmd_buffer) lds_size = next_stage->info.gs_ring_info.lds_size; } - radeon_set_sh_reg_seq(cmd_buffer->cs, vs->info.regs.pgm_rsrc1, 2); - radeon_emit(cmd_buffer->cs, rsrc1); - radeon_emit(cmd_buffer->cs, rsrc2 | S_00B22C_LDS_SIZE(lds_size)); + radeon_set_sh_reg_seq(vs->info.regs.pgm_rsrc1, 2); + radeon_emit(rsrc1); + radeon_emit(rsrc2 | S_00B22C_LDS_SIZE(lds_size)); } + + radeon_end(); } const uint32_t next_stage_pc_offset = radv_get_user_sgpr_loc(vs, AC_UD_NEXT_STAGE_PC); - radeon_emit_32bit_pointer(cmd_buffer->cs, next_stage_pc_offset, next_stage->va, &pdev->info); + radeon_begin(cmd_buffer->cs); + radeon_emit_32bit_pointer(next_stage_pc_offset, next_stage->va, &pdev->info); + radeon_end(); return; } @@ -2289,14 +2334,16 @@ radv_emit_tess_eval_shader(struct radv_cmd_buffer *cmd_buffer) lds_size = gs->info.gs_ring_info.lds_size; } - radeon_set_sh_reg(cmd_buffer->cs, tes->info.regs.pgm_lo, tes->va >> 8); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(tes->info.regs.pgm_lo, tes->va >> 8); - radeon_set_sh_reg_seq(cmd_buffer->cs, tes->info.regs.pgm_rsrc1, 2); - radeon_emit(cmd_buffer->cs, rsrc1); - radeon_emit(cmd_buffer->cs, rsrc2 | S_00B22C_LDS_SIZE(lds_size)); + radeon_set_sh_reg_seq(tes->info.regs.pgm_rsrc1, 2); + radeon_emit(rsrc1); + radeon_emit(rsrc2 | S_00B22C_LDS_SIZE(lds_size)); const uint32_t next_stage_pc_offset = radv_get_user_sgpr_loc(tes, AC_UD_NEXT_STAGE_PC); - radeon_emit_32bit_pointer(cmd_buffer->cs, next_stage_pc_offset, gs->va, &pdev->info); + radeon_emit_32bit_pointer(next_stage_pc_offset, gs->va, &pdev->info); + radeon_end(); return; } @@ -2317,6 +2364,8 @@ radv_emit_hw_gs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *gs const struct radv_legacy_gs_info *gs_state = &gs->info.gs_ring_info; const uint64_t va = radv_shader_get_va(gs); + radeon_begin(cmd_buffer->cs); + radeon_opt_set_context_reg3(cmd_buffer, R_028A60_VGT_GSVS_RING_OFFSET_1, RADV_TRACKED_VGT_GSVS_RING_OFFSET_1, gs->info.regs.gs.vgt_gsvs_ring_offset[0], gs->info.regs.gs.vgt_gsvs_ring_offset[1], gs->info.regs.gs.vgt_gsvs_ring_offset[2]); @@ -2333,11 +2382,11 @@ radv_emit_hw_gs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *gs if (pdev->info.gfx_level >= GFX9) { if (!gs->info.merged_shader_compiled_separately) { - radeon_set_sh_reg(cmd_buffer->cs, gs->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(gs->info.regs.pgm_lo, va >> 8); - radeon_set_sh_reg_seq(cmd_buffer->cs, gs->info.regs.pgm_rsrc1, 2); - radeon_emit(cmd_buffer->cs, gs->config.rsrc1); - radeon_emit(cmd_buffer->cs, gs->config.rsrc2 | S_00B22C_LDS_SIZE(gs_state->lds_size)); + radeon_set_sh_reg_seq(gs->info.regs.pgm_rsrc1, 2); + radeon_emit(gs->config.rsrc1); + radeon_emit(gs->config.rsrc2 | S_00B22C_LDS_SIZE(gs_state->lds_size)); } radeon_opt_set_context_reg(cmd_buffer, R_028A44_VGT_GS_ONCHIP_CNTL, RADV_TRACKED_VGT_GS_ONCHIP_CNTL, @@ -2349,11 +2398,11 @@ radv_emit_hw_gs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *gs gs->info.regs.gs.vgt_gs_max_prims_per_subgroup); } } else { - radeon_set_sh_reg_seq(cmd_buffer->cs, gs->info.regs.pgm_lo, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, S_00B224_MEM_BASE(va >> 40)); - radeon_emit(cmd_buffer->cs, gs->config.rsrc1); - radeon_emit(cmd_buffer->cs, gs->config.rsrc2); + radeon_set_sh_reg_seq(gs->info.regs.pgm_lo, 4); + radeon_emit(va >> 8); + radeon_emit(S_00B224_MEM_BASE(va >> 40)); + radeon_emit(gs->config.rsrc1); + radeon_emit(gs->config.rsrc2); /* GFX6-8: ESGS offchip ring buffer is allocated according to VGT_ESGS_RING_ITEMSIZE. * GFX9+: Only used to set the GS input VGPRs, emulated in shaders. @@ -2363,14 +2412,14 @@ radv_emit_hw_gs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *gs } if (pdev->info.gfx_level >= GFX7) { - radeon_set_sh_reg_idx(&pdev->info, cmd_buffer->cs, R_00B21C_SPI_SHADER_PGM_RSRC3_GS, 3, - gs->info.regs.spi_shader_pgm_rsrc3_gs); + radeon_set_sh_reg_idx(&pdev->info, R_00B21C_SPI_SHADER_PGM_RSRC3_GS, 3, gs->info.regs.spi_shader_pgm_rsrc3_gs); } if (pdev->info.gfx_level >= GFX10) { - radeon_set_sh_reg_idx(&pdev->info, cmd_buffer->cs, R_00B204_SPI_SHADER_PGM_RSRC4_GS, 3, - gs->info.regs.spi_shader_pgm_rsrc4_gs); + radeon_set_sh_reg_idx(&pdev->info, R_00B204_SPI_SHADER_PGM_RSRC4_GS, 3, gs->info.regs.spi_shader_pgm_rsrc4_gs); } + + radeon_end(); } static void @@ -2387,6 +2436,8 @@ radv_emit_geometry_shader(struct radv_cmd_buffer *cmd_buffer) radv_emit_hw_vs(cmd_buffer, cmd_buffer->state.gs_copy_shader); } + radeon_begin(cmd_buffer->cs); + radeon_opt_set_context_reg(cmd_buffer, R_028B38_VGT_GS_MAX_VERT_OUT, RADV_TRACKED_VGT_GS_MAX_VERT_OUT, gs->info.regs.vgt_gs_max_vert_out); @@ -2395,7 +2446,7 @@ radv_emit_geometry_shader(struct radv_cmd_buffer *cmd_buffer) assert(vgt_esgs_ring_itemsize_offset); - radeon_set_sh_reg(cmd_buffer->cs, vgt_esgs_ring_itemsize_offset, es->info.esgs_itemsize / 4); + radeon_set_sh_reg(vgt_esgs_ring_itemsize_offset, es->info.esgs_itemsize / 4); if (gs->info.is_ngg) { const uint32_t ngg_lds_layout_offset = radv_get_user_sgpr_loc(gs, AC_UD_NGG_LDS_LAYOUT); @@ -2403,11 +2454,13 @@ radv_emit_geometry_shader(struct radv_cmd_buffer *cmd_buffer) assert(ngg_lds_layout_offset); assert(!(gs->info.ngg_info.esgs_ring_size & 0xffff0000) && !(gs->info.ngg_info.scratch_lds_base & 0xffff0000)); - radeon_set_sh_reg(cmd_buffer->cs, ngg_lds_layout_offset, + radeon_set_sh_reg(ngg_lds_layout_offset, SET_SGPR_FIELD(NGG_LDS_LAYOUT_GS_OUT_VERTEX_BASE, gs->info.ngg_info.esgs_ring_size) | SET_SGPR_FIELD(NGG_LDS_LAYOUT_SCRATCH_BASE, gs->info.ngg_info.scratch_lds_base)); } } + + radeon_end(); } static void @@ -2416,12 +2469,14 @@ radv_emit_vgt_gs_out(struct radv_cmd_buffer *cmd_buffer, uint32_t vgt_gs_out_pri const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_physical_device *pdev = radv_device_physical(device); + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX11) { - radeon_set_uconfig_reg(cmd_buffer->cs, R_030998_VGT_GS_OUT_PRIM_TYPE, vgt_gs_out_prim_type); + radeon_set_uconfig_reg(R_030998_VGT_GS_OUT_PRIM_TYPE, vgt_gs_out_prim_type); } else { radeon_opt_set_context_reg(cmd_buffer, R_028A6C_VGT_GS_OUT_PRIM_TYPE, RADV_TRACKED_VGT_GS_OUT_PRIM_TYPE, vgt_gs_out_prim_type); } + radeon_end(); } static void @@ -2433,20 +2488,24 @@ radv_emit_mesh_shader(struct radv_cmd_buffer *cmd_buffer) const uint32_t gs_out = radv_conv_gl_prim_to_gs_out(ms->info.ms.output_prim); radv_emit_hw_ngg(cmd_buffer, NULL, ms); + + radeon_begin(cmd_buffer->cs); + radeon_opt_set_context_reg(cmd_buffer, R_028B38_VGT_GS_MAX_VERT_OUT, RADV_TRACKED_VGT_GS_MAX_VERT_OUT, ms->info.regs.vgt_gs_max_vert_out); - radeon_set_uconfig_reg_idx(&pdev->info, cmd_buffer->cs, R_030908_VGT_PRIMITIVE_TYPE, 1, V_008958_DI_PT_POINTLIST); + radeon_set_uconfig_reg_idx(&pdev->info, R_030908_VGT_PRIMITIVE_TYPE, 1, V_008958_DI_PT_POINTLIST); if (pdev->mesh_fast_launch_2) { - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B2B0_SPI_SHADER_GS_MESHLET_DIM, 2); - radeon_emit(cmd_buffer->cs, ms->info.regs.ms.spi_shader_gs_meshlet_dim); - radeon_emit(cmd_buffer->cs, ms->info.regs.ms.spi_shader_gs_meshlet_exp_alloc); + radeon_set_sh_reg_seq(R_00B2B0_SPI_SHADER_GS_MESHLET_DIM, 2); + radeon_emit(ms->info.regs.ms.spi_shader_gs_meshlet_dim); + radeon_emit(ms->info.regs.ms.spi_shader_gs_meshlet_exp_alloc); if (pdev->info.gfx_level >= GFX12) - radeon_set_sh_reg(cmd_buffer->cs, R_00B2B8_SPI_SHADER_GS_MESHLET_CTRL, - ms->info.regs.ms.spi_shader_gs_meshlet_ctrl); + radeon_set_sh_reg(R_00B2B8_SPI_SHADER_GS_MESHLET_CTRL, ms->info.regs.ms.spi_shader_gs_meshlet_ctrl); } + radeon_end(); + radv_emit_vgt_gs_out(cmd_buffer, gs_out); } @@ -2576,6 +2635,8 @@ radv_emit_ps_inputs(struct radv_cmd_buffer *cmd_buffer) /* Only GFX10.3+ support per-primitive params */ assert(pdev->info.gfx_level >= GFX10_3 || num_per_primitive_params == 0); + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { radeon_opt_set_context_regn(cmd_buffer, R_028664_SPI_PS_INPUT_CNTL_0, ps_input_cntl, cmd_buffer->tracked_regs.spi_ps_input_cntl, ps_offset); @@ -2594,6 +2655,8 @@ radv_emit_ps_inputs(struct radv_cmd_buffer *cmd_buffer) radeon_opt_set_context_regn(cmd_buffer, R_028644_SPI_PS_INPUT_CNTL_0, ps_input_cntl, cmd_buffer->tracked_regs.spi_ps_input_cntl, ps_offset); } + + radeon_end(); } static void @@ -2605,6 +2668,8 @@ radv_emit_fragment_shader_state(struct radv_cmd_buffer *cmd_buffer, const struct const uint32_t spi_ps_input_addr = ps ? ps->config.spi_ps_input_addr : 0; const uint32_t spi_ps_in_control = ps ? ps->info.regs.ps.spi_ps_in_control : 0; + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { const uint32_t pa_sc_hisz_control = ps ? ps->info.regs.ps.pa_sc_hisz_control : 0; @@ -2631,6 +2696,8 @@ radv_emit_fragment_shader_state(struct radv_cmd_buffer *cmd_buffer, const struct radeon_opt_set_context_reg(cmd_buffer, R_028C40_PA_SC_SHADER_CONTROL, RADV_TRACKED_PA_SC_SHADER_CONTROL, pa_sc_shader_control); } + + radeon_end(); } static void @@ -2639,11 +2706,13 @@ radv_emit_fragment_shader(struct radv_cmd_buffer *cmd_buffer) const struct radv_shader *ps = cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT]; const uint64_t va = radv_shader_get_va(ps); - radeon_set_sh_reg_seq(cmd_buffer->cs, ps->info.regs.pgm_lo, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, S_00B024_MEM_BASE(va >> 40)); - radeon_emit(cmd_buffer->cs, ps->config.rsrc1); - radeon_emit(cmd_buffer->cs, ps->config.rsrc2); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg_seq(ps->info.regs.pgm_lo, 4); + radeon_emit(va >> 8); + radeon_emit(S_00B024_MEM_BASE(va >> 40)); + radeon_emit(ps->config.rsrc1); + radeon_emit(ps->config.rsrc2); + radeon_end(); radv_emit_fragment_shader_state(cmd_buffer, ps); } @@ -2659,8 +2728,10 @@ radv_emit_vgt_reuse(struct radv_cmd_buffer *cmd_buffer, const struct radv_vgt_sh /* Legacy Tess+GS should disable reuse to prevent hangs on GFX10.3. */ const bool has_legacy_tess_gs = key->tess && key->gs && !key->ngg; + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028AB4_VGT_REUSE_OFF, RADV_TRACKED_VGT_REUSE_OFF, S_028AB4_REUSE_OFF(has_legacy_tess_gs)); + radeon_end(); } if (pdev->info.family >= CHIP_POLARIS10 && pdev->info.gfx_level < GFX10) { @@ -2668,8 +2739,11 @@ radv_emit_vgt_reuse(struct radv_cmd_buffer *cmd_buffer, const struct radv_vgt_sh if (tes && tes->info.tes.spacing == TESS_SPACING_FRACTIONAL_ODD) { vtx_reuse_depth = 14; } + + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028C58_VGT_VERTEX_REUSE_BLOCK_CNTL, RADV_TRACKED_VGT_VERTEX_REUSE_BLOCK_CNTL, S_028C58_VTX_REUSE_DEPTH(vtx_reuse_depth)); + radeon_end(); } } @@ -2685,7 +2759,9 @@ radv_emit_vgt_shader_config_gfx12(struct radv_cmd_buffer *cmd_buffer, const stru if (key->tess) stages |= S_028A98_HS_EN(1) | S_028A98_HS_W32_EN(key->hs_wave32); + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028A98_VGT_SHADER_STAGES_EN, RADV_TRACKED_VGT_SHADER_STAGES_EN, stages); + radeon_end(); } static void @@ -2734,7 +2810,9 @@ radv_emit_vgt_shader_config_gfx6(struct radv_cmd_buffer *cmd_buffer, const struc assert(!(key->gs && !key->ngg) || !key->gs_wave32); } + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028B54_VGT_SHADER_STAGES_EN, RADV_TRACKED_VGT_SHADER_STAGES_EN, stages); + radeon_end(); } static void @@ -2772,6 +2850,8 @@ gfx103_emit_vgt_draw_payload_cntl(struct radv_cmd_buffer *cmd_buffer) const uint32_t vgt_draw_payload_cntl = S_028A98_EN_VRS_RATE(enable_vrs) | S_028A98_EN_PRIM_PAYLOAD(enable_prim_payload); + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { radeon_opt_set_context_reg(cmd_buffer, R_028AA0_VGT_DRAW_PAYLOAD_CNTL, RADV_TRACKED_VGT_DRAW_PAYLOAD_CNTL, vgt_draw_payload_cntl); @@ -2779,6 +2859,8 @@ gfx103_emit_vgt_draw_payload_cntl(struct radv_cmd_buffer *cmd_buffer) radeon_opt_set_context_reg(cmd_buffer, R_028A98_VGT_DRAW_PAYLOAD_CNTL, RADV_TRACKED_VGT_DRAW_PAYLOAD_CNTL, vgt_draw_payload_cntl); } + + radeon_end(); } static void @@ -2803,9 +2885,11 @@ gfx103_emit_vrs_state(struct radv_cmd_buffer *cmd_buffer) * requested by the user. Note that vkd3d-proton always has to declare VRS as dynamic because * in DX12 it's fully dynamic. */ + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028848_PA_CL_VRS_CNTL, RADV_TRACKED_PA_CL_VRS_CNTL, S_028848_SAMPLE_ITER_COMBINER_MODE(V_028848_SC_VRS_COMB_MODE_OVERRIDE) | S_028848_VERTEX_RATE_COMBINER_MODE(V_028848_SC_VRS_COMB_MODE_OVERRIDE)); + radeon_end(); /* If the shader is using discard, turn off coarse shading because discard at 2x2 pixel * granularity degrades quality too much. MIN allows sample shading but not coarse shading. @@ -2814,9 +2898,11 @@ gfx103_emit_vrs_state(struct radv_cmd_buffer *cmd_buffer) } if (pdev->info.gfx_level < GFX11) { + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028064_DB_VRS_OVERRIDE_CNTL, RADV_TRACKED_DB_VRS_OVERRIDE_CNTL, S_028064_VRS_OVERRIDE_RATE_COMBINER_MODE(mode) | S_028064_VRS_OVERRIDE_RATE_X(rate_x) | S_028064_VRS_OVERRIDE_RATE_Y(rate_y)); + radeon_end(); } } @@ -2868,7 +2954,9 @@ radv_emit_graphics_shaders(struct radv_cmd_buffer *cmd_buffer) radv_emit_fragment_shader_state(cmd_buffer, NULL); } - radeon_set_sh_reg(cmd_buffer->cs, R_00B0C4_SPI_SHADER_GS_OUT_CONFIG_PS, gs_out_config_ps); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(R_00B0C4_SPI_SHADER_GS_OUT_CONFIG_PS, gs_out_config_ps); + radeon_end(); } const struct radv_vgt_shader_key vgt_shader_cfg_key = @@ -2934,7 +3022,9 @@ radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer) cmd_buffer->state.graphics_pipeline->base.shaders[MESA_SHADER_FRAGMENT]) && (settings->context_states_per_bin > 1 || settings->persistent_states_per_bin > 1)) { /* Break the batch on PS changes. */ - radeon_event_write(cmd_buffer->cs, V_028A90_BREAK_BATCH); + radeon_begin(cmd_buffer->cs); + radeon_event_write(V_028A90_BREAK_BATCH); + radeon_end(); } } @@ -2947,7 +3037,9 @@ radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer) const struct radv_sqtt_shaders_reloc *reloc = pipeline->sqtt_shaders_reloc; const uint64_t va = reloc->va[MESA_SHADER_TASK]; - radeon_set_sh_reg(cmd_buffer->gang.cs, task_shader->info.regs.pgm_lo, va >> 8); + radeon_begin(cmd_buffer->gang.cs); + radeon_set_sh_reg(task_shader->info.regs.pgm_lo, va >> 8); + radeon_end(); } } @@ -3040,8 +3132,10 @@ radv_emit_viewport(struct radv_cmd_buffer *cmd_buffer) assert(d->vk.vp.viewport_count); + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_02843C_PA_CL_VPORT_XSCALE, d->vk.vp.viewport_count * 8); + radeon_set_context_reg_seq(R_02843C_PA_CL_VPORT_XSCALE, d->vk.vp.viewport_count * 8); for (unsigned i = 0; i < d->vk.vp.viewport_count; i++) { float zscale, ztranslate, zmin, zmax; @@ -3049,41 +3143,43 @@ radv_emit_viewport(struct radv_cmd_buffer *cmd_buffer) radv_get_viewport_zscale_ztranslate(cmd_buffer, i, &zscale, &ztranslate); radv_get_viewport_zmin_zmax(cmd_buffer, &d->vk.vp.viewports[i], &zmin, &zmax); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].scale[0])); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].translate[0])); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].scale[1])); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].translate[1])); - radeon_emit(cmd_buffer->cs, fui(zscale)); - radeon_emit(cmd_buffer->cs, fui(ztranslate)); - radeon_emit(cmd_buffer->cs, fui(zmin)); - radeon_emit(cmd_buffer->cs, fui(zmax)); + radeon_emit(fui(d->hw_vp.xform[i].scale[0])); + radeon_emit(fui(d->hw_vp.xform[i].translate[0])); + radeon_emit(fui(d->hw_vp.xform[i].scale[1])); + radeon_emit(fui(d->hw_vp.xform[i].translate[1])); + radeon_emit(fui(zscale)); + radeon_emit(fui(ztranslate)); + radeon_emit(fui(zmin)); + radeon_emit(fui(zmax)); } } else { - radeon_set_context_reg_seq(cmd_buffer->cs, R_02843C_PA_CL_VPORT_XSCALE, d->vk.vp.viewport_count * 6); + radeon_set_context_reg_seq(R_02843C_PA_CL_VPORT_XSCALE, d->vk.vp.viewport_count * 6); for (unsigned i = 0; i < d->vk.vp.viewport_count; i++) { float zscale, ztranslate; radv_get_viewport_zscale_ztranslate(cmd_buffer, i, &zscale, &ztranslate); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].scale[0])); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].translate[0])); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].scale[1])); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].translate[1])); - radeon_emit(cmd_buffer->cs, fui(zscale)); - radeon_emit(cmd_buffer->cs, fui(ztranslate)); + radeon_emit(fui(d->hw_vp.xform[i].scale[0])); + radeon_emit(fui(d->hw_vp.xform[i].translate[0])); + radeon_emit(fui(d->hw_vp.xform[i].scale[1])); + radeon_emit(fui(d->hw_vp.xform[i].translate[1])); + radeon_emit(fui(zscale)); + radeon_emit(fui(ztranslate)); } - radeon_set_context_reg_seq(cmd_buffer->cs, R_0282D0_PA_SC_VPORT_ZMIN_0, d->vk.vp.viewport_count * 2); + radeon_set_context_reg_seq(R_0282D0_PA_SC_VPORT_ZMIN_0, d->vk.vp.viewport_count * 2); for (unsigned i = 0; i < d->vk.vp.viewport_count; i++) { float zmin, zmax; radv_get_viewport_zmin_zmax(cmd_buffer, &d->vk.vp.viewports[i], &zmin, &zmax); - radeon_emit(cmd_buffer->cs, fui(zmin)); - radeon_emit(cmd_buffer->cs, fui(zmax)); + radeon_emit(fui(zmin)); + radeon_emit(fui(zmax)); } } + + radeon_end(); } static VkRect2D @@ -3121,7 +3217,9 @@ radv_emit_scissor(struct radv_cmd_buffer *cmd_buffer) if (!d->vk.vp.scissor_count) return; - radeon_set_context_reg_seq(cs, R_028250_PA_SC_VPORT_SCISSOR_0_TL, d->vk.vp.scissor_count * 2); + radeon_begin(cs); + radeon_set_context_reg_seq(R_028250_PA_SC_VPORT_SCISSOR_0_TL, d->vk.vp.scissor_count * 2); + for (unsigned i = 0; i < d->vk.vp.scissor_count; i++) { VkRect2D viewport_scissor = radv_scissor_from_viewport(d->hw_vp.xform[i].scale, d->hw_vp.xform[i].translate); VkRect2D scissor = radv_intersect_scissor(&d->vk.vp.scissors[i], &viewport_scissor); @@ -3137,13 +3235,15 @@ radv_emit_scissor(struct radv_cmd_buffer *cmd_buffer) minx = miny = maxx = maxy = 1; } - radeon_emit(cs, S_028250_TL_X(minx) | S_028250_TL_Y_GFX12(miny)); - radeon_emit(cs, S_028254_BR_X(maxx - 1) | S_028254_BR_Y(maxy - 1)); + radeon_emit(S_028250_TL_X(minx) | S_028250_TL_Y_GFX12(miny)); + radeon_emit(S_028254_BR_X(maxx - 1) | S_028254_BR_Y(maxy - 1)); } else { - radeon_emit(cs, S_028250_TL_X(minx) | S_028250_TL_Y_GFX6(miny) | S_028250_WINDOW_OFFSET_DISABLE(1)); - radeon_emit(cs, S_028254_BR_X(maxx) | S_028254_BR_Y(maxy)); + radeon_emit(S_028250_TL_X(minx) | S_028250_TL_Y_GFX6(miny) | S_028250_WINDOW_OFFSET_DISABLE(1)); + radeon_emit(S_028254_BR_X(maxx) | S_028254_BR_Y(maxy)); } } + + radeon_end(); } static void @@ -3154,6 +3254,8 @@ radv_emit_discard_rectangle(struct radv_cmd_buffer *cmd_buffer) const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; uint32_t cliprect_rule = 0; + radeon_begin(cmd_buffer->cs); + if (!d->vk.dr.enable) { cliprect_rule = 0xffff; } else { @@ -3174,27 +3276,27 @@ radv_emit_discard_rectangle(struct radv_cmd_buffer *cmd_buffer) cliprect_rule |= 1u << i; } - radeon_set_context_reg_seq(cmd_buffer->cs, R_028210_PA_SC_CLIPRECT_0_TL, d->vk.dr.rectangle_count * 2); + radeon_set_context_reg_seq(R_028210_PA_SC_CLIPRECT_0_TL, d->vk.dr.rectangle_count * 2); for (unsigned i = 0; i < d->vk.dr.rectangle_count; ++i) { VkRect2D rect = d->vk.dr.rectangles[i]; - radeon_emit(cmd_buffer->cs, S_028210_TL_X(rect.offset.x) | S_028210_TL_Y(rect.offset.y)); - radeon_emit(cmd_buffer->cs, S_028214_BR_X(rect.offset.x + rect.extent.width) | - S_028214_BR_Y(rect.offset.y + rect.extent.height)); + radeon_emit(S_028210_TL_X(rect.offset.x) | S_028210_TL_Y(rect.offset.y)); + radeon_emit(S_028214_BR_X(rect.offset.x + rect.extent.width) | + S_028214_BR_Y(rect.offset.y + rect.extent.height)); } if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028374_PA_SC_CLIPRECT_0_EXT, d->vk.dr.rectangle_count); + radeon_set_context_reg_seq(R_028374_PA_SC_CLIPRECT_0_EXT, d->vk.dr.rectangle_count); for (unsigned i = 0; i < d->vk.dr.rectangle_count; ++i) { VkRect2D rect = d->vk.dr.rectangles[i]; - radeon_emit(cmd_buffer->cs, S_028374_TL_X_EXT(rect.offset.x >> 15) | - S_028374_TL_Y_EXT(rect.offset.y >> 15) | - S_028374_BR_X_EXT((rect.offset.x + rect.extent.width) >> 15) | - S_028374_BR_Y_EXT((rect.offset.y + rect.extent.height) >> 15)); + radeon_emit(S_028374_TL_X_EXT(rect.offset.x >> 15) | S_028374_TL_Y_EXT(rect.offset.y >> 15) | + S_028374_BR_X_EXT((rect.offset.x + rect.extent.width) >> 15) | + S_028374_BR_Y_EXT((rect.offset.y + rect.extent.height) >> 15)); } } } - radeon_set_context_reg(cmd_buffer->cs, R_02820C_PA_SC_CLIPRECT_RULE, cliprect_rule); + radeon_set_context_reg(R_02820C_PA_SC_CLIPRECT_RULE, cliprect_rule); + radeon_end(); } static void @@ -3202,8 +3304,10 @@ radv_emit_blend_constants(struct radv_cmd_buffer *cmd_buffer) { const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; - radeon_set_context_reg_seq(cmd_buffer->cs, R_028414_CB_BLEND_RED, 4); - radeon_emit_array(cmd_buffer->cs, (uint32_t *)d->vk.cb.blend_constants, 4); + radeon_begin(cmd_buffer->cs); + radeon_set_context_reg_seq(R_028414_CB_BLEND_RED, 4); + radeon_emit_array((uint32_t *)d->vk.cb.blend_constants, 4); + radeon_end(); } static void @@ -3232,14 +3336,16 @@ radv_emit_depth_bias(struct radv_cmd_buffer *cmd_buffer) } } - radeon_set_context_reg_seq(cmd_buffer->cs, R_028B7C_PA_SU_POLY_OFFSET_CLAMP, 5); - radeon_emit(cmd_buffer->cs, fui(d->vk.rs.depth_bias.clamp)); /* CLAMP */ - radeon_emit(cmd_buffer->cs, slope); /* FRONT SCALE */ - radeon_emit(cmd_buffer->cs, fui(d->vk.rs.depth_bias.constant_factor)); /* FRONT OFFSET */ - radeon_emit(cmd_buffer->cs, slope); /* BACK SCALE */ - radeon_emit(cmd_buffer->cs, fui(d->vk.rs.depth_bias.constant_factor)); /* BACK OFFSET */ + radeon_begin(cmd_buffer->cs); + radeon_set_context_reg_seq(R_028B7C_PA_SU_POLY_OFFSET_CLAMP, 5); + radeon_emit(fui(d->vk.rs.depth_bias.clamp)); /* CLAMP */ + radeon_emit(slope); /* FRONT SCALE */ + radeon_emit(fui(d->vk.rs.depth_bias.constant_factor)); /* FRONT OFFSET */ + radeon_emit(slope); /* BACK SCALE */ + radeon_emit(fui(d->vk.rs.depth_bias.constant_factor)); /* BACK OFFSET */ - radeon_set_context_reg(cmd_buffer->cs, R_028B78_PA_SU_POLY_OFFSET_DB_FMT_CNTL, pa_su_poly_offset_db_fmt_cntl); + radeon_set_context_reg(R_028B78_PA_SU_POLY_OFFSET_DB_FMT_CNTL, pa_su_poly_offset_db_fmt_cntl); + radeon_end(); } static void @@ -3252,16 +3358,18 @@ radv_emit_primitive_topology(struct radv_cmd_buffer *cmd_buffer) assert(!cmd_buffer->state.mesh_shading); + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX7) { uint32_t vgt_prim = d->vk.ia.primitive_topology; if (pdev->info.gfx_level >= GFX12) vgt_prim |= S_030908_NUM_INPUT_CP(d->vk.ts.patch_control_points); - radeon_set_uconfig_reg_idx(&pdev->info, cmd_buffer->cs, R_030908_VGT_PRIMITIVE_TYPE, 1, vgt_prim); + radeon_set_uconfig_reg_idx(&pdev->info, R_030908_VGT_PRIMITIVE_TYPE, 1, vgt_prim); } else { - radeon_set_config_reg(cmd_buffer->cs, R_008958_VGT_PRIMITIVE_TYPE, d->vk.ia.primitive_topology); + radeon_set_config_reg(R_008958_VGT_PRIMITIVE_TYPE, d->vk.ia.primitive_topology); } + radeon_end(); radv_emit_vgt_gs_out(cmd_buffer, vgt_gs_out_prim_type); } @@ -3354,10 +3462,14 @@ radv_emit_fragment_shading_rate(struct radv_cmd_buffer *cmd_buffer) */ pa_cl_vrs_cntl |= S_028848_HTILE_RATE_COMBINER_MODE(htile_comb_mode); - /* Emit per-draw VRS rate which is the first combiner. */ - radeon_set_uconfig_reg(cmd_buffer->cs, R_03098C_GE_VRS_RATE, S_03098C_RATE_X(rate_x) | S_03098C_RATE_Y(rate_y)); + radeon_begin(cmd_buffer->cs); - radeon_set_context_reg(cmd_buffer->cs, R_028848_PA_CL_VRS_CNTL, pa_cl_vrs_cntl); + /* Emit per-draw VRS rate which is the first combiner. */ + radeon_set_uconfig_reg(R_03098C_GE_VRS_RATE, S_03098C_RATE_X(rate_x) | S_03098C_RATE_Y(rate_y)); + + radeon_set_context_reg(R_028848_PA_CL_VRS_CNTL, pa_cl_vrs_cntl); + + radeon_end(); } static uint32_t @@ -3386,21 +3498,22 @@ radv_emit_primitive_restart_enable(struct radv_cmd_buffer *cmd_buffer) struct radeon_cmdbuf *cs = cmd_buffer->cs; const bool en = d->vk.ia.primitive_restart_enable; + radeon_begin(cs); + if (pdev->info.has_prim_restart_sync_bug) { - radeon_event_write(cmd_buffer->cs, V_028A90_SQ_NON_EVENT); + radeon_event_write(V_028A90_SQ_NON_EVENT); } if (gfx_level >= GFX11) { - radeon_set_uconfig_reg(cs, R_03092C_GE_MULTI_PRIM_IB_RESET_EN, - S_03092C_RESET_EN(en) | - /* This disables primitive restart for non-indexed draws. - * By keeping this set, we don't have to unset RESET_EN - * for non-indexed draws. */ - S_03092C_DISABLE_FOR_AUTO_INDEX(1)); + radeon_set_uconfig_reg(R_03092C_GE_MULTI_PRIM_IB_RESET_EN, S_03092C_RESET_EN(en) | + /* This disables primitive restart for non-indexed + * draws. By keeping this set, we don't have to + * unset RESET_EN for non-indexed draws. */ + S_03092C_DISABLE_FOR_AUTO_INDEX(1)); } else if (gfx_level >= GFX9) { - radeon_set_uconfig_reg(cs, R_03092C_VGT_MULTI_PRIM_IB_RESET_EN, en); + radeon_set_uconfig_reg(R_03092C_VGT_MULTI_PRIM_IB_RESET_EN, en); } else { - radeon_set_context_reg(cs, R_028A94_VGT_MULTI_PRIM_IB_RESET_EN, en); + radeon_set_context_reg(R_028A94_VGT_MULTI_PRIM_IB_RESET_EN, en); /* GFX6-7: All 32 bits are compared. * GFX8: Only index type bits are compared. @@ -3413,6 +3526,8 @@ radv_emit_primitive_restart_enable(struct radv_cmd_buffer *cmd_buffer) RADV_TRACKED_VGT_MULTI_PRIM_IB_RESET_INDX, primitive_reset_index); } } + + radeon_end(); } static void @@ -3456,11 +3571,13 @@ radv_emit_logic_op(struct radv_cmd_buffer *cmd_buffer) } } + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028858_CB_COLOR_CONTROL, cb_color_control); + radeon_set_context_reg(R_028858_CB_COLOR_CONTROL, cb_color_control); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028808_CB_COLOR_CONTROL, cb_color_control); + radeon_set_context_reg(R_028808_CB_COLOR_CONTROL, cb_color_control); } + radeon_end(); } static void @@ -3485,16 +3602,20 @@ radv_emit_color_write(struct radv_cmd_buffer *cmd_buffer) if (device->pbb_allowed && settings->context_states_per_bin > 1 && cmd_buffer->state.last_cb_target_mask != cb_target_mask) { /* Flush DFSM on CB_TARGET_MASK changes. */ - radeon_event_write(cmd_buffer->cs, V_028A90_BREAK_BATCH); + radeon_begin(cmd_buffer->cs); + radeon_event_write(V_028A90_BREAK_BATCH); + radeon_end(); cmd_buffer->state.last_cb_target_mask = cb_target_mask; } + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028850_CB_TARGET_MASK, cb_target_mask); + radeon_set_context_reg(R_028850_CB_TARGET_MASK, cb_target_mask); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028238_CB_TARGET_MASK, cb_target_mask); + radeon_set_context_reg(R_028238_CB_TARGET_MASK, cb_target_mask); } + radeon_end(); } static void @@ -3535,10 +3656,12 @@ radv_emit_patch_control_points(struct radv_cmd_buffer *cmd_buffer) S_028B58_HS_NUM_INPUT_CP(pdev->info.gfx_level < GFX12 ? d->vk.ts.patch_control_points : 0) | S_028B58_HS_NUM_OUTPUT_CP(tcs->info.tcs.tcs_vertices_out); + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX7) { - radeon_set_context_reg_idx(cmd_buffer->cs, R_028B58_VGT_LS_HS_CONFIG, 2, ls_hs_config); + radeon_set_context_reg_idx(R_028B58_VGT_LS_HS_CONFIG, 2, ls_hs_config); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028B58_VGT_LS_HS_CONFIG, ls_hs_config); + radeon_set_context_reg(R_028B58_VGT_LS_HS_CONFIG, ls_hs_config); } if (pdev->info.gfx_level >= GFX9) { @@ -3556,11 +3679,11 @@ radv_emit_patch_control_points(struct radv_cmd_buffer *cmd_buffer) hs_rsrc2 |= S_00B42C_LDS_SIZE_GFX9(cmd_buffer->state.tess_lds_size); } - radeon_set_sh_reg(cmd_buffer->cs, tcs->info.regs.pgm_rsrc2, hs_rsrc2); + radeon_set_sh_reg(tcs->info.regs.pgm_rsrc2, hs_rsrc2); } else { unsigned ls_rsrc2 = vs->config.rsrc2 | S_00B52C_LDS_SIZE(cmd_buffer->state.tess_lds_size); - radeon_set_sh_reg(cmd_buffer->cs, vs->info.regs.pgm_rsrc2, ls_rsrc2); + radeon_set_sh_reg(vs->info.regs.pgm_rsrc2, ls_rsrc2); } /* Emit user SGPRs for dynamic patch control points. */ @@ -3575,13 +3698,15 @@ radv_emit_patch_control_points(struct radv_cmd_buffer *cmd_buffer) SET_SGPR_FIELD(TCS_OFFCHIP_LAYOUT_TES_READS_TF, tes->info.tes.reads_tess_factors) | SET_SGPR_FIELD(TCS_OFFCHIP_LAYOUT_PRIMITIVE_MODE, tes->info.tes._primitive_mode); - radeon_set_sh_reg(cmd_buffer->cs, tcs_offchip_layout_offset, tcs_offchip_layout); + radeon_set_sh_reg(tcs_offchip_layout_offset, tcs_offchip_layout); tcs_offchip_layout_offset = radv_get_user_sgpr_loc(tes, AC_UD_TCS_OFFCHIP_LAYOUT); assert(tcs_offchip_layout_offset); - radeon_set_sh_reg(cmd_buffer->cs, tcs_offchip_layout_offset, tcs_offchip_layout); + radeon_set_sh_reg(tcs_offchip_layout_offset, tcs_offchip_layout); } + + radeon_end(); } static void @@ -3613,13 +3738,13 @@ radv_emit_conservative_rast_mode(struct radv_cmd_buffer *cmd_buffer) pa_sc_conservative_rast = S_028C4C_NULL_SQUAD_AA_MASK_ENABLE(1); } + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028C54_PA_SC_CONSERVATIVE_RASTERIZATION_CNTL, - pa_sc_conservative_rast); + radeon_set_context_reg(R_028C54_PA_SC_CONSERVATIVE_RASTERIZATION_CNTL, pa_sc_conservative_rast); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028C4C_PA_SC_CONSERVATIVE_RASTERIZATION_CNTL, - pa_sc_conservative_rast); + radeon_set_context_reg(R_028C4C_PA_SC_CONSERVATIVE_RASTERIZATION_CNTL, pa_sc_conservative_rast); } + radeon_end(); } } @@ -3631,13 +3756,15 @@ radv_emit_depth_clamp_enable(struct radv_cmd_buffer *cmd_buffer) enum radv_depth_clamp_mode mode = radv_get_depth_clamp_mode(cmd_buffer); + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028064_DB_VIEWPORT_CONTROL, + radeon_set_context_reg(R_028064_DB_VIEWPORT_CONTROL, S_028064_DISABLE_VIEWPORT_CLAMP(mode == RADV_DEPTH_CLAMP_MODE_DISABLED)); } else { - radeon_set_context_reg(cmd_buffer->cs, R_02800C_DB_RENDER_OVERRIDE, + radeon_set_context_reg(R_02800C_DB_RENDER_OVERRIDE, S_02800C_DISABLE_VIEWPORT_CLAMP(mode == RADV_DEPTH_CLAMP_MODE_DISABLED)); } + radeon_end(); } static void @@ -3701,13 +3828,15 @@ radv_emit_rasterization_samples(struct radv_cmd_buffer *cmd_buffer) pa_sc_mode_cntl_1 |= S_028A4C_PS_ITER_SAMPLE(1); } + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028658_SPI_BARYC_CNTL, spi_baryc_cntl); + radeon_set_context_reg(R_028658_SPI_BARYC_CNTL, spi_baryc_cntl); } else { - radeon_set_context_reg(cmd_buffer->cs, R_0286E0_SPI_BARYC_CNTL, spi_baryc_cntl); + radeon_set_context_reg(R_0286E0_SPI_BARYC_CNTL, spi_baryc_cntl); } - radeon_set_context_reg(cmd_buffer->cs, R_028A4C_PA_SC_MODE_CNTL_1, pa_sc_mode_cntl_1); + radeon_set_context_reg(R_028A4C_PA_SC_MODE_CNTL_1, pa_sc_mode_cntl_1); + radeon_end(); } static void @@ -3739,100 +3868,99 @@ radv_emit_fb_color_state(struct radv_cmd_buffer *cmd_buffer, int index, struct r if (pdev->info.gfx_level >= GFX8 && pdev->info.gfx_level < GFX11 && iview->disable_tc_compat_cmask_mrt) cb_color_info &= C_028C70_FMASK_COMPRESS_1FRAG_ONLY; + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x24, cb->ac.cb_color_base); - radeon_set_context_reg(cmd_buffer->cs, R_028C64_CB_COLOR0_VIEW + index * 0x24, cb->ac.cb_color_view); - radeon_set_context_reg(cmd_buffer->cs, R_028C68_CB_COLOR0_VIEW2 + index * 0x24, cb->ac.cb_color_view2); - radeon_set_context_reg(cmd_buffer->cs, R_028C6C_CB_COLOR0_ATTRIB + index * 0x24, cb->ac.cb_color_attrib); - radeon_set_context_reg(cmd_buffer->cs, R_028C70_CB_COLOR0_FDCC_CONTROL + index * 0x24, cb_fdcc_control); - radeon_set_context_reg(cmd_buffer->cs, R_028C78_CB_COLOR0_ATTRIB2 + index * 0x24, cb->ac.cb_color_attrib2); - radeon_set_context_reg(cmd_buffer->cs, R_028C7C_CB_COLOR0_ATTRIB3 + index * 0x24, cb->ac.cb_color_attrib3); - radeon_set_context_reg(cmd_buffer->cs, R_028E40_CB_COLOR0_BASE_EXT + index * 4, - S_028E40_BASE_256B(cb->ac.cb_color_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028EC0_CB_COLOR0_INFO + index * 4, cb->ac.cb_color_info); + radeon_set_context_reg(R_028C60_CB_COLOR0_BASE + index * 0x24, cb->ac.cb_color_base); + radeon_set_context_reg(R_028C64_CB_COLOR0_VIEW + index * 0x24, cb->ac.cb_color_view); + radeon_set_context_reg(R_028C68_CB_COLOR0_VIEW2 + index * 0x24, cb->ac.cb_color_view2); + radeon_set_context_reg(R_028C6C_CB_COLOR0_ATTRIB + index * 0x24, cb->ac.cb_color_attrib); + radeon_set_context_reg(R_028C70_CB_COLOR0_FDCC_CONTROL + index * 0x24, cb_fdcc_control); + radeon_set_context_reg(R_028C78_CB_COLOR0_ATTRIB2 + index * 0x24, cb->ac.cb_color_attrib2); + radeon_set_context_reg(R_028C7C_CB_COLOR0_ATTRIB3 + index * 0x24, cb->ac.cb_color_attrib3); + radeon_set_context_reg(R_028E40_CB_COLOR0_BASE_EXT + index * 4, S_028E40_BASE_256B(cb->ac.cb_color_base >> 32)); + radeon_set_context_reg(R_028EC0_CB_COLOR0_INFO + index * 4, cb->ac.cb_color_info); } else if (pdev->info.gfx_level >= GFX11) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C6C_CB_COLOR0_VIEW + index * 0x3c, 4); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_view); /* CB_COLOR0_VIEW */ - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_info); /* CB_COLOR0_INFO */ - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_attrib); /* CB_COLOR0_ATTRIB */ - radeon_emit(cmd_buffer->cs, cb_fdcc_control); /* CB_COLOR0_FDCC_CONTROL */ + radeon_set_context_reg_seq(R_028C6C_CB_COLOR0_VIEW + index * 0x3c, 4); + radeon_emit(cb->ac.cb_color_view); /* CB_COLOR0_VIEW */ + radeon_emit(cb->ac.cb_color_info); /* CB_COLOR0_INFO */ + radeon_emit(cb->ac.cb_color_attrib); /* CB_COLOR0_ATTRIB */ + radeon_emit(cb_fdcc_control); /* CB_COLOR0_FDCC_CONTROL */ - radeon_set_context_reg(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, cb->ac.cb_color_base); - radeon_set_context_reg(cmd_buffer->cs, R_028E40_CB_COLOR0_BASE_EXT + index * 4, - S_028E40_BASE_256B(cb->ac.cb_color_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->ac.cb_dcc_base); - radeon_set_context_reg(cmd_buffer->cs, R_028EA0_CB_COLOR0_DCC_BASE_EXT + index * 4, - S_028EA0_BASE_256B(cb->ac.cb_dcc_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028EC0_CB_COLOR0_ATTRIB2 + index * 4, cb->ac.cb_color_attrib2); - radeon_set_context_reg(cmd_buffer->cs, R_028EE0_CB_COLOR0_ATTRIB3 + index * 4, cb->ac.cb_color_attrib3); + radeon_set_context_reg(R_028C60_CB_COLOR0_BASE + index * 0x3c, cb->ac.cb_color_base); + radeon_set_context_reg(R_028E40_CB_COLOR0_BASE_EXT + index * 4, S_028E40_BASE_256B(cb->ac.cb_color_base >> 32)); + radeon_set_context_reg(R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->ac.cb_dcc_base); + radeon_set_context_reg(R_028EA0_CB_COLOR0_DCC_BASE_EXT + index * 4, S_028EA0_BASE_256B(cb->ac.cb_dcc_base >> 32)); + radeon_set_context_reg(R_028EC0_CB_COLOR0_ATTRIB2 + index * 4, cb->ac.cb_color_attrib2); + radeon_set_context_reg(R_028EE0_CB_COLOR0_ATTRIB3 + index * 4, cb->ac.cb_color_attrib3); } else if (pdev->info.gfx_level >= GFX10) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 11); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_base); - radeon_emit(cmd_buffer->cs, 0); - radeon_emit(cmd_buffer->cs, 0); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_view); - radeon_emit(cmd_buffer->cs, cb_color_info); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_attrib); - radeon_emit(cmd_buffer->cs, cb->ac.cb_dcc_control); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_cmask); - radeon_emit(cmd_buffer->cs, 0); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_fmask); - radeon_emit(cmd_buffer->cs, 0); + radeon_set_context_reg_seq(R_028C60_CB_COLOR0_BASE + index * 0x3c, 11); + radeon_emit(cb->ac.cb_color_base); + radeon_emit(0); + radeon_emit(0); + radeon_emit(cb->ac.cb_color_view); + radeon_emit(cb_color_info); + radeon_emit(cb->ac.cb_color_attrib); + radeon_emit(cb->ac.cb_dcc_control); + radeon_emit(cb->ac.cb_color_cmask); + radeon_emit(0); + radeon_emit(cb->ac.cb_color_fmask); + radeon_emit(0); - radeon_set_context_reg(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->ac.cb_dcc_base); + radeon_set_context_reg(R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->ac.cb_dcc_base); - radeon_set_context_reg(cmd_buffer->cs, R_028E40_CB_COLOR0_BASE_EXT + index * 4, - S_028E40_BASE_256B(cb->ac.cb_color_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028E60_CB_COLOR0_CMASK_BASE_EXT + index * 4, + radeon_set_context_reg(R_028E40_CB_COLOR0_BASE_EXT + index * 4, S_028E40_BASE_256B(cb->ac.cb_color_base >> 32)); + radeon_set_context_reg(R_028E60_CB_COLOR0_CMASK_BASE_EXT + index * 4, S_028E60_BASE_256B(cb->ac.cb_color_cmask >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028E80_CB_COLOR0_FMASK_BASE_EXT + index * 4, + radeon_set_context_reg(R_028E80_CB_COLOR0_FMASK_BASE_EXT + index * 4, S_028E80_BASE_256B(cb->ac.cb_color_fmask >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028EA0_CB_COLOR0_DCC_BASE_EXT + index * 4, - S_028EA0_BASE_256B(cb->ac.cb_dcc_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028EC0_CB_COLOR0_ATTRIB2 + index * 4, cb->ac.cb_color_attrib2); - radeon_set_context_reg(cmd_buffer->cs, R_028EE0_CB_COLOR0_ATTRIB3 + index * 4, cb->ac.cb_color_attrib3); + radeon_set_context_reg(R_028EA0_CB_COLOR0_DCC_BASE_EXT + index * 4, S_028EA0_BASE_256B(cb->ac.cb_dcc_base >> 32)); + radeon_set_context_reg(R_028EC0_CB_COLOR0_ATTRIB2 + index * 4, cb->ac.cb_color_attrib2); + radeon_set_context_reg(R_028EE0_CB_COLOR0_ATTRIB3 + index * 4, cb->ac.cb_color_attrib3); } else if (pdev->info.gfx_level == GFX9) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 11); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_base); - radeon_emit(cmd_buffer->cs, S_028C64_BASE_256B(cb->ac.cb_color_base >> 32)); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_attrib2); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_view); - radeon_emit(cmd_buffer->cs, cb_color_info); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_attrib); - radeon_emit(cmd_buffer->cs, cb->ac.cb_dcc_control); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_cmask); - radeon_emit(cmd_buffer->cs, S_028C80_BASE_256B(cb->ac.cb_color_cmask >> 32)); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_fmask); - radeon_emit(cmd_buffer->cs, S_028C88_BASE_256B(cb->ac.cb_color_fmask >> 32)); + radeon_set_context_reg_seq(R_028C60_CB_COLOR0_BASE + index * 0x3c, 11); + radeon_emit(cb->ac.cb_color_base); + radeon_emit(S_028C64_BASE_256B(cb->ac.cb_color_base >> 32)); + radeon_emit(cb->ac.cb_color_attrib2); + radeon_emit(cb->ac.cb_color_view); + radeon_emit(cb_color_info); + radeon_emit(cb->ac.cb_color_attrib); + radeon_emit(cb->ac.cb_dcc_control); + radeon_emit(cb->ac.cb_color_cmask); + radeon_emit(S_028C80_BASE_256B(cb->ac.cb_color_cmask >> 32)); + radeon_emit(cb->ac.cb_color_fmask); + radeon_emit(S_028C88_BASE_256B(cb->ac.cb_color_fmask >> 32)); - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, 2); - radeon_emit(cmd_buffer->cs, cb->ac.cb_dcc_base); - radeon_emit(cmd_buffer->cs, S_028C98_BASE_256B(cb->ac.cb_dcc_base >> 32)); + radeon_set_context_reg_seq(R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, 2); + radeon_emit(cb->ac.cb_dcc_base); + radeon_emit(S_028C98_BASE_256B(cb->ac.cb_dcc_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_0287A0_CB_MRT0_EPITCH + index * 4, cb->ac.cb_mrt_epitch); + radeon_set_context_reg(R_0287A0_CB_MRT0_EPITCH + index * 4, cb->ac.cb_mrt_epitch); } else { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 6); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_base); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_pitch); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_slice); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_view); - radeon_emit(cmd_buffer->cs, cb_color_info); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_attrib); + radeon_set_context_reg_seq(R_028C60_CB_COLOR0_BASE + index * 0x3c, 6); + radeon_emit(cb->ac.cb_color_base); + radeon_emit(cb->ac.cb_color_pitch); + radeon_emit(cb->ac.cb_color_slice); + radeon_emit(cb->ac.cb_color_view); + radeon_emit(cb_color_info); + radeon_emit(cb->ac.cb_color_attrib); if (pdev->info.gfx_level == GFX8) - radeon_set_context_reg(cmd_buffer->cs, R_028C78_CB_COLOR0_DCC_CONTROL + index * 0x3c, cb->ac.cb_dcc_control); + radeon_set_context_reg(R_028C78_CB_COLOR0_DCC_CONTROL + index * 0x3c, cb->ac.cb_dcc_control); - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C7C_CB_COLOR0_CMASK + index * 0x3c, 4); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_cmask); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_cmask_slice); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_fmask); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_fmask_slice); + radeon_set_context_reg_seq(R_028C7C_CB_COLOR0_CMASK + index * 0x3c, 4); + radeon_emit(cb->ac.cb_color_cmask); + radeon_emit(cb->ac.cb_color_cmask_slice); + radeon_emit(cb->ac.cb_color_fmask); + radeon_emit(cb->ac.cb_color_fmask_slice); if (is_vi) { /* DCC BASE */ - radeon_set_context_reg(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->ac.cb_dcc_base); + radeon_set_context_reg(R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->ac.cb_dcc_base); } } + radeon_end(); + if (pdev->info.gfx_level >= GFX11 ? G_028C78_FDCC_ENABLE(cb_fdcc_control) : G_028C70_DCC_ENABLE(cb_color_info)) { /* Drawing with DCC enabled also compresses colorbuffers. */ VkImageSubresourceRange range = vk_image_view_subresource_range(&iview->vk); @@ -3872,7 +4000,9 @@ radv_update_zrange_precision(struct radv_cmd_buffer *cmd_buffer, struct radv_ds_ radv_emit_cond_exec(device, cmd_buffer->cs, va, 3 /* SET_CONTEXT_REG size */); } - radeon_set_context_reg(cmd_buffer->cs, db_z_info_reg, db_z_info); + radeon_begin(cmd_buffer->cs); + radeon_set_context_reg(db_z_info_reg, db_z_info); + radeon_end(); } static struct radv_image * @@ -3929,104 +4059,104 @@ radv_emit_fb_ds_state(struct radv_cmd_buffer *cmd_buffer, struct radv_ds_buffer_ } } + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level < GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028000_DB_RENDER_CONTROL, db_render_control); - radeon_set_context_reg(cmd_buffer->cs, R_028008_DB_DEPTH_VIEW, ds->ac.db_depth_view); - radeon_set_context_reg(cmd_buffer->cs, R_028ABC_DB_HTILE_SURFACE, db_htile_surface); + radeon_set_context_reg(R_028000_DB_RENDER_CONTROL, db_render_control); + radeon_set_context_reg(R_028008_DB_DEPTH_VIEW, ds->ac.db_depth_view); + radeon_set_context_reg(R_028ABC_DB_HTILE_SURFACE, db_htile_surface); } - radeon_set_context_reg(cmd_buffer->cs, R_028010_DB_RENDER_OVERRIDE2, ds->db_render_override2); + radeon_set_context_reg(R_028010_DB_RENDER_OVERRIDE2, ds->db_render_override2); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028004_DB_DEPTH_VIEW, ds->ac.db_depth_view); - radeon_set_context_reg(cmd_buffer->cs, R_028008_DB_DEPTH_VIEW1, ds->ac.u.gfx12.db_depth_view1); - radeon_set_context_reg(cmd_buffer->cs, R_028014_DB_DEPTH_SIZE_XY, ds->ac.db_depth_size); - radeon_set_context_reg(cmd_buffer->cs, R_028018_DB_Z_INFO, ds->ac.db_z_info); - radeon_set_context_reg(cmd_buffer->cs, R_02801C_DB_STENCIL_INFO, ds->ac.db_stencil_info); - radeon_set_context_reg(cmd_buffer->cs, R_028020_DB_Z_READ_BASE, ds->ac.db_depth_base); - radeon_set_context_reg(cmd_buffer->cs, R_028024_DB_Z_READ_BASE_HI, S_028024_BASE_HI(ds->ac.db_depth_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028028_DB_Z_WRITE_BASE, ds->ac.db_depth_base); - radeon_set_context_reg(cmd_buffer->cs, R_02802C_DB_Z_WRITE_BASE_HI, S_02802C_BASE_HI(ds->ac.db_depth_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028030_DB_STENCIL_READ_BASE, ds->ac.db_stencil_base); - radeon_set_context_reg(cmd_buffer->cs, R_028034_DB_STENCIL_READ_BASE_HI, - S_028034_BASE_HI(ds->ac.db_stencil_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028038_DB_STENCIL_WRITE_BASE, ds->ac.db_stencil_base); - radeon_set_context_reg(cmd_buffer->cs, R_02803C_DB_STENCIL_WRITE_BASE_HI, - S_02803C_BASE_HI(ds->ac.db_stencil_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028B94_PA_SC_HIZ_INFO, ds->ac.u.gfx12.hiz_info); - radeon_set_context_reg(cmd_buffer->cs, R_028B98_PA_SC_HIS_INFO, ds->ac.u.gfx12.his_info); + radeon_set_context_reg(R_028004_DB_DEPTH_VIEW, ds->ac.db_depth_view); + radeon_set_context_reg(R_028008_DB_DEPTH_VIEW1, ds->ac.u.gfx12.db_depth_view1); + radeon_set_context_reg(R_028014_DB_DEPTH_SIZE_XY, ds->ac.db_depth_size); + radeon_set_context_reg(R_028018_DB_Z_INFO, ds->ac.db_z_info); + radeon_set_context_reg(R_02801C_DB_STENCIL_INFO, ds->ac.db_stencil_info); + radeon_set_context_reg(R_028020_DB_Z_READ_BASE, ds->ac.db_depth_base); + radeon_set_context_reg(R_028024_DB_Z_READ_BASE_HI, S_028024_BASE_HI(ds->ac.db_depth_base >> 32)); + radeon_set_context_reg(R_028028_DB_Z_WRITE_BASE, ds->ac.db_depth_base); + radeon_set_context_reg(R_02802C_DB_Z_WRITE_BASE_HI, S_02802C_BASE_HI(ds->ac.db_depth_base >> 32)); + radeon_set_context_reg(R_028030_DB_STENCIL_READ_BASE, ds->ac.db_stencil_base); + radeon_set_context_reg(R_028034_DB_STENCIL_READ_BASE_HI, S_028034_BASE_HI(ds->ac.db_stencil_base >> 32)); + radeon_set_context_reg(R_028038_DB_STENCIL_WRITE_BASE, ds->ac.db_stencil_base); + radeon_set_context_reg(R_02803C_DB_STENCIL_WRITE_BASE_HI, S_02803C_BASE_HI(ds->ac.db_stencil_base >> 32)); + radeon_set_context_reg(R_028B94_PA_SC_HIZ_INFO, ds->ac.u.gfx12.hiz_info); + radeon_set_context_reg(R_028B98_PA_SC_HIS_INFO, ds->ac.u.gfx12.his_info); if (ds->ac.u.gfx12.hiz_info) { - radeon_set_context_reg(cmd_buffer->cs, R_028B9C_PA_SC_HIZ_BASE, ds->ac.u.gfx12.hiz_base); - radeon_set_context_reg(cmd_buffer->cs, R_028BA0_PA_SC_HIZ_BASE_EXT, - S_028BA0_BASE_256B(ds->ac.u.gfx12.hiz_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028BA4_PA_SC_HIZ_SIZE_XY, ds->ac.u.gfx12.hiz_size_xy); + radeon_set_context_reg(R_028B9C_PA_SC_HIZ_BASE, ds->ac.u.gfx12.hiz_base); + radeon_set_context_reg(R_028BA0_PA_SC_HIZ_BASE_EXT, S_028BA0_BASE_256B(ds->ac.u.gfx12.hiz_base >> 32)); + radeon_set_context_reg(R_028BA4_PA_SC_HIZ_SIZE_XY, ds->ac.u.gfx12.hiz_size_xy); } if (ds->ac.u.gfx12.his_info) { - radeon_set_context_reg(cmd_buffer->cs, R_028BA8_PA_SC_HIS_BASE, ds->ac.u.gfx12.his_base); - radeon_set_context_reg(cmd_buffer->cs, R_028BAC_PA_SC_HIS_BASE_EXT, - S_028BAC_BASE_256B(ds->ac.u.gfx12.his_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028BB0_PA_SC_HIS_SIZE_XY, ds->ac.u.gfx12.his_size_xy); + radeon_set_context_reg(R_028BA8_PA_SC_HIS_BASE, ds->ac.u.gfx12.his_base); + radeon_set_context_reg(R_028BAC_PA_SC_HIS_BASE_EXT, S_028BAC_BASE_256B(ds->ac.u.gfx12.his_base >> 32)); + radeon_set_context_reg(R_028BB0_PA_SC_HIS_SIZE_XY, ds->ac.u.gfx12.his_size_xy); } } else if (pdev->info.gfx_level >= GFX10) { - radeon_set_context_reg(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, db_htile_data_base); - radeon_set_context_reg(cmd_buffer->cs, R_02801C_DB_DEPTH_SIZE_XY, ds->ac.db_depth_size); + radeon_set_context_reg(R_028014_DB_HTILE_DATA_BASE, db_htile_data_base); + radeon_set_context_reg(R_02801C_DB_DEPTH_SIZE_XY, ds->ac.db_depth_size); if (pdev->info.gfx_level >= GFX11) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028040_DB_Z_INFO, 6); + radeon_set_context_reg_seq(R_028040_DB_Z_INFO, 6); } else { - radeon_set_context_reg_seq(cmd_buffer->cs, R_02803C_DB_DEPTH_INFO, 7); - radeon_emit(cmd_buffer->cs, S_02803C_RESOURCE_LEVEL(1)); + radeon_set_context_reg_seq(R_02803C_DB_DEPTH_INFO, 7); + radeon_emit(S_02803C_RESOURCE_LEVEL(1)); } - radeon_emit(cmd_buffer->cs, db_z_info); - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_info); - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_base); - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_base); - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_base); - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_base); + radeon_emit(db_z_info); + radeon_emit(ds->ac.db_stencil_info); + radeon_emit(ds->ac.db_depth_base); + radeon_emit(ds->ac.db_stencil_base); + radeon_emit(ds->ac.db_depth_base); + radeon_emit(ds->ac.db_stencil_base); - radeon_set_context_reg_seq(cmd_buffer->cs, R_028068_DB_Z_READ_BASE_HI, 5); - radeon_emit(cmd_buffer->cs, S_028068_BASE_HI(ds->ac.db_depth_base >> 32)); - radeon_emit(cmd_buffer->cs, S_02806C_BASE_HI(ds->ac.db_stencil_base >> 32)); - radeon_emit(cmd_buffer->cs, S_028070_BASE_HI(ds->ac.db_depth_base >> 32)); - radeon_emit(cmd_buffer->cs, S_028074_BASE_HI(ds->ac.db_stencil_base >> 32)); - radeon_emit(cmd_buffer->cs, S_028078_BASE_HI(db_htile_data_base >> 32)); + radeon_set_context_reg_seq(R_028068_DB_Z_READ_BASE_HI, 5); + radeon_emit(S_028068_BASE_HI(ds->ac.db_depth_base >> 32)); + radeon_emit(S_02806C_BASE_HI(ds->ac.db_stencil_base >> 32)); + radeon_emit(S_028070_BASE_HI(ds->ac.db_depth_base >> 32)); + radeon_emit(S_028074_BASE_HI(ds->ac.db_stencil_base >> 32)); + radeon_emit(S_028078_BASE_HI(db_htile_data_base >> 32)); } else if (pdev->info.gfx_level == GFX9) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, 3); - radeon_emit(cmd_buffer->cs, db_htile_data_base); - radeon_emit(cmd_buffer->cs, S_028018_BASE_HI(db_htile_data_base >> 32)); - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_size); + radeon_set_context_reg_seq(R_028014_DB_HTILE_DATA_BASE, 3); + radeon_emit(db_htile_data_base); + radeon_emit(S_028018_BASE_HI(db_htile_data_base >> 32)); + radeon_emit(ds->ac.db_depth_size); - radeon_set_context_reg_seq(cmd_buffer->cs, R_028038_DB_Z_INFO, 10); - radeon_emit(cmd_buffer->cs, db_z_info); /* DB_Z_INFO */ - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_info); /* DB_STENCIL_INFO */ - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_base); /* DB_Z_READ_BASE */ - radeon_emit(cmd_buffer->cs, S_028044_BASE_HI(ds->ac.db_depth_base >> 32)); /* DB_Z_READ_BASE_HI */ - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_base); /* DB_STENCIL_READ_BASE */ - radeon_emit(cmd_buffer->cs, S_02804C_BASE_HI(ds->ac.db_stencil_base >> 32)); /* DB_STENCIL_READ_BASE_HI */ - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_base); /* DB_Z_WRITE_BASE */ - radeon_emit(cmd_buffer->cs, S_028054_BASE_HI(ds->ac.db_depth_base >> 32)); /* DB_Z_WRITE_BASE_HI */ - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_base); /* DB_STENCIL_WRITE_BASE */ - radeon_emit(cmd_buffer->cs, S_02805C_BASE_HI(ds->ac.db_stencil_base >> 32)); /* DB_STENCIL_WRITE_BASE_HI */ + radeon_set_context_reg_seq(R_028038_DB_Z_INFO, 10); + radeon_emit(db_z_info); /* DB_Z_INFO */ + radeon_emit(ds->ac.db_stencil_info); /* DB_STENCIL_INFO */ + radeon_emit(ds->ac.db_depth_base); /* DB_Z_READ_BASE */ + radeon_emit(S_028044_BASE_HI(ds->ac.db_depth_base >> 32)); /* DB_Z_READ_BASE_HI */ + radeon_emit(ds->ac.db_stencil_base); /* DB_STENCIL_READ_BASE */ + radeon_emit(S_02804C_BASE_HI(ds->ac.db_stencil_base >> 32)); /* DB_STENCIL_READ_BASE_HI */ + radeon_emit(ds->ac.db_depth_base); /* DB_Z_WRITE_BASE */ + radeon_emit(S_028054_BASE_HI(ds->ac.db_depth_base >> 32)); /* DB_Z_WRITE_BASE_HI */ + radeon_emit(ds->ac.db_stencil_base); /* DB_STENCIL_WRITE_BASE */ + radeon_emit(S_02805C_BASE_HI(ds->ac.db_stencil_base >> 32)); /* DB_STENCIL_WRITE_BASE_HI */ - radeon_set_context_reg_seq(cmd_buffer->cs, R_028068_DB_Z_INFO2, 2); - radeon_emit(cmd_buffer->cs, ds->ac.u.gfx6.db_z_info2); - radeon_emit(cmd_buffer->cs, ds->ac.u.gfx6.db_stencil_info2); + radeon_set_context_reg_seq(R_028068_DB_Z_INFO2, 2); + radeon_emit(ds->ac.u.gfx6.db_z_info2); + radeon_emit(ds->ac.u.gfx6.db_stencil_info2); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, db_htile_data_base); + radeon_set_context_reg(R_028014_DB_HTILE_DATA_BASE, db_htile_data_base); - radeon_set_context_reg_seq(cmd_buffer->cs, R_02803C_DB_DEPTH_INFO, 9); - radeon_emit(cmd_buffer->cs, ds->ac.u.gfx6.db_depth_info); /* R_02803C_DB_DEPTH_INFO */ - radeon_emit(cmd_buffer->cs, db_z_info); /* R_028040_DB_Z_INFO */ - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_info); /* R_028044_DB_STENCIL_INFO */ - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_base); /* R_028048_DB_Z_READ_BASE */ - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_base); /* R_02804C_DB_STENCIL_READ_BASE */ - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_base); /* R_028050_DB_Z_WRITE_BASE */ - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_base); /* R_028054_DB_STENCIL_WRITE_BASE */ - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_size); /* R_028058_DB_DEPTH_SIZE */ - radeon_emit(cmd_buffer->cs, ds->ac.u.gfx6.db_depth_slice); /* R_02805C_DB_DEPTH_SLICE */ + radeon_set_context_reg_seq(R_02803C_DB_DEPTH_INFO, 9); + radeon_emit(ds->ac.u.gfx6.db_depth_info); /* R_02803C_DB_DEPTH_INFO */ + radeon_emit(db_z_info); /* R_028040_DB_Z_INFO */ + radeon_emit(ds->ac.db_stencil_info); /* R_028044_DB_STENCIL_INFO */ + radeon_emit(ds->ac.db_depth_base); /* R_028048_DB_Z_READ_BASE */ + radeon_emit(ds->ac.db_stencil_base); /* R_02804C_DB_STENCIL_READ_BASE */ + radeon_emit(ds->ac.db_depth_base); /* R_028050_DB_Z_WRITE_BASE */ + radeon_emit(ds->ac.db_stencil_base); /* R_028054_DB_STENCIL_WRITE_BASE */ + radeon_emit(ds->ac.db_depth_size); /* R_028058_DB_DEPTH_SIZE */ + radeon_emit(ds->ac.u.gfx6.db_depth_slice); /* R_02805C_DB_DEPTH_SLICE */ } + radeon_end(); + /* Update the ZRANGE_PRECISION value for the TC-compat bug. */ radv_update_zrange_precision(cmd_buffer, ds, iview, true); } @@ -4038,18 +4168,20 @@ radv_emit_null_ds_state(struct radv_cmd_buffer *cmd_buffer) const struct radv_physical_device *pdev = radv_device_physical(device); const enum amd_gfx_level gfx_level = pdev->info.gfx_level; - if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028018_DB_Z_INFO, 2); - radeon_emit(cmd_buffer->cs, S_028018_FORMAT(V_028018_Z_INVALID) | S_028018_NUM_SAMPLES(3)); - radeon_emit(cmd_buffer->cs, S_02801C_FORMAT(V_02801C_STENCIL_INVALID) | S_02801C_TILE_STENCIL_DISABLE(1)); + radeon_begin(cmd_buffer->cs); - radeon_set_context_reg(cmd_buffer->cs, R_028B94_PA_SC_HIZ_INFO, S_028B94_SURFACE_ENABLE(0)); - radeon_set_context_reg(cmd_buffer->cs, R_028B98_PA_SC_HIS_INFO, S_028B98_SURFACE_ENABLE(0)); + if (pdev->info.gfx_level >= GFX12) { + radeon_set_context_reg_seq(R_028018_DB_Z_INFO, 2); + radeon_emit(S_028018_FORMAT(V_028018_Z_INVALID) | S_028018_NUM_SAMPLES(3)); + radeon_emit(S_02801C_FORMAT(V_02801C_STENCIL_INVALID) | S_02801C_TILE_STENCIL_DISABLE(1)); + + radeon_set_context_reg(R_028B94_PA_SC_HIZ_INFO, S_028B94_SURFACE_ENABLE(0)); + radeon_set_context_reg(R_028B98_PA_SC_HIS_INFO, S_028B98_SURFACE_ENABLE(0)); } else { if (gfx_level == GFX9) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028038_DB_Z_INFO, 2); + radeon_set_context_reg_seq(R_028038_DB_Z_INFO, 2); } else { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028040_DB_Z_INFO, 2); + radeon_set_context_reg_seq(R_028040_DB_Z_INFO, 2); } /* On GFX11+, the hw intentionally looks at DB_Z_INFO.NUM_SAMPLES when there is no bound @@ -4057,19 +4189,18 @@ radv_emit_null_ds_state(struct radv_cmd_buffer *cmd_buffer) * PA_SC_AA_CONFIG.MSAA_EXPOSED_SAMPLES). Use 8x for DB_Z_INFO.NUM_SAMPLES to make sure it's not * the constraining factor. This affects VRS, occlusion queries and POPS. */ - radeon_emit(cmd_buffer->cs, - S_028040_FORMAT(V_028040_Z_INVALID) | S_028040_NUM_SAMPLES(pdev->info.gfx_level >= GFX11 ? 3 : 0)); - radeon_emit(cmd_buffer->cs, S_028044_FORMAT(V_028044_STENCIL_INVALID)); + radeon_emit(S_028040_FORMAT(V_028040_Z_INVALID) | S_028040_NUM_SAMPLES(pdev->info.gfx_level >= GFX11 ? 3 : 0)); + radeon_emit(S_028044_FORMAT(V_028044_STENCIL_INVALID)); uint32_t db_render_control = 0; if (gfx_level == GFX11 || gfx_level == GFX11_5) radv_gfx11_set_db_render_control(device, 1, &db_render_control); - radeon_set_context_reg(cmd_buffer->cs, R_028000_DB_RENDER_CONTROL, db_render_control); + radeon_set_context_reg(R_028000_DB_RENDER_CONTROL, db_render_control); } - radeon_set_context_reg(cmd_buffer->cs, R_028010_DB_RENDER_OVERRIDE2, - S_028010_CENTROID_COMPUTATION_MODE(gfx_level >= GFX10_3)); + radeon_set_context_reg(R_028010_DB_RENDER_OVERRIDE2, S_028010_CENTROID_COMPUTATION_MODE(gfx_level >= GFX10_3)); + radeon_end(); } /** * Update the fast clear depth/stencil values if the image is bound as a @@ -4085,17 +4216,21 @@ radv_update_bound_fast_clear_ds(struct radv_cmd_buffer *cmd_buffer, const struct if (cmd_buffer->state.render.ds_att.iview == NULL || cmd_buffer->state.render.ds_att.iview->image != image) return; + radeon_begin(cs); + if (aspects == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) { - radeon_set_context_reg_seq(cs, R_028028_DB_STENCIL_CLEAR, 2); - radeon_emit(cs, ds_clear_value.stencil); - radeon_emit(cs, fui(ds_clear_value.depth)); + radeon_set_context_reg_seq(R_028028_DB_STENCIL_CLEAR, 2); + radeon_emit(ds_clear_value.stencil); + radeon_emit(fui(ds_clear_value.depth)); } else if (aspects == VK_IMAGE_ASPECT_DEPTH_BIT) { - radeon_set_context_reg(cs, R_02802C_DB_DEPTH_CLEAR, fui(ds_clear_value.depth)); + radeon_set_context_reg(R_02802C_DB_DEPTH_CLEAR, fui(ds_clear_value.depth)); } else { assert(aspects == VK_IMAGE_ASPECT_STENCIL_BIT); - radeon_set_context_reg(cs, R_028028_DB_STENCIL_CLEAR, ds_clear_value.stencil); + radeon_set_context_reg(R_028028_DB_STENCIL_CLEAR, ds_clear_value.stencil); } + radeon_end(); + /* Update the ZRANGE_PRECISION value for the TC-compat bug. This is * only needed when clearing Z to 0.0. */ @@ -4125,11 +4260,14 @@ radv_set_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image ASSERTED unsigned cdw_end = radv_cs_write_data_head(device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, va, 2 * level_count, cmd_buffer->state.predicating); + radeon_begin(cs); + for (uint32_t l = 0; l < level_count; l++) { - radeon_emit(cs, ds_clear_value.stencil); - radeon_emit(cs, fui(ds_clear_value.depth)); + radeon_emit(ds_clear_value.stencil); + radeon_emit(fui(ds_clear_value.depth)); } + radeon_end(); assert(cmd_buffer->cs->cdw == cdw_end); } else { /* Otherwise we need one WRITE_DATA packet per level. */ @@ -4170,9 +4308,12 @@ radv_set_tc_compat_zrange_metadata(struct radv_cmd_buffer *cmd_buffer, struct ra ASSERTED unsigned cdw_end = radv_cs_write_data_head(device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, va, level_count, cmd_buffer->state.predicating); - for (uint32_t l = 0; l < level_count; l++) - radeon_emit(cs, value); + radeon_begin(cs); + for (uint32_t l = 0; l < level_count; l++) + radeon_emit(value); + + radeon_end(); assert(cmd_buffer->cs->cdw == cdw_end); } @@ -4239,24 +4380,28 @@ radv_load_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, const struct rad uint32_t reg = R_028028_DB_STENCIL_CLEAR + 4 * reg_offset; - if (pdev->info.has_load_ctx_reg_pkt) { - radeon_emit(cs, PKT3(PKT3_LOAD_CONTEXT_REG_INDEX, 3, 0)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, (reg - SI_CONTEXT_REG_OFFSET) >> 2); - radeon_emit(cs, reg_count); - } else { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | - (reg_count == 2 ? COPY_DATA_COUNT_SEL : 0)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, reg >> 2); - radeon_emit(cs, 0); + radeon_begin(cs); - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); - radeon_emit(cs, 0); + if (pdev->info.has_load_ctx_reg_pkt) { + radeon_emit(PKT3(PKT3_LOAD_CONTEXT_REG_INDEX, 3, 0)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit((reg - SI_CONTEXT_REG_OFFSET) >> 2); + radeon_emit(reg_count); + } else { + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | + (reg_count == 2 ? COPY_DATA_COUNT_SEL : 0)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(reg >> 2); + radeon_emit(0); + + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0)); + radeon_emit(0); } + + radeon_end(); } /* @@ -4280,11 +4425,14 @@ radv_update_fce_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image * ASSERTED unsigned cdw_end = radv_cs_write_data_head(device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, va, 2 * level_count, false); + radeon_begin(cmd_buffer->cs); + for (uint32_t l = 0; l < level_count; l++) { - radeon_emit(cmd_buffer->cs, pred_val); - radeon_emit(cmd_buffer->cs, pred_val >> 32); + radeon_emit(pred_val); + radeon_emit(pred_val >> 32); } + radeon_end(); assert(cmd_buffer->cs->cdw == cdw_end); } @@ -4309,11 +4457,14 @@ radv_update_dcc_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image * ASSERTED unsigned cdw_end = radv_cs_write_data_head(device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, va, 2 * level_count, false); + radeon_begin(cmd_buffer->cs); + for (uint32_t l = 0; l < level_count; l++) { - radeon_emit(cmd_buffer->cs, pred_val); - radeon_emit(cmd_buffer->cs, pred_val >> 32); + radeon_emit(pred_val); + radeon_emit(pred_val >> 32); } + radeon_end(); assert(cmd_buffer->cs->cdw == cdw_end); } @@ -4333,9 +4484,11 @@ radv_update_bound_fast_clear_color(struct radv_cmd_buffer *cmd_buffer, struct ra ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, 4); - radeon_set_context_reg_seq(cs, R_028C8C_CB_COLOR0_CLEAR_WORD0 + cb_idx * 0x3c, 2); - radeon_emit(cs, color_values[0]); - radeon_emit(cs, color_values[1]); + radeon_begin(cs); + radeon_set_context_reg_seq(R_028C8C_CB_COLOR0_CLEAR_WORD0 + cb_idx * 0x3c, 2); + radeon_emit(color_values[0]); + radeon_emit(color_values[1]); + radeon_end(); assert(cmd_buffer->cs->cdw <= cdw_max); @@ -4361,11 +4514,14 @@ radv_set_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_im ASSERTED unsigned cdw_end = radv_cs_write_data_head(device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, va, 2 * level_count, cmd_buffer->state.predicating); + radeon_begin(cs); + for (uint32_t l = 0; l < level_count; l++) { - radeon_emit(cs, color_values[0]); - radeon_emit(cs, color_values[1]); + radeon_emit(color_values[0]); + radeon_emit(color_values[1]); } + radeon_end(); assert(cmd_buffer->cs->cdw == cdw_end); } else { /* Some default value we can set in the update. */ @@ -4422,23 +4578,27 @@ radv_load_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_i uint64_t va = radv_image_get_fast_clear_va(image, iview->vk.base_mip_level); uint32_t reg = R_028C8C_CB_COLOR0_CLEAR_WORD0 + cb_idx * 0x3c; - if (pdev->info.has_load_ctx_reg_pkt) { - radeon_emit(cs, PKT3(PKT3_LOAD_CONTEXT_REG_INDEX, 3, cmd_buffer->state.predicating)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, (reg - SI_CONTEXT_REG_OFFSET) >> 2); - radeon_emit(cs, 2); - } else { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, cmd_buffer->state.predicating)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | COPY_DATA_COUNT_SEL); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, reg >> 2); - radeon_emit(cs, 0); + radeon_begin(cs); - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); - radeon_emit(cs, 0); + if (pdev->info.has_load_ctx_reg_pkt) { + radeon_emit(PKT3(PKT3_LOAD_CONTEXT_REG_INDEX, 3, cmd_buffer->state.predicating)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit((reg - SI_CONTEXT_REG_OFFSET) >> 2); + radeon_emit(2); + } else { + radeon_emit(PKT3(PKT3_COPY_DATA, 4, cmd_buffer->state.predicating)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | COPY_DATA_COUNT_SEL); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(reg >> 2); + radeon_emit(0); + + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); + radeon_emit(0); } + + radeon_end(); } /* GFX9+ metadata cache flushing workaround. metadata cache coherency is @@ -4540,11 +4700,13 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) for (i = 0; i < render->color_att_count; ++i) { struct radv_image_view *iview = render->color_att[i].iview; if (!iview) { + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028EC0_CB_COLOR0_INFO + i * 4, color_invalid); + radeon_set_context_reg(R_028EC0_CB_COLOR0_INFO + i * 4, color_invalid); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028C70_CB_COLOR0_INFO + i * 0x3C, color_invalid); + radeon_set_context_reg(R_028C70_CB_COLOR0_INFO + i * 0x3C, color_invalid); } + radeon_end(); continue; } @@ -4579,11 +4741,13 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) extent.height = MIN2(extent.height, iview->vk.extent.height); } for (; i < cmd_buffer->state.last_subpass_color_count; i++) { + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028EC0_CB_COLOR0_INFO + i * 4, color_invalid); + radeon_set_context_reg(R_028EC0_CB_COLOR0_INFO + i * 4, color_invalid); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028C70_CB_COLOR0_INFO + i * 0x3C, color_invalid); + radeon_set_context_reg(R_028C70_CB_COLOR0_INFO + i * 0x3C, color_invalid); } + radeon_end(); } cmd_buffer->state.last_subpass_color_count = render->color_att_count; @@ -4649,6 +4813,8 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) radv_emit_null_ds_state(cmd_buffer); } + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX11) { bool vrs_surface_enable = render->vrs_att.iview != NULL; unsigned xmax = 0, ymax = 0; @@ -4667,18 +4833,17 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) ymax = vrs_iview->vk.extent.height - 1; if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_0283E0_PA_SC_VRS_INFO, + radeon_set_context_reg(R_0283E0_PA_SC_VRS_INFO, S_0283E0_RATE_SW_MODE(vrs_image->planes[0].surface.u.gfx9.swizzle_mode)); } } - radeon_set_context_reg_seq(cmd_buffer->cs, R_0283F0_PA_SC_VRS_RATE_BASE, 3); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, S_0283F4_BASE_256B(va >> 40)); - radeon_emit(cmd_buffer->cs, S_0283F8_X_MAX(xmax) | S_0283F8_Y_MAX(ymax)); + radeon_set_context_reg_seq(R_0283F0_PA_SC_VRS_RATE_BASE, 3); + radeon_emit(va >> 8); + radeon_emit(S_0283F4_BASE_256B(va >> 40)); + radeon_emit(S_0283F8_X_MAX(xmax) | S_0283F8_Y_MAX(ymax)); - radeon_set_context_reg(cmd_buffer->cs, R_0283D0_PA_SC_VRS_OVERRIDE_CNTL, - S_0283D0_VRS_SURFACE_ENABLE(vrs_surface_enable)); + radeon_set_context_reg(R_0283D0_PA_SC_VRS_OVERRIDE_CNTL, S_0283D0_VRS_SURFACE_ENABLE(vrs_surface_enable)); } if (pdev->info.gfx_level >= GFX8 && pdev->info.gfx_level < GFX12) { @@ -4688,12 +4853,12 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) if (pdev->info.gfx_level >= GFX11) { const bool has_dedicated_vram = pdev->info.has_dedicated_vram; - radeon_set_context_reg(cmd_buffer->cs, R_028424_CB_FDCC_CONTROL, + radeon_set_context_reg(R_028424_CB_FDCC_CONTROL, S_028424_SAMPLE_MASK_TRACKER_WATERMARK(has_dedicated_vram ? 0 : 15)); } else { uint8_t watermark = gfx_level >= GFX10 ? 6 : 4; - radeon_set_context_reg(cmd_buffer->cs, R_028424_CB_DCC_CONTROL, + radeon_set_context_reg(R_028424_CB_DCC_CONTROL, S_028424_OVERWRITE_COMBINER_MRT_SHARING_DISABLE(gfx_level <= GFX9) | S_028424_OVERWRITE_COMBINER_WATERMARK(watermark) | S_028424_DISABLE_CONSTANT_ENCODE_AC01(disable_constant_encode_ac01) | @@ -4702,13 +4867,14 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) } if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028184_PA_SC_SCREEN_SCISSOR_BR, + radeon_set_context_reg(R_028184_PA_SC_SCREEN_SCISSOR_BR, S_028034_BR_X(extent.width) | S_028034_BR_Y(extent.height)); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028034_PA_SC_SCREEN_SCISSOR_BR, + radeon_set_context_reg(R_028034_PA_SC_SCREEN_SCISSOR_BR, S_028034_BR_X(extent.width) | S_028034_BR_Y(extent.height)); } + radeon_end(); assert(cmd_buffer->cs->cdw <= cdw_max); cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_FRAMEBUFFER; @@ -4767,15 +4933,17 @@ radv_emit_guardband_state(struct radv_cmd_buffer *cmd_buffer) } } + radeon_begin(cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg_seq(cs, R_02842C_PA_CL_GB_VERT_CLIP_ADJ, 4); + radeon_set_context_reg_seq(R_02842C_PA_CL_GB_VERT_CLIP_ADJ, 4); } else { - radeon_set_context_reg_seq(cs, R_028BE8_PA_CL_GB_VERT_CLIP_ADJ, 4); + radeon_set_context_reg_seq(R_028BE8_PA_CL_GB_VERT_CLIP_ADJ, 4); } - radeon_emit(cs, fui(guardband_y)); - radeon_emit(cs, fui(discard_y)); - radeon_emit(cs, fui(guardband_x)); - radeon_emit(cs, fui(discard_x)); + radeon_emit(fui(guardband_y)); + radeon_emit(fui(discard_y)); + radeon_emit(fui(guardband_x)); + radeon_emit(fui(discard_x)); + radeon_end(); cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_GUARDBAND; } @@ -4818,12 +4986,14 @@ radv_emit_index_buffer(struct radv_cmd_buffer *cmd_buffer) radv_handle_zero_index_buffer_bug(cmd_buffer, &index_va, &max_index_count); } - radeon_emit(cs, PKT3(PKT3_INDEX_BASE, 1, 0)); - radeon_emit(cs, index_va); - radeon_emit(cs, index_va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_INDEX_BASE, 1, 0)); + radeon_emit(index_va); + radeon_emit(index_va >> 32); - radeon_emit(cs, PKT3(PKT3_INDEX_BUFFER_SIZE, 0, 0)); - radeon_emit(cs, max_index_count); + radeon_emit(PKT3(PKT3_INDEX_BUFFER_SIZE, 0, 0)); + radeon_emit(max_index_count); + radeon_end(); cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_INDEX_BUFFER; } @@ -4864,6 +5034,8 @@ radv_flush_occlusion_query_state(struct radv_cmd_buffer *cmd_buffer) } } + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { radeon_opt_set_context_reg(cmd_buffer, R_028060_DB_COUNT_CONTROL, RADV_TRACKED_DB_COUNT_CONTROL, db_count_control); @@ -4872,6 +5044,8 @@ radv_flush_occlusion_query_state(struct radv_cmd_buffer *cmd_buffer) db_count_control); } + radeon_end(); + cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_OCCLUSION_QUERY; } @@ -5028,8 +5202,10 @@ emit_prolog_regs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *v if (G_00B848_VGPRS(prolog->rsrc1) > G_00B848_VGPRS(rsrc1)) rsrc1 = (rsrc1 & C_00B848_VGPRS) | (prolog->rsrc1 & ~C_00B848_VGPRS); - radeon_set_sh_reg(cmd_buffer->cs, vs_shader->info.regs.pgm_lo, prolog->va >> 8); - radeon_set_sh_reg(cmd_buffer->cs, vs_shader->info.regs.pgm_rsrc1, rsrc1); + radeon_begin(cmd_buffer->cs); + + radeon_set_sh_reg(vs_shader->info.regs.pgm_lo, prolog->va >> 8); + radeon_set_sh_reg(vs_shader->info.regs.pgm_rsrc1, rsrc1); if (vs_shader->info.merged_shader_compiled_separately) { if (vs_shader->info.next_stage == MESA_SHADER_GEOMETRY) { @@ -5042,12 +5218,14 @@ emit_prolog_regs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *v lds_size = gs->info.gs_ring_info.lds_size; } - radeon_set_sh_reg(cmd_buffer->cs, vs_shader->info.regs.pgm_rsrc2, rsrc2 | S_00B22C_LDS_SIZE(lds_size)); + radeon_set_sh_reg(vs_shader->info.regs.pgm_rsrc2, rsrc2 | S_00B22C_LDS_SIZE(lds_size)); } else { - radeon_set_sh_reg(cmd_buffer->cs, vs_shader->info.regs.pgm_rsrc2, rsrc2); + radeon_set_sh_reg(vs_shader->info.regs.pgm_rsrc2, rsrc2); } } + radeon_end(); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, prolog->bo); } @@ -5092,7 +5270,9 @@ emit_prolog_inputs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader } const uint32_t vs_prolog_inputs_offset = radv_get_user_sgpr_loc(vs_shader, AC_UD_VS_PROLOG_INPUTS); - radeon_emit_64bit_pointer(cmd_buffer->cs, vs_prolog_inputs_offset, input_va); + radeon_begin(cmd_buffer->cs); + radeon_emit_64bit_pointer(vs_prolog_inputs_offset, input_va); + radeon_end(); } static void @@ -5178,13 +5358,15 @@ radv_emit_tess_domain_origin(struct radv_cmd_buffer *cmd_buffer) uint32_t vgt_tf_param = S_028B6C_TYPE(type) | S_028B6C_PARTITIONING(partitioning) | S_028B6C_TOPOLOGY(topology) | S_028B6C_DISTRIBUTION_MODE(pdev->tess_distribution_mode); + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { vgt_tf_param |= S_028AA4_TEMPORAL(gfx12_load_last_use_discard); - radeon_set_context_reg(cmd_buffer->cs, R_028AA4_VGT_TF_PARAM, vgt_tf_param); + radeon_set_context_reg(R_028AA4_VGT_TF_PARAM, vgt_tf_param); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028B6C_VGT_TF_PARAM, vgt_tf_param); + radeon_set_context_reg(R_028B6C_VGT_TF_PARAM, vgt_tf_param); } + radeon_end(); } static void @@ -5208,11 +5390,13 @@ radv_emit_alpha_to_coverage_enable(struct radv_cmd_buffer *cmd_buffer) db_alpha_to_mask |= S_028B70_ALPHA_TO_MASK_ENABLE(d->vk.ms.alpha_to_coverage_enable); + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_02807C_DB_ALPHA_TO_MASK, db_alpha_to_mask); + radeon_set_context_reg(R_02807C_DB_ALPHA_TO_MASK, db_alpha_to_mask); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028B70_DB_ALPHA_TO_MASK, db_alpha_to_mask); + radeon_set_context_reg(R_028B70_DB_ALPHA_TO_MASK, db_alpha_to_mask); } + radeon_end(); } static void @@ -5220,9 +5404,11 @@ radv_emit_sample_mask(struct radv_cmd_buffer *cmd_buffer) { const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C38_PA_SC_AA_MASK_X0Y0_X1Y0, 2); - radeon_emit(cmd_buffer->cs, d->vk.ms.sample_mask | ((uint32_t)d->vk.ms.sample_mask << 16)); - radeon_emit(cmd_buffer->cs, d->vk.ms.sample_mask | ((uint32_t)d->vk.ms.sample_mask << 16)); + radeon_begin(cmd_buffer->cs); + radeon_set_context_reg_seq(R_028C38_PA_SC_AA_MASK_X0Y0_X1Y0, 2); + radeon_emit(d->vk.ms.sample_mask | ((uint32_t)d->vk.ms.sample_mask << 16)); + radeon_emit(d->vk.ms.sample_mask | ((uint32_t)d->vk.ms.sample_mask << 16)); + radeon_end(); } static void @@ -5333,13 +5519,15 @@ radv_emit_color_blend(struct radv_cmd_buffer *cmd_buffer) } } - radeon_set_context_reg_seq(cmd_buffer->cs, R_028780_CB_BLEND0_CONTROL, MAX_RTS); - radeon_emit_array(cmd_buffer->cs, cb_blend_control, MAX_RTS); + radeon_begin(cmd_buffer->cs); + radeon_set_context_reg_seq(R_028780_CB_BLEND0_CONTROL, MAX_RTS); + radeon_emit_array(cb_blend_control, MAX_RTS); if (pdev->info.has_rbplus) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028760_SX_MRT0_BLEND_OPT, MAX_RTS); - radeon_emit_array(cmd_buffer->cs, sx_mrt_blend_opt, MAX_RTS); + radeon_set_context_reg_seq(R_028760_SX_MRT0_BLEND_OPT, MAX_RTS); + radeon_emit_array(sx_mrt_blend_opt, MAX_RTS); } + radeon_end(); } static struct radv_shader_part * @@ -5505,20 +5693,22 @@ radv_emit_msaa_state(struct radv_cmd_buffer *cmd_buffer) pa_sc_aa_config |= S_028BE0_COVERAGE_TO_SHADER_SELECT(pdev->info.gfx_level < GFX12 && ps && ps->info.ps.reads_fully_covered); - if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028C5C_PA_SC_SAMPLE_PROPERTIES, - S_028C5C_MAX_SAMPLE_DIST(max_sample_dist)); + radeon_begin(cmd_buffer->cs); - radeon_set_context_reg(cmd_buffer->cs, R_028078_DB_EQAA, db_eqaa); + if (pdev->info.gfx_level >= GFX12) { + radeon_set_context_reg(R_028C5C_PA_SC_SAMPLE_PROPERTIES, S_028C5C_MAX_SAMPLE_DIST(max_sample_dist)); + + radeon_set_context_reg(R_028078_DB_EQAA, db_eqaa); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028804_DB_EQAA, db_eqaa); + radeon_set_context_reg(R_028804_DB_EQAA, db_eqaa); } - radeon_set_context_reg(cmd_buffer->cs, R_028BE0_PA_SC_AA_CONFIG, pa_sc_aa_config); - radeon_set_context_reg( - cmd_buffer->cs, R_028A48_PA_SC_MODE_CNTL_0, - S_028A48_ALTERNATE_RBS_PER_TILE(pdev->info.gfx_level >= GFX9) | S_028A48_VPORT_SCISSOR_ENABLE(1) | - S_028A48_LINE_STIPPLE_ENABLE(d->vk.rs.line.stipple.enable) | S_028A48_MSAA_ENABLE(rasterization_samples > 1)); + radeon_set_context_reg(R_028BE0_PA_SC_AA_CONFIG, pa_sc_aa_config); + radeon_set_context_reg(R_028A48_PA_SC_MODE_CNTL_0, S_028A48_ALTERNATE_RBS_PER_TILE(pdev->info.gfx_level >= GFX9) | + S_028A48_VPORT_SCISSOR_ENABLE(1) | + S_028A48_LINE_STIPPLE_ENABLE(d->vk.rs.line.stipple.enable) | + S_028A48_MSAA_ENABLE(rasterization_samples > 1)); + radeon_end(); } static void @@ -6032,13 +6222,16 @@ radv_emit_streamout_buffers(struct radv_cmd_buffer *cmd_buffer, uint64_t va) if (!streamout_buffers_offset) return; - radeon_emit_32bit_pointer(cmd_buffer->cs, streamout_buffers_offset, va, &pdev->info); + radeon_begin(cmd_buffer->cs); + radeon_emit_32bit_pointer(streamout_buffers_offset, va, &pdev->info); if (cmd_buffer->state.gs_copy_shader) { streamout_buffers_offset = radv_get_user_sgpr_loc(cmd_buffer->state.gs_copy_shader, AC_UD_STREAMOUT_BUFFERS); if (streamout_buffers_offset) - radeon_emit_32bit_pointer(cmd_buffer->cs, streamout_buffers_offset, va, &pdev->info); + radeon_emit_32bit_pointer(streamout_buffers_offset, va, &pdev->info); } + + radeon_end(); } static void @@ -6055,7 +6248,9 @@ radv_emit_streamout_state(struct radv_cmd_buffer *cmd_buffer) if (!streamout_state_offset) return; - radeon_emit_32bit_pointer(cmd_buffer->cs, streamout_state_offset, so->state_va, &pdev->info); + radeon_begin(cmd_buffer->cs); + radeon_emit_32bit_pointer(streamout_state_offset, so->state_va, &pdev->info); + radeon_end(); } static void @@ -6153,7 +6348,9 @@ radv_flush_force_vrs_state(struct radv_cmd_buffer *cmd_buffer) if (cmd_buffer->state.last_vrs_rates != vrs_rates || cmd_buffer->state.last_force_vrs_rates_offset != force_vrs_rates_offset) { - radeon_set_sh_reg(cmd_buffer->cs, force_vrs_rates_offset, vrs_rates); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(force_vrs_rates_offset, vrs_rates); + radeon_end(); } cmd_buffer->state.last_vrs_rates = vrs_rates; @@ -6401,13 +6598,18 @@ radv_emit_ia_multi_vgt_param(struct radv_cmd_buffer *cmd_buffer, bool instanced_ patch_control_points, state->tess_num_patches); if (state->last_ia_multi_vgt_param != ia_multi_vgt_param) { + radeon_begin(cs); + if (gpu_info->gfx_level == GFX9) { - radeon_set_uconfig_reg_idx(&pdev->info, cs, R_030960_IA_MULTI_VGT_PARAM, 4, ia_multi_vgt_param); + radeon_set_uconfig_reg_idx(&pdev->info, R_030960_IA_MULTI_VGT_PARAM, 4, ia_multi_vgt_param); } else if (gpu_info->gfx_level >= GFX7) { - radeon_set_context_reg_idx(cs, R_028AA8_IA_MULTI_VGT_PARAM, 1, ia_multi_vgt_param); + radeon_set_context_reg_idx(R_028AA8_IA_MULTI_VGT_PARAM, 1, ia_multi_vgt_param); } else { - radeon_set_context_reg(cs, R_028AA8_IA_MULTI_VGT_PARAM, ia_multi_vgt_param); + radeon_set_context_reg(R_028AA8_IA_MULTI_VGT_PARAM, ia_multi_vgt_param); } + + radeon_end(); + state->last_ia_multi_vgt_param = ia_multi_vgt_param; } } @@ -6446,7 +6648,10 @@ gfx10_emit_ge_cntl(struct radv_cmd_buffer *cmd_buffer) S_03096C_BREAK_WAVE_AT_EOI(break_wave_at_eoi); if (state->last_ge_cntl != ge_cntl) { - radeon_set_uconfig_reg(cmd_buffer->cs, R_03096C_GE_CNTL, ge_cntl); + radeon_begin(cmd_buffer->cs); + radeon_set_uconfig_reg(R_03096C_GE_CNTL, ge_cntl); + radeon_end(); + state->last_ge_cntl = ge_cntl; } } @@ -6487,13 +6692,17 @@ radv_emit_draw_registers(struct radv_cmd_buffer *cmd_buffer, const struct radv_d disable_instance_packing != G_028A7C_DISABLE_INSTANCE_PACKING(state->last_index_type)))) { uint32_t index_type = state->index_type | S_028A7C_DISABLE_INSTANCE_PACKING(disable_instance_packing); + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX9) { - radeon_set_uconfig_reg_idx(&pdev->info, cs, R_03090C_VGT_INDEX_TYPE, 2, index_type); + radeon_set_uconfig_reg_idx(&pdev->info, R_03090C_VGT_INDEX_TYPE, 2, index_type); } else { - radeon_emit(cs, PKT3(PKT3_INDEX_TYPE, 0, 0)); - radeon_emit(cs, index_type); + radeon_emit(PKT3(PKT3_INDEX_TYPE, 0, 0)); + radeon_emit(index_type); } + radeon_end(); + state->last_index_type = index_type; } } @@ -7390,15 +7599,20 @@ radv_emit_compute_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_compu radv_get_user_sgpr_loc(rt_prolog, AC_UD_CS_RAY_DYNAMIC_CALLABLE_STACK_BASE); if (ray_dynamic_callback_stack_base_offset) { const struct radv_shader_info *cs_info = &rt_prolog->info; - radeon_set_sh_reg(cmd_buffer->cs, ray_dynamic_callback_stack_base_offset, + + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(ray_dynamic_callback_stack_base_offset, rt_prolog->config.scratch_bytes_per_wave / cs_info->wave_size); + radeon_end(); } const uint32_t traversal_shader_addr_offset = radv_get_user_sgpr_loc(rt_prolog, AC_UD_CS_TRAVERSAL_SHADER_ADDR); struct radv_shader *traversal_shader = cmd_buffer->state.shaders[MESA_SHADER_INTERSECTION]; if (traversal_shader_addr_offset && traversal_shader) { uint64_t traversal_va = traversal_shader->va | radv_rt_priority_traversal; - radeon_emit_64bit_pointer(cmd_buffer->cs, traversal_shader_addr_offset, traversal_va); + radeon_begin(cmd_buffer->cs); + radeon_emit_64bit_pointer(traversal_shader_addr_offset, traversal_va); + radeon_end(); } } @@ -9379,19 +9593,19 @@ radv_CmdBeginRendering(VkCommandBuffer commandBuffer, const VkRenderingInfo *pRe const uint32_t maxy = miny + render->area.extent.height; radeon_check_space(device->ws, cmd_buffer->cs, 6); + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028204_PA_SC_WINDOW_SCISSOR_TL, - S_028204_TL_X(minx) | S_028204_TL_Y_GFX12(miny)); - radeon_set_context_reg(cmd_buffer->cs, R_028208_PA_SC_WINDOW_SCISSOR_BR, + radeon_set_context_reg(R_028204_PA_SC_WINDOW_SCISSOR_TL, S_028204_TL_X(minx) | S_028204_TL_Y_GFX12(miny)); + radeon_set_context_reg(R_028208_PA_SC_WINDOW_SCISSOR_BR, S_028208_BR_X(maxx - 1) | S_028208_BR_Y(maxy - 1)); /* inclusive */ } else { - radeon_set_context_reg(cmd_buffer->cs, R_028204_PA_SC_WINDOW_SCISSOR_TL, - S_028204_TL_X(minx) | S_028204_TL_Y_GFX6(miny)); - radeon_set_context_reg(cmd_buffer->cs, R_028208_PA_SC_WINDOW_SCISSOR_BR, - S_028208_BR_X(maxx) | S_028208_BR_Y(maxy)); + radeon_set_context_reg(R_028204_PA_SC_WINDOW_SCISSOR_TL, S_028204_TL_X(minx) | S_028204_TL_Y_GFX6(miny)); + radeon_set_context_reg(R_028208_PA_SC_WINDOW_SCISSOR_BR, S_028208_BR_X(maxx) | S_028208_BR_Y(maxy)); } + radeon_end(); + radv_emit_fb_mip_change_flush(cmd_buffer); if (!(pRenderingInfo->flags & VK_RENDERING_RESUMING_BIT)) @@ -9417,7 +9631,9 @@ radv_emit_view_index_per_stage(struct radeon_cmdbuf *cs, const struct radv_shade if (!view_index_offset) return; - radeon_set_sh_reg(cs, view_index_offset, index); + radeon_begin(cs); + radeon_set_sh_reg(view_index_offset, index); + radeon_end(); } static void @@ -9439,13 +9655,15 @@ static void radv_emit_copy_data_imm(const struct radv_physical_device *pdev, struct radeon_cmdbuf *cs, uint32_t src_imm, uint64_t dst_va) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM | - (pdev->info.gfx_level == GFX6 ? COPY_DATA_ENGINE_PFP : 0)); - radeon_emit(cs, src_imm); - radeon_emit(cs, 0); - radeon_emit(cs, dst_va); - radeon_emit(cs, dst_va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM | + (pdev->info.gfx_level == GFX6 ? COPY_DATA_ENGINE_PFP : 0)); + radeon_emit(src_imm); + radeon_emit(0); + radeon_emit(dst_va); + radeon_emit(dst_va >> 32); + radeon_end(); } /** @@ -9491,9 +9709,11 @@ radv_cs_emit_compute_predication(const struct radv_device *device, struct radv_c static void radv_cs_emit_draw_packet(struct radv_cmd_buffer *cmd_buffer, uint32_t vertex_count, uint32_t use_opaque) { - radeon_emit(cmd_buffer->cs, PKT3(PKT3_DRAW_INDEX_AUTO, 1, cmd_buffer->state.predicating)); - radeon_emit(cmd_buffer->cs, vertex_count); - radeon_emit(cmd_buffer->cs, V_0287F0_DI_SRC_SEL_AUTO_INDEX | use_opaque); + radeon_begin(cmd_buffer->cs); + radeon_emit(PKT3(PKT3_DRAW_INDEX_AUTO, 1, cmd_buffer->state.predicating)); + radeon_emit(vertex_count); + radeon_emit(V_0287F0_DI_SRC_SEL_AUTO_INDEX | use_opaque); + radeon_end(); } /** @@ -9507,16 +9727,18 @@ static void radv_cs_emit_draw_indexed_packet(struct radv_cmd_buffer *cmd_buffer, uint64_t index_va, uint32_t max_index_count, uint32_t index_count, bool not_eop) { - radeon_emit(cmd_buffer->cs, PKT3(PKT3_DRAW_INDEX_2, 4, cmd_buffer->state.predicating)); - radeon_emit(cmd_buffer->cs, max_index_count); - radeon_emit(cmd_buffer->cs, index_va); - radeon_emit(cmd_buffer->cs, index_va >> 32); - radeon_emit(cmd_buffer->cs, index_count); + radeon_begin(cmd_buffer->cs); + radeon_emit(PKT3(PKT3_DRAW_INDEX_2, 4, cmd_buffer->state.predicating)); + radeon_emit(max_index_count); + radeon_emit(index_va); + radeon_emit(index_va >> 32); + radeon_emit(index_count); /* NOT_EOP allows merging multiple draws into 1 wave, but only user VGPRs * can be changed between draws and GS fast launch must be disabled. * NOT_EOP doesn't work on gfx6-gfx9 and gfx12. */ - radeon_emit(cmd_buffer->cs, V_0287F0_DI_SRC_SEL_DMA | S_0287F0_NOT_EOP(not_eop)); + radeon_emit(V_0287F0_DI_SRC_SEL_DMA | S_0287F0_NOT_EOP(not_eop)); + radeon_end(); } /* MUST inline this function to avoid massive perf loss in drawoverhead */ @@ -9544,25 +9766,29 @@ radv_cs_emit_indirect_draw_packet(struct radv_cmd_buffer *cmd_buffer, bool index if (draw_id_enable) draw_id_reg = ((base_reg + 4) - SI_SH_REG_OFFSET) >> 2; + radeon_begin(cs); + if (draw_count == 1 && !count_va && !draw_id_enable) { - radeon_emit(cs, PKT3(indexed ? PKT3_DRAW_INDEX_INDIRECT : PKT3_DRAW_INDIRECT, 3, predicating)); - radeon_emit(cs, 0); - radeon_emit(cs, vertex_offset_reg); - radeon_emit(cs, start_instance_reg); - radeon_emit(cs, di_src_sel); + radeon_emit(PKT3(indexed ? PKT3_DRAW_INDEX_INDIRECT : PKT3_DRAW_INDIRECT, 3, predicating)); + radeon_emit(0); + radeon_emit(vertex_offset_reg); + radeon_emit(start_instance_reg); + radeon_emit(di_src_sel); } else { - radeon_emit(cs, PKT3(indexed ? PKT3_DRAW_INDEX_INDIRECT_MULTI : PKT3_DRAW_INDIRECT_MULTI, 8, predicating)); - radeon_emit(cs, 0); - radeon_emit(cs, vertex_offset_reg); - radeon_emit(cs, start_instance_reg); - radeon_emit(cs, draw_id_reg | S_2C3_DRAW_INDEX_ENABLE(draw_id_enable) | S_2C3_COUNT_INDIRECT_ENABLE(!!count_va)); - radeon_emit(cs, draw_count); /* count */ - radeon_emit(cs, count_va); /* count_addr */ - radeon_emit(cs, count_va >> 32); - radeon_emit(cs, stride); /* stride */ - radeon_emit(cs, di_src_sel); + radeon_emit(PKT3(indexed ? PKT3_DRAW_INDEX_INDIRECT_MULTI : PKT3_DRAW_INDIRECT_MULTI, 8, predicating)); + radeon_emit(0); + radeon_emit(vertex_offset_reg); + radeon_emit(start_instance_reg); + radeon_emit(draw_id_reg | S_2C3_DRAW_INDEX_ENABLE(draw_id_enable) | S_2C3_COUNT_INDIRECT_ENABLE(!!count_va)); + radeon_emit(draw_count); /* count */ + radeon_emit(count_va); /* count_addr */ + radeon_emit(count_va >> 32); + radeon_emit(stride); /* stride */ + radeon_emit(di_src_sel); } + radeon_end(); + cmd_buffer->state.uses_draw_indirect = true; } @@ -9591,19 +9817,21 @@ radv_cs_emit_indirect_mesh_draw_packet(struct radv_cmd_buffer *cmd_buffer, uint3 uint32_t mode1_enable = !pdev->mesh_fast_launch_2; - radeon_emit(cs, PKT3(PKT3_DISPATCH_MESH_INDIRECT_MULTI, 7, predicating) | PKT3_RESET_FILTER_CAM_S(1)); - radeon_emit(cs, 0); /* data_offset */ - radeon_emit(cs, S_4C1_XYZ_DIM_REG(xyz_dim_reg) | S_4C1_DRAW_INDEX_REG(draw_id_reg)); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_DISPATCH_MESH_INDIRECT_MULTI, 7, predicating) | PKT3_RESET_FILTER_CAM_S(1)); + radeon_emit(0); /* data_offset */ + radeon_emit(S_4C1_XYZ_DIM_REG(xyz_dim_reg) | S_4C1_DRAW_INDEX_REG(draw_id_reg)); if (pdev->info.gfx_level >= GFX11) - radeon_emit(cs, S_4C2_DRAW_INDEX_ENABLE(draw_id_enable) | S_4C2_COUNT_INDIRECT_ENABLE(!!count_va) | - S_4C2_XYZ_DIM_ENABLE(xyz_dim_enable) | S_4C2_MODE1_ENABLE(mode1_enable)); + radeon_emit(S_4C2_DRAW_INDEX_ENABLE(draw_id_enable) | S_4C2_COUNT_INDIRECT_ENABLE(!!count_va) | + S_4C2_XYZ_DIM_ENABLE(xyz_dim_enable) | S_4C2_MODE1_ENABLE(mode1_enable)); else - radeon_emit(cs, S_4C2_DRAW_INDEX_ENABLE(draw_id_enable) | S_4C2_COUNT_INDIRECT_ENABLE(!!count_va)); - radeon_emit(cs, draw_count); - radeon_emit(cs, count_va); - radeon_emit(cs, count_va >> 32); - radeon_emit(cs, stride); - radeon_emit(cs, V_0287F0_DI_SRC_SEL_AUTO_INDEX); + radeon_emit(S_4C2_DRAW_INDEX_ENABLE(draw_id_enable) | S_4C2_COUNT_INDIRECT_ENABLE(!!count_va)); + radeon_emit(draw_count); + radeon_emit(count_va); + radeon_emit(count_va >> 32); + radeon_emit(stride); + radeon_emit(V_0287F0_DI_SRC_SEL_AUTO_INDEX); + radeon_end(); } ALWAYS_INLINE static void @@ -9617,12 +9845,14 @@ radv_cs_emit_dispatch_taskmesh_direct_ace_packet(const struct radv_device *devic device->dispatch_initiator_task | S_00B800_CS_W32_EN(task_shader->info.wave_size == 32); const uint32_t ring_entry_reg = radv_get_user_sgpr(task_shader, AC_UD_TASK_RING_ENTRY); - radeon_emit(ace_cs, PKT3(PKT3_DISPATCH_TASKMESH_DIRECT_ACE, 4, predicating) | PKT3_SHADER_TYPE_S(1)); - radeon_emit(ace_cs, x); - radeon_emit(ace_cs, y); - radeon_emit(ace_cs, z); - radeon_emit(ace_cs, dispatch_initiator); - radeon_emit(ace_cs, ring_entry_reg & 0xFFFF); + radeon_begin(ace_cs); + radeon_emit(PKT3(PKT3_DISPATCH_TASKMESH_DIRECT_ACE, 4, predicating) | PKT3_SHADER_TYPE_S(1)); + radeon_emit(x); + radeon_emit(y); + radeon_emit(z); + radeon_emit(dispatch_initiator); + radeon_emit(ring_entry_reg & 0xFFFF); + radeon_end(); } ALWAYS_INLINE static void @@ -9642,18 +9872,20 @@ radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(const struct radv_devic const uint32_t xyz_dim_reg = radv_get_user_sgpr(task_shader, AC_UD_CS_GRID_SIZE); const uint32_t draw_id_reg = radv_get_user_sgpr(task_shader, AC_UD_CS_TASK_DRAW_ID); - radeon_emit(ace_cs, PKT3(PKT3_DISPATCH_TASKMESH_INDIRECT_MULTI_ACE, 9, 0) | PKT3_SHADER_TYPE_S(1)); - radeon_emit(ace_cs, data_va); - radeon_emit(ace_cs, data_va >> 32); - radeon_emit(ace_cs, S_AD2_RING_ENTRY_REG(ring_entry_reg)); - radeon_emit(ace_cs, S_AD3_COUNT_INDIRECT_ENABLE(!!count_va) | S_AD3_DRAW_INDEX_ENABLE(!!draw_id_reg) | - S_AD3_XYZ_DIM_ENABLE(!!xyz_dim_reg) | S_AD3_DRAW_INDEX_REG(draw_id_reg)); - radeon_emit(ace_cs, S_AD4_XYZ_DIM_REG(xyz_dim_reg)); - radeon_emit(ace_cs, draw_count); - radeon_emit(ace_cs, count_va); - radeon_emit(ace_cs, count_va >> 32); - radeon_emit(ace_cs, stride); - radeon_emit(ace_cs, dispatch_initiator); + radeon_begin(ace_cs); + radeon_emit(PKT3(PKT3_DISPATCH_TASKMESH_INDIRECT_MULTI_ACE, 9, 0) | PKT3_SHADER_TYPE_S(1)); + radeon_emit(data_va); + radeon_emit(data_va >> 32); + radeon_emit(S_AD2_RING_ENTRY_REG(ring_entry_reg)); + radeon_emit(S_AD3_COUNT_INDIRECT_ENABLE(!!count_va) | S_AD3_DRAW_INDEX_ENABLE(!!draw_id_reg) | + S_AD3_XYZ_DIM_ENABLE(!!xyz_dim_reg) | S_AD3_DRAW_INDEX_REG(draw_id_reg)); + radeon_emit(S_AD4_XYZ_DIM_REG(xyz_dim_reg)); + radeon_emit(draw_count); + radeon_emit(count_va); + radeon_emit(count_va >> 32); + radeon_emit(stride); + radeon_emit(dispatch_initiator); + radeon_end(); } ALWAYS_INLINE static void @@ -9672,14 +9904,16 @@ radv_cs_emit_dispatch_taskmesh_gfx_packet(const struct radv_device *device, cons uint32_t linear_dispatch_en = cmd_state->shaders[MESA_SHADER_TASK]->info.cs.linear_taskmesh_dispatch; const bool sqtt_en = !!device->sqtt.bo; - radeon_emit(cs, PKT3(PKT3_DISPATCH_TASKMESH_GFX, 2, predicating) | PKT3_RESET_FILTER_CAM_S(1)); - radeon_emit(cs, S_4D0_RING_ENTRY_REG(ring_entry_reg) | S_4D0_XYZ_DIM_REG(xyz_dim_reg)); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_DISPATCH_TASKMESH_GFX, 2, predicating) | PKT3_RESET_FILTER_CAM_S(1)); + radeon_emit(S_4D0_RING_ENTRY_REG(ring_entry_reg) | S_4D0_XYZ_DIM_REG(xyz_dim_reg)); if (pdev->info.gfx_level >= GFX11) - radeon_emit(cs, S_4D1_XYZ_DIM_ENABLE(xyz_dim_en) | S_4D1_MODE1_ENABLE(mode1_en) | - S_4D1_LINEAR_DISPATCH_ENABLE(linear_dispatch_en) | S_4D1_THREAD_TRACE_MARKER_ENABLE(sqtt_en)); + radeon_emit(S_4D1_XYZ_DIM_ENABLE(xyz_dim_en) | S_4D1_MODE1_ENABLE(mode1_en) | + S_4D1_LINEAR_DISPATCH_ENABLE(linear_dispatch_en) | S_4D1_THREAD_TRACE_MARKER_ENABLE(sqtt_en)); else - radeon_emit(cs, S_4D1_THREAD_TRACE_MARKER_ENABLE(sqtt_en)); - radeon_emit(cs, V_0287F0_DI_SRC_SEL_AUTO_INDEX); + radeon_emit(S_4D1_THREAD_TRACE_MARKER_ENABLE(sqtt_en)); + radeon_emit(V_0287F0_DI_SRC_SEL_AUTO_INDEX); + radeon_end(); } ALWAYS_INLINE static void @@ -9691,19 +9925,22 @@ radv_emit_userdata_vertex_internal(struct radv_cmd_buffer *cmd_buffer, const str const bool uses_baseinstance = state->uses_baseinstance; const bool uses_drawid = state->uses_drawid; - radeon_set_sh_reg_seq(cs, state->vtx_base_sgpr, state->vtx_emit_num); + radeon_begin(cs); + radeon_set_sh_reg_seq(state->vtx_base_sgpr, state->vtx_emit_num); - radeon_emit(cs, vertex_offset); + radeon_emit(vertex_offset); state->last_vertex_offset_valid = true; state->last_vertex_offset = vertex_offset; if (uses_drawid) { - radeon_emit(cs, 0); + radeon_emit(0); state->last_drawid = 0; } if (uses_baseinstance) { - radeon_emit(cs, info->first_instance); + radeon_emit(info->first_instance); state->last_first_instance = info->first_instance; } + + radeon_end(); } ALWAYS_INLINE static void @@ -9725,12 +9962,15 @@ radv_emit_userdata_vertex_drawid(struct radv_cmd_buffer *cmd_buffer, uint32_t ve { struct radv_cmd_state *state = &cmd_buffer->state; struct radeon_cmdbuf *cs = cmd_buffer->cs; - radeon_set_sh_reg_seq(cs, state->vtx_base_sgpr, 1 + !!drawid); - radeon_emit(cs, vertex_offset); + + radeon_begin(cs); + radeon_set_sh_reg_seq(state->vtx_base_sgpr, 1 + !!drawid); + radeon_emit(vertex_offset); state->last_vertex_offset_valid = true; state->last_vertex_offset = vertex_offset; if (drawid) - radeon_emit(cs, drawid); + radeon_emit(drawid); + radeon_end(); } ALWAYS_INLINE static void @@ -9745,16 +9985,18 @@ radv_emit_userdata_mesh(struct radv_cmd_buffer *cmd_buffer, const uint32_t x, co if (!uses_drawid && !uses_grid_size) return; - radeon_set_sh_reg_seq(cs, state->vtx_base_sgpr, state->vtx_emit_num); + radeon_begin(cs); + radeon_set_sh_reg_seq(state->vtx_base_sgpr, state->vtx_emit_num); if (uses_grid_size) { - radeon_emit(cs, x); - radeon_emit(cs, y); - radeon_emit(cs, z); + radeon_emit(x); + radeon_emit(y); + radeon_emit(z); } if (uses_drawid) { - radeon_emit(cs, 0); + radeon_emit(0); state->last_drawid = 0; } + radeon_end(); } ALWAYS_INLINE static void @@ -9766,17 +10008,21 @@ radv_emit_userdata_task(const struct radv_cmd_state *cmd_state, struct radeon_cm const uint32_t xyz_offset = radv_get_user_sgpr_loc(task_shader, AC_UD_CS_GRID_SIZE); const uint32_t draw_id_offset = radv_get_user_sgpr_loc(task_shader, AC_UD_CS_TASK_DRAW_ID); + radeon_begin(ace_cs); + if (xyz_offset) { - radeon_set_sh_reg_seq(ace_cs, xyz_offset, 3); - radeon_emit(ace_cs, x); - radeon_emit(ace_cs, y); - radeon_emit(ace_cs, z); + radeon_set_sh_reg_seq(xyz_offset, 3); + radeon_emit(x); + radeon_emit(y); + radeon_emit(z); } if (draw_id_offset) { - radeon_set_sh_reg_seq(ace_cs, draw_id_offset, 1); - radeon_emit(ace_cs, 0); + radeon_set_sh_reg_seq(draw_id_offset, 1); + radeon_emit(0); } + + radeon_end(); } ALWAYS_INLINE static void @@ -9805,8 +10051,11 @@ radv_emit_draw_packets_indexed(struct radv_cmd_buffer *cmd_buffer, const struct if (!remaining_indexes && pdev->info.has_zero_index_buffer_bug) radv_handle_zero_index_buffer_bug(cmd_buffer, &index_va, &remaining_indexes); - if (i > 0) - radeon_set_sh_reg(cs, state->vtx_base_sgpr + sizeof(uint32_t), i); + if (i > 0) { + radeon_begin(cs); + radeon_set_sh_reg(state->vtx_base_sgpr + sizeof(uint32_t), i); + radeon_end(); + } if (!state->render.view_mask) { radv_cs_emit_draw_indexed_packet(cmd_buffer, index_va, remaining_indexes, draw->indexCount, false); @@ -9829,10 +10078,13 @@ radv_emit_draw_packets_indexed(struct radv_cmd_buffer *cmd_buffer, const struct if (i > 0) { assert(state->last_vertex_offset_valid); - if (state->last_vertex_offset != draw->vertexOffset) + if (state->last_vertex_offset != draw->vertexOffset) { radv_emit_userdata_vertex_drawid(cmd_buffer, draw->vertexOffset, i); - else - radeon_set_sh_reg(cs, state->vtx_base_sgpr + sizeof(uint32_t), i); + } else { + radeon_begin(cs); + radeon_set_sh_reg(state->vtx_base_sgpr + sizeof(uint32_t), i); + radeon_end(); + } } else radv_emit_userdata_vertex(cmd_buffer, info, draw->vertexOffset); @@ -9954,11 +10206,13 @@ radv_emit_direct_draw_packets(struct radv_cmd_buffer *cmd_buffer, const struct r static void radv_cs_emit_mesh_dispatch_packet(struct radv_cmd_buffer *cmd_buffer, uint32_t x, uint32_t y, uint32_t z) { - radeon_emit(cmd_buffer->cs, PKT3(PKT3_DISPATCH_MESH_DIRECT, 3, cmd_buffer->state.predicating)); - radeon_emit(cmd_buffer->cs, x); - radeon_emit(cmd_buffer->cs, y); - radeon_emit(cmd_buffer->cs, z); - radeon_emit(cmd_buffer->cs, S_0287F0_SOURCE_SELECT(V_0287F0_DI_SRC_SEL_AUTO_INDEX)); + radeon_begin(cmd_buffer->cs); + radeon_emit(PKT3(PKT3_DISPATCH_MESH_DIRECT, 3, cmd_buffer->state.predicating)); + radeon_emit(x); + radeon_emit(y); + radeon_emit(z); + radeon_emit(S_0287F0_SOURCE_SELECT(V_0287F0_DI_SRC_SEL_AUTO_INDEX)); + radeon_end(); } ALWAYS_INLINE static void @@ -9995,10 +10249,12 @@ radv_emit_direct_mesh_draw_packet(struct radv_cmd_buffer *cmd_buffer, uint32_t x static void radv_emit_indirect_buffer(struct radeon_cmdbuf *cs, uint64_t va, bool is_compute) { - radeon_emit(cs, PKT3(PKT3_SET_BASE, 2, 0) | (is_compute ? PKT3_SHADER_TYPE_S(1) : 0)); - radeon_emit(cs, 1); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_SET_BASE, 2, 0) | (is_compute ? PKT3_SHADER_TYPE_S(1) : 0)); + radeon_emit(1); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_end(); } ALWAYS_INLINE static void @@ -10012,8 +10268,11 @@ radv_emit_indirect_mesh_draw_packets(struct radv_cmd_buffer *cmd_buffer, const s if (state->uses_drawid) { const struct radv_shader *mesh_shader = state->shaders[MESA_SHADER_MESH]; unsigned reg = state->vtx_base_sgpr + (mesh_shader->info.cs.uses_grid_size ? 12 : 0); - radeon_set_sh_reg_seq(cs, reg, 1); - radeon_emit(cs, 0); + + radeon_begin(cs); + radeon_set_sh_reg_seq(reg, 1); + radeon_emit(0); + radeon_end(); } if (!state->render.view_mask) { @@ -10075,13 +10334,14 @@ radv_emit_indirect_taskmesh_draw_packets(const struct radv_device *device, struc * - When count != 0, write 0 to the workaround BO and execute the indirect dispatch * - When workaround BO != 0 (count was 0), execute an empty direct dispatch */ - radeon_emit(ace_cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(ace_cs, - COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(ace_cs, 1); - radeon_emit(ace_cs, 0); - radeon_emit(ace_cs, workaround_cond_va); - radeon_emit(ace_cs, workaround_cond_va >> 32); + radeon_begin(ace_cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); + radeon_emit(1); + radeon_emit(0); + radeon_emit(workaround_cond_va); + radeon_emit(workaround_cond_va >> 32); + radeon_end(); /* 2x COND_EXEC + 1x COPY_DATA + Nx DISPATCH_TASKMESH_DIRECT_ACE */ ace_predication_size += 2 * 5 + 6 + 6 * num_views; @@ -10094,13 +10354,14 @@ radv_emit_indirect_taskmesh_draw_packets(const struct radv_device *device, struc radv_emit_cond_exec(device, ace_cs, info->count_va, 6 + 11 * num_views /* 1x COPY_DATA + Nx DISPATCH_TASKMESH_INDIRECT_MULTI_ACE */); - radeon_emit(ace_cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(ace_cs, - COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(ace_cs, 0); - radeon_emit(ace_cs, 0); - radeon_emit(ace_cs, workaround_cond_va); - radeon_emit(ace_cs, workaround_cond_va >> 32); + radeon_begin(ace_cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); + radeon_emit(0); + radeon_emit(0); + radeon_emit(workaround_cond_va); + radeon_emit(workaround_cond_va >> 32); + radeon_end(); } if (!view_mask) { @@ -10281,6 +10542,8 @@ radv_emit_ngg_culling_state(struct radv_cmd_buffer *cmd_buffer) /* Get current culling settings. */ uint32_t nggc_settings = radv_get_ngg_culling_settings(cmd_buffer, vp_y_inverted); + radeon_begin(cmd_buffer->cs); + if ((cmd_buffer->state.dirty & RADV_CMD_DIRTY_PIPELINE) || (cmd_buffer->state.dirty_dynamic & (RADV_DYNAMIC_VIEWPORT | RADV_DYNAMIC_RASTERIZATION_SAMPLES))) { /* Correction for inverted Y */ @@ -10297,13 +10560,15 @@ radv_emit_ngg_culling_state(struct radv_cmd_buffer *cmd_buffer) uint32_t vp_reg_values[4] = {fui(vp_scale[0]), fui(vp_scale[1]), fui(vp_translate[0]), fui(vp_translate[1])}; const uint32_t ngg_viewport_offset = radv_get_user_sgpr_loc(last_vgt_shader, AC_UD_NGG_VIEWPORT); - radeon_set_sh_reg_seq(cmd_buffer->cs, ngg_viewport_offset, 4); - radeon_emit_array(cmd_buffer->cs, vp_reg_values, 4); + + radeon_set_sh_reg_seq(ngg_viewport_offset, 4); + radeon_emit_array(vp_reg_values, 4); } const uint32_t ngg_culling_settings_offset = radv_get_user_sgpr_loc(last_vgt_shader, AC_UD_NGG_CULLING_SETTINGS); - radeon_set_sh_reg(cmd_buffer->cs, ngg_culling_settings_offset, nggc_settings); + radeon_set_sh_reg(ngg_culling_settings_offset, nggc_settings); + radeon_end(); } static void @@ -10327,7 +10592,9 @@ radv_emit_fs_state(struct radv_cmd_buffer *cmd_buffer) SET_SGPR_FIELD(PS_STATE_LINE_RAST_MODE, radv_get_line_mode(cmd_buffer)) | SET_SGPR_FIELD(PS_STATE_RAST_PRIM, rast_prim); - radeon_set_sh_reg(cmd_buffer->cs, ps_state_offset, ps_state); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(ps_state_offset, ps_state); + radeon_end(); } static uint32_t @@ -10404,13 +10671,16 @@ radv_emit_ngg_state(struct radv_cmd_buffer *cmd_buffer) SET_SGPR_FIELD(NGG_STATE_PROVOKING_VTX, radv_get_ngg_state_provoking_vtx(cmd_buffer)) | SET_SGPR_FIELD(NGG_STATE_QUERY, radv_get_ngg_state_query(cmd_buffer)); - radeon_set_sh_reg(cmd_buffer->cs, ngg_state_offset, ngg_state); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(ngg_state_offset, ngg_state); if (pdev->info.gfx_level >= GFX12) { const uint32_t ngg_query_buf_va_offset = radv_get_user_sgpr_loc(last_vgt_shader, AC_UD_NGG_QUERY_BUF_VA); if (ngg_query_buf_va_offset) - radeon_set_sh_reg(cmd_buffer->cs, ngg_query_buf_va_offset, cmd_buffer->state.shader_query_buf_va); + radeon_set_sh_reg(ngg_query_buf_va_offset, cmd_buffer->state.shader_query_buf_va); } + + radeon_end(); } static void @@ -10437,7 +10707,9 @@ radv_emit_task_state(struct radv_cmd_buffer *cmd_buffer) (cmd_buffer->state.inherited_pipeline_statistics & VK_QUERY_PIPELINE_STATISTIC_TASK_SHADER_INVOCATIONS_BIT_EXT)) shader_query_state |= radv_shader_query_pipeline_stat; - radeon_set_sh_reg(cmd_buffer->gang.cs, task_state_offset, shader_query_state); + radeon_begin(cmd_buffer->gang.cs); + radeon_set_sh_reg(task_state_offset, shader_query_state); + radeon_end(); } static void @@ -10504,9 +10776,11 @@ radv_emit_db_shader_control(struct radv_cmd_buffer *cmd_buffer) db_shader_control |= S_02880C_POPS_OVERLAP_NUM_SAMPLES(util_logbase2(rasterization_samples)); if (gpu_info->has_pops_missed_overlap_bug) { - radeon_set_context_reg(cmd_buffer->cs, R_028060_DB_DFSM_CONTROL, + radeon_begin(cmd_buffer->cs); + radeon_set_context_reg(R_028060_DB_DFSM_CONTROL, S_028060_PUNCHOUT_MODE(V_028060_FORCE_OFF) | S_028060_POPS_DRAIN_PS_ON_OVERLAP(rasterization_samples >= 8)); + radeon_end(); } } } else if (gpu_info->has_export_conflict_bug && rasterization_samples == 1) { @@ -10518,6 +10792,8 @@ radv_emit_db_shader_control(struct radv_cmd_buffer *cmd_buffer) } } + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { radeon_opt_set_context_reg(cmd_buffer, R_02806C_DB_SHADER_CONTROL, RADV_TRACKED_DB_SHADER_CONTROL, db_shader_control); @@ -10532,6 +10808,8 @@ radv_emit_db_shader_control(struct radv_cmd_buffer *cmd_buffer) db_shader_control); } + radeon_end(); + cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_DB_SHADER_CONTROL; } @@ -10546,22 +10824,24 @@ radv_emit_streamout_enable_state(struct radv_cmd_buffer *cmd_buffer) assert(!pdev->use_ngg_streamout); + radeon_begin(cmd_buffer->cs); + if (streamout_enabled && cmd_buffer->state.last_vgt_shader) { const struct radv_shader_info *info = &cmd_buffer->state.last_vgt_shader->info; enabled_stream_buffers_mask = info->so.enabled_stream_buffers_mask; u_foreach_bit (i, so->enabled_mask) { - radeon_set_context_reg(cmd_buffer->cs, R_028AD4_VGT_STRMOUT_VTX_STRIDE_0 + 16 * i, info->so.strides[i]); + radeon_set_context_reg(R_028AD4_VGT_STRMOUT_VTX_STRIDE_0 + 16 * i, info->so.strides[i]); } } - radeon_set_context_reg_seq(cmd_buffer->cs, R_028B94_VGT_STRMOUT_CONFIG, 2); - radeon_emit(cmd_buffer->cs, S_028B94_STREAMOUT_0_EN(streamout_enabled) | S_028B94_RAST_STREAM(0) | - S_028B94_STREAMOUT_1_EN(streamout_enabled) | - S_028B94_STREAMOUT_2_EN(streamout_enabled) | - S_028B94_STREAMOUT_3_EN(streamout_enabled)); - radeon_emit(cmd_buffer->cs, so->hw_enabled_mask & enabled_stream_buffers_mask); + radeon_set_context_reg_seq(R_028B94_VGT_STRMOUT_CONFIG, 2); + radeon_emit(S_028B94_STREAMOUT_0_EN(streamout_enabled) | S_028B94_RAST_STREAM(0) | + S_028B94_STREAMOUT_1_EN(streamout_enabled) | S_028B94_STREAMOUT_2_EN(streamout_enabled) | + S_028B94_STREAMOUT_3_EN(streamout_enabled)); + radeon_emit(so->hw_enabled_mask & enabled_stream_buffers_mask); + radeon_end(); cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_STREAMOUT_ENABLE; } @@ -10605,18 +10885,22 @@ radv_emit_fragment_output_state(struct radv_cmd_buffer *cmd_buffer) uint32_t col_format_compacted = radv_compact_spi_shader_col_format(cmd_buffer->state.spi_shader_col_format); + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028854_CB_SHADER_MASK, cmd_buffer->state.cb_shader_mask); - radeon_set_context_reg_seq(cmd_buffer->cs, R_028650_SPI_SHADER_Z_FORMAT, 2); - radeon_emit(cmd_buffer->cs, cmd_buffer->state.spi_shader_z_format); - radeon_emit(cmd_buffer->cs, col_format_compacted); /* SPI_SHADER_COL_FORMAT */ + radeon_set_context_reg(R_028854_CB_SHADER_MASK, cmd_buffer->state.cb_shader_mask); + radeon_set_context_reg_seq(R_028650_SPI_SHADER_Z_FORMAT, 2); + radeon_emit(cmd_buffer->state.spi_shader_z_format); + radeon_emit(col_format_compacted); /* SPI_SHADER_COL_FORMAT */ } else { - radeon_set_context_reg(cmd_buffer->cs, R_02823C_CB_SHADER_MASK, cmd_buffer->state.cb_shader_mask); - radeon_set_context_reg_seq(cmd_buffer->cs, R_028710_SPI_SHADER_Z_FORMAT, 2); - radeon_emit(cmd_buffer->cs, cmd_buffer->state.spi_shader_z_format); - radeon_emit(cmd_buffer->cs, col_format_compacted); /* SPI_SHADER_COL_FORMAT */ + radeon_set_context_reg(R_02823C_CB_SHADER_MASK, cmd_buffer->state.cb_shader_mask); + radeon_set_context_reg_seq(R_028710_SPI_SHADER_Z_FORMAT, 2); + radeon_emit(cmd_buffer->state.spi_shader_z_format); + radeon_emit(col_format_compacted); /* SPI_SHADER_COL_FORMAT */ } + radeon_end(); + cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_FRAGMENT_OUTPUT; } @@ -10650,12 +10934,14 @@ radv_emit_depth_stencil_state(struct radv_cmd_buffer *cmd_buffer) const uint32_t depth_bounds_min = fui(d->vk.ds.depth.bounds_test.min); const uint32_t depth_bounds_max = fui(d->vk.ds.depth.bounds_test.max); + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { const bool force_s_valid = stencil_test_enable && ((d->vk.ds.stencil.front.op.pass != d->vk.ds.stencil.front.op.depth_fail) || (d->vk.ds.stencil.back.op.pass != d->vk.ds.stencil.back.op.depth_fail)); - radeon_set_context_reg(cmd_buffer->cs, R_02800C_DB_RENDER_OVERRIDE, + radeon_set_context_reg(R_02800C_DB_RENDER_OVERRIDE, S_02800C_FORCE_STENCIL_READ(1) | S_02800C_FORCE_STENCIL_VALID(force_s_valid)); radeon_opt_set_context_reg(cmd_buffer, R_028070_DB_DEPTH_CONTROL, RADV_TRACKED_DB_DEPTH_CONTROL, @@ -10704,6 +10990,8 @@ radv_emit_depth_stencil_state(struct radv_cmd_buffer *cmd_buffer) } } + radeon_end(); + cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_DEPTH_STENCIL_STATE; } @@ -10714,6 +11002,8 @@ radv_emit_raster_state(struct radv_cmd_buffer *cmd_buffer) const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; + radeon_begin(cmd_buffer->cs); + radeon_opt_set_context_reg(cmd_buffer, R_028A08_PA_SU_LINE_CNTL, RADV_TRACKED_PA_SU_LINE_CNTL, S_028A08_WIDTH(CLAMP(d->vk.rs.line.width * 8, 0, 0xFFFF))); @@ -10773,6 +11063,8 @@ radv_emit_raster_state(struct radv_cmd_buffer *cmd_buffer) pa_su_sc_mode_cntl); } + radeon_end(); + cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_RASTER_STATE; } @@ -11120,8 +11412,11 @@ radv_before_draw(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info struct radeon_cmdbuf *cs = cmd_buffer->cs; assert(state->vtx_base_sgpr); if (state->last_num_instances != info->instance_count) { - radeon_emit(cs, PKT3(PKT3_NUM_INSTANCES, 0, false)); - radeon_emit(cs, info->instance_count); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_NUM_INSTANCES, 0, false)); + radeon_emit(info->instance_count); + radeon_end(); + state->last_num_instances = info->instance_count; } } @@ -11182,8 +11477,12 @@ radv_before_taskmesh_draw(struct radv_cmd_buffer *cmd_buffer, const struct radv_ struct radv_cmd_state *state = &cmd_buffer->state; if (unlikely(state->last_num_instances != 1)) { struct radeon_cmdbuf *cs = cmd_buffer->cs; - radeon_emit(cs, PKT3(PKT3_NUM_INSTANCES, 0, false)); - radeon_emit(cs, 1); + + radeon_begin(cs); + radeon_emit(PKT3(PKT3_NUM_INSTANCES, 0, false)); + radeon_emit(1); + radeon_end(); + state->last_num_instances = 1; } } @@ -11646,8 +11945,10 @@ radv_CmdExecuteGeneratedCommandsEXT(VkCommandBuffer commandBuffer, VkBool32 isPr } if (!radv_cmd_buffer_uses_mec(cmd_buffer)) { - radeon_emit(cmd_buffer->cs, PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); - radeon_emit(cmd_buffer->cs, 0); + radeon_begin(cmd_buffer->cs); + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); + radeon_emit(0); + radeon_end(); } const uint32_t view_mask = cmd_buffer->state.render.view_mask; @@ -11725,18 +12026,21 @@ radv_save_dispatch_size(struct radv_cmd_buffer *cmd_buffer, uint64_t indirect_va uint64_t va = radv_buffer_get_va(device->trace_bo) + offsetof(struct radv_trace_data, indirect_dispatch); + radeon_begin(cs); + for (uint32_t i = 0; i < 3; i++) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, - COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, indirect_va); - radeon_emit(cs, indirect_va >> 32); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); + radeon_emit(indirect_va); + radeon_emit(indirect_va >> 32); + radeon_emit(va); + radeon_emit(va >> 32); indirect_va += 4; va += 4; } + + radeon_end(); } static void @@ -11768,15 +12072,17 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv radv_save_dispatch_size(cmd_buffer, info->indirect_va); if (info->unaligned) { - radeon_set_sh_reg_seq(cs, R_00B81C_COMPUTE_NUM_THREAD_X, 3); + radeon_begin(cs); + radeon_set_sh_reg_seq(R_00B81C_COMPUTE_NUM_THREAD_X, 3); if (pdev->info.gfx_level >= GFX12) { - radeon_emit(cs, S_00B81C_NUM_THREAD_FULL_GFX12(compute_shader->info.cs.block_size[0])); - radeon_emit(cs, S_00B820_NUM_THREAD_FULL_GFX12(compute_shader->info.cs.block_size[1])); + radeon_emit(S_00B81C_NUM_THREAD_FULL_GFX12(compute_shader->info.cs.block_size[0])); + radeon_emit(S_00B820_NUM_THREAD_FULL_GFX12(compute_shader->info.cs.block_size[1])); } else { - radeon_emit(cs, S_00B81C_NUM_THREAD_FULL_GFX6(compute_shader->info.cs.block_size[0])); - radeon_emit(cs, S_00B820_NUM_THREAD_FULL_GFX6(compute_shader->info.cs.block_size[1])); + radeon_emit(S_00B81C_NUM_THREAD_FULL_GFX6(compute_shader->info.cs.block_size[0])); + radeon_emit(S_00B820_NUM_THREAD_FULL_GFX6(compute_shader->info.cs.block_size[1])); } - radeon_emit(cs, S_00B824_NUM_THREAD_FULL(compute_shader->info.cs.block_size[2])); + radeon_emit(S_00B824_NUM_THREAD_FULL(compute_shader->info.cs.block_size[2])); + radeon_end(); dispatch_initiator |= S_00B800_USE_THREAD_DIMENSIONS(1); } @@ -11786,16 +12092,21 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv dispatch_initiator |= S_00B800_FORCE_START_AT_000(1); if (grid_size_offset) { + radeon_begin(cs); + if (device->load_grid_size_from_user_sgpr) { assert(pdev->info.gfx_level >= GFX10_3); - radeon_emit(cs, PKT3(PKT3_LOAD_SH_REG_INDEX, 3, 0)); - radeon_emit(cs, info->indirect_va); - radeon_emit(cs, info->indirect_va >> 32); - radeon_emit(cs, (grid_size_offset - SI_SH_REG_OFFSET) >> 2); - radeon_emit(cs, 3); + + radeon_emit(PKT3(PKT3_LOAD_SH_REG_INDEX, 3, 0)); + radeon_emit(info->indirect_va); + radeon_emit(info->indirect_va >> 32); + radeon_emit((grid_size_offset - SI_SH_REG_OFFSET) >> 2); + radeon_emit(3); } else { - radeon_emit_64bit_pointer(cmd_buffer->cs, grid_size_offset, info->indirect_va); + radeon_emit_64bit_pointer(grid_size_offset, info->indirect_va); } + + radeon_end(); } if (radv_cmd_buffer_uses_mec(cmd_buffer)) { @@ -11823,20 +12134,24 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv const uint64_t src_va = unaligned_va + i * 4; const uint64_t dst_va = indirect_va + i * 4; - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | - COPY_DATA_WR_CONFIRM); - radeon_emit(cs, src_va); - radeon_emit(cs, src_va >> 32); - radeon_emit(cs, dst_va); - radeon_emit(cs, dst_va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | + COPY_DATA_WR_CONFIRM); + radeon_emit(src_va); + radeon_emit(src_va >> 32); + radeon_emit(dst_va); + radeon_emit(dst_va >> 32); + radeon_end(); } } - radeon_emit(cs, PKT3(PKT3_DISPATCH_INDIRECT, 2, 0) | PKT3_SHADER_TYPE_S(1)); - radeon_emit(cs, indirect_va); - radeon_emit(cs, indirect_va >> 32); - radeon_emit(cs, dispatch_initiator); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_DISPATCH_INDIRECT, 2, 0) | PKT3_SHADER_TYPE_S(1)); + radeon_emit(indirect_va); + radeon_emit(indirect_va >> 32); + radeon_emit(dispatch_initiator); + radeon_end(); } else { radv_emit_indirect_buffer(cs, info->indirect_va, true); @@ -11846,9 +12161,11 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv predicating = false; } - radeon_emit(cs, PKT3(PKT3_DISPATCH_INDIRECT, 1, predicating) | PKT3_SHADER_TYPE_S(1)); - radeon_emit(cs, 0); - radeon_emit(cs, dispatch_initiator); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_DISPATCH_INDIRECT, 1, predicating) | PKT3_SHADER_TYPE_S(1)); + radeon_emit(0); + radeon_emit(dispatch_initiator); + radeon_end(); } } else { const unsigned *cs_block_size = compute_shader->info.cs.block_size; @@ -11874,44 +12191,49 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv offsets[i] /= cs_block_size[i]; } - radeon_set_sh_reg_seq(cs, R_00B81C_COMPUTE_NUM_THREAD_X, 3); + radeon_begin(cs); + radeon_set_sh_reg_seq(R_00B81C_COMPUTE_NUM_THREAD_X, 3); if (pdev->info.gfx_level >= GFX12) { - radeon_emit(cs, - S_00B81C_NUM_THREAD_FULL_GFX12(cs_block_size[0]) | S_00B81C_NUM_THREAD_PARTIAL(remainder[0])); - radeon_emit(cs, - S_00B820_NUM_THREAD_FULL_GFX12(cs_block_size[1]) | S_00B820_NUM_THREAD_PARTIAL(remainder[1])); + radeon_emit(S_00B81C_NUM_THREAD_FULL_GFX12(cs_block_size[0]) | S_00B81C_NUM_THREAD_PARTIAL(remainder[0])); + radeon_emit(S_00B820_NUM_THREAD_FULL_GFX12(cs_block_size[1]) | S_00B820_NUM_THREAD_PARTIAL(remainder[1])); } else { - radeon_emit(cs, - S_00B81C_NUM_THREAD_FULL_GFX6(cs_block_size[0]) | S_00B81C_NUM_THREAD_PARTIAL(remainder[0])); - radeon_emit(cs, - S_00B820_NUM_THREAD_FULL_GFX6(cs_block_size[1]) | S_00B820_NUM_THREAD_PARTIAL(remainder[1])); + radeon_emit(S_00B81C_NUM_THREAD_FULL_GFX6(cs_block_size[0]) | S_00B81C_NUM_THREAD_PARTIAL(remainder[0])); + radeon_emit(S_00B820_NUM_THREAD_FULL_GFX6(cs_block_size[1]) | S_00B820_NUM_THREAD_PARTIAL(remainder[1])); } - radeon_emit(cs, S_00B824_NUM_THREAD_FULL(cs_block_size[2]) | S_00B824_NUM_THREAD_PARTIAL(remainder[2])); + radeon_emit(S_00B824_NUM_THREAD_FULL(cs_block_size[2]) | S_00B824_NUM_THREAD_PARTIAL(remainder[2])); + radeon_end(); dispatch_initiator |= S_00B800_PARTIAL_TG_EN(1); } if (grid_size_offset) { if (device->load_grid_size_from_user_sgpr) { - radeon_set_sh_reg_seq(cs, grid_size_offset, 3); - radeon_emit(cs, blocks[0]); - radeon_emit(cs, blocks[1]); - radeon_emit(cs, blocks[2]); + radeon_begin(cs); + radeon_set_sh_reg_seq(grid_size_offset, 3); + radeon_emit(blocks[0]); + radeon_emit(blocks[1]); + radeon_emit(blocks[2]); + radeon_end(); } else { uint32_t offset; if (!radv_cmd_buffer_upload_data(cmd_buffer, 12, blocks, &offset)) return; uint64_t va = radv_buffer_get_va(cmd_buffer->upload.upload_bo) + offset; - radeon_emit_64bit_pointer(cmd_buffer->cs, grid_size_offset, va); + + radeon_begin(cs); + radeon_emit_64bit_pointer(grid_size_offset, va); + radeon_end(); } } if (offsets[0] || offsets[1] || offsets[2]) { - radeon_set_sh_reg_seq(cs, R_00B810_COMPUTE_START_X, 3); - radeon_emit(cs, offsets[0]); - radeon_emit(cs, offsets[1]); - radeon_emit(cs, offsets[2]); + radeon_begin(cs); + radeon_set_sh_reg_seq(R_00B810_COMPUTE_START_X, 3); + radeon_emit(offsets[0]); + radeon_emit(offsets[1]); + radeon_emit(offsets[2]); + radeon_end(); /* The blocks in the packet are not counts but end values. */ for (unsigned i = 0; i < 3; ++i) @@ -11940,11 +12262,13 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv } } - radeon_emit(cs, PKT3(PKT3_DISPATCH_DIRECT, 3, predicating) | PKT3_SHADER_TYPE_S(1)); - radeon_emit(cs, blocks[0]); - radeon_emit(cs, blocks[1]); - radeon_emit(cs, blocks[2]); - radeon_emit(cs, dispatch_initiator); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_DISPATCH_DIRECT, 3, predicating) | PKT3_SHADER_TYPE_S(1)); + radeon_emit(blocks[0]); + radeon_emit(blocks[1]); + radeon_emit(blocks[2]); + radeon_emit(dispatch_initiator); + radeon_end(); } assert(cmd_buffer->cs->cdw <= cdw_max); @@ -11985,7 +12309,9 @@ radv_emit_rt_stack_size(struct radv_cmd_buffer *cmd_buffer) rsrc2 |= S_00B12C_SCRATCH_EN(1); radeon_check_space(device->ws, cmd_buffer->cs, 3); - radeon_set_sh_reg(cmd_buffer->cs, rt_prolog->info.regs.pgm_rsrc2, rsrc2); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(rt_prolog->info.regs.pgm_rsrc2, rsrc2); + radeon_end(); } static void @@ -12325,12 +12651,16 @@ radv_trace_rays(struct radv_cmd_buffer *cmd_buffer, VkTraceRaysIndirectCommand2K const uint32_t sbt_descriptors_offset = radv_get_user_sgpr_loc(rt_prolog, AC_UD_CS_SBT_DESCRIPTORS); if (sbt_descriptors_offset) { - radeon_emit_64bit_pointer(cmd_buffer->cs, sbt_descriptors_offset, sbt_va); + radeon_begin(cmd_buffer->cs); + radeon_emit_64bit_pointer(sbt_descriptors_offset, sbt_va); + radeon_end(); } const uint32_t ray_launch_size_addr_offset = radv_get_user_sgpr_loc(rt_prolog, AC_UD_CS_RAY_LAUNCH_SIZE_ADDR); if (ray_launch_size_addr_offset) { - radeon_emit_64bit_pointer(cmd_buffer->cs, ray_launch_size_addr_offset, launch_size_va); + radeon_begin(cmd_buffer->cs); + radeon_emit_64bit_pointer(ray_launch_size_addr_offset, launch_size_va); + radeon_end(); } assert(cmd_buffer->cs->cdw <= cdw_max); @@ -12346,7 +12676,9 @@ radv_trace_rays(struct radv_cmd_buffer *cmd_buffer, VkTraceRaysIndirectCommand2K tables->height = 1; radv_upload_trace_rays_params(cmd_buffer, tables, mode, &launch_size_va, NULL); if (ray_launch_size_addr_offset) { - radeon_emit_64bit_pointer(cmd_buffer->cs, ray_launch_size_addr_offset, launch_size_va); + radeon_begin(cmd_buffer->cs); + radeon_emit_64bit_pointer(ray_launch_size_addr_offset, launch_size_va); + radeon_end(); } radv_dispatch(cmd_buffer, &info, pipeline, rt_prolog, VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR); @@ -13123,16 +13455,19 @@ radv_emit_set_predication_state(struct radv_cmd_buffer *cmd_buffer, bool draw_vi */ op |= draw_visible ? PREDICATION_DRAW_VISIBLE : PREDICATION_DRAW_NOT_VISIBLE; } + + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX9) { - radeon_emit(cmd_buffer->cs, PKT3(PKT3_SET_PREDICATION, 2, 0)); - radeon_emit(cmd_buffer->cs, op); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, va >> 32); + radeon_emit(PKT3(PKT3_SET_PREDICATION, 2, 0)); + radeon_emit(op); + radeon_emit(va); + radeon_emit(va >> 32); } else { - radeon_emit(cmd_buffer->cs, PKT3(PKT3_SET_PREDICATION, 1, 0)); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, op | ((va >> 32) & 0xFF)); + radeon_emit(PKT3(PKT3_SET_PREDICATION, 1, 0)); + radeon_emit(va); + radeon_emit(op | ((va >> 32) & 0xFF)); } + radeon_end(); } void @@ -13181,17 +13516,19 @@ radv_begin_conditional_rendering(struct radv_cmd_buffer *cmd_buffer, uint64_t va pred_va = radv_buffer_get_va(cmd_buffer->upload.upload_bo) + pred_offset; radeon_check_space(device->ws, cmd_buffer->cs, 8); + radeon_begin(cs); - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit( - cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, pred_va); - radeon_emit(cs, pred_va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | + COPY_DATA_WR_CONFIRM); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(pred_va); + radeon_emit(pred_va >> 32); - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0)); + radeon_emit(0); + radeon_end(); va = pred_va; pred_op = PREDICATION_OP_BOOL64; @@ -13332,32 +13669,35 @@ radv_flush_vgt_streamout(struct radv_cmd_buffer *cmd_buffer) ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, 14); + radeon_begin(cs); + /* The register is at different places on different ASICs. */ if (pdev->info.gfx_level >= GFX9) { reg_strmout_cntl = R_0300FC_CP_STRMOUT_CNTL; - radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0)); - radeon_emit(cs, S_370_DST_SEL(V_370_MEM_MAPPED_REGISTER) | S_370_ENGINE_SEL(V_370_ME)); - radeon_emit(cs, R_0300FC_CP_STRMOUT_CNTL >> 2); - radeon_emit(cs, 0); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_WRITE_DATA, 3, 0)); + radeon_emit(S_370_DST_SEL(V_370_MEM_MAPPED_REGISTER) | S_370_ENGINE_SEL(V_370_ME)); + radeon_emit(R_0300FC_CP_STRMOUT_CNTL >> 2); + radeon_emit(0); + radeon_emit(0); } else if (pdev->info.gfx_level >= GFX7) { reg_strmout_cntl = R_0300FC_CP_STRMOUT_CNTL; - radeon_set_uconfig_reg(cs, reg_strmout_cntl, 0); + radeon_set_uconfig_reg(reg_strmout_cntl, 0); } else { reg_strmout_cntl = R_0084FC_CP_STRMOUT_CNTL; - radeon_set_config_reg(cs, reg_strmout_cntl, 0); + radeon_set_config_reg(reg_strmout_cntl, 0); } - radeon_event_write(cs, V_028A90_SO_VGTSTREAMOUT_FLUSH); + radeon_event_write(V_028A90_SO_VGTSTREAMOUT_FLUSH); - radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0)); - radeon_emit(cs, WAIT_REG_MEM_EQUAL); /* wait until the register is equal to the reference value */ - radeon_emit(cs, reg_strmout_cntl >> 2); /* register */ - radeon_emit(cs, 0); - radeon_emit(cs, S_0084FC_OFFSET_UPDATE_DONE(1)); /* reference value */ - radeon_emit(cs, S_0084FC_OFFSET_UPDATE_DONE(1)); /* mask */ - radeon_emit(cs, 4); /* poll interval */ + radeon_emit(PKT3(PKT3_WAIT_REG_MEM, 5, 0)); + radeon_emit(WAIT_REG_MEM_EQUAL); /* wait until the register is equal to the reference value */ + radeon_emit(reg_strmout_cntl >> 2); /* register */ + radeon_emit(0); + radeon_emit(S_0084FC_OFFSET_UPDATE_DONE(1)); /* reference value */ + radeon_emit(S_0084FC_OFFSET_UPDATE_DONE(1)); /* mask */ + radeon_emit(4); /* poll interval */ + radeon_end(); assert(cs->cdw <= cdw_max); } @@ -13398,7 +13738,9 @@ radv_init_streamout_state(struct radv_cmd_buffer *cmd_buffer) /* Reset the ordered ID for the next GS workgroup to 0 because it must be * equal to the 4 ordered IDs in the layout. */ - radeon_set_uconfig_reg(cmd_buffer->cs, R_0309B0_GE_GS_ORDERED_ID_BASE, 0); + radeon_begin(cmd_buffer->cs); + radeon_set_uconfig_reg(R_0309B0_GE_GS_ORDERED_ID_BASE, 0); + radeon_end(); } VKAPI_ATTR void VKAPI_CALL @@ -13443,58 +13785,61 @@ radv_CmdBeginTransformFeedbackEXT(VkCommandBuffer commandBuffer, uint32_t firstC radv_cs_add_buffer(device->ws, cs, buffer->bo); } + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX12) { if (append) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit( - cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, (so->state_va + i * 8 + 4)); - radeon_emit(cs, (so->state_va + i * 8 + 4) >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | + COPY_DATA_WR_CONFIRM); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit((so->state_va + i * 8 + 4)); + radeon_emit((so->state_va + i * 8 + 4) >> 32); } } else if (pdev->use_ngg_streamout) { if (append) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, - COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, (R_031088_GDS_STRMOUT_DWORDS_WRITTEN_0 >> 2) + i); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | COPY_DATA_WR_CONFIRM); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit((R_031088_GDS_STRMOUT_DWORDS_WRITTEN_0 >> 2) + i); + radeon_emit(0); } else { /* The PKT3 CAM bit workaround seems needed for initializing this GDS register to zero. */ - radeon_set_uconfig_perfctr_reg(pdev->info.gfx_level, ring, cs, - R_031088_GDS_STRMOUT_DWORDS_WRITTEN_0 + i * 4, 0); + radeon_set_uconfig_perfctr_reg(pdev->info.gfx_level, ring, R_031088_GDS_STRMOUT_DWORDS_WRITTEN_0 + i * 4, + 0); } } else { /* AMD GCN binds streamout buffers as shader resources. * VGT only counts primitives and tells the shader through * SGPRs what to do. */ - radeon_set_context_reg(cs, R_028AD0_VGT_STRMOUT_BUFFER_SIZE_0 + 16 * i, sb[i].size >> 2); + radeon_set_context_reg(R_028AD0_VGT_STRMOUT_BUFFER_SIZE_0 + 16 * i, sb[i].size >> 2); cmd_buffer->state.context_roll_without_scissor_emitted = true; if (append) { - radeon_emit(cs, PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0)); - radeon_emit(cs, STRMOUT_SELECT_BUFFER(i) | STRMOUT_DATA_TYPE(1) | /* offset in bytes */ - STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_FROM_MEM)); /* control */ - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, va); /* src address lo */ - radeon_emit(cs, va >> 32); /* src address hi */ + radeon_emit(PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0)); + radeon_emit(STRMOUT_SELECT_BUFFER(i) | STRMOUT_DATA_TYPE(1) | /* offset in bytes */ + STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_FROM_MEM)); /* control */ + radeon_emit(0); /* unused */ + radeon_emit(0); /* unused */ + radeon_emit(va); /* src address lo */ + radeon_emit(va >> 32); /* src address hi */ } else { /* Start from the beginning. */ - radeon_emit(cs, PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0)); - radeon_emit(cs, STRMOUT_SELECT_BUFFER(i) | STRMOUT_DATA_TYPE(1) | /* offset in bytes */ - STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_FROM_PACKET)); /* control */ - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, 0); /* unused */ + radeon_emit(PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0)); + radeon_emit(STRMOUT_SELECT_BUFFER(i) | STRMOUT_DATA_TYPE(1) | /* offset in bytes */ + STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_FROM_PACKET)); /* control */ + radeon_emit(0); /* unused */ + radeon_emit(0); /* unused */ + radeon_emit(0); /* unused */ + radeon_emit(0); /* unused */ } } + + radeon_end(); } assert(cs->cdw <= cdw_max); @@ -13552,36 +13897,36 @@ radv_CmdEndTransformFeedbackEXT(VkCommandBuffer commandBuffer, uint32_t firstCou radv_cs_add_buffer(device->ws, cs, buffer->bo); } + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX12) { if (append) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit( - cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, (so->state_va + i * 8 + 4)); - radeon_emit(cs, (so->state_va + i * 8 + 4) >> 32); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | + COPY_DATA_WR_CONFIRM); + radeon_emit((so->state_va + i * 8 + 4)); + radeon_emit((so->state_va + i * 8 + 4) >> 32); + radeon_emit(va); + radeon_emit(va >> 32); } } else if (pdev->use_ngg_streamout) { if (append) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, - COPY_DATA_SRC_SEL(COPY_DATA_REG) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, (R_031088_GDS_STRMOUT_DWORDS_WRITTEN_0 >> 2) + i); - radeon_emit(cs, 0); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_REG) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); + radeon_emit((R_031088_GDS_STRMOUT_DWORDS_WRITTEN_0 >> 2) + i); + radeon_emit(0); + radeon_emit(va); + radeon_emit(va >> 32); } } else { if (append) { - radeon_emit(cs, PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0)); - radeon_emit(cs, STRMOUT_SELECT_BUFFER(i) | STRMOUT_DATA_TYPE(1) | /* offset in bytes */ - STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_NONE) | - STRMOUT_STORE_BUFFER_FILLED_SIZE); /* control */ - radeon_emit(cs, va); /* dst address lo */ - radeon_emit(cs, va >> 32); /* dst address hi */ - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, 0); /* unused */ + radeon_emit(PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0)); + radeon_emit(STRMOUT_SELECT_BUFFER(i) | STRMOUT_DATA_TYPE(1) | /* offset in bytes */ + STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_NONE) | STRMOUT_STORE_BUFFER_FILLED_SIZE); /* control */ + radeon_emit(va); /* dst address lo */ + radeon_emit(va >> 32); /* dst address hi */ + radeon_emit(0); /* unused */ + radeon_emit(0); /* unused */ } /* Deactivate transform feedback by zeroing the buffer size. @@ -13589,10 +13934,12 @@ radv_CmdEndTransformFeedbackEXT(VkCommandBuffer commandBuffer, uint32_t firstCou * be enabled even if there is not buffer bound. This ensures * that the primitives-emitted query won't increment. */ - radeon_set_context_reg(cs, R_028AD0_VGT_STRMOUT_BUFFER_SIZE_0 + 16 * i, 0); + radeon_set_context_reg(R_028AD0_VGT_STRMOUT_BUFFER_SIZE_0 + 16 * i, 0); cmd_buffer->state.context_roll_without_scissor_emitted = true; } + + radeon_end(); } assert(cmd_buffer->cs->cdw <= cdw_max); @@ -13609,30 +13956,32 @@ radv_emit_strmout_buffer(struct radv_cmd_buffer *cmd_buffer, const struct radv_d const enum amd_gfx_level gfx_level = pdev->info.gfx_level; struct radeon_cmdbuf *cs = cmd_buffer->cs; - radeon_set_context_reg(cs, R_028B30_VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE, draw_info->stride); + radeon_begin(cs); + radeon_set_context_reg(R_028B30_VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE, draw_info->stride); if (gfx_level >= GFX10) { /* Emitting a COPY_DATA packet should be enough because RADV doesn't support preemption * (shadow memory) but for unknown reasons, it can lead to GPU hangs on GFX10+. */ - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0)); + radeon_emit(0); - radeon_emit(cs, PKT3(PKT3_LOAD_CONTEXT_REG_INDEX, 3, 0)); - radeon_emit(cs, draw_info->strmout_va); - radeon_emit(cs, draw_info->strmout_va >> 32); - radeon_emit(cs, (R_028B2C_VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE - SI_CONTEXT_REG_OFFSET) >> 2); - radeon_emit(cs, 1); /* 1 DWORD */ + radeon_emit(PKT3(PKT3_LOAD_CONTEXT_REG_INDEX, 3, 0)); + radeon_emit(draw_info->strmout_va); + radeon_emit(draw_info->strmout_va >> 32); + radeon_emit((R_028B2C_VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE - SI_CONTEXT_REG_OFFSET) >> 2); + radeon_emit(1); /* 1 DWORD */ } else { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, draw_info->strmout_va); - radeon_emit(cs, draw_info->strmout_va >> 32); - radeon_emit(cs, R_028B2C_VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE >> 2); - radeon_emit(cs, 0); /* unused */ + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | COPY_DATA_WR_CONFIRM); + radeon_emit(draw_info->strmout_va); + radeon_emit(draw_info->strmout_va >> 32); + radeon_emit(R_028B2C_VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE >> 2); + radeon_emit(0); /* unused */ } - radeon_set_context_reg(cs, R_028B28_VGT_STRMOUT_DRAW_OPAQUE_OFFSET, counter_offset); + radeon_set_context_reg(R_028B28_VGT_STRMOUT_DRAW_OPAQUE_OFFSET, counter_offset); + radeon_end(); } VKAPI_ATTR void VKAPI_CALL @@ -13664,9 +14013,11 @@ radv_CmdDrawIndirectByteCountEXT(VkCommandBuffer commandBuffer, uint32_t instanc if (pdev->info.gfx_level == GFX12) { /* DrawTransformFeedback requires 3 SQ_NON_EVENTs after the packet. */ - radeon_event_write(cmd_buffer->cs, V_028A90_SQ_NON_EVENT); - radeon_event_write(cmd_buffer->cs, V_028A90_SQ_NON_EVENT); - radeon_event_write(cmd_buffer->cs, V_028A90_SQ_NON_EVENT); + radeon_begin(cmd_buffer->cs); + radeon_event_write(V_028A90_SQ_NON_EVENT); + radeon_event_write(V_028A90_SQ_NON_EVENT); + radeon_event_write(V_028A90_SQ_NON_EVENT); + radeon_end(); } radv_after_draw(cmd_buffer, false); @@ -13697,12 +14048,14 @@ radv_CmdWriteBufferMarker2AMD(VkCommandBuffer commandBuffer, VkPipelineStageFlag ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, 12); if (!(stage & ~VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT)) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, marker); - radeon_emit(cs, 0); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); + radeon_emit(marker); + radeon_emit(0); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_end(); } else { radv_cs_emit_write_event_eop(cs, pdev->info.gfx_level, cmd_buffer->qf, V_028A90_BOTTOM_OF_PIPE_TS, 0, EOP_DST_SEL_MEM, EOP_DATA_SEL_VALUE_32BIT, va, marker, cmd_buffer->gfx9_eop_bug_va); diff --git a/src/amd/vulkan/radv_cp_dma.c b/src/amd/vulkan/radv_cp_dma.c index 80cbce03e57..68dafdacf59 100644 --- a/src/amd/vulkan/radv_cp_dma.c +++ b/src/amd/vulkan/radv_cp_dma.c @@ -80,24 +80,26 @@ radv_cs_emit_cp_dma(struct radv_device *device, struct radeon_cmdbuf *cs, bool p else if (cp_dma_use_L2) header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2); + radeon_begin(cs); if (pdev->info.gfx_level >= GFX7) { - radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, predicating)); - radeon_emit(cs, header); - radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */ - radeon_emit(cs, src_va >> 32); /* SRC_ADDR_HI [31:0] */ - radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */ - radeon_emit(cs, dst_va >> 32); /* DST_ADDR_HI [31:0] */ - radeon_emit(cs, command); + radeon_emit(PKT3(PKT3_DMA_DATA, 5, predicating)); + radeon_emit(header); + radeon_emit(src_va); /* SRC_ADDR_LO [31:0] */ + radeon_emit(src_va >> 32); /* SRC_ADDR_HI [31:0] */ + radeon_emit(dst_va); /* DST_ADDR_LO [31:0] */ + radeon_emit(dst_va >> 32); /* DST_ADDR_HI [31:0] */ + radeon_emit(command); } else { assert(!cp_dma_use_L2); header |= S_411_SRC_ADDR_HI(src_va >> 32); - radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, predicating)); - radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */ - radeon_emit(cs, header); /* SRC_ADDR_HI [15:0] + flags. */ - radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */ - radeon_emit(cs, (dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */ - radeon_emit(cs, command); + radeon_emit(PKT3(PKT3_CP_DMA, 4, predicating)); + radeon_emit(src_va); /* SRC_ADDR_LO [31:0] */ + radeon_emit(header); /* SRC_ADDR_HI [15:0] + flags. */ + radeon_emit(dst_va); /* DST_ADDR_LO [31:0] */ + radeon_emit((dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */ + radeon_emit(command); } + radeon_end(); } static void @@ -116,8 +118,10 @@ radv_emit_cp_dma(struct radv_cmd_buffer *cmd_buffer, uint64_t dst_va, uint64_t s */ if (flags & CP_DMA_SYNC) { if (cmd_buffer->qf == RADV_QUEUE_GENERAL) { - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); - radeon_emit(cs, 0); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); + radeon_emit(0); + radeon_end(); } /* CP will see the sync flag and wait for all DMAs to complete. */ @@ -157,13 +161,15 @@ radv_cs_cp_dma_prefetch(const struct radv_device *device, struct radeon_cmdbuf * header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2); - radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, predicating)); - radeon_emit(cs, header); - radeon_emit(cs, aligned_va); /* SRC_ADDR_LO [31:0] */ - radeon_emit(cs, aligned_va >> 32); /* SRC_ADDR_HI [31:0] */ - radeon_emit(cs, aligned_va); /* DST_ADDR_LO [31:0] */ - radeon_emit(cs, aligned_va >> 32); /* DST_ADDR_HI [31:0] */ - radeon_emit(cs, command); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_DMA_DATA, 5, predicating)); + radeon_emit(header); + radeon_emit(aligned_va); /* SRC_ADDR_LO [31:0] */ + radeon_emit(aligned_va >> 32); /* SRC_ADDR_HI [31:0] */ + radeon_emit(aligned_va); /* DST_ADDR_LO [31:0] */ + radeon_emit(aligned_va >> 32); /* DST_ADDR_HI [31:0] */ + radeon_emit(command); + radeon_end(); } void diff --git a/src/amd/vulkan/radv_cp_reg_shadowing.c b/src/amd/vulkan/radv_cp_reg_shadowing.c index 1818b68198c..0ec33d35807 100644 --- a/src/amd/vulkan/radv_cp_reg_shadowing.c +++ b/src/amd/vulkan/radv_cp_reg_shadowing.c @@ -39,7 +39,10 @@ radv_create_shadow_regs_preamble(struct radv_device *device, struct radv_queue_s if (!pm4) goto fail_create; - radeon_emit_array(cs, pm4->pm4, pm4->ndw); + radeon_begin(cs); + radeon_emit_array(pm4->pm4, pm4->ndw); + radeon_end(); + ws->cs_pad(cs, 0); result = radv_bo_create( @@ -126,7 +129,10 @@ radv_init_shadowed_regs_buffer_state(const struct radv_device *device, struct ra goto fail; } - radeon_emit_array(cs, pm4->pm4, pm4->ndw); + radeon_begin(cs); + radeon_emit_array(pm4->pm4, pm4->ndw); + radeon_end(); + ac_pm4_free_state(pm4); } diff --git a/src/amd/vulkan/radv_cs.c b/src/amd/vulkan/radv_cs.c index 60aebcd76f6..5672c9b7dc8 100644 --- a/src/amd/vulkan/radv_cs.c +++ b/src/amd/vulkan/radv_cs.c @@ -37,27 +37,29 @@ radv_cs_emit_write_event_eop(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_le if (data_sel != EOP_DATA_SEL_DISCARD) sel |= EOP_INT_SEL(EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM); + radeon_begin(cs); + if (gfx_level >= GFX9 || is_gfx8_mec) { /* A ZPASS_DONE or PIXEL_STAT_DUMP_EVENT (of the DB occlusion * counters) must immediately precede every timestamp event to * prevent a GPU hang on GFX9. */ if (gfx_level == GFX9 && !is_mec) { - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0)); - radeon_emit(cs, EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1)); - radeon_emit(cs, gfx9_eop_bug_va); - radeon_emit(cs, gfx9_eop_bug_va >> 32); + radeon_emit(PKT3(PKT3_EVENT_WRITE, 2, 0)); + radeon_emit(EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1)); + radeon_emit(gfx9_eop_bug_va); + radeon_emit(gfx9_eop_bug_va >> 32); } - radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, is_gfx8_mec ? 5 : 6, false)); - radeon_emit(cs, op); - radeon_emit(cs, sel); - radeon_emit(cs, va); /* address lo */ - radeon_emit(cs, va >> 32); /* address hi */ - radeon_emit(cs, new_fence); /* immediate data lo */ - radeon_emit(cs, 0); /* immediate data hi */ + radeon_emit(PKT3(PKT3_RELEASE_MEM, is_gfx8_mec ? 5 : 6, false)); + radeon_emit(op); + radeon_emit(sel); + radeon_emit(va); /* address lo */ + radeon_emit(va >> 32); /* address hi */ + radeon_emit(new_fence); /* immediate data lo */ + radeon_emit(0); /* immediate data hi */ if (!is_gfx8_mec) - radeon_emit(cs, 0); /* unused */ + radeon_emit(0); /* unused */ } else { /* On GFX6, EOS events are always emitted with EVENT_WRITE_EOS. * On GFX7+, EOS events are emitted with EVENT_WRITE_EOS on @@ -68,19 +70,19 @@ radv_cs_emit_write_event_eop(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_le assert(event_flags == 0 && dst_sel == EOP_DST_SEL_MEM && data_sel == EOP_DATA_SEL_VALUE_32BIT); if (is_mec) { - radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, 5, false)); - radeon_emit(cs, op); - radeon_emit(cs, sel); - radeon_emit(cs, va); /* address lo */ - radeon_emit(cs, va >> 32); /* address hi */ - radeon_emit(cs, new_fence); /* immediate data lo */ - radeon_emit(cs, 0); /* immediate data hi */ + radeon_emit(PKT3(PKT3_RELEASE_MEM, 5, false)); + radeon_emit(op); + radeon_emit(sel); + radeon_emit(va); /* address lo */ + radeon_emit(va >> 32); /* address hi */ + radeon_emit(new_fence); /* immediate data lo */ + radeon_emit(0); /* immediate data hi */ } else { - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOS, 3, false)); - radeon_emit(cs, op); - radeon_emit(cs, va); - radeon_emit(cs, ((va >> 32) & 0xffff) | EOS_DATA_SEL(EOS_DATA_SEL_VALUE_32BIT)); - radeon_emit(cs, new_fence); + radeon_emit(PKT3(PKT3_EVENT_WRITE_EOS, 3, false)); + radeon_emit(op); + radeon_emit(va); + radeon_emit(((va >> 32) & 0xffff) | EOS_DATA_SEL(EOS_DATA_SEL_VALUE_32BIT)); + radeon_emit(new_fence); } } else { if (gfx_level == GFX7 || gfx_level == GFX8) { @@ -88,44 +90,50 @@ radv_cs_emit_write_event_eop(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_le * engines go idle (and optional cache flushes * executed) before the timestamp is written. */ - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, false)); - radeon_emit(cs, op); - radeon_emit(cs, va); - radeon_emit(cs, ((va >> 32) & 0xffff) | sel); - radeon_emit(cs, 0); /* immediate data */ - radeon_emit(cs, 0); /* unused */ + radeon_emit(PKT3(PKT3_EVENT_WRITE_EOP, 4, false)); + radeon_emit(op); + radeon_emit(va); + radeon_emit(((va >> 32) & 0xffff) | sel); + radeon_emit(0); /* immediate data */ + radeon_emit(0); /* unused */ } - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, false)); - radeon_emit(cs, op); - radeon_emit(cs, va); - radeon_emit(cs, ((va >> 32) & 0xffff) | sel); - radeon_emit(cs, new_fence); /* immediate data */ - radeon_emit(cs, 0); /* unused */ + radeon_emit(PKT3(PKT3_EVENT_WRITE_EOP, 4, false)); + radeon_emit(op); + radeon_emit(va); + radeon_emit(((va >> 32) & 0xffff) | sel); + radeon_emit(new_fence); /* immediate data */ + radeon_emit(0); /* unused */ } } + + radeon_end(); } static void radv_emit_acquire_mem(struct radeon_cmdbuf *cs, bool is_mec, bool is_gfx9, unsigned cp_coher_cntl) { + radeon_begin(cs); + if (is_mec || is_gfx9) { uint32_t hi_val = is_gfx9 ? 0xffffff : 0xff; - radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 5, false) | PKT3_SHADER_TYPE_S(is_mec)); - radeon_emit(cs, cp_coher_cntl); /* CP_COHER_CNTL */ - radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */ - radeon_emit(cs, hi_val); /* CP_COHER_SIZE_HI */ - radeon_emit(cs, 0); /* CP_COHER_BASE */ - radeon_emit(cs, 0); /* CP_COHER_BASE_HI */ - radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */ + radeon_emit(PKT3(PKT3_ACQUIRE_MEM, 5, false) | PKT3_SHADER_TYPE_S(is_mec)); + radeon_emit(cp_coher_cntl); /* CP_COHER_CNTL */ + radeon_emit(0xffffffff); /* CP_COHER_SIZE */ + radeon_emit(hi_val); /* CP_COHER_SIZE_HI */ + radeon_emit(0); /* CP_COHER_BASE */ + radeon_emit(0); /* CP_COHER_BASE_HI */ + radeon_emit(0x0000000A); /* POLL_INTERVAL */ } else { /* ACQUIRE_MEM is only required on a compute ring. */ - radeon_emit(cs, PKT3(PKT3_SURFACE_SYNC, 3, false)); - radeon_emit(cs, cp_coher_cntl); /* CP_COHER_CNTL */ - radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */ - radeon_emit(cs, 0); /* CP_COHER_BASE */ - radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */ + radeon_emit(PKT3(PKT3_SURFACE_SYNC, 3, false)); + radeon_emit(cp_coher_cntl); /* CP_COHER_CNTL */ + radeon_emit(0xffffffff); /* CP_COHER_SIZE */ + radeon_emit(0); /* CP_COHER_BASE */ + radeon_emit(0x0000000A); /* POLL_INTERVAL */ } + + radeon_end(); } static void @@ -179,7 +187,9 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level /* TODO: trigger on RADV_CMD_FLAG_FLUSH_AND_INV_CB_META */ if (gfx_level < GFX12 && flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_CB) { /* Flush CMASK/FMASK/DCC. Will wait for idle later. */ - radeon_event_write(cs, V_028A90_FLUSH_AND_INV_CB_META); + radeon_begin(cs); + radeon_event_write(V_028A90_FLUSH_AND_INV_CB_META); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_FLUSH_CB | RGP_FLUSH_INVAL_CB; } @@ -188,7 +198,9 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level /* TODO: trigger on RADV_CMD_FLAG_FLUSH_AND_INV_DB_META ? */ if (gfx_level < GFX12 && gfx_level != GFX11 && (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_DB)) { /* Flush HTILE. Will wait for idle later. */ - radeon_event_write(cs, V_028A90_FLUSH_AND_INV_DB_META); + radeon_begin(cs); + radeon_event_write(V_028A90_FLUSH_AND_INV_DB_META); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_FLUSH_DB | RGP_FLUSH_INVAL_DB; } @@ -213,18 +225,24 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level } else { /* Wait for graphics shaders to go idle if requested. */ if (flush_bits & RADV_CMD_FLAG_PS_PARTIAL_FLUSH) { - radeon_event_write(cs, V_028A90_PS_PARTIAL_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_PS_PARTIAL_FLUSH); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_PS_PARTIAL_FLUSH; } else if (flush_bits & RADV_CMD_FLAG_VS_PARTIAL_FLUSH) { - radeon_event_write(cs, V_028A90_VS_PARTIAL_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_VS_PARTIAL_FLUSH); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_VS_PARTIAL_FLUSH; } } if (flush_bits & RADV_CMD_FLAG_CS_PARTIAL_FLUSH) { - radeon_event_write(cs, V_028A90_CS_PARTIAL_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_CS_PARTIAL_FLUSH); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_CS_PARTIAL_FLUSH; } @@ -248,29 +266,33 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level gcr_cntl &= C_586_GLM_WB & C_586_GLM_INV & C_586_GLK_WB & C_586_GLK_INV & C_586_GLV_INV & C_586_GL1_INV & C_586_GL2_INV & C_586_GL2_WB; /* keep SEQ */ + radeon_begin(cs); + /* Send an event that flushes caches. */ - radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, 6, 0)); - radeon_emit(cs, S_490_EVENT_TYPE(cb_db_event) | S_490_EVENT_INDEX(5) | S_490_GLM_WB(glm_wb) | - S_490_GLM_INV(glm_inv) | S_490_GLV_INV(glv_inv) | S_490_GL1_INV(gl1_inv) | - S_490_GL2_INV(gl2_inv) | S_490_GL2_WB(gl2_wb) | S_490_SEQ(gcr_seq) | S_490_GLK_WB(glk_wb) | - S_490_GLK_INV(glk_inv) | S_490_PWS_ENABLE(1)); - radeon_emit(cs, 0); /* DST_SEL, INT_SEL, DATA_SEL */ - radeon_emit(cs, 0); /* ADDRESS_LO */ - radeon_emit(cs, 0); /* ADDRESS_HI */ - radeon_emit(cs, 0); /* DATA_LO */ - radeon_emit(cs, 0); /* DATA_HI */ - radeon_emit(cs, 0); /* INT_CTXID */ + radeon_emit(PKT3(PKT3_RELEASE_MEM, 6, 0)); + radeon_emit(S_490_EVENT_TYPE(cb_db_event) | S_490_EVENT_INDEX(5) | S_490_GLM_WB(glm_wb) | + S_490_GLM_INV(glm_inv) | S_490_GLV_INV(glv_inv) | S_490_GL1_INV(gl1_inv) | S_490_GL2_INV(gl2_inv) | + S_490_GL2_WB(gl2_wb) | S_490_SEQ(gcr_seq) | S_490_GLK_WB(glk_wb) | S_490_GLK_INV(glk_inv) | + S_490_PWS_ENABLE(1)); + radeon_emit(0); /* DST_SEL, INT_SEL, DATA_SEL */ + radeon_emit(0); /* ADDRESS_LO */ + radeon_emit(0); /* ADDRESS_HI */ + radeon_emit(0); /* DATA_LO */ + radeon_emit(0); /* DATA_HI */ + radeon_emit(0); /* INT_CTXID */ /* Wait for the event and invalidate remaining caches if needed. */ - radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 6, 0)); - radeon_emit(cs, S_580_PWS_STAGE_SEL(V_580_CP_PFP) | S_580_PWS_COUNTER_SEL(V_580_TS_SELECT) | - S_580_PWS_ENA2(1) | S_580_PWS_COUNT(0)); - radeon_emit(cs, 0xffffffff); /* GCR_SIZE */ - radeon_emit(cs, 0x01ffffff); /* GCR_SIZE_HI */ - radeon_emit(cs, 0); /* GCR_BASE_LO */ - radeon_emit(cs, 0); /* GCR_BASE_HI */ - radeon_emit(cs, S_585_PWS_ENA(1)); - radeon_emit(cs, gcr_cntl); /* GCR_CNTL */ + radeon_emit(PKT3(PKT3_ACQUIRE_MEM, 6, 0)); + radeon_emit(S_580_PWS_STAGE_SEL(V_580_CP_PFP) | S_580_PWS_COUNTER_SEL(V_580_TS_SELECT) | S_580_PWS_ENA2(1) | + S_580_PWS_COUNT(0)); + radeon_emit(0xffffffff); /* GCR_SIZE */ + radeon_emit(0x01ffffff); /* GCR_SIZE_HI */ + radeon_emit(0); /* GCR_BASE_LO */ + radeon_emit(0); /* GCR_BASE_HI */ + radeon_emit(S_585_PWS_ENA(1)); + radeon_emit(gcr_cntl); /* GCR_CNTL */ + + radeon_end(); gcr_cntl = 0; /* all done */ } else { @@ -310,9 +332,11 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level } } + radeon_begin(cs); + /* VGT state sync */ if (flush_bits & RADV_CMD_FLAG_VGT_FLUSH) { - radeon_event_write(cs, V_028A90_VGT_FLUSH); + radeon_event_write(V_028A90_VGT_FLUSH); } /* Ignore fields that only modify the behavior of other fields. */ @@ -321,37 +345,39 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level * The cache flush is executed in the ME, but the PFP waits * for completion. */ - radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 6, 0)); - radeon_emit(cs, 0); /* CP_COHER_CNTL */ - radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */ - radeon_emit(cs, 0xffffff); /* CP_COHER_SIZE_HI */ - radeon_emit(cs, 0); /* CP_COHER_BASE */ - radeon_emit(cs, 0); /* CP_COHER_BASE_HI */ - radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */ - radeon_emit(cs, gcr_cntl); /* GCR_CNTL */ + radeon_emit(PKT3(PKT3_ACQUIRE_MEM, 6, 0)); + radeon_emit(0); /* CP_COHER_CNTL */ + radeon_emit(0xffffffff); /* CP_COHER_SIZE */ + radeon_emit(0xffffff); /* CP_COHER_SIZE_HI */ + radeon_emit(0); /* CP_COHER_BASE */ + radeon_emit(0); /* CP_COHER_BASE_HI */ + radeon_emit(0x0000000A); /* POLL_INTERVAL */ + radeon_emit(gcr_cntl); /* GCR_CNTL */ } else if ((cb_db_event || (flush_bits & (RADV_CMD_FLAG_VS_PARTIAL_FLUSH | RADV_CMD_FLAG_PS_PARTIAL_FLUSH | RADV_CMD_FLAG_CS_PARTIAL_FLUSH))) && !is_mec) { /* We need to ensure that PFP waits as well. */ - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0)); + radeon_emit(0); *sqtt_flush_bits |= RGP_FLUSH_PFP_SYNC_ME; } if (flush_bits & RADV_CMD_FLAG_START_PIPELINE_STATS) { if (qf == RADV_QUEUE_GENERAL) { - radeon_event_write(cs, V_028A90_PIPELINESTAT_START); + radeon_event_write(V_028A90_PIPELINESTAT_START); } else if (qf == RADV_QUEUE_COMPUTE) { - radeon_set_sh_reg(cs, R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(1)); + radeon_set_sh_reg(R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(1)); } } else if (flush_bits & RADV_CMD_FLAG_STOP_PIPELINE_STATS) { if (qf == RADV_QUEUE_GENERAL) { - radeon_event_write(cs, V_028A90_PIPELINESTAT_STOP); + radeon_event_write(V_028A90_PIPELINESTAT_STOP); } else if (qf == RADV_QUEUE_COMPUTE) { - radeon_set_sh_reg(cs, R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(0)); + radeon_set_sh_reg(R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(0)); } } + + radeon_end(); } void @@ -405,29 +431,39 @@ radv_cs_emit_cache_flush(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, enu } if (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_CB_META) { - radeon_event_write(cs, V_028A90_FLUSH_AND_INV_CB_META); + radeon_begin(cs); + radeon_event_write(V_028A90_FLUSH_AND_INV_CB_META); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_FLUSH_CB | RGP_FLUSH_INVAL_CB; } if (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_DB_META) { - radeon_event_write(cs, V_028A90_FLUSH_AND_INV_DB_META); + radeon_begin(cs); + radeon_event_write(V_028A90_FLUSH_AND_INV_DB_META); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_FLUSH_DB | RGP_FLUSH_INVAL_DB; } if (flush_bits & RADV_CMD_FLAG_PS_PARTIAL_FLUSH) { - radeon_event_write(cs, V_028A90_PS_PARTIAL_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_PS_PARTIAL_FLUSH); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_PS_PARTIAL_FLUSH; } else if (flush_bits & RADV_CMD_FLAG_VS_PARTIAL_FLUSH) { - radeon_event_write(cs, V_028A90_VS_PARTIAL_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_VS_PARTIAL_FLUSH); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_VS_PARTIAL_FLUSH; } if (flush_bits & RADV_CMD_FLAG_CS_PARTIAL_FLUSH) { - radeon_event_write(cs, V_028A90_CS_PARTIAL_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_CS_PARTIAL_FLUSH); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_CS_PARTIAL_FLUSH; } @@ -475,12 +511,16 @@ radv_cs_emit_cache_flush(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, enu /* VGT state sync */ if (flush_bits & RADV_CMD_FLAG_VGT_FLUSH) { - radeon_event_write(cs, V_028A90_VGT_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_VGT_FLUSH); + radeon_end(); } /* VGT streamout state sync */ if (flush_bits & RADV_CMD_FLAG_VGT_STREAMOUT_SYNC) { - radeon_event_write(cs, V_028A90_VGT_STREAMOUT_SYNC); + radeon_begin(cs); + radeon_event_write(V_028A90_VGT_STREAMOUT_SYNC); + radeon_end(); } /* Make sure ME is idle (it executes most packets) before continuing. @@ -489,8 +529,10 @@ radv_cs_emit_cache_flush(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, enu if ((cp_coher_cntl || (flush_bits & (RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | RADV_CMD_FLAG_INV_L2 | RADV_CMD_FLAG_WB_L2))) && !is_mec) { - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); - radeon_emit(cs, 0); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0)); + radeon_emit(0); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_PFP_SYNC_ME; } @@ -530,19 +572,23 @@ radv_cs_emit_cache_flush(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, enu if (cp_coher_cntl) radv_emit_acquire_mem(cs, is_mec, gfx_level == GFX9, cp_coher_cntl); + radeon_begin(cs); + if (flush_bits & RADV_CMD_FLAG_START_PIPELINE_STATS) { if (qf == RADV_QUEUE_GENERAL) { - radeon_event_write(cs, V_028A90_PIPELINESTAT_START); + radeon_event_write(V_028A90_PIPELINESTAT_START); } else if (qf == RADV_QUEUE_COMPUTE) { - radeon_set_sh_reg(cs, R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(1)); + radeon_set_sh_reg(R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(1)); } } else if (flush_bits & RADV_CMD_FLAG_STOP_PIPELINE_STATS) { if (qf == RADV_QUEUE_GENERAL) { - radeon_event_write(cs, V_028A90_PIPELINESTAT_STOP); + radeon_event_write(V_028A90_PIPELINESTAT_STOP); } else if (qf == RADV_QUEUE_COMPUTE) { - radeon_set_sh_reg(cs, R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(0)); + radeon_set_sh_reg(R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(0)); } } + + radeon_end(); } void @@ -551,26 +597,32 @@ radv_emit_cond_exec(const struct radv_device *device, struct radeon_cmdbuf *cs, const struct radv_physical_device *pdev = radv_device_physical(device); const enum amd_gfx_level gfx_level = pdev->info.gfx_level; + radeon_begin(cs); + if (gfx_level >= GFX7) { - radeon_emit(cs, PKT3(PKT3_COND_EXEC, 3, 0)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, 0); - radeon_emit(cs, count); + radeon_emit(PKT3(PKT3_COND_EXEC, 3, 0)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(0); + radeon_emit(count); } else { - radeon_emit(cs, PKT3(PKT3_COND_EXEC, 2, 0)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, count); + radeon_emit(PKT3(PKT3_COND_EXEC, 2, 0)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(count); } + + radeon_end(); } void radv_cs_write_data_imm(struct radeon_cmdbuf *cs, unsigned engine_sel, uint64_t va, uint32_t imm) { - radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0)); - radeon_emit(cs, S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(engine_sel)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, imm); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_WRITE_DATA, 3, 0)); + radeon_emit(S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(engine_sel)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(imm); + radeon_end(); } diff --git a/src/amd/vulkan/radv_cs.h b/src/amd/vulkan/radv_cs.h index 697250b54ec..0b32521f701 100644 --- a/src/amd/vulkan/radv_cs.h +++ b/src/amd/vulkan/radv_cs.h @@ -27,34 +27,58 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned return cs->cdw + needed; } -/* Packet building helpers. Don't use directly. */ -#define radeon_set_reg_seq(cs, reg, num, idx, prefix_name, packet, reset_filter_cam) \ +#define radeon_begin(cs) \ + struct radeon_cmdbuf *__cs = (cs); \ + uint64_t __cs_num = __cs->cdw; \ + UNUSED uint64_t __cs_reserved_dw = __cs->reserved_dw; \ + uint32_t *__cs_buf = __cs->buf + +#define radeon_end() \ do { \ - assert((reg) >= prefix_name##_REG_OFFSET && (reg) < prefix_name##_REG_END); \ - assert(cs->cdw + 2 + num <= cs->reserved_dw); \ - radeon_emit(cs, PKT3(packet, num, 0) | PKT3_RESET_FILTER_CAM_S(reset_filter_cam)); \ - radeon_emit(cs, (((reg) - prefix_name##_REG_OFFSET) >> 2) | ((idx) << 28)); \ + __cs->cdw = __cs_num; \ + assert(__cs->cdw <= __cs->max_dw); \ + __cs = NULL; \ } while (0) -#define radeon_set_reg(cs, reg, idx, value, prefix_name, packet) \ +#define radeon_emit(value) \ do { \ - radeon_set_reg_seq(cs, reg, 1, idx, prefix_name, packet, 0); \ - radeon_emit(cs, value); \ + assert(__cs_num < __cs_reserved_dw); \ + __cs_buf[__cs_num++] = (value); \ + } while (0) + +#define radeon_emit_array(values, num) \ + do { \ + unsigned __n = (num); \ + assert(__cs_num + __n <= __cs_reserved_dw); \ + memcpy(__cs_buf + __cs_num, (values), __n * 4); \ + __cs_num += __n; \ + } while (0) + +/* Packet building helpers. Don't use directly. */ +#define __radeon_set_reg_seq(reg, num, idx, prefix_name, packet, reset_filter_cam) \ + do { \ + assert((reg) >= prefix_name##_REG_OFFSET && (reg) < prefix_name##_REG_END); \ + radeon_emit(PKT3(packet, num, 0) | PKT3_RESET_FILTER_CAM_S(reset_filter_cam)); \ + radeon_emit((((reg) - prefix_name##_REG_OFFSET) >> 2) | ((idx) << 28)); \ + } while (0) + +#define __radeon_set_reg(reg, idx, value, prefix_name, packet) \ + do { \ + __radeon_set_reg_seq(reg, 1, idx, prefix_name, packet, 0); \ + radeon_emit(value); \ } while (0) /* Packet building helpers for CONFIG registers. */ -#define radeon_set_config_reg_seq(cs, reg, num) radeon_set_reg_seq(cs, reg, num, 0, SI_CONFIG, PKT3_SET_CONFIG_REG, 0) +#define radeon_set_config_reg_seq(reg, num) __radeon_set_reg_seq(reg, num, 0, SI_CONFIG, PKT3_SET_CONFIG_REG, 0) -#define radeon_set_config_reg(cs, reg, value) radeon_set_reg(cs, reg, 0, value, SI_CONFIG, PKT3_SET_CONFIG_REG) +#define radeon_set_config_reg(reg, value) __radeon_set_reg(reg, 0, value, SI_CONFIG, PKT3_SET_CONFIG_REG) /* Packet building helpers for CONTEXT registers. */ -#define radeon_set_context_reg_seq(cs, reg, num) \ - radeon_set_reg_seq(cs, reg, num, 0, SI_CONTEXT, PKT3_SET_CONTEXT_REG, 0) +#define radeon_set_context_reg_seq(reg, num) __radeon_set_reg_seq(reg, num, 0, SI_CONTEXT, PKT3_SET_CONTEXT_REG, 0) -#define radeon_set_context_reg(cs, reg, value) radeon_set_reg(cs, reg, 0, value, SI_CONTEXT, PKT3_SET_CONTEXT_REG) +#define radeon_set_context_reg(reg, value) __radeon_set_reg(reg, 0, value, SI_CONTEXT, PKT3_SET_CONTEXT_REG) -#define radeon_set_context_reg_idx(cs, reg, idx, value) \ - radeon_set_reg(cs, reg, idx, value, SI_CONTEXT, PKT3_SET_CONTEXT_REG) +#define radeon_set_context_reg_idx(reg, idx, value) __radeon_set_reg(reg, idx, value, SI_CONTEXT, PKT3_SET_CONTEXT_REG) #define radeon_opt_set_context_reg(cmdbuf, reg, reg_enum, value) \ do { \ @@ -63,7 +87,7 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned const uint32_t __value = (value); \ if (!BITSET_TEST(__tracked_regs->reg_saved_mask, (reg_enum)) || \ __tracked_regs->reg_value[(reg_enum)] != __value) { \ - radeon_set_context_reg(__cmdbuf->cs, reg, __value); \ + radeon_set_context_reg(reg, __value); \ BITSET_SET(__tracked_regs->reg_saved_mask, (reg_enum)); \ __tracked_regs->reg_value[(reg_enum)] = __value; \ __cmdbuf->state.context_roll_without_scissor_emitted = true; \ @@ -77,9 +101,9 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned const uint32_t __v1 = (v1), __v2 = (v2); \ if (!BITSET_TEST_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 1, 0x3) || \ __tracked_regs->reg_value[(reg_enum)] != __v1 || __tracked_regs->reg_value[(reg_enum) + 1] != __v2) { \ - radeon_set_context_reg_seq(cmdbuf->cs, reg, 2); \ - radeon_emit(cmdbuf->cs, __v1); \ - radeon_emit(cmdbuf->cs, __v2); \ + radeon_set_context_reg_seq(reg, 2); \ + radeon_emit(__v1); \ + radeon_emit(__v2); \ BITSET_SET_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 1); \ __tracked_regs->reg_value[(reg_enum)] = __v1; \ __tracked_regs->reg_value[(reg_enum) + 1] = __v2; \ @@ -95,10 +119,10 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned if (!BITSET_TEST_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 2, 0x7) || \ __tracked_regs->reg_value[(reg_enum)] != __v1 || __tracked_regs->reg_value[(reg_enum) + 1] != __v2 || \ __tracked_regs->reg_value[(reg_enum) + 2] != __v3) { \ - radeon_set_context_reg_seq(cmdbuf->cs, reg, 3); \ - radeon_emit(cmdbuf->cs, __v1); \ - radeon_emit(cmdbuf->cs, __v2); \ - radeon_emit(cmdbuf->cs, __v3); \ + radeon_set_context_reg_seq(reg, 3); \ + radeon_emit(__v1); \ + radeon_emit(__v2); \ + radeon_emit(__v3); \ BITSET_SET_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 2); \ __tracked_regs->reg_value[(reg_enum)] = __v1; \ __tracked_regs->reg_value[(reg_enum) + 1] = __v2; \ @@ -115,11 +139,11 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned if (!BITSET_TEST_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 3, 0xf) || \ __tracked_regs->reg_value[(reg_enum)] != __v1 || __tracked_regs->reg_value[(reg_enum) + 1] != __v2 || \ __tracked_regs->reg_value[(reg_enum) + 2] != __v3 || __tracked_regs->reg_value[(reg_enum) + 3] != __v4) { \ - radeon_set_context_reg_seq(cmdbuf->cs, reg, 4); \ - radeon_emit(cmdbuf->cs, __v1); \ - radeon_emit(cmdbuf->cs, __v2); \ - radeon_emit(cmdbuf->cs, __v3); \ - radeon_emit(cmdbuf->cs, __v4); \ + radeon_set_context_reg_seq(reg, 4); \ + radeon_emit(__v1); \ + radeon_emit(__v2); \ + radeon_emit(__v3); \ + radeon_emit(__v4); \ BITSET_SET_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 3); \ __tracked_regs->reg_value[(reg_enum)] = __v1; \ __tracked_regs->reg_value[(reg_enum) + 1] = __v2; \ @@ -133,40 +157,39 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned do { \ struct radv_cmd_buffer *__cmdbuf = (cmdbuf); \ if (memcmp(values, saved_values, sizeof(uint32_t) * (num))) { \ - radeon_set_context_reg_seq(cmdbuf->cs, reg, num); \ - radeon_emit_array(cmdbuf->cs, values, num); \ + radeon_set_context_reg_seq(reg, num); \ + radeon_emit_array(values, num); \ memcpy(saved_values, values, sizeof(uint32_t) * (num)); \ __cmdbuf->state.context_roll_without_scissor_emitted = true; \ } \ } while (0) /* Packet building helpers for SH registers. */ -#define radeon_set_sh_reg_seq(cs, reg, num) radeon_set_reg_seq(cs, reg, num, 0, SI_SH, PKT3_SET_SH_REG, 0) +#define radeon_set_sh_reg_seq(reg, num) __radeon_set_reg_seq(reg, num, 0, SI_SH, PKT3_SET_SH_REG, 0) -#define radeon_set_sh_reg(cs, reg, value) radeon_set_reg(cs, reg, 0, value, SI_SH, PKT3_SET_SH_REG) +#define radeon_set_sh_reg(reg, value) __radeon_set_reg(reg, 0, value, SI_SH, PKT3_SET_SH_REG) -#define radeon_set_sh_reg_idx(info, cs, reg, idx, value) \ +#define radeon_set_sh_reg_idx(info, reg, idx, value) \ do { \ assert((idx)); \ unsigned __opcode = PKT3_SET_SH_REG_INDEX; \ if ((info)->gfx_level < GFX10) \ __opcode = PKT3_SET_SH_REG; \ - radeon_set_reg(cs, reg, idx, value, SI_SH, __opcode); \ + __radeon_set_reg(reg, idx, value, SI_SH, __opcode); \ } while (0) /* Packet building helpers for UCONFIG registers. */ -#define radeon_set_uconfig_reg_seq(cs, reg, num) \ - radeon_set_reg_seq(cs, reg, num, 0, CIK_UCONFIG, PKT3_SET_UCONFIG_REG, 0) +#define radeon_set_uconfig_reg_seq(reg, num) __radeon_set_reg_seq(reg, num, 0, CIK_UCONFIG, PKT3_SET_UCONFIG_REG, 0) -#define radeon_set_uconfig_reg(cs, reg, value) radeon_set_reg(cs, reg, 0, value, CIK_UCONFIG, PKT3_SET_UCONFIG_REG) +#define radeon_set_uconfig_reg(reg, value) __radeon_set_reg(reg, 0, value, CIK_UCONFIG, PKT3_SET_UCONFIG_REG) -#define radeon_set_uconfig_reg_idx(info, cs, reg, idx, value) \ +#define radeon_set_uconfig_reg_idx(info, reg, idx, value) \ do { \ assert((idx)); \ unsigned __opcode = PKT3_SET_UCONFIG_REG_INDEX; \ if ((info)->gfx_level < GFX9 || ((info)->gfx_level == GFX9 && (info)->me_fw_version < 26)) \ __opcode = PKT3_SET_UCONFIG_REG; \ - radeon_set_reg(cs, reg, idx, value, CIK_UCONFIG, __opcode); \ + __radeon_set_reg(reg, idx, value, CIK_UCONFIG, __opcode); \ } while (0) /* @@ -174,55 +197,54 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned * that means that it can skip register writes due to not taking correctly into account the * fields from the GRBM_GFX_INDEX. With this bit we can force the write. */ -#define radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, reg, num) \ +#define radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, reg, num) \ do { \ const bool __filter_cam_workaround = (gfx_level) >= GFX10 && (ring) == AMD_IP_GFX; \ - radeon_set_reg_seq(cs, reg, num, 0, CIK_UCONFIG, PKT3_SET_UCONFIG_REG, __filter_cam_workaround); \ + __radeon_set_reg_seq(reg, num, 0, CIK_UCONFIG, PKT3_SET_UCONFIG_REG, __filter_cam_workaround); \ } while (0) -#define radeon_set_uconfig_perfctr_reg(gfx_level, ring, cs, reg, value) \ +#define radeon_set_uconfig_perfctr_reg(gfx_level, ring, reg, value) \ do { \ - radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, reg, 1); \ - radeon_emit(cs, value); \ + radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, reg, 1); \ + radeon_emit(value); \ } while (0) -#define radeon_set_privileged_config_reg(cs, reg, value) \ +#define radeon_set_privileged_config_reg(reg, value) \ do { \ assert((reg) < CIK_UCONFIG_REG_OFFSET); \ - assert((cs)->cdw + 6 <= (cs)->reserved_dw); \ - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); \ - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_PERF)); \ - radeon_emit(cs, value); \ - radeon_emit(cs, 0); /* unused */ \ - radeon_emit(cs, (reg) >> 2); \ - radeon_emit(cs, 0); /* unused */ \ + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); \ + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_PERF)); \ + radeon_emit(value); \ + radeon_emit(0); /* unused */ \ + radeon_emit((reg) >> 2); \ + radeon_emit(0); /* unused */ \ } while (0) -#define radeon_event_write_predicate(cs, event_type, predicate) \ +#define radeon_event_write_predicate(event_type, predicate) \ do { \ unsigned __event_type = (event_type); \ - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, predicate)); \ - radeon_emit(cs, EVENT_TYPE(__event_type) | EVENT_INDEX(__event_type == V_028A90_VS_PARTIAL_FLUSH || \ - __event_type == V_028A90_PS_PARTIAL_FLUSH || \ - __event_type == V_028A90_CS_PARTIAL_FLUSH \ - ? 4 \ - : 0)); \ + radeon_emit(PKT3(PKT3_EVENT_WRITE, 0, predicate)); \ + radeon_emit(EVENT_TYPE(__event_type) | EVENT_INDEX(__event_type == V_028A90_VS_PARTIAL_FLUSH || \ + __event_type == V_028A90_PS_PARTIAL_FLUSH || \ + __event_type == V_028A90_CS_PARTIAL_FLUSH \ + ? 4 \ + : 0)); \ } while (0) -#define radeon_event_write(cs, event_type) radeon_event_write_predicate(cs, event_type, false) +#define radeon_event_write(event_type) radeon_event_write_predicate(event_type, false) -#define radeon_emit_32bit_pointer(cs, sh_offset, va, info) \ +#define radeon_emit_32bit_pointer(sh_offset, va, info) \ do { \ assert((va) == 0 || ((va) >> 32) == (info)->address32_hi); \ - radeon_set_sh_reg_seq(cs, sh_offset, 1); \ - radeon_emit(cs, va); \ + radeon_set_sh_reg_seq(sh_offset, 1); \ + radeon_emit(va); \ } while (0) -#define radeon_emit_64bit_pointer(cs, sh_offset, va) \ +#define radeon_emit_64bit_pointer(sh_offset, va) \ do { \ - radeon_set_sh_reg_seq(cs, sh_offset, 2); \ - radeon_emit(cs, va); \ - radeon_emit(cs, va >> 32); \ + radeon_set_sh_reg_seq(sh_offset, 2); \ + radeon_emit(va); \ + radeon_emit(va >> 32); \ } while (0) ALWAYS_INLINE static void @@ -232,13 +254,15 @@ radv_cp_wait_mem(struct radeon_cmdbuf *cs, const enum radv_queue_family qf, cons assert(op == WAIT_REG_MEM_EQUAL || op == WAIT_REG_MEM_NOT_EQUAL || op == WAIT_REG_MEM_GREATER_OR_EQUAL); if (qf == RADV_QUEUE_GENERAL || qf == RADV_QUEUE_COMPUTE) { - radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, false)); - radeon_emit(cs, op | WAIT_REG_MEM_MEM_SPACE(1)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, ref); /* reference value */ - radeon_emit(cs, mask); /* mask */ - radeon_emit(cs, 4); /* poll interval */ + radeon_begin(cs); + radeon_emit(PKT3(PKT3_WAIT_REG_MEM, 5, false)); + radeon_emit(op | WAIT_REG_MEM_MEM_SPACE(1)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(ref); /* reference value */ + radeon_emit(mask); /* mask */ + radeon_emit(4); /* poll interval */ + radeon_end(); } else if (qf == RADV_QUEUE_TRANSFER) { radv_sdma_emit_wait_mem(cs, op, va, ref, mask); } else { @@ -254,10 +278,12 @@ radv_cs_write_data_head(const struct radv_device *device, struct radeon_cmdbuf * const unsigned cdw_end = radeon_check_space(device->ws, cs, 4 + count); if (qf == RADV_QUEUE_GENERAL || qf == RADV_QUEUE_COMPUTE) { - radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + count, predicating)); - radeon_emit(cs, S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(engine_sel)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_WRITE_DATA, 2 + count, predicating)); + radeon_emit(S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(engine_sel)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_end(); } else if (qf == RADV_QUEUE_TRANSFER) { radv_sdma_emit_write_data_head(cs, va, count); } else { @@ -273,7 +299,10 @@ radv_cs_write_data(const struct radv_device *device, struct radeon_cmdbuf *cs, c const bool predicating) { ASSERTED const unsigned cdw_end = radv_cs_write_data_head(device, cs, qf, engine_sel, va, count, predicating); - radeon_emit_array(cs, dwords, count); + + radeon_begin(cs); + radeon_emit_array(dwords, count); + radeon_end(); assert(cs->cdw == cdw_end); } @@ -293,7 +322,9 @@ void radv_cs_write_data_imm(struct radeon_cmdbuf *cs, unsigned engine_sel, uint6 static inline void radv_emit_pm4_commands(struct radeon_cmdbuf *cs, const struct ac_pm4_state *pm4) { - radeon_emit_array(cs, pm4->pm4, pm4->ndw); + radeon_begin(cs); + radeon_emit_array(pm4->pm4, pm4->ndw); + radeon_end(); } #endif /* RADV_CS_H */ diff --git a/src/amd/vulkan/radv_device.c b/src/amd/vulkan/radv_device.c index 5fb48b01825..df24e89ecf3 100644 --- a/src/amd/vulkan/radv_device.c +++ b/src/amd/vulkan/radv_device.c @@ -972,40 +972,42 @@ radv_emit_default_sample_locations(const struct radv_physical_device *pdev, stru { uint64_t centroid_priority; + radeon_begin(cs); + switch (nr_samples) { default: case 1: centroid_priority = centroid_priority_1x; - radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_1x); - radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_1x); - radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_1x); - radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_1x); + radeon_set_context_reg(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_1x); + radeon_set_context_reg(R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_1x); + radeon_set_context_reg(R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_1x); + radeon_set_context_reg(R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_1x); break; case 2: centroid_priority = centroid_priority_2x; - radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_2x); - radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_2x); - radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_2x); - radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_2x); + radeon_set_context_reg(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_2x); + radeon_set_context_reg(R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_2x); + radeon_set_context_reg(R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_2x); + radeon_set_context_reg(R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_2x); break; case 4: centroid_priority = centroid_priority_4x; - radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_4x); - radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_4x); - radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_4x); - radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_4x); + radeon_set_context_reg(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_4x); + radeon_set_context_reg(R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_4x); + radeon_set_context_reg(R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_4x); + radeon_set_context_reg(R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_4x); break; case 8: centroid_priority = centroid_priority_8x; - radeon_set_context_reg_seq(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, 14); - radeon_emit_array(cs, sample_locs_8x, 4); - radeon_emit_array(cs, sample_locs_8x, 4); - radeon_emit_array(cs, sample_locs_8x, 4); - radeon_emit_array(cs, sample_locs_8x, 2); + radeon_set_context_reg_seq(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, 14); + radeon_emit_array(sample_locs_8x, 4); + radeon_emit_array(sample_locs_8x, 4); + radeon_emit_array(sample_locs_8x, 4); + radeon_emit_array(sample_locs_8x, 2); break; } @@ -1014,17 +1016,19 @@ radv_emit_default_sample_locations(const struct radv_physical_device *pdev, stru * support 16 samples. */ if (pdev->info.gfx_level >= GFX7) { - radeon_set_context_reg(cs, R_02882C_PA_SU_PRIM_FILTER_CNTL, + radeon_set_context_reg(R_02882C_PA_SU_PRIM_FILTER_CNTL, S_02882C_XMAX_RIGHT_EXCLUSION(1) | S_02882C_YMAX_BOTTOM_EXCLUSION(1)); } if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg_seq(cs, R_028BF0_PA_SC_CENTROID_PRIORITY_0, 2); + radeon_set_context_reg_seq(R_028BF0_PA_SC_CENTROID_PRIORITY_0, 2); } else { - radeon_set_context_reg_seq(cs, R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2); + radeon_set_context_reg_seq(R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2); } - radeon_emit(cs, centroid_priority); - radeon_emit(cs, centroid_priority >> 32); + radeon_emit(centroid_priority); + radeon_emit(centroid_priority >> 32); + + radeon_end(); } static void diff --git a/src/amd/vulkan/radv_perfcounter.c b/src/amd/vulkan/radv_perfcounter.c index 3d5f7721703..1e5eb70b5f8 100644 --- a/src/amd/vulkan/radv_perfcounter.c +++ b/src/amd/vulkan/radv_perfcounter.c @@ -19,25 +19,33 @@ radv_perfcounter_emit_shaders(struct radv_device *device, struct radeon_cmdbuf * { const struct radv_physical_device *pdev = radv_device_physical(device); + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX10) { - radeon_set_uconfig_reg(cs, R_036780_SQ_PERFCOUNTER_CTRL, shaders & 0x7f); + radeon_set_uconfig_reg(R_036780_SQ_PERFCOUNTER_CTRL, shaders & 0x7f); if (pdev->info.gfx_level >= GFX11) - radeon_set_uconfig_reg(cs, R_036760_SQG_PERFCOUNTER_CTRL, shaders & 0x7f); + radeon_set_uconfig_reg(R_036760_SQG_PERFCOUNTER_CTRL, shaders & 0x7f); } else { - radeon_set_uconfig_reg_seq(cs, R_036780_SQ_PERFCOUNTER_CTRL, 2); - radeon_emit(cs, shaders & 0x7f); - radeon_emit(cs, 0xffffffff); + radeon_set_uconfig_reg_seq(R_036780_SQ_PERFCOUNTER_CTRL, 2); + radeon_emit(shaders & 0x7f); + radeon_emit(0xffffffff); } + + radeon_end(); } static void radv_emit_windowed_counters(struct radv_device *device, struct radeon_cmdbuf *cs, int family, bool enable) { + radeon_begin(cs); + if (family == RADV_QUEUE_GENERAL) { - radeon_event_write(cs, enable ? V_028A90_PERFCOUNTER_START : V_028A90_PERFCOUNTER_STOP); + radeon_event_write(enable ? V_028A90_PERFCOUNTER_START : V_028A90_PERFCOUNTER_STOP); } - radeon_set_sh_reg(cs, R_00B82C_COMPUTE_PERFCOUNT_ENABLE, S_00B82C_PERFCOUNT_ENABLE(enable)); + radeon_set_sh_reg(R_00B82C_COMPUTE_PERFCOUNT_ENABLE, S_00B82C_PERFCOUNT_ENABLE(enable)); + + radeon_end(); } void @@ -52,7 +60,9 @@ radv_perfcounter_emit_reset(struct radeon_cmdbuf *cs, bool is_spm) cp_perfmon_cntl = S_036020_PERFMON_STATE(V_036020_CP_PERFMON_STATE_DISABLE_AND_RESET); } - radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl); + radeon_begin(cs); + radeon_set_uconfig_reg(R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl); + radeon_end(); } static void @@ -67,7 +77,9 @@ radv_perfcounter_emit_start(struct radeon_cmdbuf *cs, bool is_spm) cp_perfmon_cntl = S_036020_PERFMON_STATE(V_036020_CP_PERFMON_STATE_START_COUNTING); } - radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl); + radeon_begin(cs); + radeon_set_uconfig_reg(R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl); + radeon_end(); } static void @@ -83,7 +95,9 @@ radv_perfcounter_emit_stop(struct radeon_cmdbuf *cs, bool is_spm) S_036020_PERFMON_STATE(V_036020_CP_PERFMON_STATE_STOP_COUNTING) | S_036020_PERFMON_SAMPLE_ENABLE(1); } - radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl); + radeon_begin(cs); + radeon_set_uconfig_reg(R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl); + radeon_end(); } void @@ -113,7 +127,9 @@ radv_perfcounter_emit_spm_stop(struct radv_device *device, struct radeon_cmdbuf static void radv_perfcounter_emit_sample(struct radeon_cmdbuf *cs) { - radeon_event_write(cs, V_028A90_PERFCOUNTER_SAMPLE); + radeon_begin(cs); + radeon_event_write(V_028A90_PERFCOUNTER_SAMPLE); + radeon_end(); } enum radv_perfcounter_op { @@ -492,7 +508,9 @@ radv_emit_instance(struct radv_cmd_buffer *cmd_buffer, int se, int instance) value |= S_030800_INSTANCE_BROADCAST_WRITES(1); } - radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, value); + radeon_begin(cs); + radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, value); + radeon_end(); } static void @@ -512,15 +530,18 @@ radv_emit_select(struct radv_cmd_buffer *cmd_buffer, struct ac_pc_block *block, if (!regs->select0) return; + radeon_begin(cs); + for (idx = 0; idx < count; ++idx) { - radeon_set_uconfig_perfctr_reg(gfx_level, ring, cs, regs->select0[idx], - G_REG_SEL(selectors[idx]) | regs->select_or); + radeon_set_uconfig_perfctr_reg(gfx_level, ring, regs->select0[idx], G_REG_SEL(selectors[idx]) | regs->select_or); } for (idx = 0; idx < regs->num_spm_counters; idx++) { - radeon_set_uconfig_reg_seq(cs, regs->select1[idx], 1); - radeon_emit(cs, 0); + radeon_set_uconfig_reg_seq(regs->select1[idx], 1); + radeon_emit(0); } + + radeon_end(); } static void @@ -539,13 +560,15 @@ radv_pc_emit_block_instance_read(struct radv_cmd_buffer *cmd_buffer, struct ac_p if (regs->counters) reg = regs->counters[idx]; - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) | COPY_DATA_DST_SEL(COPY_DATA_TC_L2) | COPY_DATA_WR_CONFIRM | - COPY_DATA_COUNT_SEL); /* 64 bits */ - radeon_emit(cs, reg >> 2); - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_PERF) | COPY_DATA_DST_SEL(COPY_DATA_TC_L2) | COPY_DATA_WR_CONFIRM | + COPY_DATA_COUNT_SEL); /* 64 bits */ + radeon_emit(reg >> 2); + radeon_emit(0); /* unused */ + radeon_emit(va); + radeon_emit(va >> 32); + radeon_end(); va += sizeof(uint64_t) * 2 * radv_pc_get_num_instances(pdev, block); reg += reg_delta; @@ -575,19 +598,23 @@ radv_pc_wait_idle(struct radv_cmd_buffer *cmd_buffer) { struct radeon_cmdbuf *cs = cmd_buffer->cs; - radeon_event_write(cs, V_028A90_CS_PARTIAL_FLUSH); + radeon_begin(cs); - radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 6, 0)); - radeon_emit(cs, 0); /* CP_COHER_CNTL */ - radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */ - radeon_emit(cs, 0xffffff); /* CP_COHER_SIZE_HI */ - radeon_emit(cs, 0); /* CP_COHER_BASE */ - radeon_emit(cs, 0); /* CP_COHER_BASE_HI */ - radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */ - radeon_emit(cs, 0); /* GCR_CNTL */ + radeon_event_write(V_028A90_CS_PARTIAL_FLUSH); - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_ACQUIRE_MEM, 6, 0)); + radeon_emit(0); /* CP_COHER_CNTL */ + radeon_emit(0xffffffff); /* CP_COHER_SIZE */ + radeon_emit(0xffffff); /* CP_COHER_SIZE_HI */ + radeon_emit(0); /* CP_COHER_BASE */ + radeon_emit(0); /* CP_COHER_BASE_HI */ + radeon_emit(0x0000000A); /* POLL_INTERVAL */ + radeon_emit(0); /* GCR_CNTL */ + + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0)); + radeon_emit(0); + + radeon_end(); } static void @@ -607,11 +634,13 @@ radv_pc_stop_and_sample(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query uint64_t pred_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_PASS_OFFSET + 8 * pass; uint64_t reg_va = va + (end ? 8 : 0); - radeon_emit(cs, PKT3(PKT3_COND_EXEC, 3, 0)); - radeon_emit(cs, pred_va); - radeon_emit(cs, pred_va >> 32); - radeon_emit(cs, 0); /* Cache policy */ - radeon_emit(cs, 0); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COND_EXEC, 3, 0)); + radeon_emit(pred_va); + radeon_emit(pred_va >> 32); + radeon_emit(0); /* Cache policy */ + radeon_emit(0); + radeon_end(); uint32_t *skip_dwords = cs->buf + (cs->cdw - 1); @@ -677,11 +706,13 @@ radv_pc_begin_query(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query_poo for (unsigned pass = 0; pass < pool->num_passes; ++pass) { uint64_t pred_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_PASS_OFFSET + 8 * pass; - radeon_emit(cs, PKT3(PKT3_COND_EXEC, 3, 0)); - radeon_emit(cs, pred_va); - radeon_emit(cs, pred_va >> 32); - radeon_emit(cs, 0); /* Cache policy */ - radeon_emit(cs, 0); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COND_EXEC, 3, 0)); + radeon_emit(pred_va); + radeon_emit(pred_va >> 32); + radeon_emit(0); /* Cache policy */ + radeon_emit(0); + radeon_end(); uint32_t *skip_dwords = cs->buf + (cs->cdw - 1); diff --git a/src/amd/vulkan/radv_query.c b/src/amd/vulkan/radv_query.c index a31eb93214a..6a29af3b6f7 100644 --- a/src/amd/vulkan/radv_query.c +++ b/src/amd/vulkan/radv_query.c @@ -38,12 +38,14 @@ static void radv_query_shader(struct radv_cmd_buffer *cmd_buffer, VkQueryType qu static void gfx10_copy_shader_query(struct radeon_cmdbuf *cs, uint32_t src_sel, uint64_t src_va, uint64_t dst_va) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(src_sel) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, src_va); - radeon_emit(cs, src_va >> 32); - radeon_emit(cs, dst_va); - radeon_emit(cs, dst_va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(src_sel) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); + radeon_emit(src_va); + radeon_emit(src_va >> 32); + radeon_emit(dst_va); + radeon_emit(dst_va >> 32); + radeon_end(); } static void @@ -95,18 +97,20 @@ static void radv_emit_event_write(const struct radeon_info *info, struct radeon_cmdbuf *cs, enum radv_event_write event, uint64_t va) { + radeon_begin(cs); + if (event == RADV_EVENT_WRITE_PIPELINE_STAT) { - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0)); - radeon_emit(cs, EVENT_TYPE(V_028A90_SAMPLE_PIPELINESTAT) | EVENT_INDEX(2)); + radeon_emit(PKT3(PKT3_EVENT_WRITE, 2, 0)); + radeon_emit(EVENT_TYPE(V_028A90_SAMPLE_PIPELINESTAT) | EVENT_INDEX(2)); } else if (event == RADV_EVENT_WRITE_OCCLUSION_QUERY) { if (info->gfx_level >= GFX11 && info->pfp_fw_version >= EVENT_WRITE_ZPASS_PFP_VERSION) { - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_ZPASS, 1, 0)); + radeon_emit(PKT3(PKT3_EVENT_WRITE_ZPASS, 1, 0)); } else { - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0)); + radeon_emit(PKT3(PKT3_EVENT_WRITE, 2, 0)); if (info->gfx_level >= GFX11) { - radeon_emit(cs, EVENT_TYPE(V_028A90_PIXEL_PIPE_STAT_DUMP) | EVENT_INDEX(1)); + radeon_emit(EVENT_TYPE(V_028A90_PIXEL_PIPE_STAT_DUMP) | EVENT_INDEX(1)); } else { - radeon_emit(cs, EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1)); + radeon_emit(EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1)); } } } else { @@ -119,12 +123,14 @@ radv_emit_event_write(const struct radeon_info *info, struct radeon_cmdbuf *cs, V_028A90_SAMPLE_STREAMOUTSTATS3, }; - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0)); - radeon_emit(cs, EVENT_TYPE(streamout_events[event]) | EVENT_INDEX(3)); + radeon_emit(PKT3(PKT3_EVENT_WRITE, 2, 0)); + radeon_emit(EVENT_TYPE(streamout_events[event]) | EVENT_INDEX(3)); } - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_emit(va); + radeon_emit(va >> 32); + + radeon_end(); } static void @@ -2706,13 +2712,15 @@ radv_write_timestamp(struct radv_cmd_buffer *cmd_buffer, uint64_t va, VkPipeline struct radeon_cmdbuf *cs = cmd_buffer->cs; if (stage == VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_COUNT_SEL | COPY_DATA_WR_CONFIRM | COPY_DATA_SRC_SEL(COPY_DATA_TIMESTAMP) | - COPY_DATA_DST_SEL(V_370_MEM)); - radeon_emit(cs, 0); - radeon_emit(cs, 0); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_COUNT_SEL | COPY_DATA_WR_CONFIRM | COPY_DATA_SRC_SEL(COPY_DATA_TIMESTAMP) | + COPY_DATA_DST_SEL(V_370_MEM)); + radeon_emit(0); + radeon_emit(0); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_end(); } else { radv_cs_emit_write_event_eop(cs, pdev->info.gfx_level, cmd_buffer->qf, V_028A90_BOTTOM_OF_PIPE_TS, 0, EOP_DST_SEL_MEM, EOP_DATA_SEL_TIMESTAMP, va, 0, cmd_buffer->gfx9_eop_bug_va); @@ -2791,6 +2799,8 @@ radv_CmdWriteAccelerationStructuresPropertiesKHR(VkCommandBuffer commandBuffer, ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cs, 6 * accelerationStructureCount); + radeon_begin(cs); + for (uint32_t i = 0; i < accelerationStructureCount; ++i) { VK_FROM_HANDLE(vk_acceleration_structure, accel_struct, pAccelerationStructures[i]); uint64_t va = vk_acceleration_structure_get_va(accel_struct); @@ -2812,16 +2822,17 @@ radv_CmdWriteAccelerationStructuresPropertiesKHR(VkCommandBuffer commandBuffer, unreachable("Unhandle accel struct query type."); } - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | - COPY_DATA_COUNT_SEL | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, query_va); - radeon_emit(cs, query_va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL | + COPY_DATA_WR_CONFIRM); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(query_va); + radeon_emit(query_va >> 32); query_va += pool->stride; } + radeon_end(); assert(cmd_buffer->cs->cdw <= cdw_max); } diff --git a/src/amd/vulkan/radv_queue.c b/src/amd/vulkan/radv_queue.c index d5123799bea..b42fbb64201 100644 --- a/src/amd/vulkan/radv_queue.c +++ b/src/amd/vulkan/radv_queue.c @@ -373,15 +373,19 @@ radv_emit_gs_ring_sizes(struct radv_device *device, struct radeon_cmdbuf *cs, st if (gsvs_ring_bo) radv_cs_add_buffer(device->ws, cs, gsvs_ring_bo); + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX7) { - radeon_set_uconfig_reg_seq(cs, R_030900_VGT_ESGS_RING_SIZE, 2); - radeon_emit(cs, esgs_ring_size >> 8); - radeon_emit(cs, gsvs_ring_size >> 8); + radeon_set_uconfig_reg_seq(R_030900_VGT_ESGS_RING_SIZE, 2); + radeon_emit(esgs_ring_size >> 8); + radeon_emit(gsvs_ring_size >> 8); } else { - radeon_set_config_reg_seq(cs, R_0088C8_VGT_ESGS_RING_SIZE, 2); - radeon_emit(cs, esgs_ring_size >> 8); - radeon_emit(cs, gsvs_ring_size >> 8); + radeon_set_config_reg_seq(R_0088C8_VGT_ESGS_RING_SIZE, 2); + radeon_emit(esgs_ring_size >> 8); + radeon_emit(gsvs_ring_size >> 8); } + + radeon_end(); } static void @@ -398,29 +402,33 @@ radv_emit_tess_factor_ring(struct radv_device *device, struct radeon_cmdbuf *cs, radv_cs_add_buffer(device->ws, cs, tess_rings_bo); + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX7) { if (pdev->info.gfx_level >= GFX11) { /* TF_RING_SIZE is per SE on GFX11. */ tf_ring_size /= pdev->info.max_se; } - radeon_set_uconfig_reg(cs, R_030938_VGT_TF_RING_SIZE, S_030938_SIZE(tf_ring_size)); - radeon_set_uconfig_reg(cs, R_030940_VGT_TF_MEMORY_BASE, tf_va >> 8); + radeon_set_uconfig_reg(R_030938_VGT_TF_RING_SIZE, S_030938_SIZE(tf_ring_size)); + radeon_set_uconfig_reg(R_030940_VGT_TF_MEMORY_BASE, tf_va >> 8); if (pdev->info.gfx_level >= GFX12) { - radeon_set_uconfig_reg(cs, R_03099C_VGT_TF_MEMORY_BASE_HI, S_03099C_BASE_HI(tf_va >> 40)); + radeon_set_uconfig_reg(R_03099C_VGT_TF_MEMORY_BASE_HI, S_03099C_BASE_HI(tf_va >> 40)); } else if (pdev->info.gfx_level >= GFX10) { - radeon_set_uconfig_reg(cs, R_030984_VGT_TF_MEMORY_BASE_HI, S_030984_BASE_HI(tf_va >> 40)); + radeon_set_uconfig_reg(R_030984_VGT_TF_MEMORY_BASE_HI, S_030984_BASE_HI(tf_va >> 40)); } else if (pdev->info.gfx_level == GFX9) { - radeon_set_uconfig_reg(cs, R_030944_VGT_TF_MEMORY_BASE_HI, S_030944_BASE_HI(tf_va >> 40)); + radeon_set_uconfig_reg(R_030944_VGT_TF_MEMORY_BASE_HI, S_030944_BASE_HI(tf_va >> 40)); } - radeon_set_uconfig_reg(cs, R_03093C_VGT_HS_OFFCHIP_PARAM, pdev->hs.hs_offchip_param); + radeon_set_uconfig_reg(R_03093C_VGT_HS_OFFCHIP_PARAM, pdev->hs.hs_offchip_param); } else { - radeon_set_config_reg(cs, R_008988_VGT_TF_RING_SIZE, S_008988_SIZE(tf_ring_size)); - radeon_set_config_reg(cs, R_0089B8_VGT_TF_MEMORY_BASE, tf_va >> 8); - radeon_set_config_reg(cs, R_0089B0_VGT_HS_OFFCHIP_PARAM, pdev->hs.hs_offchip_param); + radeon_set_config_reg(R_008988_VGT_TF_RING_SIZE, S_008988_SIZE(tf_ring_size)); + radeon_set_config_reg(R_0089B8_VGT_TF_MEMORY_BASE, tf_va >> 8); + radeon_set_config_reg(R_0089B0_VGT_HS_OFFCHIP_PARAM, pdev->hs.hs_offchip_param); } + + radeon_end(); } static VkResult @@ -466,12 +474,16 @@ radv_emit_task_rings(struct radv_device *device, struct radeon_cmdbuf *cs, struc assert(util_is_aligned(task_ctrlbuf_va, 256)); radv_cs_add_buffer(device->ws, cs, task_rings_bo); + radeon_begin(cs); + /* Tell the GPU where the task control buffer is. */ - radeon_emit(cs, PKT3(PKT3_DISPATCH_TASK_STATE_INIT, 1, 0) | PKT3_SHADER_TYPE_S(!!compute)); + radeon_emit(PKT3(PKT3_DISPATCH_TASK_STATE_INIT, 1, 0) | PKT3_SHADER_TYPE_S(!!compute)); /* bits [31:8]: control buffer address lo, bits[7:0]: reserved (set to zero) */ - radeon_emit(cs, task_ctrlbuf_va & 0xFFFFFF00); + radeon_emit(task_ctrlbuf_va & 0xFFFFFF00); /* bits [31:0]: control buffer address hi */ - radeon_emit(cs, task_ctrlbuf_va >> 32); + radeon_emit(task_ctrlbuf_va >> 32); + + radeon_end(); } static void @@ -486,20 +498,24 @@ radv_emit_graphics_scratch(struct radv_device *device, struct radeon_cmdbuf *cs, radv_cs_add_buffer(device->ws, cs, scratch_bo); + radeon_begin(cs); + if (gpu_info->gfx_level >= GFX11) { uint64_t va = radv_buffer_get_va(scratch_bo); /* WAVES is per SE for SPI_TMPRING_SIZE. */ waves /= gpu_info->max_se; - radeon_set_context_reg_seq(cs, R_0286E8_SPI_TMPRING_SIZE, 3); - radeon_emit(cs, S_0286E8_WAVES(waves) | S_0286E8_WAVESIZE(DIV_ROUND_UP(size_per_wave, 256))); - radeon_emit(cs, va >> 8); /* SPI_GFX_SCRATCH_BASE_LO */ - radeon_emit(cs, va >> 40); /* SPI_GFX_SCRATCH_BASE_HI */ + radeon_set_context_reg_seq(R_0286E8_SPI_TMPRING_SIZE, 3); + radeon_emit(S_0286E8_WAVES(waves) | S_0286E8_WAVESIZE(DIV_ROUND_UP(size_per_wave, 256))); + radeon_emit(va >> 8); /* SPI_GFX_SCRATCH_BASE_LO */ + radeon_emit(va >> 40); /* SPI_GFX_SCRATCH_BASE_HI */ } else { - radeon_set_context_reg(cs, R_0286E8_SPI_TMPRING_SIZE, + radeon_set_context_reg(R_0286E8_SPI_TMPRING_SIZE, S_0286E8_WAVES(waves) | S_0286E8_WAVESIZE(DIV_ROUND_UP(size_per_wave, 1024))); } + + radeon_end(); } static void @@ -524,21 +540,25 @@ radv_emit_compute_scratch(struct radv_device *device, struct radeon_cmdbuf *cs, radv_cs_add_buffer(device->ws, cs, compute_scratch_bo); + radeon_begin(cs); + if (gpu_info->gfx_level >= GFX11) { - radeon_set_sh_reg_seq(cs, R_00B840_COMPUTE_DISPATCH_SCRATCH_BASE_LO, 2); - radeon_emit(cs, scratch_va >> 8); - radeon_emit(cs, scratch_va >> 40); + radeon_set_sh_reg_seq(R_00B840_COMPUTE_DISPATCH_SCRATCH_BASE_LO, 2); + radeon_emit(scratch_va >> 8); + radeon_emit(scratch_va >> 40); waves /= gpu_info->max_se; } - radeon_set_sh_reg_seq(cs, R_00B900_COMPUTE_USER_DATA_0, 2); - radeon_emit(cs, scratch_va); - radeon_emit(cs, rsrc1); + radeon_set_sh_reg_seq(R_00B900_COMPUTE_USER_DATA_0, 2); + radeon_emit(scratch_va); + radeon_emit(rsrc1); - radeon_set_sh_reg(cs, R_00B860_COMPUTE_TMPRING_SIZE, + radeon_set_sh_reg(R_00B860_COMPUTE_TMPRING_SIZE, S_00B860_WAVES(waves) | S_00B860_WAVESIZE(DIV_ROUND_UP(size_per_wave, gpu_info->gfx_level >= GFX11 ? 256 : 1024))); + + radeon_end(); } static void @@ -554,7 +574,9 @@ radv_emit_compute_shader_pointers(struct radv_device *device, struct radeon_cmdb /* Compute shader user data 0-1 have the scratch pointer (unlike GFX shaders), * so emit the descriptor pointer to user data 2-3 instead (task_ring_offsets arg). */ - radeon_emit_64bit_pointer(cs, R_00B908_COMPUTE_USER_DATA_2, va); + radeon_begin(cs); + radeon_emit_64bit_pointer(R_00B908_COMPUTE_USER_DATA_2, va); + radeon_end(); } static void @@ -571,33 +593,35 @@ radv_emit_graphics_shader_pointers(struct radv_device *device, struct radeon_cmd radv_cs_add_buffer(device->ws, cs, descriptor_bo); + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX12) { uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B410_SPI_SHADER_PGM_LO_HS, R_00B210_SPI_SHADER_PGM_LO_GS}; for (int i = 0; i < ARRAY_SIZE(regs); ++i) { - radeon_emit_64bit_pointer(cs, regs[i], va); + radeon_emit_64bit_pointer(regs[i], va); } } else if (pdev->info.gfx_level >= GFX11) { uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B420_SPI_SHADER_PGM_LO_HS, R_00B220_SPI_SHADER_PGM_LO_GS}; for (int i = 0; i < ARRAY_SIZE(regs); ++i) { - radeon_emit_64bit_pointer(cs, regs[i], va); + radeon_emit_64bit_pointer(regs[i], va); } } else if (pdev->info.gfx_level >= GFX10) { uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B130_SPI_SHADER_USER_DATA_VS_0, R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS, R_00B408_SPI_SHADER_USER_DATA_ADDR_LO_HS}; for (int i = 0; i < ARRAY_SIZE(regs); ++i) { - radeon_emit_64bit_pointer(cs, regs[i], va); + radeon_emit_64bit_pointer(regs[i], va); } } else if (pdev->info.gfx_level == GFX9) { uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B130_SPI_SHADER_USER_DATA_VS_0, R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS, R_00B408_SPI_SHADER_USER_DATA_ADDR_LO_HS}; for (int i = 0; i < ARRAY_SIZE(regs); ++i) { - radeon_emit_64bit_pointer(cs, regs[i], va); + radeon_emit_64bit_pointer(regs[i], va); } } else { uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B130_SPI_SHADER_USER_DATA_VS_0, @@ -605,9 +629,11 @@ radv_emit_graphics_shader_pointers(struct radv_device *device, struct radeon_cmd R_00B430_SPI_SHADER_USER_DATA_HS_0, R_00B530_SPI_SHADER_USER_DATA_LS_0}; for (int i = 0; i < ARRAY_SIZE(regs); ++i) { - radeon_emit_64bit_pointer(cs, regs[i], va); + radeon_emit_64bit_pointer(regs[i], va); } } + + radeon_end(); } static void @@ -626,53 +652,56 @@ radv_emit_ge_rings(struct radv_device *device, struct radeon_cmdbuf *cs, struct radv_cs_add_buffer(device->ws, cs, ge_rings_bo); + radeon_begin(cs); + /* We must wait for idle using an EOP event before changing the attribute ring registers. Use the * bottom-of-pipe EOP event, but increment the PWS counter instead of writing memory. */ - radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, 6, 0)); - radeon_emit(cs, S_490_EVENT_TYPE(V_028A90_BOTTOM_OF_PIPE_TS) | S_490_EVENT_INDEX(5) | S_490_PWS_ENABLE(1)); - radeon_emit(cs, 0); /* DST_SEL, INT_SEL, DATA_SEL */ - radeon_emit(cs, 0); /* ADDRESS_LO */ - radeon_emit(cs, 0); /* ADDRESS_HI */ - radeon_emit(cs, 0); /* DATA_LO */ - radeon_emit(cs, 0); /* DATA_HI */ - radeon_emit(cs, 0); /* INT_CTXID */ + radeon_emit(PKT3(PKT3_RELEASE_MEM, 6, 0)); + radeon_emit(S_490_EVENT_TYPE(V_028A90_BOTTOM_OF_PIPE_TS) | S_490_EVENT_INDEX(5) | S_490_PWS_ENABLE(1)); + radeon_emit(0); /* DST_SEL, INT_SEL, DATA_SEL */ + radeon_emit(0); /* ADDRESS_LO */ + radeon_emit(0); /* ADDRESS_HI */ + radeon_emit(0); /* DATA_LO */ + radeon_emit(0); /* DATA_HI */ + radeon_emit(0); /* INT_CTXID */ /* Wait for the PWS counter. */ - radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 6, 0)); - radeon_emit(cs, S_580_PWS_STAGE_SEL(V_580_CP_ME) | S_580_PWS_COUNTER_SEL(V_580_TS_SELECT) | S_580_PWS_ENA2(1) | - S_580_PWS_COUNT(0)); - radeon_emit(cs, 0xffffffff); /* GCR_SIZE */ - radeon_emit(cs, 0x01ffffff); /* GCR_SIZE_HI */ - radeon_emit(cs, 0); /* GCR_BASE_LO */ - radeon_emit(cs, 0); /* GCR_BASE_HI */ - radeon_emit(cs, S_585_PWS_ENA(1)); - radeon_emit(cs, 0); /* GCR_CNTL */ + radeon_emit(PKT3(PKT3_ACQUIRE_MEM, 6, 0)); + radeon_emit(S_580_PWS_STAGE_SEL(V_580_CP_ME) | S_580_PWS_COUNTER_SEL(V_580_TS_SELECT) | S_580_PWS_ENA2(1) | + S_580_PWS_COUNT(0)); + radeon_emit(0xffffffff); /* GCR_SIZE */ + radeon_emit(0x01ffffff); /* GCR_SIZE_HI */ + radeon_emit(0); /* GCR_BASE_LO */ + radeon_emit(0); /* GCR_BASE_HI */ + radeon_emit(S_585_PWS_ENA(1)); + radeon_emit(0); /* GCR_CNTL */ /* The PS will read inputs from this address. */ - radeon_set_uconfig_reg_seq(cs, R_031110_SPI_GS_THROTTLE_CNTL1, 4); - radeon_emit(cs, 0x12355123); /* SPI_GS_THROTTLE_CNTL1 */ - radeon_emit(cs, 0x1544D); /* SPI_GS_THROTTLE_CNTL2 */ - radeon_emit(cs, va >> 16); /* SPI_ATTRIBUTE_RING_BASE */ - radeon_emit(cs, S_03111C_MEM_SIZE((pdev->info.attribute_ring_size_per_se >> 16) - 1) | - S_03111C_BIG_PAGE(pdev->info.discardable_allows_big_page) | - S_03111C_L1_POLICY(1)); /* SPI_ATTRIBUTE_RING_SIZE */ + radeon_set_uconfig_reg_seq(R_031110_SPI_GS_THROTTLE_CNTL1, 4); + radeon_emit(0x12355123); /* SPI_GS_THROTTLE_CNTL1 */ + radeon_emit(0x1544D); /* SPI_GS_THROTTLE_CNTL2 */ + radeon_emit(va >> 16); /* SPI_ATTRIBUTE_RING_BASE */ + radeon_emit(S_03111C_MEM_SIZE((pdev->info.attribute_ring_size_per_se >> 16) - 1) | + S_03111C_BIG_PAGE(pdev->info.discardable_allows_big_page) | + S_03111C_L1_POLICY(1)); /* SPI_ATTRIBUTE_RING_SIZE */ if (pdev->info.gfx_level >= GFX12) { const uint64_t pos_address = va + pdev->info.pos_ring_offset; const uint64_t prim_address = va + pdev->info.prim_ring_offset; /* When one of these 4 registers is updated, all 4 must be updated. */ - radeon_set_uconfig_reg_seq(cs, R_0309A0_GE_POS_RING_BASE, 4); - radeon_emit(cs, pos_address >> 16); /* R_0309A0_GE_POS_RING_BASE */ - radeon_emit(cs, S_0309A4_MEM_SIZE(pdev->info.pos_ring_size_per_se >> 5)); /* R_0309A4_GE_POS_RING_SIZE */ - radeon_emit(cs, prim_address >> 16); /* R_0309A8_GE_PRIM_RING_BASE */ - radeon_emit(cs, S_0309AC_MEM_SIZE(pdev->info.prim_ring_size_per_se >> 5) | S_0309AC_SCOPE(gfx12_scope_device) | - S_0309AC_PAF_TEMPORAL(gfx12_store_high_temporal_stay_dirty) | - S_0309AC_PAB_TEMPORAL(gfx12_load_last_use_discard) | - S_0309AC_SPEC_DATA_READ(gfx12_spec_read_auto) | S_0309AC_FORCE_SE_SCOPE(1) | - S_0309AC_PAB_NOFILL(1)); /* R_0309AC_GE_PRIM_RING_SIZE */ + radeon_set_uconfig_reg_seq(R_0309A0_GE_POS_RING_BASE, 4); + radeon_emit(pos_address >> 16); /* R_0309A0_GE_POS_RING_BASE */ + radeon_emit(S_0309A4_MEM_SIZE(pdev->info.pos_ring_size_per_se >> 5)); /* R_0309A4_GE_POS_RING_SIZE */ + radeon_emit(prim_address >> 16); /* R_0309A8_GE_PRIM_RING_BASE */ + radeon_emit(S_0309AC_MEM_SIZE(pdev->info.prim_ring_size_per_se >> 5) | S_0309AC_SCOPE(gfx12_scope_device) | + S_0309AC_PAF_TEMPORAL(gfx12_store_high_temporal_stay_dirty) | + S_0309AC_PAB_TEMPORAL(gfx12_load_last_use_discard) | S_0309AC_SPEC_DATA_READ(gfx12_spec_read_auto) | + S_0309AC_FORCE_SE_SCOPE(1) | S_0309AC_PAB_NOFILL(1)); /* R_0309AC_GE_PRIM_RING_SIZE */ } + + radeon_end(); } static void @@ -1109,8 +1138,10 @@ radv_update_preamble_cs(struct radv_queue_state *queue, struct radv_device *devi radv_init_graphics_state(cs, device); if (esgs_ring_bo || gsvs_ring_bo || tess_rings_bo || task_rings_bo) { - radeon_event_write(cs, V_028A90_VS_PARTIAL_FLUSH); - radeon_event_write(cs, V_028A90_VGT_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_VS_PARTIAL_FLUSH); + radeon_event_write(V_028A90_VGT_FLUSH); + radeon_end(); } radv_emit_gs_ring_sizes(device, cs, esgs_ring_bo, needs->esgs_ring_size, gsvs_ring_bo, needs->gsvs_ring_size); @@ -1126,7 +1157,9 @@ radv_update_preamble_cs(struct radv_queue_state *queue, struct radv_device *devi radv_emit_compute(device, cs, true); if (task_rings_bo) { - radeon_event_write(cs, V_028A90_CS_PARTIAL_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_CS_PARTIAL_FLUSH); + radeon_end(); } radv_emit_task_rings(device, cs, task_rings_bo, true); @@ -1548,51 +1581,54 @@ radv_create_perf_counter_lock_cs(struct radv_device *device, unsigned pass, bool radv_cs_add_buffer(device->ws, cs, device->perf_counter_bo); + radeon_begin(cs); + if (!unlock) { uint64_t mutex_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_LOCK_OFFSET; - radeon_emit(cs, PKT3(PKT3_ATOMIC_MEM, 7, 0)); - radeon_emit(cs, ATOMIC_OP(TC_OP_ATOMIC_CMPSWAP_32) | ATOMIC_COMMAND(ATOMIC_COMMAND_LOOP)); - radeon_emit(cs, mutex_va); /* addr lo */ - radeon_emit(cs, mutex_va >> 32); /* addr hi */ - radeon_emit(cs, 1); /* data lo */ - radeon_emit(cs, 0); /* data hi */ - radeon_emit(cs, 0); /* compare data lo */ - radeon_emit(cs, 0); /* compare data hi */ - radeon_emit(cs, 10); /* loop interval */ + radeon_emit(PKT3(PKT3_ATOMIC_MEM, 7, 0)); + radeon_emit(ATOMIC_OP(TC_OP_ATOMIC_CMPSWAP_32) | ATOMIC_COMMAND(ATOMIC_COMMAND_LOOP)); + radeon_emit(mutex_va); /* addr lo */ + radeon_emit(mutex_va >> 32); /* addr hi */ + radeon_emit(1); /* data lo */ + radeon_emit(0); /* data hi */ + radeon_emit(0); /* compare data lo */ + radeon_emit(0); /* compare data hi */ + radeon_emit(10); /* loop interval */ } uint64_t va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_PASS_OFFSET; uint64_t unset_va = va + (unlock ? 8 * pass : 0); uint64_t set_va = va + (unlock ? 0 : 8 * pass); - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL | - COPY_DATA_WR_CONFIRM); - radeon_emit(cs, 0); /* immediate */ - radeon_emit(cs, 0); - radeon_emit(cs, unset_va); - radeon_emit(cs, unset_va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL | + COPY_DATA_WR_CONFIRM); + radeon_emit(0); /* immediate */ + radeon_emit(0); + radeon_emit(unset_va); + radeon_emit(unset_va >> 32); - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL | - COPY_DATA_WR_CONFIRM); - radeon_emit(cs, 1); /* immediate */ - radeon_emit(cs, 0); - radeon_emit(cs, set_va); - radeon_emit(cs, set_va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL | + COPY_DATA_WR_CONFIRM); + radeon_emit(1); /* immediate */ + radeon_emit(0); + radeon_emit(set_va); + radeon_emit(set_va >> 32); if (unlock) { uint64_t mutex_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_LOCK_OFFSET; - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL | - COPY_DATA_WR_CONFIRM); - radeon_emit(cs, 0); /* immediate */ - radeon_emit(cs, 0); - radeon_emit(cs, mutex_va); - radeon_emit(cs, mutex_va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL | + COPY_DATA_WR_CONFIRM); + radeon_emit(0); /* immediate */ + radeon_emit(0); + radeon_emit(mutex_va); + radeon_emit(mutex_va >> 32); } + radeon_end(); assert(cs->cdw <= cdw); VkResult result = device->ws->cs_finalize(cs); diff --git a/src/amd/vulkan/radv_radeon_winsys.h b/src/amd/vulkan/radv_radeon_winsys.h index 422e5bfb1bf..c6d14d259db 100644 --- a/src/amd/vulkan/radv_radeon_winsys.h +++ b/src/amd/vulkan/radv_radeon_winsys.h @@ -321,21 +321,6 @@ struct radeon_winsys { const struct vk_sync_type *const *(*get_sync_types)(struct radeon_winsys *ws); }; -static inline void -radeon_emit(struct radeon_cmdbuf *cs, uint32_t value) -{ - assert(cs->cdw < cs->reserved_dw); - cs->buf[cs->cdw++] = value; -} - -static inline void -radeon_emit_array(struct radeon_cmdbuf *cs, const uint32_t *values, unsigned count) -{ - assert(cs->cdw + count <= cs->reserved_dw); - memcpy(cs->buf + cs->cdw, values, count * 4); - cs->cdw += count; -} - static inline uint64_t radv_buffer_get_va(const struct radeon_winsys_bo *bo) { diff --git a/src/amd/vulkan/radv_sdma.c b/src/amd/vulkan/radv_sdma.c index df79b56f505..fe660f8f891 100644 --- a/src/amd/vulkan/radv_sdma.c +++ b/src/amd/vulkan/radv_sdma.c @@ -323,44 +323,54 @@ radv_sdma_emit_nop(const struct radv_device *device, struct radeon_cmdbuf *cs) { /* SDMA NOP acts as a fence command and causes the SDMA engine to wait for pending copy operations. */ radeon_check_space(device->ws, cs, 1); - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); + radeon_end(); } void radv_sdma_emit_write_timestamp(struct radeon_cmdbuf *cs, uint64_t va) { - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_TIMESTAMP, SDMA_TS_SUB_OPCODE_GET_GLOBAL_TIMESTAMP, 0)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_TIMESTAMP, SDMA_TS_SUB_OPCODE_GET_GLOBAL_TIMESTAMP, 0)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_end(); } void radv_sdma_emit_fence(struct radeon_cmdbuf *cs, uint64_t va, uint32_t fence) { - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, SDMA_FENCE_MTYPE_UC)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, fence); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_FENCE, 0, SDMA_FENCE_MTYPE_UC)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(fence); + radeon_end(); } void radv_sdma_emit_wait_mem(struct radeon_cmdbuf *cs, uint32_t op, uint64_t va, uint32_t ref, uint32_t mask) { - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_POLL_REGMEM, 0, 0) | op << 28 | SDMA_POLL_MEM); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, ref); - radeon_emit(cs, mask); - radeon_emit(cs, SDMA_POLL_INTERVAL_160_CLK | SDMA_POLL_RETRY_INDEFINITELY << 16); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_POLL_REGMEM, 0, 0) | op << 28 | SDMA_POLL_MEM); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(ref); + radeon_emit(mask); + radeon_emit(SDMA_POLL_INTERVAL_160_CLK | SDMA_POLL_RETRY_INDEFINITELY << 16); + radeon_end(); } void radv_sdma_emit_write_data_head(struct radeon_cmdbuf *cs, uint64_t va, uint32_t count) { - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, count - 1); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(count - 1); + radeon_end(); } void @@ -393,19 +403,23 @@ radv_sdma_copy_memory(const struct radv_device *device, struct radeon_cmdbuf *cs radeon_check_space(device->ws, cs, ncopy * 7); + radeon_begin(cs); + for (unsigned i = 0; i < ncopy; i++) { unsigned csize = size >= 4 ? MIN2(size & align, max_size_per_packet) : size; - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0)); - radeon_emit(cs, ver >= SDMA_4_0 ? csize - 1 : csize); - radeon_emit(cs, 0); /* src/dst endian swap */ - radeon_emit(cs, src_va); - radeon_emit(cs, src_va >> 32); - radeon_emit(cs, dst_va); - radeon_emit(cs, dst_va >> 32); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0)); + radeon_emit(ver >= SDMA_4_0 ? csize - 1 : csize); + radeon_emit(0); /* src/dst endian swap */ + radeon_emit(src_va); + radeon_emit(src_va >> 32); + radeon_emit(dst_va); + radeon_emit(dst_va >> 32); dst_va += csize; src_va += csize; size -= csize; } + + radeon_end(); } void @@ -428,18 +442,21 @@ radv_sdma_fill_memory(const struct radv_device *device, struct radeon_cmdbuf *cs const unsigned num_packets = DIV_ROUND_UP(size, max_fill_bytes); ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cs, num_packets * 5); + radeon_begin(cs); + for (unsigned i = 0; i < num_packets; ++i) { const uint64_t offset = i * max_fill_bytes; const uint64_t fill_bytes = MIN2(size - offset, max_fill_bytes); const uint64_t fill_va = va + offset; - radeon_emit(cs, constant_fill_header); - radeon_emit(cs, fill_va); - radeon_emit(cs, fill_va >> 32); - radeon_emit(cs, value); - radeon_emit(cs, fill_bytes - 1); /* Must be programmed in bytes, even if the fill is done in dwords. */ + radeon_emit(constant_fill_header); + radeon_emit(fill_va); + radeon_emit(fill_va >> 32); + radeon_emit(value); + radeon_emit(fill_bytes - 1); /* Must be programmed in bytes, even if the fill is done in dwords. */ } + radeon_end(); assert(cs->cdw <= cdw_max); } @@ -475,20 +492,22 @@ radv_sdma_emit_copy_linear_sub_window(const struct radv_device *device, struct r ASSERTED unsigned cdw_end = radeon_check_space(device->ws, cs, 13); - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR_SUB_WINDOW, 0) | util_logbase2(src->bpp) - << 29); - radeon_emit(cs, src->va); - radeon_emit(cs, src->va >> 32); - radeon_emit(cs, src_off.x | src_off.y << 16); - radeon_emit(cs, src_off.z | (src_pitch - 1) << (ver >= SDMA_7_0 ? 16 : 13)); - radeon_emit(cs, src_slice_pitch - 1); - radeon_emit(cs, dst->va); - radeon_emit(cs, dst->va >> 32); - radeon_emit(cs, dst_off.x | dst_off.y << 16); - radeon_emit(cs, dst_off.z | (dst_pitch - 1) << (ver >= SDMA_7_0 ? 16 : 13)); - radeon_emit(cs, dst_slice_pitch - 1); - radeon_emit(cs, (ext.width - 1) | (ext.height - 1) << 16); - radeon_emit(cs, (ext.depth - 1)); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR_SUB_WINDOW, 0) | util_logbase2(src->bpp) + << 29); + radeon_emit(src->va); + radeon_emit(src->va >> 32); + radeon_emit(src_off.x | src_off.y << 16); + radeon_emit(src_off.z | (src_pitch - 1) << (ver >= SDMA_7_0 ? 16 : 13)); + radeon_emit(src_slice_pitch - 1); + radeon_emit(dst->va); + radeon_emit(dst->va >> 32); + radeon_emit(dst_off.x | dst_off.y << 16); + radeon_emit(dst_off.z | (dst_pitch - 1) << (ver >= SDMA_7_0 ? 16 : 13)); + radeon_emit(dst_slice_pitch - 1); + radeon_emit((ext.width - 1) | (ext.height - 1) << 16); + radeon_emit((ext.depth - 1)); + radeon_end(); assert(cs->cdw == cdw_end); } @@ -519,29 +538,31 @@ radv_sdma_emit_copy_tiled_sub_window(const struct radv_device *device, struct ra ASSERTED unsigned cdw_end = radeon_check_space(device->ws, cs, 14 + (dcc ? 3 : 0)); - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_TILED_SUB_WINDOW, 0) | dcc << 19 | detile << 31 | - tiled->header_dword); - radeon_emit(cs, tiled->va); - radeon_emit(cs, tiled->va >> 32); - radeon_emit(cs, tiled_off.x | tiled_off.y << 16); - radeon_emit(cs, tiled_off.z | (tiled_ext.width - 1) << 16); - radeon_emit(cs, (tiled_ext.height - 1) | (tiled_ext.depth - 1) << 16); - radeon_emit(cs, tiled->info_dword); - radeon_emit(cs, linear->va); - radeon_emit(cs, linear->va >> 32); - radeon_emit(cs, linear_off.x | linear_off.y << 16); - radeon_emit(cs, linear_off.z | (linear_pitch - 1) << 16); - radeon_emit(cs, linear_slice_pitch - 1); - radeon_emit(cs, (ext.width - 1) | (ext.height - 1) << 16); - radeon_emit(cs, (ext.depth - 1)); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_TILED_SUB_WINDOW, 0) | dcc << 19 | detile << 31 | + tiled->header_dword); + radeon_emit(tiled->va); + radeon_emit(tiled->va >> 32); + radeon_emit(tiled_off.x | tiled_off.y << 16); + radeon_emit(tiled_off.z | (tiled_ext.width - 1) << 16); + radeon_emit((tiled_ext.height - 1) | (tiled_ext.depth - 1) << 16); + radeon_emit(tiled->info_dword); + radeon_emit(linear->va); + radeon_emit(linear->va >> 32); + radeon_emit(linear_off.x | linear_off.y << 16); + radeon_emit(linear_off.z | (linear_pitch - 1) << 16); + radeon_emit(linear_slice_pitch - 1); + radeon_emit((ext.width - 1) | (ext.height - 1) << 16); + radeon_emit((ext.depth - 1)); if (tiled->meta_va) { const unsigned write_compress_enable = !detile; - radeon_emit(cs, tiled->meta_va); - radeon_emit(cs, tiled->meta_va >> 32); - radeon_emit(cs, tiled->meta_config | write_compress_enable << 28); + radeon_emit(tiled->meta_va); + radeon_emit(tiled->meta_va >> 32); + radeon_emit(tiled->meta_config | write_compress_enable << 28); } + radeon_end(); assert(cs->cdw == cdw_end); } @@ -583,34 +604,36 @@ radv_sdma_emit_copy_t2t_sub_window(const struct radv_device *device, struct rade ASSERTED unsigned cdw_end = radeon_check_space(device->ws, cs, 15 + (dcc ? 3 : 0)); - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_T2T_SUB_WINDOW, 0) | dcc << 19 | dcc_dir << 31 | - src->header_dword); - radeon_emit(cs, src->va); - radeon_emit(cs, src->va >> 32); - radeon_emit(cs, src_off.x | src_off.y << 16); - radeon_emit(cs, src_off.z | (src_ext.width - 1) << 16); - radeon_emit(cs, (src_ext.height - 1) | (src_ext.depth - 1) << 16); - radeon_emit(cs, src->info_dword); - radeon_emit(cs, dst->va); - radeon_emit(cs, dst->va >> 32); - radeon_emit(cs, dst_off.x | dst_off.y << 16); - radeon_emit(cs, dst_off.z | (dst_ext.width - 1) << 16); - radeon_emit(cs, (dst_ext.height - 1) | (dst_ext.depth - 1) << 16); - radeon_emit(cs, dst->info_dword); - radeon_emit(cs, (ext.width - 1) | (ext.height - 1) << 16); - radeon_emit(cs, (ext.depth - 1)); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_T2T_SUB_WINDOW, 0) | dcc << 19 | dcc_dir << 31 | + src->header_dword); + radeon_emit(src->va); + radeon_emit(src->va >> 32); + radeon_emit(src_off.x | src_off.y << 16); + radeon_emit(src_off.z | (src_ext.width - 1) << 16); + radeon_emit((src_ext.height - 1) | (src_ext.depth - 1) << 16); + radeon_emit(src->info_dword); + radeon_emit(dst->va); + radeon_emit(dst->va >> 32); + radeon_emit(dst_off.x | dst_off.y << 16); + radeon_emit(dst_off.z | (dst_ext.width - 1) << 16); + radeon_emit((dst_ext.height - 1) | (dst_ext.depth - 1) << 16); + radeon_emit(dst->info_dword); + radeon_emit((ext.width - 1) | (ext.height - 1) << 16); + radeon_emit((ext.depth - 1)); if (dst->meta_va) { const uint32_t write_compress_enable = 1; - radeon_emit(cs, dst->meta_va); - radeon_emit(cs, dst->meta_va >> 32); - radeon_emit(cs, dst->meta_config | write_compress_enable << 28); + radeon_emit(dst->meta_va); + radeon_emit(dst->meta_va >> 32); + radeon_emit(dst->meta_config | write_compress_enable << 28); } else if (src->meta_va) { - radeon_emit(cs, src->meta_va); - radeon_emit(cs, src->meta_va >> 32); - radeon_emit(cs, src->meta_config); + radeon_emit(src->meta_va); + radeon_emit(src->meta_va >> 32); + radeon_emit(src->meta_config); } + radeon_end(); assert(cs->cdw == cdw_end); } diff --git a/src/amd/vulkan/radv_spm.c b/src/amd/vulkan/radv_spm.c index 1eea9d2d1e9..4191f0c812f 100644 --- a/src/amd/vulkan/radv_spm.c +++ b/src/amd/vulkan/radv_spm.c @@ -83,16 +83,19 @@ radv_emit_spm_counters(struct radv_device *device, struct radeon_cmdbuf *cs, enu continue; radeon_check_space(device->ws, cs, 3 + num_counters * 3); + radeon_begin(cs); - radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, spm->sq_wgp[instance].grbm_gfx_index); + radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, spm->sq_wgp[instance].grbm_gfx_index); for (uint32_t b = 0; b < num_counters; b++) { const struct ac_spm_counter_select *cntr_sel = &spm->sq_wgp[instance].counters[b]; uint32_t reg_base = R_036700_SQ_PERFCOUNTER0_SELECT; - radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, reg_base + b * 4, 1); - radeon_emit(cs, cntr_sel->sel0); + radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, reg_base + b * 4, 1); + radeon_emit(cntr_sel->sel0); } + + radeon_end(); } } @@ -103,18 +106,21 @@ radv_emit_spm_counters(struct radv_device *device, struct radeon_cmdbuf *cs, enu continue; radeon_check_space(device->ws, cs, 3 + num_counters * 3); + radeon_begin(cs); - radeon_set_uconfig_reg( - cs, R_030800_GRBM_GFX_INDEX, - S_030800_SH_BROADCAST_WRITES(1) | S_030800_INSTANCE_BROADCAST_WRITES(1) | S_030800_SE_INDEX(instance)); + radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, S_030800_SH_BROADCAST_WRITES(1) | + S_030800_INSTANCE_BROADCAST_WRITES(1) | + S_030800_SE_INDEX(instance)); for (uint32_t b = 0; b < num_counters; b++) { const struct ac_spm_counter_select *cntr_sel = &spm->sqg[instance].counters[b]; uint32_t reg_base = R_036700_SQ_PERFCOUNTER0_SELECT; - radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, reg_base + b * 4, 1); - radeon_emit(cs, cntr_sel->sel0 | S_036700_SQC_BANK_MASK(0xf)); /* SQC_BANK_MASK only gfx10 */ + radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, reg_base + b * 4, 1); + radeon_emit(cntr_sel->sel0 | S_036700_SQC_BANK_MASK(0xf)); /* SQC_BANK_MASK only gfx10 */ } + + radeon_end(); } for (uint32_t b = 0; b < spm->num_block_sel; b++) { @@ -125,8 +131,9 @@ radv_emit_spm_counters(struct radv_device *device, struct radeon_cmdbuf *cs, enu struct ac_spm_block_instance *block_instance = &block_sel->instances[i]; radeon_check_space(device->ws, cs, 3 + (AC_SPM_MAX_COUNTER_PER_BLOCK * 6)); + radeon_begin(cs); - radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, block_instance->grbm_gfx_index); + radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, block_instance->grbm_gfx_index); for (unsigned c = 0; c < block_instance->num_counters; c++) { const struct ac_spm_counter_select *cntr_sel = &block_instance->counters[c]; @@ -134,19 +141,22 @@ radv_emit_spm_counters(struct radv_device *device, struct radeon_cmdbuf *cs, enu if (!cntr_sel->active) continue; - radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, regs->select0[c], 1); - radeon_emit(cs, cntr_sel->sel0); + radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, regs->select0[c], 1); + radeon_emit(cntr_sel->sel0); - radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, regs->select1[c], 1); - radeon_emit(cs, cntr_sel->sel1); + radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, regs->select1[c], 1); + radeon_emit(cntr_sel->sel1); } + + radeon_end(); } } /* Restore global broadcasting. */ - radeon_set_uconfig_reg( - cs, R_030800_GRBM_GFX_INDEX, - S_030800_SE_BROADCAST_WRITES(1) | S_030800_SH_BROADCAST_WRITES(1) | S_030800_INSTANCE_BROADCAST_WRITES(1)); + radeon_begin(cs); + radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, S_030800_SE_BROADCAST_WRITES(1) | S_030800_SH_BROADCAST_WRITES(1) | + S_030800_INSTANCE_BROADCAST_WRITES(1)); + radeon_end(); } static void @@ -181,23 +191,26 @@ radv_emit_spm_muxsel(struct radv_device *device, struct radeon_cmdbuf *cs, enum } radeon_check_space(device->ws, cs, 3 + spm->num_muxsel_lines[s] * (7 + AC_SPM_MUXSEL_LINE_SIZE)); + radeon_begin(cs); - radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, grbm_gfx_index); + radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, grbm_gfx_index); for (unsigned l = 0; l < spm->num_muxsel_lines[s]; l++) { uint32_t *data = (uint32_t *)spm->muxsel_lines[s][l].muxsel; /* Select MUXSEL_ADDR to point to the next muxsel. */ - radeon_set_uconfig_perfctr_reg(pdev->info.gfx_level, ring, cs, rlc_muxsel_addr, l * AC_SPM_MUXSEL_LINE_SIZE); + radeon_set_uconfig_perfctr_reg(pdev->info.gfx_level, ring, rlc_muxsel_addr, l * AC_SPM_MUXSEL_LINE_SIZE); /* Write the muxsel line configuration with MUXSEL_DATA. */ - radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + AC_SPM_MUXSEL_LINE_SIZE, 0)); - radeon_emit(cs, S_370_DST_SEL(V_370_MEM_MAPPED_REGISTER) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME) | - S_370_WR_ONE_ADDR(1)); - radeon_emit(cs, rlc_muxsel_data >> 2); - radeon_emit(cs, 0); - radeon_emit_array(cs, data, AC_SPM_MUXSEL_LINE_SIZE); + radeon_emit(PKT3(PKT3_WRITE_DATA, 2 + AC_SPM_MUXSEL_LINE_SIZE, 0)); + radeon_emit(S_370_DST_SEL(V_370_MEM_MAPPED_REGISTER) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME) | + S_370_WR_ONE_ADDR(1)); + radeon_emit(rlc_muxsel_data >> 2); + radeon_emit(0); + radeon_emit_array(data, AC_SPM_MUXSEL_LINE_SIZE); } + + radeon_end(); } } @@ -215,14 +228,15 @@ radv_emit_spm_setup(struct radv_device *device, struct radeon_cmdbuf *cs, enum r assert(spm->sample_interval >= 32); radeon_check_space(device->ws, cs, 27); + radeon_begin(cs); /* Configure the SPM ring buffer. */ - radeon_set_uconfig_reg(cs, R_037200_RLC_SPM_PERFMON_CNTL, + radeon_set_uconfig_reg(R_037200_RLC_SPM_PERFMON_CNTL, S_037200_PERFMON_RING_MODE(0) | /* no stall and no interrupt on overflow */ S_037200_PERFMON_SAMPLE_INTERVAL(spm->sample_interval)); /* in sclk */ - radeon_set_uconfig_reg(cs, R_037204_RLC_SPM_PERFMON_RING_BASE_LO, va); - radeon_set_uconfig_reg(cs, R_037208_RLC_SPM_PERFMON_RING_BASE_HI, S_037208_RING_BASE_HI(va >> 32)); - radeon_set_uconfig_reg(cs, R_03720C_RLC_SPM_PERFMON_RING_SIZE, ring_size); + radeon_set_uconfig_reg(R_037204_RLC_SPM_PERFMON_RING_BASE_LO, va); + radeon_set_uconfig_reg(R_037208_RLC_SPM_PERFMON_RING_BASE_HI, S_037208_RING_BASE_HI(va >> 32)); + radeon_set_uconfig_reg(R_03720C_RLC_SPM_PERFMON_RING_SIZE, ring_size); /* Configure the muxsel. */ uint32_t total_muxsel_lines = 0; @@ -230,27 +244,29 @@ radv_emit_spm_setup(struct radv_device *device, struct radeon_cmdbuf *cs, enum r total_muxsel_lines += spm->num_muxsel_lines[s]; } - radeon_set_uconfig_reg(cs, R_03726C_RLC_SPM_ACCUM_MODE, 0); + radeon_set_uconfig_reg(R_03726C_RLC_SPM_ACCUM_MODE, 0); if (pdev->info.gfx_level >= GFX11) { - radeon_set_uconfig_reg(cs, R_03721C_RLC_SPM_PERFMON_SEGMENT_SIZE, + radeon_set_uconfig_reg(R_03721C_RLC_SPM_PERFMON_SEGMENT_SIZE, S_03721C_TOTAL_NUM_SEGMENT(total_muxsel_lines) | S_03721C_GLOBAL_NUM_SEGMENT(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_GLOBAL]) | S_03721C_SE_NUM_SEGMENT(spm->max_se_muxsel_lines)); - radeon_set_uconfig_reg(cs, R_037210_RLC_SPM_RING_WRPTR, 0); + radeon_set_uconfig_reg(R_037210_RLC_SPM_RING_WRPTR, 0); } else { - radeon_set_uconfig_reg(cs, R_037210_RLC_SPM_PERFMON_SEGMENT_SIZE, 0); - radeon_set_uconfig_reg(cs, R_03727C_RLC_SPM_PERFMON_SE3TO0_SEGMENT_SIZE, + radeon_set_uconfig_reg(R_037210_RLC_SPM_PERFMON_SEGMENT_SIZE, 0); + radeon_set_uconfig_reg(R_03727C_RLC_SPM_PERFMON_SE3TO0_SEGMENT_SIZE, S_03727C_SE0_NUM_LINE(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_SE0]) | S_03727C_SE1_NUM_LINE(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_SE1]) | S_03727C_SE2_NUM_LINE(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_SE2]) | S_03727C_SE3_NUM_LINE(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_SE3])); - radeon_set_uconfig_reg(cs, R_037280_RLC_SPM_PERFMON_GLB_SEGMENT_SIZE, + radeon_set_uconfig_reg(R_037280_RLC_SPM_PERFMON_GLB_SEGMENT_SIZE, S_037280_PERFMON_SEGMENT_SIZE(total_muxsel_lines) | S_037280_GLOBAL_NUM_LINE(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_GLOBAL])); } + radeon_end(); + /* Upload each muxsel ram to the RLC. */ radv_emit_spm_muxsel(device, cs, qf); diff --git a/src/amd/vulkan/radv_sqtt.c b/src/amd/vulkan/radv_sqtt.c index 5a37d73307f..328c4424211 100644 --- a/src/amd/vulkan/radv_sqtt.c +++ b/src/amd/vulkan/radv_sqtt.c @@ -132,14 +132,17 @@ radv_emit_sqtt_userdata(const struct radv_cmd_buffer *cmd_buffer, const void *da uint32_t count = MIN2(num_dwords, 2); radeon_check_space(device->ws, cs, 2 + count); + radeon_begin(cs); /* Without the perfctr bit the CP might not always pass the * write on correctly. */ if (pdev->info.gfx_level >= GFX10) - radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, R_030D08_SQ_THREAD_TRACE_USERDATA_2, count); + radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, R_030D08_SQ_THREAD_TRACE_USERDATA_2, count); else - radeon_set_uconfig_reg_seq(cs, R_030D08_SQ_THREAD_TRACE_USERDATA_2, count); - radeon_emit_array(cs, dwords, count); + radeon_set_uconfig_reg_seq(R_030D08_SQ_THREAD_TRACE_USERDATA_2, count); + radeon_emit_array(dwords, count); + + radeon_end(); dwords += count; num_dwords -= count; @@ -151,8 +154,10 @@ radv_emit_spi_config_cntl(const struct radv_device *device, struct radeon_cmdbuf { const struct radv_physical_device *pdev = radv_device_physical(device); + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX12) { - radeon_set_uconfig_reg(cs, R_031120_SPI_SQG_EVENT_CTL, + radeon_set_uconfig_reg(R_031120_SPI_SQG_EVENT_CTL, S_031120_ENABLE_SQG_TOP_EVENTS(enable) | S_031120_ENABLE_SQG_BOP_EVENTS(enable)); } else if (pdev->info.gfx_level >= GFX9) { uint32_t spi_config_cntl = S_031100_GPR_WRITE_PRIORITY(0x2c688) | S_031100_EXP_PRIORITY_ORDER(3) | @@ -161,12 +166,14 @@ radv_emit_spi_config_cntl(const struct radv_device *device, struct radeon_cmdbuf if (pdev->info.gfx_level >= GFX10) spi_config_cntl |= S_031100_PS_PKR_PRIORITY_CNTL(3); - radeon_set_uconfig_reg(cs, R_031100_SPI_CONFIG_CNTL, spi_config_cntl); + radeon_set_uconfig_reg(R_031100_SPI_CONFIG_CNTL, spi_config_cntl); } else { /* SPI_CONFIG_CNTL is a protected register on GFX6-GFX8. */ - radeon_set_privileged_config_reg(cs, R_009100_SPI_CONFIG_CNTL, + radeon_set_privileged_config_reg(R_009100_SPI_CONFIG_CNTL, S_009100_ENABLE_SQG_TOP_EVENTS(enable) | S_009100_ENABLE_SQG_BOP_EVENTS(enable)); } + + radeon_end(); } void @@ -177,11 +184,15 @@ radv_emit_inhibit_clockgating(const struct radv_device *device, struct radeon_cm if (pdev->info.gfx_level >= GFX11) return; /* not needed */ + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX10) { - radeon_set_uconfig_reg(cs, R_037390_RLC_PERFMON_CLK_CNTL, S_037390_PERFMON_CLOCK_STATE(inhibit)); + radeon_set_uconfig_reg(R_037390_RLC_PERFMON_CLK_CNTL, S_037390_PERFMON_CLOCK_STATE(inhibit)); } else if (pdev->info.gfx_level >= GFX8) { - radeon_set_uconfig_reg(cs, R_0372FC_RLC_PERFMON_CLK_CNTL, S_0372FC_PERFMON_CLOCK_STATE(inhibit)); + radeon_set_uconfig_reg(R_0372FC_RLC_PERFMON_CLK_CNTL, S_0372FC_PERFMON_CLOCK_STATE(inhibit)); } + + radeon_end(); } VkResult @@ -520,21 +531,25 @@ radv_begin_sqtt(struct radv_queue *queue) radeon_check_space(ws, cs, 512); + radeon_begin(cs); + switch (family) { case RADV_QUEUE_GENERAL: - radeon_emit(cs, PKT3(PKT3_CONTEXT_CONTROL, 1, 0)); - radeon_emit(cs, CC0_UPDATE_LOAD_ENABLES(1)); - radeon_emit(cs, CC1_UPDATE_SHADOW_ENABLES(1)); + radeon_emit(PKT3(PKT3_CONTEXT_CONTROL, 1, 0)); + radeon_emit(CC0_UPDATE_LOAD_ENABLES(1)); + radeon_emit(CC1_UPDATE_SHADOW_ENABLES(1)); break; case RADV_QUEUE_COMPUTE: - radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_NOP, 0, 0)); + radeon_emit(0); break; default: unreachable("Incorrect queue family"); break; } + radeon_end(); + /* Make sure to wait-for-idle before starting SQTT. */ radv_emit_wait_for_idle(device, cs, family); @@ -593,21 +608,25 @@ radv_end_sqtt(struct radv_queue *queue) radeon_check_space(ws, cs, 512); + radeon_begin(cs); + switch (family) { case RADV_QUEUE_GENERAL: - radeon_emit(cs, PKT3(PKT3_CONTEXT_CONTROL, 1, 0)); - radeon_emit(cs, CC0_UPDATE_LOAD_ENABLES(1)); - radeon_emit(cs, CC1_UPDATE_SHADOW_ENABLES(1)); + radeon_emit(PKT3(PKT3_CONTEXT_CONTROL, 1, 0)); + radeon_emit(CC0_UPDATE_LOAD_ENABLES(1)); + radeon_emit(CC1_UPDATE_SHADOW_ENABLES(1)); break; case RADV_QUEUE_COMPUTE: - radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_NOP, 0, 0)); + radeon_emit(0); break; default: unreachable("Incorrect queue family"); break; } + radeon_end(); + /* Make sure to wait-for-idle before stopping SQTT. */ radv_emit_wait_for_idle(device, cs, family); diff --git a/src/amd/vulkan/radv_video.c b/src/amd/vulkan/radv_video.c index 7d2b323b0b6..18762ad6bcf 100644 --- a/src/amd/vulkan/radv_video.c +++ b/src/amd/vulkan/radv_video.c @@ -67,10 +67,12 @@ radv_vcn_sq_header(struct radeon_cmdbuf *cs, struct rvcn_sq_var *sq, unsigned ty { if (!skip_signature) { /* vcn ib signature */ - radeon_emit(cs, RADEON_VCN_SIGNATURE_SIZE); - radeon_emit(cs, RADEON_VCN_SIGNATURE); - radeon_emit(cs, 0); - radeon_emit(cs, 0); + radeon_begin(cs); + radeon_emit(RADEON_VCN_SIGNATURE_SIZE); + radeon_emit(RADEON_VCN_SIGNATURE); + radeon_emit(0); + radeon_emit(0); + radeon_end(); sq->signature_ib_checksum = &cs->buf[cs->cdw - 2]; sq->signature_ib_total_size_in_dw = &cs->buf[cs->cdw - 1]; @@ -80,10 +82,12 @@ radv_vcn_sq_header(struct radeon_cmdbuf *cs, struct rvcn_sq_var *sq, unsigned ty } /* vcn ib engine info */ - radeon_emit(cs, RADEON_VCN_ENGINE_INFO_SIZE); - radeon_emit(cs, RADEON_VCN_ENGINE_INFO); - radeon_emit(cs, type); - radeon_emit(cs, 0); + radeon_begin(cs); + radeon_emit(RADEON_VCN_ENGINE_INFO_SIZE); + radeon_emit(RADEON_VCN_ENGINE_INFO); + radeon_emit(type); + radeon_emit(0); + radeon_end(); sq->engine_ib_size_of_packages = &cs->buf[cs->cdw - 1]; } @@ -1072,8 +1076,11 @@ static void set_reg(struct radv_cmd_buffer *cmd_buffer, unsigned reg, uint32_t val) { struct radeon_cmdbuf *cs = cmd_buffer->cs; - radeon_emit(cs, RDECODE_PKT0(reg >> 2, 0)); - radeon_emit(cs, val); + + radeon_begin(cs); + radeon_emit(RDECODE_PKT0(reg >> 2, 0)); + radeon_emit(val); + radeon_end(); } static void @@ -2584,8 +2591,10 @@ radv_vcn_cmd_reset(struct radv_cmd_buffer *cmd_buffer) if (pdev->vid_decode_ip != AMD_IP_VCN_UNIFIED) { radeon_check_space(device->ws, cmd_buffer->cs, 8); + radeon_begin(cmd_buffer->cs); for (unsigned i = 0; i < 8; i++) - radeon_emit(cmd_buffer->cs, 0x81ff); + radeon_emit(0x81ff); + radeon_end(); } else radv_vcn_sq_tail(cmd_buffer->cs, &cmd_buffer->video.sq); } @@ -2610,8 +2619,10 @@ radv_uvd_cmd_reset(struct radv_cmd_buffer *cmd_buffer) /* pad out the IB to the 16 dword boundary - otherwise the fw seems to be unhappy */ int padsize = vid->sessionctx.mem ? 4 : 6; radeon_check_space(device->ws, cmd_buffer->cs, padsize); + radeon_begin(cmd_buffer->cs); for (unsigned i = 0; i < padsize; i++) - radeon_emit(cmd_buffer->cs, PKT2_NOP_PAD); + radeon_emit(PKT2_NOP_PAD); + radeon_end(); } VKAPI_ATTR void VKAPI_CALL diff --git a/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c b/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c index 128e1d1c4ca..a478828df4d 100644 --- a/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c +++ b/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c @@ -101,6 +101,21 @@ struct radv_winsys_sem_info { struct radv_winsys_sem_counts signal; }; +static void +radeon_emit(struct radeon_cmdbuf *cs, uint32_t value) +{ + assert(cs->cdw < cs->reserved_dw); + cs->buf[cs->cdw++] = value; +} + +static void +radeon_emit_array(struct radeon_cmdbuf *cs, const uint32_t *values, unsigned count) +{ + assert(cs->cdw + count <= cs->reserved_dw); + memcpy(cs->buf + cs->cdw, values, count * 4); + cs->cdw += count; +} + static void radeon_emit_unchecked(struct radeon_cmdbuf *cs, uint32_t value) {