diff --git a/src/amd/vulkan/layers/radv_sqtt_layer.c b/src/amd/vulkan/layers/radv_sqtt_layer.c index 0351bff7ae0..e0cd3e66c0f 100644 --- a/src/amd/vulkan/layers/radv_sqtt_layer.c +++ b/src/amd/vulkan/layers/radv_sqtt_layer.c @@ -30,23 +30,25 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv radv_cs_add_buffer(device->ws, cs, reloc->bo); + radeon_begin(cs); + /* VS */ if (pipeline->base.shaders[MESA_SHADER_VERTEX]) { struct radv_shader *vs = pipeline->base.shaders[MESA_SHADER_VERTEX]; va = reloc->va[MESA_SHADER_VERTEX]; if (vs->info.vs.as_ls) { - radeon_set_sh_reg(cs, vs->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(vs->info.regs.pgm_lo, va >> 8); } else if (vs->info.vs.as_es) { - radeon_set_sh_reg_seq(cs, vs->info.regs.pgm_lo, 2); - radeon_emit(cs, va >> 8); - radeon_emit(cs, S_00B324_MEM_BASE(va >> 40)); + radeon_set_sh_reg_seq(vs->info.regs.pgm_lo, 2); + radeon_emit(va >> 8); + radeon_emit(S_00B324_MEM_BASE(va >> 40)); } else if (vs->info.is_ngg) { - radeon_set_sh_reg(cs, vs->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(vs->info.regs.pgm_lo, va >> 8); } else { - radeon_set_sh_reg_seq(cs, vs->info.regs.pgm_lo, 2); - radeon_emit(cs, va >> 8); - radeon_emit(cs, S_00B124_MEM_BASE(va >> 40)); + radeon_set_sh_reg_seq(vs->info.regs.pgm_lo, 2); + radeon_emit(va >> 8); + radeon_emit(S_00B124_MEM_BASE(va >> 40)); } } @@ -57,11 +59,11 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv va = reloc->va[MESA_SHADER_TESS_CTRL]; if (gfx_level >= GFX9) { - radeon_set_sh_reg(cs, tcs->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(tcs->info.regs.pgm_lo, va >> 8); } else { - radeon_set_sh_reg_seq(cs, tcs->info.regs.pgm_lo, 2); - radeon_emit(cs, va >> 8); - radeon_emit(cs, S_00B424_MEM_BASE(va >> 40)); + radeon_set_sh_reg_seq(tcs->info.regs.pgm_lo, 2); + radeon_emit(va >> 8); + radeon_emit(S_00B424_MEM_BASE(va >> 40)); } } @@ -71,15 +73,15 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv va = reloc->va[MESA_SHADER_TESS_EVAL]; if (tes->info.is_ngg) { - radeon_set_sh_reg(cs, tes->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(tes->info.regs.pgm_lo, va >> 8); } else if (tes->info.tes.as_es) { - radeon_set_sh_reg_seq(cs, tes->info.regs.pgm_lo, 2); - radeon_emit(cs, va >> 8); - radeon_emit(cs, S_00B324_MEM_BASE(va >> 40)); + radeon_set_sh_reg_seq(tes->info.regs.pgm_lo, 2); + radeon_emit(va >> 8); + radeon_emit(S_00B324_MEM_BASE(va >> 40)); } else { - radeon_set_sh_reg_seq(cs, tes->info.regs.pgm_lo, 2); - radeon_emit(cs, va >> 8); - radeon_emit(cs, S_00B124_MEM_BASE(va >> 40)); + radeon_set_sh_reg_seq(tes->info.regs.pgm_lo, 2); + radeon_emit(va >> 8); + radeon_emit(S_00B124_MEM_BASE(va >> 40)); } } @@ -89,14 +91,14 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv va = reloc->va[MESA_SHADER_GEOMETRY]; if (gs->info.is_ngg) { - radeon_set_sh_reg(cs, gs->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(gs->info.regs.pgm_lo, va >> 8); } else { if (gfx_level >= GFX9) { - radeon_set_sh_reg(cs, gs->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(gs->info.regs.pgm_lo, va >> 8); } else { - radeon_set_sh_reg_seq(cs, gs->info.regs.pgm_lo, 2); - radeon_emit(cs, va >> 8); - radeon_emit(cs, S_00B224_MEM_BASE(va >> 40)); + radeon_set_sh_reg_seq(gs->info.regs.pgm_lo, 2); + radeon_emit(va >> 8); + radeon_emit(S_00B224_MEM_BASE(va >> 40)); } } } @@ -107,9 +109,9 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv va = reloc->va[MESA_SHADER_FRAGMENT]; - radeon_set_sh_reg_seq(cs, ps->info.regs.pgm_lo, 2); - radeon_emit(cs, va >> 8); - radeon_emit(cs, S_00B024_MEM_BASE(va >> 40)); + radeon_set_sh_reg_seq(ps->info.regs.pgm_lo, 2); + radeon_emit(va >> 8); + radeon_emit(S_00B024_MEM_BASE(va >> 40)); } /* MS */ @@ -118,8 +120,10 @@ radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv va = reloc->va[MESA_SHADER_MESH]; - radeon_set_sh_reg(cs, ms->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(ms->info.regs.pgm_lo, va >> 8); } + + radeon_end(); } static uint64_t diff --git a/src/amd/vulkan/meta/radv_meta_buffer.c b/src/amd/vulkan/meta/radv_meta_buffer.c index 563043b3673..4b4bfe39038 100644 --- a/src/amd/vulkan/meta/radv_meta_buffer.c +++ b/src/amd/vulkan/meta/radv_meta_buffer.c @@ -305,12 +305,13 @@ radv_update_buffer_cp(struct radv_cmd_buffer *cmd_buffer, uint64_t va, const voi radv_emit_cache_flush(cmd_buffer); radeon_check_space(device->ws, cmd_buffer->cs, words + 4); - radeon_emit(cmd_buffer->cs, PKT3(PKT3_WRITE_DATA, 2 + words, 0)); - radeon_emit(cmd_buffer->cs, - S_370_DST_SEL(mec ? V_370_MEM : V_370_MEM_GRBM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME)); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, va >> 32); - radeon_emit_array(cmd_buffer->cs, data, words); + radeon_begin(cmd_buffer->cs); + radeon_emit(PKT3(PKT3_WRITE_DATA, 2 + words, 0)); + radeon_emit(S_370_DST_SEL(mec ? V_370_MEM : V_370_MEM_GRBM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit_array(data, words); + radeon_end(); if (radv_device_fault_detection_enabled(device)) radv_cmd_buffer_trace_emit(cmd_buffer); diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c index 8a1c59ea6c9..14adda707af 100644 --- a/src/amd/vulkan/radv_cmd_buffer.c +++ b/src/amd/vulkan/radv_cmd_buffer.c @@ -619,8 +619,10 @@ radv_cmd_buffer_trace_emit(struct radv_cmd_buffer *cmd_buffer) radeon_check_space(device->ws, cs, 2); - radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); - radeon_emit(cs, AC_ENCODE_TRACE_POINT(cmd_buffer->state.trace_id)); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_NOP, 0, 0)); + radeon_emit(AC_ENCODE_TRACE_POINT(cmd_buffer->state.trace_id)); + radeon_end(); } void @@ -836,8 +838,9 @@ radv_cmd_buffer_after_draw(struct radv_cmd_buffer *cmd_buffer, enum radv_cmd_flu if (unlikely(device->sqtt.bo) && !dgc) { radeon_check_space(device->ws, cmd_buffer->cs, 2); - - radeon_event_write_predicate(cmd_buffer->cs, V_028A90_THREAD_TRACE_MARKER, cmd_buffer->state.predicating); + radeon_begin(cmd_buffer->cs); + radeon_event_write_predicate(V_028A90_THREAD_TRACE_MARKER, cmd_buffer->state.predicating); + radeon_end(); } if (instance->debug_flags & RADV_DEBUG_SYNC_SHADERS) { @@ -962,7 +965,9 @@ radv_emit_userdata_address(const struct radv_device *device, struct radeon_cmdbu if (!offset) return; - radeon_emit_32bit_pointer(cs, offset, va, &pdev->info); + radeon_begin(cs); + radeon_emit_32bit_pointer(offset, va, &pdev->info); + radeon_end(); } uint64_t @@ -988,8 +993,10 @@ radv_emit_descriptors_per_stage(const struct radv_device *device, struct radeon_ const uint32_t indirect_descriptor_sets_offset = radv_get_user_sgpr_loc(shader, AC_UD_INDIRECT_DESCRIPTOR_SETS); if (indirect_descriptor_sets_offset) { - radeon_emit_32bit_pointer(cs, indirect_descriptor_sets_offset, descriptors_state->indirect_descriptor_sets_va, + radeon_begin(cs); + radeon_emit_32bit_pointer(indirect_descriptor_sets_offset, descriptors_state->indirect_descriptor_sets_va, &pdev->info); + radeon_end(); } else { const struct radv_userdata_locations *locs = &shader->info.user_sgprs_locs; const uint32_t sh_base = shader->info.user_data_0; @@ -1005,13 +1012,16 @@ radv_emit_descriptors_per_stage(const struct radv_device *device, struct radeon_ const struct radv_userdata_info *loc = &locs->descriptor_sets[start]; const unsigned sh_offset = sh_base + loc->sgpr_idx * 4; - radeon_set_sh_reg_seq(cs, sh_offset, count); + radeon_begin(cs); + radeon_set_sh_reg_seq(sh_offset, count); for (int i = 0; i < count; i++) { uint64_t va = radv_descriptor_get_va(descriptors_state, start + i); - radeon_emit(cs, va); + radeon_emit(va); } + + radeon_end(); } } } @@ -1212,36 +1222,38 @@ radv_emit_sample_locations(struct radv_cmd_buffer *cmd_buffer) /* Compute the PA_SC_CENTROID_PRIORITY_* mask. */ centroid_priority = radv_compute_centroid_priority(cmd_buffer, sample_locs[0], num_samples); + radeon_begin(cs); + /* Emit the specified user sample locations. */ switch (num_samples) { case 2: case 4: - radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_pixel[0][0]); - radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_pixel[1][0]); - radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_pixel[2][0]); - radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_pixel[3][0]); + radeon_set_context_reg(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_pixel[0][0]); + radeon_set_context_reg(R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_pixel[1][0]); + radeon_set_context_reg(R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_pixel[2][0]); + radeon_set_context_reg(R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_pixel[3][0]); break; case 8: - radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_pixel[0][0]); - radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_pixel[1][0]); - radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_pixel[2][0]); - radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_pixel[3][0]); - radeon_set_context_reg(cs, R_028BFC_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1, sample_locs_pixel[0][1]); - radeon_set_context_reg(cs, R_028C0C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1, sample_locs_pixel[1][1]); - radeon_set_context_reg(cs, R_028C1C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1, sample_locs_pixel[2][1]); - radeon_set_context_reg(cs, R_028C2C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1, sample_locs_pixel[3][1]); + radeon_set_context_reg(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_pixel[0][0]); + radeon_set_context_reg(R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_pixel[1][0]); + radeon_set_context_reg(R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_pixel[2][0]); + radeon_set_context_reg(R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_pixel[3][0]); + radeon_set_context_reg(R_028BFC_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1, sample_locs_pixel[0][1]); + radeon_set_context_reg(R_028C0C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1, sample_locs_pixel[1][1]); + radeon_set_context_reg(R_028C1C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1, sample_locs_pixel[2][1]); + radeon_set_context_reg(R_028C2C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1, sample_locs_pixel[3][1]); break; default: unreachable("invalid number of samples"); } if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg_seq(cs, R_028BF0_PA_SC_CENTROID_PRIORITY_0, 2); + radeon_set_context_reg_seq(R_028BF0_PA_SC_CENTROID_PRIORITY_0, 2); } else { - radeon_set_context_reg_seq(cs, R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2); + radeon_set_context_reg_seq(R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2); } - radeon_emit(cs, centroid_priority); - radeon_emit(cs, centroid_priority >> 32); + radeon_emit(centroid_priority); + radeon_emit(centroid_priority >> 32); if (pdev->info.gfx_level >= GFX7) { /* The exclusion bits can be set to improve rasterization efficiency if no sample lies on the pixel boundary @@ -1257,8 +1269,10 @@ radv_emit_sample_locations(struct radv_cmd_buffer *cmd_buffer) } } - radeon_set_context_reg(cs, R_02882C_PA_SU_PRIM_FILTER_CNTL, pa_su_prim_filter_cntl); + radeon_set_context_reg(R_02882C_PA_SU_PRIM_FILTER_CNTL, pa_su_prim_filter_cntl); } + + radeon_end(); } static void @@ -1272,9 +1286,10 @@ radv_emit_inline_push_consts(const struct radv_device *device, struct radeon_cmd return; radeon_check_space(device->ws, cs, 2 + loc->num_sgprs); - - radeon_set_sh_reg_seq(cs, base_reg + loc->sgpr_idx * 4, loc->num_sgprs); - radeon_emit_array(cs, values, loc->num_sgprs); + radeon_begin(cs); + radeon_set_sh_reg_seq(base_reg + loc->sgpr_idx * 4, loc->num_sgprs); + radeon_emit_array(values, loc->num_sgprs); + radeon_end(); } struct radv_bin_size_entry { @@ -1731,8 +1746,10 @@ radv_emit_binning_state(struct radv_cmd_buffer *cmd_buffer) pa_sc_binner_cntl_0 = radv_get_binning_state(cmd_buffer); + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028C44_PA_SC_BINNER_CNTL_0, RADV_TRACKED_PA_SC_BINNER_CNTL_0, pa_sc_binner_cntl_0); + radeon_end(); } static void @@ -1926,8 +1943,10 @@ radv_emit_rbplus_state(struct radv_cmd_buffer *cmd_buffer) * breaks dual source blending in SkQP and does not seem to improve * performance. */ + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg3(cmd_buffer, R_028754_SX_PS_DOWNCONVERT, RADV_TRACKED_SX_PS_DOWNCONVERT, sx_ps_downconvert, sx_blend_opt_epsilon, sx_blend_opt_control); + radeon_end(); cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_RBPLUS; } @@ -1946,7 +1965,10 @@ radv_emit_ps_epilog_state(struct radv_cmd_buffer *cmd_buffer, struct radv_shader if (G_00B848_VGPRS(ps_epilog->rsrc1) > G_00B848_VGPRS(ps_shader->config.rsrc1)) { uint32_t rsrc1 = ps_shader->config.rsrc1; rsrc1 = (rsrc1 & C_00B848_VGPRS) | (ps_epilog->rsrc1 & ~C_00B848_VGPRS); - radeon_set_sh_reg(cmd_buffer->cs, ps_shader->info.regs.pgm_rsrc1, rsrc1); + + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(ps_shader->info.regs.pgm_rsrc1, rsrc1); + radeon_end(); } radv_cs_add_buffer(device->ws, cmd_buffer->cs, ps_epilog->bo); @@ -1954,7 +1976,9 @@ radv_emit_ps_epilog_state(struct radv_cmd_buffer *cmd_buffer, struct radv_shader assert((ps_epilog->va >> 32) == pdev->info.address32_hi); const uint32_t epilog_pc_offset = radv_get_user_sgpr_loc(ps_shader, AC_UD_EPILOG_PC); - radeon_emit_32bit_pointer(cmd_buffer->cs, epilog_pc_offset, ps_epilog->va, &pdev->info); + radeon_begin(cmd_buffer->cs); + radeon_emit_32bit_pointer(epilog_pc_offset, ps_epilog->va, &pdev->info); + radeon_end(); cmd_buffer->shader_upload_seq = MAX2(cmd_buffer->shader_upload_seq, ps_epilog->upload_seq); @@ -1967,20 +1991,22 @@ radv_emit_compute_shader(const struct radv_physical_device *pdev, struct radeon_ { uint64_t va = radv_shader_get_va(shader); - radeon_set_sh_reg(cs, shader->info.regs.pgm_lo, va >> 8); + radeon_begin(cs); + radeon_set_sh_reg(shader->info.regs.pgm_lo, va >> 8); - radeon_set_sh_reg_seq(cs, shader->info.regs.pgm_rsrc1, 2); - radeon_emit(cs, shader->config.rsrc1); - radeon_emit(cs, shader->config.rsrc2); + radeon_set_sh_reg_seq(shader->info.regs.pgm_rsrc1, 2); + radeon_emit(shader->config.rsrc1); + radeon_emit(shader->config.rsrc2); if (pdev->info.gfx_level >= GFX10) { - radeon_set_sh_reg(cs, shader->info.regs.pgm_rsrc3, shader->config.rsrc3); + radeon_set_sh_reg(shader->info.regs.pgm_rsrc3, shader->config.rsrc3); } - radeon_set_sh_reg(cs, R_00B854_COMPUTE_RESOURCE_LIMITS, shader->info.regs.cs.compute_resource_limits); - radeon_set_sh_reg_seq(cs, R_00B81C_COMPUTE_NUM_THREAD_X, 3); - radeon_emit(cs, shader->info.regs.cs.compute_num_thread_x); - radeon_emit(cs, shader->info.regs.cs.compute_num_thread_y); - radeon_emit(cs, shader->info.regs.cs.compute_num_thread_z); + radeon_set_sh_reg(R_00B854_COMPUTE_RESOURCE_LIMITS, shader->info.regs.cs.compute_resource_limits); + radeon_set_sh_reg_seq(R_00B81C_COMPUTE_NUM_THREAD_X, 3); + radeon_emit(shader->info.regs.cs.compute_num_thread_x); + radeon_emit(shader->info.regs.cs.compute_num_thread_y); + radeon_emit(shader->info.regs.cs.compute_num_thread_z); + radeon_end(); } static void @@ -2002,9 +2028,11 @@ radv_emit_vgt_gs_mode(struct radv_cmd_buffer *cmd_buffer) vgt_primitiveid_en |= S_028A84_PRIMITIVEID_EN(1); } + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028A84_VGT_PRIMITIVEID_EN, RADV_TRACKED_VGT_PRIMITIVEID_EN, vgt_primitiveid_en); radeon_opt_set_context_reg(cmd_buffer, R_028A40_VGT_GS_MODE, RADV_TRACKED_VGT_GS_MODE, vgt_gs_mode); + radeon_end(); } static void @@ -2014,11 +2042,12 @@ radv_emit_hw_vs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *sh const struct radv_physical_device *pdev = radv_device_physical(device); const uint64_t va = radv_shader_get_va(shader); - radeon_set_sh_reg_seq(cmd_buffer->cs, shader->info.regs.pgm_lo, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, S_00B124_MEM_BASE(va >> 40)); - radeon_emit(cmd_buffer->cs, shader->config.rsrc1); - radeon_emit(cmd_buffer->cs, shader->config.rsrc2); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg_seq(shader->info.regs.pgm_lo, 4); + radeon_emit(va >> 8); + radeon_emit(S_00B124_MEM_BASE(va >> 40)); + radeon_emit(shader->config.rsrc1); + radeon_emit(shader->config.rsrc2); radeon_opt_set_context_reg(cmd_buffer, R_0286C4_SPI_VS_OUT_CONFIG, RADV_TRACKED_SPI_VS_OUT_CONFIG, shader->info.regs.spi_vs_out_config); @@ -2032,13 +2061,12 @@ radv_emit_hw_vs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *sh shader->info.regs.vs.vgt_reuse_off); if (pdev->info.gfx_level >= GFX7) { - radeon_set_sh_reg_idx(&pdev->info, cmd_buffer->cs, R_00B118_SPI_SHADER_PGM_RSRC3_VS, 3, + radeon_set_sh_reg_idx(&pdev->info, R_00B118_SPI_SHADER_PGM_RSRC3_VS, 3, shader->info.regs.vs.spi_shader_pgm_rsrc3_vs); - radeon_set_sh_reg(cmd_buffer->cs, R_00B11C_SPI_SHADER_LATE_ALLOC_VS, - shader->info.regs.vs.spi_shader_late_alloc_vs); + radeon_set_sh_reg(R_00B11C_SPI_SHADER_LATE_ALLOC_VS, shader->info.regs.vs.spi_shader_late_alloc_vs); if (pdev->info.gfx_level >= GFX10) { - radeon_set_uconfig_reg(cmd_buffer->cs, R_030980_GE_PC_ALLOC, shader->info.regs.ge_pc_alloc); + radeon_set_uconfig_reg(R_030980_GE_PC_ALLOC, shader->info.regs.ge_pc_alloc); if (shader->info.stage == MESA_SHADER_TESS_EVAL) { radeon_opt_set_context_reg(cmd_buffer, R_028A44_VGT_GS_ONCHIP_CNTL, RADV_TRACKED_VGT_GS_ONCHIP_CNTL, @@ -2046,6 +2074,8 @@ radv_emit_hw_vs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *sh } } } + + radeon_end(); } static void @@ -2053,11 +2083,13 @@ radv_emit_hw_es(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *sh { const uint64_t va = radv_shader_get_va(shader); - radeon_set_sh_reg_seq(cmd_buffer->cs, shader->info.regs.pgm_lo, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, S_00B324_MEM_BASE(va >> 40)); - radeon_emit(cmd_buffer->cs, shader->config.rsrc1); - radeon_emit(cmd_buffer->cs, shader->config.rsrc2); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg_seq(shader->info.regs.pgm_lo, 4); + radeon_emit(va >> 8); + radeon_emit(S_00B324_MEM_BASE(va >> 40)); + radeon_emit(shader->config.rsrc1); + radeon_emit(shader->config.rsrc2); + radeon_end(); } static void @@ -2065,9 +2097,10 @@ radv_emit_hw_ls(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *sh { const uint64_t va = radv_shader_get_va(shader); - radeon_set_sh_reg(cmd_buffer->cs, shader->info.regs.pgm_lo, va >> 8); - - radeon_set_sh_reg(cmd_buffer->cs, shader->info.regs.pgm_rsrc1, shader->config.rsrc1); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(shader->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(shader->info.regs.pgm_rsrc1, shader->config.rsrc1); + radeon_end(); } static void @@ -2089,12 +2122,14 @@ radv_emit_hw_ngg(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *e es_type = shader->info.stage; } - if (!shader->info.merged_shader_compiled_separately) { - radeon_set_sh_reg(cmd_buffer->cs, shader->info.regs.pgm_lo, va >> 8); + radeon_begin(cmd_buffer->cs); - radeon_set_sh_reg_seq(cmd_buffer->cs, shader->info.regs.pgm_rsrc1, 2); - radeon_emit(cmd_buffer->cs, shader->config.rsrc1); - radeon_emit(cmd_buffer->cs, shader->config.rsrc2); + if (!shader->info.merged_shader_compiled_separately) { + radeon_set_sh_reg(shader->info.regs.pgm_lo, va >> 8); + + radeon_set_sh_reg_seq(shader->info.regs.pgm_rsrc1, 2); + radeon_emit(shader->config.rsrc1); + radeon_emit(shader->config.rsrc2); } const struct radv_vs_output_info *outinfo = &shader->info.outinfo; @@ -2114,7 +2149,7 @@ radv_emit_hw_ngg(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *e radeon_opt_set_context_reg(cmd_buffer, R_028B3C_VGT_GS_INSTANCE_CNT, RADV_TRACKED_VGT_GS_INSTANCE_CNT, shader->info.regs.vgt_gs_instance_cnt); - radeon_set_uconfig_reg(cmd_buffer->cs, R_030988_VGT_PRIMITIVEID_EN, shader->info.regs.ngg.vgt_primitiveid_en); + radeon_set_uconfig_reg(R_030988_VGT_PRIMITIVEID_EN, shader->info.regs.ngg.vgt_primitiveid_en); radeon_opt_set_context_reg2(cmd_buffer, R_028648_SPI_SHADER_IDX_FORMAT, RADV_TRACKED_SPI_SHADER_IDX_FORMAT, shader->info.regs.ngg.spi_shader_idx_format, shader->info.regs.spi_shader_pos_format); @@ -2164,21 +2199,23 @@ radv_emit_hw_ngg(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *e shader->info.regs.vgt_gs_onchip_cntl); } - radeon_set_uconfig_reg(cmd_buffer->cs, R_03096C_GE_CNTL, ge_cntl); + radeon_set_uconfig_reg(R_03096C_GE_CNTL, ge_cntl); if (pdev->info.gfx_level >= GFX12) { - radeon_set_sh_reg(cmd_buffer->cs, R_00B220_SPI_SHADER_PGM_RSRC4_GS, shader->info.regs.spi_shader_pgm_rsrc4_gs); + radeon_set_sh_reg(R_00B220_SPI_SHADER_PGM_RSRC4_GS, shader->info.regs.spi_shader_pgm_rsrc4_gs); } else { if (pdev->info.gfx_level >= GFX7) { - radeon_set_sh_reg_idx(&pdev->info, cmd_buffer->cs, R_00B21C_SPI_SHADER_PGM_RSRC3_GS, 3, + radeon_set_sh_reg_idx(&pdev->info, R_00B21C_SPI_SHADER_PGM_RSRC3_GS, 3, shader->info.regs.spi_shader_pgm_rsrc3_gs); } - radeon_set_sh_reg_idx(&pdev->info, cmd_buffer->cs, R_00B204_SPI_SHADER_PGM_RSRC4_GS, 3, + radeon_set_sh_reg_idx(&pdev->info, R_00B204_SPI_SHADER_PGM_RSRC4_GS, 3, shader->info.regs.spi_shader_pgm_rsrc4_gs); - radeon_set_uconfig_reg(cmd_buffer->cs, R_030980_GE_PC_ALLOC, shader->info.regs.ge_pc_alloc); + radeon_set_uconfig_reg(R_030980_GE_PC_ALLOC, shader->info.regs.ge_pc_alloc); } + + radeon_end(); } static void @@ -2188,16 +2225,18 @@ radv_emit_hw_hs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *sh const struct radv_physical_device *pdev = radv_device_physical(device); const uint64_t va = radv_shader_get_va(shader); + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX9) { - radeon_set_sh_reg(cmd_buffer->cs, shader->info.regs.pgm_lo, va >> 8); - radeon_set_sh_reg(cmd_buffer->cs, shader->info.regs.pgm_rsrc1, shader->config.rsrc1); + radeon_set_sh_reg(shader->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(shader->info.regs.pgm_rsrc1, shader->config.rsrc1); } else { - radeon_set_sh_reg_seq(cmd_buffer->cs, shader->info.regs.pgm_lo, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, S_00B424_MEM_BASE(va >> 40)); - radeon_emit(cmd_buffer->cs, shader->config.rsrc1); - radeon_emit(cmd_buffer->cs, shader->config.rsrc2); + radeon_set_sh_reg_seq(shader->info.regs.pgm_lo, 4); + radeon_emit(va >> 8); + radeon_emit(S_00B424_MEM_BASE(va >> 40)); + radeon_emit(shader->config.rsrc1); + radeon_emit(shader->config.rsrc2); } + radeon_end(); } static void @@ -2215,12 +2254,14 @@ radv_emit_vertex_shader(struct radv_cmd_buffer *cmd_buffer) if (!vs->info.vs.has_prolog) { uint32_t rsrc1, rsrc2; - radeon_set_sh_reg(cmd_buffer->cs, vs->info.regs.pgm_lo, vs->va >> 8); + radeon_begin(cmd_buffer->cs); + + radeon_set_sh_reg(vs->info.regs.pgm_lo, vs->va >> 8); if (vs->info.next_stage == MESA_SHADER_TESS_CTRL) { radv_shader_combine_cfg_vs_tcs(vs, next_stage, &rsrc1, NULL); - radeon_set_sh_reg(cmd_buffer->cs, vs->info.regs.pgm_rsrc1, rsrc1); + radeon_set_sh_reg(vs->info.regs.pgm_rsrc1, rsrc1); } else { radv_shader_combine_cfg_vs_gs(vs, next_stage, &rsrc1, &rsrc2); @@ -2231,14 +2272,18 @@ radv_emit_vertex_shader(struct radv_cmd_buffer *cmd_buffer) lds_size = next_stage->info.gs_ring_info.lds_size; } - radeon_set_sh_reg_seq(cmd_buffer->cs, vs->info.regs.pgm_rsrc1, 2); - radeon_emit(cmd_buffer->cs, rsrc1); - radeon_emit(cmd_buffer->cs, rsrc2 | S_00B22C_LDS_SIZE(lds_size)); + radeon_set_sh_reg_seq(vs->info.regs.pgm_rsrc1, 2); + radeon_emit(rsrc1); + radeon_emit(rsrc2 | S_00B22C_LDS_SIZE(lds_size)); } + + radeon_end(); } const uint32_t next_stage_pc_offset = radv_get_user_sgpr_loc(vs, AC_UD_NEXT_STAGE_PC); - radeon_emit_32bit_pointer(cmd_buffer->cs, next_stage_pc_offset, next_stage->va, &pdev->info); + radeon_begin(cmd_buffer->cs); + radeon_emit_32bit_pointer(next_stage_pc_offset, next_stage->va, &pdev->info); + radeon_end(); return; } @@ -2289,14 +2334,16 @@ radv_emit_tess_eval_shader(struct radv_cmd_buffer *cmd_buffer) lds_size = gs->info.gs_ring_info.lds_size; } - radeon_set_sh_reg(cmd_buffer->cs, tes->info.regs.pgm_lo, tes->va >> 8); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(tes->info.regs.pgm_lo, tes->va >> 8); - radeon_set_sh_reg_seq(cmd_buffer->cs, tes->info.regs.pgm_rsrc1, 2); - radeon_emit(cmd_buffer->cs, rsrc1); - radeon_emit(cmd_buffer->cs, rsrc2 | S_00B22C_LDS_SIZE(lds_size)); + radeon_set_sh_reg_seq(tes->info.regs.pgm_rsrc1, 2); + radeon_emit(rsrc1); + radeon_emit(rsrc2 | S_00B22C_LDS_SIZE(lds_size)); const uint32_t next_stage_pc_offset = radv_get_user_sgpr_loc(tes, AC_UD_NEXT_STAGE_PC); - radeon_emit_32bit_pointer(cmd_buffer->cs, next_stage_pc_offset, gs->va, &pdev->info); + radeon_emit_32bit_pointer(next_stage_pc_offset, gs->va, &pdev->info); + radeon_end(); return; } @@ -2317,6 +2364,8 @@ radv_emit_hw_gs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *gs const struct radv_legacy_gs_info *gs_state = &gs->info.gs_ring_info; const uint64_t va = radv_shader_get_va(gs); + radeon_begin(cmd_buffer->cs); + radeon_opt_set_context_reg3(cmd_buffer, R_028A60_VGT_GSVS_RING_OFFSET_1, RADV_TRACKED_VGT_GSVS_RING_OFFSET_1, gs->info.regs.gs.vgt_gsvs_ring_offset[0], gs->info.regs.gs.vgt_gsvs_ring_offset[1], gs->info.regs.gs.vgt_gsvs_ring_offset[2]); @@ -2333,11 +2382,11 @@ radv_emit_hw_gs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *gs if (pdev->info.gfx_level >= GFX9) { if (!gs->info.merged_shader_compiled_separately) { - radeon_set_sh_reg(cmd_buffer->cs, gs->info.regs.pgm_lo, va >> 8); + radeon_set_sh_reg(gs->info.regs.pgm_lo, va >> 8); - radeon_set_sh_reg_seq(cmd_buffer->cs, gs->info.regs.pgm_rsrc1, 2); - radeon_emit(cmd_buffer->cs, gs->config.rsrc1); - radeon_emit(cmd_buffer->cs, gs->config.rsrc2 | S_00B22C_LDS_SIZE(gs_state->lds_size)); + radeon_set_sh_reg_seq(gs->info.regs.pgm_rsrc1, 2); + radeon_emit(gs->config.rsrc1); + radeon_emit(gs->config.rsrc2 | S_00B22C_LDS_SIZE(gs_state->lds_size)); } radeon_opt_set_context_reg(cmd_buffer, R_028A44_VGT_GS_ONCHIP_CNTL, RADV_TRACKED_VGT_GS_ONCHIP_CNTL, @@ -2349,11 +2398,11 @@ radv_emit_hw_gs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *gs gs->info.regs.gs.vgt_gs_max_prims_per_subgroup); } } else { - radeon_set_sh_reg_seq(cmd_buffer->cs, gs->info.regs.pgm_lo, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, S_00B224_MEM_BASE(va >> 40)); - radeon_emit(cmd_buffer->cs, gs->config.rsrc1); - radeon_emit(cmd_buffer->cs, gs->config.rsrc2); + radeon_set_sh_reg_seq(gs->info.regs.pgm_lo, 4); + radeon_emit(va >> 8); + radeon_emit(S_00B224_MEM_BASE(va >> 40)); + radeon_emit(gs->config.rsrc1); + radeon_emit(gs->config.rsrc2); /* GFX6-8: ESGS offchip ring buffer is allocated according to VGT_ESGS_RING_ITEMSIZE. * GFX9+: Only used to set the GS input VGPRs, emulated in shaders. @@ -2363,14 +2412,14 @@ radv_emit_hw_gs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *gs } if (pdev->info.gfx_level >= GFX7) { - radeon_set_sh_reg_idx(&pdev->info, cmd_buffer->cs, R_00B21C_SPI_SHADER_PGM_RSRC3_GS, 3, - gs->info.regs.spi_shader_pgm_rsrc3_gs); + radeon_set_sh_reg_idx(&pdev->info, R_00B21C_SPI_SHADER_PGM_RSRC3_GS, 3, gs->info.regs.spi_shader_pgm_rsrc3_gs); } if (pdev->info.gfx_level >= GFX10) { - radeon_set_sh_reg_idx(&pdev->info, cmd_buffer->cs, R_00B204_SPI_SHADER_PGM_RSRC4_GS, 3, - gs->info.regs.spi_shader_pgm_rsrc4_gs); + radeon_set_sh_reg_idx(&pdev->info, R_00B204_SPI_SHADER_PGM_RSRC4_GS, 3, gs->info.regs.spi_shader_pgm_rsrc4_gs); } + + radeon_end(); } static void @@ -2387,6 +2436,8 @@ radv_emit_geometry_shader(struct radv_cmd_buffer *cmd_buffer) radv_emit_hw_vs(cmd_buffer, cmd_buffer->state.gs_copy_shader); } + radeon_begin(cmd_buffer->cs); + radeon_opt_set_context_reg(cmd_buffer, R_028B38_VGT_GS_MAX_VERT_OUT, RADV_TRACKED_VGT_GS_MAX_VERT_OUT, gs->info.regs.vgt_gs_max_vert_out); @@ -2395,7 +2446,7 @@ radv_emit_geometry_shader(struct radv_cmd_buffer *cmd_buffer) assert(vgt_esgs_ring_itemsize_offset); - radeon_set_sh_reg(cmd_buffer->cs, vgt_esgs_ring_itemsize_offset, es->info.esgs_itemsize / 4); + radeon_set_sh_reg(vgt_esgs_ring_itemsize_offset, es->info.esgs_itemsize / 4); if (gs->info.is_ngg) { const uint32_t ngg_lds_layout_offset = radv_get_user_sgpr_loc(gs, AC_UD_NGG_LDS_LAYOUT); @@ -2403,11 +2454,13 @@ radv_emit_geometry_shader(struct radv_cmd_buffer *cmd_buffer) assert(ngg_lds_layout_offset); assert(!(gs->info.ngg_info.esgs_ring_size & 0xffff0000) && !(gs->info.ngg_info.scratch_lds_base & 0xffff0000)); - radeon_set_sh_reg(cmd_buffer->cs, ngg_lds_layout_offset, + radeon_set_sh_reg(ngg_lds_layout_offset, SET_SGPR_FIELD(NGG_LDS_LAYOUT_GS_OUT_VERTEX_BASE, gs->info.ngg_info.esgs_ring_size) | SET_SGPR_FIELD(NGG_LDS_LAYOUT_SCRATCH_BASE, gs->info.ngg_info.scratch_lds_base)); } } + + radeon_end(); } static void @@ -2416,12 +2469,14 @@ radv_emit_vgt_gs_out(struct radv_cmd_buffer *cmd_buffer, uint32_t vgt_gs_out_pri const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_physical_device *pdev = radv_device_physical(device); + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX11) { - radeon_set_uconfig_reg(cmd_buffer->cs, R_030998_VGT_GS_OUT_PRIM_TYPE, vgt_gs_out_prim_type); + radeon_set_uconfig_reg(R_030998_VGT_GS_OUT_PRIM_TYPE, vgt_gs_out_prim_type); } else { radeon_opt_set_context_reg(cmd_buffer, R_028A6C_VGT_GS_OUT_PRIM_TYPE, RADV_TRACKED_VGT_GS_OUT_PRIM_TYPE, vgt_gs_out_prim_type); } + radeon_end(); } static void @@ -2433,20 +2488,24 @@ radv_emit_mesh_shader(struct radv_cmd_buffer *cmd_buffer) const uint32_t gs_out = radv_conv_gl_prim_to_gs_out(ms->info.ms.output_prim); radv_emit_hw_ngg(cmd_buffer, NULL, ms); + + radeon_begin(cmd_buffer->cs); + radeon_opt_set_context_reg(cmd_buffer, R_028B38_VGT_GS_MAX_VERT_OUT, RADV_TRACKED_VGT_GS_MAX_VERT_OUT, ms->info.regs.vgt_gs_max_vert_out); - radeon_set_uconfig_reg_idx(&pdev->info, cmd_buffer->cs, R_030908_VGT_PRIMITIVE_TYPE, 1, V_008958_DI_PT_POINTLIST); + radeon_set_uconfig_reg_idx(&pdev->info, R_030908_VGT_PRIMITIVE_TYPE, 1, V_008958_DI_PT_POINTLIST); if (pdev->mesh_fast_launch_2) { - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B2B0_SPI_SHADER_GS_MESHLET_DIM, 2); - radeon_emit(cmd_buffer->cs, ms->info.regs.ms.spi_shader_gs_meshlet_dim); - radeon_emit(cmd_buffer->cs, ms->info.regs.ms.spi_shader_gs_meshlet_exp_alloc); + radeon_set_sh_reg_seq(R_00B2B0_SPI_SHADER_GS_MESHLET_DIM, 2); + radeon_emit(ms->info.regs.ms.spi_shader_gs_meshlet_dim); + radeon_emit(ms->info.regs.ms.spi_shader_gs_meshlet_exp_alloc); if (pdev->info.gfx_level >= GFX12) - radeon_set_sh_reg(cmd_buffer->cs, R_00B2B8_SPI_SHADER_GS_MESHLET_CTRL, - ms->info.regs.ms.spi_shader_gs_meshlet_ctrl); + radeon_set_sh_reg(R_00B2B8_SPI_SHADER_GS_MESHLET_CTRL, ms->info.regs.ms.spi_shader_gs_meshlet_ctrl); } + radeon_end(); + radv_emit_vgt_gs_out(cmd_buffer, gs_out); } @@ -2576,6 +2635,8 @@ radv_emit_ps_inputs(struct radv_cmd_buffer *cmd_buffer) /* Only GFX10.3+ support per-primitive params */ assert(pdev->info.gfx_level >= GFX10_3 || num_per_primitive_params == 0); + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { radeon_opt_set_context_regn(cmd_buffer, R_028664_SPI_PS_INPUT_CNTL_0, ps_input_cntl, cmd_buffer->tracked_regs.spi_ps_input_cntl, ps_offset); @@ -2594,6 +2655,8 @@ radv_emit_ps_inputs(struct radv_cmd_buffer *cmd_buffer) radeon_opt_set_context_regn(cmd_buffer, R_028644_SPI_PS_INPUT_CNTL_0, ps_input_cntl, cmd_buffer->tracked_regs.spi_ps_input_cntl, ps_offset); } + + radeon_end(); } static void @@ -2605,6 +2668,8 @@ radv_emit_fragment_shader_state(struct radv_cmd_buffer *cmd_buffer, const struct const uint32_t spi_ps_input_addr = ps ? ps->config.spi_ps_input_addr : 0; const uint32_t spi_ps_in_control = ps ? ps->info.regs.ps.spi_ps_in_control : 0; + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { const uint32_t pa_sc_hisz_control = ps ? ps->info.regs.ps.pa_sc_hisz_control : 0; @@ -2631,6 +2696,8 @@ radv_emit_fragment_shader_state(struct radv_cmd_buffer *cmd_buffer, const struct radeon_opt_set_context_reg(cmd_buffer, R_028C40_PA_SC_SHADER_CONTROL, RADV_TRACKED_PA_SC_SHADER_CONTROL, pa_sc_shader_control); } + + radeon_end(); } static void @@ -2639,11 +2706,13 @@ radv_emit_fragment_shader(struct radv_cmd_buffer *cmd_buffer) const struct radv_shader *ps = cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT]; const uint64_t va = radv_shader_get_va(ps); - radeon_set_sh_reg_seq(cmd_buffer->cs, ps->info.regs.pgm_lo, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, S_00B024_MEM_BASE(va >> 40)); - radeon_emit(cmd_buffer->cs, ps->config.rsrc1); - radeon_emit(cmd_buffer->cs, ps->config.rsrc2); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg_seq(ps->info.regs.pgm_lo, 4); + radeon_emit(va >> 8); + radeon_emit(S_00B024_MEM_BASE(va >> 40)); + radeon_emit(ps->config.rsrc1); + radeon_emit(ps->config.rsrc2); + radeon_end(); radv_emit_fragment_shader_state(cmd_buffer, ps); } @@ -2659,8 +2728,10 @@ radv_emit_vgt_reuse(struct radv_cmd_buffer *cmd_buffer, const struct radv_vgt_sh /* Legacy Tess+GS should disable reuse to prevent hangs on GFX10.3. */ const bool has_legacy_tess_gs = key->tess && key->gs && !key->ngg; + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028AB4_VGT_REUSE_OFF, RADV_TRACKED_VGT_REUSE_OFF, S_028AB4_REUSE_OFF(has_legacy_tess_gs)); + radeon_end(); } if (pdev->info.family >= CHIP_POLARIS10 && pdev->info.gfx_level < GFX10) { @@ -2668,8 +2739,11 @@ radv_emit_vgt_reuse(struct radv_cmd_buffer *cmd_buffer, const struct radv_vgt_sh if (tes && tes->info.tes.spacing == TESS_SPACING_FRACTIONAL_ODD) { vtx_reuse_depth = 14; } + + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028C58_VGT_VERTEX_REUSE_BLOCK_CNTL, RADV_TRACKED_VGT_VERTEX_REUSE_BLOCK_CNTL, S_028C58_VTX_REUSE_DEPTH(vtx_reuse_depth)); + radeon_end(); } } @@ -2685,7 +2759,9 @@ radv_emit_vgt_shader_config_gfx12(struct radv_cmd_buffer *cmd_buffer, const stru if (key->tess) stages |= S_028A98_HS_EN(1) | S_028A98_HS_W32_EN(key->hs_wave32); + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028A98_VGT_SHADER_STAGES_EN, RADV_TRACKED_VGT_SHADER_STAGES_EN, stages); + radeon_end(); } static void @@ -2734,7 +2810,9 @@ radv_emit_vgt_shader_config_gfx6(struct radv_cmd_buffer *cmd_buffer, const struc assert(!(key->gs && !key->ngg) || !key->gs_wave32); } + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028B54_VGT_SHADER_STAGES_EN, RADV_TRACKED_VGT_SHADER_STAGES_EN, stages); + radeon_end(); } static void @@ -2772,6 +2850,8 @@ gfx103_emit_vgt_draw_payload_cntl(struct radv_cmd_buffer *cmd_buffer) const uint32_t vgt_draw_payload_cntl = S_028A98_EN_VRS_RATE(enable_vrs) | S_028A98_EN_PRIM_PAYLOAD(enable_prim_payload); + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { radeon_opt_set_context_reg(cmd_buffer, R_028AA0_VGT_DRAW_PAYLOAD_CNTL, RADV_TRACKED_VGT_DRAW_PAYLOAD_CNTL, vgt_draw_payload_cntl); @@ -2779,6 +2859,8 @@ gfx103_emit_vgt_draw_payload_cntl(struct radv_cmd_buffer *cmd_buffer) radeon_opt_set_context_reg(cmd_buffer, R_028A98_VGT_DRAW_PAYLOAD_CNTL, RADV_TRACKED_VGT_DRAW_PAYLOAD_CNTL, vgt_draw_payload_cntl); } + + radeon_end(); } static void @@ -2803,9 +2885,11 @@ gfx103_emit_vrs_state(struct radv_cmd_buffer *cmd_buffer) * requested by the user. Note that vkd3d-proton always has to declare VRS as dynamic because * in DX12 it's fully dynamic. */ + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028848_PA_CL_VRS_CNTL, RADV_TRACKED_PA_CL_VRS_CNTL, S_028848_SAMPLE_ITER_COMBINER_MODE(V_028848_SC_VRS_COMB_MODE_OVERRIDE) | S_028848_VERTEX_RATE_COMBINER_MODE(V_028848_SC_VRS_COMB_MODE_OVERRIDE)); + radeon_end(); /* If the shader is using discard, turn off coarse shading because discard at 2x2 pixel * granularity degrades quality too much. MIN allows sample shading but not coarse shading. @@ -2814,9 +2898,11 @@ gfx103_emit_vrs_state(struct radv_cmd_buffer *cmd_buffer) } if (pdev->info.gfx_level < GFX11) { + radeon_begin(cmd_buffer->cs); radeon_opt_set_context_reg(cmd_buffer, R_028064_DB_VRS_OVERRIDE_CNTL, RADV_TRACKED_DB_VRS_OVERRIDE_CNTL, S_028064_VRS_OVERRIDE_RATE_COMBINER_MODE(mode) | S_028064_VRS_OVERRIDE_RATE_X(rate_x) | S_028064_VRS_OVERRIDE_RATE_Y(rate_y)); + radeon_end(); } } @@ -2868,7 +2954,9 @@ radv_emit_graphics_shaders(struct radv_cmd_buffer *cmd_buffer) radv_emit_fragment_shader_state(cmd_buffer, NULL); } - radeon_set_sh_reg(cmd_buffer->cs, R_00B0C4_SPI_SHADER_GS_OUT_CONFIG_PS, gs_out_config_ps); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(R_00B0C4_SPI_SHADER_GS_OUT_CONFIG_PS, gs_out_config_ps); + radeon_end(); } const struct radv_vgt_shader_key vgt_shader_cfg_key = @@ -2934,7 +3022,9 @@ radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer) cmd_buffer->state.graphics_pipeline->base.shaders[MESA_SHADER_FRAGMENT]) && (settings->context_states_per_bin > 1 || settings->persistent_states_per_bin > 1)) { /* Break the batch on PS changes. */ - radeon_event_write(cmd_buffer->cs, V_028A90_BREAK_BATCH); + radeon_begin(cmd_buffer->cs); + radeon_event_write(V_028A90_BREAK_BATCH); + radeon_end(); } } @@ -2947,7 +3037,9 @@ radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer) const struct radv_sqtt_shaders_reloc *reloc = pipeline->sqtt_shaders_reloc; const uint64_t va = reloc->va[MESA_SHADER_TASK]; - radeon_set_sh_reg(cmd_buffer->gang.cs, task_shader->info.regs.pgm_lo, va >> 8); + radeon_begin(cmd_buffer->gang.cs); + radeon_set_sh_reg(task_shader->info.regs.pgm_lo, va >> 8); + radeon_end(); } } @@ -3040,8 +3132,10 @@ radv_emit_viewport(struct radv_cmd_buffer *cmd_buffer) assert(d->vk.vp.viewport_count); + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_02843C_PA_CL_VPORT_XSCALE, d->vk.vp.viewport_count * 8); + radeon_set_context_reg_seq(R_02843C_PA_CL_VPORT_XSCALE, d->vk.vp.viewport_count * 8); for (unsigned i = 0; i < d->vk.vp.viewport_count; i++) { float zscale, ztranslate, zmin, zmax; @@ -3049,41 +3143,43 @@ radv_emit_viewport(struct radv_cmd_buffer *cmd_buffer) radv_get_viewport_zscale_ztranslate(cmd_buffer, i, &zscale, &ztranslate); radv_get_viewport_zmin_zmax(cmd_buffer, &d->vk.vp.viewports[i], &zmin, &zmax); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].scale[0])); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].translate[0])); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].scale[1])); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].translate[1])); - radeon_emit(cmd_buffer->cs, fui(zscale)); - radeon_emit(cmd_buffer->cs, fui(ztranslate)); - radeon_emit(cmd_buffer->cs, fui(zmin)); - radeon_emit(cmd_buffer->cs, fui(zmax)); + radeon_emit(fui(d->hw_vp.xform[i].scale[0])); + radeon_emit(fui(d->hw_vp.xform[i].translate[0])); + radeon_emit(fui(d->hw_vp.xform[i].scale[1])); + radeon_emit(fui(d->hw_vp.xform[i].translate[1])); + radeon_emit(fui(zscale)); + radeon_emit(fui(ztranslate)); + radeon_emit(fui(zmin)); + radeon_emit(fui(zmax)); } } else { - radeon_set_context_reg_seq(cmd_buffer->cs, R_02843C_PA_CL_VPORT_XSCALE, d->vk.vp.viewport_count * 6); + radeon_set_context_reg_seq(R_02843C_PA_CL_VPORT_XSCALE, d->vk.vp.viewport_count * 6); for (unsigned i = 0; i < d->vk.vp.viewport_count; i++) { float zscale, ztranslate; radv_get_viewport_zscale_ztranslate(cmd_buffer, i, &zscale, &ztranslate); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].scale[0])); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].translate[0])); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].scale[1])); - radeon_emit(cmd_buffer->cs, fui(d->hw_vp.xform[i].translate[1])); - radeon_emit(cmd_buffer->cs, fui(zscale)); - radeon_emit(cmd_buffer->cs, fui(ztranslate)); + radeon_emit(fui(d->hw_vp.xform[i].scale[0])); + radeon_emit(fui(d->hw_vp.xform[i].translate[0])); + radeon_emit(fui(d->hw_vp.xform[i].scale[1])); + radeon_emit(fui(d->hw_vp.xform[i].translate[1])); + radeon_emit(fui(zscale)); + radeon_emit(fui(ztranslate)); } - radeon_set_context_reg_seq(cmd_buffer->cs, R_0282D0_PA_SC_VPORT_ZMIN_0, d->vk.vp.viewport_count * 2); + radeon_set_context_reg_seq(R_0282D0_PA_SC_VPORT_ZMIN_0, d->vk.vp.viewport_count * 2); for (unsigned i = 0; i < d->vk.vp.viewport_count; i++) { float zmin, zmax; radv_get_viewport_zmin_zmax(cmd_buffer, &d->vk.vp.viewports[i], &zmin, &zmax); - radeon_emit(cmd_buffer->cs, fui(zmin)); - radeon_emit(cmd_buffer->cs, fui(zmax)); + radeon_emit(fui(zmin)); + radeon_emit(fui(zmax)); } } + + radeon_end(); } static VkRect2D @@ -3121,7 +3217,9 @@ radv_emit_scissor(struct radv_cmd_buffer *cmd_buffer) if (!d->vk.vp.scissor_count) return; - radeon_set_context_reg_seq(cs, R_028250_PA_SC_VPORT_SCISSOR_0_TL, d->vk.vp.scissor_count * 2); + radeon_begin(cs); + radeon_set_context_reg_seq(R_028250_PA_SC_VPORT_SCISSOR_0_TL, d->vk.vp.scissor_count * 2); + for (unsigned i = 0; i < d->vk.vp.scissor_count; i++) { VkRect2D viewport_scissor = radv_scissor_from_viewport(d->hw_vp.xform[i].scale, d->hw_vp.xform[i].translate); VkRect2D scissor = radv_intersect_scissor(&d->vk.vp.scissors[i], &viewport_scissor); @@ -3137,13 +3235,15 @@ radv_emit_scissor(struct radv_cmd_buffer *cmd_buffer) minx = miny = maxx = maxy = 1; } - radeon_emit(cs, S_028250_TL_X(minx) | S_028250_TL_Y_GFX12(miny)); - radeon_emit(cs, S_028254_BR_X(maxx - 1) | S_028254_BR_Y(maxy - 1)); + radeon_emit(S_028250_TL_X(minx) | S_028250_TL_Y_GFX12(miny)); + radeon_emit(S_028254_BR_X(maxx - 1) | S_028254_BR_Y(maxy - 1)); } else { - radeon_emit(cs, S_028250_TL_X(minx) | S_028250_TL_Y_GFX6(miny) | S_028250_WINDOW_OFFSET_DISABLE(1)); - radeon_emit(cs, S_028254_BR_X(maxx) | S_028254_BR_Y(maxy)); + radeon_emit(S_028250_TL_X(minx) | S_028250_TL_Y_GFX6(miny) | S_028250_WINDOW_OFFSET_DISABLE(1)); + radeon_emit(S_028254_BR_X(maxx) | S_028254_BR_Y(maxy)); } } + + radeon_end(); } static void @@ -3154,6 +3254,8 @@ radv_emit_discard_rectangle(struct radv_cmd_buffer *cmd_buffer) const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; uint32_t cliprect_rule = 0; + radeon_begin(cmd_buffer->cs); + if (!d->vk.dr.enable) { cliprect_rule = 0xffff; } else { @@ -3174,27 +3276,27 @@ radv_emit_discard_rectangle(struct radv_cmd_buffer *cmd_buffer) cliprect_rule |= 1u << i; } - radeon_set_context_reg_seq(cmd_buffer->cs, R_028210_PA_SC_CLIPRECT_0_TL, d->vk.dr.rectangle_count * 2); + radeon_set_context_reg_seq(R_028210_PA_SC_CLIPRECT_0_TL, d->vk.dr.rectangle_count * 2); for (unsigned i = 0; i < d->vk.dr.rectangle_count; ++i) { VkRect2D rect = d->vk.dr.rectangles[i]; - radeon_emit(cmd_buffer->cs, S_028210_TL_X(rect.offset.x) | S_028210_TL_Y(rect.offset.y)); - radeon_emit(cmd_buffer->cs, S_028214_BR_X(rect.offset.x + rect.extent.width) | - S_028214_BR_Y(rect.offset.y + rect.extent.height)); + radeon_emit(S_028210_TL_X(rect.offset.x) | S_028210_TL_Y(rect.offset.y)); + radeon_emit(S_028214_BR_X(rect.offset.x + rect.extent.width) | + S_028214_BR_Y(rect.offset.y + rect.extent.height)); } if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028374_PA_SC_CLIPRECT_0_EXT, d->vk.dr.rectangle_count); + radeon_set_context_reg_seq(R_028374_PA_SC_CLIPRECT_0_EXT, d->vk.dr.rectangle_count); for (unsigned i = 0; i < d->vk.dr.rectangle_count; ++i) { VkRect2D rect = d->vk.dr.rectangles[i]; - radeon_emit(cmd_buffer->cs, S_028374_TL_X_EXT(rect.offset.x >> 15) | - S_028374_TL_Y_EXT(rect.offset.y >> 15) | - S_028374_BR_X_EXT((rect.offset.x + rect.extent.width) >> 15) | - S_028374_BR_Y_EXT((rect.offset.y + rect.extent.height) >> 15)); + radeon_emit(S_028374_TL_X_EXT(rect.offset.x >> 15) | S_028374_TL_Y_EXT(rect.offset.y >> 15) | + S_028374_BR_X_EXT((rect.offset.x + rect.extent.width) >> 15) | + S_028374_BR_Y_EXT((rect.offset.y + rect.extent.height) >> 15)); } } } - radeon_set_context_reg(cmd_buffer->cs, R_02820C_PA_SC_CLIPRECT_RULE, cliprect_rule); + radeon_set_context_reg(R_02820C_PA_SC_CLIPRECT_RULE, cliprect_rule); + radeon_end(); } static void @@ -3202,8 +3304,10 @@ radv_emit_blend_constants(struct radv_cmd_buffer *cmd_buffer) { const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; - radeon_set_context_reg_seq(cmd_buffer->cs, R_028414_CB_BLEND_RED, 4); - radeon_emit_array(cmd_buffer->cs, (uint32_t *)d->vk.cb.blend_constants, 4); + radeon_begin(cmd_buffer->cs); + radeon_set_context_reg_seq(R_028414_CB_BLEND_RED, 4); + radeon_emit_array((uint32_t *)d->vk.cb.blend_constants, 4); + radeon_end(); } static void @@ -3232,14 +3336,16 @@ radv_emit_depth_bias(struct radv_cmd_buffer *cmd_buffer) } } - radeon_set_context_reg_seq(cmd_buffer->cs, R_028B7C_PA_SU_POLY_OFFSET_CLAMP, 5); - radeon_emit(cmd_buffer->cs, fui(d->vk.rs.depth_bias.clamp)); /* CLAMP */ - radeon_emit(cmd_buffer->cs, slope); /* FRONT SCALE */ - radeon_emit(cmd_buffer->cs, fui(d->vk.rs.depth_bias.constant_factor)); /* FRONT OFFSET */ - radeon_emit(cmd_buffer->cs, slope); /* BACK SCALE */ - radeon_emit(cmd_buffer->cs, fui(d->vk.rs.depth_bias.constant_factor)); /* BACK OFFSET */ + radeon_begin(cmd_buffer->cs); + radeon_set_context_reg_seq(R_028B7C_PA_SU_POLY_OFFSET_CLAMP, 5); + radeon_emit(fui(d->vk.rs.depth_bias.clamp)); /* CLAMP */ + radeon_emit(slope); /* FRONT SCALE */ + radeon_emit(fui(d->vk.rs.depth_bias.constant_factor)); /* FRONT OFFSET */ + radeon_emit(slope); /* BACK SCALE */ + radeon_emit(fui(d->vk.rs.depth_bias.constant_factor)); /* BACK OFFSET */ - radeon_set_context_reg(cmd_buffer->cs, R_028B78_PA_SU_POLY_OFFSET_DB_FMT_CNTL, pa_su_poly_offset_db_fmt_cntl); + radeon_set_context_reg(R_028B78_PA_SU_POLY_OFFSET_DB_FMT_CNTL, pa_su_poly_offset_db_fmt_cntl); + radeon_end(); } static void @@ -3252,16 +3358,18 @@ radv_emit_primitive_topology(struct radv_cmd_buffer *cmd_buffer) assert(!cmd_buffer->state.mesh_shading); + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX7) { uint32_t vgt_prim = d->vk.ia.primitive_topology; if (pdev->info.gfx_level >= GFX12) vgt_prim |= S_030908_NUM_INPUT_CP(d->vk.ts.patch_control_points); - radeon_set_uconfig_reg_idx(&pdev->info, cmd_buffer->cs, R_030908_VGT_PRIMITIVE_TYPE, 1, vgt_prim); + radeon_set_uconfig_reg_idx(&pdev->info, R_030908_VGT_PRIMITIVE_TYPE, 1, vgt_prim); } else { - radeon_set_config_reg(cmd_buffer->cs, R_008958_VGT_PRIMITIVE_TYPE, d->vk.ia.primitive_topology); + radeon_set_config_reg(R_008958_VGT_PRIMITIVE_TYPE, d->vk.ia.primitive_topology); } + radeon_end(); radv_emit_vgt_gs_out(cmd_buffer, vgt_gs_out_prim_type); } @@ -3354,10 +3462,14 @@ radv_emit_fragment_shading_rate(struct radv_cmd_buffer *cmd_buffer) */ pa_cl_vrs_cntl |= S_028848_HTILE_RATE_COMBINER_MODE(htile_comb_mode); - /* Emit per-draw VRS rate which is the first combiner. */ - radeon_set_uconfig_reg(cmd_buffer->cs, R_03098C_GE_VRS_RATE, S_03098C_RATE_X(rate_x) | S_03098C_RATE_Y(rate_y)); + radeon_begin(cmd_buffer->cs); - radeon_set_context_reg(cmd_buffer->cs, R_028848_PA_CL_VRS_CNTL, pa_cl_vrs_cntl); + /* Emit per-draw VRS rate which is the first combiner. */ + radeon_set_uconfig_reg(R_03098C_GE_VRS_RATE, S_03098C_RATE_X(rate_x) | S_03098C_RATE_Y(rate_y)); + + radeon_set_context_reg(R_028848_PA_CL_VRS_CNTL, pa_cl_vrs_cntl); + + radeon_end(); } static uint32_t @@ -3386,21 +3498,22 @@ radv_emit_primitive_restart_enable(struct radv_cmd_buffer *cmd_buffer) struct radeon_cmdbuf *cs = cmd_buffer->cs; const bool en = d->vk.ia.primitive_restart_enable; + radeon_begin(cs); + if (pdev->info.has_prim_restart_sync_bug) { - radeon_event_write(cmd_buffer->cs, V_028A90_SQ_NON_EVENT); + radeon_event_write(V_028A90_SQ_NON_EVENT); } if (gfx_level >= GFX11) { - radeon_set_uconfig_reg(cs, R_03092C_GE_MULTI_PRIM_IB_RESET_EN, - S_03092C_RESET_EN(en) | - /* This disables primitive restart for non-indexed draws. - * By keeping this set, we don't have to unset RESET_EN - * for non-indexed draws. */ - S_03092C_DISABLE_FOR_AUTO_INDEX(1)); + radeon_set_uconfig_reg(R_03092C_GE_MULTI_PRIM_IB_RESET_EN, S_03092C_RESET_EN(en) | + /* This disables primitive restart for non-indexed + * draws. By keeping this set, we don't have to + * unset RESET_EN for non-indexed draws. */ + S_03092C_DISABLE_FOR_AUTO_INDEX(1)); } else if (gfx_level >= GFX9) { - radeon_set_uconfig_reg(cs, R_03092C_VGT_MULTI_PRIM_IB_RESET_EN, en); + radeon_set_uconfig_reg(R_03092C_VGT_MULTI_PRIM_IB_RESET_EN, en); } else { - radeon_set_context_reg(cs, R_028A94_VGT_MULTI_PRIM_IB_RESET_EN, en); + radeon_set_context_reg(R_028A94_VGT_MULTI_PRIM_IB_RESET_EN, en); /* GFX6-7: All 32 bits are compared. * GFX8: Only index type bits are compared. @@ -3413,6 +3526,8 @@ radv_emit_primitive_restart_enable(struct radv_cmd_buffer *cmd_buffer) RADV_TRACKED_VGT_MULTI_PRIM_IB_RESET_INDX, primitive_reset_index); } } + + radeon_end(); } static void @@ -3456,11 +3571,13 @@ radv_emit_logic_op(struct radv_cmd_buffer *cmd_buffer) } } + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028858_CB_COLOR_CONTROL, cb_color_control); + radeon_set_context_reg(R_028858_CB_COLOR_CONTROL, cb_color_control); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028808_CB_COLOR_CONTROL, cb_color_control); + radeon_set_context_reg(R_028808_CB_COLOR_CONTROL, cb_color_control); } + radeon_end(); } static void @@ -3485,16 +3602,20 @@ radv_emit_color_write(struct radv_cmd_buffer *cmd_buffer) if (device->pbb_allowed && settings->context_states_per_bin > 1 && cmd_buffer->state.last_cb_target_mask != cb_target_mask) { /* Flush DFSM on CB_TARGET_MASK changes. */ - radeon_event_write(cmd_buffer->cs, V_028A90_BREAK_BATCH); + radeon_begin(cmd_buffer->cs); + radeon_event_write(V_028A90_BREAK_BATCH); + radeon_end(); cmd_buffer->state.last_cb_target_mask = cb_target_mask; } + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028850_CB_TARGET_MASK, cb_target_mask); + radeon_set_context_reg(R_028850_CB_TARGET_MASK, cb_target_mask); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028238_CB_TARGET_MASK, cb_target_mask); + radeon_set_context_reg(R_028238_CB_TARGET_MASK, cb_target_mask); } + radeon_end(); } static void @@ -3535,10 +3656,12 @@ radv_emit_patch_control_points(struct radv_cmd_buffer *cmd_buffer) S_028B58_HS_NUM_INPUT_CP(pdev->info.gfx_level < GFX12 ? d->vk.ts.patch_control_points : 0) | S_028B58_HS_NUM_OUTPUT_CP(tcs->info.tcs.tcs_vertices_out); + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX7) { - radeon_set_context_reg_idx(cmd_buffer->cs, R_028B58_VGT_LS_HS_CONFIG, 2, ls_hs_config); + radeon_set_context_reg_idx(R_028B58_VGT_LS_HS_CONFIG, 2, ls_hs_config); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028B58_VGT_LS_HS_CONFIG, ls_hs_config); + radeon_set_context_reg(R_028B58_VGT_LS_HS_CONFIG, ls_hs_config); } if (pdev->info.gfx_level >= GFX9) { @@ -3556,11 +3679,11 @@ radv_emit_patch_control_points(struct radv_cmd_buffer *cmd_buffer) hs_rsrc2 |= S_00B42C_LDS_SIZE_GFX9(cmd_buffer->state.tess_lds_size); } - radeon_set_sh_reg(cmd_buffer->cs, tcs->info.regs.pgm_rsrc2, hs_rsrc2); + radeon_set_sh_reg(tcs->info.regs.pgm_rsrc2, hs_rsrc2); } else { unsigned ls_rsrc2 = vs->config.rsrc2 | S_00B52C_LDS_SIZE(cmd_buffer->state.tess_lds_size); - radeon_set_sh_reg(cmd_buffer->cs, vs->info.regs.pgm_rsrc2, ls_rsrc2); + radeon_set_sh_reg(vs->info.regs.pgm_rsrc2, ls_rsrc2); } /* Emit user SGPRs for dynamic patch control points. */ @@ -3575,13 +3698,15 @@ radv_emit_patch_control_points(struct radv_cmd_buffer *cmd_buffer) SET_SGPR_FIELD(TCS_OFFCHIP_LAYOUT_TES_READS_TF, tes->info.tes.reads_tess_factors) | SET_SGPR_FIELD(TCS_OFFCHIP_LAYOUT_PRIMITIVE_MODE, tes->info.tes._primitive_mode); - radeon_set_sh_reg(cmd_buffer->cs, tcs_offchip_layout_offset, tcs_offchip_layout); + radeon_set_sh_reg(tcs_offchip_layout_offset, tcs_offchip_layout); tcs_offchip_layout_offset = radv_get_user_sgpr_loc(tes, AC_UD_TCS_OFFCHIP_LAYOUT); assert(tcs_offchip_layout_offset); - radeon_set_sh_reg(cmd_buffer->cs, tcs_offchip_layout_offset, tcs_offchip_layout); + radeon_set_sh_reg(tcs_offchip_layout_offset, tcs_offchip_layout); } + + radeon_end(); } static void @@ -3613,13 +3738,13 @@ radv_emit_conservative_rast_mode(struct radv_cmd_buffer *cmd_buffer) pa_sc_conservative_rast = S_028C4C_NULL_SQUAD_AA_MASK_ENABLE(1); } + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028C54_PA_SC_CONSERVATIVE_RASTERIZATION_CNTL, - pa_sc_conservative_rast); + radeon_set_context_reg(R_028C54_PA_SC_CONSERVATIVE_RASTERIZATION_CNTL, pa_sc_conservative_rast); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028C4C_PA_SC_CONSERVATIVE_RASTERIZATION_CNTL, - pa_sc_conservative_rast); + radeon_set_context_reg(R_028C4C_PA_SC_CONSERVATIVE_RASTERIZATION_CNTL, pa_sc_conservative_rast); } + radeon_end(); } } @@ -3631,13 +3756,15 @@ radv_emit_depth_clamp_enable(struct radv_cmd_buffer *cmd_buffer) enum radv_depth_clamp_mode mode = radv_get_depth_clamp_mode(cmd_buffer); + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028064_DB_VIEWPORT_CONTROL, + radeon_set_context_reg(R_028064_DB_VIEWPORT_CONTROL, S_028064_DISABLE_VIEWPORT_CLAMP(mode == RADV_DEPTH_CLAMP_MODE_DISABLED)); } else { - radeon_set_context_reg(cmd_buffer->cs, R_02800C_DB_RENDER_OVERRIDE, + radeon_set_context_reg(R_02800C_DB_RENDER_OVERRIDE, S_02800C_DISABLE_VIEWPORT_CLAMP(mode == RADV_DEPTH_CLAMP_MODE_DISABLED)); } + radeon_end(); } static void @@ -3701,13 +3828,15 @@ radv_emit_rasterization_samples(struct radv_cmd_buffer *cmd_buffer) pa_sc_mode_cntl_1 |= S_028A4C_PS_ITER_SAMPLE(1); } + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028658_SPI_BARYC_CNTL, spi_baryc_cntl); + radeon_set_context_reg(R_028658_SPI_BARYC_CNTL, spi_baryc_cntl); } else { - radeon_set_context_reg(cmd_buffer->cs, R_0286E0_SPI_BARYC_CNTL, spi_baryc_cntl); + radeon_set_context_reg(R_0286E0_SPI_BARYC_CNTL, spi_baryc_cntl); } - radeon_set_context_reg(cmd_buffer->cs, R_028A4C_PA_SC_MODE_CNTL_1, pa_sc_mode_cntl_1); + radeon_set_context_reg(R_028A4C_PA_SC_MODE_CNTL_1, pa_sc_mode_cntl_1); + radeon_end(); } static void @@ -3739,100 +3868,99 @@ radv_emit_fb_color_state(struct radv_cmd_buffer *cmd_buffer, int index, struct r if (pdev->info.gfx_level >= GFX8 && pdev->info.gfx_level < GFX11 && iview->disable_tc_compat_cmask_mrt) cb_color_info &= C_028C70_FMASK_COMPRESS_1FRAG_ONLY; + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x24, cb->ac.cb_color_base); - radeon_set_context_reg(cmd_buffer->cs, R_028C64_CB_COLOR0_VIEW + index * 0x24, cb->ac.cb_color_view); - radeon_set_context_reg(cmd_buffer->cs, R_028C68_CB_COLOR0_VIEW2 + index * 0x24, cb->ac.cb_color_view2); - radeon_set_context_reg(cmd_buffer->cs, R_028C6C_CB_COLOR0_ATTRIB + index * 0x24, cb->ac.cb_color_attrib); - radeon_set_context_reg(cmd_buffer->cs, R_028C70_CB_COLOR0_FDCC_CONTROL + index * 0x24, cb_fdcc_control); - radeon_set_context_reg(cmd_buffer->cs, R_028C78_CB_COLOR0_ATTRIB2 + index * 0x24, cb->ac.cb_color_attrib2); - radeon_set_context_reg(cmd_buffer->cs, R_028C7C_CB_COLOR0_ATTRIB3 + index * 0x24, cb->ac.cb_color_attrib3); - radeon_set_context_reg(cmd_buffer->cs, R_028E40_CB_COLOR0_BASE_EXT + index * 4, - S_028E40_BASE_256B(cb->ac.cb_color_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028EC0_CB_COLOR0_INFO + index * 4, cb->ac.cb_color_info); + radeon_set_context_reg(R_028C60_CB_COLOR0_BASE + index * 0x24, cb->ac.cb_color_base); + radeon_set_context_reg(R_028C64_CB_COLOR0_VIEW + index * 0x24, cb->ac.cb_color_view); + radeon_set_context_reg(R_028C68_CB_COLOR0_VIEW2 + index * 0x24, cb->ac.cb_color_view2); + radeon_set_context_reg(R_028C6C_CB_COLOR0_ATTRIB + index * 0x24, cb->ac.cb_color_attrib); + radeon_set_context_reg(R_028C70_CB_COLOR0_FDCC_CONTROL + index * 0x24, cb_fdcc_control); + radeon_set_context_reg(R_028C78_CB_COLOR0_ATTRIB2 + index * 0x24, cb->ac.cb_color_attrib2); + radeon_set_context_reg(R_028C7C_CB_COLOR0_ATTRIB3 + index * 0x24, cb->ac.cb_color_attrib3); + radeon_set_context_reg(R_028E40_CB_COLOR0_BASE_EXT + index * 4, S_028E40_BASE_256B(cb->ac.cb_color_base >> 32)); + radeon_set_context_reg(R_028EC0_CB_COLOR0_INFO + index * 4, cb->ac.cb_color_info); } else if (pdev->info.gfx_level >= GFX11) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C6C_CB_COLOR0_VIEW + index * 0x3c, 4); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_view); /* CB_COLOR0_VIEW */ - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_info); /* CB_COLOR0_INFO */ - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_attrib); /* CB_COLOR0_ATTRIB */ - radeon_emit(cmd_buffer->cs, cb_fdcc_control); /* CB_COLOR0_FDCC_CONTROL */ + radeon_set_context_reg_seq(R_028C6C_CB_COLOR0_VIEW + index * 0x3c, 4); + radeon_emit(cb->ac.cb_color_view); /* CB_COLOR0_VIEW */ + radeon_emit(cb->ac.cb_color_info); /* CB_COLOR0_INFO */ + radeon_emit(cb->ac.cb_color_attrib); /* CB_COLOR0_ATTRIB */ + radeon_emit(cb_fdcc_control); /* CB_COLOR0_FDCC_CONTROL */ - radeon_set_context_reg(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, cb->ac.cb_color_base); - radeon_set_context_reg(cmd_buffer->cs, R_028E40_CB_COLOR0_BASE_EXT + index * 4, - S_028E40_BASE_256B(cb->ac.cb_color_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->ac.cb_dcc_base); - radeon_set_context_reg(cmd_buffer->cs, R_028EA0_CB_COLOR0_DCC_BASE_EXT + index * 4, - S_028EA0_BASE_256B(cb->ac.cb_dcc_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028EC0_CB_COLOR0_ATTRIB2 + index * 4, cb->ac.cb_color_attrib2); - radeon_set_context_reg(cmd_buffer->cs, R_028EE0_CB_COLOR0_ATTRIB3 + index * 4, cb->ac.cb_color_attrib3); + radeon_set_context_reg(R_028C60_CB_COLOR0_BASE + index * 0x3c, cb->ac.cb_color_base); + radeon_set_context_reg(R_028E40_CB_COLOR0_BASE_EXT + index * 4, S_028E40_BASE_256B(cb->ac.cb_color_base >> 32)); + radeon_set_context_reg(R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->ac.cb_dcc_base); + radeon_set_context_reg(R_028EA0_CB_COLOR0_DCC_BASE_EXT + index * 4, S_028EA0_BASE_256B(cb->ac.cb_dcc_base >> 32)); + radeon_set_context_reg(R_028EC0_CB_COLOR0_ATTRIB2 + index * 4, cb->ac.cb_color_attrib2); + radeon_set_context_reg(R_028EE0_CB_COLOR0_ATTRIB3 + index * 4, cb->ac.cb_color_attrib3); } else if (pdev->info.gfx_level >= GFX10) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 11); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_base); - radeon_emit(cmd_buffer->cs, 0); - radeon_emit(cmd_buffer->cs, 0); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_view); - radeon_emit(cmd_buffer->cs, cb_color_info); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_attrib); - radeon_emit(cmd_buffer->cs, cb->ac.cb_dcc_control); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_cmask); - radeon_emit(cmd_buffer->cs, 0); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_fmask); - radeon_emit(cmd_buffer->cs, 0); + radeon_set_context_reg_seq(R_028C60_CB_COLOR0_BASE + index * 0x3c, 11); + radeon_emit(cb->ac.cb_color_base); + radeon_emit(0); + radeon_emit(0); + radeon_emit(cb->ac.cb_color_view); + radeon_emit(cb_color_info); + radeon_emit(cb->ac.cb_color_attrib); + radeon_emit(cb->ac.cb_dcc_control); + radeon_emit(cb->ac.cb_color_cmask); + radeon_emit(0); + radeon_emit(cb->ac.cb_color_fmask); + radeon_emit(0); - radeon_set_context_reg(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->ac.cb_dcc_base); + radeon_set_context_reg(R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->ac.cb_dcc_base); - radeon_set_context_reg(cmd_buffer->cs, R_028E40_CB_COLOR0_BASE_EXT + index * 4, - S_028E40_BASE_256B(cb->ac.cb_color_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028E60_CB_COLOR0_CMASK_BASE_EXT + index * 4, + radeon_set_context_reg(R_028E40_CB_COLOR0_BASE_EXT + index * 4, S_028E40_BASE_256B(cb->ac.cb_color_base >> 32)); + radeon_set_context_reg(R_028E60_CB_COLOR0_CMASK_BASE_EXT + index * 4, S_028E60_BASE_256B(cb->ac.cb_color_cmask >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028E80_CB_COLOR0_FMASK_BASE_EXT + index * 4, + radeon_set_context_reg(R_028E80_CB_COLOR0_FMASK_BASE_EXT + index * 4, S_028E80_BASE_256B(cb->ac.cb_color_fmask >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028EA0_CB_COLOR0_DCC_BASE_EXT + index * 4, - S_028EA0_BASE_256B(cb->ac.cb_dcc_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028EC0_CB_COLOR0_ATTRIB2 + index * 4, cb->ac.cb_color_attrib2); - radeon_set_context_reg(cmd_buffer->cs, R_028EE0_CB_COLOR0_ATTRIB3 + index * 4, cb->ac.cb_color_attrib3); + radeon_set_context_reg(R_028EA0_CB_COLOR0_DCC_BASE_EXT + index * 4, S_028EA0_BASE_256B(cb->ac.cb_dcc_base >> 32)); + radeon_set_context_reg(R_028EC0_CB_COLOR0_ATTRIB2 + index * 4, cb->ac.cb_color_attrib2); + radeon_set_context_reg(R_028EE0_CB_COLOR0_ATTRIB3 + index * 4, cb->ac.cb_color_attrib3); } else if (pdev->info.gfx_level == GFX9) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 11); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_base); - radeon_emit(cmd_buffer->cs, S_028C64_BASE_256B(cb->ac.cb_color_base >> 32)); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_attrib2); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_view); - radeon_emit(cmd_buffer->cs, cb_color_info); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_attrib); - radeon_emit(cmd_buffer->cs, cb->ac.cb_dcc_control); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_cmask); - radeon_emit(cmd_buffer->cs, S_028C80_BASE_256B(cb->ac.cb_color_cmask >> 32)); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_fmask); - radeon_emit(cmd_buffer->cs, S_028C88_BASE_256B(cb->ac.cb_color_fmask >> 32)); + radeon_set_context_reg_seq(R_028C60_CB_COLOR0_BASE + index * 0x3c, 11); + radeon_emit(cb->ac.cb_color_base); + radeon_emit(S_028C64_BASE_256B(cb->ac.cb_color_base >> 32)); + radeon_emit(cb->ac.cb_color_attrib2); + radeon_emit(cb->ac.cb_color_view); + radeon_emit(cb_color_info); + radeon_emit(cb->ac.cb_color_attrib); + radeon_emit(cb->ac.cb_dcc_control); + radeon_emit(cb->ac.cb_color_cmask); + radeon_emit(S_028C80_BASE_256B(cb->ac.cb_color_cmask >> 32)); + radeon_emit(cb->ac.cb_color_fmask); + radeon_emit(S_028C88_BASE_256B(cb->ac.cb_color_fmask >> 32)); - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, 2); - radeon_emit(cmd_buffer->cs, cb->ac.cb_dcc_base); - radeon_emit(cmd_buffer->cs, S_028C98_BASE_256B(cb->ac.cb_dcc_base >> 32)); + radeon_set_context_reg_seq(R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, 2); + radeon_emit(cb->ac.cb_dcc_base); + radeon_emit(S_028C98_BASE_256B(cb->ac.cb_dcc_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_0287A0_CB_MRT0_EPITCH + index * 4, cb->ac.cb_mrt_epitch); + radeon_set_context_reg(R_0287A0_CB_MRT0_EPITCH + index * 4, cb->ac.cb_mrt_epitch); } else { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 6); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_base); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_pitch); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_slice); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_view); - radeon_emit(cmd_buffer->cs, cb_color_info); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_attrib); + radeon_set_context_reg_seq(R_028C60_CB_COLOR0_BASE + index * 0x3c, 6); + radeon_emit(cb->ac.cb_color_base); + radeon_emit(cb->ac.cb_color_pitch); + radeon_emit(cb->ac.cb_color_slice); + radeon_emit(cb->ac.cb_color_view); + radeon_emit(cb_color_info); + radeon_emit(cb->ac.cb_color_attrib); if (pdev->info.gfx_level == GFX8) - radeon_set_context_reg(cmd_buffer->cs, R_028C78_CB_COLOR0_DCC_CONTROL + index * 0x3c, cb->ac.cb_dcc_control); + radeon_set_context_reg(R_028C78_CB_COLOR0_DCC_CONTROL + index * 0x3c, cb->ac.cb_dcc_control); - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C7C_CB_COLOR0_CMASK + index * 0x3c, 4); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_cmask); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_cmask_slice); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_fmask); - radeon_emit(cmd_buffer->cs, cb->ac.cb_color_fmask_slice); + radeon_set_context_reg_seq(R_028C7C_CB_COLOR0_CMASK + index * 0x3c, 4); + radeon_emit(cb->ac.cb_color_cmask); + radeon_emit(cb->ac.cb_color_cmask_slice); + radeon_emit(cb->ac.cb_color_fmask); + radeon_emit(cb->ac.cb_color_fmask_slice); if (is_vi) { /* DCC BASE */ - radeon_set_context_reg(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->ac.cb_dcc_base); + radeon_set_context_reg(R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->ac.cb_dcc_base); } } + radeon_end(); + if (pdev->info.gfx_level >= GFX11 ? G_028C78_FDCC_ENABLE(cb_fdcc_control) : G_028C70_DCC_ENABLE(cb_color_info)) { /* Drawing with DCC enabled also compresses colorbuffers. */ VkImageSubresourceRange range = vk_image_view_subresource_range(&iview->vk); @@ -3872,7 +4000,9 @@ radv_update_zrange_precision(struct radv_cmd_buffer *cmd_buffer, struct radv_ds_ radv_emit_cond_exec(device, cmd_buffer->cs, va, 3 /* SET_CONTEXT_REG size */); } - radeon_set_context_reg(cmd_buffer->cs, db_z_info_reg, db_z_info); + radeon_begin(cmd_buffer->cs); + radeon_set_context_reg(db_z_info_reg, db_z_info); + radeon_end(); } static struct radv_image * @@ -3929,104 +4059,104 @@ radv_emit_fb_ds_state(struct radv_cmd_buffer *cmd_buffer, struct radv_ds_buffer_ } } + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level < GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028000_DB_RENDER_CONTROL, db_render_control); - radeon_set_context_reg(cmd_buffer->cs, R_028008_DB_DEPTH_VIEW, ds->ac.db_depth_view); - radeon_set_context_reg(cmd_buffer->cs, R_028ABC_DB_HTILE_SURFACE, db_htile_surface); + radeon_set_context_reg(R_028000_DB_RENDER_CONTROL, db_render_control); + radeon_set_context_reg(R_028008_DB_DEPTH_VIEW, ds->ac.db_depth_view); + radeon_set_context_reg(R_028ABC_DB_HTILE_SURFACE, db_htile_surface); } - radeon_set_context_reg(cmd_buffer->cs, R_028010_DB_RENDER_OVERRIDE2, ds->db_render_override2); + radeon_set_context_reg(R_028010_DB_RENDER_OVERRIDE2, ds->db_render_override2); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028004_DB_DEPTH_VIEW, ds->ac.db_depth_view); - radeon_set_context_reg(cmd_buffer->cs, R_028008_DB_DEPTH_VIEW1, ds->ac.u.gfx12.db_depth_view1); - radeon_set_context_reg(cmd_buffer->cs, R_028014_DB_DEPTH_SIZE_XY, ds->ac.db_depth_size); - radeon_set_context_reg(cmd_buffer->cs, R_028018_DB_Z_INFO, ds->ac.db_z_info); - radeon_set_context_reg(cmd_buffer->cs, R_02801C_DB_STENCIL_INFO, ds->ac.db_stencil_info); - radeon_set_context_reg(cmd_buffer->cs, R_028020_DB_Z_READ_BASE, ds->ac.db_depth_base); - radeon_set_context_reg(cmd_buffer->cs, R_028024_DB_Z_READ_BASE_HI, S_028024_BASE_HI(ds->ac.db_depth_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028028_DB_Z_WRITE_BASE, ds->ac.db_depth_base); - radeon_set_context_reg(cmd_buffer->cs, R_02802C_DB_Z_WRITE_BASE_HI, S_02802C_BASE_HI(ds->ac.db_depth_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028030_DB_STENCIL_READ_BASE, ds->ac.db_stencil_base); - radeon_set_context_reg(cmd_buffer->cs, R_028034_DB_STENCIL_READ_BASE_HI, - S_028034_BASE_HI(ds->ac.db_stencil_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028038_DB_STENCIL_WRITE_BASE, ds->ac.db_stencil_base); - radeon_set_context_reg(cmd_buffer->cs, R_02803C_DB_STENCIL_WRITE_BASE_HI, - S_02803C_BASE_HI(ds->ac.db_stencil_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028B94_PA_SC_HIZ_INFO, ds->ac.u.gfx12.hiz_info); - radeon_set_context_reg(cmd_buffer->cs, R_028B98_PA_SC_HIS_INFO, ds->ac.u.gfx12.his_info); + radeon_set_context_reg(R_028004_DB_DEPTH_VIEW, ds->ac.db_depth_view); + radeon_set_context_reg(R_028008_DB_DEPTH_VIEW1, ds->ac.u.gfx12.db_depth_view1); + radeon_set_context_reg(R_028014_DB_DEPTH_SIZE_XY, ds->ac.db_depth_size); + radeon_set_context_reg(R_028018_DB_Z_INFO, ds->ac.db_z_info); + radeon_set_context_reg(R_02801C_DB_STENCIL_INFO, ds->ac.db_stencil_info); + radeon_set_context_reg(R_028020_DB_Z_READ_BASE, ds->ac.db_depth_base); + radeon_set_context_reg(R_028024_DB_Z_READ_BASE_HI, S_028024_BASE_HI(ds->ac.db_depth_base >> 32)); + radeon_set_context_reg(R_028028_DB_Z_WRITE_BASE, ds->ac.db_depth_base); + radeon_set_context_reg(R_02802C_DB_Z_WRITE_BASE_HI, S_02802C_BASE_HI(ds->ac.db_depth_base >> 32)); + radeon_set_context_reg(R_028030_DB_STENCIL_READ_BASE, ds->ac.db_stencil_base); + radeon_set_context_reg(R_028034_DB_STENCIL_READ_BASE_HI, S_028034_BASE_HI(ds->ac.db_stencil_base >> 32)); + radeon_set_context_reg(R_028038_DB_STENCIL_WRITE_BASE, ds->ac.db_stencil_base); + radeon_set_context_reg(R_02803C_DB_STENCIL_WRITE_BASE_HI, S_02803C_BASE_HI(ds->ac.db_stencil_base >> 32)); + radeon_set_context_reg(R_028B94_PA_SC_HIZ_INFO, ds->ac.u.gfx12.hiz_info); + radeon_set_context_reg(R_028B98_PA_SC_HIS_INFO, ds->ac.u.gfx12.his_info); if (ds->ac.u.gfx12.hiz_info) { - radeon_set_context_reg(cmd_buffer->cs, R_028B9C_PA_SC_HIZ_BASE, ds->ac.u.gfx12.hiz_base); - radeon_set_context_reg(cmd_buffer->cs, R_028BA0_PA_SC_HIZ_BASE_EXT, - S_028BA0_BASE_256B(ds->ac.u.gfx12.hiz_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028BA4_PA_SC_HIZ_SIZE_XY, ds->ac.u.gfx12.hiz_size_xy); + radeon_set_context_reg(R_028B9C_PA_SC_HIZ_BASE, ds->ac.u.gfx12.hiz_base); + radeon_set_context_reg(R_028BA0_PA_SC_HIZ_BASE_EXT, S_028BA0_BASE_256B(ds->ac.u.gfx12.hiz_base >> 32)); + radeon_set_context_reg(R_028BA4_PA_SC_HIZ_SIZE_XY, ds->ac.u.gfx12.hiz_size_xy); } if (ds->ac.u.gfx12.his_info) { - radeon_set_context_reg(cmd_buffer->cs, R_028BA8_PA_SC_HIS_BASE, ds->ac.u.gfx12.his_base); - radeon_set_context_reg(cmd_buffer->cs, R_028BAC_PA_SC_HIS_BASE_EXT, - S_028BAC_BASE_256B(ds->ac.u.gfx12.his_base >> 32)); - radeon_set_context_reg(cmd_buffer->cs, R_028BB0_PA_SC_HIS_SIZE_XY, ds->ac.u.gfx12.his_size_xy); + radeon_set_context_reg(R_028BA8_PA_SC_HIS_BASE, ds->ac.u.gfx12.his_base); + radeon_set_context_reg(R_028BAC_PA_SC_HIS_BASE_EXT, S_028BAC_BASE_256B(ds->ac.u.gfx12.his_base >> 32)); + radeon_set_context_reg(R_028BB0_PA_SC_HIS_SIZE_XY, ds->ac.u.gfx12.his_size_xy); } } else if (pdev->info.gfx_level >= GFX10) { - radeon_set_context_reg(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, db_htile_data_base); - radeon_set_context_reg(cmd_buffer->cs, R_02801C_DB_DEPTH_SIZE_XY, ds->ac.db_depth_size); + radeon_set_context_reg(R_028014_DB_HTILE_DATA_BASE, db_htile_data_base); + radeon_set_context_reg(R_02801C_DB_DEPTH_SIZE_XY, ds->ac.db_depth_size); if (pdev->info.gfx_level >= GFX11) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028040_DB_Z_INFO, 6); + radeon_set_context_reg_seq(R_028040_DB_Z_INFO, 6); } else { - radeon_set_context_reg_seq(cmd_buffer->cs, R_02803C_DB_DEPTH_INFO, 7); - radeon_emit(cmd_buffer->cs, S_02803C_RESOURCE_LEVEL(1)); + radeon_set_context_reg_seq(R_02803C_DB_DEPTH_INFO, 7); + radeon_emit(S_02803C_RESOURCE_LEVEL(1)); } - radeon_emit(cmd_buffer->cs, db_z_info); - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_info); - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_base); - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_base); - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_base); - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_base); + radeon_emit(db_z_info); + radeon_emit(ds->ac.db_stencil_info); + radeon_emit(ds->ac.db_depth_base); + radeon_emit(ds->ac.db_stencil_base); + radeon_emit(ds->ac.db_depth_base); + radeon_emit(ds->ac.db_stencil_base); - radeon_set_context_reg_seq(cmd_buffer->cs, R_028068_DB_Z_READ_BASE_HI, 5); - radeon_emit(cmd_buffer->cs, S_028068_BASE_HI(ds->ac.db_depth_base >> 32)); - radeon_emit(cmd_buffer->cs, S_02806C_BASE_HI(ds->ac.db_stencil_base >> 32)); - radeon_emit(cmd_buffer->cs, S_028070_BASE_HI(ds->ac.db_depth_base >> 32)); - radeon_emit(cmd_buffer->cs, S_028074_BASE_HI(ds->ac.db_stencil_base >> 32)); - radeon_emit(cmd_buffer->cs, S_028078_BASE_HI(db_htile_data_base >> 32)); + radeon_set_context_reg_seq(R_028068_DB_Z_READ_BASE_HI, 5); + radeon_emit(S_028068_BASE_HI(ds->ac.db_depth_base >> 32)); + radeon_emit(S_02806C_BASE_HI(ds->ac.db_stencil_base >> 32)); + radeon_emit(S_028070_BASE_HI(ds->ac.db_depth_base >> 32)); + radeon_emit(S_028074_BASE_HI(ds->ac.db_stencil_base >> 32)); + radeon_emit(S_028078_BASE_HI(db_htile_data_base >> 32)); } else if (pdev->info.gfx_level == GFX9) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, 3); - radeon_emit(cmd_buffer->cs, db_htile_data_base); - radeon_emit(cmd_buffer->cs, S_028018_BASE_HI(db_htile_data_base >> 32)); - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_size); + radeon_set_context_reg_seq(R_028014_DB_HTILE_DATA_BASE, 3); + radeon_emit(db_htile_data_base); + radeon_emit(S_028018_BASE_HI(db_htile_data_base >> 32)); + radeon_emit(ds->ac.db_depth_size); - radeon_set_context_reg_seq(cmd_buffer->cs, R_028038_DB_Z_INFO, 10); - radeon_emit(cmd_buffer->cs, db_z_info); /* DB_Z_INFO */ - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_info); /* DB_STENCIL_INFO */ - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_base); /* DB_Z_READ_BASE */ - radeon_emit(cmd_buffer->cs, S_028044_BASE_HI(ds->ac.db_depth_base >> 32)); /* DB_Z_READ_BASE_HI */ - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_base); /* DB_STENCIL_READ_BASE */ - radeon_emit(cmd_buffer->cs, S_02804C_BASE_HI(ds->ac.db_stencil_base >> 32)); /* DB_STENCIL_READ_BASE_HI */ - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_base); /* DB_Z_WRITE_BASE */ - radeon_emit(cmd_buffer->cs, S_028054_BASE_HI(ds->ac.db_depth_base >> 32)); /* DB_Z_WRITE_BASE_HI */ - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_base); /* DB_STENCIL_WRITE_BASE */ - radeon_emit(cmd_buffer->cs, S_02805C_BASE_HI(ds->ac.db_stencil_base >> 32)); /* DB_STENCIL_WRITE_BASE_HI */ + radeon_set_context_reg_seq(R_028038_DB_Z_INFO, 10); + radeon_emit(db_z_info); /* DB_Z_INFO */ + radeon_emit(ds->ac.db_stencil_info); /* DB_STENCIL_INFO */ + radeon_emit(ds->ac.db_depth_base); /* DB_Z_READ_BASE */ + radeon_emit(S_028044_BASE_HI(ds->ac.db_depth_base >> 32)); /* DB_Z_READ_BASE_HI */ + radeon_emit(ds->ac.db_stencil_base); /* DB_STENCIL_READ_BASE */ + radeon_emit(S_02804C_BASE_HI(ds->ac.db_stencil_base >> 32)); /* DB_STENCIL_READ_BASE_HI */ + radeon_emit(ds->ac.db_depth_base); /* DB_Z_WRITE_BASE */ + radeon_emit(S_028054_BASE_HI(ds->ac.db_depth_base >> 32)); /* DB_Z_WRITE_BASE_HI */ + radeon_emit(ds->ac.db_stencil_base); /* DB_STENCIL_WRITE_BASE */ + radeon_emit(S_02805C_BASE_HI(ds->ac.db_stencil_base >> 32)); /* DB_STENCIL_WRITE_BASE_HI */ - radeon_set_context_reg_seq(cmd_buffer->cs, R_028068_DB_Z_INFO2, 2); - radeon_emit(cmd_buffer->cs, ds->ac.u.gfx6.db_z_info2); - radeon_emit(cmd_buffer->cs, ds->ac.u.gfx6.db_stencil_info2); + radeon_set_context_reg_seq(R_028068_DB_Z_INFO2, 2); + radeon_emit(ds->ac.u.gfx6.db_z_info2); + radeon_emit(ds->ac.u.gfx6.db_stencil_info2); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, db_htile_data_base); + radeon_set_context_reg(R_028014_DB_HTILE_DATA_BASE, db_htile_data_base); - radeon_set_context_reg_seq(cmd_buffer->cs, R_02803C_DB_DEPTH_INFO, 9); - radeon_emit(cmd_buffer->cs, ds->ac.u.gfx6.db_depth_info); /* R_02803C_DB_DEPTH_INFO */ - radeon_emit(cmd_buffer->cs, db_z_info); /* R_028040_DB_Z_INFO */ - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_info); /* R_028044_DB_STENCIL_INFO */ - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_base); /* R_028048_DB_Z_READ_BASE */ - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_base); /* R_02804C_DB_STENCIL_READ_BASE */ - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_base); /* R_028050_DB_Z_WRITE_BASE */ - radeon_emit(cmd_buffer->cs, ds->ac.db_stencil_base); /* R_028054_DB_STENCIL_WRITE_BASE */ - radeon_emit(cmd_buffer->cs, ds->ac.db_depth_size); /* R_028058_DB_DEPTH_SIZE */ - radeon_emit(cmd_buffer->cs, ds->ac.u.gfx6.db_depth_slice); /* R_02805C_DB_DEPTH_SLICE */ + radeon_set_context_reg_seq(R_02803C_DB_DEPTH_INFO, 9); + radeon_emit(ds->ac.u.gfx6.db_depth_info); /* R_02803C_DB_DEPTH_INFO */ + radeon_emit(db_z_info); /* R_028040_DB_Z_INFO */ + radeon_emit(ds->ac.db_stencil_info); /* R_028044_DB_STENCIL_INFO */ + radeon_emit(ds->ac.db_depth_base); /* R_028048_DB_Z_READ_BASE */ + radeon_emit(ds->ac.db_stencil_base); /* R_02804C_DB_STENCIL_READ_BASE */ + radeon_emit(ds->ac.db_depth_base); /* R_028050_DB_Z_WRITE_BASE */ + radeon_emit(ds->ac.db_stencil_base); /* R_028054_DB_STENCIL_WRITE_BASE */ + radeon_emit(ds->ac.db_depth_size); /* R_028058_DB_DEPTH_SIZE */ + radeon_emit(ds->ac.u.gfx6.db_depth_slice); /* R_02805C_DB_DEPTH_SLICE */ } + radeon_end(); + /* Update the ZRANGE_PRECISION value for the TC-compat bug. */ radv_update_zrange_precision(cmd_buffer, ds, iview, true); } @@ -4038,18 +4168,20 @@ radv_emit_null_ds_state(struct radv_cmd_buffer *cmd_buffer) const struct radv_physical_device *pdev = radv_device_physical(device); const enum amd_gfx_level gfx_level = pdev->info.gfx_level; - if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028018_DB_Z_INFO, 2); - radeon_emit(cmd_buffer->cs, S_028018_FORMAT(V_028018_Z_INVALID) | S_028018_NUM_SAMPLES(3)); - radeon_emit(cmd_buffer->cs, S_02801C_FORMAT(V_02801C_STENCIL_INVALID) | S_02801C_TILE_STENCIL_DISABLE(1)); + radeon_begin(cmd_buffer->cs); - radeon_set_context_reg(cmd_buffer->cs, R_028B94_PA_SC_HIZ_INFO, S_028B94_SURFACE_ENABLE(0)); - radeon_set_context_reg(cmd_buffer->cs, R_028B98_PA_SC_HIS_INFO, S_028B98_SURFACE_ENABLE(0)); + if (pdev->info.gfx_level >= GFX12) { + radeon_set_context_reg_seq(R_028018_DB_Z_INFO, 2); + radeon_emit(S_028018_FORMAT(V_028018_Z_INVALID) | S_028018_NUM_SAMPLES(3)); + radeon_emit(S_02801C_FORMAT(V_02801C_STENCIL_INVALID) | S_02801C_TILE_STENCIL_DISABLE(1)); + + radeon_set_context_reg(R_028B94_PA_SC_HIZ_INFO, S_028B94_SURFACE_ENABLE(0)); + radeon_set_context_reg(R_028B98_PA_SC_HIS_INFO, S_028B98_SURFACE_ENABLE(0)); } else { if (gfx_level == GFX9) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028038_DB_Z_INFO, 2); + radeon_set_context_reg_seq(R_028038_DB_Z_INFO, 2); } else { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028040_DB_Z_INFO, 2); + radeon_set_context_reg_seq(R_028040_DB_Z_INFO, 2); } /* On GFX11+, the hw intentionally looks at DB_Z_INFO.NUM_SAMPLES when there is no bound @@ -4057,19 +4189,18 @@ radv_emit_null_ds_state(struct radv_cmd_buffer *cmd_buffer) * PA_SC_AA_CONFIG.MSAA_EXPOSED_SAMPLES). Use 8x for DB_Z_INFO.NUM_SAMPLES to make sure it's not * the constraining factor. This affects VRS, occlusion queries and POPS. */ - radeon_emit(cmd_buffer->cs, - S_028040_FORMAT(V_028040_Z_INVALID) | S_028040_NUM_SAMPLES(pdev->info.gfx_level >= GFX11 ? 3 : 0)); - radeon_emit(cmd_buffer->cs, S_028044_FORMAT(V_028044_STENCIL_INVALID)); + radeon_emit(S_028040_FORMAT(V_028040_Z_INVALID) | S_028040_NUM_SAMPLES(pdev->info.gfx_level >= GFX11 ? 3 : 0)); + radeon_emit(S_028044_FORMAT(V_028044_STENCIL_INVALID)); uint32_t db_render_control = 0; if (gfx_level == GFX11 || gfx_level == GFX11_5) radv_gfx11_set_db_render_control(device, 1, &db_render_control); - radeon_set_context_reg(cmd_buffer->cs, R_028000_DB_RENDER_CONTROL, db_render_control); + radeon_set_context_reg(R_028000_DB_RENDER_CONTROL, db_render_control); } - radeon_set_context_reg(cmd_buffer->cs, R_028010_DB_RENDER_OVERRIDE2, - S_028010_CENTROID_COMPUTATION_MODE(gfx_level >= GFX10_3)); + radeon_set_context_reg(R_028010_DB_RENDER_OVERRIDE2, S_028010_CENTROID_COMPUTATION_MODE(gfx_level >= GFX10_3)); + radeon_end(); } /** * Update the fast clear depth/stencil values if the image is bound as a @@ -4085,17 +4216,21 @@ radv_update_bound_fast_clear_ds(struct radv_cmd_buffer *cmd_buffer, const struct if (cmd_buffer->state.render.ds_att.iview == NULL || cmd_buffer->state.render.ds_att.iview->image != image) return; + radeon_begin(cs); + if (aspects == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) { - radeon_set_context_reg_seq(cs, R_028028_DB_STENCIL_CLEAR, 2); - radeon_emit(cs, ds_clear_value.stencil); - radeon_emit(cs, fui(ds_clear_value.depth)); + radeon_set_context_reg_seq(R_028028_DB_STENCIL_CLEAR, 2); + radeon_emit(ds_clear_value.stencil); + radeon_emit(fui(ds_clear_value.depth)); } else if (aspects == VK_IMAGE_ASPECT_DEPTH_BIT) { - radeon_set_context_reg(cs, R_02802C_DB_DEPTH_CLEAR, fui(ds_clear_value.depth)); + radeon_set_context_reg(R_02802C_DB_DEPTH_CLEAR, fui(ds_clear_value.depth)); } else { assert(aspects == VK_IMAGE_ASPECT_STENCIL_BIT); - radeon_set_context_reg(cs, R_028028_DB_STENCIL_CLEAR, ds_clear_value.stencil); + radeon_set_context_reg(R_028028_DB_STENCIL_CLEAR, ds_clear_value.stencil); } + radeon_end(); + /* Update the ZRANGE_PRECISION value for the TC-compat bug. This is * only needed when clearing Z to 0.0. */ @@ -4125,11 +4260,14 @@ radv_set_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image ASSERTED unsigned cdw_end = radv_cs_write_data_head(device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, va, 2 * level_count, cmd_buffer->state.predicating); + radeon_begin(cs); + for (uint32_t l = 0; l < level_count; l++) { - radeon_emit(cs, ds_clear_value.stencil); - radeon_emit(cs, fui(ds_clear_value.depth)); + radeon_emit(ds_clear_value.stencil); + radeon_emit(fui(ds_clear_value.depth)); } + radeon_end(); assert(cmd_buffer->cs->cdw == cdw_end); } else { /* Otherwise we need one WRITE_DATA packet per level. */ @@ -4170,9 +4308,12 @@ radv_set_tc_compat_zrange_metadata(struct radv_cmd_buffer *cmd_buffer, struct ra ASSERTED unsigned cdw_end = radv_cs_write_data_head(device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, va, level_count, cmd_buffer->state.predicating); - for (uint32_t l = 0; l < level_count; l++) - radeon_emit(cs, value); + radeon_begin(cs); + for (uint32_t l = 0; l < level_count; l++) + radeon_emit(value); + + radeon_end(); assert(cmd_buffer->cs->cdw == cdw_end); } @@ -4239,24 +4380,28 @@ radv_load_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, const struct rad uint32_t reg = R_028028_DB_STENCIL_CLEAR + 4 * reg_offset; - if (pdev->info.has_load_ctx_reg_pkt) { - radeon_emit(cs, PKT3(PKT3_LOAD_CONTEXT_REG_INDEX, 3, 0)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, (reg - SI_CONTEXT_REG_OFFSET) >> 2); - radeon_emit(cs, reg_count); - } else { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | - (reg_count == 2 ? COPY_DATA_COUNT_SEL : 0)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, reg >> 2); - radeon_emit(cs, 0); + radeon_begin(cs); - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); - radeon_emit(cs, 0); + if (pdev->info.has_load_ctx_reg_pkt) { + radeon_emit(PKT3(PKT3_LOAD_CONTEXT_REG_INDEX, 3, 0)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit((reg - SI_CONTEXT_REG_OFFSET) >> 2); + radeon_emit(reg_count); + } else { + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | + (reg_count == 2 ? COPY_DATA_COUNT_SEL : 0)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(reg >> 2); + radeon_emit(0); + + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0)); + radeon_emit(0); } + + radeon_end(); } /* @@ -4280,11 +4425,14 @@ radv_update_fce_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image * ASSERTED unsigned cdw_end = radv_cs_write_data_head(device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, va, 2 * level_count, false); + radeon_begin(cmd_buffer->cs); + for (uint32_t l = 0; l < level_count; l++) { - radeon_emit(cmd_buffer->cs, pred_val); - radeon_emit(cmd_buffer->cs, pred_val >> 32); + radeon_emit(pred_val); + radeon_emit(pred_val >> 32); } + radeon_end(); assert(cmd_buffer->cs->cdw == cdw_end); } @@ -4309,11 +4457,14 @@ radv_update_dcc_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image * ASSERTED unsigned cdw_end = radv_cs_write_data_head(device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, va, 2 * level_count, false); + radeon_begin(cmd_buffer->cs); + for (uint32_t l = 0; l < level_count; l++) { - radeon_emit(cmd_buffer->cs, pred_val); - radeon_emit(cmd_buffer->cs, pred_val >> 32); + radeon_emit(pred_val); + radeon_emit(pred_val >> 32); } + radeon_end(); assert(cmd_buffer->cs->cdw == cdw_end); } @@ -4333,9 +4484,11 @@ radv_update_bound_fast_clear_color(struct radv_cmd_buffer *cmd_buffer, struct ra ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, 4); - radeon_set_context_reg_seq(cs, R_028C8C_CB_COLOR0_CLEAR_WORD0 + cb_idx * 0x3c, 2); - radeon_emit(cs, color_values[0]); - radeon_emit(cs, color_values[1]); + radeon_begin(cs); + radeon_set_context_reg_seq(R_028C8C_CB_COLOR0_CLEAR_WORD0 + cb_idx * 0x3c, 2); + radeon_emit(color_values[0]); + radeon_emit(color_values[1]); + radeon_end(); assert(cmd_buffer->cs->cdw <= cdw_max); @@ -4361,11 +4514,14 @@ radv_set_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_im ASSERTED unsigned cdw_end = radv_cs_write_data_head(device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, va, 2 * level_count, cmd_buffer->state.predicating); + radeon_begin(cs); + for (uint32_t l = 0; l < level_count; l++) { - radeon_emit(cs, color_values[0]); - radeon_emit(cs, color_values[1]); + radeon_emit(color_values[0]); + radeon_emit(color_values[1]); } + radeon_end(); assert(cmd_buffer->cs->cdw == cdw_end); } else { /* Some default value we can set in the update. */ @@ -4422,23 +4578,27 @@ radv_load_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_i uint64_t va = radv_image_get_fast_clear_va(image, iview->vk.base_mip_level); uint32_t reg = R_028C8C_CB_COLOR0_CLEAR_WORD0 + cb_idx * 0x3c; - if (pdev->info.has_load_ctx_reg_pkt) { - radeon_emit(cs, PKT3(PKT3_LOAD_CONTEXT_REG_INDEX, 3, cmd_buffer->state.predicating)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, (reg - SI_CONTEXT_REG_OFFSET) >> 2); - radeon_emit(cs, 2); - } else { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, cmd_buffer->state.predicating)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | COPY_DATA_COUNT_SEL); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, reg >> 2); - radeon_emit(cs, 0); + radeon_begin(cs); - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); - radeon_emit(cs, 0); + if (pdev->info.has_load_ctx_reg_pkt) { + radeon_emit(PKT3(PKT3_LOAD_CONTEXT_REG_INDEX, 3, cmd_buffer->state.predicating)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit((reg - SI_CONTEXT_REG_OFFSET) >> 2); + radeon_emit(2); + } else { + radeon_emit(PKT3(PKT3_COPY_DATA, 4, cmd_buffer->state.predicating)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | COPY_DATA_COUNT_SEL); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(reg >> 2); + radeon_emit(0); + + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); + radeon_emit(0); } + + radeon_end(); } /* GFX9+ metadata cache flushing workaround. metadata cache coherency is @@ -4540,11 +4700,13 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) for (i = 0; i < render->color_att_count; ++i) { struct radv_image_view *iview = render->color_att[i].iview; if (!iview) { + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028EC0_CB_COLOR0_INFO + i * 4, color_invalid); + radeon_set_context_reg(R_028EC0_CB_COLOR0_INFO + i * 4, color_invalid); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028C70_CB_COLOR0_INFO + i * 0x3C, color_invalid); + radeon_set_context_reg(R_028C70_CB_COLOR0_INFO + i * 0x3C, color_invalid); } + radeon_end(); continue; } @@ -4579,11 +4741,13 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) extent.height = MIN2(extent.height, iview->vk.extent.height); } for (; i < cmd_buffer->state.last_subpass_color_count; i++) { + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028EC0_CB_COLOR0_INFO + i * 4, color_invalid); + radeon_set_context_reg(R_028EC0_CB_COLOR0_INFO + i * 4, color_invalid); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028C70_CB_COLOR0_INFO + i * 0x3C, color_invalid); + radeon_set_context_reg(R_028C70_CB_COLOR0_INFO + i * 0x3C, color_invalid); } + radeon_end(); } cmd_buffer->state.last_subpass_color_count = render->color_att_count; @@ -4649,6 +4813,8 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) radv_emit_null_ds_state(cmd_buffer); } + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX11) { bool vrs_surface_enable = render->vrs_att.iview != NULL; unsigned xmax = 0, ymax = 0; @@ -4667,18 +4833,17 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) ymax = vrs_iview->vk.extent.height - 1; if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_0283E0_PA_SC_VRS_INFO, + radeon_set_context_reg(R_0283E0_PA_SC_VRS_INFO, S_0283E0_RATE_SW_MODE(vrs_image->planes[0].surface.u.gfx9.swizzle_mode)); } } - radeon_set_context_reg_seq(cmd_buffer->cs, R_0283F0_PA_SC_VRS_RATE_BASE, 3); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, S_0283F4_BASE_256B(va >> 40)); - radeon_emit(cmd_buffer->cs, S_0283F8_X_MAX(xmax) | S_0283F8_Y_MAX(ymax)); + radeon_set_context_reg_seq(R_0283F0_PA_SC_VRS_RATE_BASE, 3); + radeon_emit(va >> 8); + radeon_emit(S_0283F4_BASE_256B(va >> 40)); + radeon_emit(S_0283F8_X_MAX(xmax) | S_0283F8_Y_MAX(ymax)); - radeon_set_context_reg(cmd_buffer->cs, R_0283D0_PA_SC_VRS_OVERRIDE_CNTL, - S_0283D0_VRS_SURFACE_ENABLE(vrs_surface_enable)); + radeon_set_context_reg(R_0283D0_PA_SC_VRS_OVERRIDE_CNTL, S_0283D0_VRS_SURFACE_ENABLE(vrs_surface_enable)); } if (pdev->info.gfx_level >= GFX8 && pdev->info.gfx_level < GFX12) { @@ -4688,12 +4853,12 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) if (pdev->info.gfx_level >= GFX11) { const bool has_dedicated_vram = pdev->info.has_dedicated_vram; - radeon_set_context_reg(cmd_buffer->cs, R_028424_CB_FDCC_CONTROL, + radeon_set_context_reg(R_028424_CB_FDCC_CONTROL, S_028424_SAMPLE_MASK_TRACKER_WATERMARK(has_dedicated_vram ? 0 : 15)); } else { uint8_t watermark = gfx_level >= GFX10 ? 6 : 4; - radeon_set_context_reg(cmd_buffer->cs, R_028424_CB_DCC_CONTROL, + radeon_set_context_reg(R_028424_CB_DCC_CONTROL, S_028424_OVERWRITE_COMBINER_MRT_SHARING_DISABLE(gfx_level <= GFX9) | S_028424_OVERWRITE_COMBINER_WATERMARK(watermark) | S_028424_DISABLE_CONSTANT_ENCODE_AC01(disable_constant_encode_ac01) | @@ -4702,13 +4867,14 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) } if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028184_PA_SC_SCREEN_SCISSOR_BR, + radeon_set_context_reg(R_028184_PA_SC_SCREEN_SCISSOR_BR, S_028034_BR_X(extent.width) | S_028034_BR_Y(extent.height)); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028034_PA_SC_SCREEN_SCISSOR_BR, + radeon_set_context_reg(R_028034_PA_SC_SCREEN_SCISSOR_BR, S_028034_BR_X(extent.width) | S_028034_BR_Y(extent.height)); } + radeon_end(); assert(cmd_buffer->cs->cdw <= cdw_max); cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_FRAMEBUFFER; @@ -4767,15 +4933,17 @@ radv_emit_guardband_state(struct radv_cmd_buffer *cmd_buffer) } } + radeon_begin(cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg_seq(cs, R_02842C_PA_CL_GB_VERT_CLIP_ADJ, 4); + radeon_set_context_reg_seq(R_02842C_PA_CL_GB_VERT_CLIP_ADJ, 4); } else { - radeon_set_context_reg_seq(cs, R_028BE8_PA_CL_GB_VERT_CLIP_ADJ, 4); + radeon_set_context_reg_seq(R_028BE8_PA_CL_GB_VERT_CLIP_ADJ, 4); } - radeon_emit(cs, fui(guardband_y)); - radeon_emit(cs, fui(discard_y)); - radeon_emit(cs, fui(guardband_x)); - radeon_emit(cs, fui(discard_x)); + radeon_emit(fui(guardband_y)); + radeon_emit(fui(discard_y)); + radeon_emit(fui(guardband_x)); + radeon_emit(fui(discard_x)); + radeon_end(); cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_GUARDBAND; } @@ -4818,12 +4986,14 @@ radv_emit_index_buffer(struct radv_cmd_buffer *cmd_buffer) radv_handle_zero_index_buffer_bug(cmd_buffer, &index_va, &max_index_count); } - radeon_emit(cs, PKT3(PKT3_INDEX_BASE, 1, 0)); - radeon_emit(cs, index_va); - radeon_emit(cs, index_va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_INDEX_BASE, 1, 0)); + radeon_emit(index_va); + radeon_emit(index_va >> 32); - radeon_emit(cs, PKT3(PKT3_INDEX_BUFFER_SIZE, 0, 0)); - radeon_emit(cs, max_index_count); + radeon_emit(PKT3(PKT3_INDEX_BUFFER_SIZE, 0, 0)); + radeon_emit(max_index_count); + radeon_end(); cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_INDEX_BUFFER; } @@ -4864,6 +5034,8 @@ radv_flush_occlusion_query_state(struct radv_cmd_buffer *cmd_buffer) } } + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { radeon_opt_set_context_reg(cmd_buffer, R_028060_DB_COUNT_CONTROL, RADV_TRACKED_DB_COUNT_CONTROL, db_count_control); @@ -4872,6 +5044,8 @@ radv_flush_occlusion_query_state(struct radv_cmd_buffer *cmd_buffer) db_count_control); } + radeon_end(); + cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_OCCLUSION_QUERY; } @@ -5028,8 +5202,10 @@ emit_prolog_regs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *v if (G_00B848_VGPRS(prolog->rsrc1) > G_00B848_VGPRS(rsrc1)) rsrc1 = (rsrc1 & C_00B848_VGPRS) | (prolog->rsrc1 & ~C_00B848_VGPRS); - radeon_set_sh_reg(cmd_buffer->cs, vs_shader->info.regs.pgm_lo, prolog->va >> 8); - radeon_set_sh_reg(cmd_buffer->cs, vs_shader->info.regs.pgm_rsrc1, rsrc1); + radeon_begin(cmd_buffer->cs); + + radeon_set_sh_reg(vs_shader->info.regs.pgm_lo, prolog->va >> 8); + radeon_set_sh_reg(vs_shader->info.regs.pgm_rsrc1, rsrc1); if (vs_shader->info.merged_shader_compiled_separately) { if (vs_shader->info.next_stage == MESA_SHADER_GEOMETRY) { @@ -5042,12 +5218,14 @@ emit_prolog_regs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *v lds_size = gs->info.gs_ring_info.lds_size; } - radeon_set_sh_reg(cmd_buffer->cs, vs_shader->info.regs.pgm_rsrc2, rsrc2 | S_00B22C_LDS_SIZE(lds_size)); + radeon_set_sh_reg(vs_shader->info.regs.pgm_rsrc2, rsrc2 | S_00B22C_LDS_SIZE(lds_size)); } else { - radeon_set_sh_reg(cmd_buffer->cs, vs_shader->info.regs.pgm_rsrc2, rsrc2); + radeon_set_sh_reg(vs_shader->info.regs.pgm_rsrc2, rsrc2); } } + radeon_end(); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, prolog->bo); } @@ -5092,7 +5270,9 @@ emit_prolog_inputs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader } const uint32_t vs_prolog_inputs_offset = radv_get_user_sgpr_loc(vs_shader, AC_UD_VS_PROLOG_INPUTS); - radeon_emit_64bit_pointer(cmd_buffer->cs, vs_prolog_inputs_offset, input_va); + radeon_begin(cmd_buffer->cs); + radeon_emit_64bit_pointer(vs_prolog_inputs_offset, input_va); + radeon_end(); } static void @@ -5178,13 +5358,15 @@ radv_emit_tess_domain_origin(struct radv_cmd_buffer *cmd_buffer) uint32_t vgt_tf_param = S_028B6C_TYPE(type) | S_028B6C_PARTITIONING(partitioning) | S_028B6C_TOPOLOGY(topology) | S_028B6C_DISTRIBUTION_MODE(pdev->tess_distribution_mode); + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { vgt_tf_param |= S_028AA4_TEMPORAL(gfx12_load_last_use_discard); - radeon_set_context_reg(cmd_buffer->cs, R_028AA4_VGT_TF_PARAM, vgt_tf_param); + radeon_set_context_reg(R_028AA4_VGT_TF_PARAM, vgt_tf_param); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028B6C_VGT_TF_PARAM, vgt_tf_param); + radeon_set_context_reg(R_028B6C_VGT_TF_PARAM, vgt_tf_param); } + radeon_end(); } static void @@ -5208,11 +5390,13 @@ radv_emit_alpha_to_coverage_enable(struct radv_cmd_buffer *cmd_buffer) db_alpha_to_mask |= S_028B70_ALPHA_TO_MASK_ENABLE(d->vk.ms.alpha_to_coverage_enable); + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_02807C_DB_ALPHA_TO_MASK, db_alpha_to_mask); + radeon_set_context_reg(R_02807C_DB_ALPHA_TO_MASK, db_alpha_to_mask); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028B70_DB_ALPHA_TO_MASK, db_alpha_to_mask); + radeon_set_context_reg(R_028B70_DB_ALPHA_TO_MASK, db_alpha_to_mask); } + radeon_end(); } static void @@ -5220,9 +5404,11 @@ radv_emit_sample_mask(struct radv_cmd_buffer *cmd_buffer) { const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C38_PA_SC_AA_MASK_X0Y0_X1Y0, 2); - radeon_emit(cmd_buffer->cs, d->vk.ms.sample_mask | ((uint32_t)d->vk.ms.sample_mask << 16)); - radeon_emit(cmd_buffer->cs, d->vk.ms.sample_mask | ((uint32_t)d->vk.ms.sample_mask << 16)); + radeon_begin(cmd_buffer->cs); + radeon_set_context_reg_seq(R_028C38_PA_SC_AA_MASK_X0Y0_X1Y0, 2); + radeon_emit(d->vk.ms.sample_mask | ((uint32_t)d->vk.ms.sample_mask << 16)); + radeon_emit(d->vk.ms.sample_mask | ((uint32_t)d->vk.ms.sample_mask << 16)); + radeon_end(); } static void @@ -5333,13 +5519,15 @@ radv_emit_color_blend(struct radv_cmd_buffer *cmd_buffer) } } - radeon_set_context_reg_seq(cmd_buffer->cs, R_028780_CB_BLEND0_CONTROL, MAX_RTS); - radeon_emit_array(cmd_buffer->cs, cb_blend_control, MAX_RTS); + radeon_begin(cmd_buffer->cs); + radeon_set_context_reg_seq(R_028780_CB_BLEND0_CONTROL, MAX_RTS); + radeon_emit_array(cb_blend_control, MAX_RTS); if (pdev->info.has_rbplus) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028760_SX_MRT0_BLEND_OPT, MAX_RTS); - radeon_emit_array(cmd_buffer->cs, sx_mrt_blend_opt, MAX_RTS); + radeon_set_context_reg_seq(R_028760_SX_MRT0_BLEND_OPT, MAX_RTS); + radeon_emit_array(sx_mrt_blend_opt, MAX_RTS); } + radeon_end(); } static struct radv_shader_part * @@ -5505,20 +5693,22 @@ radv_emit_msaa_state(struct radv_cmd_buffer *cmd_buffer) pa_sc_aa_config |= S_028BE0_COVERAGE_TO_SHADER_SELECT(pdev->info.gfx_level < GFX12 && ps && ps->info.ps.reads_fully_covered); - if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028C5C_PA_SC_SAMPLE_PROPERTIES, - S_028C5C_MAX_SAMPLE_DIST(max_sample_dist)); + radeon_begin(cmd_buffer->cs); - radeon_set_context_reg(cmd_buffer->cs, R_028078_DB_EQAA, db_eqaa); + if (pdev->info.gfx_level >= GFX12) { + radeon_set_context_reg(R_028C5C_PA_SC_SAMPLE_PROPERTIES, S_028C5C_MAX_SAMPLE_DIST(max_sample_dist)); + + radeon_set_context_reg(R_028078_DB_EQAA, db_eqaa); } else { - radeon_set_context_reg(cmd_buffer->cs, R_028804_DB_EQAA, db_eqaa); + radeon_set_context_reg(R_028804_DB_EQAA, db_eqaa); } - radeon_set_context_reg(cmd_buffer->cs, R_028BE0_PA_SC_AA_CONFIG, pa_sc_aa_config); - radeon_set_context_reg( - cmd_buffer->cs, R_028A48_PA_SC_MODE_CNTL_0, - S_028A48_ALTERNATE_RBS_PER_TILE(pdev->info.gfx_level >= GFX9) | S_028A48_VPORT_SCISSOR_ENABLE(1) | - S_028A48_LINE_STIPPLE_ENABLE(d->vk.rs.line.stipple.enable) | S_028A48_MSAA_ENABLE(rasterization_samples > 1)); + radeon_set_context_reg(R_028BE0_PA_SC_AA_CONFIG, pa_sc_aa_config); + radeon_set_context_reg(R_028A48_PA_SC_MODE_CNTL_0, S_028A48_ALTERNATE_RBS_PER_TILE(pdev->info.gfx_level >= GFX9) | + S_028A48_VPORT_SCISSOR_ENABLE(1) | + S_028A48_LINE_STIPPLE_ENABLE(d->vk.rs.line.stipple.enable) | + S_028A48_MSAA_ENABLE(rasterization_samples > 1)); + radeon_end(); } static void @@ -6032,13 +6222,16 @@ radv_emit_streamout_buffers(struct radv_cmd_buffer *cmd_buffer, uint64_t va) if (!streamout_buffers_offset) return; - radeon_emit_32bit_pointer(cmd_buffer->cs, streamout_buffers_offset, va, &pdev->info); + radeon_begin(cmd_buffer->cs); + radeon_emit_32bit_pointer(streamout_buffers_offset, va, &pdev->info); if (cmd_buffer->state.gs_copy_shader) { streamout_buffers_offset = radv_get_user_sgpr_loc(cmd_buffer->state.gs_copy_shader, AC_UD_STREAMOUT_BUFFERS); if (streamout_buffers_offset) - radeon_emit_32bit_pointer(cmd_buffer->cs, streamout_buffers_offset, va, &pdev->info); + radeon_emit_32bit_pointer(streamout_buffers_offset, va, &pdev->info); } + + radeon_end(); } static void @@ -6055,7 +6248,9 @@ radv_emit_streamout_state(struct radv_cmd_buffer *cmd_buffer) if (!streamout_state_offset) return; - radeon_emit_32bit_pointer(cmd_buffer->cs, streamout_state_offset, so->state_va, &pdev->info); + radeon_begin(cmd_buffer->cs); + radeon_emit_32bit_pointer(streamout_state_offset, so->state_va, &pdev->info); + radeon_end(); } static void @@ -6153,7 +6348,9 @@ radv_flush_force_vrs_state(struct radv_cmd_buffer *cmd_buffer) if (cmd_buffer->state.last_vrs_rates != vrs_rates || cmd_buffer->state.last_force_vrs_rates_offset != force_vrs_rates_offset) { - radeon_set_sh_reg(cmd_buffer->cs, force_vrs_rates_offset, vrs_rates); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(force_vrs_rates_offset, vrs_rates); + radeon_end(); } cmd_buffer->state.last_vrs_rates = vrs_rates; @@ -6401,13 +6598,18 @@ radv_emit_ia_multi_vgt_param(struct radv_cmd_buffer *cmd_buffer, bool instanced_ patch_control_points, state->tess_num_patches); if (state->last_ia_multi_vgt_param != ia_multi_vgt_param) { + radeon_begin(cs); + if (gpu_info->gfx_level == GFX9) { - radeon_set_uconfig_reg_idx(&pdev->info, cs, R_030960_IA_MULTI_VGT_PARAM, 4, ia_multi_vgt_param); + radeon_set_uconfig_reg_idx(&pdev->info, R_030960_IA_MULTI_VGT_PARAM, 4, ia_multi_vgt_param); } else if (gpu_info->gfx_level >= GFX7) { - radeon_set_context_reg_idx(cs, R_028AA8_IA_MULTI_VGT_PARAM, 1, ia_multi_vgt_param); + radeon_set_context_reg_idx(R_028AA8_IA_MULTI_VGT_PARAM, 1, ia_multi_vgt_param); } else { - radeon_set_context_reg(cs, R_028AA8_IA_MULTI_VGT_PARAM, ia_multi_vgt_param); + radeon_set_context_reg(R_028AA8_IA_MULTI_VGT_PARAM, ia_multi_vgt_param); } + + radeon_end(); + state->last_ia_multi_vgt_param = ia_multi_vgt_param; } } @@ -6446,7 +6648,10 @@ gfx10_emit_ge_cntl(struct radv_cmd_buffer *cmd_buffer) S_03096C_BREAK_WAVE_AT_EOI(break_wave_at_eoi); if (state->last_ge_cntl != ge_cntl) { - radeon_set_uconfig_reg(cmd_buffer->cs, R_03096C_GE_CNTL, ge_cntl); + radeon_begin(cmd_buffer->cs); + radeon_set_uconfig_reg(R_03096C_GE_CNTL, ge_cntl); + radeon_end(); + state->last_ge_cntl = ge_cntl; } } @@ -6487,13 +6692,17 @@ radv_emit_draw_registers(struct radv_cmd_buffer *cmd_buffer, const struct radv_d disable_instance_packing != G_028A7C_DISABLE_INSTANCE_PACKING(state->last_index_type)))) { uint32_t index_type = state->index_type | S_028A7C_DISABLE_INSTANCE_PACKING(disable_instance_packing); + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX9) { - radeon_set_uconfig_reg_idx(&pdev->info, cs, R_03090C_VGT_INDEX_TYPE, 2, index_type); + radeon_set_uconfig_reg_idx(&pdev->info, R_03090C_VGT_INDEX_TYPE, 2, index_type); } else { - radeon_emit(cs, PKT3(PKT3_INDEX_TYPE, 0, 0)); - radeon_emit(cs, index_type); + radeon_emit(PKT3(PKT3_INDEX_TYPE, 0, 0)); + radeon_emit(index_type); } + radeon_end(); + state->last_index_type = index_type; } } @@ -7390,15 +7599,20 @@ radv_emit_compute_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_compu radv_get_user_sgpr_loc(rt_prolog, AC_UD_CS_RAY_DYNAMIC_CALLABLE_STACK_BASE); if (ray_dynamic_callback_stack_base_offset) { const struct radv_shader_info *cs_info = &rt_prolog->info; - radeon_set_sh_reg(cmd_buffer->cs, ray_dynamic_callback_stack_base_offset, + + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(ray_dynamic_callback_stack_base_offset, rt_prolog->config.scratch_bytes_per_wave / cs_info->wave_size); + radeon_end(); } const uint32_t traversal_shader_addr_offset = radv_get_user_sgpr_loc(rt_prolog, AC_UD_CS_TRAVERSAL_SHADER_ADDR); struct radv_shader *traversal_shader = cmd_buffer->state.shaders[MESA_SHADER_INTERSECTION]; if (traversal_shader_addr_offset && traversal_shader) { uint64_t traversal_va = traversal_shader->va | radv_rt_priority_traversal; - radeon_emit_64bit_pointer(cmd_buffer->cs, traversal_shader_addr_offset, traversal_va); + radeon_begin(cmd_buffer->cs); + radeon_emit_64bit_pointer(traversal_shader_addr_offset, traversal_va); + radeon_end(); } } @@ -9379,19 +9593,19 @@ radv_CmdBeginRendering(VkCommandBuffer commandBuffer, const VkRenderingInfo *pRe const uint32_t maxy = miny + render->area.extent.height; radeon_check_space(device->ws, cmd_buffer->cs, 6); + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028204_PA_SC_WINDOW_SCISSOR_TL, - S_028204_TL_X(minx) | S_028204_TL_Y_GFX12(miny)); - radeon_set_context_reg(cmd_buffer->cs, R_028208_PA_SC_WINDOW_SCISSOR_BR, + radeon_set_context_reg(R_028204_PA_SC_WINDOW_SCISSOR_TL, S_028204_TL_X(minx) | S_028204_TL_Y_GFX12(miny)); + radeon_set_context_reg(R_028208_PA_SC_WINDOW_SCISSOR_BR, S_028208_BR_X(maxx - 1) | S_028208_BR_Y(maxy - 1)); /* inclusive */ } else { - radeon_set_context_reg(cmd_buffer->cs, R_028204_PA_SC_WINDOW_SCISSOR_TL, - S_028204_TL_X(minx) | S_028204_TL_Y_GFX6(miny)); - radeon_set_context_reg(cmd_buffer->cs, R_028208_PA_SC_WINDOW_SCISSOR_BR, - S_028208_BR_X(maxx) | S_028208_BR_Y(maxy)); + radeon_set_context_reg(R_028204_PA_SC_WINDOW_SCISSOR_TL, S_028204_TL_X(minx) | S_028204_TL_Y_GFX6(miny)); + radeon_set_context_reg(R_028208_PA_SC_WINDOW_SCISSOR_BR, S_028208_BR_X(maxx) | S_028208_BR_Y(maxy)); } + radeon_end(); + radv_emit_fb_mip_change_flush(cmd_buffer); if (!(pRenderingInfo->flags & VK_RENDERING_RESUMING_BIT)) @@ -9417,7 +9631,9 @@ radv_emit_view_index_per_stage(struct radeon_cmdbuf *cs, const struct radv_shade if (!view_index_offset) return; - radeon_set_sh_reg(cs, view_index_offset, index); + radeon_begin(cs); + radeon_set_sh_reg(view_index_offset, index); + radeon_end(); } static void @@ -9439,13 +9655,15 @@ static void radv_emit_copy_data_imm(const struct radv_physical_device *pdev, struct radeon_cmdbuf *cs, uint32_t src_imm, uint64_t dst_va) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM | - (pdev->info.gfx_level == GFX6 ? COPY_DATA_ENGINE_PFP : 0)); - radeon_emit(cs, src_imm); - radeon_emit(cs, 0); - radeon_emit(cs, dst_va); - radeon_emit(cs, dst_va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM | + (pdev->info.gfx_level == GFX6 ? COPY_DATA_ENGINE_PFP : 0)); + radeon_emit(src_imm); + radeon_emit(0); + radeon_emit(dst_va); + radeon_emit(dst_va >> 32); + radeon_end(); } /** @@ -9491,9 +9709,11 @@ radv_cs_emit_compute_predication(const struct radv_device *device, struct radv_c static void radv_cs_emit_draw_packet(struct radv_cmd_buffer *cmd_buffer, uint32_t vertex_count, uint32_t use_opaque) { - radeon_emit(cmd_buffer->cs, PKT3(PKT3_DRAW_INDEX_AUTO, 1, cmd_buffer->state.predicating)); - radeon_emit(cmd_buffer->cs, vertex_count); - radeon_emit(cmd_buffer->cs, V_0287F0_DI_SRC_SEL_AUTO_INDEX | use_opaque); + radeon_begin(cmd_buffer->cs); + radeon_emit(PKT3(PKT3_DRAW_INDEX_AUTO, 1, cmd_buffer->state.predicating)); + radeon_emit(vertex_count); + radeon_emit(V_0287F0_DI_SRC_SEL_AUTO_INDEX | use_opaque); + radeon_end(); } /** @@ -9507,16 +9727,18 @@ static void radv_cs_emit_draw_indexed_packet(struct radv_cmd_buffer *cmd_buffer, uint64_t index_va, uint32_t max_index_count, uint32_t index_count, bool not_eop) { - radeon_emit(cmd_buffer->cs, PKT3(PKT3_DRAW_INDEX_2, 4, cmd_buffer->state.predicating)); - radeon_emit(cmd_buffer->cs, max_index_count); - radeon_emit(cmd_buffer->cs, index_va); - radeon_emit(cmd_buffer->cs, index_va >> 32); - radeon_emit(cmd_buffer->cs, index_count); + radeon_begin(cmd_buffer->cs); + radeon_emit(PKT3(PKT3_DRAW_INDEX_2, 4, cmd_buffer->state.predicating)); + radeon_emit(max_index_count); + radeon_emit(index_va); + radeon_emit(index_va >> 32); + radeon_emit(index_count); /* NOT_EOP allows merging multiple draws into 1 wave, but only user VGPRs * can be changed between draws and GS fast launch must be disabled. * NOT_EOP doesn't work on gfx6-gfx9 and gfx12. */ - radeon_emit(cmd_buffer->cs, V_0287F0_DI_SRC_SEL_DMA | S_0287F0_NOT_EOP(not_eop)); + radeon_emit(V_0287F0_DI_SRC_SEL_DMA | S_0287F0_NOT_EOP(not_eop)); + radeon_end(); } /* MUST inline this function to avoid massive perf loss in drawoverhead */ @@ -9544,25 +9766,29 @@ radv_cs_emit_indirect_draw_packet(struct radv_cmd_buffer *cmd_buffer, bool index if (draw_id_enable) draw_id_reg = ((base_reg + 4) - SI_SH_REG_OFFSET) >> 2; + radeon_begin(cs); + if (draw_count == 1 && !count_va && !draw_id_enable) { - radeon_emit(cs, PKT3(indexed ? PKT3_DRAW_INDEX_INDIRECT : PKT3_DRAW_INDIRECT, 3, predicating)); - radeon_emit(cs, 0); - radeon_emit(cs, vertex_offset_reg); - radeon_emit(cs, start_instance_reg); - radeon_emit(cs, di_src_sel); + radeon_emit(PKT3(indexed ? PKT3_DRAW_INDEX_INDIRECT : PKT3_DRAW_INDIRECT, 3, predicating)); + radeon_emit(0); + radeon_emit(vertex_offset_reg); + radeon_emit(start_instance_reg); + radeon_emit(di_src_sel); } else { - radeon_emit(cs, PKT3(indexed ? PKT3_DRAW_INDEX_INDIRECT_MULTI : PKT3_DRAW_INDIRECT_MULTI, 8, predicating)); - radeon_emit(cs, 0); - radeon_emit(cs, vertex_offset_reg); - radeon_emit(cs, start_instance_reg); - radeon_emit(cs, draw_id_reg | S_2C3_DRAW_INDEX_ENABLE(draw_id_enable) | S_2C3_COUNT_INDIRECT_ENABLE(!!count_va)); - radeon_emit(cs, draw_count); /* count */ - radeon_emit(cs, count_va); /* count_addr */ - radeon_emit(cs, count_va >> 32); - radeon_emit(cs, stride); /* stride */ - radeon_emit(cs, di_src_sel); + radeon_emit(PKT3(indexed ? PKT3_DRAW_INDEX_INDIRECT_MULTI : PKT3_DRAW_INDIRECT_MULTI, 8, predicating)); + radeon_emit(0); + radeon_emit(vertex_offset_reg); + radeon_emit(start_instance_reg); + radeon_emit(draw_id_reg | S_2C3_DRAW_INDEX_ENABLE(draw_id_enable) | S_2C3_COUNT_INDIRECT_ENABLE(!!count_va)); + radeon_emit(draw_count); /* count */ + radeon_emit(count_va); /* count_addr */ + radeon_emit(count_va >> 32); + radeon_emit(stride); /* stride */ + radeon_emit(di_src_sel); } + radeon_end(); + cmd_buffer->state.uses_draw_indirect = true; } @@ -9591,19 +9817,21 @@ radv_cs_emit_indirect_mesh_draw_packet(struct radv_cmd_buffer *cmd_buffer, uint3 uint32_t mode1_enable = !pdev->mesh_fast_launch_2; - radeon_emit(cs, PKT3(PKT3_DISPATCH_MESH_INDIRECT_MULTI, 7, predicating) | PKT3_RESET_FILTER_CAM_S(1)); - radeon_emit(cs, 0); /* data_offset */ - radeon_emit(cs, S_4C1_XYZ_DIM_REG(xyz_dim_reg) | S_4C1_DRAW_INDEX_REG(draw_id_reg)); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_DISPATCH_MESH_INDIRECT_MULTI, 7, predicating) | PKT3_RESET_FILTER_CAM_S(1)); + radeon_emit(0); /* data_offset */ + radeon_emit(S_4C1_XYZ_DIM_REG(xyz_dim_reg) | S_4C1_DRAW_INDEX_REG(draw_id_reg)); if (pdev->info.gfx_level >= GFX11) - radeon_emit(cs, S_4C2_DRAW_INDEX_ENABLE(draw_id_enable) | S_4C2_COUNT_INDIRECT_ENABLE(!!count_va) | - S_4C2_XYZ_DIM_ENABLE(xyz_dim_enable) | S_4C2_MODE1_ENABLE(mode1_enable)); + radeon_emit(S_4C2_DRAW_INDEX_ENABLE(draw_id_enable) | S_4C2_COUNT_INDIRECT_ENABLE(!!count_va) | + S_4C2_XYZ_DIM_ENABLE(xyz_dim_enable) | S_4C2_MODE1_ENABLE(mode1_enable)); else - radeon_emit(cs, S_4C2_DRAW_INDEX_ENABLE(draw_id_enable) | S_4C2_COUNT_INDIRECT_ENABLE(!!count_va)); - radeon_emit(cs, draw_count); - radeon_emit(cs, count_va); - radeon_emit(cs, count_va >> 32); - radeon_emit(cs, stride); - radeon_emit(cs, V_0287F0_DI_SRC_SEL_AUTO_INDEX); + radeon_emit(S_4C2_DRAW_INDEX_ENABLE(draw_id_enable) | S_4C2_COUNT_INDIRECT_ENABLE(!!count_va)); + radeon_emit(draw_count); + radeon_emit(count_va); + radeon_emit(count_va >> 32); + radeon_emit(stride); + radeon_emit(V_0287F0_DI_SRC_SEL_AUTO_INDEX); + radeon_end(); } ALWAYS_INLINE static void @@ -9617,12 +9845,14 @@ radv_cs_emit_dispatch_taskmesh_direct_ace_packet(const struct radv_device *devic device->dispatch_initiator_task | S_00B800_CS_W32_EN(task_shader->info.wave_size == 32); const uint32_t ring_entry_reg = radv_get_user_sgpr(task_shader, AC_UD_TASK_RING_ENTRY); - radeon_emit(ace_cs, PKT3(PKT3_DISPATCH_TASKMESH_DIRECT_ACE, 4, predicating) | PKT3_SHADER_TYPE_S(1)); - radeon_emit(ace_cs, x); - radeon_emit(ace_cs, y); - radeon_emit(ace_cs, z); - radeon_emit(ace_cs, dispatch_initiator); - radeon_emit(ace_cs, ring_entry_reg & 0xFFFF); + radeon_begin(ace_cs); + radeon_emit(PKT3(PKT3_DISPATCH_TASKMESH_DIRECT_ACE, 4, predicating) | PKT3_SHADER_TYPE_S(1)); + radeon_emit(x); + radeon_emit(y); + radeon_emit(z); + radeon_emit(dispatch_initiator); + radeon_emit(ring_entry_reg & 0xFFFF); + radeon_end(); } ALWAYS_INLINE static void @@ -9642,18 +9872,20 @@ radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(const struct radv_devic const uint32_t xyz_dim_reg = radv_get_user_sgpr(task_shader, AC_UD_CS_GRID_SIZE); const uint32_t draw_id_reg = radv_get_user_sgpr(task_shader, AC_UD_CS_TASK_DRAW_ID); - radeon_emit(ace_cs, PKT3(PKT3_DISPATCH_TASKMESH_INDIRECT_MULTI_ACE, 9, 0) | PKT3_SHADER_TYPE_S(1)); - radeon_emit(ace_cs, data_va); - radeon_emit(ace_cs, data_va >> 32); - radeon_emit(ace_cs, S_AD2_RING_ENTRY_REG(ring_entry_reg)); - radeon_emit(ace_cs, S_AD3_COUNT_INDIRECT_ENABLE(!!count_va) | S_AD3_DRAW_INDEX_ENABLE(!!draw_id_reg) | - S_AD3_XYZ_DIM_ENABLE(!!xyz_dim_reg) | S_AD3_DRAW_INDEX_REG(draw_id_reg)); - radeon_emit(ace_cs, S_AD4_XYZ_DIM_REG(xyz_dim_reg)); - radeon_emit(ace_cs, draw_count); - radeon_emit(ace_cs, count_va); - radeon_emit(ace_cs, count_va >> 32); - radeon_emit(ace_cs, stride); - radeon_emit(ace_cs, dispatch_initiator); + radeon_begin(ace_cs); + radeon_emit(PKT3(PKT3_DISPATCH_TASKMESH_INDIRECT_MULTI_ACE, 9, 0) | PKT3_SHADER_TYPE_S(1)); + radeon_emit(data_va); + radeon_emit(data_va >> 32); + radeon_emit(S_AD2_RING_ENTRY_REG(ring_entry_reg)); + radeon_emit(S_AD3_COUNT_INDIRECT_ENABLE(!!count_va) | S_AD3_DRAW_INDEX_ENABLE(!!draw_id_reg) | + S_AD3_XYZ_DIM_ENABLE(!!xyz_dim_reg) | S_AD3_DRAW_INDEX_REG(draw_id_reg)); + radeon_emit(S_AD4_XYZ_DIM_REG(xyz_dim_reg)); + radeon_emit(draw_count); + radeon_emit(count_va); + radeon_emit(count_va >> 32); + radeon_emit(stride); + radeon_emit(dispatch_initiator); + radeon_end(); } ALWAYS_INLINE static void @@ -9672,14 +9904,16 @@ radv_cs_emit_dispatch_taskmesh_gfx_packet(const struct radv_device *device, cons uint32_t linear_dispatch_en = cmd_state->shaders[MESA_SHADER_TASK]->info.cs.linear_taskmesh_dispatch; const bool sqtt_en = !!device->sqtt.bo; - radeon_emit(cs, PKT3(PKT3_DISPATCH_TASKMESH_GFX, 2, predicating) | PKT3_RESET_FILTER_CAM_S(1)); - radeon_emit(cs, S_4D0_RING_ENTRY_REG(ring_entry_reg) | S_4D0_XYZ_DIM_REG(xyz_dim_reg)); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_DISPATCH_TASKMESH_GFX, 2, predicating) | PKT3_RESET_FILTER_CAM_S(1)); + radeon_emit(S_4D0_RING_ENTRY_REG(ring_entry_reg) | S_4D0_XYZ_DIM_REG(xyz_dim_reg)); if (pdev->info.gfx_level >= GFX11) - radeon_emit(cs, S_4D1_XYZ_DIM_ENABLE(xyz_dim_en) | S_4D1_MODE1_ENABLE(mode1_en) | - S_4D1_LINEAR_DISPATCH_ENABLE(linear_dispatch_en) | S_4D1_THREAD_TRACE_MARKER_ENABLE(sqtt_en)); + radeon_emit(S_4D1_XYZ_DIM_ENABLE(xyz_dim_en) | S_4D1_MODE1_ENABLE(mode1_en) | + S_4D1_LINEAR_DISPATCH_ENABLE(linear_dispatch_en) | S_4D1_THREAD_TRACE_MARKER_ENABLE(sqtt_en)); else - radeon_emit(cs, S_4D1_THREAD_TRACE_MARKER_ENABLE(sqtt_en)); - radeon_emit(cs, V_0287F0_DI_SRC_SEL_AUTO_INDEX); + radeon_emit(S_4D1_THREAD_TRACE_MARKER_ENABLE(sqtt_en)); + radeon_emit(V_0287F0_DI_SRC_SEL_AUTO_INDEX); + radeon_end(); } ALWAYS_INLINE static void @@ -9691,19 +9925,22 @@ radv_emit_userdata_vertex_internal(struct radv_cmd_buffer *cmd_buffer, const str const bool uses_baseinstance = state->uses_baseinstance; const bool uses_drawid = state->uses_drawid; - radeon_set_sh_reg_seq(cs, state->vtx_base_sgpr, state->vtx_emit_num); + radeon_begin(cs); + radeon_set_sh_reg_seq(state->vtx_base_sgpr, state->vtx_emit_num); - radeon_emit(cs, vertex_offset); + radeon_emit(vertex_offset); state->last_vertex_offset_valid = true; state->last_vertex_offset = vertex_offset; if (uses_drawid) { - radeon_emit(cs, 0); + radeon_emit(0); state->last_drawid = 0; } if (uses_baseinstance) { - radeon_emit(cs, info->first_instance); + radeon_emit(info->first_instance); state->last_first_instance = info->first_instance; } + + radeon_end(); } ALWAYS_INLINE static void @@ -9725,12 +9962,15 @@ radv_emit_userdata_vertex_drawid(struct radv_cmd_buffer *cmd_buffer, uint32_t ve { struct radv_cmd_state *state = &cmd_buffer->state; struct radeon_cmdbuf *cs = cmd_buffer->cs; - radeon_set_sh_reg_seq(cs, state->vtx_base_sgpr, 1 + !!drawid); - radeon_emit(cs, vertex_offset); + + radeon_begin(cs); + radeon_set_sh_reg_seq(state->vtx_base_sgpr, 1 + !!drawid); + radeon_emit(vertex_offset); state->last_vertex_offset_valid = true; state->last_vertex_offset = vertex_offset; if (drawid) - radeon_emit(cs, drawid); + radeon_emit(drawid); + radeon_end(); } ALWAYS_INLINE static void @@ -9745,16 +9985,18 @@ radv_emit_userdata_mesh(struct radv_cmd_buffer *cmd_buffer, const uint32_t x, co if (!uses_drawid && !uses_grid_size) return; - radeon_set_sh_reg_seq(cs, state->vtx_base_sgpr, state->vtx_emit_num); + radeon_begin(cs); + radeon_set_sh_reg_seq(state->vtx_base_sgpr, state->vtx_emit_num); if (uses_grid_size) { - radeon_emit(cs, x); - radeon_emit(cs, y); - radeon_emit(cs, z); + radeon_emit(x); + radeon_emit(y); + radeon_emit(z); } if (uses_drawid) { - radeon_emit(cs, 0); + radeon_emit(0); state->last_drawid = 0; } + radeon_end(); } ALWAYS_INLINE static void @@ -9766,17 +10008,21 @@ radv_emit_userdata_task(const struct radv_cmd_state *cmd_state, struct radeon_cm const uint32_t xyz_offset = radv_get_user_sgpr_loc(task_shader, AC_UD_CS_GRID_SIZE); const uint32_t draw_id_offset = radv_get_user_sgpr_loc(task_shader, AC_UD_CS_TASK_DRAW_ID); + radeon_begin(ace_cs); + if (xyz_offset) { - radeon_set_sh_reg_seq(ace_cs, xyz_offset, 3); - radeon_emit(ace_cs, x); - radeon_emit(ace_cs, y); - radeon_emit(ace_cs, z); + radeon_set_sh_reg_seq(xyz_offset, 3); + radeon_emit(x); + radeon_emit(y); + radeon_emit(z); } if (draw_id_offset) { - radeon_set_sh_reg_seq(ace_cs, draw_id_offset, 1); - radeon_emit(ace_cs, 0); + radeon_set_sh_reg_seq(draw_id_offset, 1); + radeon_emit(0); } + + radeon_end(); } ALWAYS_INLINE static void @@ -9805,8 +10051,11 @@ radv_emit_draw_packets_indexed(struct radv_cmd_buffer *cmd_buffer, const struct if (!remaining_indexes && pdev->info.has_zero_index_buffer_bug) radv_handle_zero_index_buffer_bug(cmd_buffer, &index_va, &remaining_indexes); - if (i > 0) - radeon_set_sh_reg(cs, state->vtx_base_sgpr + sizeof(uint32_t), i); + if (i > 0) { + radeon_begin(cs); + radeon_set_sh_reg(state->vtx_base_sgpr + sizeof(uint32_t), i); + radeon_end(); + } if (!state->render.view_mask) { radv_cs_emit_draw_indexed_packet(cmd_buffer, index_va, remaining_indexes, draw->indexCount, false); @@ -9829,10 +10078,13 @@ radv_emit_draw_packets_indexed(struct radv_cmd_buffer *cmd_buffer, const struct if (i > 0) { assert(state->last_vertex_offset_valid); - if (state->last_vertex_offset != draw->vertexOffset) + if (state->last_vertex_offset != draw->vertexOffset) { radv_emit_userdata_vertex_drawid(cmd_buffer, draw->vertexOffset, i); - else - radeon_set_sh_reg(cs, state->vtx_base_sgpr + sizeof(uint32_t), i); + } else { + radeon_begin(cs); + radeon_set_sh_reg(state->vtx_base_sgpr + sizeof(uint32_t), i); + radeon_end(); + } } else radv_emit_userdata_vertex(cmd_buffer, info, draw->vertexOffset); @@ -9954,11 +10206,13 @@ radv_emit_direct_draw_packets(struct radv_cmd_buffer *cmd_buffer, const struct r static void radv_cs_emit_mesh_dispatch_packet(struct radv_cmd_buffer *cmd_buffer, uint32_t x, uint32_t y, uint32_t z) { - radeon_emit(cmd_buffer->cs, PKT3(PKT3_DISPATCH_MESH_DIRECT, 3, cmd_buffer->state.predicating)); - radeon_emit(cmd_buffer->cs, x); - radeon_emit(cmd_buffer->cs, y); - radeon_emit(cmd_buffer->cs, z); - radeon_emit(cmd_buffer->cs, S_0287F0_SOURCE_SELECT(V_0287F0_DI_SRC_SEL_AUTO_INDEX)); + radeon_begin(cmd_buffer->cs); + radeon_emit(PKT3(PKT3_DISPATCH_MESH_DIRECT, 3, cmd_buffer->state.predicating)); + radeon_emit(x); + radeon_emit(y); + radeon_emit(z); + radeon_emit(S_0287F0_SOURCE_SELECT(V_0287F0_DI_SRC_SEL_AUTO_INDEX)); + radeon_end(); } ALWAYS_INLINE static void @@ -9995,10 +10249,12 @@ radv_emit_direct_mesh_draw_packet(struct radv_cmd_buffer *cmd_buffer, uint32_t x static void radv_emit_indirect_buffer(struct radeon_cmdbuf *cs, uint64_t va, bool is_compute) { - radeon_emit(cs, PKT3(PKT3_SET_BASE, 2, 0) | (is_compute ? PKT3_SHADER_TYPE_S(1) : 0)); - radeon_emit(cs, 1); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_SET_BASE, 2, 0) | (is_compute ? PKT3_SHADER_TYPE_S(1) : 0)); + radeon_emit(1); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_end(); } ALWAYS_INLINE static void @@ -10012,8 +10268,11 @@ radv_emit_indirect_mesh_draw_packets(struct radv_cmd_buffer *cmd_buffer, const s if (state->uses_drawid) { const struct radv_shader *mesh_shader = state->shaders[MESA_SHADER_MESH]; unsigned reg = state->vtx_base_sgpr + (mesh_shader->info.cs.uses_grid_size ? 12 : 0); - radeon_set_sh_reg_seq(cs, reg, 1); - radeon_emit(cs, 0); + + radeon_begin(cs); + radeon_set_sh_reg_seq(reg, 1); + radeon_emit(0); + radeon_end(); } if (!state->render.view_mask) { @@ -10075,13 +10334,14 @@ radv_emit_indirect_taskmesh_draw_packets(const struct radv_device *device, struc * - When count != 0, write 0 to the workaround BO and execute the indirect dispatch * - When workaround BO != 0 (count was 0), execute an empty direct dispatch */ - radeon_emit(ace_cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(ace_cs, - COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(ace_cs, 1); - radeon_emit(ace_cs, 0); - radeon_emit(ace_cs, workaround_cond_va); - radeon_emit(ace_cs, workaround_cond_va >> 32); + radeon_begin(ace_cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); + radeon_emit(1); + radeon_emit(0); + radeon_emit(workaround_cond_va); + radeon_emit(workaround_cond_va >> 32); + radeon_end(); /* 2x COND_EXEC + 1x COPY_DATA + Nx DISPATCH_TASKMESH_DIRECT_ACE */ ace_predication_size += 2 * 5 + 6 + 6 * num_views; @@ -10094,13 +10354,14 @@ radv_emit_indirect_taskmesh_draw_packets(const struct radv_device *device, struc radv_emit_cond_exec(device, ace_cs, info->count_va, 6 + 11 * num_views /* 1x COPY_DATA + Nx DISPATCH_TASKMESH_INDIRECT_MULTI_ACE */); - radeon_emit(ace_cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(ace_cs, - COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(ace_cs, 0); - radeon_emit(ace_cs, 0); - radeon_emit(ace_cs, workaround_cond_va); - radeon_emit(ace_cs, workaround_cond_va >> 32); + radeon_begin(ace_cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); + radeon_emit(0); + radeon_emit(0); + radeon_emit(workaround_cond_va); + radeon_emit(workaround_cond_va >> 32); + radeon_end(); } if (!view_mask) { @@ -10281,6 +10542,8 @@ radv_emit_ngg_culling_state(struct radv_cmd_buffer *cmd_buffer) /* Get current culling settings. */ uint32_t nggc_settings = radv_get_ngg_culling_settings(cmd_buffer, vp_y_inverted); + radeon_begin(cmd_buffer->cs); + if ((cmd_buffer->state.dirty & RADV_CMD_DIRTY_PIPELINE) || (cmd_buffer->state.dirty_dynamic & (RADV_DYNAMIC_VIEWPORT | RADV_DYNAMIC_RASTERIZATION_SAMPLES))) { /* Correction for inverted Y */ @@ -10297,13 +10560,15 @@ radv_emit_ngg_culling_state(struct radv_cmd_buffer *cmd_buffer) uint32_t vp_reg_values[4] = {fui(vp_scale[0]), fui(vp_scale[1]), fui(vp_translate[0]), fui(vp_translate[1])}; const uint32_t ngg_viewport_offset = radv_get_user_sgpr_loc(last_vgt_shader, AC_UD_NGG_VIEWPORT); - radeon_set_sh_reg_seq(cmd_buffer->cs, ngg_viewport_offset, 4); - radeon_emit_array(cmd_buffer->cs, vp_reg_values, 4); + + radeon_set_sh_reg_seq(ngg_viewport_offset, 4); + radeon_emit_array(vp_reg_values, 4); } const uint32_t ngg_culling_settings_offset = radv_get_user_sgpr_loc(last_vgt_shader, AC_UD_NGG_CULLING_SETTINGS); - radeon_set_sh_reg(cmd_buffer->cs, ngg_culling_settings_offset, nggc_settings); + radeon_set_sh_reg(ngg_culling_settings_offset, nggc_settings); + radeon_end(); } static void @@ -10327,7 +10592,9 @@ radv_emit_fs_state(struct radv_cmd_buffer *cmd_buffer) SET_SGPR_FIELD(PS_STATE_LINE_RAST_MODE, radv_get_line_mode(cmd_buffer)) | SET_SGPR_FIELD(PS_STATE_RAST_PRIM, rast_prim); - radeon_set_sh_reg(cmd_buffer->cs, ps_state_offset, ps_state); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(ps_state_offset, ps_state); + radeon_end(); } static uint32_t @@ -10404,13 +10671,16 @@ radv_emit_ngg_state(struct radv_cmd_buffer *cmd_buffer) SET_SGPR_FIELD(NGG_STATE_PROVOKING_VTX, radv_get_ngg_state_provoking_vtx(cmd_buffer)) | SET_SGPR_FIELD(NGG_STATE_QUERY, radv_get_ngg_state_query(cmd_buffer)); - radeon_set_sh_reg(cmd_buffer->cs, ngg_state_offset, ngg_state); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(ngg_state_offset, ngg_state); if (pdev->info.gfx_level >= GFX12) { const uint32_t ngg_query_buf_va_offset = radv_get_user_sgpr_loc(last_vgt_shader, AC_UD_NGG_QUERY_BUF_VA); if (ngg_query_buf_va_offset) - radeon_set_sh_reg(cmd_buffer->cs, ngg_query_buf_va_offset, cmd_buffer->state.shader_query_buf_va); + radeon_set_sh_reg(ngg_query_buf_va_offset, cmd_buffer->state.shader_query_buf_va); } + + radeon_end(); } static void @@ -10437,7 +10707,9 @@ radv_emit_task_state(struct radv_cmd_buffer *cmd_buffer) (cmd_buffer->state.inherited_pipeline_statistics & VK_QUERY_PIPELINE_STATISTIC_TASK_SHADER_INVOCATIONS_BIT_EXT)) shader_query_state |= radv_shader_query_pipeline_stat; - radeon_set_sh_reg(cmd_buffer->gang.cs, task_state_offset, shader_query_state); + radeon_begin(cmd_buffer->gang.cs); + radeon_set_sh_reg(task_state_offset, shader_query_state); + radeon_end(); } static void @@ -10504,9 +10776,11 @@ radv_emit_db_shader_control(struct radv_cmd_buffer *cmd_buffer) db_shader_control |= S_02880C_POPS_OVERLAP_NUM_SAMPLES(util_logbase2(rasterization_samples)); if (gpu_info->has_pops_missed_overlap_bug) { - radeon_set_context_reg(cmd_buffer->cs, R_028060_DB_DFSM_CONTROL, + radeon_begin(cmd_buffer->cs); + radeon_set_context_reg(R_028060_DB_DFSM_CONTROL, S_028060_PUNCHOUT_MODE(V_028060_FORCE_OFF) | S_028060_POPS_DRAIN_PS_ON_OVERLAP(rasterization_samples >= 8)); + radeon_end(); } } } else if (gpu_info->has_export_conflict_bug && rasterization_samples == 1) { @@ -10518,6 +10792,8 @@ radv_emit_db_shader_control(struct radv_cmd_buffer *cmd_buffer) } } + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { radeon_opt_set_context_reg(cmd_buffer, R_02806C_DB_SHADER_CONTROL, RADV_TRACKED_DB_SHADER_CONTROL, db_shader_control); @@ -10532,6 +10808,8 @@ radv_emit_db_shader_control(struct radv_cmd_buffer *cmd_buffer) db_shader_control); } + radeon_end(); + cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_DB_SHADER_CONTROL; } @@ -10546,22 +10824,24 @@ radv_emit_streamout_enable_state(struct radv_cmd_buffer *cmd_buffer) assert(!pdev->use_ngg_streamout); + radeon_begin(cmd_buffer->cs); + if (streamout_enabled && cmd_buffer->state.last_vgt_shader) { const struct radv_shader_info *info = &cmd_buffer->state.last_vgt_shader->info; enabled_stream_buffers_mask = info->so.enabled_stream_buffers_mask; u_foreach_bit (i, so->enabled_mask) { - radeon_set_context_reg(cmd_buffer->cs, R_028AD4_VGT_STRMOUT_VTX_STRIDE_0 + 16 * i, info->so.strides[i]); + radeon_set_context_reg(R_028AD4_VGT_STRMOUT_VTX_STRIDE_0 + 16 * i, info->so.strides[i]); } } - radeon_set_context_reg_seq(cmd_buffer->cs, R_028B94_VGT_STRMOUT_CONFIG, 2); - radeon_emit(cmd_buffer->cs, S_028B94_STREAMOUT_0_EN(streamout_enabled) | S_028B94_RAST_STREAM(0) | - S_028B94_STREAMOUT_1_EN(streamout_enabled) | - S_028B94_STREAMOUT_2_EN(streamout_enabled) | - S_028B94_STREAMOUT_3_EN(streamout_enabled)); - radeon_emit(cmd_buffer->cs, so->hw_enabled_mask & enabled_stream_buffers_mask); + radeon_set_context_reg_seq(R_028B94_VGT_STRMOUT_CONFIG, 2); + radeon_emit(S_028B94_STREAMOUT_0_EN(streamout_enabled) | S_028B94_RAST_STREAM(0) | + S_028B94_STREAMOUT_1_EN(streamout_enabled) | S_028B94_STREAMOUT_2_EN(streamout_enabled) | + S_028B94_STREAMOUT_3_EN(streamout_enabled)); + radeon_emit(so->hw_enabled_mask & enabled_stream_buffers_mask); + radeon_end(); cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_STREAMOUT_ENABLE; } @@ -10605,18 +10885,22 @@ radv_emit_fragment_output_state(struct radv_cmd_buffer *cmd_buffer) uint32_t col_format_compacted = radv_compact_spi_shader_col_format(cmd_buffer->state.spi_shader_col_format); + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg(cmd_buffer->cs, R_028854_CB_SHADER_MASK, cmd_buffer->state.cb_shader_mask); - radeon_set_context_reg_seq(cmd_buffer->cs, R_028650_SPI_SHADER_Z_FORMAT, 2); - radeon_emit(cmd_buffer->cs, cmd_buffer->state.spi_shader_z_format); - radeon_emit(cmd_buffer->cs, col_format_compacted); /* SPI_SHADER_COL_FORMAT */ + radeon_set_context_reg(R_028854_CB_SHADER_MASK, cmd_buffer->state.cb_shader_mask); + radeon_set_context_reg_seq(R_028650_SPI_SHADER_Z_FORMAT, 2); + radeon_emit(cmd_buffer->state.spi_shader_z_format); + radeon_emit(col_format_compacted); /* SPI_SHADER_COL_FORMAT */ } else { - radeon_set_context_reg(cmd_buffer->cs, R_02823C_CB_SHADER_MASK, cmd_buffer->state.cb_shader_mask); - radeon_set_context_reg_seq(cmd_buffer->cs, R_028710_SPI_SHADER_Z_FORMAT, 2); - radeon_emit(cmd_buffer->cs, cmd_buffer->state.spi_shader_z_format); - radeon_emit(cmd_buffer->cs, col_format_compacted); /* SPI_SHADER_COL_FORMAT */ + radeon_set_context_reg(R_02823C_CB_SHADER_MASK, cmd_buffer->state.cb_shader_mask); + radeon_set_context_reg_seq(R_028710_SPI_SHADER_Z_FORMAT, 2); + radeon_emit(cmd_buffer->state.spi_shader_z_format); + radeon_emit(col_format_compacted); /* SPI_SHADER_COL_FORMAT */ } + radeon_end(); + cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_FRAGMENT_OUTPUT; } @@ -10650,12 +10934,14 @@ radv_emit_depth_stencil_state(struct radv_cmd_buffer *cmd_buffer) const uint32_t depth_bounds_min = fui(d->vk.ds.depth.bounds_test.min); const uint32_t depth_bounds_max = fui(d->vk.ds.depth.bounds_test.max); + radeon_begin(cmd_buffer->cs); + if (pdev->info.gfx_level >= GFX12) { const bool force_s_valid = stencil_test_enable && ((d->vk.ds.stencil.front.op.pass != d->vk.ds.stencil.front.op.depth_fail) || (d->vk.ds.stencil.back.op.pass != d->vk.ds.stencil.back.op.depth_fail)); - radeon_set_context_reg(cmd_buffer->cs, R_02800C_DB_RENDER_OVERRIDE, + radeon_set_context_reg(R_02800C_DB_RENDER_OVERRIDE, S_02800C_FORCE_STENCIL_READ(1) | S_02800C_FORCE_STENCIL_VALID(force_s_valid)); radeon_opt_set_context_reg(cmd_buffer, R_028070_DB_DEPTH_CONTROL, RADV_TRACKED_DB_DEPTH_CONTROL, @@ -10704,6 +10990,8 @@ radv_emit_depth_stencil_state(struct radv_cmd_buffer *cmd_buffer) } } + radeon_end(); + cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_DEPTH_STENCIL_STATE; } @@ -10714,6 +11002,8 @@ radv_emit_raster_state(struct radv_cmd_buffer *cmd_buffer) const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; + radeon_begin(cmd_buffer->cs); + radeon_opt_set_context_reg(cmd_buffer, R_028A08_PA_SU_LINE_CNTL, RADV_TRACKED_PA_SU_LINE_CNTL, S_028A08_WIDTH(CLAMP(d->vk.rs.line.width * 8, 0, 0xFFFF))); @@ -10773,6 +11063,8 @@ radv_emit_raster_state(struct radv_cmd_buffer *cmd_buffer) pa_su_sc_mode_cntl); } + radeon_end(); + cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_RASTER_STATE; } @@ -11120,8 +11412,11 @@ radv_before_draw(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info struct radeon_cmdbuf *cs = cmd_buffer->cs; assert(state->vtx_base_sgpr); if (state->last_num_instances != info->instance_count) { - radeon_emit(cs, PKT3(PKT3_NUM_INSTANCES, 0, false)); - radeon_emit(cs, info->instance_count); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_NUM_INSTANCES, 0, false)); + radeon_emit(info->instance_count); + radeon_end(); + state->last_num_instances = info->instance_count; } } @@ -11182,8 +11477,12 @@ radv_before_taskmesh_draw(struct radv_cmd_buffer *cmd_buffer, const struct radv_ struct radv_cmd_state *state = &cmd_buffer->state; if (unlikely(state->last_num_instances != 1)) { struct radeon_cmdbuf *cs = cmd_buffer->cs; - radeon_emit(cs, PKT3(PKT3_NUM_INSTANCES, 0, false)); - radeon_emit(cs, 1); + + radeon_begin(cs); + radeon_emit(PKT3(PKT3_NUM_INSTANCES, 0, false)); + radeon_emit(1); + radeon_end(); + state->last_num_instances = 1; } } @@ -11646,8 +11945,10 @@ radv_CmdExecuteGeneratedCommandsEXT(VkCommandBuffer commandBuffer, VkBool32 isPr } if (!radv_cmd_buffer_uses_mec(cmd_buffer)) { - radeon_emit(cmd_buffer->cs, PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); - radeon_emit(cmd_buffer->cs, 0); + radeon_begin(cmd_buffer->cs); + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); + radeon_emit(0); + radeon_end(); } const uint32_t view_mask = cmd_buffer->state.render.view_mask; @@ -11725,18 +12026,21 @@ radv_save_dispatch_size(struct radv_cmd_buffer *cmd_buffer, uint64_t indirect_va uint64_t va = radv_buffer_get_va(device->trace_bo) + offsetof(struct radv_trace_data, indirect_dispatch); + radeon_begin(cs); + for (uint32_t i = 0; i < 3; i++) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, - COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, indirect_va); - radeon_emit(cs, indirect_va >> 32); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); + radeon_emit(indirect_va); + radeon_emit(indirect_va >> 32); + radeon_emit(va); + radeon_emit(va >> 32); indirect_va += 4; va += 4; } + + radeon_end(); } static void @@ -11768,15 +12072,17 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv radv_save_dispatch_size(cmd_buffer, info->indirect_va); if (info->unaligned) { - radeon_set_sh_reg_seq(cs, R_00B81C_COMPUTE_NUM_THREAD_X, 3); + radeon_begin(cs); + radeon_set_sh_reg_seq(R_00B81C_COMPUTE_NUM_THREAD_X, 3); if (pdev->info.gfx_level >= GFX12) { - radeon_emit(cs, S_00B81C_NUM_THREAD_FULL_GFX12(compute_shader->info.cs.block_size[0])); - radeon_emit(cs, S_00B820_NUM_THREAD_FULL_GFX12(compute_shader->info.cs.block_size[1])); + radeon_emit(S_00B81C_NUM_THREAD_FULL_GFX12(compute_shader->info.cs.block_size[0])); + radeon_emit(S_00B820_NUM_THREAD_FULL_GFX12(compute_shader->info.cs.block_size[1])); } else { - radeon_emit(cs, S_00B81C_NUM_THREAD_FULL_GFX6(compute_shader->info.cs.block_size[0])); - radeon_emit(cs, S_00B820_NUM_THREAD_FULL_GFX6(compute_shader->info.cs.block_size[1])); + radeon_emit(S_00B81C_NUM_THREAD_FULL_GFX6(compute_shader->info.cs.block_size[0])); + radeon_emit(S_00B820_NUM_THREAD_FULL_GFX6(compute_shader->info.cs.block_size[1])); } - radeon_emit(cs, S_00B824_NUM_THREAD_FULL(compute_shader->info.cs.block_size[2])); + radeon_emit(S_00B824_NUM_THREAD_FULL(compute_shader->info.cs.block_size[2])); + radeon_end(); dispatch_initiator |= S_00B800_USE_THREAD_DIMENSIONS(1); } @@ -11786,16 +12092,21 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv dispatch_initiator |= S_00B800_FORCE_START_AT_000(1); if (grid_size_offset) { + radeon_begin(cs); + if (device->load_grid_size_from_user_sgpr) { assert(pdev->info.gfx_level >= GFX10_3); - radeon_emit(cs, PKT3(PKT3_LOAD_SH_REG_INDEX, 3, 0)); - radeon_emit(cs, info->indirect_va); - radeon_emit(cs, info->indirect_va >> 32); - radeon_emit(cs, (grid_size_offset - SI_SH_REG_OFFSET) >> 2); - radeon_emit(cs, 3); + + radeon_emit(PKT3(PKT3_LOAD_SH_REG_INDEX, 3, 0)); + radeon_emit(info->indirect_va); + radeon_emit(info->indirect_va >> 32); + radeon_emit((grid_size_offset - SI_SH_REG_OFFSET) >> 2); + radeon_emit(3); } else { - radeon_emit_64bit_pointer(cmd_buffer->cs, grid_size_offset, info->indirect_va); + radeon_emit_64bit_pointer(grid_size_offset, info->indirect_va); } + + radeon_end(); } if (radv_cmd_buffer_uses_mec(cmd_buffer)) { @@ -11823,20 +12134,24 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv const uint64_t src_va = unaligned_va + i * 4; const uint64_t dst_va = indirect_va + i * 4; - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | - COPY_DATA_WR_CONFIRM); - radeon_emit(cs, src_va); - radeon_emit(cs, src_va >> 32); - radeon_emit(cs, dst_va); - radeon_emit(cs, dst_va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | + COPY_DATA_WR_CONFIRM); + radeon_emit(src_va); + radeon_emit(src_va >> 32); + radeon_emit(dst_va); + radeon_emit(dst_va >> 32); + radeon_end(); } } - radeon_emit(cs, PKT3(PKT3_DISPATCH_INDIRECT, 2, 0) | PKT3_SHADER_TYPE_S(1)); - radeon_emit(cs, indirect_va); - radeon_emit(cs, indirect_va >> 32); - radeon_emit(cs, dispatch_initiator); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_DISPATCH_INDIRECT, 2, 0) | PKT3_SHADER_TYPE_S(1)); + radeon_emit(indirect_va); + radeon_emit(indirect_va >> 32); + radeon_emit(dispatch_initiator); + radeon_end(); } else { radv_emit_indirect_buffer(cs, info->indirect_va, true); @@ -11846,9 +12161,11 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv predicating = false; } - radeon_emit(cs, PKT3(PKT3_DISPATCH_INDIRECT, 1, predicating) | PKT3_SHADER_TYPE_S(1)); - radeon_emit(cs, 0); - radeon_emit(cs, dispatch_initiator); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_DISPATCH_INDIRECT, 1, predicating) | PKT3_SHADER_TYPE_S(1)); + radeon_emit(0); + radeon_emit(dispatch_initiator); + radeon_end(); } } else { const unsigned *cs_block_size = compute_shader->info.cs.block_size; @@ -11874,44 +12191,49 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv offsets[i] /= cs_block_size[i]; } - radeon_set_sh_reg_seq(cs, R_00B81C_COMPUTE_NUM_THREAD_X, 3); + radeon_begin(cs); + radeon_set_sh_reg_seq(R_00B81C_COMPUTE_NUM_THREAD_X, 3); if (pdev->info.gfx_level >= GFX12) { - radeon_emit(cs, - S_00B81C_NUM_THREAD_FULL_GFX12(cs_block_size[0]) | S_00B81C_NUM_THREAD_PARTIAL(remainder[0])); - radeon_emit(cs, - S_00B820_NUM_THREAD_FULL_GFX12(cs_block_size[1]) | S_00B820_NUM_THREAD_PARTIAL(remainder[1])); + radeon_emit(S_00B81C_NUM_THREAD_FULL_GFX12(cs_block_size[0]) | S_00B81C_NUM_THREAD_PARTIAL(remainder[0])); + radeon_emit(S_00B820_NUM_THREAD_FULL_GFX12(cs_block_size[1]) | S_00B820_NUM_THREAD_PARTIAL(remainder[1])); } else { - radeon_emit(cs, - S_00B81C_NUM_THREAD_FULL_GFX6(cs_block_size[0]) | S_00B81C_NUM_THREAD_PARTIAL(remainder[0])); - radeon_emit(cs, - S_00B820_NUM_THREAD_FULL_GFX6(cs_block_size[1]) | S_00B820_NUM_THREAD_PARTIAL(remainder[1])); + radeon_emit(S_00B81C_NUM_THREAD_FULL_GFX6(cs_block_size[0]) | S_00B81C_NUM_THREAD_PARTIAL(remainder[0])); + radeon_emit(S_00B820_NUM_THREAD_FULL_GFX6(cs_block_size[1]) | S_00B820_NUM_THREAD_PARTIAL(remainder[1])); } - radeon_emit(cs, S_00B824_NUM_THREAD_FULL(cs_block_size[2]) | S_00B824_NUM_THREAD_PARTIAL(remainder[2])); + radeon_emit(S_00B824_NUM_THREAD_FULL(cs_block_size[2]) | S_00B824_NUM_THREAD_PARTIAL(remainder[2])); + radeon_end(); dispatch_initiator |= S_00B800_PARTIAL_TG_EN(1); } if (grid_size_offset) { if (device->load_grid_size_from_user_sgpr) { - radeon_set_sh_reg_seq(cs, grid_size_offset, 3); - radeon_emit(cs, blocks[0]); - radeon_emit(cs, blocks[1]); - radeon_emit(cs, blocks[2]); + radeon_begin(cs); + radeon_set_sh_reg_seq(grid_size_offset, 3); + radeon_emit(blocks[0]); + radeon_emit(blocks[1]); + radeon_emit(blocks[2]); + radeon_end(); } else { uint32_t offset; if (!radv_cmd_buffer_upload_data(cmd_buffer, 12, blocks, &offset)) return; uint64_t va = radv_buffer_get_va(cmd_buffer->upload.upload_bo) + offset; - radeon_emit_64bit_pointer(cmd_buffer->cs, grid_size_offset, va); + + radeon_begin(cs); + radeon_emit_64bit_pointer(grid_size_offset, va); + radeon_end(); } } if (offsets[0] || offsets[1] || offsets[2]) { - radeon_set_sh_reg_seq(cs, R_00B810_COMPUTE_START_X, 3); - radeon_emit(cs, offsets[0]); - radeon_emit(cs, offsets[1]); - radeon_emit(cs, offsets[2]); + radeon_begin(cs); + radeon_set_sh_reg_seq(R_00B810_COMPUTE_START_X, 3); + radeon_emit(offsets[0]); + radeon_emit(offsets[1]); + radeon_emit(offsets[2]); + radeon_end(); /* The blocks in the packet are not counts but end values. */ for (unsigned i = 0; i < 3; ++i) @@ -11940,11 +12262,13 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv } } - radeon_emit(cs, PKT3(PKT3_DISPATCH_DIRECT, 3, predicating) | PKT3_SHADER_TYPE_S(1)); - radeon_emit(cs, blocks[0]); - radeon_emit(cs, blocks[1]); - radeon_emit(cs, blocks[2]); - radeon_emit(cs, dispatch_initiator); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_DISPATCH_DIRECT, 3, predicating) | PKT3_SHADER_TYPE_S(1)); + radeon_emit(blocks[0]); + radeon_emit(blocks[1]); + radeon_emit(blocks[2]); + radeon_emit(dispatch_initiator); + radeon_end(); } assert(cmd_buffer->cs->cdw <= cdw_max); @@ -11985,7 +12309,9 @@ radv_emit_rt_stack_size(struct radv_cmd_buffer *cmd_buffer) rsrc2 |= S_00B12C_SCRATCH_EN(1); radeon_check_space(device->ws, cmd_buffer->cs, 3); - radeon_set_sh_reg(cmd_buffer->cs, rt_prolog->info.regs.pgm_rsrc2, rsrc2); + radeon_begin(cmd_buffer->cs); + radeon_set_sh_reg(rt_prolog->info.regs.pgm_rsrc2, rsrc2); + radeon_end(); } static void @@ -12325,12 +12651,16 @@ radv_trace_rays(struct radv_cmd_buffer *cmd_buffer, VkTraceRaysIndirectCommand2K const uint32_t sbt_descriptors_offset = radv_get_user_sgpr_loc(rt_prolog, AC_UD_CS_SBT_DESCRIPTORS); if (sbt_descriptors_offset) { - radeon_emit_64bit_pointer(cmd_buffer->cs, sbt_descriptors_offset, sbt_va); + radeon_begin(cmd_buffer->cs); + radeon_emit_64bit_pointer(sbt_descriptors_offset, sbt_va); + radeon_end(); } const uint32_t ray_launch_size_addr_offset = radv_get_user_sgpr_loc(rt_prolog, AC_UD_CS_RAY_LAUNCH_SIZE_ADDR); if (ray_launch_size_addr_offset) { - radeon_emit_64bit_pointer(cmd_buffer->cs, ray_launch_size_addr_offset, launch_size_va); + radeon_begin(cmd_buffer->cs); + radeon_emit_64bit_pointer(ray_launch_size_addr_offset, launch_size_va); + radeon_end(); } assert(cmd_buffer->cs->cdw <= cdw_max); @@ -12346,7 +12676,9 @@ radv_trace_rays(struct radv_cmd_buffer *cmd_buffer, VkTraceRaysIndirectCommand2K tables->height = 1; radv_upload_trace_rays_params(cmd_buffer, tables, mode, &launch_size_va, NULL); if (ray_launch_size_addr_offset) { - radeon_emit_64bit_pointer(cmd_buffer->cs, ray_launch_size_addr_offset, launch_size_va); + radeon_begin(cmd_buffer->cs); + radeon_emit_64bit_pointer(ray_launch_size_addr_offset, launch_size_va); + radeon_end(); } radv_dispatch(cmd_buffer, &info, pipeline, rt_prolog, VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR); @@ -13123,16 +13455,19 @@ radv_emit_set_predication_state(struct radv_cmd_buffer *cmd_buffer, bool draw_vi */ op |= draw_visible ? PREDICATION_DRAW_VISIBLE : PREDICATION_DRAW_NOT_VISIBLE; } + + radeon_begin(cmd_buffer->cs); if (pdev->info.gfx_level >= GFX9) { - radeon_emit(cmd_buffer->cs, PKT3(PKT3_SET_PREDICATION, 2, 0)); - radeon_emit(cmd_buffer->cs, op); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, va >> 32); + radeon_emit(PKT3(PKT3_SET_PREDICATION, 2, 0)); + radeon_emit(op); + radeon_emit(va); + radeon_emit(va >> 32); } else { - radeon_emit(cmd_buffer->cs, PKT3(PKT3_SET_PREDICATION, 1, 0)); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, op | ((va >> 32) & 0xFF)); + radeon_emit(PKT3(PKT3_SET_PREDICATION, 1, 0)); + radeon_emit(va); + radeon_emit(op | ((va >> 32) & 0xFF)); } + radeon_end(); } void @@ -13181,17 +13516,19 @@ radv_begin_conditional_rendering(struct radv_cmd_buffer *cmd_buffer, uint64_t va pred_va = radv_buffer_get_va(cmd_buffer->upload.upload_bo) + pred_offset; radeon_check_space(device->ws, cmd_buffer->cs, 8); + radeon_begin(cs); - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit( - cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, pred_va); - radeon_emit(cs, pred_va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | + COPY_DATA_WR_CONFIRM); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(pred_va); + radeon_emit(pred_va >> 32); - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0)); + radeon_emit(0); + radeon_end(); va = pred_va; pred_op = PREDICATION_OP_BOOL64; @@ -13332,32 +13669,35 @@ radv_flush_vgt_streamout(struct radv_cmd_buffer *cmd_buffer) ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, 14); + radeon_begin(cs); + /* The register is at different places on different ASICs. */ if (pdev->info.gfx_level >= GFX9) { reg_strmout_cntl = R_0300FC_CP_STRMOUT_CNTL; - radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0)); - radeon_emit(cs, S_370_DST_SEL(V_370_MEM_MAPPED_REGISTER) | S_370_ENGINE_SEL(V_370_ME)); - radeon_emit(cs, R_0300FC_CP_STRMOUT_CNTL >> 2); - radeon_emit(cs, 0); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_WRITE_DATA, 3, 0)); + radeon_emit(S_370_DST_SEL(V_370_MEM_MAPPED_REGISTER) | S_370_ENGINE_SEL(V_370_ME)); + radeon_emit(R_0300FC_CP_STRMOUT_CNTL >> 2); + radeon_emit(0); + radeon_emit(0); } else if (pdev->info.gfx_level >= GFX7) { reg_strmout_cntl = R_0300FC_CP_STRMOUT_CNTL; - radeon_set_uconfig_reg(cs, reg_strmout_cntl, 0); + radeon_set_uconfig_reg(reg_strmout_cntl, 0); } else { reg_strmout_cntl = R_0084FC_CP_STRMOUT_CNTL; - radeon_set_config_reg(cs, reg_strmout_cntl, 0); + radeon_set_config_reg(reg_strmout_cntl, 0); } - radeon_event_write(cs, V_028A90_SO_VGTSTREAMOUT_FLUSH); + radeon_event_write(V_028A90_SO_VGTSTREAMOUT_FLUSH); - radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0)); - radeon_emit(cs, WAIT_REG_MEM_EQUAL); /* wait until the register is equal to the reference value */ - radeon_emit(cs, reg_strmout_cntl >> 2); /* register */ - radeon_emit(cs, 0); - radeon_emit(cs, S_0084FC_OFFSET_UPDATE_DONE(1)); /* reference value */ - radeon_emit(cs, S_0084FC_OFFSET_UPDATE_DONE(1)); /* mask */ - radeon_emit(cs, 4); /* poll interval */ + radeon_emit(PKT3(PKT3_WAIT_REG_MEM, 5, 0)); + radeon_emit(WAIT_REG_MEM_EQUAL); /* wait until the register is equal to the reference value */ + radeon_emit(reg_strmout_cntl >> 2); /* register */ + radeon_emit(0); + radeon_emit(S_0084FC_OFFSET_UPDATE_DONE(1)); /* reference value */ + radeon_emit(S_0084FC_OFFSET_UPDATE_DONE(1)); /* mask */ + radeon_emit(4); /* poll interval */ + radeon_end(); assert(cs->cdw <= cdw_max); } @@ -13398,7 +13738,9 @@ radv_init_streamout_state(struct radv_cmd_buffer *cmd_buffer) /* Reset the ordered ID for the next GS workgroup to 0 because it must be * equal to the 4 ordered IDs in the layout. */ - radeon_set_uconfig_reg(cmd_buffer->cs, R_0309B0_GE_GS_ORDERED_ID_BASE, 0); + radeon_begin(cmd_buffer->cs); + radeon_set_uconfig_reg(R_0309B0_GE_GS_ORDERED_ID_BASE, 0); + radeon_end(); } VKAPI_ATTR void VKAPI_CALL @@ -13443,58 +13785,61 @@ radv_CmdBeginTransformFeedbackEXT(VkCommandBuffer commandBuffer, uint32_t firstC radv_cs_add_buffer(device->ws, cs, buffer->bo); } + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX12) { if (append) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit( - cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, (so->state_va + i * 8 + 4)); - radeon_emit(cs, (so->state_va + i * 8 + 4) >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | + COPY_DATA_WR_CONFIRM); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit((so->state_va + i * 8 + 4)); + radeon_emit((so->state_va + i * 8 + 4) >> 32); } } else if (pdev->use_ngg_streamout) { if (append) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, - COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, (R_031088_GDS_STRMOUT_DWORDS_WRITTEN_0 >> 2) + i); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | COPY_DATA_WR_CONFIRM); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit((R_031088_GDS_STRMOUT_DWORDS_WRITTEN_0 >> 2) + i); + radeon_emit(0); } else { /* The PKT3 CAM bit workaround seems needed for initializing this GDS register to zero. */ - radeon_set_uconfig_perfctr_reg(pdev->info.gfx_level, ring, cs, - R_031088_GDS_STRMOUT_DWORDS_WRITTEN_0 + i * 4, 0); + radeon_set_uconfig_perfctr_reg(pdev->info.gfx_level, ring, R_031088_GDS_STRMOUT_DWORDS_WRITTEN_0 + i * 4, + 0); } } else { /* AMD GCN binds streamout buffers as shader resources. * VGT only counts primitives and tells the shader through * SGPRs what to do. */ - radeon_set_context_reg(cs, R_028AD0_VGT_STRMOUT_BUFFER_SIZE_0 + 16 * i, sb[i].size >> 2); + radeon_set_context_reg(R_028AD0_VGT_STRMOUT_BUFFER_SIZE_0 + 16 * i, sb[i].size >> 2); cmd_buffer->state.context_roll_without_scissor_emitted = true; if (append) { - radeon_emit(cs, PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0)); - radeon_emit(cs, STRMOUT_SELECT_BUFFER(i) | STRMOUT_DATA_TYPE(1) | /* offset in bytes */ - STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_FROM_MEM)); /* control */ - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, va); /* src address lo */ - radeon_emit(cs, va >> 32); /* src address hi */ + radeon_emit(PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0)); + radeon_emit(STRMOUT_SELECT_BUFFER(i) | STRMOUT_DATA_TYPE(1) | /* offset in bytes */ + STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_FROM_MEM)); /* control */ + radeon_emit(0); /* unused */ + radeon_emit(0); /* unused */ + radeon_emit(va); /* src address lo */ + radeon_emit(va >> 32); /* src address hi */ } else { /* Start from the beginning. */ - radeon_emit(cs, PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0)); - radeon_emit(cs, STRMOUT_SELECT_BUFFER(i) | STRMOUT_DATA_TYPE(1) | /* offset in bytes */ - STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_FROM_PACKET)); /* control */ - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, 0); /* unused */ + radeon_emit(PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0)); + radeon_emit(STRMOUT_SELECT_BUFFER(i) | STRMOUT_DATA_TYPE(1) | /* offset in bytes */ + STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_FROM_PACKET)); /* control */ + radeon_emit(0); /* unused */ + radeon_emit(0); /* unused */ + radeon_emit(0); /* unused */ + radeon_emit(0); /* unused */ } } + + radeon_end(); } assert(cs->cdw <= cdw_max); @@ -13552,36 +13897,36 @@ radv_CmdEndTransformFeedbackEXT(VkCommandBuffer commandBuffer, uint32_t firstCou radv_cs_add_buffer(device->ws, cs, buffer->bo); } + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX12) { if (append) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit( - cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, (so->state_va + i * 8 + 4)); - radeon_emit(cs, (so->state_va + i * 8 + 4) >> 32); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | + COPY_DATA_WR_CONFIRM); + radeon_emit((so->state_va + i * 8 + 4)); + radeon_emit((so->state_va + i * 8 + 4) >> 32); + radeon_emit(va); + radeon_emit(va >> 32); } } else if (pdev->use_ngg_streamout) { if (append) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, - COPY_DATA_SRC_SEL(COPY_DATA_REG) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, (R_031088_GDS_STRMOUT_DWORDS_WRITTEN_0 >> 2) + i); - radeon_emit(cs, 0); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_REG) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); + radeon_emit((R_031088_GDS_STRMOUT_DWORDS_WRITTEN_0 >> 2) + i); + radeon_emit(0); + radeon_emit(va); + radeon_emit(va >> 32); } } else { if (append) { - radeon_emit(cs, PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0)); - radeon_emit(cs, STRMOUT_SELECT_BUFFER(i) | STRMOUT_DATA_TYPE(1) | /* offset in bytes */ - STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_NONE) | - STRMOUT_STORE_BUFFER_FILLED_SIZE); /* control */ - radeon_emit(cs, va); /* dst address lo */ - radeon_emit(cs, va >> 32); /* dst address hi */ - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, 0); /* unused */ + radeon_emit(PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0)); + radeon_emit(STRMOUT_SELECT_BUFFER(i) | STRMOUT_DATA_TYPE(1) | /* offset in bytes */ + STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_NONE) | STRMOUT_STORE_BUFFER_FILLED_SIZE); /* control */ + radeon_emit(va); /* dst address lo */ + radeon_emit(va >> 32); /* dst address hi */ + radeon_emit(0); /* unused */ + radeon_emit(0); /* unused */ } /* Deactivate transform feedback by zeroing the buffer size. @@ -13589,10 +13934,12 @@ radv_CmdEndTransformFeedbackEXT(VkCommandBuffer commandBuffer, uint32_t firstCou * be enabled even if there is not buffer bound. This ensures * that the primitives-emitted query won't increment. */ - radeon_set_context_reg(cs, R_028AD0_VGT_STRMOUT_BUFFER_SIZE_0 + 16 * i, 0); + radeon_set_context_reg(R_028AD0_VGT_STRMOUT_BUFFER_SIZE_0 + 16 * i, 0); cmd_buffer->state.context_roll_without_scissor_emitted = true; } + + radeon_end(); } assert(cmd_buffer->cs->cdw <= cdw_max); @@ -13609,30 +13956,32 @@ radv_emit_strmout_buffer(struct radv_cmd_buffer *cmd_buffer, const struct radv_d const enum amd_gfx_level gfx_level = pdev->info.gfx_level; struct radeon_cmdbuf *cs = cmd_buffer->cs; - radeon_set_context_reg(cs, R_028B30_VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE, draw_info->stride); + radeon_begin(cs); + radeon_set_context_reg(R_028B30_VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE, draw_info->stride); if (gfx_level >= GFX10) { /* Emitting a COPY_DATA packet should be enough because RADV doesn't support preemption * (shadow memory) but for unknown reasons, it can lead to GPU hangs on GFX10+. */ - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0)); + radeon_emit(0); - radeon_emit(cs, PKT3(PKT3_LOAD_CONTEXT_REG_INDEX, 3, 0)); - radeon_emit(cs, draw_info->strmout_va); - radeon_emit(cs, draw_info->strmout_va >> 32); - radeon_emit(cs, (R_028B2C_VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE - SI_CONTEXT_REG_OFFSET) >> 2); - radeon_emit(cs, 1); /* 1 DWORD */ + radeon_emit(PKT3(PKT3_LOAD_CONTEXT_REG_INDEX, 3, 0)); + radeon_emit(draw_info->strmout_va); + radeon_emit(draw_info->strmout_va >> 32); + radeon_emit((R_028B2C_VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE - SI_CONTEXT_REG_OFFSET) >> 2); + radeon_emit(1); /* 1 DWORD */ } else { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, draw_info->strmout_va); - radeon_emit(cs, draw_info->strmout_va >> 32); - radeon_emit(cs, R_028B2C_VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE >> 2); - radeon_emit(cs, 0); /* unused */ + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) | COPY_DATA_WR_CONFIRM); + radeon_emit(draw_info->strmout_va); + radeon_emit(draw_info->strmout_va >> 32); + radeon_emit(R_028B2C_VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE >> 2); + radeon_emit(0); /* unused */ } - radeon_set_context_reg(cs, R_028B28_VGT_STRMOUT_DRAW_OPAQUE_OFFSET, counter_offset); + radeon_set_context_reg(R_028B28_VGT_STRMOUT_DRAW_OPAQUE_OFFSET, counter_offset); + radeon_end(); } VKAPI_ATTR void VKAPI_CALL @@ -13664,9 +14013,11 @@ radv_CmdDrawIndirectByteCountEXT(VkCommandBuffer commandBuffer, uint32_t instanc if (pdev->info.gfx_level == GFX12) { /* DrawTransformFeedback requires 3 SQ_NON_EVENTs after the packet. */ - radeon_event_write(cmd_buffer->cs, V_028A90_SQ_NON_EVENT); - radeon_event_write(cmd_buffer->cs, V_028A90_SQ_NON_EVENT); - radeon_event_write(cmd_buffer->cs, V_028A90_SQ_NON_EVENT); + radeon_begin(cmd_buffer->cs); + radeon_event_write(V_028A90_SQ_NON_EVENT); + radeon_event_write(V_028A90_SQ_NON_EVENT); + radeon_event_write(V_028A90_SQ_NON_EVENT); + radeon_end(); } radv_after_draw(cmd_buffer, false); @@ -13697,12 +14048,14 @@ radv_CmdWriteBufferMarker2AMD(VkCommandBuffer commandBuffer, VkPipelineStageFlag ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, 12); if (!(stage & ~VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT)) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, marker); - radeon_emit(cs, 0); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); + radeon_emit(marker); + radeon_emit(0); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_end(); } else { radv_cs_emit_write_event_eop(cs, pdev->info.gfx_level, cmd_buffer->qf, V_028A90_BOTTOM_OF_PIPE_TS, 0, EOP_DST_SEL_MEM, EOP_DATA_SEL_VALUE_32BIT, va, marker, cmd_buffer->gfx9_eop_bug_va); diff --git a/src/amd/vulkan/radv_cp_dma.c b/src/amd/vulkan/radv_cp_dma.c index 80cbce03e57..68dafdacf59 100644 --- a/src/amd/vulkan/radv_cp_dma.c +++ b/src/amd/vulkan/radv_cp_dma.c @@ -80,24 +80,26 @@ radv_cs_emit_cp_dma(struct radv_device *device, struct radeon_cmdbuf *cs, bool p else if (cp_dma_use_L2) header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2); + radeon_begin(cs); if (pdev->info.gfx_level >= GFX7) { - radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, predicating)); - radeon_emit(cs, header); - radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */ - radeon_emit(cs, src_va >> 32); /* SRC_ADDR_HI [31:0] */ - radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */ - radeon_emit(cs, dst_va >> 32); /* DST_ADDR_HI [31:0] */ - radeon_emit(cs, command); + radeon_emit(PKT3(PKT3_DMA_DATA, 5, predicating)); + radeon_emit(header); + radeon_emit(src_va); /* SRC_ADDR_LO [31:0] */ + radeon_emit(src_va >> 32); /* SRC_ADDR_HI [31:0] */ + radeon_emit(dst_va); /* DST_ADDR_LO [31:0] */ + radeon_emit(dst_va >> 32); /* DST_ADDR_HI [31:0] */ + radeon_emit(command); } else { assert(!cp_dma_use_L2); header |= S_411_SRC_ADDR_HI(src_va >> 32); - radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, predicating)); - radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */ - radeon_emit(cs, header); /* SRC_ADDR_HI [15:0] + flags. */ - radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */ - radeon_emit(cs, (dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */ - radeon_emit(cs, command); + radeon_emit(PKT3(PKT3_CP_DMA, 4, predicating)); + radeon_emit(src_va); /* SRC_ADDR_LO [31:0] */ + radeon_emit(header); /* SRC_ADDR_HI [15:0] + flags. */ + radeon_emit(dst_va); /* DST_ADDR_LO [31:0] */ + radeon_emit((dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */ + radeon_emit(command); } + radeon_end(); } static void @@ -116,8 +118,10 @@ radv_emit_cp_dma(struct radv_cmd_buffer *cmd_buffer, uint64_t dst_va, uint64_t s */ if (flags & CP_DMA_SYNC) { if (cmd_buffer->qf == RADV_QUEUE_GENERAL) { - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); - radeon_emit(cs, 0); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); + radeon_emit(0); + radeon_end(); } /* CP will see the sync flag and wait for all DMAs to complete. */ @@ -157,13 +161,15 @@ radv_cs_cp_dma_prefetch(const struct radv_device *device, struct radeon_cmdbuf * header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2); - radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, predicating)); - radeon_emit(cs, header); - radeon_emit(cs, aligned_va); /* SRC_ADDR_LO [31:0] */ - radeon_emit(cs, aligned_va >> 32); /* SRC_ADDR_HI [31:0] */ - radeon_emit(cs, aligned_va); /* DST_ADDR_LO [31:0] */ - radeon_emit(cs, aligned_va >> 32); /* DST_ADDR_HI [31:0] */ - radeon_emit(cs, command); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_DMA_DATA, 5, predicating)); + radeon_emit(header); + radeon_emit(aligned_va); /* SRC_ADDR_LO [31:0] */ + radeon_emit(aligned_va >> 32); /* SRC_ADDR_HI [31:0] */ + radeon_emit(aligned_va); /* DST_ADDR_LO [31:0] */ + radeon_emit(aligned_va >> 32); /* DST_ADDR_HI [31:0] */ + radeon_emit(command); + radeon_end(); } void diff --git a/src/amd/vulkan/radv_cp_reg_shadowing.c b/src/amd/vulkan/radv_cp_reg_shadowing.c index 1818b68198c..0ec33d35807 100644 --- a/src/amd/vulkan/radv_cp_reg_shadowing.c +++ b/src/amd/vulkan/radv_cp_reg_shadowing.c @@ -39,7 +39,10 @@ radv_create_shadow_regs_preamble(struct radv_device *device, struct radv_queue_s if (!pm4) goto fail_create; - radeon_emit_array(cs, pm4->pm4, pm4->ndw); + radeon_begin(cs); + radeon_emit_array(pm4->pm4, pm4->ndw); + radeon_end(); + ws->cs_pad(cs, 0); result = radv_bo_create( @@ -126,7 +129,10 @@ radv_init_shadowed_regs_buffer_state(const struct radv_device *device, struct ra goto fail; } - radeon_emit_array(cs, pm4->pm4, pm4->ndw); + radeon_begin(cs); + radeon_emit_array(pm4->pm4, pm4->ndw); + radeon_end(); + ac_pm4_free_state(pm4); } diff --git a/src/amd/vulkan/radv_cs.c b/src/amd/vulkan/radv_cs.c index 60aebcd76f6..5672c9b7dc8 100644 --- a/src/amd/vulkan/radv_cs.c +++ b/src/amd/vulkan/radv_cs.c @@ -37,27 +37,29 @@ radv_cs_emit_write_event_eop(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_le if (data_sel != EOP_DATA_SEL_DISCARD) sel |= EOP_INT_SEL(EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM); + radeon_begin(cs); + if (gfx_level >= GFX9 || is_gfx8_mec) { /* A ZPASS_DONE or PIXEL_STAT_DUMP_EVENT (of the DB occlusion * counters) must immediately precede every timestamp event to * prevent a GPU hang on GFX9. */ if (gfx_level == GFX9 && !is_mec) { - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0)); - radeon_emit(cs, EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1)); - radeon_emit(cs, gfx9_eop_bug_va); - radeon_emit(cs, gfx9_eop_bug_va >> 32); + radeon_emit(PKT3(PKT3_EVENT_WRITE, 2, 0)); + radeon_emit(EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1)); + radeon_emit(gfx9_eop_bug_va); + radeon_emit(gfx9_eop_bug_va >> 32); } - radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, is_gfx8_mec ? 5 : 6, false)); - radeon_emit(cs, op); - radeon_emit(cs, sel); - radeon_emit(cs, va); /* address lo */ - radeon_emit(cs, va >> 32); /* address hi */ - radeon_emit(cs, new_fence); /* immediate data lo */ - radeon_emit(cs, 0); /* immediate data hi */ + radeon_emit(PKT3(PKT3_RELEASE_MEM, is_gfx8_mec ? 5 : 6, false)); + radeon_emit(op); + radeon_emit(sel); + radeon_emit(va); /* address lo */ + radeon_emit(va >> 32); /* address hi */ + radeon_emit(new_fence); /* immediate data lo */ + radeon_emit(0); /* immediate data hi */ if (!is_gfx8_mec) - radeon_emit(cs, 0); /* unused */ + radeon_emit(0); /* unused */ } else { /* On GFX6, EOS events are always emitted with EVENT_WRITE_EOS. * On GFX7+, EOS events are emitted with EVENT_WRITE_EOS on @@ -68,19 +70,19 @@ radv_cs_emit_write_event_eop(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_le assert(event_flags == 0 && dst_sel == EOP_DST_SEL_MEM && data_sel == EOP_DATA_SEL_VALUE_32BIT); if (is_mec) { - radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, 5, false)); - radeon_emit(cs, op); - radeon_emit(cs, sel); - radeon_emit(cs, va); /* address lo */ - radeon_emit(cs, va >> 32); /* address hi */ - radeon_emit(cs, new_fence); /* immediate data lo */ - radeon_emit(cs, 0); /* immediate data hi */ + radeon_emit(PKT3(PKT3_RELEASE_MEM, 5, false)); + radeon_emit(op); + radeon_emit(sel); + radeon_emit(va); /* address lo */ + radeon_emit(va >> 32); /* address hi */ + radeon_emit(new_fence); /* immediate data lo */ + radeon_emit(0); /* immediate data hi */ } else { - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOS, 3, false)); - radeon_emit(cs, op); - radeon_emit(cs, va); - radeon_emit(cs, ((va >> 32) & 0xffff) | EOS_DATA_SEL(EOS_DATA_SEL_VALUE_32BIT)); - radeon_emit(cs, new_fence); + radeon_emit(PKT3(PKT3_EVENT_WRITE_EOS, 3, false)); + radeon_emit(op); + radeon_emit(va); + radeon_emit(((va >> 32) & 0xffff) | EOS_DATA_SEL(EOS_DATA_SEL_VALUE_32BIT)); + radeon_emit(new_fence); } } else { if (gfx_level == GFX7 || gfx_level == GFX8) { @@ -88,44 +90,50 @@ radv_cs_emit_write_event_eop(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_le * engines go idle (and optional cache flushes * executed) before the timestamp is written. */ - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, false)); - radeon_emit(cs, op); - radeon_emit(cs, va); - radeon_emit(cs, ((va >> 32) & 0xffff) | sel); - radeon_emit(cs, 0); /* immediate data */ - radeon_emit(cs, 0); /* unused */ + radeon_emit(PKT3(PKT3_EVENT_WRITE_EOP, 4, false)); + radeon_emit(op); + radeon_emit(va); + radeon_emit(((va >> 32) & 0xffff) | sel); + radeon_emit(0); /* immediate data */ + radeon_emit(0); /* unused */ } - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, false)); - radeon_emit(cs, op); - radeon_emit(cs, va); - radeon_emit(cs, ((va >> 32) & 0xffff) | sel); - radeon_emit(cs, new_fence); /* immediate data */ - radeon_emit(cs, 0); /* unused */ + radeon_emit(PKT3(PKT3_EVENT_WRITE_EOP, 4, false)); + radeon_emit(op); + radeon_emit(va); + radeon_emit(((va >> 32) & 0xffff) | sel); + radeon_emit(new_fence); /* immediate data */ + radeon_emit(0); /* unused */ } } + + radeon_end(); } static void radv_emit_acquire_mem(struct radeon_cmdbuf *cs, bool is_mec, bool is_gfx9, unsigned cp_coher_cntl) { + radeon_begin(cs); + if (is_mec || is_gfx9) { uint32_t hi_val = is_gfx9 ? 0xffffff : 0xff; - radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 5, false) | PKT3_SHADER_TYPE_S(is_mec)); - radeon_emit(cs, cp_coher_cntl); /* CP_COHER_CNTL */ - radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */ - radeon_emit(cs, hi_val); /* CP_COHER_SIZE_HI */ - radeon_emit(cs, 0); /* CP_COHER_BASE */ - radeon_emit(cs, 0); /* CP_COHER_BASE_HI */ - radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */ + radeon_emit(PKT3(PKT3_ACQUIRE_MEM, 5, false) | PKT3_SHADER_TYPE_S(is_mec)); + radeon_emit(cp_coher_cntl); /* CP_COHER_CNTL */ + radeon_emit(0xffffffff); /* CP_COHER_SIZE */ + radeon_emit(hi_val); /* CP_COHER_SIZE_HI */ + radeon_emit(0); /* CP_COHER_BASE */ + radeon_emit(0); /* CP_COHER_BASE_HI */ + radeon_emit(0x0000000A); /* POLL_INTERVAL */ } else { /* ACQUIRE_MEM is only required on a compute ring. */ - radeon_emit(cs, PKT3(PKT3_SURFACE_SYNC, 3, false)); - radeon_emit(cs, cp_coher_cntl); /* CP_COHER_CNTL */ - radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */ - radeon_emit(cs, 0); /* CP_COHER_BASE */ - radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */ + radeon_emit(PKT3(PKT3_SURFACE_SYNC, 3, false)); + radeon_emit(cp_coher_cntl); /* CP_COHER_CNTL */ + radeon_emit(0xffffffff); /* CP_COHER_SIZE */ + radeon_emit(0); /* CP_COHER_BASE */ + radeon_emit(0x0000000A); /* POLL_INTERVAL */ } + + radeon_end(); } static void @@ -179,7 +187,9 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level /* TODO: trigger on RADV_CMD_FLAG_FLUSH_AND_INV_CB_META */ if (gfx_level < GFX12 && flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_CB) { /* Flush CMASK/FMASK/DCC. Will wait for idle later. */ - radeon_event_write(cs, V_028A90_FLUSH_AND_INV_CB_META); + radeon_begin(cs); + radeon_event_write(V_028A90_FLUSH_AND_INV_CB_META); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_FLUSH_CB | RGP_FLUSH_INVAL_CB; } @@ -188,7 +198,9 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level /* TODO: trigger on RADV_CMD_FLAG_FLUSH_AND_INV_DB_META ? */ if (gfx_level < GFX12 && gfx_level != GFX11 && (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_DB)) { /* Flush HTILE. Will wait for idle later. */ - radeon_event_write(cs, V_028A90_FLUSH_AND_INV_DB_META); + radeon_begin(cs); + radeon_event_write(V_028A90_FLUSH_AND_INV_DB_META); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_FLUSH_DB | RGP_FLUSH_INVAL_DB; } @@ -213,18 +225,24 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level } else { /* Wait for graphics shaders to go idle if requested. */ if (flush_bits & RADV_CMD_FLAG_PS_PARTIAL_FLUSH) { - radeon_event_write(cs, V_028A90_PS_PARTIAL_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_PS_PARTIAL_FLUSH); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_PS_PARTIAL_FLUSH; } else if (flush_bits & RADV_CMD_FLAG_VS_PARTIAL_FLUSH) { - radeon_event_write(cs, V_028A90_VS_PARTIAL_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_VS_PARTIAL_FLUSH); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_VS_PARTIAL_FLUSH; } } if (flush_bits & RADV_CMD_FLAG_CS_PARTIAL_FLUSH) { - radeon_event_write(cs, V_028A90_CS_PARTIAL_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_CS_PARTIAL_FLUSH); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_CS_PARTIAL_FLUSH; } @@ -248,29 +266,33 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level gcr_cntl &= C_586_GLM_WB & C_586_GLM_INV & C_586_GLK_WB & C_586_GLK_INV & C_586_GLV_INV & C_586_GL1_INV & C_586_GL2_INV & C_586_GL2_WB; /* keep SEQ */ + radeon_begin(cs); + /* Send an event that flushes caches. */ - radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, 6, 0)); - radeon_emit(cs, S_490_EVENT_TYPE(cb_db_event) | S_490_EVENT_INDEX(5) | S_490_GLM_WB(glm_wb) | - S_490_GLM_INV(glm_inv) | S_490_GLV_INV(glv_inv) | S_490_GL1_INV(gl1_inv) | - S_490_GL2_INV(gl2_inv) | S_490_GL2_WB(gl2_wb) | S_490_SEQ(gcr_seq) | S_490_GLK_WB(glk_wb) | - S_490_GLK_INV(glk_inv) | S_490_PWS_ENABLE(1)); - radeon_emit(cs, 0); /* DST_SEL, INT_SEL, DATA_SEL */ - radeon_emit(cs, 0); /* ADDRESS_LO */ - radeon_emit(cs, 0); /* ADDRESS_HI */ - radeon_emit(cs, 0); /* DATA_LO */ - radeon_emit(cs, 0); /* DATA_HI */ - radeon_emit(cs, 0); /* INT_CTXID */ + radeon_emit(PKT3(PKT3_RELEASE_MEM, 6, 0)); + radeon_emit(S_490_EVENT_TYPE(cb_db_event) | S_490_EVENT_INDEX(5) | S_490_GLM_WB(glm_wb) | + S_490_GLM_INV(glm_inv) | S_490_GLV_INV(glv_inv) | S_490_GL1_INV(gl1_inv) | S_490_GL2_INV(gl2_inv) | + S_490_GL2_WB(gl2_wb) | S_490_SEQ(gcr_seq) | S_490_GLK_WB(glk_wb) | S_490_GLK_INV(glk_inv) | + S_490_PWS_ENABLE(1)); + radeon_emit(0); /* DST_SEL, INT_SEL, DATA_SEL */ + radeon_emit(0); /* ADDRESS_LO */ + radeon_emit(0); /* ADDRESS_HI */ + radeon_emit(0); /* DATA_LO */ + radeon_emit(0); /* DATA_HI */ + radeon_emit(0); /* INT_CTXID */ /* Wait for the event and invalidate remaining caches if needed. */ - radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 6, 0)); - radeon_emit(cs, S_580_PWS_STAGE_SEL(V_580_CP_PFP) | S_580_PWS_COUNTER_SEL(V_580_TS_SELECT) | - S_580_PWS_ENA2(1) | S_580_PWS_COUNT(0)); - radeon_emit(cs, 0xffffffff); /* GCR_SIZE */ - radeon_emit(cs, 0x01ffffff); /* GCR_SIZE_HI */ - radeon_emit(cs, 0); /* GCR_BASE_LO */ - radeon_emit(cs, 0); /* GCR_BASE_HI */ - radeon_emit(cs, S_585_PWS_ENA(1)); - radeon_emit(cs, gcr_cntl); /* GCR_CNTL */ + radeon_emit(PKT3(PKT3_ACQUIRE_MEM, 6, 0)); + radeon_emit(S_580_PWS_STAGE_SEL(V_580_CP_PFP) | S_580_PWS_COUNTER_SEL(V_580_TS_SELECT) | S_580_PWS_ENA2(1) | + S_580_PWS_COUNT(0)); + radeon_emit(0xffffffff); /* GCR_SIZE */ + radeon_emit(0x01ffffff); /* GCR_SIZE_HI */ + radeon_emit(0); /* GCR_BASE_LO */ + radeon_emit(0); /* GCR_BASE_HI */ + radeon_emit(S_585_PWS_ENA(1)); + radeon_emit(gcr_cntl); /* GCR_CNTL */ + + radeon_end(); gcr_cntl = 0; /* all done */ } else { @@ -310,9 +332,11 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level } } + radeon_begin(cs); + /* VGT state sync */ if (flush_bits & RADV_CMD_FLAG_VGT_FLUSH) { - radeon_event_write(cs, V_028A90_VGT_FLUSH); + radeon_event_write(V_028A90_VGT_FLUSH); } /* Ignore fields that only modify the behavior of other fields. */ @@ -321,37 +345,39 @@ gfx10_cs_emit_cache_flush(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level * The cache flush is executed in the ME, but the PFP waits * for completion. */ - radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 6, 0)); - radeon_emit(cs, 0); /* CP_COHER_CNTL */ - radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */ - radeon_emit(cs, 0xffffff); /* CP_COHER_SIZE_HI */ - radeon_emit(cs, 0); /* CP_COHER_BASE */ - radeon_emit(cs, 0); /* CP_COHER_BASE_HI */ - radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */ - radeon_emit(cs, gcr_cntl); /* GCR_CNTL */ + radeon_emit(PKT3(PKT3_ACQUIRE_MEM, 6, 0)); + radeon_emit(0); /* CP_COHER_CNTL */ + radeon_emit(0xffffffff); /* CP_COHER_SIZE */ + radeon_emit(0xffffff); /* CP_COHER_SIZE_HI */ + radeon_emit(0); /* CP_COHER_BASE */ + radeon_emit(0); /* CP_COHER_BASE_HI */ + radeon_emit(0x0000000A); /* POLL_INTERVAL */ + radeon_emit(gcr_cntl); /* GCR_CNTL */ } else if ((cb_db_event || (flush_bits & (RADV_CMD_FLAG_VS_PARTIAL_FLUSH | RADV_CMD_FLAG_PS_PARTIAL_FLUSH | RADV_CMD_FLAG_CS_PARTIAL_FLUSH))) && !is_mec) { /* We need to ensure that PFP waits as well. */ - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0)); + radeon_emit(0); *sqtt_flush_bits |= RGP_FLUSH_PFP_SYNC_ME; } if (flush_bits & RADV_CMD_FLAG_START_PIPELINE_STATS) { if (qf == RADV_QUEUE_GENERAL) { - radeon_event_write(cs, V_028A90_PIPELINESTAT_START); + radeon_event_write(V_028A90_PIPELINESTAT_START); } else if (qf == RADV_QUEUE_COMPUTE) { - radeon_set_sh_reg(cs, R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(1)); + radeon_set_sh_reg(R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(1)); } } else if (flush_bits & RADV_CMD_FLAG_STOP_PIPELINE_STATS) { if (qf == RADV_QUEUE_GENERAL) { - radeon_event_write(cs, V_028A90_PIPELINESTAT_STOP); + radeon_event_write(V_028A90_PIPELINESTAT_STOP); } else if (qf == RADV_QUEUE_COMPUTE) { - radeon_set_sh_reg(cs, R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(0)); + radeon_set_sh_reg(R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(0)); } } + + radeon_end(); } void @@ -405,29 +431,39 @@ radv_cs_emit_cache_flush(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, enu } if (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_CB_META) { - radeon_event_write(cs, V_028A90_FLUSH_AND_INV_CB_META); + radeon_begin(cs); + radeon_event_write(V_028A90_FLUSH_AND_INV_CB_META); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_FLUSH_CB | RGP_FLUSH_INVAL_CB; } if (flush_bits & RADV_CMD_FLAG_FLUSH_AND_INV_DB_META) { - radeon_event_write(cs, V_028A90_FLUSH_AND_INV_DB_META); + radeon_begin(cs); + radeon_event_write(V_028A90_FLUSH_AND_INV_DB_META); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_FLUSH_DB | RGP_FLUSH_INVAL_DB; } if (flush_bits & RADV_CMD_FLAG_PS_PARTIAL_FLUSH) { - radeon_event_write(cs, V_028A90_PS_PARTIAL_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_PS_PARTIAL_FLUSH); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_PS_PARTIAL_FLUSH; } else if (flush_bits & RADV_CMD_FLAG_VS_PARTIAL_FLUSH) { - radeon_event_write(cs, V_028A90_VS_PARTIAL_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_VS_PARTIAL_FLUSH); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_VS_PARTIAL_FLUSH; } if (flush_bits & RADV_CMD_FLAG_CS_PARTIAL_FLUSH) { - radeon_event_write(cs, V_028A90_CS_PARTIAL_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_CS_PARTIAL_FLUSH); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_CS_PARTIAL_FLUSH; } @@ -475,12 +511,16 @@ radv_cs_emit_cache_flush(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, enu /* VGT state sync */ if (flush_bits & RADV_CMD_FLAG_VGT_FLUSH) { - radeon_event_write(cs, V_028A90_VGT_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_VGT_FLUSH); + radeon_end(); } /* VGT streamout state sync */ if (flush_bits & RADV_CMD_FLAG_VGT_STREAMOUT_SYNC) { - radeon_event_write(cs, V_028A90_VGT_STREAMOUT_SYNC); + radeon_begin(cs); + radeon_event_write(V_028A90_VGT_STREAMOUT_SYNC); + radeon_end(); } /* Make sure ME is idle (it executes most packets) before continuing. @@ -489,8 +529,10 @@ radv_cs_emit_cache_flush(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, enu if ((cp_coher_cntl || (flush_bits & (RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | RADV_CMD_FLAG_INV_L2 | RADV_CMD_FLAG_WB_L2))) && !is_mec) { - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); - radeon_emit(cs, 0); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0)); + radeon_emit(0); + radeon_end(); *sqtt_flush_bits |= RGP_FLUSH_PFP_SYNC_ME; } @@ -530,19 +572,23 @@ radv_cs_emit_cache_flush(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, enu if (cp_coher_cntl) radv_emit_acquire_mem(cs, is_mec, gfx_level == GFX9, cp_coher_cntl); + radeon_begin(cs); + if (flush_bits & RADV_CMD_FLAG_START_PIPELINE_STATS) { if (qf == RADV_QUEUE_GENERAL) { - radeon_event_write(cs, V_028A90_PIPELINESTAT_START); + radeon_event_write(V_028A90_PIPELINESTAT_START); } else if (qf == RADV_QUEUE_COMPUTE) { - radeon_set_sh_reg(cs, R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(1)); + radeon_set_sh_reg(R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(1)); } } else if (flush_bits & RADV_CMD_FLAG_STOP_PIPELINE_STATS) { if (qf == RADV_QUEUE_GENERAL) { - radeon_event_write(cs, V_028A90_PIPELINESTAT_STOP); + radeon_event_write(V_028A90_PIPELINESTAT_STOP); } else if (qf == RADV_QUEUE_COMPUTE) { - radeon_set_sh_reg(cs, R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(0)); + radeon_set_sh_reg(R_00B828_COMPUTE_PIPELINESTAT_ENABLE, S_00B828_PIPELINESTAT_ENABLE(0)); } } + + radeon_end(); } void @@ -551,26 +597,32 @@ radv_emit_cond_exec(const struct radv_device *device, struct radeon_cmdbuf *cs, const struct radv_physical_device *pdev = radv_device_physical(device); const enum amd_gfx_level gfx_level = pdev->info.gfx_level; + radeon_begin(cs); + if (gfx_level >= GFX7) { - radeon_emit(cs, PKT3(PKT3_COND_EXEC, 3, 0)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, 0); - radeon_emit(cs, count); + radeon_emit(PKT3(PKT3_COND_EXEC, 3, 0)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(0); + radeon_emit(count); } else { - radeon_emit(cs, PKT3(PKT3_COND_EXEC, 2, 0)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, count); + radeon_emit(PKT3(PKT3_COND_EXEC, 2, 0)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(count); } + + radeon_end(); } void radv_cs_write_data_imm(struct radeon_cmdbuf *cs, unsigned engine_sel, uint64_t va, uint32_t imm) { - radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0)); - radeon_emit(cs, S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(engine_sel)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, imm); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_WRITE_DATA, 3, 0)); + radeon_emit(S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(engine_sel)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(imm); + radeon_end(); } diff --git a/src/amd/vulkan/radv_cs.h b/src/amd/vulkan/radv_cs.h index 697250b54ec..0b32521f701 100644 --- a/src/amd/vulkan/radv_cs.h +++ b/src/amd/vulkan/radv_cs.h @@ -27,34 +27,58 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned return cs->cdw + needed; } -/* Packet building helpers. Don't use directly. */ -#define radeon_set_reg_seq(cs, reg, num, idx, prefix_name, packet, reset_filter_cam) \ +#define radeon_begin(cs) \ + struct radeon_cmdbuf *__cs = (cs); \ + uint64_t __cs_num = __cs->cdw; \ + UNUSED uint64_t __cs_reserved_dw = __cs->reserved_dw; \ + uint32_t *__cs_buf = __cs->buf + +#define radeon_end() \ do { \ - assert((reg) >= prefix_name##_REG_OFFSET && (reg) < prefix_name##_REG_END); \ - assert(cs->cdw + 2 + num <= cs->reserved_dw); \ - radeon_emit(cs, PKT3(packet, num, 0) | PKT3_RESET_FILTER_CAM_S(reset_filter_cam)); \ - radeon_emit(cs, (((reg) - prefix_name##_REG_OFFSET) >> 2) | ((idx) << 28)); \ + __cs->cdw = __cs_num; \ + assert(__cs->cdw <= __cs->max_dw); \ + __cs = NULL; \ } while (0) -#define radeon_set_reg(cs, reg, idx, value, prefix_name, packet) \ +#define radeon_emit(value) \ do { \ - radeon_set_reg_seq(cs, reg, 1, idx, prefix_name, packet, 0); \ - radeon_emit(cs, value); \ + assert(__cs_num < __cs_reserved_dw); \ + __cs_buf[__cs_num++] = (value); \ + } while (0) + +#define radeon_emit_array(values, num) \ + do { \ + unsigned __n = (num); \ + assert(__cs_num + __n <= __cs_reserved_dw); \ + memcpy(__cs_buf + __cs_num, (values), __n * 4); \ + __cs_num += __n; \ + } while (0) + +/* Packet building helpers. Don't use directly. */ +#define __radeon_set_reg_seq(reg, num, idx, prefix_name, packet, reset_filter_cam) \ + do { \ + assert((reg) >= prefix_name##_REG_OFFSET && (reg) < prefix_name##_REG_END); \ + radeon_emit(PKT3(packet, num, 0) | PKT3_RESET_FILTER_CAM_S(reset_filter_cam)); \ + radeon_emit((((reg) - prefix_name##_REG_OFFSET) >> 2) | ((idx) << 28)); \ + } while (0) + +#define __radeon_set_reg(reg, idx, value, prefix_name, packet) \ + do { \ + __radeon_set_reg_seq(reg, 1, idx, prefix_name, packet, 0); \ + radeon_emit(value); \ } while (0) /* Packet building helpers for CONFIG registers. */ -#define radeon_set_config_reg_seq(cs, reg, num) radeon_set_reg_seq(cs, reg, num, 0, SI_CONFIG, PKT3_SET_CONFIG_REG, 0) +#define radeon_set_config_reg_seq(reg, num) __radeon_set_reg_seq(reg, num, 0, SI_CONFIG, PKT3_SET_CONFIG_REG, 0) -#define radeon_set_config_reg(cs, reg, value) radeon_set_reg(cs, reg, 0, value, SI_CONFIG, PKT3_SET_CONFIG_REG) +#define radeon_set_config_reg(reg, value) __radeon_set_reg(reg, 0, value, SI_CONFIG, PKT3_SET_CONFIG_REG) /* Packet building helpers for CONTEXT registers. */ -#define radeon_set_context_reg_seq(cs, reg, num) \ - radeon_set_reg_seq(cs, reg, num, 0, SI_CONTEXT, PKT3_SET_CONTEXT_REG, 0) +#define radeon_set_context_reg_seq(reg, num) __radeon_set_reg_seq(reg, num, 0, SI_CONTEXT, PKT3_SET_CONTEXT_REG, 0) -#define radeon_set_context_reg(cs, reg, value) radeon_set_reg(cs, reg, 0, value, SI_CONTEXT, PKT3_SET_CONTEXT_REG) +#define radeon_set_context_reg(reg, value) __radeon_set_reg(reg, 0, value, SI_CONTEXT, PKT3_SET_CONTEXT_REG) -#define radeon_set_context_reg_idx(cs, reg, idx, value) \ - radeon_set_reg(cs, reg, idx, value, SI_CONTEXT, PKT3_SET_CONTEXT_REG) +#define radeon_set_context_reg_idx(reg, idx, value) __radeon_set_reg(reg, idx, value, SI_CONTEXT, PKT3_SET_CONTEXT_REG) #define radeon_opt_set_context_reg(cmdbuf, reg, reg_enum, value) \ do { \ @@ -63,7 +87,7 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned const uint32_t __value = (value); \ if (!BITSET_TEST(__tracked_regs->reg_saved_mask, (reg_enum)) || \ __tracked_regs->reg_value[(reg_enum)] != __value) { \ - radeon_set_context_reg(__cmdbuf->cs, reg, __value); \ + radeon_set_context_reg(reg, __value); \ BITSET_SET(__tracked_regs->reg_saved_mask, (reg_enum)); \ __tracked_regs->reg_value[(reg_enum)] = __value; \ __cmdbuf->state.context_roll_without_scissor_emitted = true; \ @@ -77,9 +101,9 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned const uint32_t __v1 = (v1), __v2 = (v2); \ if (!BITSET_TEST_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 1, 0x3) || \ __tracked_regs->reg_value[(reg_enum)] != __v1 || __tracked_regs->reg_value[(reg_enum) + 1] != __v2) { \ - radeon_set_context_reg_seq(cmdbuf->cs, reg, 2); \ - radeon_emit(cmdbuf->cs, __v1); \ - radeon_emit(cmdbuf->cs, __v2); \ + radeon_set_context_reg_seq(reg, 2); \ + radeon_emit(__v1); \ + radeon_emit(__v2); \ BITSET_SET_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 1); \ __tracked_regs->reg_value[(reg_enum)] = __v1; \ __tracked_regs->reg_value[(reg_enum) + 1] = __v2; \ @@ -95,10 +119,10 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned if (!BITSET_TEST_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 2, 0x7) || \ __tracked_regs->reg_value[(reg_enum)] != __v1 || __tracked_regs->reg_value[(reg_enum) + 1] != __v2 || \ __tracked_regs->reg_value[(reg_enum) + 2] != __v3) { \ - radeon_set_context_reg_seq(cmdbuf->cs, reg, 3); \ - radeon_emit(cmdbuf->cs, __v1); \ - radeon_emit(cmdbuf->cs, __v2); \ - radeon_emit(cmdbuf->cs, __v3); \ + radeon_set_context_reg_seq(reg, 3); \ + radeon_emit(__v1); \ + radeon_emit(__v2); \ + radeon_emit(__v3); \ BITSET_SET_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 2); \ __tracked_regs->reg_value[(reg_enum)] = __v1; \ __tracked_regs->reg_value[(reg_enum) + 1] = __v2; \ @@ -115,11 +139,11 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned if (!BITSET_TEST_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 3, 0xf) || \ __tracked_regs->reg_value[(reg_enum)] != __v1 || __tracked_regs->reg_value[(reg_enum) + 1] != __v2 || \ __tracked_regs->reg_value[(reg_enum) + 2] != __v3 || __tracked_regs->reg_value[(reg_enum) + 3] != __v4) { \ - radeon_set_context_reg_seq(cmdbuf->cs, reg, 4); \ - radeon_emit(cmdbuf->cs, __v1); \ - radeon_emit(cmdbuf->cs, __v2); \ - radeon_emit(cmdbuf->cs, __v3); \ - radeon_emit(cmdbuf->cs, __v4); \ + radeon_set_context_reg_seq(reg, 4); \ + radeon_emit(__v1); \ + radeon_emit(__v2); \ + radeon_emit(__v3); \ + radeon_emit(__v4); \ BITSET_SET_RANGE_INSIDE_WORD(__tracked_regs->reg_saved_mask, (reg_enum), (reg_enum) + 3); \ __tracked_regs->reg_value[(reg_enum)] = __v1; \ __tracked_regs->reg_value[(reg_enum) + 1] = __v2; \ @@ -133,40 +157,39 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned do { \ struct radv_cmd_buffer *__cmdbuf = (cmdbuf); \ if (memcmp(values, saved_values, sizeof(uint32_t) * (num))) { \ - radeon_set_context_reg_seq(cmdbuf->cs, reg, num); \ - radeon_emit_array(cmdbuf->cs, values, num); \ + radeon_set_context_reg_seq(reg, num); \ + radeon_emit_array(values, num); \ memcpy(saved_values, values, sizeof(uint32_t) * (num)); \ __cmdbuf->state.context_roll_without_scissor_emitted = true; \ } \ } while (0) /* Packet building helpers for SH registers. */ -#define radeon_set_sh_reg_seq(cs, reg, num) radeon_set_reg_seq(cs, reg, num, 0, SI_SH, PKT3_SET_SH_REG, 0) +#define radeon_set_sh_reg_seq(reg, num) __radeon_set_reg_seq(reg, num, 0, SI_SH, PKT3_SET_SH_REG, 0) -#define radeon_set_sh_reg(cs, reg, value) radeon_set_reg(cs, reg, 0, value, SI_SH, PKT3_SET_SH_REG) +#define radeon_set_sh_reg(reg, value) __radeon_set_reg(reg, 0, value, SI_SH, PKT3_SET_SH_REG) -#define radeon_set_sh_reg_idx(info, cs, reg, idx, value) \ +#define radeon_set_sh_reg_idx(info, reg, idx, value) \ do { \ assert((idx)); \ unsigned __opcode = PKT3_SET_SH_REG_INDEX; \ if ((info)->gfx_level < GFX10) \ __opcode = PKT3_SET_SH_REG; \ - radeon_set_reg(cs, reg, idx, value, SI_SH, __opcode); \ + __radeon_set_reg(reg, idx, value, SI_SH, __opcode); \ } while (0) /* Packet building helpers for UCONFIG registers. */ -#define radeon_set_uconfig_reg_seq(cs, reg, num) \ - radeon_set_reg_seq(cs, reg, num, 0, CIK_UCONFIG, PKT3_SET_UCONFIG_REG, 0) +#define radeon_set_uconfig_reg_seq(reg, num) __radeon_set_reg_seq(reg, num, 0, CIK_UCONFIG, PKT3_SET_UCONFIG_REG, 0) -#define radeon_set_uconfig_reg(cs, reg, value) radeon_set_reg(cs, reg, 0, value, CIK_UCONFIG, PKT3_SET_UCONFIG_REG) +#define radeon_set_uconfig_reg(reg, value) __radeon_set_reg(reg, 0, value, CIK_UCONFIG, PKT3_SET_UCONFIG_REG) -#define radeon_set_uconfig_reg_idx(info, cs, reg, idx, value) \ +#define radeon_set_uconfig_reg_idx(info, reg, idx, value) \ do { \ assert((idx)); \ unsigned __opcode = PKT3_SET_UCONFIG_REG_INDEX; \ if ((info)->gfx_level < GFX9 || ((info)->gfx_level == GFX9 && (info)->me_fw_version < 26)) \ __opcode = PKT3_SET_UCONFIG_REG; \ - radeon_set_reg(cs, reg, idx, value, CIK_UCONFIG, __opcode); \ + __radeon_set_reg(reg, idx, value, CIK_UCONFIG, __opcode); \ } while (0) /* @@ -174,55 +197,54 @@ radeon_check_space(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, unsigned * that means that it can skip register writes due to not taking correctly into account the * fields from the GRBM_GFX_INDEX. With this bit we can force the write. */ -#define radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, reg, num) \ +#define radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, reg, num) \ do { \ const bool __filter_cam_workaround = (gfx_level) >= GFX10 && (ring) == AMD_IP_GFX; \ - radeon_set_reg_seq(cs, reg, num, 0, CIK_UCONFIG, PKT3_SET_UCONFIG_REG, __filter_cam_workaround); \ + __radeon_set_reg_seq(reg, num, 0, CIK_UCONFIG, PKT3_SET_UCONFIG_REG, __filter_cam_workaround); \ } while (0) -#define radeon_set_uconfig_perfctr_reg(gfx_level, ring, cs, reg, value) \ +#define radeon_set_uconfig_perfctr_reg(gfx_level, ring, reg, value) \ do { \ - radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, reg, 1); \ - radeon_emit(cs, value); \ + radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, reg, 1); \ + radeon_emit(value); \ } while (0) -#define radeon_set_privileged_config_reg(cs, reg, value) \ +#define radeon_set_privileged_config_reg(reg, value) \ do { \ assert((reg) < CIK_UCONFIG_REG_OFFSET); \ - assert((cs)->cdw + 6 <= (cs)->reserved_dw); \ - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); \ - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_PERF)); \ - radeon_emit(cs, value); \ - radeon_emit(cs, 0); /* unused */ \ - radeon_emit(cs, (reg) >> 2); \ - radeon_emit(cs, 0); /* unused */ \ + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); \ + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_PERF)); \ + radeon_emit(value); \ + radeon_emit(0); /* unused */ \ + radeon_emit((reg) >> 2); \ + radeon_emit(0); /* unused */ \ } while (0) -#define radeon_event_write_predicate(cs, event_type, predicate) \ +#define radeon_event_write_predicate(event_type, predicate) \ do { \ unsigned __event_type = (event_type); \ - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, predicate)); \ - radeon_emit(cs, EVENT_TYPE(__event_type) | EVENT_INDEX(__event_type == V_028A90_VS_PARTIAL_FLUSH || \ - __event_type == V_028A90_PS_PARTIAL_FLUSH || \ - __event_type == V_028A90_CS_PARTIAL_FLUSH \ - ? 4 \ - : 0)); \ + radeon_emit(PKT3(PKT3_EVENT_WRITE, 0, predicate)); \ + radeon_emit(EVENT_TYPE(__event_type) | EVENT_INDEX(__event_type == V_028A90_VS_PARTIAL_FLUSH || \ + __event_type == V_028A90_PS_PARTIAL_FLUSH || \ + __event_type == V_028A90_CS_PARTIAL_FLUSH \ + ? 4 \ + : 0)); \ } while (0) -#define radeon_event_write(cs, event_type) radeon_event_write_predicate(cs, event_type, false) +#define radeon_event_write(event_type) radeon_event_write_predicate(event_type, false) -#define radeon_emit_32bit_pointer(cs, sh_offset, va, info) \ +#define radeon_emit_32bit_pointer(sh_offset, va, info) \ do { \ assert((va) == 0 || ((va) >> 32) == (info)->address32_hi); \ - radeon_set_sh_reg_seq(cs, sh_offset, 1); \ - radeon_emit(cs, va); \ + radeon_set_sh_reg_seq(sh_offset, 1); \ + radeon_emit(va); \ } while (0) -#define radeon_emit_64bit_pointer(cs, sh_offset, va) \ +#define radeon_emit_64bit_pointer(sh_offset, va) \ do { \ - radeon_set_sh_reg_seq(cs, sh_offset, 2); \ - radeon_emit(cs, va); \ - radeon_emit(cs, va >> 32); \ + radeon_set_sh_reg_seq(sh_offset, 2); \ + radeon_emit(va); \ + radeon_emit(va >> 32); \ } while (0) ALWAYS_INLINE static void @@ -232,13 +254,15 @@ radv_cp_wait_mem(struct radeon_cmdbuf *cs, const enum radv_queue_family qf, cons assert(op == WAIT_REG_MEM_EQUAL || op == WAIT_REG_MEM_NOT_EQUAL || op == WAIT_REG_MEM_GREATER_OR_EQUAL); if (qf == RADV_QUEUE_GENERAL || qf == RADV_QUEUE_COMPUTE) { - radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, false)); - radeon_emit(cs, op | WAIT_REG_MEM_MEM_SPACE(1)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, ref); /* reference value */ - radeon_emit(cs, mask); /* mask */ - radeon_emit(cs, 4); /* poll interval */ + radeon_begin(cs); + radeon_emit(PKT3(PKT3_WAIT_REG_MEM, 5, false)); + radeon_emit(op | WAIT_REG_MEM_MEM_SPACE(1)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(ref); /* reference value */ + radeon_emit(mask); /* mask */ + radeon_emit(4); /* poll interval */ + radeon_end(); } else if (qf == RADV_QUEUE_TRANSFER) { radv_sdma_emit_wait_mem(cs, op, va, ref, mask); } else { @@ -254,10 +278,12 @@ radv_cs_write_data_head(const struct radv_device *device, struct radeon_cmdbuf * const unsigned cdw_end = radeon_check_space(device->ws, cs, 4 + count); if (qf == RADV_QUEUE_GENERAL || qf == RADV_QUEUE_COMPUTE) { - radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + count, predicating)); - radeon_emit(cs, S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(engine_sel)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_WRITE_DATA, 2 + count, predicating)); + radeon_emit(S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(engine_sel)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_end(); } else if (qf == RADV_QUEUE_TRANSFER) { radv_sdma_emit_write_data_head(cs, va, count); } else { @@ -273,7 +299,10 @@ radv_cs_write_data(const struct radv_device *device, struct radeon_cmdbuf *cs, c const bool predicating) { ASSERTED const unsigned cdw_end = radv_cs_write_data_head(device, cs, qf, engine_sel, va, count, predicating); - radeon_emit_array(cs, dwords, count); + + radeon_begin(cs); + radeon_emit_array(dwords, count); + radeon_end(); assert(cs->cdw == cdw_end); } @@ -293,7 +322,9 @@ void radv_cs_write_data_imm(struct radeon_cmdbuf *cs, unsigned engine_sel, uint6 static inline void radv_emit_pm4_commands(struct radeon_cmdbuf *cs, const struct ac_pm4_state *pm4) { - radeon_emit_array(cs, pm4->pm4, pm4->ndw); + radeon_begin(cs); + radeon_emit_array(pm4->pm4, pm4->ndw); + radeon_end(); } #endif /* RADV_CS_H */ diff --git a/src/amd/vulkan/radv_device.c b/src/amd/vulkan/radv_device.c index 5fb48b01825..df24e89ecf3 100644 --- a/src/amd/vulkan/radv_device.c +++ b/src/amd/vulkan/radv_device.c @@ -972,40 +972,42 @@ radv_emit_default_sample_locations(const struct radv_physical_device *pdev, stru { uint64_t centroid_priority; + radeon_begin(cs); + switch (nr_samples) { default: case 1: centroid_priority = centroid_priority_1x; - radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_1x); - radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_1x); - radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_1x); - radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_1x); + radeon_set_context_reg(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_1x); + radeon_set_context_reg(R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_1x); + radeon_set_context_reg(R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_1x); + radeon_set_context_reg(R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_1x); break; case 2: centroid_priority = centroid_priority_2x; - radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_2x); - radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_2x); - radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_2x); - radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_2x); + radeon_set_context_reg(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_2x); + radeon_set_context_reg(R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_2x); + radeon_set_context_reg(R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_2x); + radeon_set_context_reg(R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_2x); break; case 4: centroid_priority = centroid_priority_4x; - radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_4x); - radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_4x); - radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_4x); - radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_4x); + radeon_set_context_reg(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_4x); + radeon_set_context_reg(R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_4x); + radeon_set_context_reg(R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_4x); + radeon_set_context_reg(R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_4x); break; case 8: centroid_priority = centroid_priority_8x; - radeon_set_context_reg_seq(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, 14); - radeon_emit_array(cs, sample_locs_8x, 4); - radeon_emit_array(cs, sample_locs_8x, 4); - radeon_emit_array(cs, sample_locs_8x, 4); - radeon_emit_array(cs, sample_locs_8x, 2); + radeon_set_context_reg_seq(R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, 14); + radeon_emit_array(sample_locs_8x, 4); + radeon_emit_array(sample_locs_8x, 4); + radeon_emit_array(sample_locs_8x, 4); + radeon_emit_array(sample_locs_8x, 2); break; } @@ -1014,17 +1016,19 @@ radv_emit_default_sample_locations(const struct radv_physical_device *pdev, stru * support 16 samples. */ if (pdev->info.gfx_level >= GFX7) { - radeon_set_context_reg(cs, R_02882C_PA_SU_PRIM_FILTER_CNTL, + radeon_set_context_reg(R_02882C_PA_SU_PRIM_FILTER_CNTL, S_02882C_XMAX_RIGHT_EXCLUSION(1) | S_02882C_YMAX_BOTTOM_EXCLUSION(1)); } if (pdev->info.gfx_level >= GFX12) { - radeon_set_context_reg_seq(cs, R_028BF0_PA_SC_CENTROID_PRIORITY_0, 2); + radeon_set_context_reg_seq(R_028BF0_PA_SC_CENTROID_PRIORITY_0, 2); } else { - radeon_set_context_reg_seq(cs, R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2); + radeon_set_context_reg_seq(R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2); } - radeon_emit(cs, centroid_priority); - radeon_emit(cs, centroid_priority >> 32); + radeon_emit(centroid_priority); + radeon_emit(centroid_priority >> 32); + + radeon_end(); } static void diff --git a/src/amd/vulkan/radv_perfcounter.c b/src/amd/vulkan/radv_perfcounter.c index 3d5f7721703..1e5eb70b5f8 100644 --- a/src/amd/vulkan/radv_perfcounter.c +++ b/src/amd/vulkan/radv_perfcounter.c @@ -19,25 +19,33 @@ radv_perfcounter_emit_shaders(struct radv_device *device, struct radeon_cmdbuf * { const struct radv_physical_device *pdev = radv_device_physical(device); + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX10) { - radeon_set_uconfig_reg(cs, R_036780_SQ_PERFCOUNTER_CTRL, shaders & 0x7f); + radeon_set_uconfig_reg(R_036780_SQ_PERFCOUNTER_CTRL, shaders & 0x7f); if (pdev->info.gfx_level >= GFX11) - radeon_set_uconfig_reg(cs, R_036760_SQG_PERFCOUNTER_CTRL, shaders & 0x7f); + radeon_set_uconfig_reg(R_036760_SQG_PERFCOUNTER_CTRL, shaders & 0x7f); } else { - radeon_set_uconfig_reg_seq(cs, R_036780_SQ_PERFCOUNTER_CTRL, 2); - radeon_emit(cs, shaders & 0x7f); - radeon_emit(cs, 0xffffffff); + radeon_set_uconfig_reg_seq(R_036780_SQ_PERFCOUNTER_CTRL, 2); + radeon_emit(shaders & 0x7f); + radeon_emit(0xffffffff); } + + radeon_end(); } static void radv_emit_windowed_counters(struct radv_device *device, struct radeon_cmdbuf *cs, int family, bool enable) { + radeon_begin(cs); + if (family == RADV_QUEUE_GENERAL) { - radeon_event_write(cs, enable ? V_028A90_PERFCOUNTER_START : V_028A90_PERFCOUNTER_STOP); + radeon_event_write(enable ? V_028A90_PERFCOUNTER_START : V_028A90_PERFCOUNTER_STOP); } - radeon_set_sh_reg(cs, R_00B82C_COMPUTE_PERFCOUNT_ENABLE, S_00B82C_PERFCOUNT_ENABLE(enable)); + radeon_set_sh_reg(R_00B82C_COMPUTE_PERFCOUNT_ENABLE, S_00B82C_PERFCOUNT_ENABLE(enable)); + + radeon_end(); } void @@ -52,7 +60,9 @@ radv_perfcounter_emit_reset(struct radeon_cmdbuf *cs, bool is_spm) cp_perfmon_cntl = S_036020_PERFMON_STATE(V_036020_CP_PERFMON_STATE_DISABLE_AND_RESET); } - radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl); + radeon_begin(cs); + radeon_set_uconfig_reg(R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl); + radeon_end(); } static void @@ -67,7 +77,9 @@ radv_perfcounter_emit_start(struct radeon_cmdbuf *cs, bool is_spm) cp_perfmon_cntl = S_036020_PERFMON_STATE(V_036020_CP_PERFMON_STATE_START_COUNTING); } - radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl); + radeon_begin(cs); + radeon_set_uconfig_reg(R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl); + radeon_end(); } static void @@ -83,7 +95,9 @@ radv_perfcounter_emit_stop(struct radeon_cmdbuf *cs, bool is_spm) S_036020_PERFMON_STATE(V_036020_CP_PERFMON_STATE_STOP_COUNTING) | S_036020_PERFMON_SAMPLE_ENABLE(1); } - radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl); + radeon_begin(cs); + radeon_set_uconfig_reg(R_036020_CP_PERFMON_CNTL, cp_perfmon_cntl); + radeon_end(); } void @@ -113,7 +127,9 @@ radv_perfcounter_emit_spm_stop(struct radv_device *device, struct radeon_cmdbuf static void radv_perfcounter_emit_sample(struct radeon_cmdbuf *cs) { - radeon_event_write(cs, V_028A90_PERFCOUNTER_SAMPLE); + radeon_begin(cs); + radeon_event_write(V_028A90_PERFCOUNTER_SAMPLE); + radeon_end(); } enum radv_perfcounter_op { @@ -492,7 +508,9 @@ radv_emit_instance(struct radv_cmd_buffer *cmd_buffer, int se, int instance) value |= S_030800_INSTANCE_BROADCAST_WRITES(1); } - radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, value); + radeon_begin(cs); + radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, value); + radeon_end(); } static void @@ -512,15 +530,18 @@ radv_emit_select(struct radv_cmd_buffer *cmd_buffer, struct ac_pc_block *block, if (!regs->select0) return; + radeon_begin(cs); + for (idx = 0; idx < count; ++idx) { - radeon_set_uconfig_perfctr_reg(gfx_level, ring, cs, regs->select0[idx], - G_REG_SEL(selectors[idx]) | regs->select_or); + radeon_set_uconfig_perfctr_reg(gfx_level, ring, regs->select0[idx], G_REG_SEL(selectors[idx]) | regs->select_or); } for (idx = 0; idx < regs->num_spm_counters; idx++) { - radeon_set_uconfig_reg_seq(cs, regs->select1[idx], 1); - radeon_emit(cs, 0); + radeon_set_uconfig_reg_seq(regs->select1[idx], 1); + radeon_emit(0); } + + radeon_end(); } static void @@ -539,13 +560,15 @@ radv_pc_emit_block_instance_read(struct radv_cmd_buffer *cmd_buffer, struct ac_p if (regs->counters) reg = regs->counters[idx]; - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) | COPY_DATA_DST_SEL(COPY_DATA_TC_L2) | COPY_DATA_WR_CONFIRM | - COPY_DATA_COUNT_SEL); /* 64 bits */ - radeon_emit(cs, reg >> 2); - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_PERF) | COPY_DATA_DST_SEL(COPY_DATA_TC_L2) | COPY_DATA_WR_CONFIRM | + COPY_DATA_COUNT_SEL); /* 64 bits */ + radeon_emit(reg >> 2); + radeon_emit(0); /* unused */ + radeon_emit(va); + radeon_emit(va >> 32); + radeon_end(); va += sizeof(uint64_t) * 2 * radv_pc_get_num_instances(pdev, block); reg += reg_delta; @@ -575,19 +598,23 @@ radv_pc_wait_idle(struct radv_cmd_buffer *cmd_buffer) { struct radeon_cmdbuf *cs = cmd_buffer->cs; - radeon_event_write(cs, V_028A90_CS_PARTIAL_FLUSH); + radeon_begin(cs); - radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 6, 0)); - radeon_emit(cs, 0); /* CP_COHER_CNTL */ - radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */ - radeon_emit(cs, 0xffffff); /* CP_COHER_SIZE_HI */ - radeon_emit(cs, 0); /* CP_COHER_BASE */ - radeon_emit(cs, 0); /* CP_COHER_BASE_HI */ - radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */ - radeon_emit(cs, 0); /* GCR_CNTL */ + radeon_event_write(V_028A90_CS_PARTIAL_FLUSH); - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_ACQUIRE_MEM, 6, 0)); + radeon_emit(0); /* CP_COHER_CNTL */ + radeon_emit(0xffffffff); /* CP_COHER_SIZE */ + radeon_emit(0xffffff); /* CP_COHER_SIZE_HI */ + radeon_emit(0); /* CP_COHER_BASE */ + radeon_emit(0); /* CP_COHER_BASE_HI */ + radeon_emit(0x0000000A); /* POLL_INTERVAL */ + radeon_emit(0); /* GCR_CNTL */ + + radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0)); + radeon_emit(0); + + radeon_end(); } static void @@ -607,11 +634,13 @@ radv_pc_stop_and_sample(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query uint64_t pred_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_PASS_OFFSET + 8 * pass; uint64_t reg_va = va + (end ? 8 : 0); - radeon_emit(cs, PKT3(PKT3_COND_EXEC, 3, 0)); - radeon_emit(cs, pred_va); - radeon_emit(cs, pred_va >> 32); - radeon_emit(cs, 0); /* Cache policy */ - radeon_emit(cs, 0); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COND_EXEC, 3, 0)); + radeon_emit(pred_va); + radeon_emit(pred_va >> 32); + radeon_emit(0); /* Cache policy */ + radeon_emit(0); + radeon_end(); uint32_t *skip_dwords = cs->buf + (cs->cdw - 1); @@ -677,11 +706,13 @@ radv_pc_begin_query(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query_poo for (unsigned pass = 0; pass < pool->num_passes; ++pass) { uint64_t pred_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_PASS_OFFSET + 8 * pass; - radeon_emit(cs, PKT3(PKT3_COND_EXEC, 3, 0)); - radeon_emit(cs, pred_va); - radeon_emit(cs, pred_va >> 32); - radeon_emit(cs, 0); /* Cache policy */ - radeon_emit(cs, 0); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COND_EXEC, 3, 0)); + radeon_emit(pred_va); + radeon_emit(pred_va >> 32); + radeon_emit(0); /* Cache policy */ + radeon_emit(0); + radeon_end(); uint32_t *skip_dwords = cs->buf + (cs->cdw - 1); diff --git a/src/amd/vulkan/radv_query.c b/src/amd/vulkan/radv_query.c index a31eb93214a..6a29af3b6f7 100644 --- a/src/amd/vulkan/radv_query.c +++ b/src/amd/vulkan/radv_query.c @@ -38,12 +38,14 @@ static void radv_query_shader(struct radv_cmd_buffer *cmd_buffer, VkQueryType qu static void gfx10_copy_shader_query(struct radeon_cmdbuf *cs, uint32_t src_sel, uint64_t src_va, uint64_t dst_va) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(src_sel) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, src_va); - radeon_emit(cs, src_va >> 32); - radeon_emit(cs, dst_va); - radeon_emit(cs, dst_va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(src_sel) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_WR_CONFIRM); + radeon_emit(src_va); + radeon_emit(src_va >> 32); + radeon_emit(dst_va); + radeon_emit(dst_va >> 32); + radeon_end(); } static void @@ -95,18 +97,20 @@ static void radv_emit_event_write(const struct radeon_info *info, struct radeon_cmdbuf *cs, enum radv_event_write event, uint64_t va) { + radeon_begin(cs); + if (event == RADV_EVENT_WRITE_PIPELINE_STAT) { - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0)); - radeon_emit(cs, EVENT_TYPE(V_028A90_SAMPLE_PIPELINESTAT) | EVENT_INDEX(2)); + radeon_emit(PKT3(PKT3_EVENT_WRITE, 2, 0)); + radeon_emit(EVENT_TYPE(V_028A90_SAMPLE_PIPELINESTAT) | EVENT_INDEX(2)); } else if (event == RADV_EVENT_WRITE_OCCLUSION_QUERY) { if (info->gfx_level >= GFX11 && info->pfp_fw_version >= EVENT_WRITE_ZPASS_PFP_VERSION) { - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_ZPASS, 1, 0)); + radeon_emit(PKT3(PKT3_EVENT_WRITE_ZPASS, 1, 0)); } else { - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0)); + radeon_emit(PKT3(PKT3_EVENT_WRITE, 2, 0)); if (info->gfx_level >= GFX11) { - radeon_emit(cs, EVENT_TYPE(V_028A90_PIXEL_PIPE_STAT_DUMP) | EVENT_INDEX(1)); + radeon_emit(EVENT_TYPE(V_028A90_PIXEL_PIPE_STAT_DUMP) | EVENT_INDEX(1)); } else { - radeon_emit(cs, EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1)); + radeon_emit(EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1)); } } } else { @@ -119,12 +123,14 @@ radv_emit_event_write(const struct radeon_info *info, struct radeon_cmdbuf *cs, V_028A90_SAMPLE_STREAMOUTSTATS3, }; - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0)); - radeon_emit(cs, EVENT_TYPE(streamout_events[event]) | EVENT_INDEX(3)); + radeon_emit(PKT3(PKT3_EVENT_WRITE, 2, 0)); + radeon_emit(EVENT_TYPE(streamout_events[event]) | EVENT_INDEX(3)); } - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_emit(va); + radeon_emit(va >> 32); + + radeon_end(); } static void @@ -2706,13 +2712,15 @@ radv_write_timestamp(struct radv_cmd_buffer *cmd_buffer, uint64_t va, VkPipeline struct radeon_cmdbuf *cs = cmd_buffer->cs; if (stage == VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_COUNT_SEL | COPY_DATA_WR_CONFIRM | COPY_DATA_SRC_SEL(COPY_DATA_TIMESTAMP) | - COPY_DATA_DST_SEL(V_370_MEM)); - radeon_emit(cs, 0); - radeon_emit(cs, 0); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_begin(cs); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_COUNT_SEL | COPY_DATA_WR_CONFIRM | COPY_DATA_SRC_SEL(COPY_DATA_TIMESTAMP) | + COPY_DATA_DST_SEL(V_370_MEM)); + radeon_emit(0); + radeon_emit(0); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_end(); } else { radv_cs_emit_write_event_eop(cs, pdev->info.gfx_level, cmd_buffer->qf, V_028A90_BOTTOM_OF_PIPE_TS, 0, EOP_DST_SEL_MEM, EOP_DATA_SEL_TIMESTAMP, va, 0, cmd_buffer->gfx9_eop_bug_va); @@ -2791,6 +2799,8 @@ radv_CmdWriteAccelerationStructuresPropertiesKHR(VkCommandBuffer commandBuffer, ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cs, 6 * accelerationStructureCount); + radeon_begin(cs); + for (uint32_t i = 0; i < accelerationStructureCount; ++i) { VK_FROM_HANDLE(vk_acceleration_structure, accel_struct, pAccelerationStructures[i]); uint64_t va = vk_acceleration_structure_get_va(accel_struct); @@ -2812,16 +2822,17 @@ radv_CmdWriteAccelerationStructuresPropertiesKHR(VkCommandBuffer commandBuffer, unreachable("Unhandle accel struct query type."); } - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | - COPY_DATA_COUNT_SEL | COPY_DATA_WR_CONFIRM); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, query_va); - radeon_emit(cs, query_va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL | + COPY_DATA_WR_CONFIRM); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(query_va); + radeon_emit(query_va >> 32); query_va += pool->stride; } + radeon_end(); assert(cmd_buffer->cs->cdw <= cdw_max); } diff --git a/src/amd/vulkan/radv_queue.c b/src/amd/vulkan/radv_queue.c index d5123799bea..b42fbb64201 100644 --- a/src/amd/vulkan/radv_queue.c +++ b/src/amd/vulkan/radv_queue.c @@ -373,15 +373,19 @@ radv_emit_gs_ring_sizes(struct radv_device *device, struct radeon_cmdbuf *cs, st if (gsvs_ring_bo) radv_cs_add_buffer(device->ws, cs, gsvs_ring_bo); + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX7) { - radeon_set_uconfig_reg_seq(cs, R_030900_VGT_ESGS_RING_SIZE, 2); - radeon_emit(cs, esgs_ring_size >> 8); - radeon_emit(cs, gsvs_ring_size >> 8); + radeon_set_uconfig_reg_seq(R_030900_VGT_ESGS_RING_SIZE, 2); + radeon_emit(esgs_ring_size >> 8); + radeon_emit(gsvs_ring_size >> 8); } else { - radeon_set_config_reg_seq(cs, R_0088C8_VGT_ESGS_RING_SIZE, 2); - radeon_emit(cs, esgs_ring_size >> 8); - radeon_emit(cs, gsvs_ring_size >> 8); + radeon_set_config_reg_seq(R_0088C8_VGT_ESGS_RING_SIZE, 2); + radeon_emit(esgs_ring_size >> 8); + radeon_emit(gsvs_ring_size >> 8); } + + radeon_end(); } static void @@ -398,29 +402,33 @@ radv_emit_tess_factor_ring(struct radv_device *device, struct radeon_cmdbuf *cs, radv_cs_add_buffer(device->ws, cs, tess_rings_bo); + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX7) { if (pdev->info.gfx_level >= GFX11) { /* TF_RING_SIZE is per SE on GFX11. */ tf_ring_size /= pdev->info.max_se; } - radeon_set_uconfig_reg(cs, R_030938_VGT_TF_RING_SIZE, S_030938_SIZE(tf_ring_size)); - radeon_set_uconfig_reg(cs, R_030940_VGT_TF_MEMORY_BASE, tf_va >> 8); + radeon_set_uconfig_reg(R_030938_VGT_TF_RING_SIZE, S_030938_SIZE(tf_ring_size)); + radeon_set_uconfig_reg(R_030940_VGT_TF_MEMORY_BASE, tf_va >> 8); if (pdev->info.gfx_level >= GFX12) { - radeon_set_uconfig_reg(cs, R_03099C_VGT_TF_MEMORY_BASE_HI, S_03099C_BASE_HI(tf_va >> 40)); + radeon_set_uconfig_reg(R_03099C_VGT_TF_MEMORY_BASE_HI, S_03099C_BASE_HI(tf_va >> 40)); } else if (pdev->info.gfx_level >= GFX10) { - radeon_set_uconfig_reg(cs, R_030984_VGT_TF_MEMORY_BASE_HI, S_030984_BASE_HI(tf_va >> 40)); + radeon_set_uconfig_reg(R_030984_VGT_TF_MEMORY_BASE_HI, S_030984_BASE_HI(tf_va >> 40)); } else if (pdev->info.gfx_level == GFX9) { - radeon_set_uconfig_reg(cs, R_030944_VGT_TF_MEMORY_BASE_HI, S_030944_BASE_HI(tf_va >> 40)); + radeon_set_uconfig_reg(R_030944_VGT_TF_MEMORY_BASE_HI, S_030944_BASE_HI(tf_va >> 40)); } - radeon_set_uconfig_reg(cs, R_03093C_VGT_HS_OFFCHIP_PARAM, pdev->hs.hs_offchip_param); + radeon_set_uconfig_reg(R_03093C_VGT_HS_OFFCHIP_PARAM, pdev->hs.hs_offchip_param); } else { - radeon_set_config_reg(cs, R_008988_VGT_TF_RING_SIZE, S_008988_SIZE(tf_ring_size)); - radeon_set_config_reg(cs, R_0089B8_VGT_TF_MEMORY_BASE, tf_va >> 8); - radeon_set_config_reg(cs, R_0089B0_VGT_HS_OFFCHIP_PARAM, pdev->hs.hs_offchip_param); + radeon_set_config_reg(R_008988_VGT_TF_RING_SIZE, S_008988_SIZE(tf_ring_size)); + radeon_set_config_reg(R_0089B8_VGT_TF_MEMORY_BASE, tf_va >> 8); + radeon_set_config_reg(R_0089B0_VGT_HS_OFFCHIP_PARAM, pdev->hs.hs_offchip_param); } + + radeon_end(); } static VkResult @@ -466,12 +474,16 @@ radv_emit_task_rings(struct radv_device *device, struct radeon_cmdbuf *cs, struc assert(util_is_aligned(task_ctrlbuf_va, 256)); radv_cs_add_buffer(device->ws, cs, task_rings_bo); + radeon_begin(cs); + /* Tell the GPU where the task control buffer is. */ - radeon_emit(cs, PKT3(PKT3_DISPATCH_TASK_STATE_INIT, 1, 0) | PKT3_SHADER_TYPE_S(!!compute)); + radeon_emit(PKT3(PKT3_DISPATCH_TASK_STATE_INIT, 1, 0) | PKT3_SHADER_TYPE_S(!!compute)); /* bits [31:8]: control buffer address lo, bits[7:0]: reserved (set to zero) */ - radeon_emit(cs, task_ctrlbuf_va & 0xFFFFFF00); + radeon_emit(task_ctrlbuf_va & 0xFFFFFF00); /* bits [31:0]: control buffer address hi */ - radeon_emit(cs, task_ctrlbuf_va >> 32); + radeon_emit(task_ctrlbuf_va >> 32); + + radeon_end(); } static void @@ -486,20 +498,24 @@ radv_emit_graphics_scratch(struct radv_device *device, struct radeon_cmdbuf *cs, radv_cs_add_buffer(device->ws, cs, scratch_bo); + radeon_begin(cs); + if (gpu_info->gfx_level >= GFX11) { uint64_t va = radv_buffer_get_va(scratch_bo); /* WAVES is per SE for SPI_TMPRING_SIZE. */ waves /= gpu_info->max_se; - radeon_set_context_reg_seq(cs, R_0286E8_SPI_TMPRING_SIZE, 3); - radeon_emit(cs, S_0286E8_WAVES(waves) | S_0286E8_WAVESIZE(DIV_ROUND_UP(size_per_wave, 256))); - radeon_emit(cs, va >> 8); /* SPI_GFX_SCRATCH_BASE_LO */ - radeon_emit(cs, va >> 40); /* SPI_GFX_SCRATCH_BASE_HI */ + radeon_set_context_reg_seq(R_0286E8_SPI_TMPRING_SIZE, 3); + radeon_emit(S_0286E8_WAVES(waves) | S_0286E8_WAVESIZE(DIV_ROUND_UP(size_per_wave, 256))); + radeon_emit(va >> 8); /* SPI_GFX_SCRATCH_BASE_LO */ + radeon_emit(va >> 40); /* SPI_GFX_SCRATCH_BASE_HI */ } else { - radeon_set_context_reg(cs, R_0286E8_SPI_TMPRING_SIZE, + radeon_set_context_reg(R_0286E8_SPI_TMPRING_SIZE, S_0286E8_WAVES(waves) | S_0286E8_WAVESIZE(DIV_ROUND_UP(size_per_wave, 1024))); } + + radeon_end(); } static void @@ -524,21 +540,25 @@ radv_emit_compute_scratch(struct radv_device *device, struct radeon_cmdbuf *cs, radv_cs_add_buffer(device->ws, cs, compute_scratch_bo); + radeon_begin(cs); + if (gpu_info->gfx_level >= GFX11) { - radeon_set_sh_reg_seq(cs, R_00B840_COMPUTE_DISPATCH_SCRATCH_BASE_LO, 2); - radeon_emit(cs, scratch_va >> 8); - radeon_emit(cs, scratch_va >> 40); + radeon_set_sh_reg_seq(R_00B840_COMPUTE_DISPATCH_SCRATCH_BASE_LO, 2); + radeon_emit(scratch_va >> 8); + radeon_emit(scratch_va >> 40); waves /= gpu_info->max_se; } - radeon_set_sh_reg_seq(cs, R_00B900_COMPUTE_USER_DATA_0, 2); - radeon_emit(cs, scratch_va); - radeon_emit(cs, rsrc1); + radeon_set_sh_reg_seq(R_00B900_COMPUTE_USER_DATA_0, 2); + radeon_emit(scratch_va); + radeon_emit(rsrc1); - radeon_set_sh_reg(cs, R_00B860_COMPUTE_TMPRING_SIZE, + radeon_set_sh_reg(R_00B860_COMPUTE_TMPRING_SIZE, S_00B860_WAVES(waves) | S_00B860_WAVESIZE(DIV_ROUND_UP(size_per_wave, gpu_info->gfx_level >= GFX11 ? 256 : 1024))); + + radeon_end(); } static void @@ -554,7 +574,9 @@ radv_emit_compute_shader_pointers(struct radv_device *device, struct radeon_cmdb /* Compute shader user data 0-1 have the scratch pointer (unlike GFX shaders), * so emit the descriptor pointer to user data 2-3 instead (task_ring_offsets arg). */ - radeon_emit_64bit_pointer(cs, R_00B908_COMPUTE_USER_DATA_2, va); + radeon_begin(cs); + radeon_emit_64bit_pointer(R_00B908_COMPUTE_USER_DATA_2, va); + radeon_end(); } static void @@ -571,33 +593,35 @@ radv_emit_graphics_shader_pointers(struct radv_device *device, struct radeon_cmd radv_cs_add_buffer(device->ws, cs, descriptor_bo); + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX12) { uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B410_SPI_SHADER_PGM_LO_HS, R_00B210_SPI_SHADER_PGM_LO_GS}; for (int i = 0; i < ARRAY_SIZE(regs); ++i) { - radeon_emit_64bit_pointer(cs, regs[i], va); + radeon_emit_64bit_pointer(regs[i], va); } } else if (pdev->info.gfx_level >= GFX11) { uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B420_SPI_SHADER_PGM_LO_HS, R_00B220_SPI_SHADER_PGM_LO_GS}; for (int i = 0; i < ARRAY_SIZE(regs); ++i) { - radeon_emit_64bit_pointer(cs, regs[i], va); + radeon_emit_64bit_pointer(regs[i], va); } } else if (pdev->info.gfx_level >= GFX10) { uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B130_SPI_SHADER_USER_DATA_VS_0, R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS, R_00B408_SPI_SHADER_USER_DATA_ADDR_LO_HS}; for (int i = 0; i < ARRAY_SIZE(regs); ++i) { - radeon_emit_64bit_pointer(cs, regs[i], va); + radeon_emit_64bit_pointer(regs[i], va); } } else if (pdev->info.gfx_level == GFX9) { uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B130_SPI_SHADER_USER_DATA_VS_0, R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS, R_00B408_SPI_SHADER_USER_DATA_ADDR_LO_HS}; for (int i = 0; i < ARRAY_SIZE(regs); ++i) { - radeon_emit_64bit_pointer(cs, regs[i], va); + radeon_emit_64bit_pointer(regs[i], va); } } else { uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B130_SPI_SHADER_USER_DATA_VS_0, @@ -605,9 +629,11 @@ radv_emit_graphics_shader_pointers(struct radv_device *device, struct radeon_cmd R_00B430_SPI_SHADER_USER_DATA_HS_0, R_00B530_SPI_SHADER_USER_DATA_LS_0}; for (int i = 0; i < ARRAY_SIZE(regs); ++i) { - radeon_emit_64bit_pointer(cs, regs[i], va); + radeon_emit_64bit_pointer(regs[i], va); } } + + radeon_end(); } static void @@ -626,53 +652,56 @@ radv_emit_ge_rings(struct radv_device *device, struct radeon_cmdbuf *cs, struct radv_cs_add_buffer(device->ws, cs, ge_rings_bo); + radeon_begin(cs); + /* We must wait for idle using an EOP event before changing the attribute ring registers. Use the * bottom-of-pipe EOP event, but increment the PWS counter instead of writing memory. */ - radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, 6, 0)); - radeon_emit(cs, S_490_EVENT_TYPE(V_028A90_BOTTOM_OF_PIPE_TS) | S_490_EVENT_INDEX(5) | S_490_PWS_ENABLE(1)); - radeon_emit(cs, 0); /* DST_SEL, INT_SEL, DATA_SEL */ - radeon_emit(cs, 0); /* ADDRESS_LO */ - radeon_emit(cs, 0); /* ADDRESS_HI */ - radeon_emit(cs, 0); /* DATA_LO */ - radeon_emit(cs, 0); /* DATA_HI */ - radeon_emit(cs, 0); /* INT_CTXID */ + radeon_emit(PKT3(PKT3_RELEASE_MEM, 6, 0)); + radeon_emit(S_490_EVENT_TYPE(V_028A90_BOTTOM_OF_PIPE_TS) | S_490_EVENT_INDEX(5) | S_490_PWS_ENABLE(1)); + radeon_emit(0); /* DST_SEL, INT_SEL, DATA_SEL */ + radeon_emit(0); /* ADDRESS_LO */ + radeon_emit(0); /* ADDRESS_HI */ + radeon_emit(0); /* DATA_LO */ + radeon_emit(0); /* DATA_HI */ + radeon_emit(0); /* INT_CTXID */ /* Wait for the PWS counter. */ - radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 6, 0)); - radeon_emit(cs, S_580_PWS_STAGE_SEL(V_580_CP_ME) | S_580_PWS_COUNTER_SEL(V_580_TS_SELECT) | S_580_PWS_ENA2(1) | - S_580_PWS_COUNT(0)); - radeon_emit(cs, 0xffffffff); /* GCR_SIZE */ - radeon_emit(cs, 0x01ffffff); /* GCR_SIZE_HI */ - radeon_emit(cs, 0); /* GCR_BASE_LO */ - radeon_emit(cs, 0); /* GCR_BASE_HI */ - radeon_emit(cs, S_585_PWS_ENA(1)); - radeon_emit(cs, 0); /* GCR_CNTL */ + radeon_emit(PKT3(PKT3_ACQUIRE_MEM, 6, 0)); + radeon_emit(S_580_PWS_STAGE_SEL(V_580_CP_ME) | S_580_PWS_COUNTER_SEL(V_580_TS_SELECT) | S_580_PWS_ENA2(1) | + S_580_PWS_COUNT(0)); + radeon_emit(0xffffffff); /* GCR_SIZE */ + radeon_emit(0x01ffffff); /* GCR_SIZE_HI */ + radeon_emit(0); /* GCR_BASE_LO */ + radeon_emit(0); /* GCR_BASE_HI */ + radeon_emit(S_585_PWS_ENA(1)); + radeon_emit(0); /* GCR_CNTL */ /* The PS will read inputs from this address. */ - radeon_set_uconfig_reg_seq(cs, R_031110_SPI_GS_THROTTLE_CNTL1, 4); - radeon_emit(cs, 0x12355123); /* SPI_GS_THROTTLE_CNTL1 */ - radeon_emit(cs, 0x1544D); /* SPI_GS_THROTTLE_CNTL2 */ - radeon_emit(cs, va >> 16); /* SPI_ATTRIBUTE_RING_BASE */ - radeon_emit(cs, S_03111C_MEM_SIZE((pdev->info.attribute_ring_size_per_se >> 16) - 1) | - S_03111C_BIG_PAGE(pdev->info.discardable_allows_big_page) | - S_03111C_L1_POLICY(1)); /* SPI_ATTRIBUTE_RING_SIZE */ + radeon_set_uconfig_reg_seq(R_031110_SPI_GS_THROTTLE_CNTL1, 4); + radeon_emit(0x12355123); /* SPI_GS_THROTTLE_CNTL1 */ + radeon_emit(0x1544D); /* SPI_GS_THROTTLE_CNTL2 */ + radeon_emit(va >> 16); /* SPI_ATTRIBUTE_RING_BASE */ + radeon_emit(S_03111C_MEM_SIZE((pdev->info.attribute_ring_size_per_se >> 16) - 1) | + S_03111C_BIG_PAGE(pdev->info.discardable_allows_big_page) | + S_03111C_L1_POLICY(1)); /* SPI_ATTRIBUTE_RING_SIZE */ if (pdev->info.gfx_level >= GFX12) { const uint64_t pos_address = va + pdev->info.pos_ring_offset; const uint64_t prim_address = va + pdev->info.prim_ring_offset; /* When one of these 4 registers is updated, all 4 must be updated. */ - radeon_set_uconfig_reg_seq(cs, R_0309A0_GE_POS_RING_BASE, 4); - radeon_emit(cs, pos_address >> 16); /* R_0309A0_GE_POS_RING_BASE */ - radeon_emit(cs, S_0309A4_MEM_SIZE(pdev->info.pos_ring_size_per_se >> 5)); /* R_0309A4_GE_POS_RING_SIZE */ - radeon_emit(cs, prim_address >> 16); /* R_0309A8_GE_PRIM_RING_BASE */ - radeon_emit(cs, S_0309AC_MEM_SIZE(pdev->info.prim_ring_size_per_se >> 5) | S_0309AC_SCOPE(gfx12_scope_device) | - S_0309AC_PAF_TEMPORAL(gfx12_store_high_temporal_stay_dirty) | - S_0309AC_PAB_TEMPORAL(gfx12_load_last_use_discard) | - S_0309AC_SPEC_DATA_READ(gfx12_spec_read_auto) | S_0309AC_FORCE_SE_SCOPE(1) | - S_0309AC_PAB_NOFILL(1)); /* R_0309AC_GE_PRIM_RING_SIZE */ + radeon_set_uconfig_reg_seq(R_0309A0_GE_POS_RING_BASE, 4); + radeon_emit(pos_address >> 16); /* R_0309A0_GE_POS_RING_BASE */ + radeon_emit(S_0309A4_MEM_SIZE(pdev->info.pos_ring_size_per_se >> 5)); /* R_0309A4_GE_POS_RING_SIZE */ + radeon_emit(prim_address >> 16); /* R_0309A8_GE_PRIM_RING_BASE */ + radeon_emit(S_0309AC_MEM_SIZE(pdev->info.prim_ring_size_per_se >> 5) | S_0309AC_SCOPE(gfx12_scope_device) | + S_0309AC_PAF_TEMPORAL(gfx12_store_high_temporal_stay_dirty) | + S_0309AC_PAB_TEMPORAL(gfx12_load_last_use_discard) | S_0309AC_SPEC_DATA_READ(gfx12_spec_read_auto) | + S_0309AC_FORCE_SE_SCOPE(1) | S_0309AC_PAB_NOFILL(1)); /* R_0309AC_GE_PRIM_RING_SIZE */ } + + radeon_end(); } static void @@ -1109,8 +1138,10 @@ radv_update_preamble_cs(struct radv_queue_state *queue, struct radv_device *devi radv_init_graphics_state(cs, device); if (esgs_ring_bo || gsvs_ring_bo || tess_rings_bo || task_rings_bo) { - radeon_event_write(cs, V_028A90_VS_PARTIAL_FLUSH); - radeon_event_write(cs, V_028A90_VGT_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_VS_PARTIAL_FLUSH); + radeon_event_write(V_028A90_VGT_FLUSH); + radeon_end(); } radv_emit_gs_ring_sizes(device, cs, esgs_ring_bo, needs->esgs_ring_size, gsvs_ring_bo, needs->gsvs_ring_size); @@ -1126,7 +1157,9 @@ radv_update_preamble_cs(struct radv_queue_state *queue, struct radv_device *devi radv_emit_compute(device, cs, true); if (task_rings_bo) { - radeon_event_write(cs, V_028A90_CS_PARTIAL_FLUSH); + radeon_begin(cs); + radeon_event_write(V_028A90_CS_PARTIAL_FLUSH); + radeon_end(); } radv_emit_task_rings(device, cs, task_rings_bo, true); @@ -1548,51 +1581,54 @@ radv_create_perf_counter_lock_cs(struct radv_device *device, unsigned pass, bool radv_cs_add_buffer(device->ws, cs, device->perf_counter_bo); + radeon_begin(cs); + if (!unlock) { uint64_t mutex_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_LOCK_OFFSET; - radeon_emit(cs, PKT3(PKT3_ATOMIC_MEM, 7, 0)); - radeon_emit(cs, ATOMIC_OP(TC_OP_ATOMIC_CMPSWAP_32) | ATOMIC_COMMAND(ATOMIC_COMMAND_LOOP)); - radeon_emit(cs, mutex_va); /* addr lo */ - radeon_emit(cs, mutex_va >> 32); /* addr hi */ - radeon_emit(cs, 1); /* data lo */ - radeon_emit(cs, 0); /* data hi */ - radeon_emit(cs, 0); /* compare data lo */ - radeon_emit(cs, 0); /* compare data hi */ - radeon_emit(cs, 10); /* loop interval */ + radeon_emit(PKT3(PKT3_ATOMIC_MEM, 7, 0)); + radeon_emit(ATOMIC_OP(TC_OP_ATOMIC_CMPSWAP_32) | ATOMIC_COMMAND(ATOMIC_COMMAND_LOOP)); + radeon_emit(mutex_va); /* addr lo */ + radeon_emit(mutex_va >> 32); /* addr hi */ + radeon_emit(1); /* data lo */ + radeon_emit(0); /* data hi */ + radeon_emit(0); /* compare data lo */ + radeon_emit(0); /* compare data hi */ + radeon_emit(10); /* loop interval */ } uint64_t va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_PASS_OFFSET; uint64_t unset_va = va + (unlock ? 8 * pass : 0); uint64_t set_va = va + (unlock ? 0 : 8 * pass); - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL | - COPY_DATA_WR_CONFIRM); - radeon_emit(cs, 0); /* immediate */ - radeon_emit(cs, 0); - radeon_emit(cs, unset_va); - radeon_emit(cs, unset_va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL | + COPY_DATA_WR_CONFIRM); + radeon_emit(0); /* immediate */ + radeon_emit(0); + radeon_emit(unset_va); + radeon_emit(unset_va >> 32); - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL | - COPY_DATA_WR_CONFIRM); - radeon_emit(cs, 1); /* immediate */ - radeon_emit(cs, 0); - radeon_emit(cs, set_va); - radeon_emit(cs, set_va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL | + COPY_DATA_WR_CONFIRM); + radeon_emit(1); /* immediate */ + radeon_emit(0); + radeon_emit(set_va); + radeon_emit(set_va >> 32); if (unlock) { uint64_t mutex_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_LOCK_OFFSET; - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL | - COPY_DATA_WR_CONFIRM); - radeon_emit(cs, 0); /* immediate */ - radeon_emit(cs, 0); - radeon_emit(cs, mutex_va); - radeon_emit(cs, mutex_va >> 32); + radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | COPY_DATA_COUNT_SEL | + COPY_DATA_WR_CONFIRM); + radeon_emit(0); /* immediate */ + radeon_emit(0); + radeon_emit(mutex_va); + radeon_emit(mutex_va >> 32); } + radeon_end(); assert(cs->cdw <= cdw); VkResult result = device->ws->cs_finalize(cs); diff --git a/src/amd/vulkan/radv_radeon_winsys.h b/src/amd/vulkan/radv_radeon_winsys.h index 422e5bfb1bf..c6d14d259db 100644 --- a/src/amd/vulkan/radv_radeon_winsys.h +++ b/src/amd/vulkan/radv_radeon_winsys.h @@ -321,21 +321,6 @@ struct radeon_winsys { const struct vk_sync_type *const *(*get_sync_types)(struct radeon_winsys *ws); }; -static inline void -radeon_emit(struct radeon_cmdbuf *cs, uint32_t value) -{ - assert(cs->cdw < cs->reserved_dw); - cs->buf[cs->cdw++] = value; -} - -static inline void -radeon_emit_array(struct radeon_cmdbuf *cs, const uint32_t *values, unsigned count) -{ - assert(cs->cdw + count <= cs->reserved_dw); - memcpy(cs->buf + cs->cdw, values, count * 4); - cs->cdw += count; -} - static inline uint64_t radv_buffer_get_va(const struct radeon_winsys_bo *bo) { diff --git a/src/amd/vulkan/radv_sdma.c b/src/amd/vulkan/radv_sdma.c index df79b56f505..fe660f8f891 100644 --- a/src/amd/vulkan/radv_sdma.c +++ b/src/amd/vulkan/radv_sdma.c @@ -323,44 +323,54 @@ radv_sdma_emit_nop(const struct radv_device *device, struct radeon_cmdbuf *cs) { /* SDMA NOP acts as a fence command and causes the SDMA engine to wait for pending copy operations. */ radeon_check_space(device->ws, cs, 1); - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); + radeon_end(); } void radv_sdma_emit_write_timestamp(struct radeon_cmdbuf *cs, uint64_t va) { - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_TIMESTAMP, SDMA_TS_SUB_OPCODE_GET_GLOBAL_TIMESTAMP, 0)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_TIMESTAMP, SDMA_TS_SUB_OPCODE_GET_GLOBAL_TIMESTAMP, 0)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_end(); } void radv_sdma_emit_fence(struct radeon_cmdbuf *cs, uint64_t va, uint32_t fence) { - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, SDMA_FENCE_MTYPE_UC)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, fence); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_FENCE, 0, SDMA_FENCE_MTYPE_UC)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(fence); + radeon_end(); } void radv_sdma_emit_wait_mem(struct radeon_cmdbuf *cs, uint32_t op, uint64_t va, uint32_t ref, uint32_t mask) { - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_POLL_REGMEM, 0, 0) | op << 28 | SDMA_POLL_MEM); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, ref); - radeon_emit(cs, mask); - radeon_emit(cs, SDMA_POLL_INTERVAL_160_CLK | SDMA_POLL_RETRY_INDEFINITELY << 16); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_POLL_REGMEM, 0, 0) | op << 28 | SDMA_POLL_MEM); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(ref); + radeon_emit(mask); + radeon_emit(SDMA_POLL_INTERVAL_160_CLK | SDMA_POLL_RETRY_INDEFINITELY << 16); + radeon_end(); } void radv_sdma_emit_write_data_head(struct radeon_cmdbuf *cs, uint64_t va, uint32_t count) { - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, count - 1); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0)); + radeon_emit(va); + radeon_emit(va >> 32); + radeon_emit(count - 1); + radeon_end(); } void @@ -393,19 +403,23 @@ radv_sdma_copy_memory(const struct radv_device *device, struct radeon_cmdbuf *cs radeon_check_space(device->ws, cs, ncopy * 7); + radeon_begin(cs); + for (unsigned i = 0; i < ncopy; i++) { unsigned csize = size >= 4 ? MIN2(size & align, max_size_per_packet) : size; - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0)); - radeon_emit(cs, ver >= SDMA_4_0 ? csize - 1 : csize); - radeon_emit(cs, 0); /* src/dst endian swap */ - radeon_emit(cs, src_va); - radeon_emit(cs, src_va >> 32); - radeon_emit(cs, dst_va); - radeon_emit(cs, dst_va >> 32); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0)); + radeon_emit(ver >= SDMA_4_0 ? csize - 1 : csize); + radeon_emit(0); /* src/dst endian swap */ + radeon_emit(src_va); + radeon_emit(src_va >> 32); + radeon_emit(dst_va); + radeon_emit(dst_va >> 32); dst_va += csize; src_va += csize; size -= csize; } + + radeon_end(); } void @@ -428,18 +442,21 @@ radv_sdma_fill_memory(const struct radv_device *device, struct radeon_cmdbuf *cs const unsigned num_packets = DIV_ROUND_UP(size, max_fill_bytes); ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cs, num_packets * 5); + radeon_begin(cs); + for (unsigned i = 0; i < num_packets; ++i) { const uint64_t offset = i * max_fill_bytes; const uint64_t fill_bytes = MIN2(size - offset, max_fill_bytes); const uint64_t fill_va = va + offset; - radeon_emit(cs, constant_fill_header); - radeon_emit(cs, fill_va); - radeon_emit(cs, fill_va >> 32); - radeon_emit(cs, value); - radeon_emit(cs, fill_bytes - 1); /* Must be programmed in bytes, even if the fill is done in dwords. */ + radeon_emit(constant_fill_header); + radeon_emit(fill_va); + radeon_emit(fill_va >> 32); + radeon_emit(value); + radeon_emit(fill_bytes - 1); /* Must be programmed in bytes, even if the fill is done in dwords. */ } + radeon_end(); assert(cs->cdw <= cdw_max); } @@ -475,20 +492,22 @@ radv_sdma_emit_copy_linear_sub_window(const struct radv_device *device, struct r ASSERTED unsigned cdw_end = radeon_check_space(device->ws, cs, 13); - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR_SUB_WINDOW, 0) | util_logbase2(src->bpp) - << 29); - radeon_emit(cs, src->va); - radeon_emit(cs, src->va >> 32); - radeon_emit(cs, src_off.x | src_off.y << 16); - radeon_emit(cs, src_off.z | (src_pitch - 1) << (ver >= SDMA_7_0 ? 16 : 13)); - radeon_emit(cs, src_slice_pitch - 1); - radeon_emit(cs, dst->va); - radeon_emit(cs, dst->va >> 32); - radeon_emit(cs, dst_off.x | dst_off.y << 16); - radeon_emit(cs, dst_off.z | (dst_pitch - 1) << (ver >= SDMA_7_0 ? 16 : 13)); - radeon_emit(cs, dst_slice_pitch - 1); - radeon_emit(cs, (ext.width - 1) | (ext.height - 1) << 16); - radeon_emit(cs, (ext.depth - 1)); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR_SUB_WINDOW, 0) | util_logbase2(src->bpp) + << 29); + radeon_emit(src->va); + radeon_emit(src->va >> 32); + radeon_emit(src_off.x | src_off.y << 16); + radeon_emit(src_off.z | (src_pitch - 1) << (ver >= SDMA_7_0 ? 16 : 13)); + radeon_emit(src_slice_pitch - 1); + radeon_emit(dst->va); + radeon_emit(dst->va >> 32); + radeon_emit(dst_off.x | dst_off.y << 16); + radeon_emit(dst_off.z | (dst_pitch - 1) << (ver >= SDMA_7_0 ? 16 : 13)); + radeon_emit(dst_slice_pitch - 1); + radeon_emit((ext.width - 1) | (ext.height - 1) << 16); + radeon_emit((ext.depth - 1)); + radeon_end(); assert(cs->cdw == cdw_end); } @@ -519,29 +538,31 @@ radv_sdma_emit_copy_tiled_sub_window(const struct radv_device *device, struct ra ASSERTED unsigned cdw_end = radeon_check_space(device->ws, cs, 14 + (dcc ? 3 : 0)); - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_TILED_SUB_WINDOW, 0) | dcc << 19 | detile << 31 | - tiled->header_dword); - radeon_emit(cs, tiled->va); - radeon_emit(cs, tiled->va >> 32); - radeon_emit(cs, tiled_off.x | tiled_off.y << 16); - radeon_emit(cs, tiled_off.z | (tiled_ext.width - 1) << 16); - radeon_emit(cs, (tiled_ext.height - 1) | (tiled_ext.depth - 1) << 16); - radeon_emit(cs, tiled->info_dword); - radeon_emit(cs, linear->va); - radeon_emit(cs, linear->va >> 32); - radeon_emit(cs, linear_off.x | linear_off.y << 16); - radeon_emit(cs, linear_off.z | (linear_pitch - 1) << 16); - radeon_emit(cs, linear_slice_pitch - 1); - radeon_emit(cs, (ext.width - 1) | (ext.height - 1) << 16); - radeon_emit(cs, (ext.depth - 1)); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_TILED_SUB_WINDOW, 0) | dcc << 19 | detile << 31 | + tiled->header_dword); + radeon_emit(tiled->va); + radeon_emit(tiled->va >> 32); + radeon_emit(tiled_off.x | tiled_off.y << 16); + radeon_emit(tiled_off.z | (tiled_ext.width - 1) << 16); + radeon_emit((tiled_ext.height - 1) | (tiled_ext.depth - 1) << 16); + radeon_emit(tiled->info_dword); + radeon_emit(linear->va); + radeon_emit(linear->va >> 32); + radeon_emit(linear_off.x | linear_off.y << 16); + radeon_emit(linear_off.z | (linear_pitch - 1) << 16); + radeon_emit(linear_slice_pitch - 1); + radeon_emit((ext.width - 1) | (ext.height - 1) << 16); + radeon_emit((ext.depth - 1)); if (tiled->meta_va) { const unsigned write_compress_enable = !detile; - radeon_emit(cs, tiled->meta_va); - radeon_emit(cs, tiled->meta_va >> 32); - radeon_emit(cs, tiled->meta_config | write_compress_enable << 28); + radeon_emit(tiled->meta_va); + radeon_emit(tiled->meta_va >> 32); + radeon_emit(tiled->meta_config | write_compress_enable << 28); } + radeon_end(); assert(cs->cdw == cdw_end); } @@ -583,34 +604,36 @@ radv_sdma_emit_copy_t2t_sub_window(const struct radv_device *device, struct rade ASSERTED unsigned cdw_end = radeon_check_space(device->ws, cs, 15 + (dcc ? 3 : 0)); - radeon_emit(cs, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_T2T_SUB_WINDOW, 0) | dcc << 19 | dcc_dir << 31 | - src->header_dword); - radeon_emit(cs, src->va); - radeon_emit(cs, src->va >> 32); - radeon_emit(cs, src_off.x | src_off.y << 16); - radeon_emit(cs, src_off.z | (src_ext.width - 1) << 16); - radeon_emit(cs, (src_ext.height - 1) | (src_ext.depth - 1) << 16); - radeon_emit(cs, src->info_dword); - radeon_emit(cs, dst->va); - radeon_emit(cs, dst->va >> 32); - radeon_emit(cs, dst_off.x | dst_off.y << 16); - radeon_emit(cs, dst_off.z | (dst_ext.width - 1) << 16); - radeon_emit(cs, (dst_ext.height - 1) | (dst_ext.depth - 1) << 16); - radeon_emit(cs, dst->info_dword); - radeon_emit(cs, (ext.width - 1) | (ext.height - 1) << 16); - radeon_emit(cs, (ext.depth - 1)); + radeon_begin(cs); + radeon_emit(SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_T2T_SUB_WINDOW, 0) | dcc << 19 | dcc_dir << 31 | + src->header_dword); + radeon_emit(src->va); + radeon_emit(src->va >> 32); + radeon_emit(src_off.x | src_off.y << 16); + radeon_emit(src_off.z | (src_ext.width - 1) << 16); + radeon_emit((src_ext.height - 1) | (src_ext.depth - 1) << 16); + radeon_emit(src->info_dword); + radeon_emit(dst->va); + radeon_emit(dst->va >> 32); + radeon_emit(dst_off.x | dst_off.y << 16); + radeon_emit(dst_off.z | (dst_ext.width - 1) << 16); + radeon_emit((dst_ext.height - 1) | (dst_ext.depth - 1) << 16); + radeon_emit(dst->info_dword); + radeon_emit((ext.width - 1) | (ext.height - 1) << 16); + radeon_emit((ext.depth - 1)); if (dst->meta_va) { const uint32_t write_compress_enable = 1; - radeon_emit(cs, dst->meta_va); - radeon_emit(cs, dst->meta_va >> 32); - radeon_emit(cs, dst->meta_config | write_compress_enable << 28); + radeon_emit(dst->meta_va); + radeon_emit(dst->meta_va >> 32); + radeon_emit(dst->meta_config | write_compress_enable << 28); } else if (src->meta_va) { - radeon_emit(cs, src->meta_va); - radeon_emit(cs, src->meta_va >> 32); - radeon_emit(cs, src->meta_config); + radeon_emit(src->meta_va); + radeon_emit(src->meta_va >> 32); + radeon_emit(src->meta_config); } + radeon_end(); assert(cs->cdw == cdw_end); } diff --git a/src/amd/vulkan/radv_spm.c b/src/amd/vulkan/radv_spm.c index 1eea9d2d1e9..4191f0c812f 100644 --- a/src/amd/vulkan/radv_spm.c +++ b/src/amd/vulkan/radv_spm.c @@ -83,16 +83,19 @@ radv_emit_spm_counters(struct radv_device *device, struct radeon_cmdbuf *cs, enu continue; radeon_check_space(device->ws, cs, 3 + num_counters * 3); + radeon_begin(cs); - radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, spm->sq_wgp[instance].grbm_gfx_index); + radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, spm->sq_wgp[instance].grbm_gfx_index); for (uint32_t b = 0; b < num_counters; b++) { const struct ac_spm_counter_select *cntr_sel = &spm->sq_wgp[instance].counters[b]; uint32_t reg_base = R_036700_SQ_PERFCOUNTER0_SELECT; - radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, reg_base + b * 4, 1); - radeon_emit(cs, cntr_sel->sel0); + radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, reg_base + b * 4, 1); + radeon_emit(cntr_sel->sel0); } + + radeon_end(); } } @@ -103,18 +106,21 @@ radv_emit_spm_counters(struct radv_device *device, struct radeon_cmdbuf *cs, enu continue; radeon_check_space(device->ws, cs, 3 + num_counters * 3); + radeon_begin(cs); - radeon_set_uconfig_reg( - cs, R_030800_GRBM_GFX_INDEX, - S_030800_SH_BROADCAST_WRITES(1) | S_030800_INSTANCE_BROADCAST_WRITES(1) | S_030800_SE_INDEX(instance)); + radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, S_030800_SH_BROADCAST_WRITES(1) | + S_030800_INSTANCE_BROADCAST_WRITES(1) | + S_030800_SE_INDEX(instance)); for (uint32_t b = 0; b < num_counters; b++) { const struct ac_spm_counter_select *cntr_sel = &spm->sqg[instance].counters[b]; uint32_t reg_base = R_036700_SQ_PERFCOUNTER0_SELECT; - radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, reg_base + b * 4, 1); - radeon_emit(cs, cntr_sel->sel0 | S_036700_SQC_BANK_MASK(0xf)); /* SQC_BANK_MASK only gfx10 */ + radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, reg_base + b * 4, 1); + radeon_emit(cntr_sel->sel0 | S_036700_SQC_BANK_MASK(0xf)); /* SQC_BANK_MASK only gfx10 */ } + + radeon_end(); } for (uint32_t b = 0; b < spm->num_block_sel; b++) { @@ -125,8 +131,9 @@ radv_emit_spm_counters(struct radv_device *device, struct radeon_cmdbuf *cs, enu struct ac_spm_block_instance *block_instance = &block_sel->instances[i]; radeon_check_space(device->ws, cs, 3 + (AC_SPM_MAX_COUNTER_PER_BLOCK * 6)); + radeon_begin(cs); - radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, block_instance->grbm_gfx_index); + radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, block_instance->grbm_gfx_index); for (unsigned c = 0; c < block_instance->num_counters; c++) { const struct ac_spm_counter_select *cntr_sel = &block_instance->counters[c]; @@ -134,19 +141,22 @@ radv_emit_spm_counters(struct radv_device *device, struct radeon_cmdbuf *cs, enu if (!cntr_sel->active) continue; - radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, regs->select0[c], 1); - radeon_emit(cs, cntr_sel->sel0); + radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, regs->select0[c], 1); + radeon_emit(cntr_sel->sel0); - radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, regs->select1[c], 1); - radeon_emit(cs, cntr_sel->sel1); + radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, regs->select1[c], 1); + radeon_emit(cntr_sel->sel1); } + + radeon_end(); } } /* Restore global broadcasting. */ - radeon_set_uconfig_reg( - cs, R_030800_GRBM_GFX_INDEX, - S_030800_SE_BROADCAST_WRITES(1) | S_030800_SH_BROADCAST_WRITES(1) | S_030800_INSTANCE_BROADCAST_WRITES(1)); + radeon_begin(cs); + radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, S_030800_SE_BROADCAST_WRITES(1) | S_030800_SH_BROADCAST_WRITES(1) | + S_030800_INSTANCE_BROADCAST_WRITES(1)); + radeon_end(); } static void @@ -181,23 +191,26 @@ radv_emit_spm_muxsel(struct radv_device *device, struct radeon_cmdbuf *cs, enum } radeon_check_space(device->ws, cs, 3 + spm->num_muxsel_lines[s] * (7 + AC_SPM_MUXSEL_LINE_SIZE)); + radeon_begin(cs); - radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, grbm_gfx_index); + radeon_set_uconfig_reg(R_030800_GRBM_GFX_INDEX, grbm_gfx_index); for (unsigned l = 0; l < spm->num_muxsel_lines[s]; l++) { uint32_t *data = (uint32_t *)spm->muxsel_lines[s][l].muxsel; /* Select MUXSEL_ADDR to point to the next muxsel. */ - radeon_set_uconfig_perfctr_reg(pdev->info.gfx_level, ring, cs, rlc_muxsel_addr, l * AC_SPM_MUXSEL_LINE_SIZE); + radeon_set_uconfig_perfctr_reg(pdev->info.gfx_level, ring, rlc_muxsel_addr, l * AC_SPM_MUXSEL_LINE_SIZE); /* Write the muxsel line configuration with MUXSEL_DATA. */ - radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + AC_SPM_MUXSEL_LINE_SIZE, 0)); - radeon_emit(cs, S_370_DST_SEL(V_370_MEM_MAPPED_REGISTER) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME) | - S_370_WR_ONE_ADDR(1)); - radeon_emit(cs, rlc_muxsel_data >> 2); - radeon_emit(cs, 0); - radeon_emit_array(cs, data, AC_SPM_MUXSEL_LINE_SIZE); + radeon_emit(PKT3(PKT3_WRITE_DATA, 2 + AC_SPM_MUXSEL_LINE_SIZE, 0)); + radeon_emit(S_370_DST_SEL(V_370_MEM_MAPPED_REGISTER) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME) | + S_370_WR_ONE_ADDR(1)); + radeon_emit(rlc_muxsel_data >> 2); + radeon_emit(0); + radeon_emit_array(data, AC_SPM_MUXSEL_LINE_SIZE); } + + radeon_end(); } } @@ -215,14 +228,15 @@ radv_emit_spm_setup(struct radv_device *device, struct radeon_cmdbuf *cs, enum r assert(spm->sample_interval >= 32); radeon_check_space(device->ws, cs, 27); + radeon_begin(cs); /* Configure the SPM ring buffer. */ - radeon_set_uconfig_reg(cs, R_037200_RLC_SPM_PERFMON_CNTL, + radeon_set_uconfig_reg(R_037200_RLC_SPM_PERFMON_CNTL, S_037200_PERFMON_RING_MODE(0) | /* no stall and no interrupt on overflow */ S_037200_PERFMON_SAMPLE_INTERVAL(spm->sample_interval)); /* in sclk */ - radeon_set_uconfig_reg(cs, R_037204_RLC_SPM_PERFMON_RING_BASE_LO, va); - radeon_set_uconfig_reg(cs, R_037208_RLC_SPM_PERFMON_RING_BASE_HI, S_037208_RING_BASE_HI(va >> 32)); - radeon_set_uconfig_reg(cs, R_03720C_RLC_SPM_PERFMON_RING_SIZE, ring_size); + radeon_set_uconfig_reg(R_037204_RLC_SPM_PERFMON_RING_BASE_LO, va); + radeon_set_uconfig_reg(R_037208_RLC_SPM_PERFMON_RING_BASE_HI, S_037208_RING_BASE_HI(va >> 32)); + radeon_set_uconfig_reg(R_03720C_RLC_SPM_PERFMON_RING_SIZE, ring_size); /* Configure the muxsel. */ uint32_t total_muxsel_lines = 0; @@ -230,27 +244,29 @@ radv_emit_spm_setup(struct radv_device *device, struct radeon_cmdbuf *cs, enum r total_muxsel_lines += spm->num_muxsel_lines[s]; } - radeon_set_uconfig_reg(cs, R_03726C_RLC_SPM_ACCUM_MODE, 0); + radeon_set_uconfig_reg(R_03726C_RLC_SPM_ACCUM_MODE, 0); if (pdev->info.gfx_level >= GFX11) { - radeon_set_uconfig_reg(cs, R_03721C_RLC_SPM_PERFMON_SEGMENT_SIZE, + radeon_set_uconfig_reg(R_03721C_RLC_SPM_PERFMON_SEGMENT_SIZE, S_03721C_TOTAL_NUM_SEGMENT(total_muxsel_lines) | S_03721C_GLOBAL_NUM_SEGMENT(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_GLOBAL]) | S_03721C_SE_NUM_SEGMENT(spm->max_se_muxsel_lines)); - radeon_set_uconfig_reg(cs, R_037210_RLC_SPM_RING_WRPTR, 0); + radeon_set_uconfig_reg(R_037210_RLC_SPM_RING_WRPTR, 0); } else { - radeon_set_uconfig_reg(cs, R_037210_RLC_SPM_PERFMON_SEGMENT_SIZE, 0); - radeon_set_uconfig_reg(cs, R_03727C_RLC_SPM_PERFMON_SE3TO0_SEGMENT_SIZE, + radeon_set_uconfig_reg(R_037210_RLC_SPM_PERFMON_SEGMENT_SIZE, 0); + radeon_set_uconfig_reg(R_03727C_RLC_SPM_PERFMON_SE3TO0_SEGMENT_SIZE, S_03727C_SE0_NUM_LINE(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_SE0]) | S_03727C_SE1_NUM_LINE(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_SE1]) | S_03727C_SE2_NUM_LINE(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_SE2]) | S_03727C_SE3_NUM_LINE(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_SE3])); - radeon_set_uconfig_reg(cs, R_037280_RLC_SPM_PERFMON_GLB_SEGMENT_SIZE, + radeon_set_uconfig_reg(R_037280_RLC_SPM_PERFMON_GLB_SEGMENT_SIZE, S_037280_PERFMON_SEGMENT_SIZE(total_muxsel_lines) | S_037280_GLOBAL_NUM_LINE(spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_GLOBAL])); } + radeon_end(); + /* Upload each muxsel ram to the RLC. */ radv_emit_spm_muxsel(device, cs, qf); diff --git a/src/amd/vulkan/radv_sqtt.c b/src/amd/vulkan/radv_sqtt.c index 5a37d73307f..328c4424211 100644 --- a/src/amd/vulkan/radv_sqtt.c +++ b/src/amd/vulkan/radv_sqtt.c @@ -132,14 +132,17 @@ radv_emit_sqtt_userdata(const struct radv_cmd_buffer *cmd_buffer, const void *da uint32_t count = MIN2(num_dwords, 2); radeon_check_space(device->ws, cs, 2 + count); + radeon_begin(cs); /* Without the perfctr bit the CP might not always pass the * write on correctly. */ if (pdev->info.gfx_level >= GFX10) - radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, cs, R_030D08_SQ_THREAD_TRACE_USERDATA_2, count); + radeon_set_uconfig_perfctr_reg_seq(gfx_level, ring, R_030D08_SQ_THREAD_TRACE_USERDATA_2, count); else - radeon_set_uconfig_reg_seq(cs, R_030D08_SQ_THREAD_TRACE_USERDATA_2, count); - radeon_emit_array(cs, dwords, count); + radeon_set_uconfig_reg_seq(R_030D08_SQ_THREAD_TRACE_USERDATA_2, count); + radeon_emit_array(dwords, count); + + radeon_end(); dwords += count; num_dwords -= count; @@ -151,8 +154,10 @@ radv_emit_spi_config_cntl(const struct radv_device *device, struct radeon_cmdbuf { const struct radv_physical_device *pdev = radv_device_physical(device); + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX12) { - radeon_set_uconfig_reg(cs, R_031120_SPI_SQG_EVENT_CTL, + radeon_set_uconfig_reg(R_031120_SPI_SQG_EVENT_CTL, S_031120_ENABLE_SQG_TOP_EVENTS(enable) | S_031120_ENABLE_SQG_BOP_EVENTS(enable)); } else if (pdev->info.gfx_level >= GFX9) { uint32_t spi_config_cntl = S_031100_GPR_WRITE_PRIORITY(0x2c688) | S_031100_EXP_PRIORITY_ORDER(3) | @@ -161,12 +166,14 @@ radv_emit_spi_config_cntl(const struct radv_device *device, struct radeon_cmdbuf if (pdev->info.gfx_level >= GFX10) spi_config_cntl |= S_031100_PS_PKR_PRIORITY_CNTL(3); - radeon_set_uconfig_reg(cs, R_031100_SPI_CONFIG_CNTL, spi_config_cntl); + radeon_set_uconfig_reg(R_031100_SPI_CONFIG_CNTL, spi_config_cntl); } else { /* SPI_CONFIG_CNTL is a protected register on GFX6-GFX8. */ - radeon_set_privileged_config_reg(cs, R_009100_SPI_CONFIG_CNTL, + radeon_set_privileged_config_reg(R_009100_SPI_CONFIG_CNTL, S_009100_ENABLE_SQG_TOP_EVENTS(enable) | S_009100_ENABLE_SQG_BOP_EVENTS(enable)); } + + radeon_end(); } void @@ -177,11 +184,15 @@ radv_emit_inhibit_clockgating(const struct radv_device *device, struct radeon_cm if (pdev->info.gfx_level >= GFX11) return; /* not needed */ + radeon_begin(cs); + if (pdev->info.gfx_level >= GFX10) { - radeon_set_uconfig_reg(cs, R_037390_RLC_PERFMON_CLK_CNTL, S_037390_PERFMON_CLOCK_STATE(inhibit)); + radeon_set_uconfig_reg(R_037390_RLC_PERFMON_CLK_CNTL, S_037390_PERFMON_CLOCK_STATE(inhibit)); } else if (pdev->info.gfx_level >= GFX8) { - radeon_set_uconfig_reg(cs, R_0372FC_RLC_PERFMON_CLK_CNTL, S_0372FC_PERFMON_CLOCK_STATE(inhibit)); + radeon_set_uconfig_reg(R_0372FC_RLC_PERFMON_CLK_CNTL, S_0372FC_PERFMON_CLOCK_STATE(inhibit)); } + + radeon_end(); } VkResult @@ -520,21 +531,25 @@ radv_begin_sqtt(struct radv_queue *queue) radeon_check_space(ws, cs, 512); + radeon_begin(cs); + switch (family) { case RADV_QUEUE_GENERAL: - radeon_emit(cs, PKT3(PKT3_CONTEXT_CONTROL, 1, 0)); - radeon_emit(cs, CC0_UPDATE_LOAD_ENABLES(1)); - radeon_emit(cs, CC1_UPDATE_SHADOW_ENABLES(1)); + radeon_emit(PKT3(PKT3_CONTEXT_CONTROL, 1, 0)); + radeon_emit(CC0_UPDATE_LOAD_ENABLES(1)); + radeon_emit(CC1_UPDATE_SHADOW_ENABLES(1)); break; case RADV_QUEUE_COMPUTE: - radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_NOP, 0, 0)); + radeon_emit(0); break; default: unreachable("Incorrect queue family"); break; } + radeon_end(); + /* Make sure to wait-for-idle before starting SQTT. */ radv_emit_wait_for_idle(device, cs, family); @@ -593,21 +608,25 @@ radv_end_sqtt(struct radv_queue *queue) radeon_check_space(ws, cs, 512); + radeon_begin(cs); + switch (family) { case RADV_QUEUE_GENERAL: - radeon_emit(cs, PKT3(PKT3_CONTEXT_CONTROL, 1, 0)); - radeon_emit(cs, CC0_UPDATE_LOAD_ENABLES(1)); - radeon_emit(cs, CC1_UPDATE_SHADOW_ENABLES(1)); + radeon_emit(PKT3(PKT3_CONTEXT_CONTROL, 1, 0)); + radeon_emit(CC0_UPDATE_LOAD_ENABLES(1)); + radeon_emit(CC1_UPDATE_SHADOW_ENABLES(1)); break; case RADV_QUEUE_COMPUTE: - radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); - radeon_emit(cs, 0); + radeon_emit(PKT3(PKT3_NOP, 0, 0)); + radeon_emit(0); break; default: unreachable("Incorrect queue family"); break; } + radeon_end(); + /* Make sure to wait-for-idle before stopping SQTT. */ radv_emit_wait_for_idle(device, cs, family); diff --git a/src/amd/vulkan/radv_video.c b/src/amd/vulkan/radv_video.c index 7d2b323b0b6..18762ad6bcf 100644 --- a/src/amd/vulkan/radv_video.c +++ b/src/amd/vulkan/radv_video.c @@ -67,10 +67,12 @@ radv_vcn_sq_header(struct radeon_cmdbuf *cs, struct rvcn_sq_var *sq, unsigned ty { if (!skip_signature) { /* vcn ib signature */ - radeon_emit(cs, RADEON_VCN_SIGNATURE_SIZE); - radeon_emit(cs, RADEON_VCN_SIGNATURE); - radeon_emit(cs, 0); - radeon_emit(cs, 0); + radeon_begin(cs); + radeon_emit(RADEON_VCN_SIGNATURE_SIZE); + radeon_emit(RADEON_VCN_SIGNATURE); + radeon_emit(0); + radeon_emit(0); + radeon_end(); sq->signature_ib_checksum = &cs->buf[cs->cdw - 2]; sq->signature_ib_total_size_in_dw = &cs->buf[cs->cdw - 1]; @@ -80,10 +82,12 @@ radv_vcn_sq_header(struct radeon_cmdbuf *cs, struct rvcn_sq_var *sq, unsigned ty } /* vcn ib engine info */ - radeon_emit(cs, RADEON_VCN_ENGINE_INFO_SIZE); - radeon_emit(cs, RADEON_VCN_ENGINE_INFO); - radeon_emit(cs, type); - radeon_emit(cs, 0); + radeon_begin(cs); + radeon_emit(RADEON_VCN_ENGINE_INFO_SIZE); + radeon_emit(RADEON_VCN_ENGINE_INFO); + radeon_emit(type); + radeon_emit(0); + radeon_end(); sq->engine_ib_size_of_packages = &cs->buf[cs->cdw - 1]; } @@ -1072,8 +1076,11 @@ static void set_reg(struct radv_cmd_buffer *cmd_buffer, unsigned reg, uint32_t val) { struct radeon_cmdbuf *cs = cmd_buffer->cs; - radeon_emit(cs, RDECODE_PKT0(reg >> 2, 0)); - radeon_emit(cs, val); + + radeon_begin(cs); + radeon_emit(RDECODE_PKT0(reg >> 2, 0)); + radeon_emit(val); + radeon_end(); } static void @@ -2584,8 +2591,10 @@ radv_vcn_cmd_reset(struct radv_cmd_buffer *cmd_buffer) if (pdev->vid_decode_ip != AMD_IP_VCN_UNIFIED) { radeon_check_space(device->ws, cmd_buffer->cs, 8); + radeon_begin(cmd_buffer->cs); for (unsigned i = 0; i < 8; i++) - radeon_emit(cmd_buffer->cs, 0x81ff); + radeon_emit(0x81ff); + radeon_end(); } else radv_vcn_sq_tail(cmd_buffer->cs, &cmd_buffer->video.sq); } @@ -2610,8 +2619,10 @@ radv_uvd_cmd_reset(struct radv_cmd_buffer *cmd_buffer) /* pad out the IB to the 16 dword boundary - otherwise the fw seems to be unhappy */ int padsize = vid->sessionctx.mem ? 4 : 6; radeon_check_space(device->ws, cmd_buffer->cs, padsize); + radeon_begin(cmd_buffer->cs); for (unsigned i = 0; i < padsize; i++) - radeon_emit(cmd_buffer->cs, PKT2_NOP_PAD); + radeon_emit(PKT2_NOP_PAD); + radeon_end(); } VKAPI_ATTR void VKAPI_CALL diff --git a/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c b/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c index 128e1d1c4ca..a478828df4d 100644 --- a/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c +++ b/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c @@ -101,6 +101,21 @@ struct radv_winsys_sem_info { struct radv_winsys_sem_counts signal; }; +static void +radeon_emit(struct radeon_cmdbuf *cs, uint32_t value) +{ + assert(cs->cdw < cs->reserved_dw); + cs->buf[cs->cdw++] = value; +} + +static void +radeon_emit_array(struct radeon_cmdbuf *cs, const uint32_t *values, unsigned count) +{ + assert(cs->cdw + count <= cs->reserved_dw); + memcpy(cs->buf + cs->cdw, values, count * 4); + cs->cdw += count; +} + static void radeon_emit_unchecked(struct radeon_cmdbuf *cs, uint32_t value) {