AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

radv: emit graphics pipelines directly from the cmdbuf

Browse Source 
This allows us to unify emitting monolithic graphics pipelines and
shader objects.

Though, this temporarily reduces performance in some games due to more
context rolls. But this will be fixed in the following commits by
using the recent mechanism to track context register writes.

Signed-off-by: Samuel Pitoiset <samuel.pitoiset@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/28983>
This commit is contained in:
Samuel Pitoiset
2024-05-08 16:10:54 +02:00
committed by Marge Bot
parent fca40bcce3
commit 9532b0f1b2
5 changed files with 820 additions and 942 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/amd/vulkan/radv_cmd_buffer.c
+820 -97
View File
@@ -31,6 +31,7 @@
#include "vk_util.h"
#include "ac_debug.h"
#include "ac_nir.h"
#include "ac_shader_args.h"
#include "aco_interface.h"
@@ -1914,6 +1915,821 @@ radv_emit_compute_shader(const struct radv_physical_device *pdev, struct radeon_
radeon_emit(cs, shader->info.regs.cs.compute_num_thread_z);
}
static void
radv_emit_vgt_gs_mode(struct radv_cmd_buffer *cmd_buffer)
{
const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
const struct radv_shader_info *info = &cmd_buffer->state.last_vgt_shader->info;
unsigned vgt_primitiveid_en = 0;
uint32_t vgt_gs_mode = 0;
if (info->is_ngg)
return;
if (info->stage == MESA_SHADER_GEOMETRY) {
vgt_gs_mode = ac_vgt_gs_mode(info->gs.vertices_out, pdev->info.gfx_level);
} else if (info->outinfo.export_prim_id || info->uses_prim_id) {
vgt_gs_mode = S_028A40_MODE(V_028A40_GS_SCENARIO_A);
vgt_primitiveid_en |= S_028A84_PRIMITIVEID_EN(1);
}
radeon_set_context_reg(cmd_buffer->cs, R_028A84_VGT_PRIMITIVEID_EN, vgt_primitiveid_en);
radeon_set_context_reg(cmd_buffer->cs, R_028A40_VGT_GS_MODE, vgt_gs_mode);
}
static void
radv_emit_hw_vs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *shader)
{
const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
const uint64_t va = radv_shader_get_va(shader);
radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B120_SPI_SHADER_PGM_LO_VS, 4);
radeon_emit(cmd_buffer->cs, va >> 8);
radeon_emit(cmd_buffer->cs, S_00B124_MEM_BASE(va >> 40));
radeon_emit(cmd_buffer->cs, shader->config.rsrc1);
radeon_emit(cmd_buffer->cs, shader->config.rsrc2);
radeon_set_context_reg(cmd_buffer->cs, R_0286C4_SPI_VS_OUT_CONFIG, shader->info.regs.spi_vs_out_config);
radeon_set_context_reg(cmd_buffer->cs, R_02870C_SPI_SHADER_POS_FORMAT, shader->info.regs.spi_shader_pos_format);
radeon_set_context_reg(cmd_buffer->cs, R_02881C_PA_CL_VS_OUT_CNTL, shader->info.regs.pa_cl_vs_out_cntl);
if (pdev->info.gfx_level <= GFX8)
radeon_set_context_reg(cmd_buffer->cs, R_028AB4_VGT_REUSE_OFF, shader->info.regs.vs.vgt_reuse_off);
if (pdev->info.gfx_level >= GFX7) {
radeon_set_sh_reg_idx(pdev, cmd_buffer->cs, R_00B118_SPI_SHADER_PGM_RSRC3_VS, 3,
shader->info.regs.vs.spi_shader_pgm_rsrc3_vs);
radeon_set_sh_reg(cmd_buffer->cs, R_00B11C_SPI_SHADER_LATE_ALLOC_VS,
shader->info.regs.vs.spi_shader_late_alloc_vs);
if (pdev->info.gfx_level >= GFX10) {
radeon_set_uconfig_reg(cmd_buffer->cs, R_030980_GE_PC_ALLOC, shader->info.regs.ge_pc_alloc);
if (shader->info.stage == MESA_SHADER_TESS_EVAL) {
radeon_set_context_reg(cmd_buffer->cs, R_028A44_VGT_GS_ONCHIP_CNTL, shader->info.regs.vgt_gs_onchip_cntl);
}
}
}
}
static void
radv_emit_hw_es(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *shader)
{
const uint64_t va = radv_shader_get_va(shader);
radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B320_SPI_SHADER_PGM_LO_ES, 4);
radeon_emit(cmd_buffer->cs, va >> 8);
radeon_emit(cmd_buffer->cs, S_00B324_MEM_BASE(va >> 40));
radeon_emit(cmd_buffer->cs, shader->config.rsrc1);
radeon_emit(cmd_buffer->cs, shader->config.rsrc2);
}
static void
radv_emit_hw_ls(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *shader)
{
const uint64_t va = radv_shader_get_va(shader);
radeon_set_sh_reg(cmd_buffer->cs, R_00B520_SPI_SHADER_PGM_LO_LS, va >> 8);
radeon_set_sh_reg(cmd_buffer->cs, R_00B528_SPI_SHADER_PGM_RSRC1_LS, shader->config.rsrc1);
}
static void
radv_emit_hw_ngg(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *es, const struct radv_shader *shader)
{
const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
const uint64_t va = radv_shader_get_va(shader);
gl_shader_stage es_type;
const struct gfx10_ngg_info *ngg_state = &shader->info.ngg_info;
if (shader->info.stage == MESA_SHADER_GEOMETRY) {
if (shader->info.merged_shader_compiled_separately) {
es_type = es->info.stage;
} else {
es_type = shader->info.gs.es_type;
}
} else {
es_type = shader->info.stage;
}
if (!shader->info.merged_shader_compiled_separately) {
radeon_set_sh_reg(cmd_buffer->cs, R_00B320_SPI_SHADER_PGM_LO_ES, va >> 8);
radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B228_SPI_SHADER_PGM_RSRC1_GS, 2);
radeon_emit(cmd_buffer->cs, shader->config.rsrc1);
radeon_emit(cmd_buffer->cs, shader->config.rsrc2);
}
const struct radv_vs_output_info *outinfo = &shader->info.outinfo;
const bool es_enable_prim_id = outinfo->export_prim_id || (es && es->info.uses_prim_id);
bool break_wave_at_eoi = false;
if (es_type == MESA_SHADER_TESS_EVAL) {
if (es_enable_prim_id || (shader->info.uses_prim_id))
break_wave_at_eoi = true;
}
radeon_set_context_reg(cmd_buffer->cs, R_0286C4_SPI_VS_OUT_CONFIG, shader->info.regs.spi_vs_out_config);
radeon_set_context_reg_seq(cmd_buffer->cs, R_028708_SPI_SHADER_IDX_FORMAT, 2);
radeon_emit(cmd_buffer->cs, shader->info.regs.ngg.spi_shader_idx_format);
radeon_emit(cmd_buffer->cs, shader->info.regs.spi_shader_pos_format);
radeon_set_context_reg(cmd_buffer->cs, R_02881C_PA_CL_VS_OUT_CNTL, shader->info.regs.pa_cl_vs_out_cntl);
radeon_set_context_reg(cmd_buffer->cs, R_028A84_VGT_PRIMITIVEID_EN,
shader->info.regs.ngg.vgt_primitiveid_en | S_028A84_PRIMITIVEID_EN(es_enable_prim_id));
radeon_set_context_reg(cmd_buffer->cs, R_0287FC_GE_MAX_OUTPUT_PER_SUBGROUP,
shader->info.regs.ngg.ge_max_output_per_subgroup);
radeon_set_context_reg(cmd_buffer->cs, R_028B4C_GE_NGG_SUBGRP_CNTL, shader->info.regs.ngg.ge_ngg_subgrp_cntl);
radeon_set_context_reg(cmd_buffer->cs, R_028B90_VGT_GS_INSTANCE_CNT, shader->info.regs.vgt_gs_instance_cnt);
uint32_t ge_cntl = shader->info.regs.ngg.ge_cntl;
if (pdev->info.gfx_level >= GFX11) {
ge_cntl |= S_03096C_BREAK_PRIMGRP_AT_EOI(break_wave_at_eoi);
} else {
ge_cntl |= S_03096C_BREAK_WAVE_AT_EOI(break_wave_at_eoi);
/* Bug workaround for a possible hang with non-tessellation cases.
* Tessellation always sets GE_CNTL.VERT_GRP_SIZE = 0
*
* Requirement: GE_CNTL.VERT_GRP_SIZE = VGT_GS_ONCHIP_CNTL.ES_VERTS_PER_SUBGRP - 5
*/
if (pdev->info.gfx_level == GFX10 && es_type != MESA_SHADER_TESS_EVAL && ngg_state->hw_max_esverts != 256) {
ge_cntl &= C_03096C_VERT_GRP_SIZE;
if (ngg_state->hw_max_esverts > 5) {
ge_cntl |= S_03096C_VERT_GRP_SIZE(ngg_state->hw_max_esverts - 5);
}
}
radeon_set_context_reg(cmd_buffer->cs, R_028A44_VGT_GS_ONCHIP_CNTL, shader->info.regs.vgt_gs_onchip_cntl);
}
radeon_set_uconfig_reg(cmd_buffer->cs, R_03096C_GE_CNTL, ge_cntl);
radeon_set_sh_reg_idx(pdev, cmd_buffer->cs, R_00B21C_SPI_SHADER_PGM_RSRC3_GS, 3,
shader->info.regs.spi_shader_pgm_rsrc3_gs);
radeon_set_sh_reg_idx(pdev, cmd_buffer->cs, R_00B204_SPI_SHADER_PGM_RSRC4_GS, 3,
shader->info.regs.spi_shader_pgm_rsrc4_gs);
radeon_set_uconfig_reg(cmd_buffer->cs, R_030980_GE_PC_ALLOC, shader->info.regs.ge_pc_alloc);
}
static void
radv_emit_hw_hs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *shader)
{
const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
const uint64_t va = radv_shader_get_va(shader);
if (pdev->info.gfx_level >= GFX9) {
if (pdev->info.gfx_level >= GFX10) {
radeon_set_sh_reg(cmd_buffer->cs, R_00B520_SPI_SHADER_PGM_LO_LS, va >> 8);
} else {
radeon_set_sh_reg(cmd_buffer->cs, R_00B410_SPI_SHADER_PGM_LO_LS, va >> 8);
}
radeon_set_sh_reg(cmd_buffer->cs, R_00B428_SPI_SHADER_PGM_RSRC1_HS, shader->config.rsrc1);
} else {
radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B420_SPI_SHADER_PGM_LO_HS, 4);
radeon_emit(cmd_buffer->cs, va >> 8);
radeon_emit(cmd_buffer->cs, S_00B424_MEM_BASE(va >> 40));
radeon_emit(cmd_buffer->cs, shader->config.rsrc1);
radeon_emit(cmd_buffer->cs, shader->config.rsrc2);
}
}
static void
radv_emit_vertex_shader(struct radv_cmd_buffer *cmd_buffer)
{
const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
const struct radv_shader *vs = cmd_buffer->state.shaders[MESA_SHADER_VERTEX];
if (vs->info.merged_shader_compiled_separately) {
assert(vs->info.next_stage == MESA_SHADER_TESS_CTRL || vs->info.next_stage == MESA_SHADER_GEOMETRY);
const struct radv_userdata_info *loc = &vs->info.user_sgprs_locs.shader_data[AC_UD_NEXT_STAGE_PC];
const struct radv_shader *next_stage = cmd_buffer->state.shaders[vs->info.next_stage];
const uint32_t base_reg = vs->info.user_data_0;
assert(loc->sgpr_idx != -1 && loc->num_sgprs == 1);
if (!vs->info.vs.has_prolog) {
uint32_t rsrc1, rsrc2;
if (vs->info.next_stage == MESA_SHADER_TESS_CTRL) {
radv_shader_combine_cfg_vs_tcs(vs, next_stage, &rsrc1, NULL);
if (pdev->info.gfx_level >= GFX10) {
radeon_set_sh_reg(cmd_buffer->cs, R_00B520_SPI_SHADER_PGM_LO_LS, vs->va >> 8);
} else {
radeon_set_sh_reg(cmd_buffer->cs, R_00B410_SPI_SHADER_PGM_LO_LS, vs->va >> 8);
}
radeon_set_sh_reg(cmd_buffer->cs, R_00B428_SPI_SHADER_PGM_RSRC1_HS, rsrc1);
} else {
radv_shader_combine_cfg_vs_gs(vs, next_stage, &rsrc1, &rsrc2);
if (pdev->info.gfx_level >= GFX10) {
radeon_set_sh_reg(cmd_buffer->cs, R_00B320_SPI_SHADER_PGM_LO_ES, vs->va >> 8);
} else {
radeon_set_sh_reg(cmd_buffer->cs, R_00B210_SPI_SHADER_PGM_LO_ES, vs->va >> 8);
}
unsigned lds_size;
if (next_stage->info.is_ngg) {
lds_size = DIV_ROUND_UP(next_stage->info.ngg_info.lds_size, pdev->info.lds_encode_granularity);
} else {
lds_size = next_stage->info.gs_ring_info.lds_size;
}
radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B228_SPI_SHADER_PGM_RSRC1_GS, 2);
radeon_emit(cmd_buffer->cs, rsrc1);
radeon_emit(cmd_buffer->cs, rsrc2 | S_00B22C_LDS_SIZE(lds_size));
}
}
radv_emit_shader_pointer(device, cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, next_stage->va, false);
return;
}
if (vs->info.vs.as_ls)
radv_emit_hw_ls(cmd_buffer, vs);
else if (vs->info.vs.as_es)
radv_emit_hw_es(cmd_buffer, vs);
else if (vs->info.is_ngg)
radv_emit_hw_ngg(cmd_buffer, NULL, vs);
else
radv_emit_hw_vs(cmd_buffer, vs);
}
static void
radv_emit_tess_ctrl_shader(struct radv_cmd_buffer *cmd_buffer)
{
const struct radv_shader *tcs = cmd_buffer->state.shaders[MESA_SHADER_TESS_CTRL];
if (tcs->info.merged_shader_compiled_separately) {
/* When VS+TCS are compiled separately on GFX9+, the VS will jump to the TCS and everything is
* emitted as part of the VS.
*/
return;
}
radv_emit_hw_hs(cmd_buffer, tcs);
}
static void
radv_emit_tess_eval_shader(struct radv_cmd_buffer *cmd_buffer)
{
const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
const struct radv_shader *tes = cmd_buffer->state.shaders[MESA_SHADER_TESS_EVAL];
if (tes->info.merged_shader_compiled_separately) {
assert(tes->info.next_stage == MESA_SHADER_GEOMETRY);
const struct radv_userdata_info *loc = &tes->info.user_sgprs_locs.shader_data[AC_UD_NEXT_STAGE_PC];
const struct radv_shader *gs = cmd_buffer->state.shaders[MESA_SHADER_GEOMETRY];
const uint32_t base_reg = tes->info.user_data_0;
uint32_t rsrc1, rsrc2;
assert(loc->sgpr_idx != -1 && loc->num_sgprs == 1);
radv_shader_combine_cfg_tes_gs(tes, gs, &rsrc1, &rsrc2);
radeon_set_sh_reg(cmd_buffer->cs, R_00B210_SPI_SHADER_PGM_LO_ES, tes->va >> 8);
unsigned lds_size;
if (gs->info.is_ngg) {
lds_size = DIV_ROUND_UP(gs->info.ngg_info.lds_size, pdev->info.lds_encode_granularity);
} else {
lds_size = gs->info.gs_ring_info.lds_size;
}
radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B228_SPI_SHADER_PGM_RSRC1_GS, 2);
radeon_emit(cmd_buffer->cs, rsrc1);
radeon_emit(cmd_buffer->cs, rsrc2 | S_00B22C_LDS_SIZE(lds_size));
radv_emit_shader_pointer(device, cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, gs->va, false);
return;
}
if (tes->info.is_ngg) {
radv_emit_hw_ngg(cmd_buffer, NULL, tes);
} else if (tes->info.tes.as_es) {
radv_emit_hw_es(cmd_buffer, tes);
} else {
radv_emit_hw_vs(cmd_buffer, tes);
}
}
static void
radv_emit_hw_gs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *gs)
{
const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
const struct radv_legacy_gs_info *gs_state = &gs->info.gs_ring_info;
const uint64_t va = radv_shader_get_va(gs);
radeon_set_context_reg_seq(cmd_buffer->cs, R_028A60_VGT_GSVS_RING_OFFSET_1, 3);
radeon_emit(cmd_buffer->cs, gs->info.regs.gs.vgt_gsvs_ring_offset[0]);
radeon_emit(cmd_buffer->cs, gs->info.regs.gs.vgt_gsvs_ring_offset[1]);
radeon_emit(cmd_buffer->cs, gs->info.regs.gs.vgt_gsvs_ring_offset[2]);
radeon_set_context_reg(cmd_buffer->cs, R_028AB0_VGT_GSVS_RING_ITEMSIZE, gs->info.regs.gs.vgt_gsvs_ring_itemsize);
radeon_set_context_reg_seq(cmd_buffer->cs, R_028B5C_VGT_GS_VERT_ITEMSIZE, 4);
radeon_emit(cmd_buffer->cs, gs->info.regs.gs.vgt_gs_vert_itemsize[0]);
radeon_emit(cmd_buffer->cs, gs->info.regs.gs.vgt_gs_vert_itemsize[1]);
radeon_emit(cmd_buffer->cs, gs->info.regs.gs.vgt_gs_vert_itemsize[2]);
radeon_emit(cmd_buffer->cs, gs->info.regs.gs.vgt_gs_vert_itemsize[3]);
radeon_set_context_reg(cmd_buffer->cs, R_028B90_VGT_GS_INSTANCE_CNT, gs->info.regs.gs.vgt_gs_instance_cnt);
if (pdev->info.gfx_level >= GFX9) {
if (!gs->info.merged_shader_compiled_separately) {
if (pdev->info.gfx_level >= GFX10) {
radeon_set_sh_reg(cmd_buffer->cs, R_00B320_SPI_SHADER_PGM_LO_ES, va >> 8);
} else {
radeon_set_sh_reg(cmd_buffer->cs, R_00B210_SPI_SHADER_PGM_LO_ES, va >> 8);
}
radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B228_SPI_SHADER_PGM_RSRC1_GS, 2);
radeon_emit(cmd_buffer->cs, gs->config.rsrc1);
radeon_emit(cmd_buffer->cs, gs->config.rsrc2 | S_00B22C_LDS_SIZE(gs_state->lds_size));
}
radeon_set_context_reg(cmd_buffer->cs, R_028A44_VGT_GS_ONCHIP_CNTL, gs->info.regs.vgt_gs_onchip_cntl);
radeon_set_context_reg(cmd_buffer->cs, R_028A94_VGT_GS_MAX_PRIMS_PER_SUBGROUP,
gs->info.regs.gs.vgt_gs_max_prims_per_subgroup);
} else {
radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B220_SPI_SHADER_PGM_LO_GS, 4);
radeon_emit(cmd_buffer->cs, va >> 8);
radeon_emit(cmd_buffer->cs, S_00B224_MEM_BASE(va >> 40));
radeon_emit(cmd_buffer->cs, gs->config.rsrc1);
radeon_emit(cmd_buffer->cs, gs->config.rsrc2);
/* GFX6-8: ESGS offchip ring buffer is allocated according to VGT_ESGS_RING_ITEMSIZE.
* GFX9+: Only used to set the GS input VGPRs, emulated in shaders.
*/
radeon_set_context_reg(cmd_buffer->cs, R_028AAC_VGT_ESGS_RING_ITEMSIZE, gs->info.regs.gs.vgt_esgs_ring_itemsize);
}
radeon_set_sh_reg_idx(pdev, cmd_buffer->cs, R_00B21C_SPI_SHADER_PGM_RSRC3_GS, 3,
gs->info.regs.spi_shader_pgm_rsrc3_gs);
if (pdev->info.gfx_level >= GFX10) {
radeon_set_sh_reg_idx(pdev, cmd_buffer->cs, R_00B204_SPI_SHADER_PGM_RSRC4_GS, 3,
gs->info.regs.spi_shader_pgm_rsrc4_gs);
}
}
static void
radv_emit_geometry_shader(struct radv_cmd_buffer *cmd_buffer)
{
const struct radv_shader *gs = cmd_buffer->state.shaders[MESA_SHADER_GEOMETRY];
const struct radv_shader *es = cmd_buffer->state.shaders[MESA_SHADER_TESS_EVAL]
? cmd_buffer->state.shaders[MESA_SHADER_TESS_EVAL]
: cmd_buffer->state.shaders[MESA_SHADER_VERTEX];
if (gs->info.is_ngg) {
radv_emit_hw_ngg(cmd_buffer, es, gs);
} else {
radv_emit_hw_gs(cmd_buffer, gs);
radv_emit_hw_vs(cmd_buffer, cmd_buffer->state.gs_copy_shader);
}
radeon_set_context_reg(cmd_buffer->cs, R_028B38_VGT_GS_MAX_VERT_OUT, gs->info.regs.vgt_gs_max_vert_out);
if (gs->info.merged_shader_compiled_separately) {
const struct radv_userdata_info *vgt_esgs_ring_itemsize = radv_get_user_sgpr(gs, AC_UD_VGT_ESGS_RING_ITEMSIZE);
assert(vgt_esgs_ring_itemsize->sgpr_idx != -1 && vgt_esgs_ring_itemsize->num_sgprs == 1);
radeon_set_sh_reg(cmd_buffer->cs, gs->info.user_data_0 + vgt_esgs_ring_itemsize->sgpr_idx * 4,
es->info.esgs_itemsize / 4);
if (gs->info.is_ngg) {
const struct radv_userdata_info *ngg_lds_layout = radv_get_user_sgpr(gs, AC_UD_NGG_LDS_LAYOUT);
assert(ngg_lds_layout->sgpr_idx != -1 && ngg_lds_layout->num_sgprs == 1);
assert(!(gs->info.ngg_info.esgs_ring_size & 0xffff0000) && !(gs->info.ngg_info.scratch_lds_base & 0xffff0000));
radeon_set_sh_reg(cmd_buffer->cs, gs->info.user_data_0 + ngg_lds_layout->sgpr_idx * 4,
SET_SGPR_FIELD(NGG_LDS_LAYOUT_GS_OUT_VERTEX_BASE, gs->info.ngg_info.esgs_ring_size) |
SET_SGPR_FIELD(NGG_LDS_LAYOUT_SCRATCH_BASE, gs->info.ngg_info.scratch_lds_base));
}
}
}
static void
radv_emit_vgt_gs_out(struct radv_cmd_buffer *cmd_buffer, uint32_t vgt_gs_out_prim_type)
{
const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
if (pdev->info.gfx_level >= GFX11) {
radeon_set_uconfig_reg(cmd_buffer->cs, R_030998_VGT_GS_OUT_PRIM_TYPE, vgt_gs_out_prim_type);
} else {
radeon_set_context_reg(cmd_buffer->cs, R_028A6C_VGT_GS_OUT_PRIM_TYPE, vgt_gs_out_prim_type);
}
}
static void
radv_emit_mesh_shader(struct radv_cmd_buffer *cmd_buffer)
{
const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
const struct radv_shader *ms = cmd_buffer->state.shaders[MESA_SHADER_MESH];
const uint32_t gs_out = radv_conv_gl_prim_to_gs_out(ms->info.ms.output_prim);
radv_emit_hw_ngg(cmd_buffer, NULL, ms);
radeon_set_context_reg(cmd_buffer->cs, R_028B38_VGT_GS_MAX_VERT_OUT, ms->info.regs.vgt_gs_max_vert_out);
radeon_set_uconfig_reg_idx(pdev, cmd_buffer->cs, R_030908_VGT_PRIMITIVE_TYPE, 1, V_008958_DI_PT_POINTLIST);
if (pdev->mesh_fast_launch_2) {
radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B2B0_SPI_SHADER_GS_MESHLET_DIM, 2);
radeon_emit(cmd_buffer->cs, ms->info.regs.ms.spi_shader_gs_meshlet_dim);
radeon_emit(cmd_buffer->cs, ms->info.regs.ms.spi_shader_gs_meshlet_exp_alloc);
}
radv_emit_vgt_gs_out(cmd_buffer, gs_out);
}
enum radv_ps_in_type {
radv_ps_in_interpolated,
radv_ps_in_flat,
radv_ps_in_explicit,
radv_ps_in_explicit_strict,
radv_ps_in_interpolated_fp16,
radv_ps_in_interpolated_fp16_hi,
radv_ps_in_per_prim_gfx103,
radv_ps_in_per_prim_gfx11,
};
static uint32_t
offset_to_ps_input(const uint32_t offset, const enum radv_ps_in_type type)
{
assert(offset != AC_EXP_PARAM_UNDEFINED);
if (offset >= AC_EXP_PARAM_DEFAULT_VAL_0000 && offset <= AC_EXP_PARAM_DEFAULT_VAL_1111) {
/* The input is a DEFAULT_VAL constant. */
return S_028644_OFFSET(0x20) | S_028644_DEFAULT_VAL(offset - AC_EXP_PARAM_DEFAULT_VAL_0000);
}
assert(offset <= AC_EXP_PARAM_OFFSET_31);
uint32_t ps_input_cntl = S_028644_OFFSET(offset);
switch (type) {
case radv_ps_in_explicit_strict:
/* Rotate parameter cache contents to strict vertex order. */
ps_input_cntl |= S_028644_ROTATE_PC_PTR(1);
FALLTHROUGH;
case radv_ps_in_explicit:
/* Force parameter cache to be read in passthrough mode. */
ps_input_cntl |= S_028644_OFFSET(1 << 5);
FALLTHROUGH;
case radv_ps_in_flat:
ps_input_cntl |= S_028644_FLAT_SHADE(1);
break;
case radv_ps_in_interpolated_fp16_hi:
ps_input_cntl |= S_028644_ATTR1_VALID(1);
FALLTHROUGH;
case radv_ps_in_interpolated_fp16:
/* These must be set even if only the high 16 bits are used. */
ps_input_cntl |= S_028644_FP16_INTERP_MODE(1) | S_028644_ATTR0_VALID(1);
break;
case radv_ps_in_per_prim_gfx11:
ps_input_cntl |= S_028644_PRIM_ATTR(1);
break;
case radv_ps_in_interpolated:
case radv_ps_in_per_prim_gfx103:
break;
}
return ps_input_cntl;
}
static void
slot_to_ps_input(const struct radv_vs_output_info *outinfo, unsigned slot, uint32_t *ps_input_cntl, unsigned *ps_offset,
const bool use_default_0, const enum radv_ps_in_type type)
{
unsigned vs_offset = outinfo->vs_output_param_offset[slot];
if (vs_offset == AC_EXP_PARAM_UNDEFINED) {
if (use_default_0)
vs_offset = AC_EXP_PARAM_DEFAULT_VAL_0000;
else
return;
}
ps_input_cntl[*ps_offset] = offset_to_ps_input(vs_offset, type);
++(*ps_offset);
}
static void
input_mask_to_ps_inputs(const struct radv_vs_output_info *outinfo, const struct radv_shader *ps, uint32_t input_mask,
uint32_t *ps_input_cntl, unsigned *ps_offset, const enum radv_ps_in_type default_type)
{
u_foreach_bit (i, input_mask) {
unsigned vs_offset = outinfo->vs_output_param_offset[VARYING_SLOT_VAR0 + i];
if (vs_offset == AC_EXP_PARAM_UNDEFINED) {
ps_input_cntl[*ps_offset] = S_028644_OFFSET(0x20);
++(*ps_offset);
continue;
}
enum radv_ps_in_type type = default_type;
if (ps->info.ps.flat_shaded_mask & BITFIELD_BIT(*ps_offset))
type = radv_ps_in_flat;
else if (ps->info.ps.explicit_shaded_mask & BITFIELD_BIT(*ps_offset))
type = radv_ps_in_explicit;
else if (ps->info.ps.explicit_strict_shaded_mask & BITFIELD_BIT(*ps_offset))
type = radv_ps_in_explicit_strict;
else if (ps->info.ps.float16_hi_shaded_mask & BITFIELD_BIT(*ps_offset))
type = radv_ps_in_interpolated_fp16_hi;
else if (ps->info.ps.float16_shaded_mask & BITFIELD_BIT(*ps_offset))
type = radv_ps_in_interpolated_fp16;
ps_input_cntl[*ps_offset] = offset_to_ps_input(vs_offset, type);
++(*ps_offset);
}
}
static void
radv_emit_ps_inputs(struct radv_cmd_buffer *cmd_buffer)
{
const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
const struct radv_shader *ps = cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT];
const struct radv_shader *last_vgt_shader = cmd_buffer->state.last_vgt_shader;
const struct radv_vs_output_info *outinfo = &last_vgt_shader->info.outinfo;
const bool mesh = last_vgt_shader->info.stage == MESA_SHADER_MESH;
const bool gfx11plus = pdev->info.gfx_level >= GFX11;
const enum radv_ps_in_type per_prim = gfx11plus ? radv_ps_in_per_prim_gfx11 : radv_ps_in_per_prim_gfx103;
uint32_t ps_input_cntl[32];
unsigned ps_offset = 0;
if (ps->info.ps.prim_id_input && !mesh)
slot_to_ps_input(outinfo, VARYING_SLOT_PRIMITIVE_ID, ps_input_cntl, &ps_offset, false, radv_ps_in_flat);
if (ps->info.ps.layer_input && !mesh)
slot_to_ps_input(outinfo, VARYING_SLOT_LAYER, ps_input_cntl, &ps_offset, true, radv_ps_in_flat);
if (ps->info.ps.viewport_index_input && !mesh)
slot_to_ps_input(outinfo, VARYING_SLOT_VIEWPORT, ps_input_cntl, &ps_offset, true, radv_ps_in_flat);
if (ps->info.ps.has_pcoord)
ps_input_cntl[ps_offset++] = S_028644_PT_SPRITE_TEX(1) | S_028644_OFFSET(0x20);
if (ps->info.ps.input_clips_culls_mask & 0x0f)
slot_to_ps_input(outinfo, VARYING_SLOT_CLIP_DIST0, ps_input_cntl, &ps_offset, false, radv_ps_in_interpolated);
if (ps->info.ps.input_clips_culls_mask & 0xf0)
slot_to_ps_input(outinfo, VARYING_SLOT_CLIP_DIST1, ps_input_cntl, &ps_offset, false, radv_ps_in_interpolated);
input_mask_to_ps_inputs(outinfo, ps, ps->info.ps.input_mask, ps_input_cntl, &ps_offset, radv_ps_in_interpolated);
/* Per-primitive PS inputs: the HW needs these to be last. */
if (ps->info.ps.prim_id_input && mesh)
slot_to_ps_input(outinfo, VARYING_SLOT_PRIMITIVE_ID, ps_input_cntl, &ps_offset, false, per_prim);
if (ps->info.ps.layer_input && mesh)
slot_to_ps_input(outinfo, VARYING_SLOT_LAYER, ps_input_cntl, &ps_offset, true, per_prim);
if (ps->info.ps.viewport_index_input && mesh)
slot_to_ps_input(outinfo, VARYING_SLOT_VIEWPORT, ps_input_cntl, &ps_offset, true, per_prim);
input_mask_to_ps_inputs(outinfo, ps, ps->info.ps.input_per_primitive_mask, ps_input_cntl, &ps_offset, per_prim);
if (ps_offset) {
radeon_set_context_reg_seq(cmd_buffer->cs, R_028644_SPI_PS_INPUT_CNTL_0, ps_offset);
for (unsigned i = 0; i < ps_offset; i++) {
radeon_emit(cmd_buffer->cs, ps_input_cntl[i]);
}
}
}
static void
radv_emit_fragment_shader(struct radv_cmd_buffer *cmd_buffer)
{
const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
const struct radv_shader *ps = cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT];
const uint64_t va = radv_shader_get_va(ps);
radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B020_SPI_SHADER_PGM_LO_PS, 4);
radeon_emit(cmd_buffer->cs, va >> 8);
radeon_emit(cmd_buffer->cs, S_00B024_MEM_BASE(va >> 40));
radeon_emit(cmd_buffer->cs, ps->config.rsrc1);
radeon_emit(cmd_buffer->cs, ps->config.rsrc2);
radeon_set_context_reg_seq(cmd_buffer->cs, R_0286CC_SPI_PS_INPUT_ENA, 2);
radeon_emit(cmd_buffer->cs, ps->config.spi_ps_input_ena);
radeon_emit(cmd_buffer->cs, ps->config.spi_ps_input_addr);
radeon_set_context_reg(cmd_buffer->cs, R_0286D8_SPI_PS_IN_CONTROL, ps->info.regs.ps.spi_ps_in_control);
radeon_set_context_reg(cmd_buffer->cs, R_028710_SPI_SHADER_Z_FORMAT, ps->info.regs.ps.spi_shader_z_format);
if (pdev->info.gfx_level >= GFX9 && pdev->info.gfx_level < GFX11)
radeon_set_context_reg(cmd_buffer->cs, R_028C40_PA_SC_SHADER_CONTROL, ps->info.regs.ps.pa_sc_shader_control);
}
static void
radv_emit_vgt_reuse(struct radv_cmd_buffer *cmd_buffer, const struct radv_vgt_shader_key *key)
{
const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
const struct radv_shader *tes = radv_get_shader(cmd_buffer->state.shaders, MESA_SHADER_TESS_EVAL);
if (pdev->info.gfx_level == GFX10_3) {
/* Legacy Tess+GS should disable reuse to prevent hangs on GFX10.3. */
const bool has_legacy_tess_gs = key->tess && key->gs && !key->ngg;
radeon_set_context_reg(cmd_buffer->cs, R_028AB4_VGT_REUSE_OFF, S_028AB4_REUSE_OFF(has_legacy_tess_gs));
}
if (pdev->info.family >= CHIP_POLARIS10 && pdev->info.gfx_level < GFX10) {
unsigned vtx_reuse_depth = 30;
if (tes && tes->info.tes.spacing == TESS_SPACING_FRACTIONAL_ODD) {
vtx_reuse_depth = 14;
}
radeon_set_context_reg(cmd_buffer->cs, R_028C58_VGT_VERTEX_REUSE_BLOCK_CNTL,
S_028C58_VTX_REUSE_DEPTH(vtx_reuse_depth));
}
}
static void
radv_emit_vgt_shader_config(struct radv_cmd_buffer *cmd_buffer, const struct radv_vgt_shader_key *key)
{
const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
uint32_t stages = 0;
if (key->tess) {
stages |= S_028B54_LS_EN(V_028B54_LS_STAGE_ON) | S_028B54_HS_EN(1) | S_028B54_DYNAMIC_HS(1);
if (key->gs)
stages |= S_028B54_ES_EN(V_028B54_ES_STAGE_DS) | S_028B54_GS_EN(1);
else if (key->ngg)
stages |= S_028B54_ES_EN(V_028B54_ES_STAGE_DS);
else
stages |= S_028B54_VS_EN(V_028B54_VS_STAGE_DS);
} else if (key->gs) {
stages |= S_028B54_ES_EN(V_028B54_ES_STAGE_REAL) | S_028B54_GS_EN(1);
} else if (key->mesh) {
assert(!key->ngg_passthrough);
unsigned gs_fast_launch = pdev->mesh_fast_launch_2 ? 2 : 1;
stages |=
S_028B54_GS_EN(1) | S_028B54_GS_FAST_LAUNCH(gs_fast_launch) | S_028B54_NGG_WAVE_ID_EN(key->mesh_scratch_ring);
} else if (key->ngg) {
stages |= S_028B54_ES_EN(V_028B54_ES_STAGE_REAL);
}
if (key->ngg) {
stages |= S_028B54_PRIMGEN_EN(1) | S_028B54_NGG_WAVE_ID_EN(key->ngg_streamout) |
S_028B54_PRIMGEN_PASSTHRU_EN(key->ngg_passthrough) |
S_028B54_PRIMGEN_PASSTHRU_NO_MSG(key->ngg_passthrough && pdev->info.family >= CHIP_NAVI23);
} else if (key->gs) {
stages |= S_028B54_VS_EN(V_028B54_VS_STAGE_COPY_SHADER);
}
if (pdev->info.gfx_level >= GFX9)
stages |= S_028B54_MAX_PRIMGRP_IN_WAVE(2);
if (pdev->info.gfx_level >= GFX10) {
stages |= S_028B54_HS_W32_EN(key->hs_wave32) | S_028B54_GS_W32_EN(key->gs_wave32) |
S_028B54_VS_W32_EN(pdev->info.gfx_level < GFX11 && key->vs_wave32);
/* Legacy GS only supports Wave64. Read it as an implication. */
assert(!(key->gs && !key->ngg) || !key->gs_wave32);
}
radeon_set_context_reg(cmd_buffer->cs, R_028B54_VGT_SHADER_STAGES_EN, stages);
}
static void
gfx103_emit_vgt_draw_payload_cntl(struct radv_cmd_buffer *cmd_buffer)
{
const struct radv_shader *mesh_shader = cmd_buffer->state.shaders[MESA_SHADER_MESH];
const bool enable_vrs = cmd_buffer->state.uses_vrs;
bool enable_prim_payload = false;
/* Enables the second channel of the primitive export instruction.
* This channel contains: VRS rate x, y, viewport and layer.
*/
if (mesh_shader) {
const struct radv_vs_output_info *outinfo = &mesh_shader->info.outinfo;
enable_prim_payload = (outinfo->writes_viewport_index_per_primitive || outinfo->writes_layer_per_primitive ||
outinfo->writes_primitive_shading_rate_per_primitive);
}
radeon_set_context_reg(cmd_buffer->cs, R_028A98_VGT_DRAW_PAYLOAD_CNTL,
S_028A98_EN_VRS_RATE(enable_vrs) | S_028A98_EN_PRIM_PAYLOAD(enable_prim_payload));
}
static void
gfx103_emit_vrs_state(struct radv_cmd_buffer *cmd_buffer)
{
const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
const struct radv_shader *ps = cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT];
const bool force_vrs_per_vertex = cmd_buffer->state.last_vgt_shader->info.force_vrs_per_vertex;
const bool enable_vrs_coarse_shading = cmd_buffer->state.uses_vrs_coarse_shading;
uint32_t mode = V_028064_SC_VRS_COMB_MODE_PASSTHRU;
uint8_t rate_x = 0, rate_y = 0;
if (enable_vrs_coarse_shading) {
/* When per-draw VRS is not enabled at all, try enabling VRS coarse shading 2x2 if the driver
* determined that it's safe to enable.
*/
mode = V_028064_SC_VRS_COMB_MODE_OVERRIDE;
rate_x = rate_y = 1;
} else if (force_vrs_per_vertex) {
/* Otherwise, if per-draw VRS is not enabled statically, try forcing per-vertex VRS if
* requested by the user. Note that vkd3d-proton always has to declare VRS as dynamic because
* in DX12 it's fully dynamic.
*/
radeon_set_context_reg(cmd_buffer->cs, R_028848_PA_CL_VRS_CNTL,
S_028848_SAMPLE_ITER_COMBINER_MODE(V_028848_SC_VRS_COMB_MODE_OVERRIDE) |
S_028848_VERTEX_RATE_COMBINER_MODE(V_028848_SC_VRS_COMB_MODE_OVERRIDE));
/* If the shader is using discard, turn off coarse shading because discard at 2x2 pixel
* granularity degrades quality too much. MIN allows sample shading but not coarse shading.
*/
mode = ps->info.ps.can_discard ? V_028064_SC_VRS_COMB_MODE_MIN : V_028064_SC_VRS_COMB_MODE_PASSTHRU;
}
if (pdev->info.gfx_level < GFX11) {
radeon_set_context_reg(cmd_buffer->cs, R_028064_DB_VRS_OVERRIDE_CNTL,
S_028064_VRS_OVERRIDE_RATE_COMBINER_MODE(mode) | S_028064_VRS_OVERRIDE_RATE_X(rate_x) |
S_028064_VRS_OVERRIDE_RATE_Y(rate_y));
}
}
static void
radv_emit_graphics_shaders(struct radv_cmd_buffer *cmd_buffer)
{
struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
radv_foreach_stage(s, cmd_buffer->state.active_stages & RADV_GRAPHICS_STAGE_BITS)
{
switch (s) {
case MESA_SHADER_VERTEX:
radv_emit_vertex_shader(cmd_buffer);
break;
case MESA_SHADER_TESS_CTRL:
radv_emit_tess_ctrl_shader(cmd_buffer);
break;
case MESA_SHADER_TESS_EVAL:
radv_emit_tess_eval_shader(cmd_buffer);
break;
case MESA_SHADER_GEOMETRY:
radv_emit_geometry_shader(cmd_buffer);
break;
case MESA_SHADER_FRAGMENT:
radv_emit_fragment_shader(cmd_buffer);
radv_emit_ps_inputs(cmd_buffer);
break;
case MESA_SHADER_MESH:
radv_emit_mesh_shader(cmd_buffer);
break;
case MESA_SHADER_TASK:
radv_emit_compute_shader(pdev, cmd_buffer->gang.cs, cmd_buffer->state.shaders[MESA_SHADER_TASK]);
break;
default:
unreachable("invalid bind stage");
}
}
const struct radv_vgt_shader_key vgt_shader_cfg_key =
radv_get_vgt_shader_key(device, cmd_buffer->state.shaders, cmd_buffer->state.gs_copy_shader);
radv_emit_vgt_gs_mode(cmd_buffer);
radv_emit_vgt_reuse(cmd_buffer, &vgt_shader_cfg_key);
radv_emit_vgt_shader_config(cmd_buffer, &vgt_shader_cfg_key);
if (pdev->info.gfx_level >= GFX10_3) {
gfx103_emit_vgt_draw_payload_cntl(cmd_buffer);
gfx103_emit_vrs_state(cmd_buffer);
}
cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_GRAPHICS_SHADERS;
}
static void
radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer)
{
@@ -1948,15 +2764,7 @@ radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer)
cmd_buffer->state.dirty |= RADV_CMD_DIRTY_FRAMEBUFFER;
}
radeon_emit_array(cmd_buffer->cs, pipeline->base.cs.buf, pipeline->base.cs.cdw);
if (!cmd_buffer->state.emitted_graphics_pipeline ||
cmd_buffer->state.emitted_graphics_pipeline->base.ctx_cs.cdw != pipeline->base.ctx_cs.cdw ||
cmd_buffer->state.emitted_graphics_pipeline->base.ctx_cs_hash != pipeline->base.ctx_cs_hash ||
memcmp(cmd_buffer->state.emitted_graphics_pipeline->base.ctx_cs.buf, pipeline->base.ctx_cs.buf,
pipeline->base.ctx_cs.cdw * 4)) {
radeon_emit_array(cmd_buffer->cs, pipeline->base.ctx_cs.buf, pipeline->base.ctx_cs.cdw);
}
radv_emit_graphics_shaders(cmd_buffer);
if (device->pbb_allowed) {
const struct radv_binning_settings *settings = &pdev->binning_settings;
@@ -1974,14 +2782,9 @@ radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer)
if (pipeline->sqtt_shaders_reloc) {
/* Emit shaders relocation because RGP requires them to be contiguous in memory. */
radv_sqtt_emit_relocated_shaders(cmd_buffer, pipeline);
}
struct radv_shader *task_shader = cmd_buffer->state.shaders[MESA_SHADER_TASK];
if (task_shader) {
radv_emit_compute_shader(pdev, cmd_buffer->gang.cs, task_shader);
/* Relocate the task shader because RGP requires shaders to be contiguous in memory. */
if (pipeline->sqtt_shaders_reloc) {
struct radv_shader *task_shader = cmd_buffer->state.shaders[MESA_SHADER_TASK];
if (task_shader) {
const struct radv_sqtt_shaders_reloc *reloc = pipeline->sqtt_shaders_reloc;
const uint64_t va = reloc->va[MESA_SHADER_TASK];
@@ -2340,7 +3143,7 @@ radv_emit_primitive_topology(struct radv_cmd_buffer *cmd_buffer)
radeon_set_config_reg(cmd_buffer->cs, R_008958_VGT_PRIMITIVE_TYPE, d->vk.ia.primitive_topology);
}
radv_emit_vgt_gs_out(device, cmd_buffer->cs, cmd_buffer->cs, vgt_gs_out_prim_type);
radv_emit_vgt_gs_out(cmd_buffer, vgt_gs_out_prim_type);
if (loc->sgpr_idx == -1)
return;
@@ -9482,86 +10285,6 @@ radv_cmdbuf_get_last_vgt_api_stage(const struct radv_cmd_buffer *cmd_buffer)
return util_last_bit(cmd_buffer->state.active_stages & BITFIELD_MASK(MESA_SHADER_FRAGMENT)) - 1;
}
static void
radv_emit_graphics_shaders(struct radv_cmd_buffer *cmd_buffer)
{
struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
const struct radv_physical_device *pdev = radv_device_physical(device);
const struct radv_shader *last_vgt_shader = cmd_buffer->state.last_vgt_shader;
struct radeon_cmdbuf *cs = cmd_buffer->cs;
radv_foreach_stage(s, cmd_buffer->state.active_stages & RADV_GRAPHICS_STAGE_BITS)
{
switch (s) {
case MESA_SHADER_VERTEX: {
const struct radv_shader *vs = cmd_buffer->state.shaders[MESA_SHADER_VERTEX];
struct radv_shader *next_stage = NULL;
if (vs->info.merged_shader_compiled_separately) {
assert(vs->info.next_stage == MESA_SHADER_TESS_CTRL || vs->info.next_stage == MESA_SHADER_GEOMETRY);
next_stage = cmd_buffer->state.shaders[vs->info.next_stage];
}
radv_emit_vertex_shader(device, cs, cs, vs, next_stage);
break;
}
case MESA_SHADER_TESS_CTRL:
radv_emit_tess_ctrl_shader(device, cs, cmd_buffer->state.shaders[MESA_SHADER_TESS_CTRL]);
break;
case MESA_SHADER_TESS_EVAL: {
const struct radv_shader *tes = cmd_buffer->state.shaders[MESA_SHADER_TESS_EVAL];
struct radv_shader *gs = NULL;
if (tes->info.merged_shader_compiled_separately) {
assert(tes->info.next_stage == MESA_SHADER_GEOMETRY);
gs = cmd_buffer->state.shaders[MESA_SHADER_GEOMETRY];
}
radv_emit_tess_eval_shader(device, cs, cs, tes, gs);
break;
}
case MESA_SHADER_GEOMETRY: {
struct radv_shader *es = cmd_buffer->state.shaders[MESA_SHADER_TESS_EVAL]
? cmd_buffer->state.shaders[MESA_SHADER_TESS_EVAL]
: cmd_buffer->state.shaders[MESA_SHADER_VERTEX];
radv_emit_geometry_shader(device, cs, cs, cmd_buffer->state.shaders[MESA_SHADER_GEOMETRY], es,
cmd_buffer->state.gs_copy_shader);
break;
}
case MESA_SHADER_FRAGMENT:
radv_emit_fragment_shader(device, cs, cs, cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT]);
radv_emit_ps_inputs(device, cs, last_vgt_shader, cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT]);
break;
case MESA_SHADER_MESH:
radv_emit_mesh_shader(device, cs, cs, cmd_buffer->state.shaders[MESA_SHADER_MESH]);
break;
case MESA_SHADER_TASK:
radv_emit_compute_shader(pdev, cmd_buffer->gang.cs, cmd_buffer->state.shaders[MESA_SHADER_TASK]);
break;
default:
unreachable("invalid bind stage");
}
}
/* Emit graphics states related to shaders. */
const struct radv_vgt_shader_key vgt_shader_cfg_key =
radv_get_vgt_shader_key(device, cmd_buffer->state.shaders, cmd_buffer->state.gs_copy_shader);
radv_emit_vgt_gs_mode(device, cs, last_vgt_shader);
radv_emit_vgt_reuse(device, cs, radv_get_shader(cmd_buffer->state.shaders, MESA_SHADER_TESS_EVAL),
&vgt_shader_cfg_key);
radv_emit_vgt_shader_config(device, cs, &vgt_shader_cfg_key);
if (pdev->info.gfx_level >= GFX10_3) {
gfx103_emit_vgt_draw_payload_cntl(cs, cmd_buffer->state.shaders[MESA_SHADER_MESH], cmd_buffer->state.uses_vrs);
gfx103_emit_vrs_state(device, cs, cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT],
cmd_buffer->state.uses_vrs_coarse_shading, last_vgt_shader->info.force_vrs_per_vertex);
}
cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_GRAPHICS_SHADERS;
}
static void
radv_emit_color_output_state(struct radv_cmd_buffer *cmd_buffer)
{
src/amd/vulkan/radv_pipeline.c
-3
View File
@@ -97,9 +97,6 @@ radv_pipeline_destroy(struct radv_device *device, struct radv_pipeline *pipeline
unreachable("invalid pipeline type");
}
if (pipeline->cs.buf)
free(pipeline->cs.buf);
radv_rmv_log_resource_destroy(device, (uint64_t)radv_pipeline_to_handle(pipeline));
vk_object_base_finish(&pipeline->base);
vk_free2(&device->vk.alloc, allocator, pipeline);
src/amd/vulkan/radv_pipeline.h
-4
View File
@@ -51,10 +51,6 @@ struct radv_pipeline {
struct radv_shader *shaders[MESA_VULKAN_SHADER_STAGES];
struct radv_shader *gs_copy_shader;
struct radeon_cmdbuf cs;
uint32_t ctx_cs_hash;
struct radeon_cmdbuf ctx_cs;
uint32_t user_data_0[MESA_VULKAN_SHADER_STAGES];
/* Unique pipeline hash identifier. */
src/amd/vulkan/radv_pipeline_graphics.c
-798
View File
@@ -2753,624 +2753,6 @@ done:
return result;
}
void
radv_emit_vgt_gs_mode(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs,
const struct radv_shader *last_vgt_api_shader)
{
const struct radv_physical_device *pdev = radv_device_physical(device);
const struct radv_shader_info *info = &last_vgt_api_shader->info;
unsigned vgt_primitiveid_en = 0;
uint32_t vgt_gs_mode = 0;
if (info->is_ngg)
return;
if (info->stage == MESA_SHADER_GEOMETRY) {
vgt_gs_mode = ac_vgt_gs_mode(info->gs.vertices_out, pdev->info.gfx_level);
} else if (info->outinfo.export_prim_id || info->uses_prim_id) {
vgt_gs_mode = S_028A40_MODE(V_028A40_GS_SCENARIO_A);
vgt_primitiveid_en |= S_028A84_PRIMITIVEID_EN(1);
}
radeon_set_context_reg(ctx_cs, R_028A84_VGT_PRIMITIVEID_EN, vgt_primitiveid_en);
radeon_set_context_reg(ctx_cs, R_028A40_VGT_GS_MODE, vgt_gs_mode);
}
static void
radv_emit_hw_vs(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, struct radeon_cmdbuf *cs,
const struct radv_shader *shader)
{
const struct radv_physical_device *pdev = radv_device_physical(device);
uint64_t va = radv_shader_get_va(shader);
radeon_set_sh_reg_seq(cs, R_00B120_SPI_SHADER_PGM_LO_VS, 4);
radeon_emit(cs, va >> 8);
radeon_emit(cs, S_00B124_MEM_BASE(va >> 40));
radeon_emit(cs, shader->config.rsrc1);
radeon_emit(cs, shader->config.rsrc2);
radeon_set_context_reg(ctx_cs, R_0286C4_SPI_VS_OUT_CONFIG, shader->info.regs.spi_vs_out_config);
radeon_set_context_reg(ctx_cs, R_02870C_SPI_SHADER_POS_FORMAT, shader->info.regs.spi_shader_pos_format);
radeon_set_context_reg(ctx_cs, R_02881C_PA_CL_VS_OUT_CNTL, shader->info.regs.pa_cl_vs_out_cntl);
if (pdev->info.gfx_level <= GFX8)
radeon_set_context_reg(ctx_cs, R_028AB4_VGT_REUSE_OFF, shader->info.regs.vs.vgt_reuse_off);
if (pdev->info.gfx_level >= GFX7) {
radeon_set_sh_reg_idx(pdev, cs, R_00B118_SPI_SHADER_PGM_RSRC3_VS, 3,
shader->info.regs.vs.spi_shader_pgm_rsrc3_vs);
radeon_set_sh_reg(cs, R_00B11C_SPI_SHADER_LATE_ALLOC_VS, shader->info.regs.vs.spi_shader_late_alloc_vs);
if (pdev->info.gfx_level >= GFX10) {
radeon_set_uconfig_reg(cs, R_030980_GE_PC_ALLOC, shader->info.regs.ge_pc_alloc);
if (shader->info.stage == MESA_SHADER_TESS_EVAL) {
radeon_set_context_reg(ctx_cs, R_028A44_VGT_GS_ONCHIP_CNTL, shader->info.regs.vgt_gs_onchip_cntl);
}
}
}
}
static void
radv_emit_hw_es(struct radeon_cmdbuf *cs, const struct radv_shader *shader)
{
uint64_t va = radv_shader_get_va(shader);
radeon_set_sh_reg_seq(cs, R_00B320_SPI_SHADER_PGM_LO_ES, 4);
radeon_emit(cs, va >> 8);
radeon_emit(cs, S_00B324_MEM_BASE(va >> 40));
radeon_emit(cs, shader->config.rsrc1);
radeon_emit(cs, shader->config.rsrc2);
}
static void
radv_emit_hw_ls(struct radeon_cmdbuf *cs, const struct radv_shader *shader)
{
uint64_t va = radv_shader_get_va(shader);
radeon_set_sh_reg(cs, R_00B520_SPI_SHADER_PGM_LO_LS, va >> 8);
radeon_set_sh_reg(cs, R_00B528_SPI_SHADER_PGM_RSRC1_LS, shader->config.rsrc1);
}
static void
radv_emit_hw_ngg(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, struct radeon_cmdbuf *cs,
const struct radv_shader *es, const struct radv_shader *shader)
{
const struct radv_physical_device *pdev = radv_device_physical(device);
uint64_t va = radv_shader_get_va(shader);
gl_shader_stage es_type;
const struct gfx10_ngg_info *ngg_state = &shader->info.ngg_info;
if (shader->info.stage == MESA_SHADER_GEOMETRY) {
if (shader->info.merged_shader_compiled_separately) {
es_type = es->info.stage;
} else {
es_type = shader->info.gs.es_type;
}
} else {
es_type = shader->info.stage;
}
if (!shader->info.merged_shader_compiled_separately) {
radeon_set_sh_reg(cs, R_00B320_SPI_SHADER_PGM_LO_ES, va >> 8);
radeon_set_sh_reg_seq(cs, R_00B228_SPI_SHADER_PGM_RSRC1_GS, 2);
radeon_emit(cs, shader->config.rsrc1);
radeon_emit(cs, shader->config.rsrc2);
}
const struct radv_vs_output_info *outinfo = &shader->info.outinfo;
bool es_enable_prim_id = outinfo->export_prim_id || (es && es->info.uses_prim_id);
bool break_wave_at_eoi = false;
if (es_type == MESA_SHADER_TESS_EVAL) {
if (es_enable_prim_id || (shader->info.uses_prim_id))
break_wave_at_eoi = true;
}
radeon_set_context_reg(ctx_cs, R_0286C4_SPI_VS_OUT_CONFIG, shader->info.regs.spi_vs_out_config);
radeon_set_context_reg_seq(ctx_cs, R_028708_SPI_SHADER_IDX_FORMAT, 2);
radeon_emit(ctx_cs, shader->info.regs.ngg.spi_shader_idx_format);
radeon_emit(ctx_cs, shader->info.regs.spi_shader_pos_format);
radeon_set_context_reg(ctx_cs, R_02881C_PA_CL_VS_OUT_CNTL, shader->info.regs.pa_cl_vs_out_cntl);
radeon_set_context_reg(ctx_cs, R_028A84_VGT_PRIMITIVEID_EN,
shader->info.regs.ngg.vgt_primitiveid_en | S_028A84_PRIMITIVEID_EN(es_enable_prim_id));
radeon_set_context_reg(ctx_cs, R_0287FC_GE_MAX_OUTPUT_PER_SUBGROUP,
shader->info.regs.ngg.ge_max_output_per_subgroup);
radeon_set_context_reg(ctx_cs, R_028B4C_GE_NGG_SUBGRP_CNTL, shader->info.regs.ngg.ge_ngg_subgrp_cntl);
radeon_set_context_reg(ctx_cs, R_028B90_VGT_GS_INSTANCE_CNT, shader->info.regs.vgt_gs_instance_cnt);
uint32_t ge_cntl = shader->info.regs.ngg.ge_cntl;
if (pdev->info.gfx_level >= GFX11) {
ge_cntl |= S_03096C_BREAK_PRIMGRP_AT_EOI(break_wave_at_eoi);
} else {
ge_cntl |= S_03096C_BREAK_WAVE_AT_EOI(break_wave_at_eoi);
/* Bug workaround for a possible hang with non-tessellation cases.
* Tessellation always sets GE_CNTL.VERT_GRP_SIZE = 0
*
* Requirement: GE_CNTL.VERT_GRP_SIZE = VGT_GS_ONCHIP_CNTL.ES_VERTS_PER_SUBGRP - 5
*/
if (pdev->info.gfx_level == GFX10 && es_type != MESA_SHADER_TESS_EVAL && ngg_state->hw_max_esverts != 256) {
ge_cntl &= C_03096C_VERT_GRP_SIZE;
if (ngg_state->hw_max_esverts > 5) {
ge_cntl |= S_03096C_VERT_GRP_SIZE(ngg_state->hw_max_esverts - 5);
}
}
radeon_set_context_reg(ctx_cs, R_028A44_VGT_GS_ONCHIP_CNTL, shader->info.regs.vgt_gs_onchip_cntl);
}
radeon_set_uconfig_reg(cs, R_03096C_GE_CNTL, ge_cntl);
radeon_set_sh_reg_idx(pdev, cs, R_00B21C_SPI_SHADER_PGM_RSRC3_GS, 3, shader->info.regs.spi_shader_pgm_rsrc3_gs);
radeon_set_sh_reg_idx(pdev, cs, R_00B204_SPI_SHADER_PGM_RSRC4_GS, 3, shader->info.regs.spi_shader_pgm_rsrc4_gs);
radeon_set_uconfig_reg(cs, R_030980_GE_PC_ALLOC, shader->info.regs.ge_pc_alloc);
}
static void
radv_emit_hw_hs(const struct radv_device *device, struct radeon_cmdbuf *cs, const struct radv_shader *shader)
{
const struct radv_physical_device *pdev = radv_device_physical(device);
uint64_t va = radv_shader_get_va(shader);
if (pdev->info.gfx_level >= GFX9) {
if (pdev->info.gfx_level >= GFX10) {
radeon_set_sh_reg(cs, R_00B520_SPI_SHADER_PGM_LO_LS, va >> 8);
} else {
radeon_set_sh_reg(cs, R_00B410_SPI_SHADER_PGM_LO_LS, va >> 8);
}
radeon_set_sh_reg(cs, R_00B428_SPI_SHADER_PGM_RSRC1_HS, shader->config.rsrc1);
} else {
radeon_set_sh_reg_seq(cs, R_00B420_SPI_SHADER_PGM_LO_HS, 4);
radeon_emit(cs, va >> 8);
radeon_emit(cs, S_00B424_MEM_BASE(va >> 40));
radeon_emit(cs, shader->config.rsrc1);
radeon_emit(cs, shader->config.rsrc2);
}
}
void
radv_emit_vertex_shader(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, struct radeon_cmdbuf *cs,
const struct radv_shader *vs, const struct radv_shader *next_stage)
{
const struct radv_physical_device *pdev = radv_device_physical(device);
if (vs->info.merged_shader_compiled_separately) {
const struct radv_userdata_info *loc = &vs->info.user_sgprs_locs.shader_data[AC_UD_NEXT_STAGE_PC];
const uint32_t base_reg = vs->info.user_data_0;
assert(loc->sgpr_idx != -1 && loc->num_sgprs == 1);
if (!vs->info.vs.has_prolog) {
uint32_t rsrc1, rsrc2;
if (vs->info.next_stage == MESA_SHADER_TESS_CTRL) {
radv_shader_combine_cfg_vs_tcs(vs, next_stage, &rsrc1, NULL);
if (pdev->info.gfx_level >= GFX10) {
radeon_set_sh_reg(cs, R_00B520_SPI_SHADER_PGM_LO_LS, vs->va >> 8);
} else {
radeon_set_sh_reg(cs, R_00B410_SPI_SHADER_PGM_LO_LS, vs->va >> 8);
}
radeon_set_sh_reg(cs, R_00B428_SPI_SHADER_PGM_RSRC1_HS, rsrc1);
} else {
radv_shader_combine_cfg_vs_gs(vs, next_stage, &rsrc1, &rsrc2);
if (pdev->info.gfx_level >= GFX10) {
radeon_set_sh_reg(cs, R_00B320_SPI_SHADER_PGM_LO_ES, vs->va >> 8);
} else {
radeon_set_sh_reg(cs, R_00B210_SPI_SHADER_PGM_LO_ES, vs->va >> 8);
}
unsigned lds_size;
if (next_stage->info.is_ngg) {
lds_size = DIV_ROUND_UP(next_stage->info.ngg_info.lds_size, pdev->info.lds_encode_granularity);
} else {
lds_size = next_stage->info.gs_ring_info.lds_size;
}
radeon_set_sh_reg_seq(cs, R_00B228_SPI_SHADER_PGM_RSRC1_GS, 2);
radeon_emit(cs, rsrc1);
radeon_emit(cs, rsrc2 | S_00B22C_LDS_SIZE(lds_size));
}
}
radv_emit_shader_pointer(device, cs, base_reg + loc->sgpr_idx * 4, next_stage->va, false);
return;
}
if (vs->info.vs.as_ls)
radv_emit_hw_ls(cs, vs);
else if (vs->info.vs.as_es)
radv_emit_hw_es(cs, vs);
else if (vs->info.is_ngg)
radv_emit_hw_ngg(device, ctx_cs, cs, NULL, vs);
else
radv_emit_hw_vs(device, ctx_cs, cs, vs);
}
void
radv_emit_tess_ctrl_shader(const struct radv_device *device, struct radeon_cmdbuf *cs, const struct radv_shader *tcs)
{
if (tcs->info.merged_shader_compiled_separately) {
/* When VS+TCS are compiled separately on GFX9+, the VS will jump to the TCS and everything is
* emitted as part of the VS.
*/
return;
}
radv_emit_hw_hs(device, cs, tcs);
}
void
radv_emit_tess_eval_shader(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, struct radeon_cmdbuf *cs,
const struct radv_shader *tes, const struct radv_shader *gs)
{
const struct radv_physical_device *pdev = radv_device_physical(device);
if (tes->info.merged_shader_compiled_separately) {
const struct radv_userdata_info *loc = &tes->info.user_sgprs_locs.shader_data[AC_UD_NEXT_STAGE_PC];
const uint32_t base_reg = tes->info.user_data_0;
uint32_t rsrc1, rsrc2;
assert(loc->sgpr_idx != -1 && loc->num_sgprs == 1);
radv_shader_combine_cfg_tes_gs(tes, gs, &rsrc1, &rsrc2);
radeon_set_sh_reg(cs, R_00B210_SPI_SHADER_PGM_LO_ES, tes->va >> 8);
unsigned lds_size;
if (gs->info.is_ngg) {
lds_size = DIV_ROUND_UP(gs->info.ngg_info.lds_size, pdev->info.lds_encode_granularity);
} else {
lds_size = gs->info.gs_ring_info.lds_size;
}
radeon_set_sh_reg_seq(cs, R_00B228_SPI_SHADER_PGM_RSRC1_GS, 2);
radeon_emit(cs, rsrc1);
radeon_emit(cs, rsrc2 | S_00B22C_LDS_SIZE(lds_size));
radv_emit_shader_pointer(device, cs, base_reg + loc->sgpr_idx * 4, gs->va, false);
return;
}
if (tes->info.is_ngg) {
radv_emit_hw_ngg(device, ctx_cs, cs, NULL, tes);
} else if (tes->info.tes.as_es) {
radv_emit_hw_es(cs, tes);
} else {
radv_emit_hw_vs(device, ctx_cs, cs, tes);
}
}
static void
radv_emit_hw_gs(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, struct radeon_cmdbuf *cs,
const struct radv_shader *gs)
{
const struct radv_physical_device *pdev = radv_device_physical(device);
const struct radv_legacy_gs_info *gs_state = &gs->info.gs_ring_info;
const uint64_t va = radv_shader_get_va(gs);
radeon_set_context_reg_seq(ctx_cs, R_028A60_VGT_GSVS_RING_OFFSET_1, 3);
radeon_emit(ctx_cs, gs->info.regs.gs.vgt_gsvs_ring_offset[0]);
radeon_emit(ctx_cs, gs->info.regs.gs.vgt_gsvs_ring_offset[1]);
radeon_emit(ctx_cs, gs->info.regs.gs.vgt_gsvs_ring_offset[2]);
radeon_set_context_reg(ctx_cs, R_028AB0_VGT_GSVS_RING_ITEMSIZE, gs->info.regs.gs.vgt_gsvs_ring_itemsize);
radeon_set_context_reg_seq(ctx_cs, R_028B5C_VGT_GS_VERT_ITEMSIZE, 4);
radeon_emit(ctx_cs, gs->info.regs.gs.vgt_gs_vert_itemsize[0]);
radeon_emit(ctx_cs, gs->info.regs.gs.vgt_gs_vert_itemsize[1]);
radeon_emit(ctx_cs, gs->info.regs.gs.vgt_gs_vert_itemsize[2]);
radeon_emit(ctx_cs, gs->info.regs.gs.vgt_gs_vert_itemsize[3]);
radeon_set_context_reg(ctx_cs, R_028B90_VGT_GS_INSTANCE_CNT, gs->info.regs.gs.vgt_gs_instance_cnt);
if (pdev->info.gfx_level >= GFX9) {
if (!gs->info.merged_shader_compiled_separately) {
if (pdev->info.gfx_level >= GFX10) {
radeon_set_sh_reg(cs, R_00B320_SPI_SHADER_PGM_LO_ES, va >> 8);
} else {
radeon_set_sh_reg(cs, R_00B210_SPI_SHADER_PGM_LO_ES, va >> 8);
}
radeon_set_sh_reg_seq(cs, R_00B228_SPI_SHADER_PGM_RSRC1_GS, 2);
radeon_emit(cs, gs->config.rsrc1);
radeon_emit(cs, gs->config.rsrc2 | S_00B22C_LDS_SIZE(gs_state->lds_size));
}
radeon_set_context_reg(ctx_cs, R_028A44_VGT_GS_ONCHIP_CNTL, gs->info.regs.vgt_gs_onchip_cntl);
radeon_set_context_reg(ctx_cs, R_028A94_VGT_GS_MAX_PRIMS_PER_SUBGROUP,
gs->info.regs.gs.vgt_gs_max_prims_per_subgroup);
} else {
radeon_set_sh_reg_seq(cs, R_00B220_SPI_SHADER_PGM_LO_GS, 4);
radeon_emit(cs, va >> 8);
radeon_emit(cs, S_00B224_MEM_BASE(va >> 40));
radeon_emit(cs, gs->config.rsrc1);
radeon_emit(cs, gs->config.rsrc2);
/* GFX6-8: ESGS offchip ring buffer is allocated according to VGT_ESGS_RING_ITEMSIZE.
* GFX9+: Only used to set the GS input VGPRs, emulated in shaders.
*/
radeon_set_context_reg(ctx_cs, R_028AAC_VGT_ESGS_RING_ITEMSIZE, gs->info.regs.gs.vgt_esgs_ring_itemsize);
}
radeon_set_sh_reg_idx(pdev, cs, R_00B21C_SPI_SHADER_PGM_RSRC3_GS, 3, gs->info.regs.spi_shader_pgm_rsrc3_gs);
if (pdev->info.gfx_level >= GFX10) {
radeon_set_sh_reg_idx(pdev, cs, R_00B204_SPI_SHADER_PGM_RSRC4_GS, 3, gs->info.regs.spi_shader_pgm_rsrc4_gs);
}
}
void
radv_emit_geometry_shader(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, struct radeon_cmdbuf *cs,
const struct radv_shader *gs, const struct radv_shader *es,
const struct radv_shader *gs_copy_shader)
{
if (gs->info.is_ngg) {
radv_emit_hw_ngg(device, ctx_cs, cs, es, gs);
} else {
radv_emit_hw_gs(device, ctx_cs, cs, gs);
radv_emit_hw_vs(device, ctx_cs, cs, gs_copy_shader);
}
radeon_set_context_reg(ctx_cs, R_028B38_VGT_GS_MAX_VERT_OUT, gs->info.regs.vgt_gs_max_vert_out);
if (gs->info.merged_shader_compiled_separately) {
const struct radv_userdata_info *vgt_esgs_ring_itemsize = radv_get_user_sgpr(gs, AC_UD_VGT_ESGS_RING_ITEMSIZE);
assert(vgt_esgs_ring_itemsize->sgpr_idx != -1 && vgt_esgs_ring_itemsize->num_sgprs == 1);
radeon_set_sh_reg(cs, gs->info.user_data_0 + vgt_esgs_ring_itemsize->sgpr_idx * 4, es->info.esgs_itemsize / 4);
if (gs->info.is_ngg) {
const struct radv_userdata_info *ngg_lds_layout = radv_get_user_sgpr(gs, AC_UD_NGG_LDS_LAYOUT);
assert(ngg_lds_layout->sgpr_idx != -1 && ngg_lds_layout->num_sgprs == 1);
assert(!(gs->info.ngg_info.esgs_ring_size & 0xffff0000) && !(gs->info.ngg_info.scratch_lds_base & 0xffff0000));
radeon_set_sh_reg(cs, gs->info.user_data_0 + ngg_lds_layout->sgpr_idx * 4,
SET_SGPR_FIELD(NGG_LDS_LAYOUT_GS_OUT_VERTEX_BASE, gs->info.ngg_info.esgs_ring_size) |
SET_SGPR_FIELD(NGG_LDS_LAYOUT_SCRATCH_BASE, gs->info.ngg_info.scratch_lds_base));
}
}
}
void
radv_emit_mesh_shader(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, struct radeon_cmdbuf *cs,
const struct radv_shader *ms)
{
const struct radv_physical_device *pdev = radv_device_physical(device);
const uint32_t gs_out = radv_conv_gl_prim_to_gs_out(ms->info.ms.output_prim);
radv_emit_hw_ngg(device, ctx_cs, cs, NULL, ms);
radeon_set_context_reg(ctx_cs, R_028B38_VGT_GS_MAX_VERT_OUT, ms->info.regs.vgt_gs_max_vert_out);
radeon_set_uconfig_reg_idx(pdev, cs, R_030908_VGT_PRIMITIVE_TYPE, 1, V_008958_DI_PT_POINTLIST);
if (pdev->mesh_fast_launch_2) {
radeon_set_sh_reg_seq(cs, R_00B2B0_SPI_SHADER_GS_MESHLET_DIM, 2);
radeon_emit(cs, ms->info.regs.ms.spi_shader_gs_meshlet_dim);
radeon_emit(cs, ms->info.regs.ms.spi_shader_gs_meshlet_exp_alloc);
}
radv_emit_vgt_gs_out(device, ctx_cs, cs, gs_out);
}
enum radv_ps_in_type {
radv_ps_in_interpolated,
radv_ps_in_flat,
radv_ps_in_explicit,
radv_ps_in_explicit_strict,
radv_ps_in_interpolated_fp16,
radv_ps_in_interpolated_fp16_hi,
radv_ps_in_per_prim_gfx103,
radv_ps_in_per_prim_gfx11,
};
static uint32_t
offset_to_ps_input(const uint32_t offset, const enum radv_ps_in_type type)
{
assert(offset != AC_EXP_PARAM_UNDEFINED);
if (offset >= AC_EXP_PARAM_DEFAULT_VAL_0000 && offset <= AC_EXP_PARAM_DEFAULT_VAL_1111) {
/* The input is a DEFAULT_VAL constant. */
return S_028644_OFFSET(0x20) | S_028644_DEFAULT_VAL(offset - AC_EXP_PARAM_DEFAULT_VAL_0000);
}
assert(offset <= AC_EXP_PARAM_OFFSET_31);
uint32_t ps_input_cntl = S_028644_OFFSET(offset);
switch (type) {
case radv_ps_in_explicit_strict:
/* Rotate parameter cache contents to strict vertex order. */
ps_input_cntl |= S_028644_ROTATE_PC_PTR(1);
FALLTHROUGH;
case radv_ps_in_explicit:
/* Force parameter cache to be read in passthrough mode. */
ps_input_cntl |= S_028644_OFFSET(1 << 5);
FALLTHROUGH;
case radv_ps_in_flat:
ps_input_cntl |= S_028644_FLAT_SHADE(1);
break;
case radv_ps_in_interpolated_fp16_hi:
ps_input_cntl |= S_028644_ATTR1_VALID(1);
FALLTHROUGH;
case radv_ps_in_interpolated_fp16:
/* These must be set even if only the high 16 bits are used. */
ps_input_cntl |= S_028644_FP16_INTERP_MODE(1) | S_028644_ATTR0_VALID(1);
break;
case radv_ps_in_per_prim_gfx11:
ps_input_cntl |= S_028644_PRIM_ATTR(1);
break;
case radv_ps_in_interpolated:
case radv_ps_in_per_prim_gfx103:
break;
}
return ps_input_cntl;
}
static void
slot_to_ps_input(const struct radv_vs_output_info *outinfo, unsigned slot, uint32_t *ps_input_cntl, unsigned *ps_offset,
const bool use_default_0, const enum radv_ps_in_type type)
{
unsigned vs_offset = outinfo->vs_output_param_offset[slot];
if (vs_offset == AC_EXP_PARAM_UNDEFINED) {
if (use_default_0)
vs_offset = AC_EXP_PARAM_DEFAULT_VAL_0000;
else
return;
}
ps_input_cntl[*ps_offset] = offset_to_ps_input(vs_offset, type);
++(*ps_offset);
}
static void
input_mask_to_ps_inputs(const struct radv_vs_output_info *outinfo, const struct radv_shader *ps, uint32_t input_mask,
uint32_t *ps_input_cntl, unsigned *ps_offset, const enum radv_ps_in_type default_type)
{
u_foreach_bit (i, input_mask) {
unsigned vs_offset = outinfo->vs_output_param_offset[VARYING_SLOT_VAR0 + i];
if (vs_offset == AC_EXP_PARAM_UNDEFINED) {
ps_input_cntl[*ps_offset] = S_028644_OFFSET(0x20);
++(*ps_offset);
continue;
}
enum radv_ps_in_type type = default_type;
if (ps->info.ps.flat_shaded_mask & BITFIELD_BIT(*ps_offset))
type = radv_ps_in_flat;
else if (ps->info.ps.explicit_shaded_mask & BITFIELD_BIT(*ps_offset))
type = radv_ps_in_explicit;
else if (ps->info.ps.explicit_strict_shaded_mask & BITFIELD_BIT(*ps_offset))
type = radv_ps_in_explicit_strict;
else if (ps->info.ps.float16_hi_shaded_mask & BITFIELD_BIT(*ps_offset))
type = radv_ps_in_interpolated_fp16_hi;
else if (ps->info.ps.float16_shaded_mask & BITFIELD_BIT(*ps_offset))
type = radv_ps_in_interpolated_fp16;
ps_input_cntl[*ps_offset] = offset_to_ps_input(vs_offset, type);
++(*ps_offset);
}
}
void
radv_emit_ps_inputs(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs,
const struct radv_shader *last_vgt_shader, const struct radv_shader *ps)
{
const struct radv_physical_device *pdev = radv_device_physical(device);
const struct radv_vs_output_info *outinfo = &last_vgt_shader->info.outinfo;
const bool mesh = last_vgt_shader->info.stage == MESA_SHADER_MESH;
const bool gfx11plus = pdev->info.gfx_level >= GFX11;
const enum radv_ps_in_type per_prim = gfx11plus ? radv_ps_in_per_prim_gfx11 : radv_ps_in_per_prim_gfx103;
uint32_t ps_input_cntl[32];
unsigned ps_offset = 0;
if (ps->info.ps.prim_id_input && !mesh)
slot_to_ps_input(outinfo, VARYING_SLOT_PRIMITIVE_ID, ps_input_cntl, &ps_offset, false, radv_ps_in_flat);
if (ps->info.ps.layer_input && !mesh)
slot_to_ps_input(outinfo, VARYING_SLOT_LAYER, ps_input_cntl, &ps_offset, true, radv_ps_in_flat);
if (ps->info.ps.viewport_index_input && !mesh)
slot_to_ps_input(outinfo, VARYING_SLOT_VIEWPORT, ps_input_cntl, &ps_offset, true, radv_ps_in_flat);
if (ps->info.ps.has_pcoord)
ps_input_cntl[ps_offset++] = S_028644_PT_SPRITE_TEX(1) | S_028644_OFFSET(0x20);
if (ps->info.ps.input_clips_culls_mask & 0x0f)
slot_to_ps_input(outinfo, VARYING_SLOT_CLIP_DIST0, ps_input_cntl, &ps_offset, false, radv_ps_in_interpolated);
if (ps->info.ps.input_clips_culls_mask & 0xf0)
slot_to_ps_input(outinfo, VARYING_SLOT_CLIP_DIST1, ps_input_cntl, &ps_offset, false, radv_ps_in_interpolated);
input_mask_to_ps_inputs(outinfo, ps, ps->info.ps.input_mask, ps_input_cntl, &ps_offset, radv_ps_in_interpolated);
/* Per-primitive PS inputs: the HW needs these to be last. */
if (ps->info.ps.prim_id_input && mesh)
slot_to_ps_input(outinfo, VARYING_SLOT_PRIMITIVE_ID, ps_input_cntl, &ps_offset, false, per_prim);
if (ps->info.ps.layer_input && mesh)
slot_to_ps_input(outinfo, VARYING_SLOT_LAYER, ps_input_cntl, &ps_offset, true, per_prim);
if (ps->info.ps.viewport_index_input && mesh)
slot_to_ps_input(outinfo, VARYING_SLOT_VIEWPORT, ps_input_cntl, &ps_offset, true, per_prim);
input_mask_to_ps_inputs(outinfo, ps, ps->info.ps.input_per_primitive_mask, ps_input_cntl, &ps_offset, per_prim);
if (ps_offset) {
radeon_set_context_reg_seq(ctx_cs, R_028644_SPI_PS_INPUT_CNTL_0, ps_offset);
for (unsigned i = 0; i < ps_offset; i++) {
radeon_emit(ctx_cs, ps_input_cntl[i]);
}
}
}
void
radv_emit_fragment_shader(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, struct radeon_cmdbuf *cs,
const struct radv_shader *ps)
{
const struct radv_physical_device *pdev = radv_device_physical(device);
uint64_t va;
va = radv_shader_get_va(ps);
radeon_set_sh_reg_seq(cs, R_00B020_SPI_SHADER_PGM_LO_PS, 4);
radeon_emit(cs, va >> 8);
radeon_emit(cs, S_00B024_MEM_BASE(va >> 40));
radeon_emit(cs, ps->config.rsrc1);
radeon_emit(cs, ps->config.rsrc2);
radeon_set_context_reg_seq(ctx_cs, R_0286CC_SPI_PS_INPUT_ENA, 2);
radeon_emit(ctx_cs, ps->config.spi_ps_input_ena);
radeon_emit(ctx_cs, ps->config.spi_ps_input_addr);
radeon_set_context_reg(ctx_cs, R_0286D8_SPI_PS_IN_CONTROL, ps->info.regs.ps.spi_ps_in_control);
radeon_set_context_reg(ctx_cs, R_028710_SPI_SHADER_Z_FORMAT, ps->info.regs.ps.spi_shader_z_format);
if (pdev->info.gfx_level >= GFX9 && pdev->info.gfx_level < GFX11)
radeon_set_context_reg(ctx_cs, R_028C40_PA_SC_SHADER_CONTROL, ps->info.regs.ps.pa_sc_shader_control);
}
void
radv_emit_vgt_reuse(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, const struct radv_shader *tes,
const struct radv_vgt_shader_key *key)
{
const struct radv_physical_device *pdev = radv_device_physical(device);
if (pdev->info.gfx_level == GFX10_3) {
/* Legacy Tess+GS should disable reuse to prevent hangs on GFX10.3. */
const bool has_legacy_tess_gs = key->tess && key->gs && !key->ngg;
radeon_set_context_reg(ctx_cs, R_028AB4_VGT_REUSE_OFF, S_028AB4_REUSE_OFF(has_legacy_tess_gs));
}
if (pdev->info.family >= CHIP_POLARIS10 && pdev->info.gfx_level < GFX10) {
unsigned vtx_reuse_depth = 30;
if (tes && tes->info.tes.spacing == TESS_SPACING_FRACTIONAL_ODD) {
vtx_reuse_depth = 14;
}
radeon_set_context_reg(ctx_cs, R_028C58_VGT_VERTEX_REUSE_BLOCK_CNTL, S_028C58_VTX_REUSE_DEPTH(vtx_reuse_depth));
}
}
struct radv_vgt_shader_key
radv_get_vgt_shader_key(const struct radv_device *device, struct radv_shader **shaders,
const struct radv_shader *gs_copy_shader)
@@ -3418,86 +2800,6 @@ radv_get_vgt_shader_key(const struct radv_device *device, struct radv_shader **s
return key;
}
void
radv_emit_vgt_shader_config(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs,
const struct radv_vgt_shader_key *key)
{
const struct radv_physical_device *pdev = radv_device_physical(device);
uint32_t stages = 0;
if (key->tess) {
stages |= S_028B54_LS_EN(V_028B54_LS_STAGE_ON) | S_028B54_HS_EN(1) | S_028B54_DYNAMIC_HS(1);
if (key->gs)
stages |= S_028B54_ES_EN(V_028B54_ES_STAGE_DS) | S_028B54_GS_EN(1);
else if (key->ngg)
stages |= S_028B54_ES_EN(V_028B54_ES_STAGE_DS);
else
stages |= S_028B54_VS_EN(V_028B54_VS_STAGE_DS);
} else if (key->gs) {
stages |= S_028B54_ES_EN(V_028B54_ES_STAGE_REAL) | S_028B54_GS_EN(1);
} else if (key->mesh) {
assert(!key->ngg_passthrough);
unsigned gs_fast_launch = pdev->mesh_fast_launch_2 ? 2 : 1;
stages |=
S_028B54_GS_EN(1) | S_028B54_GS_FAST_LAUNCH(gs_fast_launch) | S_028B54_NGG_WAVE_ID_EN(key->mesh_scratch_ring);
} else if (key->ngg) {
stages |= S_028B54_ES_EN(V_028B54_ES_STAGE_REAL);
}
if (key->ngg) {
stages |= S_028B54_PRIMGEN_EN(1) | S_028B54_NGG_WAVE_ID_EN(key->ngg_streamout) |
S_028B54_PRIMGEN_PASSTHRU_EN(key->ngg_passthrough) |
S_028B54_PRIMGEN_PASSTHRU_NO_MSG(key->ngg_passthrough && pdev->info.family >= CHIP_NAVI23);
} else if (key->gs) {
stages |= S_028B54_VS_EN(V_028B54_VS_STAGE_COPY_SHADER);
}
if (pdev->info.gfx_level >= GFX9)
stages |= S_028B54_MAX_PRIMGRP_IN_WAVE(2);
if (pdev->info.gfx_level >= GFX10) {
stages |= S_028B54_HS_W32_EN(key->hs_wave32) | S_028B54_GS_W32_EN(key->gs_wave32) |
S_028B54_VS_W32_EN(pdev->info.gfx_level < GFX11 && key->vs_wave32);
/* Legacy GS only supports Wave64. Read it as an implication. */
assert(!(key->gs && !key->ngg) || !key->gs_wave32);
}
radeon_set_context_reg(ctx_cs, R_028B54_VGT_SHADER_STAGES_EN, stages);
}
void
radv_emit_vgt_gs_out(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, struct radeon_cmdbuf *cs,
uint32_t vgt_gs_out_prim_type)
{
const struct radv_physical_device *pdev = radv_device_physical(device);
if (pdev->info.gfx_level >= GFX11) {
radeon_set_uconfig_reg(cs, R_030998_VGT_GS_OUT_PRIM_TYPE, vgt_gs_out_prim_type);
} else {
radeon_set_context_reg(ctx_cs, R_028A6C_VGT_GS_OUT_PRIM_TYPE, vgt_gs_out_prim_type);
}
}
void
gfx103_emit_vgt_draw_payload_cntl(struct radeon_cmdbuf *ctx_cs, const struct radv_shader *mesh_shader, bool enable_vrs)
{
bool enable_prim_payload = false;
/* Enables the second channel of the primitive export instruction.
* This channel contains: VRS rate x, y, viewport and layer.
*/
if (mesh_shader) {
const struct radv_vs_output_info *outinfo = &mesh_shader->info.outinfo;
enable_prim_payload = (outinfo->writes_viewport_index_per_primitive || outinfo->writes_layer_per_primitive ||
outinfo->writes_primitive_shading_rate_per_primitive);
}
radeon_set_context_reg(ctx_cs, R_028A98_VGT_DRAW_PAYLOAD_CNTL,
S_028A98_EN_VRS_RATE(enable_vrs) | S_028A98_EN_PRIM_PAYLOAD(enable_prim_payload));
}
static bool
gfx103_pipeline_vrs_coarse_shading(const struct radv_device *device, const struct radv_graphics_pipeline *pipeline)
{
@@ -3517,104 +2819,6 @@ gfx103_pipeline_vrs_coarse_shading(const struct radv_device *device, const struc
return true;
}
void
gfx103_emit_vrs_state(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, const struct radv_shader *ps,
bool enable_vrs_coarse_shading, bool force_vrs_per_vertex)
{
const struct radv_physical_device *pdev = radv_device_physical(device);
uint32_t mode = V_028064_SC_VRS_COMB_MODE_PASSTHRU;
uint8_t rate_x = 0, rate_y = 0;
if (enable_vrs_coarse_shading) {
/* When per-draw VRS is not enabled at all, try enabling VRS coarse shading 2x2 if the driver
* determined that it's safe to enable.
*/
mode = V_028064_SC_VRS_COMB_MODE_OVERRIDE;
rate_x = rate_y = 1;
} else if (force_vrs_per_vertex) {
/* Otherwise, if per-draw VRS is not enabled statically, try forcing per-vertex VRS if
* requested by the user. Note that vkd3d-proton always has to declare VRS as dynamic because
* in DX12 it's fully dynamic.
*/
radeon_set_context_reg(ctx_cs, R_028848_PA_CL_VRS_CNTL,
S_028848_SAMPLE_ITER_COMBINER_MODE(V_028848_SC_VRS_COMB_MODE_OVERRIDE) |
S_028848_VERTEX_RATE_COMBINER_MODE(V_028848_SC_VRS_COMB_MODE_OVERRIDE));
/* If the shader is using discard, turn off coarse shading because discard at 2x2 pixel
* granularity degrades quality too much. MIN allows sample shading but not coarse shading.
*/
mode = ps->info.ps.can_discard ? V_028064_SC_VRS_COMB_MODE_MIN : V_028064_SC_VRS_COMB_MODE_PASSTHRU;
}
if (pdev->info.gfx_level < GFX11) {
radeon_set_context_reg(ctx_cs, R_028064_DB_VRS_OVERRIDE_CNTL,
S_028064_VRS_OVERRIDE_RATE_COMBINER_MODE(mode) | S_028064_VRS_OVERRIDE_RATE_X(rate_x) |
S_028064_VRS_OVERRIDE_RATE_Y(rate_y));
}
}
static void
radv_pipeline_emit_pm4(const struct radv_device *device, struct radv_graphics_pipeline *pipeline)
{
const struct radv_physical_device *pdev = radv_device_physical(device);
const struct radv_shader *last_vgt_shader = radv_get_last_vgt_shader(pipeline);
const struct radv_shader *ps = pipeline->base.shaders[MESA_SHADER_FRAGMENT];
struct radeon_cmdbuf *ctx_cs = &pipeline->base.ctx_cs;
struct radeon_cmdbuf *cs = &pipeline->base.cs;
cs->reserved_dw = cs->max_dw = 64;
ctx_cs->reserved_dw = ctx_cs->max_dw = 256;
cs->buf = malloc(4 * (cs->max_dw + ctx_cs->max_dw));
ctx_cs->buf = cs->buf + cs->max_dw;
const struct radv_vgt_shader_key vgt_shader_key =
radv_get_vgt_shader_key(device, pipeline->base.shaders, pipeline->base.gs_copy_shader);
radv_emit_vgt_gs_mode(device, ctx_cs, pipeline->base.shaders[pipeline->last_vgt_api_stage]);
if (radv_pipeline_has_stage(pipeline, MESA_SHADER_VERTEX)) {
radv_emit_vertex_shader(device, ctx_cs, cs, pipeline->base.shaders[MESA_SHADER_VERTEX], NULL);
}
if (radv_pipeline_has_stage(pipeline, MESA_SHADER_MESH)) {
radv_emit_mesh_shader(device, ctx_cs, cs, pipeline->base.shaders[MESA_SHADER_MESH]);
}
if (radv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_CTRL)) {
radv_emit_tess_ctrl_shader(device, cs, pipeline->base.shaders[MESA_SHADER_TESS_CTRL]);
if (radv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_EVAL)) {
radv_emit_tess_eval_shader(device, ctx_cs, cs, pipeline->base.shaders[MESA_SHADER_TESS_EVAL], NULL);
}
}
if (radv_pipeline_has_stage(pipeline, MESA_SHADER_GEOMETRY)) {
const struct radv_shader *gs = pipeline->base.shaders[MESA_SHADER_GEOMETRY];
const struct radv_shader *es = pipeline->base.shaders[gs->info.gs.es_type];
radv_emit_geometry_shader(device, ctx_cs, cs, gs, es, pipeline->base.gs_copy_shader);
}
if (ps) {
radv_emit_fragment_shader(device, ctx_cs, cs, ps);
radv_emit_ps_inputs(device, ctx_cs, last_vgt_shader, ps);
}
radv_emit_vgt_reuse(device, ctx_cs, radv_get_shader(pipeline->base.shaders, MESA_SHADER_TESS_EVAL), &vgt_shader_key);
radv_emit_vgt_shader_config(device, ctx_cs, &vgt_shader_key);
if (pdev->info.gfx_level >= GFX10_3) {
gfx103_emit_vgt_draw_payload_cntl(ctx_cs, pipeline->base.shaders[MESA_SHADER_MESH], pipeline->uses_vrs);
gfx103_emit_vrs_state(device, ctx_cs, pipeline->base.shaders[MESA_SHADER_FRAGMENT],
pipeline->uses_vrs_coarse_shading, pipeline->force_vrs_per_vertex);
}
pipeline->base.ctx_cs_hash = _mesa_hash_data(ctx_cs->buf, ctx_cs->cdw * 4);
assert(ctx_cs->cdw <= ctx_cs->max_dw);
assert(cs->cdw <= cs->max_dw);
}
static void
radv_pipeline_init_vertex_input_state(const struct radv_device *device, struct radv_graphics_pipeline *pipeline,
const struct vk_graphics_pipeline_state *state)
@@ -3940,8 +3144,6 @@ radv_graphics_pipeline_init(struct radv_graphics_pipeline *pipeline, struct radv
radv_pipeline_init_extra(pipeline, extra, &state);
}
radv_pipeline_emit_pm4(device, pipeline);
return result;
}
src/amd/vulkan/radv_pipeline_graphics.h
-40
View File
@@ -608,31 +608,6 @@ void radv_graphics_shaders_compile(struct radv_device *device, struct vk_pipelin
struct radv_shader **shaders, struct radv_shader_binary **binaries,
struct radv_shader **gs_copy_shader, struct radv_shader_binary **gs_copy_binary);
void radv_emit_vgt_gs_mode(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs,
const struct radv_shader *last_vgt_api_shader);
void radv_emit_vertex_shader(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, struct radeon_cmdbuf *cs,
const struct radv_shader *vs, const struct radv_shader *next_stage);
void radv_emit_tess_ctrl_shader(const struct radv_device *device, struct radeon_cmdbuf *cs,
const struct radv_shader *tcs);
void radv_emit_tess_eval_shader(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs,
struct radeon_cmdbuf *cs, const struct radv_shader *tes, const struct radv_shader *gs);
void radv_emit_geometry_shader(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, struct radeon_cmdbuf *cs,
const struct radv_shader *gs, const struct radv_shader *es,
const struct radv_shader *gs_copy_shader);
void radv_emit_mesh_shader(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, struct radeon_cmdbuf *cs,
const struct radv_shader *ms);
void radv_emit_ps_inputs(const struct radv_device *device, struct radeon_cmdbuf *cs,
const struct radv_shader *last_vgt_shader, const struct radv_shader *ps);
void radv_emit_fragment_shader(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, struct radeon_cmdbuf *cs,
const struct radv_shader *ps);
struct radv_vgt_shader_key {
uint8_t tess : 1;
uint8_t gs : 1;
@@ -646,24 +621,9 @@ struct radv_vgt_shader_key {
uint8_t vs_wave32 : 1;
};
void radv_emit_vgt_reuse(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, const struct radv_shader *tes,
const struct radv_vgt_shader_key *key);
struct radv_vgt_shader_key radv_get_vgt_shader_key(const struct radv_device *device, struct radv_shader **shaders,
const struct radv_shader *gs_copy_shader);
void radv_emit_vgt_shader_config(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs,
const struct radv_vgt_shader_key *key);
void radv_emit_vgt_gs_out(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, struct radeon_cmdbuf *cs,
uint32_t vgt_gs_out_prim_type);
void gfx103_emit_vgt_draw_payload_cntl(struct radeon_cmdbuf *ctx_cs, const struct radv_shader *mesh_shader,
bool enable_vrs);
void gfx103_emit_vrs_state(const struct radv_device *device, struct radeon_cmdbuf *ctx_cs, const struct radv_shader *ps,
bool enable_vrs_coarse_shading, bool force_vrs_per_vertex);
uint32_t radv_get_vgt_gs_out(struct radv_shader **shaders, uint32_t primitive_topology);
bool radv_needs_null_export_workaround(const struct radv_device *device, const struct radv_shader *ps,