AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ac/nir,llvm: add GS VGPR changes for gfx12

Browse Source 
See the big comment.

Reviewed-by: Timur Kristóf <timur.kristof@gmail.com>
Acked-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/29007>
This commit is contained in:
Marek Olšák
2023-04-29 08:37:23 -04:00
parent 4e1abe5d8c
commit 12bca6123a
7 changed files with 91 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/amd/common/ac_nir_lower_esgs_io_to_mem.c
+26 -3
View File
@@ -237,15 +237,38 @@ gs_per_vertex_input_vertex_offset_gfx9(nir_builder *b, lower_esgs_io_state *st,
return nir_iand_imm(b, vertex_offset, 0xffffu);
}
static nir_def *
gs_per_vertex_input_vertex_offset_gfx12(nir_builder *b, lower_esgs_io_state *st,
nir_src *vertex_src)
{
if (nir_src_is_const(*vertex_src)) {
unsigned vertex = nir_src_as_uint(*vertex_src);
return nir_ubfe_imm(b, gs_get_vertex_offset(b, st, vertex / 3),
(vertex % 3) * 9, 8);
}
nir_def *bitoffset = nir_imul_imm(b, nir_umod_imm(b, vertex_src->ssa, 3), 9);
nir_def *lt3 = nir_ult(b, vertex_src->ssa, nir_imm_int(b, 3));
return nir_bcsel(b, lt3,
nir_ubfe(b, gs_get_vertex_offset(b, st, 0), bitoffset, nir_imm_int(b, 8)),
nir_ubfe(b, gs_get_vertex_offset(b, st, 1), bitoffset, nir_imm_int(b, 8)));
}
static nir_def *
gs_per_vertex_input_offset(nir_builder *b,
lower_esgs_io_state *st,
nir_intrinsic_instr *instr)
{
nir_src *vertex_src = nir_get_io_arrayed_index_src(instr);
nir_def *vertex_offset = st->gfx_level >= GFX9
? gs_per_vertex_input_vertex_offset_gfx9(b, st, vertex_src)
: gs_per_vertex_input_vertex_offset_gfx6(b, st, vertex_src);
nir_def *vertex_offset;
if (st->gfx_level >= GFX12)
vertex_offset = gs_per_vertex_input_vertex_offset_gfx12(b, st, vertex_src);
else if (st->gfx_level >= GFX9)
vertex_offset = gs_per_vertex_input_vertex_offset_gfx9(b, st, vertex_src);
else
vertex_offset = gs_per_vertex_input_vertex_offset_gfx6(b, st, vertex_src);
/* Gfx6-8 can't emulate VGT_ESGS_RING_ITEMSIZE because it uses the register to determine
* the allocation size of the ESGS ring buffer in memory.
src/amd/common/ac_nir_lower_ngg.c
+26 -14
View File
@@ -400,13 +400,15 @@ pervertex_lds_addr(nir_builder *b, nir_def *vertex_idx, unsigned per_vtx_bytes)
static nir_def *
emit_pack_ngg_prim_exp_arg(nir_builder *b, unsigned num_vertices_per_primitives,
nir_def *vertex_indices[3], nir_def *is_null_prim)
nir_def *vertex_indices[3], nir_def *is_null_prim,
enum amd_gfx_level gfx_level)
{
nir_def *arg = nir_load_initial_edgeflags_amd(b);
for (unsigned i = 0; i < num_vertices_per_primitives; ++i) {
assert(vertex_indices[i]);
arg = nir_ior(b, arg, nir_ishl_imm(b, vertex_indices[i], 10u * i));
arg = nir_ior(b, arg, nir_ishl_imm(b, vertex_indices[i],
(gfx_level >= GFX12 ? 9u : 10u) * i));
}
if (is_null_prim) {
@@ -486,11 +488,16 @@ ngg_nogs_init_vertex_indices_vars(nir_builder *b, nir_function_impl *impl, lower
for (unsigned v = 0; v < s->options->num_vertices_per_primitive; ++v) {
s->gs_vtx_indices_vars[v] = nir_local_variable_create(impl, glsl_uint_type(), "gs_vtx_addr");
nir_def *vtx = s->options->passthrough ?
nir_ubfe_imm(b, nir_load_packed_passthrough_primitive_amd(b),
10 * v, 9) :
nir_ubfe_imm(b, nir_load_gs_vertex_offset_amd(b, .base = v / 2u),
(v & 1u) * 16u, 16u);
nir_def *vtx;
if (s->options->gfx_level >= GFX12) {
vtx = nir_ubfe_imm(b, nir_load_packed_passthrough_primitive_amd(b), 9 * v, 8);
} else if (s->options->passthrough) {
vtx = nir_ubfe_imm(b, nir_load_packed_passthrough_primitive_amd(b), 10 * v, 9);
} else {
vtx = nir_ubfe_imm(b, nir_load_gs_vertex_offset_amd(b, .base = v / 2u),
(v & 1u) * 16u, 16u);
}
nir_store_var(b, s->gs_vtx_indices_vars[v], vtx, 0x1);
}
@@ -499,7 +506,7 @@ ngg_nogs_init_vertex_indices_vars(nir_builder *b, nir_function_impl *impl, lower
static nir_def *
emit_ngg_nogs_prim_exp_arg(nir_builder *b, lower_ngg_nogs_state *s)
{
if (s->options->passthrough) {
if (s->options->gfx_level >= GFX12 || s->options->passthrough) {
return nir_load_packed_passthrough_primitive_amd(b);
} else {
nir_def *vtx_idx[3] = {0};
@@ -507,7 +514,8 @@ emit_ngg_nogs_prim_exp_arg(nir_builder *b, lower_ngg_nogs_state *s)
for (unsigned v = 0; v < s->options->num_vertices_per_primitive; ++v)
vtx_idx[v] = nir_load_var(b, s->gs_vtx_indices_vars[v]);
return emit_pack_ngg_prim_exp_arg(b, s->options->num_vertices_per_primitive, vtx_idx, NULL);
return emit_pack_ngg_prim_exp_arg(b, s->options->num_vertices_per_primitive, vtx_idx, NULL,
s->options->gfx_level);
}
}
@@ -563,7 +571,7 @@ emit_ngg_nogs_prim_export(nir_builder *b, lower_ngg_nogs_state *s, nir_def *arg)
.memory_semantics = NIR_MEMORY_ACQ_REL,
.memory_modes = nir_var_mem_shared);
unsigned edge_flag_bits = ac_get_all_edge_flag_bits();
unsigned edge_flag_bits = ac_get_all_edge_flag_bits(s->options->gfx_level);
nir_def *mask = nir_imm_intN_t(b, ~edge_flag_bits, 32);
unsigned edge_flag_offset = 0;
@@ -578,7 +586,11 @@ emit_ngg_nogs_prim_export(nir_builder *b, lower_ngg_nogs_state *s, nir_def *arg)
nir_def *vtx_idx = nir_load_var(b, s->gs_vtx_indices_vars[i]);
nir_def *addr = pervertex_lds_addr(b, vtx_idx, s->pervertex_lds_bytes);
nir_def *edge = nir_load_shared(b, 1, 32, addr, .base = edge_flag_offset);
mask = nir_ior(b, mask, nir_ishl_imm(b, edge, 9 + i * 10));
if (s->options->gfx_level >= GFX12)
mask = nir_ior(b, mask, nir_ishl_imm(b, edge, 8 + i * 9));
else
mask = nir_ior(b, mask, nir_ishl_imm(b, edge, 9 + i * 10));
}
arg = nir_iand(b, arg, mask);
}
@@ -1032,7 +1044,7 @@ compact_vertices_after_culling(nir_builder *b,
nir_def *prim_exp_arg =
emit_pack_ngg_prim_exp_arg(b, s->options->num_vertices_per_primitive,
exporter_vtx_indices, NULL);
exporter_vtx_indices, NULL, s->options->gfx_level);
nir_store_var(b, prim_exp_arg_var, prim_exp_arg, 0x1u);
}
nir_pop_if(b, if_gs_accepted);
@@ -2932,7 +2944,7 @@ ngg_gs_export_primitives(nir_builder *b, nir_def *max_num_out_prims, nir_def *ti
}
nir_def *arg = emit_pack_ngg_prim_exp_arg(b, s->num_vertices_per_primitive, vtx_indices,
is_null_prim);
is_null_prim, s->options->gfx_level);
ac_nir_export_primitive(b, arg, NULL);
nir_pop_if(b, if_prim_export_thread);
}
@@ -4200,7 +4212,7 @@ ms_prim_exp_arg_ch1(nir_builder *b, nir_def *invocation_index, nir_def *num_vtx,
indices[i] = nir_umin(b, indices[i], max_vtx_idx);
}
return emit_pack_ngg_prim_exp_arg(b, s->vertices_per_prim, indices, cull_flag);
return emit_pack_ngg_prim_exp_arg(b, s->vertices_per_prim, indices, cull_flag, s->gfx_level);
}
static nir_def *
src/amd/common/ac_shader_args.h
+17 -1
View File
@@ -117,10 +117,26 @@ struct ac_shader_args {
* [20:28] vertex index 2
* [29] edgeflag 2
* [31] 0 (valid prim)
*
* GFX12+: [0-1] 2x uint32 with the following bitfields matching the prim export except
* the GS invocation ID, which is 0 without a user GS, so it doesn't have to be masked
* out for the prim export:
* [0]:
* [0:7] vertex index 0
* [8] edgeflag 0
* [9:16] vertex index 1
* [17] edgeflag 1
* [18:25] vertex index 2
* [26] edgeflag 2
* [27:31] GS invocation ID
* [1]:
* [0:7] vertex index 3
* [9:16] vertex index 4
* [18:25] vertex index 5
*/
struct ac_arg gs_vtx_offset[6];
struct ac_arg gs_prim_id;
struct ac_arg gs_invocation_id;
struct ac_arg gs_invocation_id; /* GFX6-11 only. GFX12+ uses gs_vtx_offset[0]. */
/* Streamout */
struct ac_arg streamout_config;
src/amd/common/ac_shader_util.c
+4 -3
View File
@@ -1328,10 +1328,11 @@ union ac_hw_cache_flags ac_get_hw_cache_flags(const struct radeon_info *info,
return result;
}
unsigned ac_get_all_edge_flag_bits(void)
unsigned ac_get_all_edge_flag_bits(enum amd_gfx_level gfx_level)
{
/* This will be extended in the future. */
return (1u << 9) | (1u << 19) | (1u << 29);
return gfx_level >= GFX12 ?
((1u << 8) | (1u << 17) | (1u << 26)) :
((1u << 9) | (1u << 19) | (1u << 29));
}
/**
src/amd/common/ac_shader_util.h
+1 -1
View File
@@ -234,7 +234,7 @@ enum gl_access_qualifier ac_get_mem_access_flags(const nir_intrinsic_instr *inst
union ac_hw_cache_flags ac_get_hw_cache_flags(const struct radeon_info *info,
enum gl_access_qualifier access);
unsigned ac_get_all_edge_flag_bits(void);
unsigned ac_get_all_edge_flag_bits(enum amd_gfx_level gfx_level);
unsigned ac_shader_io_get_unique_index_patch(unsigned semantic);
src/amd/llvm/ac_nir_to_llvm.c
+2
View File
@@ -3060,6 +3060,8 @@ static bool visit_intrinsic(struct ac_nir_context *ctx, nir_intrinsic_instr *ins
assert(ctx->stage == MESA_SHADER_TESS_CTRL || ctx->stage == MESA_SHADER_GEOMETRY);
if (ctx->stage == MESA_SHADER_TESS_CTRL) {
result = ac_unpack_param(&ctx->ac, ac_get_arg(&ctx->ac, ctx->args->tcs_rel_ids), 8, 5);
} else if (ctx->ac.gfx_level >= GFX12) {
result = ac_unpack_param(&ctx->ac, ac_get_arg(&ctx->ac, ctx->args->gs_vtx_offset[0]), 27, 5);
} else if (ctx->ac.gfx_level >= GFX10) {
result = ac_unpack_param(&ctx->ac, ac_get_arg(&ctx->ac, ctx->args->gs_invocation_id), 0, 7);
} else {
src/gallium/drivers/radeonsi/si_nir_lower_abi.c
+15 -10
View File
@@ -680,21 +680,26 @@ static bool lower_intrinsic(nir_builder *b, nir_instr *instr, struct lower_abi_s
/* Line primitives and blits don't need edge flags. */
replacement = nir_imm_int(b, 0);
} else if (shader->selector->stage == MESA_SHADER_VERTEX) {
/* Use the following trick to extract the edge flags:
* extracted = v_and_b32 gs_invocation_id, 0x700 ; get edge flags at bits 8, 9, 10
* shifted = v_mul_u32_u24 extracted, 0x80402u ; shift the bits: 8->9, 9->19, 10->29
* result = v_and_b32 shifted, 0x20080200 ; remove garbage
*/
nir_def *tmp = ac_nir_load_arg(b, &args->ac, args->ac.gs_invocation_id);
tmp = nir_iand_imm(b, tmp, 0x700);
tmp = nir_imul_imm(b, tmp, 0x80402);
replacement = nir_iand_imm(b, tmp, 0x20080200);
if (sel->screen->info.gfx_level >= GFX12) {
replacement = nir_iand_imm(b, ac_nir_load_arg(b, &args->ac, args->ac.gs_vtx_offset[0]),
ac_get_all_edge_flag_bits(sel->screen->info.gfx_level));
} else {
/* Use the following trick to extract the edge flags:
* extracted = v_and_b32 gs_invocation_id, 0x700 ; get edge flags at bits 8, 9, 10
* shifted = v_mul_u32_u24 extracted, 0x80402u ; shift the bits: 8->9, 9->19, 10->29
* result = v_and_b32 shifted, 0x20080200 ; remove garbage
*/
nir_def *tmp = ac_nir_load_arg(b, &args->ac, args->ac.gs_invocation_id);
tmp = nir_iand_imm(b, tmp, 0x700);
tmp = nir_imul_imm(b, tmp, 0x80402);
replacement = nir_iand_imm(b, tmp, 0x20080200);
}
} else {
/* Edge flags are always enabled when polygon mode is enabled, so we always have to
* return valid edge flags if the primitive type is not lines and if we are not blitting
* because the shader doesn't know when polygon mode is enabled.
*/
replacement = nir_imm_int(b, ac_get_all_edge_flag_bits());
replacement = nir_imm_int(b, ac_get_all_edge_flag_bits(sel->screen->info.gfx_level));
}
break;
case nir_intrinsic_load_packed_passthrough_primitive_amd: