From 12bca6123a80a771f812b499b6543b02ee1e3441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Ol=C5=A1=C3=A1k?= Date: Sat, 29 Apr 2023 08:37:23 -0400 Subject: [PATCH] ac/nir,llvm: add GS VGPR changes for gfx12 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See the big comment. Reviewed-by: Timur Kristóf Acked-by: Pierre-Eric Pelloux-Prayer Part-of: --- src/amd/common/ac_nir_lower_esgs_io_to_mem.c | 29 ++++++++++++-- src/amd/common/ac_nir_lower_ngg.c | 40 ++++++++++++------- src/amd/common/ac_shader_args.h | 18 ++++++++- src/amd/common/ac_shader_util.c | 7 ++-- src/amd/common/ac_shader_util.h | 2 +- src/amd/llvm/ac_nir_to_llvm.c | 2 + .../drivers/radeonsi/si_nir_lower_abi.c | 25 +++++++----- 7 files changed, 91 insertions(+), 32 deletions(-) diff --git a/src/amd/common/ac_nir_lower_esgs_io_to_mem.c b/src/amd/common/ac_nir_lower_esgs_io_to_mem.c index 7c16abadee0..4d6fadbbbdb 100644 --- a/src/amd/common/ac_nir_lower_esgs_io_to_mem.c +++ b/src/amd/common/ac_nir_lower_esgs_io_to_mem.c @@ -237,15 +237,38 @@ gs_per_vertex_input_vertex_offset_gfx9(nir_builder *b, lower_esgs_io_state *st, return nir_iand_imm(b, vertex_offset, 0xffffu); } +static nir_def * +gs_per_vertex_input_vertex_offset_gfx12(nir_builder *b, lower_esgs_io_state *st, + nir_src *vertex_src) +{ + if (nir_src_is_const(*vertex_src)) { + unsigned vertex = nir_src_as_uint(*vertex_src); + return nir_ubfe_imm(b, gs_get_vertex_offset(b, st, vertex / 3), + (vertex % 3) * 9, 8); + } + + nir_def *bitoffset = nir_imul_imm(b, nir_umod_imm(b, vertex_src->ssa, 3), 9); + nir_def *lt3 = nir_ult(b, vertex_src->ssa, nir_imm_int(b, 3)); + + return nir_bcsel(b, lt3, + nir_ubfe(b, gs_get_vertex_offset(b, st, 0), bitoffset, nir_imm_int(b, 8)), + nir_ubfe(b, gs_get_vertex_offset(b, st, 1), bitoffset, nir_imm_int(b, 8))); +} + static nir_def * gs_per_vertex_input_offset(nir_builder *b, lower_esgs_io_state *st, nir_intrinsic_instr *instr) { nir_src *vertex_src = nir_get_io_arrayed_index_src(instr); - nir_def *vertex_offset = st->gfx_level >= GFX9 - ? gs_per_vertex_input_vertex_offset_gfx9(b, st, vertex_src) - : gs_per_vertex_input_vertex_offset_gfx6(b, st, vertex_src); + nir_def *vertex_offset; + + if (st->gfx_level >= GFX12) + vertex_offset = gs_per_vertex_input_vertex_offset_gfx12(b, st, vertex_src); + else if (st->gfx_level >= GFX9) + vertex_offset = gs_per_vertex_input_vertex_offset_gfx9(b, st, vertex_src); + else + vertex_offset = gs_per_vertex_input_vertex_offset_gfx6(b, st, vertex_src); /* Gfx6-8 can't emulate VGT_ESGS_RING_ITEMSIZE because it uses the register to determine * the allocation size of the ESGS ring buffer in memory. diff --git a/src/amd/common/ac_nir_lower_ngg.c b/src/amd/common/ac_nir_lower_ngg.c index 6400aa32eb3..ba6c7e5a5ef 100644 --- a/src/amd/common/ac_nir_lower_ngg.c +++ b/src/amd/common/ac_nir_lower_ngg.c @@ -400,13 +400,15 @@ pervertex_lds_addr(nir_builder *b, nir_def *vertex_idx, unsigned per_vtx_bytes) static nir_def * emit_pack_ngg_prim_exp_arg(nir_builder *b, unsigned num_vertices_per_primitives, - nir_def *vertex_indices[3], nir_def *is_null_prim) + nir_def *vertex_indices[3], nir_def *is_null_prim, + enum amd_gfx_level gfx_level) { nir_def *arg = nir_load_initial_edgeflags_amd(b); for (unsigned i = 0; i < num_vertices_per_primitives; ++i) { assert(vertex_indices[i]); - arg = nir_ior(b, arg, nir_ishl_imm(b, vertex_indices[i], 10u * i)); + arg = nir_ior(b, arg, nir_ishl_imm(b, vertex_indices[i], + (gfx_level >= GFX12 ? 9u : 10u) * i)); } if (is_null_prim) { @@ -486,11 +488,16 @@ ngg_nogs_init_vertex_indices_vars(nir_builder *b, nir_function_impl *impl, lower for (unsigned v = 0; v < s->options->num_vertices_per_primitive; ++v) { s->gs_vtx_indices_vars[v] = nir_local_variable_create(impl, glsl_uint_type(), "gs_vtx_addr"); - nir_def *vtx = s->options->passthrough ? - nir_ubfe_imm(b, nir_load_packed_passthrough_primitive_amd(b), - 10 * v, 9) : - nir_ubfe_imm(b, nir_load_gs_vertex_offset_amd(b, .base = v / 2u), - (v & 1u) * 16u, 16u); + nir_def *vtx; + + if (s->options->gfx_level >= GFX12) { + vtx = nir_ubfe_imm(b, nir_load_packed_passthrough_primitive_amd(b), 9 * v, 8); + } else if (s->options->passthrough) { + vtx = nir_ubfe_imm(b, nir_load_packed_passthrough_primitive_amd(b), 10 * v, 9); + } else { + vtx = nir_ubfe_imm(b, nir_load_gs_vertex_offset_amd(b, .base = v / 2u), + (v & 1u) * 16u, 16u); + } nir_store_var(b, s->gs_vtx_indices_vars[v], vtx, 0x1); } @@ -499,7 +506,7 @@ ngg_nogs_init_vertex_indices_vars(nir_builder *b, nir_function_impl *impl, lower static nir_def * emit_ngg_nogs_prim_exp_arg(nir_builder *b, lower_ngg_nogs_state *s) { - if (s->options->passthrough) { + if (s->options->gfx_level >= GFX12 || s->options->passthrough) { return nir_load_packed_passthrough_primitive_amd(b); } else { nir_def *vtx_idx[3] = {0}; @@ -507,7 +514,8 @@ emit_ngg_nogs_prim_exp_arg(nir_builder *b, lower_ngg_nogs_state *s) for (unsigned v = 0; v < s->options->num_vertices_per_primitive; ++v) vtx_idx[v] = nir_load_var(b, s->gs_vtx_indices_vars[v]); - return emit_pack_ngg_prim_exp_arg(b, s->options->num_vertices_per_primitive, vtx_idx, NULL); + return emit_pack_ngg_prim_exp_arg(b, s->options->num_vertices_per_primitive, vtx_idx, NULL, + s->options->gfx_level); } } @@ -563,7 +571,7 @@ emit_ngg_nogs_prim_export(nir_builder *b, lower_ngg_nogs_state *s, nir_def *arg) .memory_semantics = NIR_MEMORY_ACQ_REL, .memory_modes = nir_var_mem_shared); - unsigned edge_flag_bits = ac_get_all_edge_flag_bits(); + unsigned edge_flag_bits = ac_get_all_edge_flag_bits(s->options->gfx_level); nir_def *mask = nir_imm_intN_t(b, ~edge_flag_bits, 32); unsigned edge_flag_offset = 0; @@ -578,7 +586,11 @@ emit_ngg_nogs_prim_export(nir_builder *b, lower_ngg_nogs_state *s, nir_def *arg) nir_def *vtx_idx = nir_load_var(b, s->gs_vtx_indices_vars[i]); nir_def *addr = pervertex_lds_addr(b, vtx_idx, s->pervertex_lds_bytes); nir_def *edge = nir_load_shared(b, 1, 32, addr, .base = edge_flag_offset); - mask = nir_ior(b, mask, nir_ishl_imm(b, edge, 9 + i * 10)); + + if (s->options->gfx_level >= GFX12) + mask = nir_ior(b, mask, nir_ishl_imm(b, edge, 8 + i * 9)); + else + mask = nir_ior(b, mask, nir_ishl_imm(b, edge, 9 + i * 10)); } arg = nir_iand(b, arg, mask); } @@ -1032,7 +1044,7 @@ compact_vertices_after_culling(nir_builder *b, nir_def *prim_exp_arg = emit_pack_ngg_prim_exp_arg(b, s->options->num_vertices_per_primitive, - exporter_vtx_indices, NULL); + exporter_vtx_indices, NULL, s->options->gfx_level); nir_store_var(b, prim_exp_arg_var, prim_exp_arg, 0x1u); } nir_pop_if(b, if_gs_accepted); @@ -2932,7 +2944,7 @@ ngg_gs_export_primitives(nir_builder *b, nir_def *max_num_out_prims, nir_def *ti } nir_def *arg = emit_pack_ngg_prim_exp_arg(b, s->num_vertices_per_primitive, vtx_indices, - is_null_prim); + is_null_prim, s->options->gfx_level); ac_nir_export_primitive(b, arg, NULL); nir_pop_if(b, if_prim_export_thread); } @@ -4200,7 +4212,7 @@ ms_prim_exp_arg_ch1(nir_builder *b, nir_def *invocation_index, nir_def *num_vtx, indices[i] = nir_umin(b, indices[i], max_vtx_idx); } - return emit_pack_ngg_prim_exp_arg(b, s->vertices_per_prim, indices, cull_flag); + return emit_pack_ngg_prim_exp_arg(b, s->vertices_per_prim, indices, cull_flag, s->gfx_level); } static nir_def * diff --git a/src/amd/common/ac_shader_args.h b/src/amd/common/ac_shader_args.h index 1b2b6b30dea..4d16fc26b65 100644 --- a/src/amd/common/ac_shader_args.h +++ b/src/amd/common/ac_shader_args.h @@ -117,10 +117,26 @@ struct ac_shader_args { * [20:28] vertex index 2 * [29] edgeflag 2 * [31] 0 (valid prim) + * + * GFX12+: [0-1] 2x uint32 with the following bitfields matching the prim export except + * the GS invocation ID, which is 0 without a user GS, so it doesn't have to be masked + * out for the prim export: + * [0]: + * [0:7] vertex index 0 + * [8] edgeflag 0 + * [9:16] vertex index 1 + * [17] edgeflag 1 + * [18:25] vertex index 2 + * [26] edgeflag 2 + * [27:31] GS invocation ID + * [1]: + * [0:7] vertex index 3 + * [9:16] vertex index 4 + * [18:25] vertex index 5 */ struct ac_arg gs_vtx_offset[6]; struct ac_arg gs_prim_id; - struct ac_arg gs_invocation_id; + struct ac_arg gs_invocation_id; /* GFX6-11 only. GFX12+ uses gs_vtx_offset[0]. */ /* Streamout */ struct ac_arg streamout_config; diff --git a/src/amd/common/ac_shader_util.c b/src/amd/common/ac_shader_util.c index 87f0ffffae7..8fd0af6375e 100644 --- a/src/amd/common/ac_shader_util.c +++ b/src/amd/common/ac_shader_util.c @@ -1328,10 +1328,11 @@ union ac_hw_cache_flags ac_get_hw_cache_flags(const struct radeon_info *info, return result; } -unsigned ac_get_all_edge_flag_bits(void) +unsigned ac_get_all_edge_flag_bits(enum amd_gfx_level gfx_level) { - /* This will be extended in the future. */ - return (1u << 9) | (1u << 19) | (1u << 29); + return gfx_level >= GFX12 ? + ((1u << 8) | (1u << 17) | (1u << 26)) : + ((1u << 9) | (1u << 19) | (1u << 29)); } /** diff --git a/src/amd/common/ac_shader_util.h b/src/amd/common/ac_shader_util.h index e364b6b41db..1f7390b693f 100644 --- a/src/amd/common/ac_shader_util.h +++ b/src/amd/common/ac_shader_util.h @@ -234,7 +234,7 @@ enum gl_access_qualifier ac_get_mem_access_flags(const nir_intrinsic_instr *inst union ac_hw_cache_flags ac_get_hw_cache_flags(const struct radeon_info *info, enum gl_access_qualifier access); -unsigned ac_get_all_edge_flag_bits(void); +unsigned ac_get_all_edge_flag_bits(enum amd_gfx_level gfx_level); unsigned ac_shader_io_get_unique_index_patch(unsigned semantic); diff --git a/src/amd/llvm/ac_nir_to_llvm.c b/src/amd/llvm/ac_nir_to_llvm.c index a41f7dba776..75b8097ea4d 100644 --- a/src/amd/llvm/ac_nir_to_llvm.c +++ b/src/amd/llvm/ac_nir_to_llvm.c @@ -3060,6 +3060,8 @@ static bool visit_intrinsic(struct ac_nir_context *ctx, nir_intrinsic_instr *ins assert(ctx->stage == MESA_SHADER_TESS_CTRL || ctx->stage == MESA_SHADER_GEOMETRY); if (ctx->stage == MESA_SHADER_TESS_CTRL) { result = ac_unpack_param(&ctx->ac, ac_get_arg(&ctx->ac, ctx->args->tcs_rel_ids), 8, 5); + } else if (ctx->ac.gfx_level >= GFX12) { + result = ac_unpack_param(&ctx->ac, ac_get_arg(&ctx->ac, ctx->args->gs_vtx_offset[0]), 27, 5); } else if (ctx->ac.gfx_level >= GFX10) { result = ac_unpack_param(&ctx->ac, ac_get_arg(&ctx->ac, ctx->args->gs_invocation_id), 0, 7); } else { diff --git a/src/gallium/drivers/radeonsi/si_nir_lower_abi.c b/src/gallium/drivers/radeonsi/si_nir_lower_abi.c index cf6a113d484..936c0635a63 100644 --- a/src/gallium/drivers/radeonsi/si_nir_lower_abi.c +++ b/src/gallium/drivers/radeonsi/si_nir_lower_abi.c @@ -680,21 +680,26 @@ static bool lower_intrinsic(nir_builder *b, nir_instr *instr, struct lower_abi_s /* Line primitives and blits don't need edge flags. */ replacement = nir_imm_int(b, 0); } else if (shader->selector->stage == MESA_SHADER_VERTEX) { - /* Use the following trick to extract the edge flags: - * extracted = v_and_b32 gs_invocation_id, 0x700 ; get edge flags at bits 8, 9, 10 - * shifted = v_mul_u32_u24 extracted, 0x80402u ; shift the bits: 8->9, 9->19, 10->29 - * result = v_and_b32 shifted, 0x20080200 ; remove garbage - */ - nir_def *tmp = ac_nir_load_arg(b, &args->ac, args->ac.gs_invocation_id); - tmp = nir_iand_imm(b, tmp, 0x700); - tmp = nir_imul_imm(b, tmp, 0x80402); - replacement = nir_iand_imm(b, tmp, 0x20080200); + if (sel->screen->info.gfx_level >= GFX12) { + replacement = nir_iand_imm(b, ac_nir_load_arg(b, &args->ac, args->ac.gs_vtx_offset[0]), + ac_get_all_edge_flag_bits(sel->screen->info.gfx_level)); + } else { + /* Use the following trick to extract the edge flags: + * extracted = v_and_b32 gs_invocation_id, 0x700 ; get edge flags at bits 8, 9, 10 + * shifted = v_mul_u32_u24 extracted, 0x80402u ; shift the bits: 8->9, 9->19, 10->29 + * result = v_and_b32 shifted, 0x20080200 ; remove garbage + */ + nir_def *tmp = ac_nir_load_arg(b, &args->ac, args->ac.gs_invocation_id); + tmp = nir_iand_imm(b, tmp, 0x700); + tmp = nir_imul_imm(b, tmp, 0x80402); + replacement = nir_iand_imm(b, tmp, 0x20080200); + } } else { /* Edge flags are always enabled when polygon mode is enabled, so we always have to * return valid edge flags if the primitive type is not lines and if we are not blitting * because the shader doesn't know when polygon mode is enabled. */ - replacement = nir_imm_int(b, ac_get_all_edge_flag_bits()); + replacement = nir_imm_int(b, ac_get_all_edge_flag_bits(sel->screen->info.gfx_level)); } break; case nir_intrinsic_load_packed_passthrough_primitive_amd: