From 28fe7e2a0e08a95ca498e9d656f2d4ca1d0e3c7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timur=20Krist=C3=B3f?= Date: Mon, 27 Jan 2025 12:36:19 +0100 Subject: [PATCH] ac/nir/ngg: Rework GS output code for better attribute ring handling. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create separate branches for output processing and exports. Normally, emit attribute ring stores at the end of the shader, but with the attribute ring wait bug, insert them between the primitive and position export branches. Reviewed-by: Marek Olšák Part-of: --- src/amd/common/nir/ac_nir_lower_ngg.c | 154 ++++++++++++++++---------- 1 file changed, 97 insertions(+), 57 deletions(-) diff --git a/src/amd/common/nir/ac_nir_lower_ngg.c b/src/amd/common/nir/ac_nir_lower_ngg.c index f292102b031..690da678e53 100644 --- a/src/amd/common/nir/ac_nir_lower_ngg.c +++ b/src/amd/common/nir/ac_nir_lower_ngg.c @@ -2326,19 +2326,6 @@ ngg_gs_process_out_primitive(nir_builder *b, is_null_prim, s->options->hw_info->gfx_level); } -static void -ngg_gs_export_primitives(nir_builder *b, nir_def *max_num_out_prims, nir_def *tid_in_tg, - nir_def *exporter_tid_in_tg, nir_def *primflag_0, - lower_ngg_gs_state *s) -{ - nir_if *if_prim_export_thread = nir_push_if(b, nir_ilt(b, tid_in_tg, max_num_out_prims)); - { - nir_def *arg = ngg_gs_process_out_primitive(b, exporter_tid_in_tg, primflag_0, s); - ac_nir_export_primitive(b, arg, NULL); - } - nir_pop_if(b, if_prim_export_thread); -} - static void ngg_gs_process_out_vertex(nir_builder *b, nir_def *out_vtx_lds_addr, lower_ngg_gs_state *s) { @@ -2408,54 +2395,111 @@ ngg_gs_process_out_vertex(nir_builder *b, nir_def *out_vtx_lds_addr, lower_ngg_g ac_nir_clamp_vertex_color_outputs(b, &s->out); } +/** + * Emit NGG GS output, including vertex and primitive exports and attribute ring stores (if any). + * The exact sequence emitted, depends on the current GPU and its workarounds. + * + * The order mainly depends on whether the current GPU has an attribute ring, and + * whether it has the bug that requires us to emit a wait for the attribute ring stores. + * + * The basic structure looks like this: + * + * if (has primitive) { + * + * + * if (!(wait for attr ring)) { + * + * } + * } + * if (has vertex) { + * + * + * if (!(wait for attr ring)) { + * + * + * } + * } + * + * if (wait for attr ring) { + * + * if (has primitive) { + * + * } + * if (has vertex) { + * + * + * } + * } + * + */ static void -ngg_gs_export_vertices(nir_builder *b, nir_def *max_num_out_vtx, nir_def *tid_in_tg, - nir_def *out_vtx_lds_addr, lower_ngg_gs_state *s) +ngg_gs_emit_output(nir_builder *b, nir_def *max_num_out_vtx, nir_def *max_num_out_prims, + nir_def *tid_in_tg, nir_def *out_vtx_lds_addr, nir_def *prim_exporter_tid_in_tg, + nir_def *primflag_0, lower_ngg_gs_state *s) { - nir_if *if_vtx_export_thread = nir_push_if(b, nir_ilt(b, tid_in_tg, max_num_out_vtx)); + nir_def *undef = nir_undef(b, 1, 32); - ngg_gs_process_out_vertex(b, out_vtx_lds_addr, s); - - uint64_t export_outputs = b->shader->info.outputs_written | VARYING_BIT_POS; - if (s->options->kill_pointsize) - export_outputs &= ~VARYING_BIT_PSIZ; - if (s->options->kill_layer) - export_outputs &= ~VARYING_BIT_LAYER; - - const bool wait_attr_ring = s->options->has_param_exports && s->options->hw_info->has_attr_ring_wait_bug; - if (wait_attr_ring) - export_outputs &= ~VARYING_BIT_POS; - - ac_nir_export_position(b, s->options->hw_info->gfx_level, - s->options->clip_cull_dist_mask, - !s->options->has_param_exports, - s->options->force_vrs, !wait_attr_ring, - export_outputs, &s->out, NULL); - - if (s->options->has_param_exports && !s->options->hw_info->has_attr_ring) { - /* Emit vertex parameter exports. - * Only the vertex export threads should do this. - */ - ac_nir_export_parameters(b, s->options->vs_output_param_offset, - b->shader->info.outputs_written, - b->shader->info.outputs_written_16bit, - &s->out); + /* Primitive processing */ + nir_def *prim_exp_arg = NULL; + nir_if *if_process_primitive = nir_push_if(b, nir_ilt(b, tid_in_tg, max_num_out_prims)); + { + prim_exp_arg = ngg_gs_process_out_primitive(b, prim_exporter_tid_in_tg, primflag_0, s); } + nir_pop_if(b, if_process_primitive); + prim_exp_arg = nir_if_phi(b, prim_exp_arg, undef); - nir_pop_if(b, if_vtx_export_thread); + /* Vertex processing */ + nir_if *if_process_vertex = nir_push_if(b, nir_ilt(b, tid_in_tg, max_num_out_vtx)); + { + ngg_gs_process_out_vertex(b, out_vtx_lds_addr, s); + } + nir_pop_if(b, if_process_vertex); + ac_nir_create_output_phis(b, b->shader->info.outputs_written, b->shader->info.outputs_written_16bit, &s->out); + + nir_if *if_export_primitive = nir_push_if(b, if_process_primitive->condition.ssa); + { + ac_nir_export_primitive(b, prim_exp_arg, NULL); + } + nir_pop_if(b, if_export_primitive); + + nir_if *if_export_vertex = nir_push_if(b, if_process_vertex->condition.ssa); + { + uint64_t export_outputs = b->shader->info.outputs_written | VARYING_BIT_POS; + if (s->options->kill_pointsize) + export_outputs &= ~VARYING_BIT_PSIZ; + if (s->options->kill_layer) + export_outputs &= ~VARYING_BIT_LAYER; + + ac_nir_export_position(b, s->options->hw_info->gfx_level, + s->options->clip_cull_dist_mask, + !s->options->has_param_exports, + s->options->force_vrs, true, + export_outputs, &s->out, NULL); + + if (s->options->has_param_exports && !s->options->hw_info->has_attr_ring) + ac_nir_export_parameters(b, s->options->vs_output_param_offset, + b->shader->info.outputs_written, + b->shader->info.outputs_written_16bit, + &s->out); + } + nir_pop_if(b, if_export_vertex); if (s->options->has_param_exports && s->options->hw_info->has_attr_ring) { - /* Store vertex parameters to attribute ring. - * For optimal attribute ring access, this should happen in top level CF. - */ - ac_nir_create_output_phis(b, b->shader->info.outputs_written, b->shader->info.outputs_written_16bit, &s->out); + if (s->options->hw_info->has_attr_ring_wait_bug) + b->cursor = nir_after_cf_node_and_phis(&if_export_primitive->cf_node); + ac_nir_store_parameters_to_attr_ring(b, s->options->vs_output_param_offset, b->shader->info.outputs_written, b->shader->info.outputs_written_16bit, &s->out, tid_in_tg, max_num_out_vtx); - if (wait_attr_ring) - export_pos0_wait_attr_ring(b, if_vtx_export_thread, s->out.outputs, s->options); + if (s->options->hw_info->has_attr_ring_wait_bug) { + /* Wait for attribute ring stores to finish. */ + nir_barrier(b, .execution_scope = SCOPE_SUBGROUP, + .memory_scope = SCOPE_DEVICE, + .memory_semantics = NIR_MEMORY_RELEASE, + .memory_modes = nir_var_mem_ssbo | nir_var_shader_out | nir_var_mem_global | nir_var_image); + } } } @@ -2746,9 +2790,8 @@ ngg_gs_finale(nir_builder *b, lower_ngg_gs_state *s) nir_def *out_vtx_primflag_0 = ngg_gs_load_out_vtx_primflag(b, 0, tid_in_tg, out_vtx_lds_addr, max_vtxcnt, s); - if (s->output_compile_time_known) { - ngg_gs_export_primitives(b, max_vtxcnt, tid_in_tg, tid_in_tg, out_vtx_primflag_0, s); - ngg_gs_export_vertices(b, max_vtxcnt, tid_in_tg, out_vtx_lds_addr, s); + if (s->output_compile_time_known && b->shader->info.gs.vertices_out) { + ngg_gs_emit_output(b, max_vtxcnt, max_prmcnt, tid_in_tg, out_vtx_lds_addr, tid_in_tg, out_vtx_primflag_0, s); return; } @@ -2798,8 +2841,7 @@ ngg_gs_finale(nir_builder *b, lower_ngg_gs_state *s) nir_barrier(b, .execution_scope=SCOPE_WORKGROUP, .memory_scope=SCOPE_WORKGROUP, .memory_semantics=NIR_MEMORY_ACQ_REL, .memory_modes=nir_var_mem_shared); - ngg_gs_export_primitives(b, max_prmcnt, tid_in_tg, exporter_tid_in_tg, out_vtx_primflag_0, s); - ngg_gs_export_vertices(b, workgroup_num_vertices, tid_in_tg, out_vtx_lds_addr, s); + ngg_gs_emit_output(b, workgroup_num_vertices, max_prmcnt, tid_in_tg, out_vtx_lds_addr, exporter_tid_in_tg, out_vtx_primflag_0, s); } void @@ -2820,9 +2862,7 @@ ac_nir_lower_ngg_gs(nir_shader *shader, const ac_nir_lower_ngg_options *options) if (!options->can_cull) { nir_gs_count_vertices_and_primitives(shader, state.const_out_vtxcnt, state.const_out_prmcnt, NULL, 4u); - state.output_compile_time_known = - state.const_out_vtxcnt[0] == shader->info.gs.vertices_out && - state.const_out_prmcnt[0] != -1; + state.output_compile_time_known = false; } if (shader->info.gs.output_primitive == MESA_PRIM_POINTS)