diff --git a/src/amd/common/ac_nir.c b/src/amd/common/ac_nir.c index 2ad58c38d71..a91e48b575a 100644 --- a/src/amd/common/ac_nir.c +++ b/src/amd/common/ac_nir.c @@ -98,9 +98,31 @@ ac_nir_lower_sin_cos(nir_shader *shader) typedef struct { const struct ac_shader_args *const args; const enum amd_gfx_level gfx_level; + bool has_ls_vgpr_init_bug; const enum ac_hw_stage hw_stage; + + nir_def *vertex_id; + nir_def *instance_id; } lower_intrinsics_to_args_state; +static nir_def * +preload_arg(lower_intrinsics_to_args_state *s, nir_function_impl *impl, struct ac_arg arg, + struct ac_arg ls_buggy_arg) +{ + nir_builder start_b = nir_builder_at(nir_before_impl(impl)); + nir_def *value = ac_nir_load_arg(&start_b, s->args, arg); + + /* If there are no HS threads, SPI mistakenly loads the LS VGPRs starting at VGPR 0. */ + if ((s->hw_stage == AC_HW_LOCAL_SHADER || s->hw_stage == AC_HW_HULL_SHADER) && + s->has_ls_vgpr_init_bug) { + nir_def *count = ac_nir_unpack_arg(&start_b, s->args, s->args->merged_wave_info, 8, 8); + nir_def *hs_empty = nir_ieq_imm(&start_b, count, 0); + value = nir_bcsel(&start_b, hs_empty, ac_nir_load_arg(&start_b, s->args, ls_buggy_arg), + value); + } + return value; +} + static bool lower_intrinsic_to_arg(nir_builder *b, nir_instr *instr, void *state) { @@ -332,6 +354,21 @@ lower_intrinsic_to_arg(nir_builder *b, nir_instr *instr, void *state) else unreachable("Shader doesn't have GS wave ID."); break; + case nir_intrinsic_overwrite_vs_arguments_amd: + s->vertex_id = intrin->src[0].ssa; + s->instance_id = intrin->src[1].ssa; + nir_instr_remove(instr); + return true; + case nir_intrinsic_load_vertex_id_zero_base: + if (!s->vertex_id) + s->vertex_id = preload_arg(s, b->impl, s->args->vertex_id, s->args->tcs_patch_id); + replacement = s->vertex_id; + break; + case nir_intrinsic_load_instance_id: + if (!s->instance_id) + s->instance_id = preload_arg(s, b->impl, s->args->instance_id, s->args->vertex_id); + replacement = s->instance_id; + break; default: return false; } @@ -343,12 +380,13 @@ lower_intrinsic_to_arg(nir_builder *b, nir_instr *instr, void *state) bool ac_nir_lower_intrinsics_to_args(nir_shader *shader, const enum amd_gfx_level gfx_level, - const enum ac_hw_stage hw_stage, + bool has_ls_vgpr_init_bug, const enum ac_hw_stage hw_stage, const struct ac_shader_args *ac_args) { lower_intrinsics_to_args_state state = { .gfx_level = gfx_level, .hw_stage = hw_stage, + .has_ls_vgpr_init_bug = has_ls_vgpr_init_bug, .args = ac_args, }; diff --git a/src/amd/common/ac_nir.h b/src/amd/common/ac_nir.h index 6285045e9c5..3b2c8c9f1c9 100644 --- a/src/amd/common/ac_nir.h +++ b/src/amd/common/ac_nir.h @@ -77,7 +77,7 @@ ac_nir_unpack_arg(nir_builder *b, const struct ac_shader_args *ac_args, struct a bool ac_nir_lower_sin_cos(nir_shader *shader); bool ac_nir_lower_intrinsics_to_args(nir_shader *shader, const enum amd_gfx_level gfx_level, - const enum ac_hw_stage hw_stage, + bool has_ls_vgpr_init_bug, const enum ac_hw_stage hw_stage, const struct ac_shader_args *ac_args); bool ac_nir_optimize_outputs(nir_shader *nir, bool sprite_tex_disallowed, diff --git a/src/amd/compiler/aco_instruction_selection.cpp b/src/amd/compiler/aco_instruction_selection.cpp index 343cd375638..54c6012b623 100644 --- a/src/amd/compiler/aco_instruction_selection.cpp +++ b/src/amd/compiler/aco_instruction_selection.cpp @@ -8852,16 +8852,6 @@ visit_intrinsic(isel_context* ctx, nir_intrinsic_instr* instr) emit_split_vector(ctx, dst, 2); break; } - case nir_intrinsic_load_vertex_id_zero_base: { - Temp dst = get_ssa_temp(ctx, &instr->def); - bld.copy(Definition(dst), get_arg(ctx, ctx->args->vertex_id)); - break; - } - case nir_intrinsic_load_instance_id: { - Temp dst = get_ssa_temp(ctx, &instr->def); - bld.copy(Definition(dst), get_arg(ctx, ctx->args->instance_id)); - break; - } case nir_intrinsic_load_primitive_id: { Temp dst = get_ssa_temp(ctx, &instr->def); @@ -8924,11 +8914,6 @@ visit_intrinsic(isel_context* ctx, nir_intrinsic_instr* instr) bld.def(s1, scc), Operand::c32(nir_intrinsic_call_idx(instr))); break; } - case nir_intrinsic_overwrite_vs_arguments_amd: { - ctx->arg_temps[ctx->args->vertex_id.arg_index] = get_ssa_temp(ctx, instr->src[0].ssa); - ctx->arg_temps[ctx->args->instance_id.arg_index] = get_ssa_temp(ctx, instr->src[1].ssa); - break; - } case nir_intrinsic_overwrite_tes_arguments_amd: { ctx->arg_temps[ctx->args->tes_u.arg_index] = get_ssa_temp(ctx, instr->src[0].ssa); ctx->arg_temps[ctx->args->tes_v.arg_index] = get_ssa_temp(ctx, instr->src[1].ssa); @@ -11118,20 +11103,11 @@ fix_ls_vgpr_init_bug(isel_context* ctx) Temp ls_has_nonzero_hs_threads = bool_to_vector_condition(ctx, hs_thread_count.def(1).getTemp()); /* If there are no HS threads, SPI mistakenly loads the LS VGPRs starting at VGPR 0. */ - - Temp instance_id = - bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), get_arg(ctx, ctx->args->vertex_id), - get_arg(ctx, ctx->args->instance_id), ls_has_nonzero_hs_threads); Temp vs_rel_patch_id = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), get_arg(ctx, ctx->args->tcs_rel_ids), get_arg(ctx, ctx->args->vs_rel_patch_id), ls_has_nonzero_hs_threads); - Temp vertex_id = - bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), get_arg(ctx, ctx->args->tcs_patch_id), - get_arg(ctx, ctx->args->vertex_id), ls_has_nonzero_hs_threads); - ctx->arg_temps[ctx->args->instance_id.arg_index] = instance_id; ctx->arg_temps[ctx->args->vs_rel_patch_id.arg_index] = vs_rel_patch_id; - ctx->arg_temps[ctx->args->vertex_id.arg_index] = vertex_id; } void diff --git a/src/amd/compiler/aco_instruction_selection_setup.cpp b/src/amd/compiler/aco_instruction_selection_setup.cpp index e7d7928cdca..f30461ec0b9 100644 --- a/src/amd/compiler/aco_instruction_selection_setup.cpp +++ b/src/amd/compiler/aco_instruction_selection_setup.cpp @@ -549,7 +549,6 @@ init_context(isel_context* ctx, nir_shader* shader) case nir_intrinsic_load_input_vertex: case nir_intrinsic_load_per_vertex_input: case nir_intrinsic_load_per_vertex_output: - case nir_intrinsic_load_vertex_id_zero_base: case nir_intrinsic_load_interpolated_input: case nir_intrinsic_load_local_invocation_index: case nir_intrinsic_load_subgroup_invocation: @@ -558,7 +557,6 @@ init_context(isel_context* ctx, nir_shader* shader) case nir_intrinsic_mbcnt_amd: case nir_intrinsic_lane_permute_16_amd: case nir_intrinsic_dpp16_shift_amd: - case nir_intrinsic_load_instance_id: case nir_intrinsic_ssbo_atomic: case nir_intrinsic_ssbo_atomic_swap: case nir_intrinsic_global_atomic_amd: diff --git a/src/amd/llvm/ac_nir_to_llvm.c b/src/amd/llvm/ac_nir_to_llvm.c index dc0fe4032b6..c5ae7c9d531 100644 --- a/src/amd/llvm/ac_nir_to_llvm.c +++ b/src/amd/llvm/ac_nir_to_llvm.c @@ -2822,9 +2822,6 @@ static bool visit_intrinsic(struct ac_nir_context *ctx, nir_intrinsic_instr *ins case nir_intrinsic_load_lds_ngg_gs_out_vertex_base_amd: result = ctx->abi->intrinsic_load(ctx->abi, instr); break; - case nir_intrinsic_load_vertex_id_zero_base: - result = ctx->abi->vertex_id_replaced ? ctx->abi->vertex_id_replaced : ctx->abi->vertex_id; - break; case nir_intrinsic_load_primitive_id: if (ctx->stage == MESA_SHADER_GEOMETRY) { result = ac_get_arg(&ctx->ac, ctx->args->gs_prim_id); @@ -2845,10 +2842,6 @@ static bool visit_intrinsic(struct ac_nir_context *ctx, nir_intrinsic_instr *ins case nir_intrinsic_is_helper_invocation: result = ac_build_load_helper_invocation(&ctx->ac); break; - case nir_intrinsic_load_instance_id: - result = ctx->abi->instance_id_replaced ? - ctx->abi->instance_id_replaced : ctx->abi->instance_id; - break; case nir_intrinsic_load_num_workgroups: if (ctx->abi->load_grid_size_from_user_sgpr) { result = ac_get_arg(&ctx->ac, ctx->args->num_work_groups); @@ -3262,10 +3255,6 @@ static bool visit_intrinsic(struct ac_nir_context *ctx, nir_intrinsic_instr *ins result = LLVMBuildICmp(ctx->ac.builder, LLVMIntULT, ac_get_thread_id(&ctx->ac), count, ""); break; } - case nir_intrinsic_overwrite_vs_arguments_amd: - ctx->abi->vertex_id_replaced = get_src(ctx, instr->src[0]); - ctx->abi->instance_id_replaced = get_src(ctx, instr->src[1]); - break; case nir_intrinsic_overwrite_tes_arguments_amd: ctx->abi->tes_u_replaced = ac_to_float(&ctx->ac, get_src(ctx, instr->src[0])); ctx->abi->tes_v_replaced = ac_to_float(&ctx->ac, get_src(ctx, instr->src[1])); @@ -4207,15 +4196,7 @@ void ac_fixup_ls_hs_input_vgprs(struct ac_llvm_context *ac, struct ac_shader_abi LLVMValueRef count = ac_unpack_param(ac, ac_get_arg(ac, args->merged_wave_info), 8, 8); LLVMValueRef hs_empty = LLVMBuildICmp(ac->builder, LLVMIntEQ, count, ac->i32_0, ""); - abi->instance_id = - LLVMBuildSelect(ac->builder, hs_empty, ac_get_arg(ac, args->vertex_id), - abi->instance_id, ""); - abi->vs_rel_patch_id = LLVMBuildSelect(ac->builder, hs_empty, ac_get_arg(ac, args->tcs_rel_ids), abi->vs_rel_patch_id, ""); - - abi->vertex_id = - LLVMBuildSelect(ac->builder, hs_empty, ac_get_arg(ac, args->tcs_patch_id), - abi->vertex_id, ""); } diff --git a/src/amd/llvm/ac_shader_abi.h b/src/amd/llvm/ac_shader_abi.h index a438192d5f6..b2c16717960 100644 --- a/src/amd/llvm/ac_shader_abi.h +++ b/src/amd/llvm/ac_shader_abi.h @@ -26,13 +26,9 @@ struct ac_shader_abi { bool is_16bit[AC_LLVM_MAX_OUTPUTS * 4]; /* These input registers sometimes need to be fixed up. */ - LLVMValueRef vertex_id; LLVMValueRef vs_rel_patch_id; - LLVMValueRef instance_id; /* replaced registers when culling enabled */ - LLVMValueRef vertex_id_replaced; - LLVMValueRef instance_id_replaced; LLVMValueRef tes_u_replaced; LLVMValueRef tes_v_replaced; LLVMValueRef tes_rel_patch_id_replaced; diff --git a/src/amd/vulkan/radv_nir_to_llvm.c b/src/amd/vulkan/radv_nir_to_llvm.c index 7c10f26b896..1fc854388ea 100644 --- a/src/amd/vulkan/radv_nir_to_llvm.c +++ b/src/amd/vulkan/radv_nir_to_llvm.c @@ -279,12 +279,8 @@ ac_translate_nir_to_llvm(struct ac_llvm_compiler *ac_llvm, const struct radv_nir if (shader_count >= 2 || is_ngg) ac_init_exec_full_mask(&ctx.ac); - if (args->ac.vertex_id.used) - ctx.abi.vertex_id = ac_get_arg(&ctx.ac, args->ac.vertex_id); if (args->ac.vs_rel_patch_id.used) ctx.abi.vs_rel_patch_id = ac_get_arg(&ctx.ac, args->ac.vs_rel_patch_id); - if (args->ac.instance_id.used) - ctx.abi.instance_id = ac_get_arg(&ctx.ac, args->ac.instance_id); if (options->info->has_ls_vgpr_init_bug && shaders[shader_count - 1]->info.stage == MESA_SHADER_TESS_CTRL) ac_fixup_ls_hs_input_vgprs(&ctx.ac, &ctx.abi, &args->ac); diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c index e986fa3dde2..e43edf6b568 100644 --- a/src/amd/vulkan/radv_pipeline.c +++ b/src/amd/vulkan/radv_pipeline.c @@ -515,8 +515,9 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_graphics_stat }); NIR_PASS(_, stage->nir, ac_nir_lower_global_access); - NIR_PASS_V(stage->nir, ac_nir_lower_intrinsics_to_args, gfx_level, radv_select_hw_stage(&stage->info, gfx_level), - &stage->args.ac); + NIR_PASS_V(stage->nir, ac_nir_lower_intrinsics_to_args, gfx_level, + pdev->info.has_ls_vgpr_init_bug && gfx_state && !gfx_state->vs.has_prolog, + radv_select_hw_stage(&stage->info, gfx_level), &stage->args.ac); NIR_PASS_V(stage->nir, radv_nir_lower_abi, gfx_level, stage, gfx_state, pdev->info.address32_hi); radv_optimize_nir_algebraic( stage->nir, io_to_mem || lowered_ngg || stage->stage == MESA_SHADER_COMPUTE || stage->stage == MESA_SHADER_TASK, diff --git a/src/amd/vulkan/radv_pipeline_graphics.c b/src/amd/vulkan/radv_pipeline_graphics.c index 0de369dafe6..1efb9a05f74 100644 --- a/src/amd/vulkan/radv_pipeline_graphics.c +++ b/src/amd/vulkan/radv_pipeline_graphics.c @@ -2277,7 +2277,8 @@ radv_create_gs_copy_shader(struct radv_device *device, struct vk_pipeline_cache gs_copy_stage.info.user_sgprs_locs = gs_copy_stage.args.user_sgprs_locs; gs_copy_stage.info.inline_push_constant_mask = gs_copy_stage.args.ac.inline_push_const_mask; - NIR_PASS_V(nir, ac_nir_lower_intrinsics_to_args, pdev->info.gfx_level, AC_HW_VERTEX_SHADER, &gs_copy_stage.args.ac); + NIR_PASS_V(nir, ac_nir_lower_intrinsics_to_args, pdev->info.gfx_level, pdev->info.has_ls_vgpr_init_bug, + AC_HW_VERTEX_SHADER, &gs_copy_stage.args.ac); NIR_PASS_V(nir, radv_nir_lower_abi, pdev->info.gfx_level, &gs_copy_stage, gfx_state, pdev->info.address32_hi); struct radv_graphics_pipeline_key key = {0}; diff --git a/src/gallium/drivers/radeonsi/si_shader.c b/src/gallium/drivers/radeonsi/si_shader.c index efed3026f6c..259f04d8c3f 100644 --- a/src/gallium/drivers/radeonsi/si_shader.c +++ b/src/gallium/drivers/radeonsi/si_shader.c @@ -2592,6 +2592,7 @@ static struct nir_shader *si_get_nir_shader(struct si_shader *shader, struct si_ NIR_PASS(progress, nir, si_nir_lower_abi, shader, args); NIR_PASS(progress, nir, ac_nir_lower_intrinsics_to_args, sel->screen->info.gfx_level, + sel->screen->info.has_ls_vgpr_init_bug, si_select_hw_stage(nir->info.stage, key, sel->screen->info.gfx_level), &args->ac); @@ -2762,7 +2763,8 @@ si_nir_generate_gs_copy_shader(struct si_screen *sscreen, si_init_shader_args(shader, &args, &gs_nir->info); NIR_PASS_V(nir, si_nir_lower_abi, shader, &args); - NIR_PASS_V(nir, ac_nir_lower_intrinsics_to_args, sscreen->info.gfx_level, AC_HW_VERTEX_SHADER, &args.ac); + NIR_PASS_V(nir, ac_nir_lower_intrinsics_to_args, sscreen->info.gfx_level, + sscreen->info.has_ls_vgpr_init_bug, AC_HW_VERTEX_SHADER, &args.ac); si_nir_opts(gs_selector->screen, nir, false); diff --git a/src/gallium/drivers/radeonsi/si_shader_llvm.c b/src/gallium/drivers/radeonsi/si_shader_llvm.c index 7629b51fc8d..ae78628f07a 100644 --- a/src/gallium/drivers/radeonsi/si_shader_llvm.c +++ b/src/gallium/drivers/radeonsi/si_shader_llvm.c @@ -219,8 +219,6 @@ void si_llvm_create_main_func(struct si_shader_context *ctx) } if (ctx->stage == MESA_SHADER_VERTEX) { - ctx->abi.vertex_id = ac_get_arg(&ctx->ac, ctx->args->ac.vertex_id); - ctx->abi.instance_id = ac_get_arg(&ctx->ac, ctx->args->ac.instance_id); if (ctx->args->ac.vs_rel_patch_id.used) ctx->abi.vs_rel_patch_id = ac_get_arg(&ctx->ac, ctx->args->ac.vs_rel_patch_id);