From 3fe651f6079584f0e07ab683e4a59df64a26e2c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Ol=C5=A1=C3=A1k?= Date: Fri, 8 Aug 2025 18:00:45 -0400 Subject: [PATCH] nir: remove load_smem_amd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit replaced by load_global_amd + ACCESS_SMEM_AMD Reviewed-by: Daniel Schürmann Part-of: --- src/amd/common/nir/ac_nir.c | 4 +-- .../instruction_selection/aco_isel_setup.cpp | 4 +-- .../aco_select_nir_intrinsics.cpp | 27 ------------------- src/amd/llvm/ac_nir_to_llvm.c | 20 -------------- src/compiler/nir/nir.h | 2 +- src/compiler/nir/nir_divergence_analysis.c | 1 - src/compiler/nir/nir_intrinsics.py | 9 ------- src/compiler/nir/nir_lower_io.c | 2 -- src/compiler/nir/nir_opt_call.c | 1 - src/compiler/nir/nir_opt_load_skip_helpers.c | 1 - .../nir/nir_opt_load_store_vectorize.c | 1 - src/compiler/nir/nir_opt_move_to_top.c | 7 ++--- src/compiler/nir/nir_opt_sink.c | 6 ++--- 13 files changed, 7 insertions(+), 78 deletions(-) diff --git a/src/amd/common/nir/ac_nir.c b/src/amd/common/nir/ac_nir.c index 7d79f217aa4..a0a536d00c2 100644 --- a/src/amd/common/nir/ac_nir.c +++ b/src/amd/common/nir/ac_nir.c @@ -490,7 +490,6 @@ ac_nir_mem_vectorize_callback(unsigned align_mul, unsigned align_offset, unsigne bool uses_smem = (nir_intrinsic_has_access(low) && nir_intrinsic_access(low) & ACCESS_SMEM_AMD) || /* These don't have the "access" field. */ - low->intrinsic == nir_intrinsic_load_smem_amd || low->intrinsic == nir_intrinsic_load_push_constant; bool is_store = !nir_intrinsic_infos[low->intrinsic].has_dest; bool swizzled = low->intrinsic == nir_intrinsic_load_stack || @@ -513,12 +512,11 @@ ac_nir_mem_vectorize_callback(unsigned align_mul, unsigned align_offset, unsigne nir_deref_mode_is(nir_src_as_deref(low->src[0]), nir_var_mem_shared)); /* Don't vectorize descriptor loads for LLVM due to excessive SGPR and VGPR spilling. */ - if (!config->uses_aco && low->intrinsic == nir_intrinsic_load_smem_amd) + if (!config->uses_aco && low->intrinsic == nir_intrinsic_load_global && uses_smem) return false; /* Reject opcodes we don't vectorize. */ switch (low->intrinsic) { - case nir_intrinsic_load_smem_amd: case nir_intrinsic_load_push_constant: case nir_intrinsic_load_ubo: case nir_intrinsic_load_stack: diff --git a/src/amd/compiler/instruction_selection/aco_isel_setup.cpp b/src/amd/compiler/instruction_selection/aco_isel_setup.cpp index d57105cec31..3909d1a769f 100644 --- a/src/amd/compiler/instruction_selection/aco_isel_setup.cpp +++ b/src/amd/compiler/instruction_selection/aco_isel_setup.cpp @@ -202,8 +202,7 @@ apply_nuw_to_offsets(isel_context* ctx, nir_function_impl* impl) apply_nuw_to_ssa(ctx, intrin->src[2].ssa); break; case nir_intrinsic_load_scratch: apply_nuw_to_ssa(ctx, intrin->src[0].ssa); break; - case nir_intrinsic_store_scratch: - case nir_intrinsic_load_smem_amd: apply_nuw_to_ssa(ctx, intrin->src[1].ssa); break; + case nir_intrinsic_store_scratch: apply_nuw_to_ssa(ctx, intrin->src[1].ssa); break; case nir_intrinsic_load_global_amd: if (nir_intrinsic_access(intrin) & ACCESS_SMEM_AMD) apply_nuw_to_ssa(ctx, intrin->src[1].ssa); @@ -566,7 +565,6 @@ init_context(isel_context* ctx, nir_shader* shader) case nir_intrinsic_ballot_relaxed: case nir_intrinsic_bindless_image_samples: case nir_intrinsic_load_scalar_arg_amd: - case nir_intrinsic_load_smem_amd: case nir_intrinsic_unit_test_uniform_amd: type = RegType::sgpr; break; case nir_intrinsic_load_input: case nir_intrinsic_load_per_primitive_input: diff --git a/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp b/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp index 8d2e347dceb..2e0ca601414 100644 --- a/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp +++ b/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp @@ -2853,32 +2853,6 @@ visit_store_buffer(isel_context* ctx, nir_intrinsic_instr* intrin) } } -void -visit_load_smem(isel_context* ctx, nir_intrinsic_instr* instr) -{ - Builder bld(ctx->program, ctx->block); - Temp dst = get_ssa_temp(ctx, &instr->def); - Temp base = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa)); - Temp offset = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa)); - - assert(base.bytes() == 8); - - aco_opcode opcode; - unsigned size; - assert(dst.bytes() <= 64); - std::tie(opcode, size) = get_smem_opcode(ctx->program->gfx_level, dst.bytes(), false, false); - size = util_next_power_of_two(size); - - if (dst.size() != DIV_ROUND_UP(size, 4)) { - bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), - bld.smem(opcode, bld.def(RegClass::get(RegType::sgpr, size)), base, offset), - Operand::c32(0u)); - } else { - bld.smem(opcode, Definition(dst), base, offset); - } - emit_split_vector(ctx, dst, instr->def.num_components); -} - sync_scope translate_nir_scope(mesa_scope scope) { @@ -4090,7 +4064,6 @@ visit_intrinsic(isel_context* ctx, nir_intrinsic_instr* instr) case nir_intrinsic_load_typed_buffer_amd: case nir_intrinsic_load_buffer_amd: visit_load_buffer(ctx, instr); break; case nir_intrinsic_store_buffer_amd: visit_store_buffer(ctx, instr); break; - case nir_intrinsic_load_smem_amd: visit_load_smem(ctx, instr); break; case nir_intrinsic_load_global_amd: visit_load_global(ctx, instr); break; case nir_intrinsic_store_global_amd: visit_store_global(ctx, instr); break; case nir_intrinsic_global_atomic_amd: diff --git a/src/amd/llvm/ac_nir_to_llvm.c b/src/amd/llvm/ac_nir_to_llvm.c index 520b17c0c81..ece97735712 100644 --- a/src/amd/llvm/ac_nir_to_llvm.c +++ b/src/amd/llvm/ac_nir_to_llvm.c @@ -3232,26 +3232,6 @@ static bool visit_intrinsic(struct ac_nir_context *ctx, nir_intrinsic_instr *ins result = LLVMBuildBitCast(ctx->ac.builder, result, get_def_type(ctx, &instr->def), ""); break; } - case nir_intrinsic_load_smem_amd: { - LLVMValueRef base = get_src(ctx, instr->src[0]); - LLVMValueRef offset = get_src(ctx, instr->src[1]); - - bool is_addr_32bit = nir_src_bit_size(instr->src[0]) == 32; - int addr_space = is_addr_32bit ? AC_ADDR_SPACE_CONST_32BIT : AC_ADDR_SPACE_CONST; - - LLVMTypeRef result_type = get_def_type(ctx, &instr->def); - LLVMValueRef addr = LLVMBuildIntToPtr(ctx->ac.builder, base, - LLVMPointerTypeInContext(ctx->ac.context, addr_space), ""); - /* see ac_build_load_custom() for 32bit/64bit addr GEP difference */ - addr = is_addr_32bit ? - LLVMBuildInBoundsGEP2(ctx->ac.builder, ctx->ac.i8, addr, &offset, 1, "") : - LLVMBuildGEP2(ctx->ac.builder, ctx->ac.i8, addr, &offset, 1, ""); - - LLVMSetMetadata(addr, ctx->ac.uniform_md_kind, ctx->ac.empty_md); - result = LLVMBuildLoad2(ctx->ac.builder, result_type, addr, ""); - LLVMSetMetadata(result, ctx->ac.invariant_load_md_kind, ctx->ac.empty_md); - break; - } case nir_intrinsic_ordered_xfb_counter_add_gfx11_amd: { /* Gfx11 GDS instructions only operate on the first active lane. All other lanes are * ignored. So are their EXEC bits. This uses the mutex feature of ds_ordered_count diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index 12b345ccaf8..0a78572a2ff 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -6317,7 +6317,7 @@ typedef enum { nir_move_load_buffer_amd = BITFIELD_BIT(21), nir_move_load_frag_coord = BITFIELD_BIT(22), - /* The following options only impact load_global/ubo/ssbo/smem_amd. */ + /* The following options only impact load_global/ubo/ssbo. */ nir_move_only_convergent = BITFIELD_BIT(30), nir_move_only_divergent = BITFIELD_BIT(31), } nir_move_options; diff --git a/src/compiler/nir/nir_divergence_analysis.c b/src/compiler/nir/nir_divergence_analysis.c index 9fa6e27c537..57d8c8a0c15 100644 --- a/src/compiler/nir/nir_divergence_analysis.c +++ b/src/compiler/nir/nir_divergence_analysis.c @@ -298,7 +298,6 @@ visit_intrinsic(nir_intrinsic_instr *instr, struct divergence_state *state) case nir_intrinsic_load_tess_level_inner_default: case nir_intrinsic_load_tess_level_outer_default: case nir_intrinsic_load_scalar_arg_amd: - case nir_intrinsic_load_smem_amd: case nir_intrinsic_load_resume_shader_address_amd: case nir_intrinsic_load_reloc_const_intel: case nir_intrinsic_load_btd_global_arg_addr_intel: diff --git a/src/compiler/nir/nir_intrinsics.py b/src/compiler/nir/nir_intrinsics.py index 0e677fd5ff9..41371db0be7 100644 --- a/src/compiler/nir/nir_intrinsics.py +++ b/src/compiler/nir/nir_intrinsics.py @@ -1953,15 +1953,6 @@ intrinsic("load_vector_arg_amd", dest_comp=0, bit_sizes=[32], store("scalar_arg_amd", [], [BASE]) store("vector_arg_amd", [], [BASE]) -# src[] = { 32/64-bit base address, 32-bit offset }. -# -# Similar to load_global_constant, the memory accessed must be read-only. This -# restriction justifies the CAN_REORDER flag. Additionally, the base/offset must -# be subgroup uniform. -intrinsic("load_smem_amd", src_comp=[1, 1], dest_comp=0, bit_sizes=[32], - indices=[ALIGN_MUL, ALIGN_OFFSET, ACCESS], - flags=[CAN_ELIMINATE, CAN_REORDER]) - # src[] = { offset }. intrinsic("load_shared2_amd", [1], dest_comp=2, indices=[ACCESS, OFFSET0, OFFSET1, ST64], flags=[CAN_ELIMINATE]) diff --git a/src/compiler/nir/nir_lower_io.c b/src/compiler/nir/nir_lower_io.c index 588ea1f381c..e167e441463 100644 --- a/src/compiler/nir/nir_lower_io.c +++ b/src/compiler/nir/nir_lower_io.c @@ -973,7 +973,6 @@ nir_get_io_offset_src_number(const nir_intrinsic_instr *instr) case nir_intrinsic_load_per_view_output: case nir_intrinsic_load_per_primitive_output: case nir_intrinsic_load_interpolated_input: - case nir_intrinsic_load_smem_amd: case nir_intrinsic_load_global_amd: case nir_intrinsic_store_output: case nir_intrinsic_store_shared: @@ -1051,7 +1050,6 @@ nir_get_io_index_src_number(const nir_intrinsic_instr *instr) case nir_intrinsic_load_per_view_output: case nir_intrinsic_load_per_primitive_output: case nir_intrinsic_load_interpolated_input: - case nir_intrinsic_load_smem_amd: case nir_intrinsic_load_global_amd: case nir_intrinsic_global_atomic_amd: case nir_intrinsic_global_atomic_swap_amd: diff --git a/src/compiler/nir/nir_opt_call.c b/src/compiler/nir/nir_opt_call.c index f90ffcefe61..59d355489df 100644 --- a/src/compiler/nir/nir_opt_call.c +++ b/src/compiler/nir/nir_opt_call.c @@ -30,7 +30,6 @@ can_remat_instr(nir_instr *instr) case nir_intrinsic_load_vulkan_descriptor: case nir_intrinsic_load_push_constant: case nir_intrinsic_load_global_constant: - case nir_intrinsic_load_smem_amd: case nir_intrinsic_load_scalar_arg_amd: case nir_intrinsic_load_vector_arg_amd: return true; diff --git a/src/compiler/nir/nir_opt_load_skip_helpers.c b/src/compiler/nir/nir_opt_load_skip_helpers.c index fc66daeb9f6..e0a17c12bb0 100644 --- a/src/compiler/nir/nir_opt_load_skip_helpers.c +++ b/src/compiler/nir/nir_opt_load_skip_helpers.c @@ -178,7 +178,6 @@ nir_opt_load_skip_helpers(nir_shader *shader, nir_opt_load_skip_helpers_options add_load_to_worklist(&hs, instr)) { switch (intr->intrinsic) { case nir_intrinsic_load_global_amd: - case nir_intrinsic_load_smem_amd: break; default: { /* Even if this load is skipped for helpers, the handle must diff --git a/src/compiler/nir/nir_opt_load_store_vectorize.c b/src/compiler/nir/nir_opt_load_store_vectorize.c index c12b83f7d20..0171a2a31f0 100644 --- a/src/compiler/nir/nir_opt_load_store_vectorize.c +++ b/src/compiler/nir/nir_opt_load_store_vectorize.c @@ -119,7 +119,6 @@ get_info(nir_intrinsic_op op) STORE(nir_var_uniform, const_ir3, -1, -1, -1, 0, 4) INFO(nir_var_mem_shared, shared_append_amd, true, -1, -1, -1, -1, 1) INFO(nir_var_mem_shared, shared_consume_amd, true, -1, -1, -1, -1, 1) - LOAD(nir_var_mem_global, smem_amd, 0, 1, -1, 1) LOAD(0, buffer_amd, 0, 1, -1, 1) STORE(0, buffer_amd, 1, 2, -1, 0, 1) default: diff --git a/src/compiler/nir/nir_opt_move_to_top.c b/src/compiler/nir/nir_opt_move_to_top.c index 95306e5fd7d..fd4b7151bcc 100644 --- a/src/compiler/nir/nir_opt_move_to_top.c +++ b/src/compiler/nir/nir_opt_move_to_top.c @@ -72,9 +72,7 @@ can_move_src_to_top(nir_src *src, void *_state) case nir_intrinsic_load_interpolated_input: case nir_intrinsic_load_per_primitive_input: case nir_intrinsic_load_per_vertex_input: - /* nir_move_to_top_load_smem_amd and its sources. */ case nir_intrinsic_load_scalar_arg_amd: - case nir_intrinsic_load_smem_amd: break; case nir_intrinsic_load_global_amd: if (!(nir_intrinsic_access(nir_instr_as_intrinsic(instr)) & @@ -146,9 +144,8 @@ handle_load(nir_builder *b, nir_intrinsic_instr *intr, void *_state) !nir_is_output_load(intr); move |= state->options & nir_move_to_top_load_smem_amd && - ((intr->intrinsic == nir_intrinsic_load_global_amd && - nir_intrinsic_access(intr) & ACCESS_SMEM_AMD) || - intr->intrinsic == nir_intrinsic_load_smem_amd); + (intr->intrinsic == nir_intrinsic_load_global_amd && + nir_intrinsic_access(intr) & ACCESS_SMEM_AMD); if (!move) return false; diff --git a/src/compiler/nir/nir_opt_sink.c b/src/compiler/nir/nir_opt_sink.c index 161b491f595..764c89c622e 100644 --- a/src/compiler/nir/nir_opt_sink.c +++ b/src/compiler/nir/nir_opt_sink.c @@ -150,8 +150,7 @@ can_sink_instr(nir_instr *instr, nir_move_options options, bool *can_mov_out_of_ if (intrin->intrinsic == nir_intrinsic_load_global || intrin->intrinsic == nir_intrinsic_load_global_amd || intrin->intrinsic == nir_intrinsic_load_ubo || - intrin->intrinsic == nir_intrinsic_load_ssbo || - intrin->intrinsic == nir_intrinsic_load_smem_amd) { + intrin->intrinsic == nir_intrinsic_load_ssbo) { if (intrin->def.divergent) { if (options & nir_move_only_convergent) return false; @@ -201,8 +200,7 @@ can_sink_instr(nir_instr *instr, nir_move_options options, bool *can_mov_out_of_ return options & nir_move_load_input; case nir_intrinsic_load_global: - case nir_intrinsic_load_global_amd: - case nir_intrinsic_load_smem_amd: /* = global + convergent */ + case nir_intrinsic_load_global_amd: /* = global + convergent */ return options & nir_move_load_global; case nir_intrinsic_ldc_nv: