From 09e607c3853c4621919ada7e395b2f2a5059a3b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Ol=C5=A1=C3=A1k?= Date: Tue, 1 Apr 2025 15:43:17 -0400 Subject: [PATCH] nir: add access to load_smem_amd (for ACCESS_CAN_SPECULATE) Part-of: --- src/amd/vulkan/nir/radv_nir_lower_abi.c | 6 +++-- src/amd/vulkan/nir/radv_nir_lower_vs_inputs.c | 3 ++- src/compiler/nir/nir_intrinsics.py | 2 +- .../drivers/radeonsi/si_nir_lower_abi.c | 12 ++++++---- .../drivers/radeonsi/si_nir_lower_resource.c | 22 ++++++++++--------- .../drivers/radeonsi/si_nir_lower_vs_inputs.c | 3 ++- 6 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/amd/vulkan/nir/radv_nir_lower_abi.c b/src/amd/vulkan/nir/radv_nir_lower_abi.c index 14d2893e41a..6de91165ee0 100644 --- a/src/amd/vulkan/nir/radv_nir_lower_abi.c +++ b/src/amd/vulkan/nir/radv_nir_lower_abi.c @@ -34,7 +34,8 @@ load_ring(nir_builder *b, unsigned ring, lower_abi_state *s) nir_def *ring_offsets = ac_nir_load_arg(b, &s->args->ac, arg); ring_offsets = nir_pack_64_2x32_split(b, nir_channel(b, ring_offsets, 0), nir_channel(b, ring_offsets, 1)); - return nir_load_smem_amd(b, 4, ring_offsets, nir_imm_int(b, ring * 16u), .align_mul = 4u); + return nir_load_smem_amd(b, 4, ring_offsets, nir_imm_int(b, ring * 16u), .align_mul = 4u, + .access = ACCESS_CAN_SPECULATE); } static nir_def * @@ -366,7 +367,8 @@ lower_abi_instr(nir_builder *b, nir_intrinsic_instr *intrin, void *state) case nir_intrinsic_load_streamout_buffer_amd: { nir_def *ptr = nir_pack_64_2x32_split(b, ac_nir_load_arg(b, &s->args->ac, s->args->streamout_buffers), nir_imm_int(b, s->address32_hi)); - replacement = nir_load_smem_amd(b, 4, ptr, nir_imm_int(b, nir_intrinsic_base(intrin) * 16)); + replacement = nir_load_smem_amd(b, 4, ptr, nir_imm_int(b, nir_intrinsic_base(intrin) * 16), + .access = ACCESS_CAN_SPECULATE); break; } case nir_intrinsic_load_xfb_state_address_gfx12_amd: diff --git a/src/amd/vulkan/nir/radv_nir_lower_vs_inputs.c b/src/amd/vulkan/nir/radv_nir_lower_vs_inputs.c index d5261137f28..a32cd27a936 100644 --- a/src/amd/vulkan/nir/radv_nir_lower_vs_inputs.c +++ b/src/amd/vulkan/nir/radv_nir_lower_vs_inputs.c @@ -234,7 +234,8 @@ lower_load_vs_input(nir_builder *b, nir_intrinsic_instr *intrin, lower_vs_inputs nir_def *vertex_buffers_arg = ac_nir_load_arg(b, &s->args->ac, s->args->ac.vertex_buffers); nir_def *vertex_buffers = nir_pack_64_2x32_split(b, vertex_buffers_arg, nir_imm_int(b, s->gpu_info->address32_hi)); - nir_def *descriptor = nir_load_smem_amd(b, 4, vertex_buffers, nir_imm_int(b, desc_index * 16)); + nir_def *descriptor = nir_load_smem_amd(b, 4, vertex_buffers, nir_imm_int(b, desc_index * 16), + .access = ACCESS_CAN_SPECULATE); nir_def *base_index = calc_vs_input_index(b, location, s); nir_def *zero = nir_imm_int(b, 0); diff --git a/src/compiler/nir/nir_intrinsics.py b/src/compiler/nir/nir_intrinsics.py index 1ce7147293d..4fa6b66f0e1 100644 --- a/src/compiler/nir/nir_intrinsics.py +++ b/src/compiler/nir/nir_intrinsics.py @@ -1930,7 +1930,7 @@ store("vector_arg_amd", [], [BASE]) # restriction justifies the CAN_REORDER flag. Additionally, the base/offset must # be subgroup uniform. intrinsic("load_smem_amd", src_comp=[1, 1], dest_comp=0, bit_sizes=[32], - indices=[ALIGN_MUL, ALIGN_OFFSET], + indices=[ALIGN_MUL, ALIGN_OFFSET, ACCESS], flags=[CAN_ELIMINATE, CAN_REORDER]) # src[] = { offset }. diff --git a/src/gallium/drivers/radeonsi/si_nir_lower_abi.c b/src/gallium/drivers/radeonsi/si_nir_lower_abi.c index 02549f8a0ab..669925542a2 100644 --- a/src/gallium/drivers/radeonsi/si_nir_lower_abi.c +++ b/src/gallium/drivers/radeonsi/si_nir_lower_abi.c @@ -29,7 +29,8 @@ nir_def *si_nir_load_internal_binding(nir_builder *b, struct si_shader_args *arg unsigned slot, unsigned num_components) { nir_def *addr = ac_nir_load_arg(b, &args->ac, args->internal_bindings); - return nir_load_smem_amd(b, num_components, addr, nir_imm_int(b, slot * 16)); + return nir_load_smem_amd(b, num_components, addr, nir_imm_int(b, slot * 16), + .access = ACCESS_CAN_SPECULATE); } static nir_def *build_attr_ring_desc(nir_builder *b, struct si_shader *shader, @@ -321,17 +322,20 @@ static bool lower_intrinsic(nir_builder *b, nir_instr *instr, struct lower_abi_s } case nir_intrinsic_load_clip_half_line_width_amd: { nir_def *addr = ac_nir_load_arg(b, &args->ac, args->small_prim_cull_info); - replacement = nir_load_smem_amd(b, 2, addr, nir_imm_int(b, 32)); + replacement = nir_load_smem_amd(b, 2, addr, nir_imm_int(b, 32), + .access = ACCESS_CAN_SPECULATE); break; } case nir_intrinsic_load_cull_triangle_viewport_xy_scale_and_offset_amd: { nir_def *addr = ac_nir_load_arg(b, &args->ac, args->small_prim_cull_info); - replacement = nir_load_smem_amd(b, 4, addr, nir_imm_int(b, 0)); + replacement = nir_load_smem_amd(b, 4, addr, nir_imm_int(b, 0), + .access = ACCESS_CAN_SPECULATE); break; } case nir_intrinsic_load_cull_line_viewport_xy_scale_and_offset_amd: { nir_def *addr = ac_nir_load_arg(b, &args->ac, args->small_prim_cull_info); - replacement = nir_load_smem_amd(b, 4, addr, nir_imm_int(b, 16)); + replacement = nir_load_smem_amd(b, 4, addr, nir_imm_int(b, 16), + .access = ACCESS_CAN_SPECULATE); break; } case nir_intrinsic_load_num_vertices_per_primitive_amd: diff --git a/src/gallium/drivers/radeonsi/si_nir_lower_resource.c b/src/gallium/drivers/radeonsi/si_nir_lower_resource.c index 462ecc0cba8..d78abe4bee6 100644 --- a/src/gallium/drivers/radeonsi/si_nir_lower_resource.c +++ b/src/gallium/drivers/radeonsi/si_nir_lower_resource.c @@ -74,7 +74,7 @@ static nir_def *load_ubo_desc(nir_builder *b, nir_def *index, index = nir_iadd_imm(b, index, SI_NUM_SHADER_BUFFERS); nir_def *offset = nir_ishl_imm(b, index, 4); - return nir_load_smem_amd(b, 4, addr, offset); + return nir_load_smem_amd(b, 4, addr, offset, .access = ACCESS_CAN_SPECULATE); } static nir_def *load_ssbo_desc(nir_builder *b, nir_src *index, @@ -94,7 +94,7 @@ static nir_def *load_ssbo_desc(nir_builder *b, nir_src *index, slot = nir_isub_imm(b, SI_NUM_SHADER_BUFFERS - 1, slot); nir_def *offset = nir_ishl_imm(b, slot, 4); - return nir_load_smem_amd(b, 4, addr, offset); + return nir_load_smem_amd(b, 4, addr, offset, .access = ACCESS_CAN_SPECULATE); } static nir_def *fixup_image_desc(nir_builder *b, nir_def *rsrc, bool uses_store, @@ -138,7 +138,7 @@ static nir_def *fixup_image_desc(nir_builder *b, nir_def *rsrc, bool uses_store, */ static nir_def *load_image_desc(nir_builder *b, nir_def *list, nir_def *index, enum ac_descriptor_type desc_type, bool uses_store, - struct lower_resource_state *s) + bool bindless, struct lower_resource_state *s) { /* index is in uvec8 unit, convert to offset in bytes */ nir_def *offset = nir_ishl_imm(b, index, 5); @@ -151,7 +151,8 @@ static nir_def *load_image_desc(nir_builder *b, nir_def *list, nir_def *index, num_channels = 8; } - nir_def *rsrc = nir_load_smem_amd(b, num_channels, list, offset); + nir_def *rsrc = nir_load_smem_amd(b, num_channels, list, offset, + .access = bindless ? 0 : ACCESS_CAN_SPECULATE); if (desc_type == AC_DESC_IMAGE) rsrc = fixup_image_desc(b, rsrc, uses_store, s); @@ -237,7 +238,7 @@ static nir_def *load_deref_image_desc(nir_builder *b, nir_deref_instr *deref, index = nir_isub_imm(b, SI_NUM_IMAGE_SLOTS - 1, index); nir_def *list = ac_nir_load_arg(b, &s->args->ac, s->args->samplers_and_images); - desc = load_image_desc(b, list, index, desc_type, !is_load, s); + desc = load_image_desc(b, list, index, desc_type, !is_load, false, s); } return desc; @@ -255,7 +256,7 @@ static nir_def *load_bindless_image_desc(nir_builder *b, nir_def *index, index = nir_iadd_imm(b, index, 1); nir_def *list = ac_nir_load_arg(b, &s->args->ac, s->args->bindless_samplers_and_images); - return load_image_desc(b, list, index, desc_type, !is_load, s); + return load_image_desc(b, list, index, desc_type, !is_load, true, s); } static bool lower_resource_intrinsic(nir_builder *b, nir_intrinsic_instr *intrin, @@ -397,7 +398,7 @@ static bool lower_resource_intrinsic(nir_builder *b, nir_intrinsic_instr *intrin } static nir_def *load_sampler_desc(nir_builder *b, nir_def *list, nir_def *index, - enum ac_descriptor_type desc_type) + enum ac_descriptor_type desc_type, bool bindless) { /* index is in 16 dword unit, convert to offset in bytes */ nir_def *offset = nir_ishl_imm(b, index, 6); @@ -427,7 +428,8 @@ static nir_def *load_sampler_desc(nir_builder *b, nir_def *list, nir_def *index, break; } - return nir_load_smem_amd(b, num_channels, list, offset); + return nir_load_smem_amd(b, num_channels, list, offset, + .access = bindless ? 0 : ACCESS_CAN_SPECULATE); } static nir_def *load_deref_sampler_desc(nir_builder *b, nir_deref_instr *deref, @@ -442,7 +444,7 @@ static nir_def *load_deref_sampler_desc(nir_builder *b, nir_deref_instr *deref, /* return actual desc when required by caller */ if (return_descriptor) { nir_def *list = ac_nir_load_arg(b, &s->args->ac, s->args->samplers_and_images); - return load_sampler_desc(b, list, index, desc_type); + return load_sampler_desc(b, list, index, desc_type, false); } /* Just use index here and let nir-to-llvm backend to translate to actual @@ -461,7 +463,7 @@ static nir_def *load_bindless_sampler_desc(nir_builder *b, nir_def *index, /* 64 bit to 32 bit */ index = nir_u2u32(b, index); - return load_sampler_desc(b, list, index, desc_type); + return load_sampler_desc(b, list, index, desc_type, true); } static nir_def *fixup_sampler_desc(nir_builder *b, diff --git a/src/gallium/drivers/radeonsi/si_nir_lower_vs_inputs.c b/src/gallium/drivers/radeonsi/si_nir_lower_vs_inputs.c index 6cdc990f826..bdfe0be1be1 100644 --- a/src/gallium/drivers/radeonsi/si_nir_lower_vs_inputs.c +++ b/src/gallium/drivers/radeonsi/si_nir_lower_vs_inputs.c @@ -433,7 +433,8 @@ load_vs_input_from_vertex_buffer(nir_builder *b, unsigned input_index, } else { unsigned index = input_index - sel->info.num_vbos_in_user_sgprs; nir_def *addr = ac_nir_load_arg(b, &s->args->ac, s->args->ac.vertex_buffers); - vb_desc = nir_load_smem_amd(b, 4, addr, nir_imm_int(b, index * 16)); + vb_desc = nir_load_smem_amd(b, 4, addr, nir_imm_int(b, index * 16), + .access = ACCESS_CAN_SPECULATE); } nir_def *vertex_index = s->vertex_index[input_index];