diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index 0814d263a7d..2ef2b7dce26 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -6640,6 +6640,7 @@ bool nir_dedup_inline_samplers(nir_shader *shader); typedef struct nir_lower_ssbo_options { bool native_loads; + bool native_offset; } nir_lower_ssbo_options; bool nir_lower_ssbo(nir_shader *shader, const nir_lower_ssbo_options *opts); diff --git a/src/compiler/nir/nir_intrinsics.py b/src/compiler/nir/nir_intrinsics.py index 4b7c08aa271..e07730547ed 100644 --- a/src/compiler/nir/nir_intrinsics.py +++ b/src/compiler/nir/nir_intrinsics.py @@ -1162,8 +1162,8 @@ load("per_primitive_input", [1], [BASE, COMPONENT, DEST_TYPE, IO_SEMANTICS], [CA # src[] = { buffer_index, offset }. load("ssbo", [-1, 1], [ACCESS, ALIGN_MUL, ALIGN_OFFSET], [CAN_ELIMINATE]) -# src[] = { buffer_index } -load("ssbo_address", [1], [], [CAN_ELIMINATE, CAN_REORDER]) +# src[] = { buffer_index, offset } +load("ssbo_address", [1, 1], [], [CAN_ELIMINATE, CAN_REORDER]) # src[] = { offset }. load("output", [1], [BASE, RANGE, COMPONENT, DEST_TYPE, IO_SEMANTICS], flags=[CAN_ELIMINATE]) # src[] = { vertex, offset }. diff --git a/src/compiler/nir/nir_lower_ssbo.c b/src/compiler/nir/nir_lower_ssbo.c index f8be6fdfcd3..7bd26850035 100644 --- a/src/compiler/nir/nir_lower_ssbo.c +++ b/src/compiler/nir/nir_lower_ssbo.c @@ -13,12 +13,20 @@ */ static nir_def * -calc_address(nir_builder *b, nir_intrinsic_instr *intr) +calc_address(nir_builder *b, nir_intrinsic_instr *intr, + const nir_lower_ssbo_options *opts) { unsigned index_src = intr->intrinsic == nir_intrinsic_store_ssbo ? 1 : 0; - nir_def *base = nir_load_ssbo_address(b, 1, 64, intr->src[index_src].ssa); + bool lower_offset = !opts || !opts->native_offset; + nir_def *offset = nir_get_io_offset_src(intr)->ssa; + nir_def *addr = + nir_load_ssbo_address(b, 1, 64, intr->src[index_src].ssa, + lower_offset ? nir_imm_int(b, 0) : offset); - return nir_iadd(b, base, nir_u2u64(b, nir_get_io_offset_src(intr)->ssa)); + if (lower_offset) + addr = nir_iadd(b, addr, nir_u2u64(b, offset)); + + return addr; } static bool @@ -35,26 +43,30 @@ pass(nir_builder *b, nir_intrinsic_instr *intr, void *data) return false; def = nir_build_load_global(b, intr->def.num_components, - intr->def.bit_size, calc_address(b, intr), + intr->def.bit_size, + calc_address(b, intr, opts), .align_mul = nir_intrinsic_align_mul(intr), .align_offset = nir_intrinsic_align_offset(intr)); break; case nir_intrinsic_store_ssbo: - nir_build_store_global(b, intr->src[0].ssa, calc_address(b, intr), + nir_build_store_global(b, intr->src[0].ssa, + calc_address(b, intr, opts), .align_mul = nir_intrinsic_align_mul(intr), .align_offset = nir_intrinsic_align_offset(intr), .write_mask = nir_intrinsic_write_mask(intr)); break; case nir_intrinsic_ssbo_atomic: - def = nir_global_atomic(b, intr->def.bit_size, calc_address(b, intr), + def = nir_global_atomic(b, intr->def.bit_size, + calc_address(b, intr, opts), intr->src[2].ssa, .atomic_op = nir_intrinsic_atomic_op(intr)); break; case nir_intrinsic_ssbo_atomic_swap: - def = nir_global_atomic_swap(b, intr->def.bit_size, calc_address(b, intr), + def = nir_global_atomic_swap(b, intr->def.bit_size, + calc_address(b, intr, opts), intr->src[2].ssa, intr->src[3].ssa, .atomic_op = nir_intrinsic_atomic_op(intr)); break; diff --git a/src/gallium/drivers/asahi/agx_nir_lower_sysvals.c b/src/gallium/drivers/asahi/agx_nir_lower_sysvals.c index 143606d0809..93113887184 100644 --- a/src/gallium/drivers/asahi/agx_nir_lower_sysvals.c +++ b/src/gallium/drivers/asahi/agx_nir_lower_sysvals.c @@ -170,6 +170,7 @@ lower_intrinsic(nir_builder *b, nir_intrinsic_instr *intr, return load_sysval_root( b, 1, 64, &u->pipeline_statistics[nir_intrinsic_base(intr)]); case nir_intrinsic_load_ssbo_address: + assert(nir_src_as_uint(intr->src[1]) == 0); return load_sysval_indirect(b, 1, 64, stage_table(b), &s->ssbo_base, intr->src[0].ssa); case nir_intrinsic_get_ubo_size: diff --git a/src/gallium/drivers/panfrost/pan_nir_lower_sysvals.c b/src/gallium/drivers/panfrost/pan_nir_lower_sysvals.c index a2bc499d67f..50e3f6b00eb 100644 --- a/src/gallium/drivers/panfrost/pan_nir_lower_sysvals.c +++ b/src/gallium/drivers/panfrost/pan_nir_lower_sysvals.c @@ -58,6 +58,7 @@ sysval_for_intrinsic(nir_intrinsic_instr *intr, unsigned *offset) { switch (intr->intrinsic) { case nir_intrinsic_load_ssbo_address: + assert(nir_src_as_uint(intr->src[1]) == 0); return PAN_SYSVAL(SSBO, nir_src_as_uint(intr->src[0])); case nir_intrinsic_get_ssbo_size: *offset = 8; diff --git a/src/gallium/drivers/radeonsi/si_nir_lower_resource.c b/src/gallium/drivers/radeonsi/si_nir_lower_resource.c index 1bb84e278ac..f8b5d3b0326 100644 --- a/src/gallium/drivers/radeonsi/si_nir_lower_resource.c +++ b/src/gallium/drivers/radeonsi/si_nir_lower_resource.c @@ -299,6 +299,7 @@ static bool lower_resource_intrinsic(nir_builder *b, nir_intrinsic_instr *intrin break; } case nir_intrinsic_load_ssbo_address: { + assert(nir_src_as_uint(intrin->src[1]) == 0); nir_def *desc = load_ssbo_desc(b, &intrin->src[0], s); nir_def *lo = nir_channel(b, desc, 0); nir_def *hi = nir_i2i32(b, nir_u2u16(b, nir_channel(b, desc, 1)));