From 9cf232c9a8296a589f76707e81f500accd6ff93f Mon Sep 17 00:00:00 2001 From: Emma Anholt Date: Thu, 2 Sep 2021 12:14:20 -0700 Subject: [PATCH] freedreno/ir3: Clarify what's going on in a4xx SSBO atomics. Part-of: --- src/freedreno/ir3/ir3_a4xx.c | 57 ++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/src/freedreno/ir3/ir3_a4xx.c b/src/freedreno/ir3/ir3_a4xx.c index ea93285bc76..820a692f091 100644 --- a/src/freedreno/ir3/ir3_a4xx.c +++ b/src/freedreno/ir3/ir3_a4xx.c @@ -105,70 +105,69 @@ emit_intrinsic_store_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr) * sources represent: * * 0: The SSBO buffer index. - * 1: The offset into the SSBO buffer of the variable that the atomic + * 1: The byte offset into the SSBO buffer of the variable that the atomic * operation will operate on. * 2: The data parameter to the atomic function (i.e. the value to add * in ssbo_atomic_add, etc). - * 3: For CompSwap only: the second data parameter. + * 3: CompSwap: the second data parameter. + * Non-CompSwap: The dword offset into the SSBO buffer variable. + * 4: CompSwap: The dword offset into the SSBO buffer variable. + * + * We use custom ssbo_*_ir3 intrinsics generated by ir3_nir_lower_io_offsets() + * so we can have the dword offset generated in NIR. */ static struct ir3_instruction * emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr) { struct ir3_block *b = ctx->block; - struct ir3_instruction *atomic, *ssbo, *src0, *src1, *src2, *byte_offset, - *offset; + struct ir3_instruction *atomic; type_t type = TYPE_U32; - ssbo = ir3_ssbo_to_ibo(ctx, intr->src[0]); + struct ir3_instruction *ssbo = ir3_ssbo_to_ibo(ctx, intr->src[0]); - byte_offset = ir3_get_src(ctx, &intr->src[1])[0]; - offset = ir3_get_src(ctx, &intr->src[3])[0]; - - /* src0 is data (or uvec2(data, compare)) - * src1 is offset - * src2 is uvec2(offset*4, 0) (appears to be 64b byte offset) - * - * Note that nir already multiplies the offset by four - */ - src0 = ir3_get_src(ctx, &intr->src[2])[0]; - src1 = offset; - src2 = ir3_collect(ctx, byte_offset, create_immed(b, 0)); + struct ir3_instruction *data = ir3_get_src(ctx, &intr->src[2])[0]; + /* 64b byte offset */ + struct ir3_instruction *byte_offset = + ir3_collect(ctx, ir3_get_src(ctx, &intr->src[1])[0], create_immed(b, 0)); + /* dword offset for everything but comp_swap */ + struct ir3_instruction *src3 = ir3_get_src(ctx, &intr->src[3])[0]; switch (intr->intrinsic) { case nir_intrinsic_ssbo_atomic_add_ir3: - atomic = ir3_ATOMIC_ADD_G(b, ssbo, 0, src0, 0, src1, 0, src2, 0); + atomic = ir3_ATOMIC_ADD_G(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0); break; case nir_intrinsic_ssbo_atomic_imin_ir3: - atomic = ir3_ATOMIC_MIN_G(b, ssbo, 0, src0, 0, src1, 0, src2, 0); + atomic = ir3_ATOMIC_MIN_G(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0); type = TYPE_S32; break; case nir_intrinsic_ssbo_atomic_umin_ir3: - atomic = ir3_ATOMIC_MIN_G(b, ssbo, 0, src0, 0, src1, 0, src2, 0); + atomic = ir3_ATOMIC_MIN_G(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0); break; case nir_intrinsic_ssbo_atomic_imax_ir3: - atomic = ir3_ATOMIC_MAX_G(b, ssbo, 0, src0, 0, src1, 0, src2, 0); + atomic = ir3_ATOMIC_MAX_G(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0); type = TYPE_S32; break; case nir_intrinsic_ssbo_atomic_umax_ir3: - atomic = ir3_ATOMIC_MAX_G(b, ssbo, 0, src0, 0, src1, 0, src2, 0); + atomic = ir3_ATOMIC_MAX_G(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0); break; case nir_intrinsic_ssbo_atomic_and_ir3: - atomic = ir3_ATOMIC_AND_G(b, ssbo, 0, src0, 0, src1, 0, src2, 0); + atomic = ir3_ATOMIC_AND_G(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0); break; case nir_intrinsic_ssbo_atomic_or_ir3: - atomic = ir3_ATOMIC_OR_G(b, ssbo, 0, src0, 0, src1, 0, src2, 0); + atomic = ir3_ATOMIC_OR_G(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0); break; case nir_intrinsic_ssbo_atomic_xor_ir3: - atomic = ir3_ATOMIC_XOR_G(b, ssbo, 0, src0, 0, src1, 0, src2, 0); + atomic = ir3_ATOMIC_XOR_G(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0); break; case nir_intrinsic_ssbo_atomic_exchange_ir3: - atomic = ir3_ATOMIC_XCHG_G(b, ssbo, 0, src0, 0, src1, 0, src2, 0); + atomic = ir3_ATOMIC_XCHG_G(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0); break; case nir_intrinsic_ssbo_atomic_comp_swap_ir3: /* for cmpxchg, src0 is [ui]vec2(data, compare): */ - src0 = ir3_collect(ctx, ir3_get_src(ctx, &intr->src[3])[0], src0); - src1 = ir3_get_src(ctx, &intr->src[4])[0]; - atomic = ir3_ATOMIC_CMPXCHG_G(b, ssbo, 0, src0, 0, src1, 0, src2, 0); + data = ir3_collect(ctx, src3, data); + struct ir3_instruction *dword_offset = ir3_get_src(ctx, &intr->src[4])[0]; + atomic = ir3_ATOMIC_CMPXCHG_G(b, ssbo, 0, data, 0, dword_offset, 0, + byte_offset, 0); break; default: unreachable("boo");