From 7d0870e5d538a598c481a76b3bb964818d0e72bb Mon Sep 17 00:00:00 2001 From: Amber Date: Sat, 4 May 2024 18:15:45 +0200 Subject: [PATCH] ir3: add support for 64 bit atomics Signed-off-by: Amber Harmonia Part-of: --- src/freedreno/common/freedreno_dev_info.h | 2 + src/freedreno/common/freedreno_devices.py | 3 ++ src/freedreno/ir3/instr-a3xx.h | 2 + src/freedreno/ir3/ir3_a6xx.c | 48 +++++++++++++++++--- src/freedreno/ir3/ir3_compiler_nir.c | 29 ++++++++++-- src/freedreno/ir3/ir3_nir_lower_io_offsets.c | 5 ++ src/freedreno/ir3/ir3_validate.c | 32 +++++++++++++ 7 files changed, 110 insertions(+), 11 deletions(-) diff --git a/src/freedreno/common/freedreno_dev_info.h b/src/freedreno/common/freedreno_dev_info.h index d41728878ba..4d73d974612 100644 --- a/src/freedreno/common/freedreno_dev_info.h +++ b/src/freedreno/common/freedreno_dev_info.h @@ -225,6 +225,8 @@ struct fd_dev_info { /* Whether there is CP_EVENT_WRITE7::WRITE_SAMPLE_COUNT */ bool has_event_write_sample_count; + bool has_64b_ssbo_atomics; + /* Blob executes a special compute dispatch at the start of each * command buffers. We copy this dispatch as is. */ diff --git a/src/freedreno/common/freedreno_devices.py b/src/freedreno/common/freedreno_devices.py index 28ad880f789..809751a6967 100644 --- a/src/freedreno/common/freedreno_devices.py +++ b/src/freedreno/common/freedreno_devices.py @@ -893,6 +893,7 @@ a7xx_740 = A7XXProps( # Most devices with a740 have blob v6xx which doesn't have # this hint set. Match them for better compatibility by default. enable_tp_ubwc_flag_hint = False, + has_64b_ssbo_atomics = True, ) a7xx_740_a32 = A7XXProps( @@ -903,6 +904,7 @@ a7xx_740_a32 = A7XXProps( supports_ibo_ubwc = True, fs_must_have_non_zero_constlen_quirk = True, enable_tp_ubwc_flag_hint = False, + has_64b_ssbo_atomics = True, ) a7xx_740v3 = A7XXProps( @@ -931,6 +933,7 @@ a7xx_750 = A7XXProps( has_compliant_dp4acc = True, ubwc_coherency_quirk = True, has_persistent_counter = True, + has_64b_ssbo_atomics = True, ) a730_magic_regs = dict( diff --git a/src/freedreno/ir3/instr-a3xx.h b/src/freedreno/ir3/instr-a3xx.h index e9a7e54a330..40e7db6a892 100644 --- a/src/freedreno/ir3/instr-a3xx.h +++ b/src/freedreno/ir3/instr-a3xx.h @@ -444,6 +444,8 @@ type_uint_size(unsigned bit_size) case 1: /* 1b bools are treated as normal half-regs */ case 16: return TYPE_U16; case 32: return TYPE_U32; + case 64: + return TYPE_U32; default: ir3_assert(0); /* invalid size */ return (type_t)0; diff --git a/src/freedreno/ir3/ir3_a6xx.c b/src/freedreno/ir3/ir3_a6xx.c index 44ab994e945..d1aace1af02 100644 --- a/src/freedreno/ir3/ir3_a6xx.c +++ b/src/freedreno/ir3/ir3_a6xx.c @@ -206,6 +206,9 @@ emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr) struct ir3_instruction *atomic, *ibo, *src0, *src1, *data, *dummy; nir_atomic_op op = nir_intrinsic_atomic_op(intr); type_t type = nir_atomic_op_type(op) == nir_type_int ? TYPE_S32 : TYPE_U32; + if (intr->def.bit_size == 64) { + type = TYPE_ATOMIC_U64; + } ibo = ir3_ssbo_to_ibo(ctx, intr->src[0]); @@ -230,10 +233,23 @@ emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr) if (op == nir_atomic_op_cmpxchg) { src0 = ir3_get_src(ctx, &intr->src[4])[0]; struct ir3_instruction *compare = ir3_get_src(ctx, &intr->src[3])[0]; - src1 = ir3_collect(b, dummy, compare, data); + if (intr->def.bit_size == 64) { + struct ir3_instruction *dummy2 = create_immed(b, 0); + struct ir3_instruction *compare2 = ir3_get_src(ctx, &intr->src[3])[1]; + struct ir3_instruction *data2 = ir3_get_src(ctx, &intr->src[2])[1]; + src1 = ir3_collect(b, dummy, dummy2, compare, compare2, data, data2); + } else { + src1 = ir3_collect(b, dummy, compare, data); + } } else { src0 = ir3_get_src(ctx, &intr->src[3])[0]; - src1 = ir3_collect(b, dummy, data); + if (intr->def.bit_size == 64) { + struct ir3_instruction *dummy2 = create_immed(b, 0); + struct ir3_instruction *data2 = ir3_get_src(ctx, &intr->src[2])[1]; + src1 = ir3_collect(b, dummy, dummy2, data, data2); + } else { + src1 = ir3_collect(b, dummy, data); + } } atomic = emit_atomic(b, op, ibo, src0, src1); @@ -250,10 +266,12 @@ emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr) atomic->dsts[0]->wrmask = src1->dsts[0]->wrmask; ir3_reg_tie(atomic->dsts[0], atomic->srcs[2]); ir3_handle_nonuniform(atomic, intr); - struct ir3_instruction *split; - ir3_split_dest(b, &split, atomic, 0, 1); - return split; -} + + size_t num_results = intr->def.bit_size == 64 ? 2 : 1; + struct ir3_instruction *defs[num_results]; + ir3_split_dest(b, defs, atomic, 0, num_results); + return ir3_create_collect(b, defs, num_results); + } /* src[] = { deref, coord, sample_index }. const_index[] = {} */ static void @@ -482,6 +500,9 @@ emit_intrinsic_atomic_global(struct ir3_context *ctx, nir_intrinsic_instr *intr) struct ir3_instruction *value = ir3_get_src(ctx, &intr->src[1])[0]; nir_atomic_op op = nir_intrinsic_atomic_op(intr); type_t type = nir_atomic_op_type(op) == nir_type_int ? TYPE_S32 : TYPE_U32; + if (intr->def.bit_size == 64) { + type = TYPE_ATOMIC_U64; + } addr = ir3_collect(b, ir3_get_src(ctx, &intr->src[0])[0], ir3_get_src(ctx, &intr->src[0])[1]); @@ -489,8 +510,20 @@ emit_intrinsic_atomic_global(struct ir3_context *ctx, nir_intrinsic_instr *intr) if (op == nir_atomic_op_cmpxchg) { struct ir3_instruction *compare = ir3_get_src(ctx, &intr->src[2])[0]; src1 = ir3_collect(b, compare, value); + if (intr->def.bit_size == 64) { + struct ir3_instruction *compare2 = ir3_get_src(ctx, &intr->src[2])[1]; + struct ir3_instruction *value2 = ir3_get_src(ctx, &intr->src[1])[1]; + src1 = ir3_collect(b, compare, compare2, value, value2); + } else { + src1 = ir3_collect(b, compare, value); + } } else { - src1 = value; + if (intr->def.bit_size == 64) { + struct ir3_instruction *value2 = ir3_get_src(ctx, &intr->src[1])[1]; + src1 = ir3_collect(b, value, value2); + } else { + src1 = value; + } } switch (op) { @@ -535,6 +568,7 @@ emit_intrinsic_atomic_global(struct ir3_context *ctx, nir_intrinsic_instr *intr) atomic->cat6.type = type; atomic->barrier_class = IR3_BARRIER_BUFFER_W; atomic->barrier_conflict = IR3_BARRIER_BUFFER_R | IR3_BARRIER_BUFFER_W; + atomic->dsts[0]->wrmask = MASK(intr->def.bit_size == 64 ? 2 : 1); /* even if nothing consume the result, we can't DCE the instruction: */ array_insert(b, b->keeps, atomic); diff --git a/src/freedreno/ir3/ir3_compiler_nir.c b/src/freedreno/ir3/ir3_compiler_nir.c index c7357114d5f..fbb60d64522 100644 --- a/src/freedreno/ir3/ir3_compiler_nir.c +++ b/src/freedreno/ir3/ir3_compiler_nir.c @@ -1027,7 +1027,22 @@ emit_alu(struct ir3_context *ctx, nir_alu_instr *alu) dst = ir3_SUB_S_rpt(b, dst_sz, src[0], 0, src[1], 0); set_instr_flags(dst.rpts, dst_sz, IR3_INSTR_SAT); break; - + case nir_op_pack_64_2x32_split: { + struct ir3_instruction *r0 = ir3_MOV(b, src[0].rpts[0], TYPE_U32); + struct ir3_instruction *r1 = ir3_MOV(b, src[1].rpts[0], TYPE_U32); + dst.rpts[0] = r0; + dst.rpts[1] = r1; + dst_sz = 2; + break; + } + case nir_op_unpack_64_2x32_split_x: { + ir3_split_dest(b, &dst.rpts[0], src[0].rpts[0], 0, 1); + break; + } + case nir_op_unpack_64_2x32_split_y: { + ir3_split_dest(b, &dst.rpts[0], src[0].rpts[0], 1, 1); + break; + } case nir_op_udot_4x8_uadd: case nir_op_udot_4x8_uadd_sat: case nir_op_sdot_4x8_iadd: @@ -3311,9 +3326,9 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr) static void emit_load_const(struct ir3_context *ctx, nir_load_const_instr *instr) { - struct ir3_instruction **dst = - ir3_get_dst_ssa(ctx, &instr->def, instr->def.num_components); unsigned bit_size = ir3_bitsize(ctx, instr->def.bit_size); + struct ir3_instruction **dst = + ir3_get_dst_ssa(ctx, &instr->def, instr->def.num_components * ((bit_size == 64) ? 2 : 1)); if (bit_size <= 8) { for (int i = 0; i < instr->def.num_components; i++) @@ -3321,9 +3336,15 @@ emit_load_const(struct ir3_context *ctx, nir_load_const_instr *instr) } else if (bit_size <= 16) { for (int i = 0; i < instr->def.num_components; i++) dst[i] = create_immed_typed(ctx->block, instr->value[i].u16, TYPE_U16); - } else { + } else if (bit_size <= 32) { for (int i = 0; i < instr->def.num_components; i++) dst[i] = create_immed_typed(ctx->block, instr->value[i].u32, TYPE_U32); + } else { + assert(instr->def.num_components == 1); + for (int i = 0; i < instr->def.num_components; i++) { + dst[2 * i] = create_immed_typed(ctx->block, (uint32_t)(instr->value[i].u64), TYPE_U32); + dst[2 * i + 1] = create_immed_typed(ctx->block, (uint32_t)(instr->value[i].u64 >> 32), TYPE_U32); + } } } diff --git a/src/freedreno/ir3/ir3_nir_lower_io_offsets.c b/src/freedreno/ir3/ir3_nir_lower_io_offsets.c index b65ae333546..2eb2608518b 100644 --- a/src/freedreno/ir3/ir3_nir_lower_io_offsets.c +++ b/src/freedreno/ir3/ir3_nir_lower_io_offsets.c @@ -174,6 +174,11 @@ lower_offset_for_ssbo(nir_intrinsic_instr *intrinsic, nir_builder *b, (!has_dest && intrinsic->src[0].ssa->bit_size == 8)) shift = 0; + if ((has_dest && intrinsic->def.bit_size == 64) || + (!has_dest && intrinsic->src[0].ssa->bit_size == 64)) { + shift = 1; + } + /* Here we create a new intrinsic and copy over all contents from the old * one. */ diff --git a/src/freedreno/ir3/ir3_validate.c b/src/freedreno/ir3/ir3_validate.c index 78f526366b8..3041e46f3f8 100644 --- a/src/freedreno/ir3/ir3_validate.c +++ b/src/freedreno/ir3/ir3_validate.c @@ -7,6 +7,7 @@ #include "util/ralloc.h" +#include "instr-a3xx.h" #include "ir3.h" #include "ir3_compiler.h" @@ -442,6 +443,37 @@ validate_instr(struct ir3_validate_ctx *ctx, struct ir3_instruction *instr) validate_assert(ctx, !(instr->srcs[1]->flags & IR3_REG_HALF)); validate_assert(ctx, !(instr->srcs[2]->flags & IR3_REG_HALF)); validate_reg_size(ctx, instr->dsts[0], instr->cat6.type); + break; + case OPC_ATOMIC_B_CMPXCHG: + if (instr->cat6.type == TYPE_ATOMIC_U64) { + validate_assert(ctx, !(instr->dsts[0]->flags & IR3_REG_HALF)); + validate_assert(ctx, instr->dsts[0]->wrmask == 0x3f); + } else { + validate_reg_size(ctx, instr->dsts[0], instr->cat6.type); + } + validate_assert(ctx, !(instr->srcs[0]->flags & IR3_REG_HALF)); + validate_assert(ctx, !(instr->srcs[1]->flags & IR3_REG_HALF)); + break; + case OPC_ATOMIC_B_XCHG: + if (instr->cat6.type == TYPE_ATOMIC_U64) { + validate_assert(ctx, !(instr->dsts[0]->flags & IR3_REG_HALF)); + validate_assert(ctx, instr->dsts[0]->wrmask == 0xf); + } else { + validate_reg_size(ctx, instr->dsts[0], instr->cat6.type); + } + validate_assert(ctx, !(instr->srcs[0]->flags & IR3_REG_HALF)); + validate_assert(ctx, !(instr->srcs[1]->flags & IR3_REG_HALF)); + break; + case OPC_ATOMIC_G_CMPXCHG: + case OPC_ATOMIC_G_XCHG: + if (instr->cat6.type == TYPE_ATOMIC_U64) { + validate_assert(ctx, !(instr->dsts[0]->flags & IR3_REG_HALF)); + validate_assert(ctx, instr->dsts[0]->wrmask == 0x3); + } else { + validate_reg_size(ctx, instr->dsts[0], instr->cat6.type); + } + validate_assert(ctx, !(instr->srcs[0]->flags & IR3_REG_HALF)); + validate_assert(ctx, !(instr->srcs[1]->flags & IR3_REG_HALF)); break; case OPC_SHFL: validate_reg_size(ctx, instr->srcs[0], instr->cat6.type);