From c88c66754cc551bceae199768d261909668cb619 Mon Sep 17 00:00:00 2001 From: Ashley Smith Date: Fri, 11 Jul 2025 09:49:47 +0100 Subject: [PATCH] pan/va: Add support for 64-bit atomic operations Adds support for 64-bit atomic operations for KHR_shader_atomic_int64 using 64-bit atomic instructions. Valhall is working but Bifrost will require some more work to implement as it requires two instructions to execute a 64-bit atomic. Reviewed-by: Mary Guillemard Reviewed-by: Eric R. Smith Signed-off-by: Ashley Smith Part-of: --- src/panfrost/compiler/bi_ra.c | 3 +- src/panfrost/compiler/bifrost/bi_schedule.c | 46 +++++++ src/panfrost/compiler/bifrost_compile.c | 112 ++++++++++++++---- src/panfrost/compiler/bir.c | 12 ++ src/panfrost/compiler/valhall/va_lower_isel.c | 13 ++ src/panfrost/compiler/valhall/va_pack.c | 33 ++++++ 6 files changed, 193 insertions(+), 26 deletions(-) diff --git a/src/panfrost/compiler/bi_ra.c b/src/panfrost/compiler/bi_ra.c index f9b99272419..0f0cd61f3f7 100644 --- a/src/panfrost/compiler/bi_ra.c +++ b/src/panfrost/compiler/bi_ra.c @@ -854,7 +854,8 @@ bi_is_tied(const bi_instr *I) { return (I->op == BI_OPCODE_TEXC || I->op == BI_OPCODE_TEXC_DUAL || I->op == BI_OPCODE_ATOM_RETURN_I32 || I->op == BI_OPCODE_AXCHG_I32 || - I->op == BI_OPCODE_ACMPXCHG_I32) && + I->op == BI_OPCODE_ACMPXCHG_I32 || I->op == BI_OPCODE_AXCHG_I64 || + I->op == BI_OPCODE_ACMPXCHG_I64) && !bi_is_null(I->src[0]); } diff --git a/src/panfrost/compiler/bifrost/bi_schedule.c b/src/panfrost/compiler/bifrost/bi_schedule.c index c583e27630a..851cfef0699 100644 --- a/src/panfrost/compiler/bifrost/bi_schedule.c +++ b/src/panfrost/compiler/bifrost/bi_schedule.c @@ -396,6 +396,48 @@ bi_lower_atom_c1(bi_context *ctx, struct bi_clause_state *clause, return atom_c; } +static bi_instr * +bi_lower_atom_c_64(bi_context *ctx, struct bi_clause_state *clause, + struct bi_tuple_state *tuple) +{ + bi_instr *pinstr = tuple->add; + bi_builder b = bi_init_builder(ctx, bi_before_instr(pinstr)); + bi_instr *atom_c = bi_atom_c_return_i64(&b, pinstr->src[1], pinstr->src[2], + pinstr->src[0], pinstr->atom_opc); + + if (bi_is_null(pinstr->dest[0])) + bi_set_opcode(atom_c, BI_OPCODE_ATOM_C_I64); + + bi_instr *atom_cx = + bi_atom_cx_to(&b, pinstr->dest[0], pinstr->src[0], pinstr->src[1], + pinstr->src[2], pinstr->src[0], pinstr->sr_count); + tuple->add = atom_cx; + bi_remove_instruction(pinstr); + + return atom_c; +} + +static bi_instr * +bi_lower_atom_c1_64(bi_context *ctx, struct bi_clause_state *clause, + struct bi_tuple_state *tuple) +{ + bi_instr *pinstr = tuple->add; + bi_builder b = bi_init_builder(ctx, bi_before_instr(pinstr)); + bi_instr *atom_c = bi_atom_c1_return_i64(&b, pinstr->src[0], pinstr->src[1], + pinstr->atom_opc); + + if (bi_is_null(pinstr->dest[0])) + bi_set_opcode(atom_c, BI_OPCODE_ATOM_C1_I64); + + bi_instr *atom_cx = + bi_atom_cx_to(&b, pinstr->dest[0], bi_null(), pinstr->src[0], + pinstr->src[1], bi_dontcare(&b), pinstr->sr_count); + tuple->add = atom_cx; + bi_remove_instruction(pinstr); + + return atom_c; +} + static bi_instr * bi_lower_seg_add(bi_context *ctx, struct bi_clause_state *clause, struct bi_tuple_state *tuple) @@ -1289,6 +1331,10 @@ bi_take_instr(bi_context *ctx, struct bi_worklist st, return bi_lower_atom_c(ctx, clause, tuple); else if (tuple->add && tuple->add->op == BI_OPCODE_ATOM1_RETURN_I32) return bi_lower_atom_c1(ctx, clause, tuple); + else if (tuple->add && tuple->add->op == BI_OPCODE_ATOM_RETURN_I64) + return bi_lower_atom_c_64(ctx, clause, tuple); + else if (tuple->add && tuple->add->op == BI_OPCODE_ATOM1_RETURN_I64) + return bi_lower_atom_c1_64(ctx, clause, tuple); else if (tuple->add && tuple->add->op == BI_OPCODE_SEG_ADD_I64) return bi_lower_seg_add(ctx, clause, tuple); else if (tuple->add && tuple->add->table) diff --git a/src/panfrost/compiler/bifrost_compile.c b/src/panfrost/compiler/bifrost_compile.c index fefbbe526c6..fc8503d72ec 100644 --- a/src/panfrost/compiler/bifrost_compile.c +++ b/src/panfrost/compiler/bifrost_compile.c @@ -1597,6 +1597,8 @@ bi_atom_opc_for_nir(nir_atomic_op op) case nir_atomic_op_iand: return BI_ATOM_OPC_AAND; case nir_atomic_op_ior: return BI_ATOM_OPC_AOR; case nir_atomic_op_ixor: return BI_ATOM_OPC_AXOR; + case nir_atomic_op_xchg: return BI_ATOM_OPC_AXCHG; + case nir_atomic_op_cmpxchg: return BI_ATOM_OPC_AXCHG; default: unreachable("Unexpected computational atomic"); } /* clang-format on */ @@ -1829,6 +1831,36 @@ bi_emit_image_store(bi_builder *b, nir_intrinsic_instr *instr) BI_REGISTER_FORMAT_AUTO, instr->num_components - 1); } +static void +bi_emit_atomic_i64_to(bi_builder *b, bi_index dst, bi_index addr, bi_index arg, + nir_atomic_op op) +{ + enum bi_atom_opc opc = bi_atom_opc_for_nir(op); + enum bi_atom_opc post_opc = opc; + bool bifrost = b->shader->arch <= 8; + + /* ATOM_C.i64 takes a vector with {arg, coalesced}, ATOM_C1.i64 doesn't + * take any vector but can still output in RETURN mode */ + bi_index tmp_dest = bifrost ? bi_temp(b->shader) : dst; + unsigned sr_count = bifrost ? 4 : 2; + + /* Generate either ATOM or ATOM1 as required */ + if (bi_promote_atom_c1(opc, arg, &opc)) { + bi_atom1_return_i64_to(b, tmp_dest, bi_extract(b, addr, 0), + bi_extract(b, addr, 1), opc, sr_count); + } else { + bi_atom_return_i64_to(b, tmp_dest, arg, bi_extract(b, addr, 0), + bi_extract(b, addr, 1), opc, sr_count); + } + + if (bifrost) { + /* Post-process it */ + bi_emit_cached_split(b, tmp_dest, 64 * 2); + bi_atom_post_i64_to(b, dst, bi_extract(b, tmp_dest, 0), + bi_extract(b, tmp_dest, 2), post_opc); + bi_emit_cached_split(b, dst, 64); + } +} static void bi_emit_atomic_i32_to(bi_builder *b, bi_index dst, bi_index addr, bi_index arg, nir_atomic_op op) @@ -2121,24 +2153,32 @@ bi_emit_intrinsic(bi_builder *b, nir_intrinsic_instr *instr) case nir_intrinsic_shared_atomic: { nir_atomic_op op = nir_intrinsic_atomic_op(instr); - if (op == nir_atomic_op_xchg) { - bi_emit_axchg_to(b, dst, bi_src_index(&instr->src[0]), &instr->src[1], - BI_SEG_WLS); + bi_index addr = bi_src_index(&instr->src[0]); + bi_index addr_hi; + + if (b->shader->arch >= 9) { + bi_handle_segment(b, &addr, &addr_hi, BI_SEG_WLS, NULL); + addr = bi_collect_v2i32(b, addr, addr_hi); + + if (nir_src_bit_size(instr->src[1]) == 32) { + bi_emit_atomic_i32_to(b, dst, addr, bi_src_index(&instr->src[1]), op); + } else { + bi_emit_atomic_i64_to(b, dst, addr, bi_src_index(&instr->src[1]), op); + } } else { - assert(nir_src_bit_size(instr->src[1]) == 32); - - bi_index addr = bi_src_index(&instr->src[0]); - bi_index addr_hi; - - if (b->shader->arch >= 9) { - bi_handle_segment(b, &addr, &addr_hi, BI_SEG_WLS, NULL); - addr = bi_collect_v2i32(b, addr, addr_hi); + if (op == nir_atomic_op_xchg) { + bi_emit_axchg_to(b, dst, addr, &instr->src[1], + BI_SEG_WLS); } else { addr = bi_seg_add_i64(b, addr, bi_zero(), false, BI_SEG_WLS); bi_emit_cached_split(b, addr, 64); - } - bi_emit_atomic_i32_to(b, dst, addr, bi_src_index(&instr->src[1]), op); + if (nir_src_bit_size(instr->src[1]) == 32) { + bi_emit_atomic_i32_to(b, dst, addr, bi_src_index(&instr->src[1]), op); + } else { + bi_emit_atomic_i64_to(b, dst, addr, bi_src_index(&instr->src[1]), op); + } + } } bi_split_def(b, &instr->def); @@ -2148,14 +2188,27 @@ bi_emit_intrinsic(bi_builder *b, nir_intrinsic_instr *instr) case nir_intrinsic_global_atomic: { nir_atomic_op op = nir_intrinsic_atomic_op(instr); - if (op == nir_atomic_op_xchg) { - bi_emit_axchg_to(b, dst, bi_src_index(&instr->src[0]), &instr->src[1], - BI_SEG_NONE); + if (b->shader->arch >= 9) { + if (nir_src_bit_size(instr->src[1]) == 32) { + bi_emit_atomic_i32_to(b, dst, bi_src_index(&instr->src[0]), + bi_src_index(&instr->src[1]), op); + } else { + bi_emit_atomic_i64_to(b, dst, bi_src_index(&instr->src[0]), + bi_src_index(&instr->src[1]), op); + } } else { - assert(nir_src_bit_size(instr->src[1]) == 32); - bi_emit_atomic_i32_to(b, dst, bi_src_index(&instr->src[0]), - bi_src_index(&instr->src[1]), op); + if (op == nir_atomic_op_xchg) { + bi_emit_axchg_to(b, dst, bi_src_index(&instr->src[0]), &instr->src[1], BI_SEG_NONE); + } else { + if (nir_src_bit_size(instr->src[1]) == 32) { + bi_emit_atomic_i32_to(b, dst, bi_src_index(&instr->src[0]), + bi_src_index(&instr->src[1]), op); + } else { + bi_emit_atomic_i64_to(b, dst, bi_src_index(&instr->src[0]), + bi_src_index(&instr->src[1]), op); + } + } } bi_split_def(b, &instr->def); @@ -2383,20 +2436,29 @@ bi_emit_intrinsic(bi_builder *b, nir_intrinsic_instr *instr) static void bi_emit_load_const(bi_builder *b, nir_load_const_instr *instr) { - /* Make sure we've been lowered */ - assert(instr->def.num_components <= (32 / instr->def.bit_size)); - /* Accumulate all the channels of the constant, as if we did an * implicit SEL over them */ - uint32_t acc = 0; + uint64_t acc = 0; for (unsigned i = 0; i < instr->def.num_components; ++i) { - unsigned v = + uint64_t v = nir_const_value_as_uint(instr->value[i], instr->def.bit_size); acc |= (v << (i * instr->def.bit_size)); } - bi_mov_i32_to(b, bi_get_index(instr->def.index), bi_imm_u32(acc)); + if (instr->def.bit_size <= 32) { + bi_mov_i32_to(b, bi_get_index(instr->def.index), bi_imm_u32(acc)); + } else { + uint32_t imm_2x32[2] = { acc & 0xffffffff, (acc >> 32) & 0xffffffff }; + bi_index tempa = bi_temp(b->shader); + bi_index tempb = bi_temp(b->shader); + bi_mov_i32_to(b, tempa, bi_imm_u32(imm_2x32[0])); + bi_mov_i32_to(b, tempb, bi_imm_u32(imm_2x32[1])); + + bi_instr *collect = bi_collect_i32_to(b, bi_get_index(instr->def.index), 2); + collect->src[0] = tempa; + collect->src[1] = tempb; + } } static bi_index diff --git a/src/panfrost/compiler/bir.c b/src/panfrost/compiler/bir.c index 245d480aa2c..00b03955b08 100644 --- a/src/panfrost/compiler/bir.c +++ b/src/panfrost/compiler/bir.c @@ -90,6 +90,8 @@ bi_count_read_registers(const bi_instr *ins, unsigned s) /* ATOM reads 1 but writes 2. Exception for ACMPXCHG */ if (s == 0 && ins->op == BI_OPCODE_ATOM_RETURN_I32) return (ins->atom_opc == BI_ATOM_OPC_ACMPXCHG) ? 2 : 1; + else if (s == 0 && ins->op == BI_OPCODE_ATOM_RETURN_I64) + return (ins->atom_opc == BI_ATOM_OPC_ACMPXCHG) ? 4 : 2; else if (s == 0 && bi_get_opcode_props(ins)->sr_read) return bi_count_staging_registers(ins); else if (s == 4 && ins->op == BI_OPCODE_BLEND) @@ -125,8 +127,18 @@ bi_count_write_registers(const bi_instr *ins, unsigned d) case BI_OPCODE_ACMPXCHG_I32: /* Reads 2 but writes 1 */ return 1; + case BI_OPCODE_ACMPXCHG_I64: + /* Reads 4 but writes 2 */ + return 2; + case BI_OPCODE_ATOM_POST_I32: + /* Reads 2 but writes 1 */ + return 1; + case BI_OPCODE_ATOM_POST_I64: + /* Reads 4 but writes 2 */ + return 2; case BI_OPCODE_ATOM1_RETURN_I32: + case BI_OPCODE_ATOM1_RETURN_I64: /* Allow omitting the destination for plain ATOM1 */ return bi_is_null(ins->dest[0]) ? 0 : ins->sr_count; default: diff --git a/src/panfrost/compiler/valhall/va_lower_isel.c b/src/panfrost/compiler/valhall/va_lower_isel.c index d69b37f6e10..8e464510bdc 100644 --- a/src/panfrost/compiler/valhall/va_lower_isel.c +++ b/src/panfrost/compiler/valhall/va_lower_isel.c @@ -110,12 +110,25 @@ lower(bi_builder *b, bi_instr *I) return bi_branchzi(b, bi_zero(), I->src[0], BI_CMPF_EQ); } + case BI_OPCODE_AXCHG_I64: + bi_set_opcode(I, BI_OPCODE_ATOM_RETURN_I64); + I->atom_opc = BI_ATOM_OPC_AXCHG; + I->sr_count = 2; + return NULL; + case BI_OPCODE_AXCHG_I32: bi_set_opcode(I, BI_OPCODE_ATOM_RETURN_I32); I->atom_opc = BI_ATOM_OPC_AXCHG; I->sr_count = 1; return NULL; + case BI_OPCODE_ACMPXCHG_I64: + bi_set_opcode(I, BI_OPCODE_ATOM_RETURN_I64); + I->atom_opc = BI_ATOM_OPC_ACMPXCHG; + /* Reads 4, this is special cased in bir.c */ + I->sr_count = 2; + return NULL; + case BI_OPCODE_ACMPXCHG_I32: bi_set_opcode(I, BI_OPCODE_ATOM_RETURN_I32); I->atom_opc = BI_ATOM_OPC_ACMPXCHG; diff --git a/src/panfrost/compiler/valhall/va_pack.c b/src/panfrost/compiler/valhall/va_pack.c index 2db72e6e637..b177403dfe7 100644 --- a/src/panfrost/compiler/valhall/va_pack.c +++ b/src/panfrost/compiler/valhall/va_pack.c @@ -909,6 +909,37 @@ va_pack_instr(const bi_instr *I, unsigned arch) hex |= va_pack_store(I); break; + case BI_OPCODE_ATOM1_RETURN_I64: + /* Permit omitting the destination for plain ATOM1 */ + if (!bi_count_write_registers(I, 0)) { + hex |= (0x40ull << 40); // fake read + } + + /* 64-bit source */ + va_validate_register_pair(I, 0); + hex |= (uint64_t)va_pack_src(I, 0) << 0; + hex |= va_pack_byte_offset_8(I); + hex |= ((uint64_t)va_pack_atom_opc_1(I)) << 22; + break; + + case BI_OPCODE_ACMPXCHG_I64: + case BI_OPCODE_AXCHG_I64: + case BI_OPCODE_ATOM_I64: + case BI_OPCODE_ATOM_RETURN_I64: + /* 64-bit source */ + va_validate_register_pair(I, 1); + hex |= (uint64_t)va_pack_src(I, 1) << 0; + hex |= va_pack_byte_offset_8(I); + hex |= ((uint64_t)va_pack_atom_opc(I)) << 22; + + if (I->op == BI_OPCODE_ATOM_RETURN_I64) + hex |= (0xc0ull << 40); // flags + + if (I->atom_opc == BI_ATOM_OPC_ACMPXCHG) + hex |= (1 << 26); /* .compare */ + + break; + case BI_OPCODE_ATOM1_RETURN_I32: /* Permit omitting the destination for plain ATOM1 */ if (!bi_count_write_registers(I, 0)) { @@ -922,6 +953,8 @@ va_pack_instr(const bi_instr *I, unsigned arch) hex |= ((uint64_t)va_pack_atom_opc_1(I)) << 22; break; + case BI_OPCODE_ACMPXCHG_I32: + case BI_OPCODE_AXCHG_I32: case BI_OPCODE_ATOM_I32: case BI_OPCODE_ATOM_RETURN_I32: /* 64-bit source */