AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pan/va: Add support for 64-bit atomic operations

Browse Source 
Adds support for 64-bit atomic operations for KHR_shader_atomic_int64
using 64-bit atomic instructions. Valhall is working but Bifrost will
require some more work to implement as it requires two instructions to
execute a 64-bit atomic.

Reviewed-by: Mary Guillemard <mary.guillemard@collabora.com>
Reviewed-by: Eric R. Smith <eric.smith@collabora.com>
Signed-off-by: Ashley Smith <ashley.smith@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/35789>
This commit is contained in:
Ashley Smith
2025-07-11 09:49:47 +01:00
committed by Marge Bot
parent c3a21fb0af
commit c88c66754c
6 changed files with 193 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/panfrost/compiler/bi_ra.c
+2 -1
View File
@@ -854,7 +854,8 @@ bi_is_tied(const bi_instr *I)
{
return (I->op == BI_OPCODE_TEXC || I->op == BI_OPCODE_TEXC_DUAL ||
I->op == BI_OPCODE_ATOM_RETURN_I32 || I->op == BI_OPCODE_AXCHG_I32 ||
I->op == BI_OPCODE_ACMPXCHG_I32) &&
I->op == BI_OPCODE_ACMPXCHG_I32 || I->op == BI_OPCODE_AXCHG_I64 ||
I->op == BI_OPCODE_ACMPXCHG_I64) &&
!bi_is_null(I->src[0]);
}
src/panfrost/compiler/bifrost/bi_schedule.c
+46
View File
@@ -396,6 +396,48 @@ bi_lower_atom_c1(bi_context *ctx, struct bi_clause_state *clause,
return atom_c;
}
static bi_instr *
bi_lower_atom_c_64(bi_context *ctx, struct bi_clause_state *clause,
struct bi_tuple_state *tuple)
{
bi_instr *pinstr = tuple->add;
bi_builder b = bi_init_builder(ctx, bi_before_instr(pinstr));
bi_instr *atom_c = bi_atom_c_return_i64(&b, pinstr->src[1], pinstr->src[2],
pinstr->src[0], pinstr->atom_opc);
if (bi_is_null(pinstr->dest[0]))
bi_set_opcode(atom_c, BI_OPCODE_ATOM_C_I64);
bi_instr *atom_cx =
bi_atom_cx_to(&b, pinstr->dest[0], pinstr->src[0], pinstr->src[1],
pinstr->src[2], pinstr->src[0], pinstr->sr_count);
tuple->add = atom_cx;
bi_remove_instruction(pinstr);
return atom_c;
}
static bi_instr *
bi_lower_atom_c1_64(bi_context *ctx, struct bi_clause_state *clause,
struct bi_tuple_state *tuple)
{
bi_instr *pinstr = tuple->add;
bi_builder b = bi_init_builder(ctx, bi_before_instr(pinstr));
bi_instr *atom_c = bi_atom_c1_return_i64(&b, pinstr->src[0], pinstr->src[1],
pinstr->atom_opc);
if (bi_is_null(pinstr->dest[0]))
bi_set_opcode(atom_c, BI_OPCODE_ATOM_C1_I64);
bi_instr *atom_cx =
bi_atom_cx_to(&b, pinstr->dest[0], bi_null(), pinstr->src[0],
pinstr->src[1], bi_dontcare(&b), pinstr->sr_count);
tuple->add = atom_cx;
bi_remove_instruction(pinstr);
return atom_c;
}
static bi_instr *
bi_lower_seg_add(bi_context *ctx, struct bi_clause_state *clause,
struct bi_tuple_state *tuple)
@@ -1289,6 +1331,10 @@ bi_take_instr(bi_context *ctx, struct bi_worklist st,
return bi_lower_atom_c(ctx, clause, tuple);
else if (tuple->add && tuple->add->op == BI_OPCODE_ATOM1_RETURN_I32)
return bi_lower_atom_c1(ctx, clause, tuple);
else if (tuple->add && tuple->add->op == BI_OPCODE_ATOM_RETURN_I64)
return bi_lower_atom_c_64(ctx, clause, tuple);
else if (tuple->add && tuple->add->op == BI_OPCODE_ATOM1_RETURN_I64)
return bi_lower_atom_c1_64(ctx, clause, tuple);
else if (tuple->add && tuple->add->op == BI_OPCODE_SEG_ADD_I64)
return bi_lower_seg_add(ctx, clause, tuple);
else if (tuple->add && tuple->add->table)
src/panfrost/compiler/bifrost_compile.c
+87 -25
View File
@@ -1597,6 +1597,8 @@ bi_atom_opc_for_nir(nir_atomic_op op)
case nir_atomic_op_iand: return BI_ATOM_OPC_AAND;
case nir_atomic_op_ior: return BI_ATOM_OPC_AOR;
case nir_atomic_op_ixor: return BI_ATOM_OPC_AXOR;
case nir_atomic_op_xchg: return BI_ATOM_OPC_AXCHG;
case nir_atomic_op_cmpxchg: return BI_ATOM_OPC_AXCHG;
default: unreachable("Unexpected computational atomic");
}
/* clang-format on */
@@ -1829,6 +1831,36 @@ bi_emit_image_store(bi_builder *b, nir_intrinsic_instr *instr)
BI_REGISTER_FORMAT_AUTO, instr->num_components - 1);
}
static void
bi_emit_atomic_i64_to(bi_builder *b, bi_index dst, bi_index addr, bi_index arg,
nir_atomic_op op)
{
enum bi_atom_opc opc = bi_atom_opc_for_nir(op);
enum bi_atom_opc post_opc = opc;
bool bifrost = b->shader->arch <= 8;
/* ATOM_C.i64 takes a vector with {arg, coalesced}, ATOM_C1.i64 doesn't
* take any vector but can still output in RETURN mode */
bi_index tmp_dest = bifrost ? bi_temp(b->shader) : dst;
unsigned sr_count = bifrost ? 4 : 2;
/* Generate either ATOM or ATOM1 as required */
if (bi_promote_atom_c1(opc, arg, &opc)) {
bi_atom1_return_i64_to(b, tmp_dest, bi_extract(b, addr, 0),
bi_extract(b, addr, 1), opc, sr_count);
} else {
bi_atom_return_i64_to(b, tmp_dest, arg, bi_extract(b, addr, 0),
bi_extract(b, addr, 1), opc, sr_count);
}
if (bifrost) {
/* Post-process it */
bi_emit_cached_split(b, tmp_dest, 64 * 2);
bi_atom_post_i64_to(b, dst, bi_extract(b, tmp_dest, 0),
bi_extract(b, tmp_dest, 2), post_opc);
bi_emit_cached_split(b, dst, 64);
}
}
static void
bi_emit_atomic_i32_to(bi_builder *b, bi_index dst, bi_index addr, bi_index arg,
nir_atomic_op op)
@@ -2121,24 +2153,32 @@ bi_emit_intrinsic(bi_builder *b, nir_intrinsic_instr *instr)
case nir_intrinsic_shared_atomic: {
nir_atomic_op op = nir_intrinsic_atomic_op(instr);
if (op == nir_atomic_op_xchg) {
bi_emit_axchg_to(b, dst, bi_src_index(&instr->src[0]), &instr->src[1],
BI_SEG_WLS);
bi_index addr = bi_src_index(&instr->src[0]);
bi_index addr_hi;
if (b->shader->arch >= 9) {
bi_handle_segment(b, &addr, &addr_hi, BI_SEG_WLS, NULL);
addr = bi_collect_v2i32(b, addr, addr_hi);
if (nir_src_bit_size(instr->src[1]) == 32) {
bi_emit_atomic_i32_to(b, dst, addr, bi_src_index(&instr->src[1]), op);
} else {
bi_emit_atomic_i64_to(b, dst, addr, bi_src_index(&instr->src[1]), op);
}
} else {
assert(nir_src_bit_size(instr->src[1]) == 32);
bi_index addr = bi_src_index(&instr->src[0]);
bi_index addr_hi;
if (b->shader->arch >= 9) {
bi_handle_segment(b, &addr, &addr_hi, BI_SEG_WLS, NULL);
addr = bi_collect_v2i32(b, addr, addr_hi);
if (op == nir_atomic_op_xchg) {
bi_emit_axchg_to(b, dst, addr, &instr->src[1],
BI_SEG_WLS);
} else {
addr = bi_seg_add_i64(b, addr, bi_zero(), false, BI_SEG_WLS);
bi_emit_cached_split(b, addr, 64);
}
bi_emit_atomic_i32_to(b, dst, addr, bi_src_index(&instr->src[1]), op);
if (nir_src_bit_size(instr->src[1]) == 32) {
bi_emit_atomic_i32_to(b, dst, addr, bi_src_index(&instr->src[1]), op);
} else {
bi_emit_atomic_i64_to(b, dst, addr, bi_src_index(&instr->src[1]), op);
}
}
}
bi_split_def(b, &instr->def);
@@ -2148,14 +2188,27 @@ bi_emit_intrinsic(bi_builder *b, nir_intrinsic_instr *instr)
case nir_intrinsic_global_atomic: {
nir_atomic_op op = nir_intrinsic_atomic_op(instr);
if (op == nir_atomic_op_xchg) {
bi_emit_axchg_to(b, dst, bi_src_index(&instr->src[0]), &instr->src[1],
BI_SEG_NONE);
if (b->shader->arch >= 9) {
if (nir_src_bit_size(instr->src[1]) == 32) {
bi_emit_atomic_i32_to(b, dst, bi_src_index(&instr->src[0]),
bi_src_index(&instr->src[1]), op);
} else {
bi_emit_atomic_i64_to(b, dst, bi_src_index(&instr->src[0]),
bi_src_index(&instr->src[1]), op);
}
} else {
assert(nir_src_bit_size(instr->src[1]) == 32);
bi_emit_atomic_i32_to(b, dst, bi_src_index(&instr->src[0]),
bi_src_index(&instr->src[1]), op);
if (op == nir_atomic_op_xchg) {
bi_emit_axchg_to(b, dst, bi_src_index(&instr->src[0]), &instr->src[1], BI_SEG_NONE);
} else {
if (nir_src_bit_size(instr->src[1]) == 32) {
bi_emit_atomic_i32_to(b, dst, bi_src_index(&instr->src[0]),
bi_src_index(&instr->src[1]), op);
} else {
bi_emit_atomic_i64_to(b, dst, bi_src_index(&instr->src[0]),
bi_src_index(&instr->src[1]), op);
}
}
}
bi_split_def(b, &instr->def);
@@ -2383,20 +2436,29 @@ bi_emit_intrinsic(bi_builder *b, nir_intrinsic_instr *instr)
static void
bi_emit_load_const(bi_builder *b, nir_load_const_instr *instr)
{
/* Make sure we've been lowered */
assert(instr->def.num_components <= (32 / instr->def.bit_size));
/* Accumulate all the channels of the constant, as if we did an
* implicit SEL over them */
uint32_t acc = 0;
uint64_t acc = 0;
for (unsigned i = 0; i < instr->def.num_components; ++i) {
unsigned v =
uint64_t v =
nir_const_value_as_uint(instr->value[i], instr->def.bit_size);
acc |= (v << (i * instr->def.bit_size));
}
bi_mov_i32_to(b, bi_get_index(instr->def.index), bi_imm_u32(acc));
if (instr->def.bit_size <= 32) {
bi_mov_i32_to(b, bi_get_index(instr->def.index), bi_imm_u32(acc));
} else {
uint32_t imm_2x32[2] = { acc & 0xffffffff, (acc >> 32) & 0xffffffff };
bi_index tempa = bi_temp(b->shader);
bi_index tempb = bi_temp(b->shader);
bi_mov_i32_to(b, tempa, bi_imm_u32(imm_2x32[0]));
bi_mov_i32_to(b, tempb, bi_imm_u32(imm_2x32[1]));
bi_instr *collect = bi_collect_i32_to(b, bi_get_index(instr->def.index), 2);
collect->src[0] = tempa;
collect->src[1] = tempb;
}
}
static bi_index
src/panfrost/compiler/bir.c
+12
View File
@@ -90,6 +90,8 @@ bi_count_read_registers(const bi_instr *ins, unsigned s)
/* ATOM reads 1 but writes 2. Exception for ACMPXCHG */
if (s == 0 && ins->op == BI_OPCODE_ATOM_RETURN_I32)
return (ins->atom_opc == BI_ATOM_OPC_ACMPXCHG) ? 2 : 1;
else if (s == 0 && ins->op == BI_OPCODE_ATOM_RETURN_I64)
return (ins->atom_opc == BI_ATOM_OPC_ACMPXCHG) ? 4 : 2;
else if (s == 0 && bi_get_opcode_props(ins)->sr_read)
return bi_count_staging_registers(ins);
else if (s == 4 && ins->op == BI_OPCODE_BLEND)
@@ -125,8 +127,18 @@ bi_count_write_registers(const bi_instr *ins, unsigned d)
case BI_OPCODE_ACMPXCHG_I32:
/* Reads 2 but writes 1 */
return 1;
case BI_OPCODE_ACMPXCHG_I64:
/* Reads 4 but writes 2 */
return 2;
case BI_OPCODE_ATOM_POST_I32:
/* Reads 2 but writes 1 */
return 1;
case BI_OPCODE_ATOM_POST_I64:
/* Reads 4 but writes 2 */
return 2;
case BI_OPCODE_ATOM1_RETURN_I32:
case BI_OPCODE_ATOM1_RETURN_I64:
/* Allow omitting the destination for plain ATOM1 */
return bi_is_null(ins->dest[0]) ? 0 : ins->sr_count;
default:
src/panfrost/compiler/valhall/va_lower_isel.c
+13
View File
@@ -110,12 +110,25 @@ lower(bi_builder *b, bi_instr *I)
return bi_branchzi(b, bi_zero(), I->src[0], BI_CMPF_EQ);
}
case BI_OPCODE_AXCHG_I64:
bi_set_opcode(I, BI_OPCODE_ATOM_RETURN_I64);
I->atom_opc = BI_ATOM_OPC_AXCHG;
I->sr_count = 2;
return NULL;
case BI_OPCODE_AXCHG_I32:
bi_set_opcode(I, BI_OPCODE_ATOM_RETURN_I32);
I->atom_opc = BI_ATOM_OPC_AXCHG;
I->sr_count = 1;
return NULL;
case BI_OPCODE_ACMPXCHG_I64:
bi_set_opcode(I, BI_OPCODE_ATOM_RETURN_I64);
I->atom_opc = BI_ATOM_OPC_ACMPXCHG;
/* Reads 4, this is special cased in bir.c */
I->sr_count = 2;
return NULL;
case BI_OPCODE_ACMPXCHG_I32:
bi_set_opcode(I, BI_OPCODE_ATOM_RETURN_I32);
I->atom_opc = BI_ATOM_OPC_ACMPXCHG;
src/panfrost/compiler/valhall/va_pack.c
+33
View File
@@ -909,6 +909,37 @@ va_pack_instr(const bi_instr *I, unsigned arch)
hex |= va_pack_store(I);
break;
case BI_OPCODE_ATOM1_RETURN_I64:
/* Permit omitting the destination for plain ATOM1 */
if (!bi_count_write_registers(I, 0)) {
hex |= (0x40ull << 40); // fake read
}
/* 64-bit source */
va_validate_register_pair(I, 0);
hex |= (uint64_t)va_pack_src(I, 0) << 0;
hex |= va_pack_byte_offset_8(I);
hex |= ((uint64_t)va_pack_atom_opc_1(I)) << 22;
break;
case BI_OPCODE_ACMPXCHG_I64:
case BI_OPCODE_AXCHG_I64:
case BI_OPCODE_ATOM_I64:
case BI_OPCODE_ATOM_RETURN_I64:
/* 64-bit source */
va_validate_register_pair(I, 1);
hex |= (uint64_t)va_pack_src(I, 1) << 0;
hex |= va_pack_byte_offset_8(I);
hex |= ((uint64_t)va_pack_atom_opc(I)) << 22;
if (I->op == BI_OPCODE_ATOM_RETURN_I64)
hex |= (0xc0ull << 40); // flags
if (I->atom_opc == BI_ATOM_OPC_ACMPXCHG)
hex |= (1 << 26); /* .compare */
break;
case BI_OPCODE_ATOM1_RETURN_I32:
/* Permit omitting the destination for plain ATOM1 */
if (!bi_count_write_registers(I, 0)) {
@@ -922,6 +953,8 @@ va_pack_instr(const bi_instr *I, unsigned arch)
hex |= ((uint64_t)va_pack_atom_opc_1(I)) << 22;
break;
case BI_OPCODE_ACMPXCHG_I32:
case BI_OPCODE_AXCHG_I32:
case BI_OPCODE_ATOM_I32:
case BI_OPCODE_ATOM_RETURN_I32:
/* 64-bit source */