AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ir3: add support for 64 bit atomics

Browse Source 
Signed-off-by: Amber Harmonia <amber@igalia.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/27776>
This commit is contained in:
Amber
2024-05-04 18:15:45 +02:00
committed by Marge Bot
parent 5628a01523
commit 7d0870e5d5
7 changed files with 110 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/freedreno/common/freedreno_dev_info.h
+2
View File
@@ -225,6 +225,8 @@ struct fd_dev_info {
/* Whether there is CP_EVENT_WRITE7::WRITE_SAMPLE_COUNT */
bool has_event_write_sample_count;
bool has_64b_ssbo_atomics;
/* Blob executes a special compute dispatch at the start of each
* command buffers. We copy this dispatch as is.
*/
src/freedreno/common/freedreno_devices.py
+3
View File
@@ -893,6 +893,7 @@ a7xx_740 = A7XXProps(
# Most devices with a740 have blob v6xx which doesn't have
# this hint set. Match them for better compatibility by default.
enable_tp_ubwc_flag_hint = False,
has_64b_ssbo_atomics = True,
)
a7xx_740_a32 = A7XXProps(
@@ -903,6 +904,7 @@ a7xx_740_a32 = A7XXProps(
supports_ibo_ubwc = True,
fs_must_have_non_zero_constlen_quirk = True,
enable_tp_ubwc_flag_hint = False,
has_64b_ssbo_atomics = True,
)
a7xx_740v3 = A7XXProps(
@@ -931,6 +933,7 @@ a7xx_750 = A7XXProps(
has_compliant_dp4acc = True,
ubwc_coherency_quirk = True,
has_persistent_counter = True,
has_64b_ssbo_atomics = True,
)
a730_magic_regs = dict(
src/freedreno/ir3/instr-a3xx.h
+2
View File
@@ -444,6 +444,8 @@ type_uint_size(unsigned bit_size)
case 1: /* 1b bools are treated as normal half-regs */
case 16: return TYPE_U16;
case 32: return TYPE_U32;
case 64:
return TYPE_U32;
default:
ir3_assert(0); /* invalid size */
return (type_t)0;
src/freedreno/ir3/ir3_a6xx.c
+41 -7
View File
@@ -206,6 +206,9 @@ emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
struct ir3_instruction *atomic, *ibo, *src0, *src1, *data, *dummy;
nir_atomic_op op = nir_intrinsic_atomic_op(intr);
type_t type = nir_atomic_op_type(op) == nir_type_int ? TYPE_S32 : TYPE_U32;
if (intr->def.bit_size == 64) {
type = TYPE_ATOMIC_U64;
}
ibo = ir3_ssbo_to_ibo(ctx, intr->src[0]);
@@ -230,10 +233,23 @@ emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
if (op == nir_atomic_op_cmpxchg) {
src0 = ir3_get_src(ctx, &intr->src[4])[0];
struct ir3_instruction *compare = ir3_get_src(ctx, &intr->src[3])[0];
src1 = ir3_collect(b, dummy, compare, data);
if (intr->def.bit_size == 64) {
struct ir3_instruction *dummy2 = create_immed(b, 0);
struct ir3_instruction *compare2 = ir3_get_src(ctx, &intr->src[3])[1];
struct ir3_instruction *data2 = ir3_get_src(ctx, &intr->src[2])[1];
src1 = ir3_collect(b, dummy, dummy2, compare, compare2, data, data2);
} else {
src1 = ir3_collect(b, dummy, compare, data);
}
} else {
src0 = ir3_get_src(ctx, &intr->src[3])[0];
src1 = ir3_collect(b, dummy, data);
if (intr->def.bit_size == 64) {
struct ir3_instruction *dummy2 = create_immed(b, 0);
struct ir3_instruction *data2 = ir3_get_src(ctx, &intr->src[2])[1];
src1 = ir3_collect(b, dummy, dummy2, data, data2);
} else {
src1 = ir3_collect(b, dummy, data);
}
}
atomic = emit_atomic(b, op, ibo, src0, src1);
@@ -250,10 +266,12 @@ emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
atomic->dsts[0]->wrmask = src1->dsts[0]->wrmask;
ir3_reg_tie(atomic->dsts[0], atomic->srcs[2]);
ir3_handle_nonuniform(atomic, intr);
struct ir3_instruction *split;
ir3_split_dest(b, &split, atomic, 0, 1);
return split;
}
size_t num_results = intr->def.bit_size == 64 ? 2 : 1;
struct ir3_instruction *defs[num_results];
ir3_split_dest(b, defs, atomic, 0, num_results);
return ir3_create_collect(b, defs, num_results);
}
/* src[] = { deref, coord, sample_index }. const_index[] = {} */
static void
@@ -482,6 +500,9 @@ emit_intrinsic_atomic_global(struct ir3_context *ctx, nir_intrinsic_instr *intr)
struct ir3_instruction *value = ir3_get_src(ctx, &intr->src[1])[0];
nir_atomic_op op = nir_intrinsic_atomic_op(intr);
type_t type = nir_atomic_op_type(op) == nir_type_int ? TYPE_S32 : TYPE_U32;
if (intr->def.bit_size == 64) {
type = TYPE_ATOMIC_U64;
}
addr = ir3_collect(b, ir3_get_src(ctx, &intr->src[0])[0],
ir3_get_src(ctx, &intr->src[0])[1]);
@@ -489,8 +510,20 @@ emit_intrinsic_atomic_global(struct ir3_context *ctx, nir_intrinsic_instr *intr)
if (op == nir_atomic_op_cmpxchg) {
struct ir3_instruction *compare = ir3_get_src(ctx, &intr->src[2])[0];
src1 = ir3_collect(b, compare, value);
if (intr->def.bit_size == 64) {
struct ir3_instruction *compare2 = ir3_get_src(ctx, &intr->src[2])[1];
struct ir3_instruction *value2 = ir3_get_src(ctx, &intr->src[1])[1];
src1 = ir3_collect(b, compare, compare2, value, value2);
} else {
src1 = ir3_collect(b, compare, value);
}
} else {
src1 = value;
if (intr->def.bit_size == 64) {
struct ir3_instruction *value2 = ir3_get_src(ctx, &intr->src[1])[1];
src1 = ir3_collect(b, value, value2);
} else {
src1 = value;
}
}
switch (op) {
@@ -535,6 +568,7 @@ emit_intrinsic_atomic_global(struct ir3_context *ctx, nir_intrinsic_instr *intr)
atomic->cat6.type = type;
atomic->barrier_class = IR3_BARRIER_BUFFER_W;
atomic->barrier_conflict = IR3_BARRIER_BUFFER_R | IR3_BARRIER_BUFFER_W;
atomic->dsts[0]->wrmask = MASK(intr->def.bit_size == 64 ? 2 : 1);
/* even if nothing consume the result, we can't DCE the instruction: */
array_insert(b, b->keeps, atomic);
src/freedreno/ir3/ir3_compiler_nir.c
+25 -4
View File
@@ -1027,7 +1027,22 @@ emit_alu(struct ir3_context *ctx, nir_alu_instr *alu)
dst = ir3_SUB_S_rpt(b, dst_sz, src[0], 0, src[1], 0);
set_instr_flags(dst.rpts, dst_sz, IR3_INSTR_SAT);
break;
case nir_op_pack_64_2x32_split: {
struct ir3_instruction *r0 = ir3_MOV(b, src[0].rpts[0], TYPE_U32);
struct ir3_instruction *r1 = ir3_MOV(b, src[1].rpts[0], TYPE_U32);
dst.rpts[0] = r0;
dst.rpts[1] = r1;
dst_sz = 2;
break;
}
case nir_op_unpack_64_2x32_split_x: {
ir3_split_dest(b, &dst.rpts[0], src[0].rpts[0], 0, 1);
break;
}
case nir_op_unpack_64_2x32_split_y: {
ir3_split_dest(b, &dst.rpts[0], src[0].rpts[0], 1, 1);
break;
}
case nir_op_udot_4x8_uadd:
case nir_op_udot_4x8_uadd_sat:
case nir_op_sdot_4x8_iadd:
@@ -3311,9 +3326,9 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr)
static void
emit_load_const(struct ir3_context *ctx, nir_load_const_instr *instr)
{
struct ir3_instruction **dst =
ir3_get_dst_ssa(ctx, &instr->def, instr->def.num_components);
unsigned bit_size = ir3_bitsize(ctx, instr->def.bit_size);
struct ir3_instruction **dst =
ir3_get_dst_ssa(ctx, &instr->def, instr->def.num_components * ((bit_size == 64) ? 2 : 1));
if (bit_size <= 8) {
for (int i = 0; i < instr->def.num_components; i++)
@@ -3321,9 +3336,15 @@ emit_load_const(struct ir3_context *ctx, nir_load_const_instr *instr)
} else if (bit_size <= 16) {
for (int i = 0; i < instr->def.num_components; i++)
dst[i] = create_immed_typed(ctx->block, instr->value[i].u16, TYPE_U16);
} else {
} else if (bit_size <= 32) {
for (int i = 0; i < instr->def.num_components; i++)
dst[i] = create_immed_typed(ctx->block, instr->value[i].u32, TYPE_U32);
} else {
assert(instr->def.num_components == 1);
for (int i = 0; i < instr->def.num_components; i++) {
dst[2 * i] = create_immed_typed(ctx->block, (uint32_t)(instr->value[i].u64), TYPE_U32);
dst[2 * i + 1] = create_immed_typed(ctx->block, (uint32_t)(instr->value[i].u64 >> 32), TYPE_U32);
}
}
}
src/freedreno/ir3/ir3_nir_lower_io_offsets.c
+5
View File
@@ -174,6 +174,11 @@ lower_offset_for_ssbo(nir_intrinsic_instr *intrinsic, nir_builder *b,
(!has_dest && intrinsic->src[0].ssa->bit_size == 8))
shift = 0;
if ((has_dest && intrinsic->def.bit_size == 64) ||
(!has_dest && intrinsic->src[0].ssa->bit_size == 64)) {
shift = 1;
}
/* Here we create a new intrinsic and copy over all contents from the old
* one. */
src/freedreno/ir3/ir3_validate.c
+32
View File
@@ -7,6 +7,7 @@
#include "util/ralloc.h"
#include "instr-a3xx.h"
#include "ir3.h"
#include "ir3_compiler.h"
@@ -442,6 +443,37 @@ validate_instr(struct ir3_validate_ctx *ctx, struct ir3_instruction *instr)
validate_assert(ctx, !(instr->srcs[1]->flags & IR3_REG_HALF));
validate_assert(ctx, !(instr->srcs[2]->flags & IR3_REG_HALF));
validate_reg_size(ctx, instr->dsts[0], instr->cat6.type);
break;
case OPC_ATOMIC_B_CMPXCHG:
if (instr->cat6.type == TYPE_ATOMIC_U64) {
validate_assert(ctx, !(instr->dsts[0]->flags & IR3_REG_HALF));
validate_assert(ctx, instr->dsts[0]->wrmask == 0x3f);
} else {
validate_reg_size(ctx, instr->dsts[0], instr->cat6.type);
}
validate_assert(ctx, !(instr->srcs[0]->flags & IR3_REG_HALF));
validate_assert(ctx, !(instr->srcs[1]->flags & IR3_REG_HALF));
break;
case OPC_ATOMIC_B_XCHG:
if (instr->cat6.type == TYPE_ATOMIC_U64) {
validate_assert(ctx, !(instr->dsts[0]->flags & IR3_REG_HALF));
validate_assert(ctx, instr->dsts[0]->wrmask == 0xf);
} else {
validate_reg_size(ctx, instr->dsts[0], instr->cat6.type);
}
validate_assert(ctx, !(instr->srcs[0]->flags & IR3_REG_HALF));
validate_assert(ctx, !(instr->srcs[1]->flags & IR3_REG_HALF));
break;
case OPC_ATOMIC_G_CMPXCHG:
case OPC_ATOMIC_G_XCHG:
if (instr->cat6.type == TYPE_ATOMIC_U64) {
validate_assert(ctx, !(instr->dsts[0]->flags & IR3_REG_HALF));
validate_assert(ctx, instr->dsts[0]->wrmask == 0x3);
} else {
validate_reg_size(ctx, instr->dsts[0], instr->cat6.type);
}
validate_assert(ctx, !(instr->srcs[0]->flags & IR3_REG_HALF));
validate_assert(ctx, !(instr->srcs[1]->flags & IR3_REG_HALF));
break;
case OPC_SHFL:
validate_reg_size(ctx, instr->srcs[0], instr->cat6.type);