From cd66a6813b6bf93ceb38e03c411779d326b16872 Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Fri, 6 Sep 2024 17:31:25 -0400 Subject: [PATCH] agx: plumb COHERENT set the magic caching bits. this fixes memory model fails on g13d. Signed-off-by: Alyssa Rosenzweig Part-of: --- src/asahi/compiler/agx_compile.c | 16 ++++++++++++---- src/asahi/compiler/agx_compiler.h | 3 +++ src/asahi/compiler/agx_opcodes.h.py | 6 +++--- src/asahi/compiler/agx_opcodes.py | 12 +++++++----- src/asahi/compiler/agx_pack.c | 18 +++++++++++++----- 5 files changed, 38 insertions(+), 17 deletions(-) diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index 30202ef3093..6b4504feeef 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -673,6 +673,12 @@ agx_emit_local_load_pixel(agx_builder *b, agx_index dest, agx_emit_cached_split(b, dest, nr_comps); } +static bool +nir_is_coherent(nir_intrinsic_instr *instr) +{ + return nir_intrinsic_access(instr) & (ACCESS_COHERENT | ACCESS_VOLATILE); +} + static void agx_emit_load(agx_builder *b, agx_index dest, nir_intrinsic_instr *instr) { @@ -686,7 +692,8 @@ agx_emit_load(agx_builder *b, agx_index dest, nir_intrinsic_instr *instr) offset = agx_abs(offset); agx_device_load_to(b, dest, addr, offset, fmt, - BITFIELD_MASK(instr->def.num_components), shift); + BITFIELD_MASK(instr->def.num_components), shift, + nir_is_coherent(instr)); agx_emit_cached_split(b, dest, instr->def.num_components); } @@ -704,7 +711,7 @@ agx_emit_store(agx_builder *b, nir_intrinsic_instr *instr) agx_device_store(b, agx_recollect_vector(b, instr->src[0]), addr, offset, fmt, BITFIELD_MASK(nir_src_num_components(instr->src[0])), - shift); + shift, nir_is_coherent(instr)); } /* Preambles write directly to uniform registers, so move from uniform to GPR */ @@ -1092,7 +1099,7 @@ agx_emit_image_load(agx_builder *b, agx_index dst, nir_intrinsic_instr *intr) agx_instr *I = agx_image_load_to( b, tmp, coords, lod, bindless, texture, agx_immediate(0), agx_null(), - agx_tex_dim(dim, is_array), lod_mode, 0, false); + agx_tex_dim(dim, is_array), lod_mode, 0, false, nir_is_coherent(intr)); I->mask = agx_expand_tex_to(b, &intr->def, tmp, true); b->shader->out->uses_txf = true; @@ -1195,7 +1202,8 @@ agx_emit_image_store(agx_builder *b, nir_intrinsic_instr *instr) /* Image stores act like tilebuffer stores when used for tib spilling */ b->shader->out->tag_write_disable = false; - return agx_image_write(b, data, coords, lod, base, index, dim); + return agx_image_write(b, data, coords, lod, base, index, dim, + nir_is_coherent(instr)); } static enum agx_simd_op diff --git a/src/asahi/compiler/agx_compiler.h b/src/asahi/compiler/agx_compiler.h index e120ed5e946..32b6b5f2d7e 100644 --- a/src/asahi/compiler/agx_compiler.h +++ b/src/asahi/compiler/agx_compiler.h @@ -373,6 +373,9 @@ typedef struct { /* TODO: Handle iter ops more efficient */ enum agx_interpolation interpolation : 2; + /* TODO: Handle loads more efficiently */ + bool coherent : 1; + /* Final st_vary op */ bool last : 1; diff --git a/src/asahi/compiler/agx_opcodes.h.py b/src/asahi/compiler/agx_opcodes.h.py index eec899c2943..72c7c54c55c 100644 --- a/src/asahi/compiler/agx_opcodes.h.py +++ b/src/asahi/compiler/agx_opcodes.h.py @@ -52,11 +52,11 @@ agx_${name}_as_str(enum agx_${name} x) /* Runtime accessible info on each defined opcode */ -<% assert(len(immediates) < 32); %> +<% assert(len(immediates) < 64); %> enum agx_immediate { % for i, imm in enumerate(immediates): - AGX_IMMEDIATE_${imm.upper()} = (1 << ${i}), + AGX_IMMEDIATE_${imm.upper()} = (1ull << ${i}), % endfor }; @@ -70,7 +70,7 @@ struct agx_opcode_info { const char *name; unsigned nr_srcs; unsigned nr_dests; - enum agx_immediate immediates; + uint64_t immediates; struct agx_encoding encoding; struct agx_encoding encoding_16; enum agx_schedule_class schedule_class; diff --git a/src/asahi/compiler/agx_opcodes.py b/src/asahi/compiler/agx_opcodes.py index 4e39ca406d6..ec7765da52b 100644 --- a/src/asahi/compiler/agx_opcodes.py +++ b/src/asahi/compiler/agx_opcodes.py @@ -106,6 +106,7 @@ GATHER = enum("gather", { OFFSET = immediate("offset", "bool") SHADOW = immediate("shadow", "bool") QUERY_LOD = immediate("query_lod", "bool") +COHERENT = immediate("coherent", "bool") SCOREBOARD = immediate("scoreboard") ICOND = immediate("icond", "enum agx_icond") FCOND = immediate("fcond", "enum agx_fcond") @@ -310,15 +311,16 @@ op("texture_sample", srcs = 6, imms = [DIM, LOD_MODE, MASK, SCOREBOARD, OFFSET, SHADOW, QUERY_LOD, GATHER]) for memory, can_reorder in [("texture", True), ("image", False)]: + coherency = [COHERENT] if not can_reorder else [] op(f"{memory}_load", encoding_32 = (0x71, 0x7F, 8, 10), # XXX WRONG SIZE - srcs = 6, imms = [DIM, LOD_MODE, MASK, SCOREBOARD, OFFSET], + srcs = 6, imms = [DIM, LOD_MODE, MASK, SCOREBOARD, OFFSET] + coherency, can_reorder = can_reorder, schedule_class = "none" if can_reorder else "load") # sources are base, index op("device_load", encoding_32 = (0x05, 0x7F, 6, 8), - srcs = 2, imms = [FORMAT, MASK, SHIFT, SCOREBOARD], can_reorder = False, + srcs = 2, imms = [FORMAT, MASK, SHIFT, SCOREBOARD, COHERENT], can_reorder = False, schedule_class = "load") # sources are base (relative to workgroup memory), index @@ -331,7 +333,7 @@ op("local_load", # TODO: Consider permitting the short form op("device_store", encoding_32 = (0x45 | (1 << 47), 0, 8, _), - dests = 0, srcs = 3, imms = [FORMAT, MASK, SHIFT, SCOREBOARD], can_eliminate = False, + dests = 0, srcs = 3, imms = [FORMAT, MASK, SHIFT, SCOREBOARD, COHERENT], can_eliminate = False, schedule_class = "store") # sources are value, base, index @@ -432,8 +434,8 @@ op("signal_pix", (0x58, 0xFF, 4, _), dests = 0, imms = [WRITEOUT], # Sources are the data vector, the coordinate vector, the LOD, the bindless # table if present (zero for texture state registers), and texture index. -op("image_write", (0xF1 | (1 << 23) | (9 << 43), 0xFF, 6, 8), dests = 0, srcs = 5, imms - = [DIM], can_eliminate = False, schedule_class = "store") +op("image_write", (0xF1 | (1 << 23), 0xFF, 6, 8), dests = 0, srcs = 5, imms + = [DIM, COHERENT], can_eliminate = False, schedule_class = "store") # Sources are the image base, image index, the offset within shared memory, and # the coordinates (or just the layer if implicit). diff --git a/src/asahi/compiler/agx_pack.c b/src/asahi/compiler/agx_pack.c index cd918ef8b25..c184ad0368b 100644 --- a/src/asahi/compiler/agx_pack.c +++ b/src/asahi/compiler/agx_pack.c @@ -731,7 +731,7 @@ agx_pack_instr(struct util_dynarray *emission, struct util_dynarray *fixups, unsigned O = agx_pack_memory_index(I, I->src[offset_src], &Ot); unsigned u1 = is_uniform_store ? 0 : 1; // XXX unsigned u3 = 0; - unsigned u4 = is_uniform_store ? 0 : 4; // XXX + unsigned u4 = is_uniform_store ? 0 : I->coherent ? 7 : 4; unsigned u5 = 0; bool L = true; /* TODO: when would you want short? */ @@ -856,7 +856,7 @@ agx_pack_instr(struct util_dynarray *emission, struct util_dynarray *fixups, unsigned q1 = I->shadow; unsigned q2 = I->query_lod ? 2 : 0; - unsigned q3 = 12; // XXX + unsigned q3 = 0xc; // XXX unsigned kill = 0; // helper invocation kill bit /* Set bit 43 for image loads. This seems to makes sure that image loads @@ -868,9 +868,15 @@ agx_pack_instr(struct util_dynarray *emission, struct util_dynarray *fixups, * Apple seems to set this bit unconditionally for read/write image loads * and never for readonly image loads. Some sort of cache control. */ - if (I->op == AGX_OPCODE_IMAGE_LOAD) + if (I->op == AGX_OPCODE_IMAGE_LOAD) { q3 |= 1; + /* Cache bypass for multidie coherency */ + if (I->coherent) { + q3 |= 2; + } + } + uint32_t extend = ((U & BITFIELD_MASK(5)) << 0) | (kill << 5) | ((I->dim >> 3) << 7) | ((R >> 6) << 8) | ((C >> 6) << 10) | ((D >> 6) << 12) | ((T >> 6) << 14) | @@ -917,6 +923,8 @@ agx_pack_instr(struct util_dynarray *emission, struct util_dynarray *fixups, pack_assert(I, T < (1 << 8)); pack_assert(I, Tt < (1 << 2)); + unsigned coherency = I->coherent ? 0xf : 0x9; + uint64_t raw = agx_opcodes_info[I->op].encoding.exact | (Rt ? (1 << 8) : 0) | ((R & BITFIELD_MASK(6)) << 9) | ((C & BITFIELD_MASK(6)) << 16) | (Ct ? (1 << 22) : 0) | @@ -924,8 +932,8 @@ agx_pack_instr(struct util_dynarray *emission, struct util_dynarray *fixups, (((uint64_t)(T & BITFIELD_MASK(6))) << 32) | (((uint64_t)Tt) << 38) | (((uint64_t)I->dim & BITFIELD_MASK(3)) << 40) | - (Cs ? (1ull << 47) : 0) | (((uint64_t)U) << 48) | - (rtz ? (1ull << 53) : 0) | + (((uint64_t)coherency) << 43) | (Cs ? (1ull << 47) : 0) | + (((uint64_t)U) << 48) | (rtz ? (1ull << 53) : 0) | ((I->dim & BITFIELD_BIT(4)) ? (1ull << 55) : 0) | (((uint64_t)R >> 6) << 56) | (((uint64_t)C >> 6) << 58) | (((uint64_t)D >> 6) << 60) | (((uint64_t)T >> 6) << 62);