AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

agx: plumb COHERENT

Browse Source 
set the magic caching bits. this fixes memory model fails on g13d.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/31532>
This commit is contained in:
Alyssa Rosenzweig
2024-09-06 17:31:25 -04:00
committed by Marge Bot
parent d4e34bc829
commit cd66a6813b
5 changed files with 38 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/asahi/compiler/agx_compile.c
+12 -4
View File
@@ -673,6 +673,12 @@ agx_emit_local_load_pixel(agx_builder *b, agx_index dest,
agx_emit_cached_split(b, dest, nr_comps);
}
static bool
nir_is_coherent(nir_intrinsic_instr *instr)
{
return nir_intrinsic_access(instr) & (ACCESS_COHERENT | ACCESS_VOLATILE);
}
static void
agx_emit_load(agx_builder *b, agx_index dest, nir_intrinsic_instr *instr)
{
@@ -686,7 +692,8 @@ agx_emit_load(agx_builder *b, agx_index dest, nir_intrinsic_instr *instr)
offset = agx_abs(offset);
agx_device_load_to(b, dest, addr, offset, fmt,
BITFIELD_MASK(instr->def.num_components), shift);
BITFIELD_MASK(instr->def.num_components), shift,
nir_is_coherent(instr));
agx_emit_cached_split(b, dest, instr->def.num_components);
}
@@ -704,7 +711,7 @@ agx_emit_store(agx_builder *b, nir_intrinsic_instr *instr)
agx_device_store(b, agx_recollect_vector(b, instr->src[0]), addr, offset,
fmt, BITFIELD_MASK(nir_src_num_components(instr->src[0])),
shift);
shift, nir_is_coherent(instr));
}
/* Preambles write directly to uniform registers, so move from uniform to GPR */
@@ -1092,7 +1099,7 @@ agx_emit_image_load(agx_builder *b, agx_index dst, nir_intrinsic_instr *intr)
agx_instr *I = agx_image_load_to(
b, tmp, coords, lod, bindless, texture, agx_immediate(0), agx_null(),
agx_tex_dim(dim, is_array), lod_mode, 0, false);
agx_tex_dim(dim, is_array), lod_mode, 0, false, nir_is_coherent(intr));
I->mask = agx_expand_tex_to(b, &intr->def, tmp, true);
b->shader->out->uses_txf = true;
@@ -1195,7 +1202,8 @@ agx_emit_image_store(agx_builder *b, nir_intrinsic_instr *instr)
/* Image stores act like tilebuffer stores when used for tib spilling */
b->shader->out->tag_write_disable = false;
return agx_image_write(b, data, coords, lod, base, index, dim);
return agx_image_write(b, data, coords, lod, base, index, dim,
nir_is_coherent(instr));
}
static enum agx_simd_op
src/asahi/compiler/agx_compiler.h
+3
View File
@@ -373,6 +373,9 @@ typedef struct {
/* TODO: Handle iter ops more efficient */
enum agx_interpolation interpolation : 2;
/* TODO: Handle loads more efficiently */
bool coherent : 1;
/* Final st_vary op */
bool last : 1;
src/asahi/compiler/agx_opcodes.h.py
+3 -3
View File
@@ -52,11 +52,11 @@ agx_${name}_as_str(enum agx_${name} x)
/* Runtime accessible info on each defined opcode */
<% assert(len(immediates) < 32); %>
<% assert(len(immediates) < 64); %>
enum agx_immediate {
% for i, imm in enumerate(immediates):
AGX_IMMEDIATE_${imm.upper()} = (1 << ${i}),
AGX_IMMEDIATE_${imm.upper()} = (1ull << ${i}),
% endfor
};
@@ -70,7 +70,7 @@ struct agx_opcode_info {
const char *name;
unsigned nr_srcs;
unsigned nr_dests;
enum agx_immediate immediates;
uint64_t immediates;
struct agx_encoding encoding;
struct agx_encoding encoding_16;
enum agx_schedule_class schedule_class;
src/asahi/compiler/agx_opcodes.py
+7 -5
View File
@@ -106,6 +106,7 @@ GATHER = enum("gather", {
OFFSET = immediate("offset", "bool")
SHADOW = immediate("shadow", "bool")
QUERY_LOD = immediate("query_lod", "bool")
COHERENT = immediate("coherent", "bool")
SCOREBOARD = immediate("scoreboard")
ICOND = immediate("icond", "enum agx_icond")
FCOND = immediate("fcond", "enum agx_fcond")
@@ -310,15 +311,16 @@ op("texture_sample",
srcs = 6, imms = [DIM, LOD_MODE, MASK, SCOREBOARD, OFFSET, SHADOW,
QUERY_LOD, GATHER])
for memory, can_reorder in [("texture", True), ("image", False)]:
coherency = [COHERENT] if not can_reorder else []
op(f"{memory}_load", encoding_32 = (0x71, 0x7F, 8, 10), # XXX WRONG SIZE
srcs = 6, imms = [DIM, LOD_MODE, MASK, SCOREBOARD, OFFSET],
srcs = 6, imms = [DIM, LOD_MODE, MASK, SCOREBOARD, OFFSET] + coherency,
can_reorder = can_reorder,
schedule_class = "none" if can_reorder else "load")
# sources are base, index
op("device_load",
encoding_32 = (0x05, 0x7F, 6, 8),
srcs = 2, imms = [FORMAT, MASK, SHIFT, SCOREBOARD], can_reorder = False,
srcs = 2, imms = [FORMAT, MASK, SHIFT, SCOREBOARD, COHERENT], can_reorder = False,
schedule_class = "load")
# sources are base (relative to workgroup memory), index
@@ -331,7 +333,7 @@ op("local_load",
# TODO: Consider permitting the short form
op("device_store",
encoding_32 = (0x45 | (1 << 47), 0, 8, _),
dests = 0, srcs = 3, imms = [FORMAT, MASK, SHIFT, SCOREBOARD], can_eliminate = False,
dests = 0, srcs = 3, imms = [FORMAT, MASK, SHIFT, SCOREBOARD, COHERENT], can_eliminate = False,
schedule_class = "store")
# sources are value, base, index
@@ -432,8 +434,8 @@ op("signal_pix", (0x58, 0xFF, 4, _), dests = 0, imms = [WRITEOUT],
# Sources are the data vector, the coordinate vector, the LOD, the bindless
# table if present (zero for texture state registers), and texture index.
op("image_write", (0xF1 | (1 << 23) | (9 << 43), 0xFF, 6, 8), dests = 0, srcs = 5, imms
= [DIM], can_eliminate = False, schedule_class = "store")
op("image_write", (0xF1 | (1 << 23), 0xFF, 6, 8), dests = 0, srcs = 5, imms
= [DIM, COHERENT], can_eliminate = False, schedule_class = "store")
# Sources are the image base, image index, the offset within shared memory, and
# the coordinates (or just the layer if implicit).
src/asahi/compiler/agx_pack.c
+13 -5
View File
@@ -731,7 +731,7 @@ agx_pack_instr(struct util_dynarray *emission, struct util_dynarray *fixups,
unsigned O = agx_pack_memory_index(I, I->src[offset_src], &Ot);
unsigned u1 = is_uniform_store ? 0 : 1; // XXX
unsigned u3 = 0;
unsigned u4 = is_uniform_store ? 0 : 4; // XXX
unsigned u4 = is_uniform_store ? 0 : I->coherent ? 7 : 4;
unsigned u5 = 0;
bool L = true; /* TODO: when would you want short? */
@@ -856,7 +856,7 @@ agx_pack_instr(struct util_dynarray *emission, struct util_dynarray *fixups,
unsigned q1 = I->shadow;
unsigned q2 = I->query_lod ? 2 : 0;
unsigned q3 = 12; // XXX
unsigned q3 = 0xc; // XXX
unsigned kill = 0; // helper invocation kill bit
/* Set bit 43 for image loads. This seems to makes sure that image loads
@@ -868,9 +868,15 @@ agx_pack_instr(struct util_dynarray *emission, struct util_dynarray *fixups,
* Apple seems to set this bit unconditionally for read/write image loads
* and never for readonly image loads. Some sort of cache control.
*/
if (I->op == AGX_OPCODE_IMAGE_LOAD)
if (I->op == AGX_OPCODE_IMAGE_LOAD) {
q3 |= 1;
/* Cache bypass for multidie coherency */
if (I->coherent) {
q3 |= 2;
}
}
uint32_t extend = ((U & BITFIELD_MASK(5)) << 0) | (kill << 5) |
((I->dim >> 3) << 7) | ((R >> 6) << 8) |
((C >> 6) << 10) | ((D >> 6) << 12) | ((T >> 6) << 14) |
@@ -917,6 +923,8 @@ agx_pack_instr(struct util_dynarray *emission, struct util_dynarray *fixups,
pack_assert(I, T < (1 << 8));
pack_assert(I, Tt < (1 << 2));
unsigned coherency = I->coherent ? 0xf : 0x9;
uint64_t raw = agx_opcodes_info[I->op].encoding.exact |
(Rt ? (1 << 8) : 0) | ((R & BITFIELD_MASK(6)) << 9) |
((C & BITFIELD_MASK(6)) << 16) | (Ct ? (1 << 22) : 0) |
@@ -924,8 +932,8 @@ agx_pack_instr(struct util_dynarray *emission, struct util_dynarray *fixups,
(((uint64_t)(T & BITFIELD_MASK(6))) << 32) |
(((uint64_t)Tt) << 38) |
(((uint64_t)I->dim & BITFIELD_MASK(3)) << 40) |
(Cs ? (1ull << 47) : 0) | (((uint64_t)U) << 48) |
(rtz ? (1ull << 53) : 0) |
(((uint64_t)coherency) << 43) | (Cs ? (1ull << 47) : 0) |
(((uint64_t)U) << 48) | (rtz ? (1ull << 53) : 0) |
((I->dim & BITFIELD_BIT(4)) ? (1ull << 55) : 0) |
(((uint64_t)R >> 6) << 56) | (((uint64_t)C >> 6) << 58) |
(((uint64_t)D >> 6) << 60) | (((uint64_t)T >> 6) << 62);