From b049b1c98f8862e0c0fe349ba079ea37f0e2f6d4 Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Fri, 11 Aug 2023 15:31:28 -0400 Subject: [PATCH] asahi: Split out per-stage sysvals Signed-off-by: Alyssa Rosenzweig Part-of: --- .../drivers/asahi/agx_nir_lower_sysvals.c | 56 +++++++++++-------- src/gallium/drivers/asahi/agx_state.c | 11 ++-- src/gallium/drivers/asahi/agx_state.h | 43 ++++++++------ src/gallium/drivers/asahi/agx_uniforms.c | 51 +++++++++++------ 4 files changed, 100 insertions(+), 61 deletions(-) diff --git a/src/gallium/drivers/asahi/agx_nir_lower_sysvals.c b/src/gallium/drivers/asahi/agx_nir_lower_sysvals.c index 051e01843e3..ffc35b16926 100644 --- a/src/gallium/drivers/asahi/agx_nir_lower_sysvals.c +++ b/src/gallium/drivers/asahi/agx_nir_lower_sysvals.c @@ -20,7 +20,8 @@ * 3. Fill in the load_preamble instructions with the real uniforms. */ -#define MAX_TABLE_SIZE sizeof(struct agx_draw_uniforms) +#define MAX_TABLE_SIZE sizeof(struct agx_stage_uniforms) +static_assert(sizeof(struct agx_draw_uniforms) <= MAX_TABLE_SIZE, "packed"); struct table_state { /* Bitset of 16-bit uniforms pushed */ @@ -81,11 +82,18 @@ load_sysval_indirect(nir_builder *b, unsigned dim, unsigned bitsize, } } +static unsigned +stage_table(nir_builder *b) +{ + assert(b->shader->info.stage < PIPE_SHADER_TYPES); + return AGX_SYSVAL_STAGE(b->shader->info.stage); +} + static nir_def * load_ubo(nir_builder *b, nir_intrinsic_instr *intr, void *bases) { - nir_def *base = load_sysval_indirect(b, 1, 64, AGX_SYSVAL_TABLE_ROOT, bases, - intr->src[0].ssa); + nir_def *base = + load_sysval_indirect(b, 1, 64, stage_table(b), bases, intr->src[0].ssa); nir_def *address = nir_iadd(b, base, nir_u2u64(b, intr->src[1].ssa)); @@ -97,10 +105,11 @@ static nir_def * lower_intrinsic(nir_builder *b, nir_intrinsic_instr *intr) { struct agx_draw_uniforms *u = NULL; + struct agx_stage_uniforms *s = NULL; switch (intr->intrinsic) { case nir_intrinsic_load_ubo: - return load_ubo(b, intr, u->ubo_base); + return load_ubo(b, intr, s->ubo_base); case nir_intrinsic_load_vbo_base_agx: return load_sysval_indirect(b, 1, 64, AGX_SYSVAL_TABLE_ROOT, &u->vs.vbo_base, intr->src[0].ssa); @@ -117,11 +126,11 @@ lower_intrinsic(nir_builder *b, nir_intrinsic_instr *intr) case nir_intrinsic_load_sample_positions_agx: return load_sysval_root(b, 1, 32, &u->fs.ppp_multisamplectl); case nir_intrinsic_load_ssbo_address: - return load_sysval_indirect(b, 1, 64, AGX_SYSVAL_TABLE_ROOT, - &u->ssbo_base, intr->src[0].ssa); + return load_sysval_indirect(b, 1, 64, stage_table(b), &s->ssbo_base, + intr->src[0].ssa); case nir_intrinsic_get_ssbo_size: - return load_sysval_indirect(b, 1, 32, AGX_SYSVAL_TABLE_ROOT, - &u->ssbo_size, intr->src[0].ssa); + return load_sysval_indirect(b, 1, 32, stage_table(b), &s->ssbo_size, + intr->src[0].ssa); case nir_intrinsic_load_num_workgroups: return load_sysval(b, 3, 32, AGX_SYSVAL_TABLE_GRID, 0); case nir_intrinsic_load_xfb_address: @@ -160,16 +169,15 @@ lower_sysvals(nir_builder *b, nir_instr *instr, void *data) if (tex->op != nir_texop_lod_bias_agx) return false; - struct agx_draw_uniforms *u = NULL; + struct agx_stage_uniforms *s = NULL; int src_idx = nir_tex_instr_src_index(tex, nir_tex_src_texture_offset); if (src_idx >= 0) { - replacement = - load_sysval_indirect(b, 1, 16, AGX_SYSVAL_TABLE_ROOT, u->lod_bias, - tex->src[src_idx].src.ssa); + replacement = load_sysval_indirect( + b, 1, 16, stage_table(b), s->lod_bias, tex->src[src_idx].src.ssa); } else { - replacement = - load_sysval_root(b, 1, 16, &u->lod_bias[tex->sampler_index]); + replacement = load_sysval(b, 1, 16, stage_table(b), + (uintptr_t)&s->lod_bias[tex->sampler_index]); } } @@ -292,14 +300,14 @@ lay_out_table(struct agx_compiled_shader *shader, struct table_state *state, * otherwise bound to texture state registers. */ static void -reserve_internal_bindless(struct state *state) +reserve_internal_bindless(struct state *state, enum pipe_shader_type stage) { - struct table_state *table = &state->tables[AGX_SYSVAL_TABLE_ROOT]; - struct agx_draw_uniforms *u = NULL; - const unsigned len_words = sizeof(u->texture_base) / sizeof(uint16_t); + struct table_state *table = &state->tables[AGX_SYSVAL_STAGE(stage)]; + struct agx_stage_uniforms *s = NULL; + const unsigned len_words = sizeof(s->texture_base) / sizeof(uint16_t); - static_assert(offsetof(struct agx_draw_uniforms, texture_base) == 0, "ABI"); - static_assert(sizeof(u->texture_base) == 8, "64-bit pointer"); + static_assert(offsetof(struct agx_stage_uniforms, texture_base) == 0, "ABI"); + static_assert(sizeof(s->texture_base) == 8, "64-bit pointer"); BITSET_SET_RANGE(table->pushed, 0, len_words - 1); @@ -312,8 +320,10 @@ lay_out_uniforms(struct agx_compiled_shader *shader, struct state *state) { unsigned uniform = 0; - /* Lay out each system value table */ - for (uint8_t t = 0; t < AGX_NUM_SYSVAL_TABLES; ++t) + /* Lay out each system value table. We do this backwards to ensure the first + * uniform goes to the bindless texture base. + */ + for (int t = AGX_NUM_SYSVAL_TABLES - 1; t >= 0; --t) uniform = lay_out_table(shader, &state->tables[t], t, uniform); /* Step 4: Fill in the loads */ @@ -351,7 +361,7 @@ agx_nir_lower_sysvals(nir_shader *shader, bool internal_bindless, &state); if (internal_bindless) - reserve_internal_bindless(&state); + reserve_internal_bindless(&state, shader->info.stage); *push_size = lay_out_uniforms(compiled, &state); diff --git a/src/gallium/drivers/asahi/agx_state.c b/src/gallium/drivers/asahi/agx_state.c index a2906a5077c..167af8338ad 100644 --- a/src/gallium/drivers/asahi/agx_state.c +++ b/src/gallium/drivers/asahi/agx_state.c @@ -2234,14 +2234,15 @@ agx_build_pipeline(struct agx_batch *batch, struct agx_compiled_shader *cs, /* Must only upload uniforms after uploading textures so we can implement the * AGX_PUSH_TEXTURE_BASE sysval correctly. */ - uint64_t uniform_tables[AGX_NUM_SYSVAL_TABLES] = { - agx_upload_uniforms(batch, T_tex.gpu, stage), - ctx->grid_info, - }; + batch->tables[AGX_SYSVAL_STAGE(stage)] = + agx_upload_stage_uniforms(batch, T_tex.gpu, stage); + + batch->tables[AGX_SYSVAL_TABLE_ROOT] = agx_upload_uniforms(batch, stage); + batch->tables[AGX_SYSVAL_TABLE_GRID] = ctx->grid_info; for (unsigned i = 0; i < cs->push_range_count; ++i) { agx_usc_uniform(&b, cs->push[i].uniform, cs->push[i].length, - uniform_tables[cs->push[i].table] + cs->push[i].offset); + batch->tables[cs->push[i].table] + cs->push[i].offset); } if (stage == PIPE_SHADER_FRAGMENT) { diff --git a/src/gallium/drivers/asahi/agx_state.h b/src/gallium/drivers/asahi/agx_state.h index 3bdff57d087..cf6fe6ddb43 100644 --- a/src/gallium/drivers/asahi/agx_state.h +++ b/src/gallium/drivers/asahi/agx_state.h @@ -16,6 +16,7 @@ #include "asahi/lib/agx_tilebuffer.h" #include "asahi/lib/pool.h" #include "compiler/nir/nir_lower_blend.h" +#include "compiler/shader_enums.h" #include "gallium/auxiliary/util/u_blitter.h" #include "gallium/include/pipe/p_context.h" #include "gallium/include/pipe/p_screen.h" @@ -118,25 +119,9 @@ static_assert(AGX_SYSVAL_STAGE(PIPE_SHADER_COMPUTE) == AGX_SYSVAL_TABLE_CS, /* Root system value table */ struct PACKED agx_draw_uniforms { - /* Pointer to binding table for texture descriptor, or 0 if none. This must - * be first so that u0_u1 is always available for lowering binding - * tables to bindless access. - */ - uint64_t texture_base; - /* Pointers to the system value tables themselves (for indirection) */ uint64_t tables[AGX_NUM_SYSVAL_TABLES]; - /* Uniform buffer objects */ - uint64_t ubo_base[PIPE_MAX_CONSTANT_BUFFERS]; - - /* Shader storage buffer objects */ - uint64_t ssbo_base[PIPE_MAX_SHADER_BUFFERS]; - uint32_t ssbo_size[PIPE_MAX_SHADER_BUFFERS]; - - /* LOD bias as float16 */ - uint16_t lod_bias[PIPE_MAX_SAMPLERS]; - union { struct { /* Vertex buffer object bases, if present */ @@ -159,6 +144,24 @@ struct PACKED agx_draw_uniforms { }; }; +struct PACKED agx_stage_uniforms { + /* Pointer to binding table for texture descriptor, or 0 if none. This must + * be first so that u0_u1 is always available for lowering binding + * tables to bindless access. + */ + uint64_t texture_base; + + /* Uniform buffer objects */ + uint64_t ubo_base[PIPE_MAX_CONSTANT_BUFFERS]; + + /* Shader storage buffer objects */ + uint64_t ssbo_base[PIPE_MAX_SHADER_BUFFERS]; + uint32_t ssbo_size[PIPE_MAX_SHADER_BUFFERS]; + + /* LOD bias as float16 */ + uint16_t lod_bias[PIPE_MAX_SAMPLERS]; +}; + /* In the architecture, there are 512 uniform registers, each 16-bits. In a * theoretical worst case, we could push to all of them. We use a worst-case * maximum because the expression for a tight upper bound is too messy and easy @@ -267,6 +270,9 @@ struct agx_batch { */ uint32_t ppp_multisamplectl; + /* Pointers to the system value tables */ + uint64_t tables[AGX_NUM_SYSVAL_TABLES]; + /* Resource list requirements, represented as a bit set indexed by BO * handles (GEM handles on Linux, or IOGPU's equivalent on macOS) */ @@ -704,9 +710,12 @@ agx_transfer(struct pipe_transfer *p) return (struct agx_transfer *)p; } -uint64_t agx_upload_uniforms(struct agx_batch *batch, uint64_t textures, +uint64_t agx_upload_uniforms(struct agx_batch *batch, enum pipe_shader_type stage); +uint64_t agx_upload_stage_uniforms(struct agx_batch *batch, uint64_t textures, + enum pipe_shader_type stage); + bool agx_nir_lower_sysvals(nir_shader *shader, bool internal_bindless, struct agx_compiled_shader *compiled, unsigned *push_size); diff --git a/src/gallium/drivers/asahi/agx_uniforms.c b/src/gallium/drivers/asahi/agx_uniforms.c index a9858abd1da..bd9465b0047 100644 --- a/src/gallium/drivers/asahi/agx_uniforms.c +++ b/src/gallium/drivers/asahi/agx_uniforms.c @@ -51,11 +51,9 @@ agx_vertex_buffer_ptr(struct agx_batch *batch, unsigned vbo) } uint64_t -agx_upload_uniforms(struct agx_batch *batch, uint64_t textures, - enum pipe_shader_type stage) +agx_upload_uniforms(struct agx_batch *batch, enum pipe_shader_type stage) { struct agx_context *ctx = batch->ctx; - struct agx_stage *st = &ctx->stage[stage]; struct agx_ptr root_ptr = agx_pool_alloc_aligned( &batch->pool, sizeof(struct agx_draw_uniforms), 16); @@ -65,21 +63,11 @@ agx_upload_uniforms(struct agx_batch *batch, uint64_t textures, { [AGX_SYSVAL_TABLE_ROOT] = root_ptr.gpu, }, - .texture_base = textures, }; - u_foreach_bit(s, st->valid_samplers) { - uniforms.lod_bias[s] = st->samplers[s]->lod_bias_as_fp16; - } - - u_foreach_bit(cb, st->cb_mask) { - uniforms.ubo_base[cb] = agx_const_buffer_ptr(batch, &st->cb[cb]); - } - - u_foreach_bit(cb, st->ssbo_mask) { - uniforms.ssbo_base[cb] = agx_shader_buffer_ptr(batch, &st->ssbo[cb]); - uniforms.ssbo_size[cb] = st->ssbo[cb].buffer_size; - } + STATIC_ASSERT(AGX_SYSVAL_TABLE_ROOT == 0); + memcpy(&uniforms.tables[1], &batch->tables[1], + sizeof(batch->tables[0]) * (ARRAY_SIZE(batch->tables) - 1)); if (stage == PIPE_SHADER_VERTEX) { u_foreach_bit(vbo, ctx->vb_mask) { @@ -106,3 +94,34 @@ agx_upload_uniforms(struct agx_batch *batch, uint64_t textures, memcpy(root_ptr.cpu, &uniforms, sizeof(uniforms)); return root_ptr.gpu; } + +uint64_t +agx_upload_stage_uniforms(struct agx_batch *batch, uint64_t textures, + enum pipe_shader_type stage) +{ + struct agx_context *ctx = batch->ctx; + struct agx_stage *st = &ctx->stage[stage]; + + struct agx_ptr root_ptr = agx_pool_alloc_aligned( + &batch->pool, sizeof(struct agx_stage_uniforms), 16); + + struct agx_stage_uniforms uniforms = { + .texture_base = textures, + }; + + u_foreach_bit(s, st->valid_samplers) { + uniforms.lod_bias[s] = st->samplers[s]->lod_bias_as_fp16; + } + + u_foreach_bit(cb, st->cb_mask) { + uniforms.ubo_base[cb] = agx_const_buffer_ptr(batch, &st->cb[cb]); + } + + u_foreach_bit(cb, st->ssbo_mask) { + uniforms.ssbo_base[cb] = agx_shader_buffer_ptr(batch, &st->ssbo[cb]); + uniforms.ssbo_size[cb] = st->ssbo[cb].buffer_size; + } + + memcpy(root_ptr.cpu, &uniforms, sizeof(uniforms)); + return root_ptr.gpu; +}