From 02fe57b7e9cec6d1e0f6f7a9b7a15965db7b3d90 Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Wed, 28 Dec 2022 21:16:35 -0500 Subject: [PATCH] agx: Lower system values in NIR in the driver To comply with The Ekstrand Rule. AGX has a large number of "uniform registers" available. These may be loaded with arbitrary ranges of GPU memory by the driver, or they can be written by the preamble shader. Currently, the compiler runs nir_opt_preamble on the first half of the uniform file, and then translates NIR sysvals to moves from the second half of the uniform file, passing back a uniform->sysval map for the GL driver to respect. This has (at least) two issues: * Since nir_opt_preamble runs before gathering sysvals, it has to assume the maximum number of sysvals are pushed, which can prevent it from moving some computation to the preamble due to running out of partitioned uniform registers. This is a problem for Dolphin's ubershaders, though it's unclear how much it matters for Dolphin perf. * This violates The Ekstrand Rule and apparently will be a problem for our Vulkan driver. I'm just a compiler+GL girl, so I wouldn't know. To fix this, we invert the order of operations. At the end of this series, we instead lower NIR system values to NIR load_preamble instructions in the GL driver. The compiler just translates directly to uniform registers reads. The Vulkan driver will need its own version of this code, but maybe it can do something clever and descriptor set aware. This means that there will already be some load_preamble instructions when nir_opt_preamble runs, so I've made minor changes to nir_opt_preamble to handle that gracefully. This is a bit lazy... The alternative is to introduce a `load_uniform_agx` intrinsic which `load_preamble` gets lowered to trivially. But that's another pass over the IR (and due to AGX's shader variant hell I'm sensitive to backend compile time) and it would be more complicated than what's implemented here. Signed-off-by: Alyssa Rosenzweig Acked-by: Ella Stanforth Part-of: --- src/asahi/compiler/agx_compile.c | 37 +--- src/asahi/compiler/agx_compile.h | 66 +------ src/asahi/compiler/agx_compiler.h | 8 - src/asahi/compiler/agx_uniforms.c | 96 --------- src/asahi/compiler/meson.build | 1 - .../drivers/asahi/agx_nir_lower_sysvals.c | 184 ++++++++++++++++++ src/gallium/drivers/asahi/agx_state.c | 16 +- src/gallium/drivers/asahi/agx_state.h | 58 +++++- src/gallium/drivers/asahi/agx_uniforms.c | 93 +++------ src/gallium/drivers/asahi/meson.build | 1 + 10 files changed, 283 insertions(+), 277 deletions(-) delete mode 100644 src/asahi/compiler/agx_uniforms.c create mode 100644 src/gallium/drivers/asahi/agx_nir_lower_sysvals.c diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index bd15c59f03b..baef2287a26 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -23,12 +23,12 @@ * SOFTWARE. */ -#include "agx_compile.h" #include "compiler/nir/nir_builder.h" #include "compiler/nir_types.h" #include "util/glheader.h" #include "util/u_debug.h" #include "agx_builder.h" +#include "agx_compile.h" #include "agx_compiler.h" #include "agx_internal_formats.h" @@ -652,15 +652,6 @@ agx_emit_load_frag_coord(agx_builder *b, agx_index dst, agx_emit_collect_to(b, dst, 4, dests); } -static agx_instr * -agx_blend_const(agx_builder *b, agx_index dst, unsigned comp) -{ - agx_index val = agx_indexed_sysval(b->shader, AGX_PUSH_BLEND_CONST, - AGX_SIZE_32, comp * 2, 4 * 2); - - return agx_mov_to(b, dst, val); -} - /* * Demoting a helper invocation is logically equivalent to zeroing the sample * mask. Metal implement discard as such. @@ -742,22 +733,6 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr) case nir_intrinsic_load_back_face_agx: return agx_get_sr_to(b, dst, AGX_SR_BACKFACING); - case nir_intrinsic_load_texture_base_agx: - return agx_mov_to(b, dst, - agx_indexed_sysval(b->shader, AGX_PUSH_TEXTURE_BASE, - AGX_SIZE_64, 0, 4)); - - case nir_intrinsic_load_ubo_base_agx: - return agx_mov_to( - b, dst, - agx_indexed_sysval(b->shader, AGX_PUSH_UBO_BASES, AGX_SIZE_64, - nir_src_as_uint(instr->src[0]) * 4, - b->shader->nir->info.num_ubos * 4)); - - case nir_intrinsic_load_vbo_base_agx: - return agx_mov_to( - b, dst, agx_vbo_base(b->shader, nir_src_as_uint(instr->src[0]))); - case nir_intrinsic_load_vertex_id: return agx_mov_to(b, dst, agx_abs(agx_vertex_id(b))); @@ -773,15 +748,6 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr) case nir_intrinsic_block_image_store_agx: return agx_emit_block_image_store(b, instr); - case nir_intrinsic_load_blend_const_color_r_float: - return agx_blend_const(b, dst, 0); - case nir_intrinsic_load_blend_const_color_g_float: - return agx_blend_const(b, dst, 1); - case nir_intrinsic_load_blend_const_color_b_float: - return agx_blend_const(b, dst, 2); - case nir_intrinsic_load_blend_const_color_a_float: - return agx_blend_const(b, dst, 3); - default: fprintf(stderr, "Unhandled intrinsic %s\n", nir_intrinsic_infos[instr->intrinsic].name); @@ -2015,6 +1981,7 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key, out->depth_layout = layout; } + out->push_count = key->reserved_preamble; agx_optimize_nir(nir, &out->push_count); /* Implement conditional discard with real control flow like Metal */ diff --git a/src/asahi/compiler/agx_compile.h b/src/asahi/compiler/agx_compile.h index b2388669f43..0c5a289738a 100644 --- a/src/asahi/compiler/agx_compile.h +++ b/src/asahi/compiler/agx_compile.h @@ -27,63 +27,6 @@ #include "compiler/nir/nir.h" #include "util/u_dynarray.h" -enum agx_push_type { - /* Array of 64-bit pointers to the base addresses (BASES) and array of - * 16-bit sizes for optional bounds checking (SIZES) */ - AGX_PUSH_UBO_BASES, - AGX_PUSH_UBO_SIZES, - AGX_PUSH_VBO_SIZES, - AGX_PUSH_SSBO_BASES, - AGX_PUSH_SSBO_SIZES, - - /* 64-bit VBO base pointer */ - AGX_PUSH_VBO_BASE, - - /* Push the attached constant memory */ - AGX_PUSH_CONSTANTS, - - /* Push the content of a UBO */ - AGX_PUSH_UBO_DATA, - - /* RGBA blend constant (FP32) */ - AGX_PUSH_BLEND_CONST, - - AGX_PUSH_TEXTURE_BASE, - - /* Keep last */ - AGX_PUSH_NUM_TYPES -}; - -static_assert(AGX_PUSH_NUM_TYPES < (1 << 8), "type overflow"); - -struct agx_push { - /* Contents to push */ - enum agx_push_type type : 8; - - /* Base of where to push, indexed in 16-bit units. The uniform file contains - * 512 = 2^9 such units. */ - unsigned base : 9; - - /* Number of 16-bit units to push */ - unsigned length : 9; - - /* If set, rather than pushing the specified data, push a pointer to the - * specified data. This is slower to access but enables indirect access, as - * the uniform file does not support indirection. */ - bool indirect : 1; - - union { - struct { - uint16_t ubo; - uint16_t offset; - } ubo_data; - - uint32_t vbo; - }; -}; - -/* All possible push types except VBO, plus up to 16 VBOs */ -#define AGX_MAX_PUSH_RANGES (AGX_PUSH_NUM_TYPES - 1 + 16) /* Arbitrary */ #define AGX_MAX_VARYINGS (32) @@ -153,11 +96,11 @@ union agx_varyings { }; struct agx_shader_info { - unsigned push_count; - unsigned push_ranges; - struct agx_push push[AGX_MAX_PUSH_RANGES]; union agx_varyings varyings; + /* Number of uniforms */ + unsigned push_count; + /* Does the shader have a preamble? If so, it is at offset preamble_offset. * The main shader is at offset main_offset. The preamble is executed first. */ @@ -223,6 +166,9 @@ struct agx_fs_shader_key { }; struct agx_shader_key { + /* Number of reserved preamble slots at the start */ + unsigned reserved_preamble; + union { struct agx_fs_shader_key fs; }; diff --git a/src/asahi/compiler/agx_compiler.h b/src/asahi/compiler/agx_compiler.h index 80c25750178..cdb1c65e08f 100644 --- a/src/asahi/compiler/agx_compiler.h +++ b/src/asahi/compiler/agx_compiler.h @@ -775,14 +775,6 @@ agx_builder_insert(agx_cursor *cursor, agx_instr *I) unreachable("Invalid cursor option"); } -/* Uniform file management */ - -agx_index agx_indexed_sysval(agx_context *ctx, enum agx_push_type type, - enum agx_size size, unsigned index, - unsigned length); - -agx_index agx_vbo_base(agx_context *ctx, unsigned vbo); - /* Routines defined for AIR */ void agx_print_instr(agx_instr *I, FILE *fp); diff --git a/src/asahi/compiler/agx_uniforms.c b/src/asahi/compiler/agx_uniforms.c deleted file mode 100644 index 99ba92a9c89..00000000000 --- a/src/asahi/compiler/agx_uniforms.c +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2021 Alyssa Rosenzweig - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include "agx_compiler.h" - -/* Manages the uniform file. We can push certain fixed items (during initial - * code generation), where we're gauranteed to have sufficient space. After - * that, UBO ranges can be selectively pushed while there's space. */ - -/* Directly index an array sysval. Index must be in bounds. Index specified in - * 16-bit units regardless of the underlying sysval's unit. */ - -agx_index -agx_indexed_sysval(agx_context *ctx, enum agx_push_type type, - enum agx_size size, unsigned index, unsigned length) -{ - /* Check if we already pushed */ - for (unsigned i = 0; i < ctx->out->push_ranges; ++i) { - struct agx_push push = ctx->out->push[i]; - - if (push.type == type && !push.indirect) { - assert(length == push.length); - assert(index < push.length); - return agx_uniform(push.base + index, size); - } - } - - /* Otherwise, push */ - assert(ctx->out->push_ranges < AGX_MAX_PUSH_RANGES); - - ctx->out->push_count = - ALIGN_POT(ctx->out->push_count, agx_size_align_16(size)); - - unsigned base = ctx->out->push_count; - ctx->out->push_count += length; - assert(ctx->out->push_count <= AGX_NUM_UNIFORMS); - - ctx->out->push[ctx->out->push_ranges++] = - (struct agx_push){.type = type, - .base = base, - .length = length, - .indirect = false}; - - return agx_uniform(base + index, size); -} - -agx_index -agx_vbo_base(agx_context *ctx, unsigned vbo) -{ - /* Check if we already pushed */ - for (unsigned i = 0; i < ctx->out->push_ranges; ++i) { - struct agx_push push = ctx->out->push[i]; - - if (push.type == AGX_PUSH_VBO_BASE && push.vbo == vbo) { - return agx_uniform(push.base, AGX_SIZE_64); - } - } - - /* Otherwise, push */ - assert(ctx->out->push_ranges < AGX_MAX_PUSH_RANGES); - - ctx->out->push_count = ALIGN_POT(ctx->out->push_count, 4); - - unsigned base = ctx->out->push_count; - ctx->out->push_count += 4; - assert(ctx->out->push_count <= AGX_NUM_UNIFORMS); - - ctx->out->push[ctx->out->push_ranges++] = (struct agx_push){ - .type = AGX_PUSH_VBO_BASE, - .base = base, - .length = 4, - .vbo = vbo, - }; - - return agx_uniform(base, AGX_SIZE_64); -} diff --git a/src/asahi/compiler/meson.build b/src/asahi/compiler/meson.build index 4f449d7a259..ad76f14001c 100644 --- a/src/asahi/compiler/meson.build +++ b/src/asahi/compiler/meson.build @@ -39,7 +39,6 @@ libasahi_agx_files = files( 'agx_opt_cse.c', 'agx_optimizer.c', 'agx_register_allocate.c', - 'agx_uniforms.c', 'agx_validate.c', ) diff --git a/src/gallium/drivers/asahi/agx_nir_lower_sysvals.c b/src/gallium/drivers/asahi/agx_nir_lower_sysvals.c new file mode 100644 index 00000000000..433d005293c --- /dev/null +++ b/src/gallium/drivers/asahi/agx_nir_lower_sysvals.c @@ -0,0 +1,184 @@ +/* + * Copyright 2022 Alyssa Rosenzweig + * SPDX-License-Identifier: MIT + */ + +#include "compiler/nir/nir_builder.h" +#include "util/bitset.h" +#include "util/u_dynarray.h" +#include "agx_state.h" + +/* + * Lower all system values to uniform loads. This pass tries to compact ranges + * of contiguous uploaded uniforms to reduce the draw-time overhead of uploading + * many tiny ranges. To do so, it works in 3 steps: + * + * 1. Walk the NIR, converting system values to placeholder load_preambles. + * 2. Walk the ranges of uniforms needed, compacting into contiguous ranges. + * 3. Fill in the load_preamble instructions with the real uniforms. + */ +struct state { + /* Array of load_preamble nir_intrinsic_instr's to fix up at the end */ + struct util_dynarray load_preambles; + + /* Bitset of 16-bit uniforms pushed */ + BITSET_DECLARE(pushed, sizeof(struct agx_draw_uniforms) / 2); + + /* Element size in 16-bit units, so we may split ranges of different sizes + * to guarantee natural alignment. + */ + uint8_t element_size[sizeof(struct agx_draw_uniforms) / 2]; +}; + +static bool +pass(struct nir_builder *b, nir_instr *instr, void *data) +{ + if (instr->type != nir_instr_type_intrinsic) + return false; + + nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); + b->cursor = nir_before_instr(instr); + struct state *state = data; + + /* For offsetof with dynamic array elements */ + struct agx_draw_uniforms *u = NULL; + void *ptr = NULL; + + switch (intr->intrinsic) { + case nir_intrinsic_load_vbo_base_agx: + ptr = &u->vs.vbo_base[nir_src_as_uint(intr->src[0])]; + break; + case nir_intrinsic_load_ubo_base_agx: + ptr = &u->ubo_base[nir_src_as_uint(intr->src[0])]; + break; + case nir_intrinsic_load_texture_base_agx: + ptr = &u->texture_base; + break; + case nir_intrinsic_load_blend_const_color_r_float: + ptr = &u->fs.blend_constant[0]; + break; + case nir_intrinsic_load_blend_const_color_g_float: + ptr = &u->fs.blend_constant[1]; + break; + case nir_intrinsic_load_blend_const_color_b_float: + ptr = &u->fs.blend_constant[2]; + break; + case nir_intrinsic_load_blend_const_color_a_float: + ptr = &u->fs.blend_constant[3]; + break; + default: + return false; + } + + assert(nir_dest_bit_size(intr->dest) >= 16 && "no 8-bit sysvals"); + + unsigned dim = nir_dest_num_components(intr->dest); + unsigned element_size = nir_dest_bit_size(intr->dest) / 16; + unsigned length = dim * element_size; + + unsigned offset = (uintptr_t)ptr; + assert((offset % 2) == 0 && "all entries are aligned by ABI"); + + nir_ssa_def *value = + nir_load_preamble(b, dim, nir_dest_bit_size(intr->dest), .base = offset); + nir_ssa_def_rewrite_uses(&intr->dest.ssa, value); + + BITSET_SET_RANGE(state->pushed, (offset / 2), (offset / 2) + length - 1); + + for (unsigned i = 0; i < length; ++i) { + if (state->element_size[(offset / 2) + i]) + assert((state->element_size[(offset / 2) + i]) == element_size); + else + state->element_size[(offset / 2) + i] = element_size; + } + + util_dynarray_append(&state->load_preambles, nir_intrinsic_instr *, + nir_instr_as_intrinsic(value->parent_instr)); + return true; +} + +static struct agx_push_range * +find_push_range_containing(struct agx_compiled_shader *shader, unsigned offset) +{ + for (unsigned i = 0; i < shader->push_range_count; ++i) { + struct agx_push_range *range = &shader->push[i]; + + /* range->length is 16-bit words, need to convert. offset is bytes. */ + unsigned length_B = range->length * 2; + + if (range->offset <= offset && offset < (range->offset + length_B)) + return range; + } + + unreachable("no containing range"); +} + +static unsigned +lay_out_uniforms(struct agx_compiled_shader *shader, struct state *state) +{ + unsigned uniform = 0; + + unsigned start, end; + BITSET_FOREACH_RANGE(start, end, state->pushed, sizeof(state->pushed) * 8) { + unsigned range_start = start; + + do { + uint8_t size = state->element_size[range_start]; + + /* Find a range of constant element size. [range_start, range_end) */ + unsigned range_end; + for (range_end = range_start + 1; + range_end < end && state->element_size[range_end] == size; + ++range_end) + ; + + /* Now make the range with the given size (naturally aligned) */ + uniform = ALIGN_POT(uniform, size); + + assert((shader->push_range_count < ARRAY_SIZE(shader->push)) && + "AGX_MAX_PUSH_RANGES must be an upper bound"); + + /* Offsets must be aligned to 8 bytes, this may require pushing a + * little more than intended (otherwise we would need extra copies) + */ + range_start = ROUND_DOWN_TO(range_start, 8 / 2); + + shader->push[shader->push_range_count++] = (struct agx_push_range){ + .uniform = uniform, + .offset = range_start * 2 /* bytes, not elements */, + .length = (range_end - range_start), + }; + + uniform += (range_end - range_start); + range_start = range_end; + } while (range_start < end); + } + + util_dynarray_foreach(&state->load_preambles, nir_intrinsic_instr *, intr) { + unsigned offset = nir_intrinsic_base(*intr); + struct agx_push_range *range = find_push_range_containing(shader, offset); + + nir_intrinsic_set_base(*intr, + range->uniform + ((offset - range->offset) / 2)); + } + + return uniform; +} + +bool +agx_nir_lower_sysvals(nir_shader *shader, struct agx_compiled_shader *compiled, + unsigned *push_size) +{ + struct state state = {0}; + + bool progress = nir_shader_instructions_pass( + shader, pass, nir_metadata_block_index | nir_metadata_dominance, &state); + + if (progress) { + *push_size = lay_out_uniforms(compiled, &state); + } else { + *push_size = 0; + } + + return progress; +} diff --git a/src/gallium/drivers/asahi/agx_state.c b/src/gallium/drivers/asahi/agx_state.c index a59bab8af3f..b24be584808 100644 --- a/src/gallium/drivers/asahi/agx_state.c +++ b/src/gallium/drivers/asahi/agx_state.c @@ -1291,6 +1291,10 @@ agx_compile_variant(struct agx_device *dev, struct agx_uncompiled_shader *so, } struct agx_shader_key base_key = {0}; + + NIR_PASS_V(nir, agx_nir_lower_sysvals, compiled, + &base_key.reserved_preamble); + agx_compile_shader_nir(nir, &base_key, debug, &binary, &compiled->info); if (binary.size) { @@ -1549,7 +1553,7 @@ agx_build_pipeline(struct agx_batch *batch, struct agx_compiled_shader *cs, } struct agx_usc_builder b = - agx_alloc_usc_control(&batch->pipeline_pool, cs->info.push_ranges + 2); + agx_alloc_usc_control(&batch->pipeline_pool, cs->push_range_count + 2); if (nr_textures) { agx_usc_pack(&b, TEXTURE, cfg) { @@ -1557,8 +1561,6 @@ agx_build_pipeline(struct agx_batch *batch, struct agx_compiled_shader *cs, cfg.count = nr_textures; cfg.buffer = T_tex.gpu; } - - batch->textures = T_tex.gpu; } if (nr_samplers) { @@ -1572,9 +1574,11 @@ agx_build_pipeline(struct agx_batch *batch, struct agx_compiled_shader *cs, /* Must only upload uniforms after uploading textures so we can implement the * AGX_PUSH_TEXTURE_BASE sysval correctly. */ - for (unsigned i = 0; i < cs->info.push_ranges; ++i) { - agx_usc_uniform(&b, cs->info.push[i].base, cs->info.push[i].length, - agx_push_location(batch, cs->info.push[i], stage)); + uint64_t uniforms = agx_upload_uniforms(batch, T_tex.gpu, stage); + + for (unsigned i = 0; i < cs->push_range_count; ++i) { + agx_usc_uniform(&b, cs->push[i].uniform, cs->push[i].length, + uniforms + cs->push[i].offset); } if (stage == PIPE_SHADER_FRAGMENT) diff --git a/src/gallium/drivers/asahi/agx_state.h b/src/gallium/drivers/asahi/agx_state.h index a0be811b4f9..eeac74e2dce 100644 --- a/src/gallium/drivers/asahi/agx_state.h +++ b/src/gallium/drivers/asahi/agx_state.h @@ -58,12 +58,59 @@ agx_so_target(struct pipe_stream_output_target *target) return (struct agx_streamout_target *)target; } +/* Shaders can access fixed-function state through system values. + * It is convenient to stash all of this information into a single "root" + * descriptor, then push individual parts as needed. + * + * In the future, we could optimize this to reduce CPU overhead, e.g. splitting + * into multiple descriptors for finer dirty tracking. This is not ABI with the + * compiler. The layout is up to us and handled by our code lowering system + * values to uniforms. + */ +struct PACKED agx_draw_uniforms { + /* Pointer to binding table for texture descriptor, or 0 if none */ + uint64_t texture_base; + + /* Uniform buffer objects */ + uint64_t ubo_base[PIPE_MAX_CONSTANT_BUFFERS]; + + union { + struct { + /* Vertex buffer object bases, if present */ + uint64_t vbo_base[PIPE_MAX_ATTRIBS]; + } vs; + + struct { + /* Blend constant if any */ + float blend_constant[4]; + } fs; + }; +}; + +/* We only push whole elements at a time so we can calculate an upper bound */ +#define AGX_MAX_PUSH_RANGES (1 + PIPE_MAX_CONSTANT_BUFFERS + PIPE_MAX_ATTRIBS) + +struct agx_push_range { + /* Base 16-bit uniform to push to */ + uint16_t uniform; + + /* Offset into agx_draw_uniforms to push in bytes */ + uint16_t offset; + + /* Number of consecutive 16-bit uniforms to push */ + size_t length; +}; + struct agx_compiled_shader { /* Mapped executable memory */ struct agx_bo *bo; /* Metadata returned from the compiler */ struct agx_shader_info info; + + /* Uniforms the driver must push */ + unsigned push_range_count; + struct agx_push_range push[AGX_MAX_PUSH_RANGES]; }; struct agx_uncompiled_shader { @@ -100,9 +147,6 @@ struct agx_batch { /* PIPE_CLEAR_* bitmask */ uint32_t clear, draw, load, resolve; - /* Base of uploaded texture descriptors */ - uint64_t textures; - uint64_t uploaded_clear_color[PIPE_MAX_COLOR_BUFS]; double clear_depth; unsigned clear_stencil; @@ -411,8 +455,12 @@ agx_transfer(struct pipe_transfer *p) return (struct agx_transfer *)p; } -uint64_t agx_push_location(struct agx_batch *batch, struct agx_push push, - enum pipe_shader_type stage); +uint64_t agx_upload_uniforms(struct agx_batch *batch, uint64_t textures, + enum pipe_shader_type stage); + +bool agx_nir_lower_sysvals(nir_shader *shader, + struct agx_compiled_shader *compiled, + unsigned *push_size); bool agx_batch_is_active(struct agx_batch *batch); diff --git a/src/gallium/drivers/asahi/agx_uniforms.c b/src/gallium/drivers/asahi/agx_uniforms.c index 01a6a411444..4d2c5b509a6 100644 --- a/src/gallium/drivers/asahi/agx_uniforms.c +++ b/src/gallium/drivers/asahi/agx_uniforms.c @@ -24,11 +24,6 @@ #include "asahi/lib/agx_pack.h" #include "agx_state.h" -/* Computes the address for a push uniform, adding referenced BOs to the - * current batch as necessary. Note anything uploaded via the batch's pool does - * not require an update to the BO list, since the entire pool will be added - * once at submit time. */ - static uint64_t agx_const_buffer_ptr(struct agx_batch *batch, struct pipe_constant_buffer *cb) { @@ -45,76 +40,42 @@ agx_const_buffer_ptr(struct agx_batch *batch, struct pipe_constant_buffer *cb) } static uint64_t -agx_push_location_direct(struct agx_batch *batch, struct agx_push push, - enum pipe_shader_type stage) +agx_vertex_buffer_ptr(struct agx_batch *batch, unsigned vbo) { - struct agx_context *ctx = batch->ctx; - struct agx_stage *st = &ctx->stage[stage]; - - switch (push.type) { - case AGX_PUSH_UBO_BASES: { - unsigned count = util_last_bit(st->cb_mask); - struct agx_ptr ptr = - agx_pool_alloc_aligned(&batch->pool, count * sizeof(uint64_t), 8); - uint64_t *addresses = ptr.cpu; - - for (unsigned i = 0; i < count; ++i) { - struct pipe_constant_buffer *cb = &st->cb[i]; - addresses[i] = agx_const_buffer_ptr(batch, cb); - } - - return ptr.gpu; - } - - case AGX_PUSH_VBO_BASE: { - struct agx_ptr ptr = - agx_pool_alloc_aligned(&batch->pool, sizeof(uint64_t), 8); - uint64_t *address = ptr.cpu; - - assert(ctx->vb_mask & BITFIELD_BIT(push.vbo) && "oob"); - - struct pipe_vertex_buffer vb = ctx->vertex_buffers[push.vbo]; - assert(!vb.is_user_buffer); - - if (!vb.buffer.resource) { - *address = 0; - return ptr.gpu; - } + struct pipe_vertex_buffer vb = batch->ctx->vertex_buffers[vbo]; + assert(!vb.is_user_buffer); + if (vb.buffer.resource) { struct agx_resource *rsrc = agx_resource(vb.buffer.resource); agx_batch_reads(batch, rsrc); - *address = rsrc->bo->ptr.gpu + vb.buffer_offset; - return ptr.gpu; - } - - case AGX_PUSH_BLEND_CONST: { - return agx_pool_upload_aligned(&batch->pool, &ctx->blend_color, - sizeof(ctx->blend_color), 8); - } - - case AGX_PUSH_TEXTURE_BASE: { - struct agx_ptr ptr = - agx_pool_alloc_aligned(&batch->pool, sizeof(uint64_t), 8); - uint64_t *address = ptr.cpu; - *address = batch->textures; - return ptr.gpu; - } - - default: - unreachable("todo: push more"); + return rsrc->bo->ptr.gpu + vb.buffer_offset; + } else { + return 0; } } uint64_t -agx_push_location(struct agx_batch *batch, struct agx_push push, - enum pipe_shader_type stage) +agx_upload_uniforms(struct agx_batch *batch, uint64_t textures, + enum pipe_shader_type stage) { - uint64_t direct = agx_push_location_direct(batch, push, stage); - struct agx_pool *pool = &batch->pool; + struct agx_context *ctx = batch->ctx; + struct agx_stage *st = &ctx->stage[stage]; - if (push.indirect) - return agx_pool_upload(pool, &direct, sizeof(direct)); - else - return direct; + struct agx_draw_uniforms uniforms = {.texture_base = textures}; + + u_foreach_bit(cb, st->cb_mask) { + uniforms.ubo_base[cb] = agx_const_buffer_ptr(batch, &st->cb[cb]); + } + + if (stage == PIPE_SHADER_VERTEX) { + u_foreach_bit(vbo, ctx->vb_mask) { + uniforms.vs.vbo_base[vbo] = agx_vertex_buffer_ptr(batch, vbo); + } + } else if (stage == PIPE_SHADER_FRAGMENT) { + memcpy(uniforms.fs.blend_constant, &ctx->blend_color, + sizeof(ctx->blend_color)); + } + + return agx_pool_upload(&batch->pool, &uniforms, sizeof(uniforms)); } diff --git a/src/gallium/drivers/asahi/meson.build b/src/gallium/drivers/asahi/meson.build index 34ce6cf4df4..e7889df517e 100644 --- a/src/gallium/drivers/asahi/meson.build +++ b/src/gallium/drivers/asahi/meson.build @@ -22,6 +22,7 @@ files_asahi = files( 'agx_batch.c', 'agx_blit.c', 'agx_pipe.c', + 'agx_nir_lower_sysvals.c', 'agx_query.c', 'agx_state.c', 'agx_uniforms.c',