diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index 915996ac6a9..ad5060f2006 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -2931,7 +2931,7 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key, if (nir->info.stage == MESA_SHADER_FRAGMENT) out->tag_write_disable = !nir->info.writes_memory; - bool needs_libagx = false; + bool needs_libagx = nir->info.stage == MESA_SHADER_GEOMETRY; /* Late tilebuffer lowering creates multisampled image stores */ NIR_PASS(needs_libagx, nir, agx_nir_lower_multisampled_image_store); diff --git a/src/asahi/lib/meson.build b/src/asahi/lib/meson.build index ebb76eb0b28..99a84325115 100644 --- a/src/asahi/lib/meson.build +++ b/src/asahi/lib/meson.build @@ -13,6 +13,7 @@ libasahi_lib_files = files( 'agx_meta.c', 'agx_tilebuffer.c', 'agx_nir_lower_alpha.c', + 'agx_nir_lower_gs.c', 'agx_nir_lower_msaa.c', 'agx_nir_lower_sample_intrinsics.c', 'agx_nir_lower_tilebuffer.c', @@ -28,6 +29,8 @@ libasahi_decode_files = files( libagx_shader_files = files( 'shaders/libagx.h', + 'shaders/geometry.cl', + 'shaders/geometry.h', 'shaders/texture.cl', ) @@ -75,6 +78,7 @@ libagx_shaders = custom_target( prepended_input_args, '-o', '@OUTPUT@', '--', '-cl-std=cl2.0', '-D__OPENCL_VERSION__=200', '-I' + join_paths(meson.current_source_dir(), '.'), + '-I' + join_paths(meson.current_source_dir(), '../../'), '-I' + join_paths(meson.current_source_dir(), 'shaders'), '-I' + join_paths(meson.current_build_dir(), '.'), ], diff --git a/src/gallium/drivers/asahi/agx_batch.c b/src/gallium/drivers/asahi/agx_batch.c index 933cf914198..a1c965a249c 100644 --- a/src/gallium/drivers/asahi/agx_batch.c +++ b/src/gallium/drivers/asahi/agx_batch.c @@ -133,6 +133,7 @@ agx_batch_init(struct agx_context *ctx, batch->clear_depth = 0; batch->clear_stencil = 0; batch->varyings = 0; + batch->geometry_state = 0; batch->any_draws = false; batch->initialized = false; batch->draws = 0; diff --git a/src/gallium/drivers/asahi/agx_blit.c b/src/gallium/drivers/asahi/agx_blit.c index 71763acf269..5293c8e474f 100644 --- a/src/gallium/drivers/asahi/agx_blit.c +++ b/src/gallium/drivers/asahi/agx_blit.c @@ -19,6 +19,8 @@ agx_blitter_save(struct agx_context *ctx, struct blitter_context *blitter, util_blitter_save_vertex_elements(blitter, ctx->attributes); util_blitter_save_vertex_shader(blitter, ctx->stage[PIPE_SHADER_VERTEX].shader); + util_blitter_save_geometry_shader(blitter, + ctx->stage[PIPE_SHADER_GEOMETRY].shader); util_blitter_save_rasterizer(blitter, ctx->rast); util_blitter_save_viewport(blitter, &ctx->viewport); util_blitter_save_scissor(blitter, &ctx->scissor); diff --git a/src/gallium/drivers/asahi/agx_disk_cache.c b/src/gallium/drivers/asahi/agx_disk_cache.c index f33a971baed..e33a819098c 100644 --- a/src/gallium/drivers/asahi/agx_disk_cache.c +++ b/src/gallium/drivers/asahi/agx_disk_cache.c @@ -35,6 +35,8 @@ agx_disk_cache_compute_key(struct disk_cache *cache, int key_size; if (uncompiled->type == PIPE_SHADER_VERTEX) key_size = sizeof(shader_key->vs); + else if (uncompiled->type == PIPE_SHADER_GEOMETRY) + key_size = sizeof(shader_key->gs); else if (uncompiled->type == PIPE_SHADER_FRAGMENT) key_size = sizeof(shader_key->fs); else if (uncompiled->type == PIPE_SHADER_COMPUTE) @@ -66,6 +68,10 @@ agx_disk_cache_store(struct disk_cache *cache, if (!cache) return; + /* TODO: Support caching GS */ + if (uncompiled->type == PIPE_SHADER_GEOMETRY) + return; + assert(binary->bo->ptr.cpu != NULL && "shaders must be CPU mapped"); cache_key cache_key; @@ -100,6 +106,10 @@ agx_disk_cache_retrieve(struct agx_screen *screen, if (!cache) return NULL; + /* TODO: Support caching GS */ + if (uncompiled->type == PIPE_SHADER_GEOMETRY) + return NULL; + cache_key cache_key; agx_disk_cache_compute_key(cache, uncompiled, key, cache_key); diff --git a/src/gallium/drivers/asahi/agx_nir_lower_sysvals.c b/src/gallium/drivers/asahi/agx_nir_lower_sysvals.c index 08af274f074..a31ad2be5e8 100644 --- a/src/gallium/drivers/asahi/agx_nir_lower_sysvals.c +++ b/src/gallium/drivers/asahi/agx_nir_lower_sysvals.c @@ -153,18 +153,14 @@ lower_intrinsic(nir_builder *b, nir_intrinsic_instr *intr) intr->src[0].ssa); case nir_intrinsic_load_num_workgroups: return load_sysval(b, 3, 32, AGX_SYSVAL_TABLE_GRID, 0); + case nir_intrinsic_load_first_vertex: + return load_sysval(b, 1, 32, AGX_SYSVAL_TABLE_PARAMS, 0); + case nir_intrinsic_load_base_instance: + return load_sysval(b, 1, 32, AGX_SYSVAL_TABLE_PARAMS, 4); case nir_intrinsic_load_layer_id_written_agx: return load_sysval_root(b, 1, 16, &u->layer_id_written); - case nir_intrinsic_load_xfb_address: - return load_sysval_root(b, 1, 64, &u->xfb.base[nir_intrinsic_base(intr)]); - case nir_intrinsic_load_xfb_size: - return load_sysval_root(b, 1, 32, &u->xfb.size[nir_intrinsic_base(intr)]); - case nir_intrinsic_load_xfb_index_buffer: - return load_sysval_root(b, 1, 64, &u->xfb.index_buffer); - case nir_intrinsic_load_base_vertex: - return load_sysval_root(b, 1, 32, &u->xfb.base_vertex); - case nir_intrinsic_load_num_vertices: - return load_sysval_root(b, 1, 32, &u->xfb.num_vertices); + case nir_intrinsic_load_geometry_param_buffer_agx: + return load_sysval_root(b, 1, 64, &u->geometry_params); default: return NULL; } diff --git a/src/gallium/drivers/asahi/agx_state.c b/src/gallium/drivers/asahi/agx_state.c index df81abe2395..43b0bb6a0c4 100644 --- a/src/gallium/drivers/asahi/agx_state.c +++ b/src/gallium/drivers/asahi/agx_state.c @@ -35,6 +35,8 @@ #include "util/format/u_format.h" #include "util/format_srgb.h" #include "util/half_float.h" +#include "util/macros.h" +#include "util/u_dump.h" #include "util/u_inlines.h" #include "util/u_memory.h" #include "util/u_prim.h" @@ -43,6 +45,7 @@ #include "util/u_upload_mgr.h" #include "agx_device.h" #include "agx_disk_cache.h" +#include "agx_nir_lower_gs.h" #include "agx_tilebuffer.h" #include "pool.h" @@ -1415,6 +1418,18 @@ asahi_vs_shader_key_equal(const void *a, const void *b) return memcmp(a, b, sizeof(struct asahi_vs_shader_key)) == 0; } +static uint32_t +asahi_gs_shader_key_hash(const void *key) +{ + return _mesa_hash_data(key, sizeof(struct asahi_gs_shader_key)); +} + +static bool +asahi_gs_shader_key_equal(const void *a, const void *b) +{ + return memcmp(a, b, sizeof(struct asahi_gs_shader_key)) == 0; +} + static uint32_t asahi_fs_shader_key_hash(const void *key) { @@ -1562,20 +1577,61 @@ agx_link_varyings_vs_fs(struct agx_pool *pool, struct agx_varyings_vs *vs, return ptr.gpu; } -/* Does not take ownership of key. Clones if necessary. */ +/* + * Compile a NIR shader. The only lowering left at this point is sysvals. The + * shader key should have already been applied. agx_compile_variant may call + * this multiple times if there are auxiliary shaders. + */ static struct agx_compiled_shader * -agx_compile_variant(struct agx_device *dev, struct agx_uncompiled_shader *so, - struct util_debug_callback *debug, - union asahi_shader_key *key_) +agx_compile_nir(struct agx_device *dev, nir_shader *nir, + const struct agx_shader_key *base_key, + struct util_debug_callback *debug) { struct agx_compiled_shader *compiled = CALLOC_STRUCT(agx_compiled_shader); struct util_dynarray binary; util_dynarray_init(&binary, NULL); + struct agx_shader_key key = *base_key; + key.needs_g13x_coherency = (dev->params.gpu_generation == 13 && + dev->params.num_clusters_total > 1) || + dev->params.num_dies > 1; + key.libagx = dev->libagx; + + NIR_PASS_V(nir, agx_nir_lower_sysvals); + NIR_PASS_V(nir, agx_nir_layout_uniforms, compiled, &key.reserved_preamble); + + agx_compile_shader_nir(nir, &key, debug, &binary, &compiled->info); + + if (binary.size) { + compiled->bo = agx_bo_create(dev, binary.size, + AGX_BO_EXEC | AGX_BO_LOW_VA, "Executable"); + + memcpy(compiled->bo->ptr.cpu, binary.data, binary.size); + } + + util_dynarray_fini(&binary); + return compiled; +} + +/* Does not take ownership of key. Clones if necessary. */ +static struct agx_compiled_shader * +agx_compile_variant(struct agx_device *dev, struct pipe_context *pctx, + struct agx_uncompiled_shader *so, + struct util_debug_callback *debug, + union asahi_shader_key *key_, + struct agx_uncompiled_shader *linked_so) +{ struct blob_reader reader; blob_reader_init(&reader, so->serialized_nir.data, so->serialized_nir.size); nir_shader *nir = nir_deserialize(NULL, &agx_nir_options, &reader); + /* Auxiliary programs */ + enum mesa_prim gs_out_prim = MESA_PRIM_MAX; + unsigned gs_out_count_words = 0; + nir_shader *gs_count = NULL; + nir_shader *gs_copy = NULL; + nir_shader *pre_gs = NULL; + /* This can happen at inopportune times and cause jank, log it */ perf_debug(dev, "Compiling shader variant #%u", _mesa_hash_table_num_entries(so->variants)); @@ -1586,9 +1642,37 @@ agx_compile_variant(struct agx_device *dev, struct agx_uncompiled_shader *so, struct asahi_vs_shader_key *key = &key_->vs; NIR_PASS_V(nir, agx_nir_lower_vbo, &key->vbuf); + } else if (nir->info.stage == MESA_SHADER_GEOMETRY) { + struct asahi_gs_shader_key *key = &key_->gs; - if (key->xfb.active && nir->xfb_info != NULL) - NIR_PASS_V(nir, agx_nir_lower_xfb, &key->xfb); + /* XFB occurs for GS, not VS. TODO: Check if active. */ + if (nir->xfb_info != NULL) { + NIR_PASS_V(nir, nir_io_add_const_offset_to_base, + nir_var_shader_in | nir_var_shader_out); + NIR_PASS_V(nir, nir_io_add_intrinsic_xfb_info); + } + + struct blob_reader vs_reader; + blob_reader_init(&vs_reader, linked_so->serialized_nir.data, + linked_so->serialized_nir.size); + nir_shader *vs = nir_deserialize(NULL, &agx_nir_options, &vs_reader); + + /* Apply the VS key to the VS before linking it in */ + NIR_PASS_V(vs, agx_nir_lower_vbo, &key->vbuf); + NIR_PASS_V(vs, nir_lower_io_to_scalar, nir_var_shader_out, NULL, NULL); + + NIR_PASS_V(nir, nir_lower_io_to_scalar, nir_var_shader_out, NULL, NULL); + + /* Lower VS sysvals before it's merged in, so we access the correct shader + * stage for UBOs etc. + */ + NIR_PASS_V(vs, agx_nir_lower_sysvals); + + /* Link VS with GS */ + NIR_PASS_V(nir, agx_nir_lower_gs, vs, dev->libagx, &key->ia, + key->rasterizer_discard, &gs_count, &gs_copy, &pre_gs, + &gs_out_prim, &gs_out_count_words); + ralloc_free(vs); } else if (nir->info.stage == MESA_SHADER_FRAGMENT) { struct asahi_fs_shader_key *key = &key_->fs; @@ -1686,12 +1770,7 @@ agx_compile_variant(struct agx_device *dev, struct agx_uncompiled_shader *so, NIR_PASS_V(nir, agx_nir_predicate_layer_id); } - struct agx_shader_key base_key = { - .needs_g13x_coherency = (dev->params.gpu_generation == 13 && - dev->params.num_clusters_total > 1) || - dev->params.num_dies > 1, - .libagx = dev->libagx, - }; + struct agx_shader_key base_key = {0}; if (nir->info.stage == MESA_SHADER_FRAGMENT) base_key.fs.nr_samples = key_->fs.nr_samples; @@ -1701,46 +1780,59 @@ agx_compile_variant(struct agx_device *dev, struct agx_uncompiled_shader *so, base_key.vs.outputs_linear_shaded = key_->vs.outputs_linear_shaded; } - NIR_PASS_V(nir, agx_nir_lower_sysvals); - NIR_PASS_V(nir, agx_nir_layout_uniforms, compiled, - &base_key.reserved_preamble); - - agx_compile_shader_nir(nir, &base_key, debug, &binary, &compiled->info); + struct agx_compiled_shader *compiled = + agx_compile_nir(dev, nir, &base_key, debug); /* reads_tib => Translucent pass type */ compiled->info.reads_tib |= force_translucent; - /* Could be optimized to use non-translucent pass types with the appropriate - * HSR configuration, but that mechanism is not yet understood. Warn that - * we're leaving perf on the table when used. + /* Could be optimized to use non-translucent pass types with the + * appropriate HSR configuration, but that mechanism is not yet + * understood. Warn that we're leaving perf on the table when used. */ if (force_translucent) perf_debug(dev, "Translucency forced due to colour masking"); - if (binary.size) { - compiled->bo = agx_bo_create(dev, binary.size, - AGX_BO_EXEC | AGX_BO_LOW_VA, "Executable"); + /* Compile auxiliary programs */ + if (gs_count) + compiled->gs_count = agx_compile_nir(dev, gs_count, &base_key, debug); - memcpy(compiled->bo->ptr.cpu, binary.data, binary.size); + if (pre_gs) + compiled->pre_gs = agx_compile_nir(dev, pre_gs, &base_key, debug); + + if (gs_copy) { + const struct pipe_shader_state templ = { + .type = PIPE_SHADER_IR_NIR, + .ir.nir = gs_copy, + }; + + void *cso = pctx->create_vs_state(pctx, &templ); + struct agx_uncompiled_shader *cs = cso; + + compiled->gs_copy = cs; } - ralloc_free(nir); - util_dynarray_fini(&binary); + compiled->gs_output_mode = gs_out_prim; + compiled->gs_count_words = gs_out_count_words; + ralloc_free(nir); + ralloc_free(gs_count); return compiled; } static struct agx_compiled_shader * -agx_get_shader_variant(struct agx_screen *screen, +agx_get_shader_variant(struct agx_screen *screen, struct pipe_context *pctx, struct agx_uncompiled_shader *so, struct util_debug_callback *debug, - union asahi_shader_key *key) + union asahi_shader_key *key, + struct agx_uncompiled_shader *linked_so) { struct agx_compiled_shader *compiled = agx_disk_cache_retrieve(screen, so, key); if (!compiled) { - compiled = agx_compile_variant(&screen->dev, so, debug, key); + compiled = + agx_compile_variant(&screen->dev, pctx, so, debug, key, linked_so); agx_disk_cache_store(screen->disk_cache, so, key, compiled); } @@ -1754,6 +1846,8 @@ agx_get_shader_variant(struct agx_screen *screen, memcpy(cloned_key, key, sizeof(struct asahi_fs_shader_key)); } else if (so->type == PIPE_SHADER_VERTEX) { memcpy(cloned_key, key, sizeof(struct asahi_vs_shader_key)); + } else if (so->type == PIPE_SHADER_GEOMETRY) { + memcpy(cloned_key, key, sizeof(struct asahi_gs_shader_key)); } else { assert(gl_shader_stage_is_compute(so->type)); /* No key */ @@ -1773,6 +1867,9 @@ agx_shader_initialize(struct agx_device *dev, struct agx_uncompiled_shader *so, so->type = pipe_shader_type_from_mesa(nir->info.stage); + blob_init(&so->early_serialized_nir); + nir_serialize(&so->early_serialized_nir, nir, true); + nir_lower_robust_access_options robustness = { /* Images accessed through the texture or PBE hardware are robust, so we * don't set lower_image. However, buffer images and image atomics are @@ -1835,6 +1932,9 @@ agx_create_shader_state(struct pipe_context *pctx, if (nir->info.stage == MESA_SHADER_VERTEX) { so->variants = _mesa_hash_table_create(so, asahi_vs_shader_key_hash, asahi_vs_shader_key_equal); + } else if (nir->info.stage == MESA_SHADER_GEOMETRY) { + so->variants = _mesa_hash_table_create(NULL, asahi_gs_shader_key_hash, + asahi_gs_shader_key_equal); } else { so->variants = _mesa_hash_table_create(so, asahi_fs_shader_key_hash, asahi_fs_shader_key_equal); @@ -1865,6 +1965,11 @@ agx_create_shader_state(struct pipe_context *pctx, break; } + + case PIPE_SHADER_GEOMETRY: + /* TODO: Geometry shaders with shader-db */ + return so; + case PIPE_SHADER_FRAGMENT: key.fs.nr_cbufs = 1; key.fs.nr_samples = 1; @@ -1886,7 +1991,7 @@ agx_create_shader_state(struct pipe_context *pctx, unreachable("Unknown shader stage in shader-db precompile"); } - agx_compile_variant(dev, so, &pctx->debug, &key); + agx_compile_variant(dev, pctx, so, &pctx->debug, &key, NULL); } return so; @@ -1913,7 +2018,8 @@ agx_create_compute_state(struct pipe_context *pctx, nir_shader *nir = (void *)cso->prog; agx_shader_initialize(dev, so, nir, ctx->support_lod_bias); - agx_get_shader_variant(agx_screen(pctx->screen), so, &pctx->debug, &key); + agx_get_shader_variant(agx_screen(pctx->screen), pctx, so, &pctx->debug, + &key, NULL); /* We're done with the NIR, throw it away */ ralloc_free(nir); @@ -1925,8 +2031,8 @@ agx_get_compute_state_info(struct pipe_context *pctx, void *cso, struct pipe_compute_state_object_info *info) { union asahi_shader_key key = {0}; - struct agx_compiled_shader *so = - agx_get_shader_variant(agx_screen(pctx->screen), cso, &pctx->debug, &key); + struct agx_compiled_shader *so = agx_get_shader_variant( + agx_screen(pctx->screen), pctx, cso, &pctx->debug, &key, NULL); info->max_threads = agx_occupancy_for_register_count(so->info.nr_gprs).max_threads; @@ -1958,8 +2064,13 @@ agx_update_shader(struct agx_context *ctx, struct agx_compiled_shader **out, return true; } + struct agx_uncompiled_shader *linked_so = NULL; + if (stage == PIPE_SHADER_GEOMETRY) + linked_so = ctx->stage[PIPE_SHADER_VERTEX].shader; + struct agx_screen *screen = agx_screen(ctx->base.screen); - *out = agx_get_shader_variant(screen, so, &ctx->base.debug, key); + *out = agx_get_shader_variant(screen, &ctx->base, so, &ctx->base.debug, key, + linked_so); return true; } @@ -1969,7 +2080,6 @@ agx_update_vs(struct agx_context *ctx) /* Only proceed if the shader or anything the key depends on changes * * vb_mask, attributes, vertex_buffers: VERTEX - * streamout.active: XFB * outputs_{flat,linear}_shaded: FS_PROG */ if (!(ctx->dirty & (AGX_DIRTY_VS_PROG | AGX_DIRTY_VERTEX | AGX_DIRTY_XFB | @@ -1978,7 +2088,6 @@ agx_update_vs(struct agx_context *ctx) struct asahi_vs_shader_key key = { .vbuf.count = util_last_bit(ctx->vb_mask), - .xfb = ctx->streamout.key, .outputs_flat_shaded = ctx->stage[PIPE_SHADER_FRAGMENT].shader->info.inputs_flat_shaded, .outputs_linear_shaded = @@ -1992,6 +2101,69 @@ agx_update_vs(struct agx_context *ctx) (union asahi_shader_key *)&key); } +static enum mesa_prim +translate_ia_mode(enum mesa_prim prim) +{ + switch (prim) { + case MESA_PRIM_POINTS: + case MESA_PRIM_LINE_STRIP: + case MESA_PRIM_LINE_STRIP_ADJACENCY: + /* Every vertex defines a primitive */ + return MESA_PRIM_POINTS; + + default: + return prim; + } +} + +/* + * Triangle strips are rotated based on the provoking vertex, but other + * primitive types are not and do not need to know the provoking vertex. + */ +static bool +ia_needs_provoking(enum mesa_prim prim) +{ + return prim == MESA_PRIM_TRIANGLE_STRIP || + prim == MESA_PRIM_TRIANGLE_STRIP_ADJACENCY; +} + +static bool +agx_update_gs(struct agx_context *ctx, const struct pipe_draw_info *info) +{ + /* Only proceed if there is a geometry shader. Due to input assembly + * dependence, we don't bother to dirty track right now. + */ + if (!ctx->stage[PIPE_SHADER_GEOMETRY].shader) { + ctx->gs = NULL; + return false; + } + + /* XXX: Deduplicate this code from regular vertex */ + struct asahi_gs_shader_key key = { + .vbuf.count = util_last_bit(ctx->vb_mask), + + .ia.index_size = info->index_size, + .ia.mode = translate_ia_mode(info->mode), + .ia.flatshade_first = + ia_needs_provoking(info->mode) && ctx->rast->base.flatshade_first, + + .rasterizer_discard = ctx->rast->base.rasterizer_discard, + }; + + memcpy(key.vbuf.attributes, ctx->attributes, + sizeof(key.vbuf.attributes[0]) * AGX_MAX_ATTRIBS); + + static_assert(sizeof(key.input_nir_sha1) == + sizeof(ctx->stage[PIPE_SHADER_VERTEX].shader->nir_sha1), + "common size for shader sha-1"); + + memcpy(key.input_nir_sha1, ctx->stage[PIPE_SHADER_VERTEX].shader->nir_sha1, + sizeof(key.input_nir_sha1)); + + return agx_update_shader(ctx, &ctx->gs, PIPE_SHADER_GEOMETRY, + (union asahi_shader_key *)&key); +} + static bool agx_update_fs(struct agx_batch *batch) { @@ -2043,20 +2215,41 @@ agx_update_fs(struct agx_batch *batch) } static void -agx_bind_shader_state(struct pipe_context *pctx, void *cso) +agx_bind_shader_state(struct pipe_context *pctx, void *cso, + enum pipe_shader_type stage) { - if (!cso) - return; - struct agx_context *ctx = agx_context(pctx); - struct agx_uncompiled_shader *so = cso; - if (so->type == PIPE_SHADER_VERTEX) + if (stage == PIPE_SHADER_VERTEX) ctx->dirty |= AGX_DIRTY_VS_PROG; - else if (so->type == PIPE_SHADER_FRAGMENT) + else if (stage == PIPE_SHADER_FRAGMENT) ctx->dirty |= AGX_DIRTY_FS_PROG; - ctx->stage[so->type].shader = so; + ctx->stage[stage].shader = cso; +} + +static void +agx_bind_vs_state(struct pipe_context *pctx, void *cso) +{ + agx_bind_shader_state(pctx, cso, PIPE_SHADER_VERTEX); +} + +static void +agx_bind_fs_state(struct pipe_context *pctx, void *cso) +{ + agx_bind_shader_state(pctx, cso, PIPE_SHADER_FRAGMENT); +} + +static void +agx_bind_gs_state(struct pipe_context *pctx, void *cso) +{ + agx_bind_shader_state(pctx, cso, PIPE_SHADER_GEOMETRY); +} + +static void +agx_bind_cs_state(struct pipe_context *pctx, void *cso) +{ + agx_bind_shader_state(pctx, cso, PIPE_SHADER_COMPUTE); } static void @@ -2073,6 +2266,12 @@ agx_delete_shader_state(struct pipe_context *ctx, void *cso) struct agx_uncompiled_shader *so = cso; _mesa_hash_table_destroy(so->variants, agx_delete_compiled_shader); blob_finish(&so->serialized_nir); + + for (unsigned i = 0; i < ARRAY_SIZE(so->passthrough_progs); ++i) { + if (so->passthrough_progs[i]) + agx_delete_shader_state(ctx, so->passthrough_progs[i]); + } + ralloc_free(so); } @@ -2343,6 +2542,9 @@ agx_update_descriptors(struct agx_batch *batch, struct agx_compiled_shader *cs, { struct agx_context *ctx = batch->ctx; + if (!cs) + return; + if (ctx->stage[stage].dirty & AGX_STAGE_DIRTY_IMAGE) agx_upload_textures(batch, cs, stage); @@ -2724,6 +2926,7 @@ agx_point_object_type(struct agx_rasterizer *rast) } #define MAX_PPP_UPDATES 2 +#define IS_DIRTY(ST) !!(ctx->dirty & AGX_DIRTY_##ST) static uint8_t * agx_encode_state(struct agx_batch *batch, uint8_t *out, bool is_lines, @@ -2733,25 +2936,7 @@ agx_encode_state(struct agx_batch *batch, uint8_t *out, bool is_lines, struct agx_rasterizer *rast = ctx->rast; unsigned ppp_updates = 0; -#define IS_DIRTY(ST) !!(ctx->dirty & AGX_DIRTY_##ST) - - agx_update_descriptors(batch, ctx->vs, PIPE_SHADER_VERTEX); - agx_update_descriptors(batch, ctx->fs, PIPE_SHADER_FRAGMENT); - - if (IS_DIRTY(VERTEX)) { - agx_upload_vbos(batch); - } - - if (IS_DIRTY(BLEND_COLOR)) { - memcpy(batch->uniforms.blend_constant, &ctx->blend_color, - sizeof(ctx->blend_color)); - } - - if (IS_DIRTY(VS) || IS_DIRTY(FS) || IS_DIRTY(VERTEX) || - IS_DIRTY(BLEND_COLOR)) { - - agx_upload_uniforms(batch); - } + struct agx_compiled_shader *vs = ctx->vs, *fs = ctx->fs; if (IS_DIRTY(VS)) { agx_pack(out, VDM_STATE, cfg) { @@ -2763,23 +2948,22 @@ agx_encode_state(struct agx_batch *batch, uint8_t *out, bool is_lines, out += AGX_VDM_STATE_LENGTH; agx_pack(out, VDM_STATE_VERTEX_SHADER_WORD_0, cfg) { - cfg.uniform_register_count = ctx->vs->info.push_count; - cfg.preshader_register_count = ctx->vs->info.nr_preamble_gprs; + cfg.uniform_register_count = vs->info.push_count; + cfg.preshader_register_count = vs->info.nr_preamble_gprs; cfg.texture_state_register_count = - agx_nr_tex_descriptors(batch, PIPE_SHADER_VERTEX, ctx->vs); + agx_nr_tex_descriptors(batch, PIPE_SHADER_VERTEX, vs); cfg.sampler_state_register_count = - translate_sampler_state_count(ctx, ctx->vs, PIPE_SHADER_VERTEX); + translate_sampler_state_count(ctx, vs, PIPE_SHADER_VERTEX); } out += AGX_VDM_STATE_VERTEX_SHADER_WORD_0_LENGTH; agx_pack(out, VDM_STATE_VERTEX_SHADER_WORD_1, cfg) { - cfg.pipeline = - agx_build_pipeline(batch, ctx->vs, PIPE_SHADER_VERTEX, 0); + cfg.pipeline = agx_build_pipeline(batch, vs, PIPE_SHADER_VERTEX, 0); } out += AGX_VDM_STATE_VERTEX_SHADER_WORD_1_LENGTH; agx_pack(out, VDM_STATE_VERTEX_OUTPUTS, cfg) { - cfg.output_count_1 = ctx->vs->info.varyings.vs.nr_index; + cfg.output_count_1 = vs->info.varyings.vs.nr_index; cfg.output_count_2 = cfg.output_count_1; } out += AGX_VDM_STATE_VERTEX_OUTPUTS_LENGTH; @@ -2798,7 +2982,6 @@ agx_encode_state(struct agx_batch *batch, uint8_t *out, bool is_lines, } struct agx_pool *pool = &batch->pool; - struct agx_compiled_shader *vs = ctx->vs, *fs = ctx->fs; if ((ctx->dirty & AGX_DIRTY_RS) && ctx->rast->base.offset_tri) { agx_upload_depth_bias(batch, &ctx->rast->base); @@ -2815,7 +2998,7 @@ agx_encode_state(struct agx_batch *batch, uint8_t *out, bool is_lines, if (IS_DIRTY(VS_PROG) || IS_DIRTY(FS_PROG) || IS_DIRTY(RS)) { batch->varyings = agx_link_varyings_vs_fs( - &batch->pipeline_pool, &ctx->vs->info.varyings.vs, + &batch->pipeline_pool, &vs->info.varyings.vs, &ctx->fs->info.varyings.fs, ctx->rast->base.flatshade_first); varyings_dirty = true; @@ -2994,8 +3177,6 @@ agx_encode_state(struct agx_batch *batch, uint8_t *out, bool is_lines, agx_ppp_fini(&out, &ppp); ppp_updates++; -#undef IS_DIRTY - assert(ppp_updates <= MAX_PPP_UPDATES); return out; } @@ -3108,6 +3289,361 @@ agx_ensure_vdm_cmdbuf_has_space(struct agx_batch *batch, size_t space) batch->vdm.end = batch->vdm.current + size; } +static uint64_t +agx_allocate_geometry_count_buffer( + struct agx_batch *batch, const struct pipe_draw_info *info, + const struct pipe_draw_start_count_bias *draws) +{ + unsigned prim_per_instance = + u_decomposed_prims_for_vertices(info->mode, draws->count); + unsigned prims = prim_per_instance * info->instance_count; + + unsigned stride = batch->ctx->gs->gs_count_words * 4; + unsigned size = prims * stride; + + if (size) + return agx_pool_alloc_aligned(&batch->pool, size, 4).gpu; + else + return 0; +} + +static uint64_t +agx_batch_geometry_state(struct agx_batch *batch) +{ + if (!batch->geometry_state) { + struct agx_geometry_state state = { + .heap = agx_resource(batch->ctx->heap)->bo->ptr.gpu, + }; + + agx_batch_writes(batch, agx_resource(batch->ctx->heap)); + + batch->geometry_state = + agx_pool_upload_aligned(&batch->pool, &state, sizeof(state), 8); + } + + return batch->geometry_state; +} + +static uint64_t +agx_batch_geometry_params(struct agx_batch *batch, uint64_t input_index_buffer, + const struct pipe_draw_info *info, + const struct pipe_draw_start_count_bias *draw, + const struct pipe_draw_indirect_info *indirect) +{ + struct agx_geometry_params params = { + .state = agx_batch_geometry_state(batch), + .indirect_desc = batch->geom_indirect, + .input_index_buffer = input_index_buffer, + .index_size_B = info->index_size, + }; + + for (unsigned i = 0; i < ARRAY_SIZE(batch->ctx->streamout.targets); ++i) { + struct agx_streamout_target *so = + agx_so_target(batch->ctx->streamout.targets[i]); + struct agx_resource *rsrc = so ? agx_resource(so->offset) : NULL; + + uint32_t size; + params.xfb_base_original[i] = agx_batch_get_so_address(batch, i, &size); + params.xfb_size[i] = size; + + if (rsrc) { + params.xfb_offs_ptrs[i] = rsrc->bo->ptr.gpu; + agx_batch_writes(batch, rsrc); + } else { + params.xfb_offs_ptrs[i] = 0; + } + } + + /* TODO: Multiple streams! */ + if (batch->ctx->prims_generated) { + params.prims_generated_counter[0] = + agx_get_query_address(batch, batch->ctx->prims_generated); + } + + /* TODO: Multiple streams! */ + if (batch->ctx->tf_prims_generated) { + params.xfb_prims_generated_counter[0] = + agx_get_query_address(batch, batch->ctx->tf_prims_generated); + } + + /* Calculate input primitive count for direct draws, and allocate the count + * buffer. GPU calculates and allocates for indirect draws. + */ + unsigned count_buffer_stride = batch->ctx->gs->gs_count_words * 4; + + if (indirect) { + struct agx_resource *rsrc = agx_resource(indirect->buffer); + params.input_indirect_desc = rsrc->bo->ptr.gpu + indirect->offset; + params.count_buffer_stride = count_buffer_stride; + agx_batch_reads(batch, rsrc); + } else { + unsigned prim_per_instance = + u_decomposed_prims_for_vertices(info->mode, draw->count); + params.input_primitives = prim_per_instance * info->instance_count; + + unsigned size = params.input_primitives * count_buffer_stride; + + if (size) { + params.count_buffer = + agx_pool_alloc_aligned(&batch->pool, size, 4).gpu; + } + } + + return agx_pool_upload_aligned_with_bo(&batch->pool, ¶ms, sizeof(params), + 8, &batch->geom_params_bo); +} + +static void +agx_launch_gs(struct agx_batch *batch, const struct pipe_draw_info *info, + const struct pipe_draw_start_count_bias *draws, + const struct pipe_draw_indirect_info *indirect) +{ + struct agx_context *ctx = batch->ctx; + struct agx_device *dev = agx_device(ctx->base.screen); + struct agx_compiled_shader *gs = ctx->gs; + + perf_debug(dev, "Geometry shader or XFB"); + + /* This is a graphics batch, so it may not have had a CDM encoder allocated + * yet. Allocate that so we can start enqueueing compute work. + */ + if (!batch->cdm.bo) { + batch->cdm = agx_encoder_allocate(batch, dev); + } + + if (info->primitive_restart) { + fprintf(stderr, "Mode: %s\n", util_str_prim_mode(info->mode, true)); + unreachable("TODO: Primitive restart with GS"); + } + + /* Update after we're done touching uniforms */ + agx_upload_uniforms(batch); + + struct pipe_grid_info grid = {.block = {1, 1, 1}}; + struct agx_resource grid_indirect_rsrc = {.bo = batch->geom_params_bo}; + + if (indirect) { + assert(indirect->buffer && "drawauto already handled"); + + if (!ctx->gs_setup_indirect[info->mode]) { + struct agx_shader_key base_key = {0}; + + ctx->gs_setup_indirect[info->mode] = agx_compile_nir( + dev, agx_nir_gs_setup_indirect(dev->libagx, info->mode), &base_key, + NULL); + } + + const struct pipe_grid_info grid_1x1 = { + .block = {1, 1, 1}, + .grid = {1, 1, 1}, + }; + + agx_launch(batch, &grid_1x1, ctx->gs_setup_indirect[info->mode], + PIPE_SHADER_COMPUTE); + + /* Wrap the pool allocation in a fake resource for meta-Gallium use */ + assert(batch->geom_params_bo != NULL); + grid.indirect = &grid_indirect_rsrc.base; + grid.indirect_offset = + (batch->uniforms.geometry_params - grid_indirect_rsrc.bo->ptr.gpu) + + offsetof(struct agx_geometry_params, gs_grid); + } else { + unsigned prim_per_instance = + u_decomposed_prims_for_vertices(info->mode, draws->count); + + grid.grid[0] = prim_per_instance; + grid.grid[1] = info->instance_count; + grid.grid[2] = 1; + } + + /* If there is a count shader, launch it and prefix sum the results. */ + if (gs->gs_count) { + perf_debug(dev, "Geometry shader count"); + agx_launch(batch, &grid, gs->gs_count, PIPE_SHADER_GEOMETRY); + + assert(gs->gs_count_words < ARRAY_SIZE(ctx->gs_prefix_sums)); + if (!ctx->gs_prefix_sums[gs->gs_count_words]) { + struct agx_shader_key base_key = {0}; + + ctx->gs_prefix_sums[gs->gs_count_words] = agx_compile_nir( + dev, agx_nir_prefix_sum_gs(dev->libagx, gs->gs_count_words), + &base_key, NULL); + } + + agx_launch(batch, + &(const struct pipe_grid_info){ + .block = {32, gs->gs_count_words, 1}, + .grid = {1, 1, 1}, + }, + ctx->gs_prefix_sums[gs->gs_count_words], PIPE_SHADER_COMPUTE); + } + + /* Pre-GS shader */ + agx_launch(batch, + &(const struct pipe_grid_info){ + .block = {1, 1, 1}, + .grid = {1, 1, 1}, + }, + gs->pre_gs, PIPE_SHADER_COMPUTE); + + /* Launch the actual geometry shader */ + agx_launch(batch, &grid, gs, PIPE_SHADER_GEOMETRY); + batch->any_draws = true; + + /* Run a GS-less draw consuming those results */ + void *vs_cso = ctx->stage[PIPE_SHADER_VERTEX].shader; + void *gs_cso = ctx->stage[PIPE_SHADER_GEOMETRY].shader; + struct agx_query *prim_query = ctx->prims_generated; + + ctx->base.bind_vs_state(&ctx->base, gs->gs_copy); + ctx->base.bind_gs_state(&ctx->base, NULL); + ctx->prims_generated = NULL; + + bool indexed = gs->gs_output_mode != MESA_PRIM_POINTS; + + struct pipe_draw_info draw_info = { + .mode = gs->gs_output_mode, + .index_size = indexed ? 4 : 0, + .primitive_restart = indexed, + .restart_index = ~0, + .index.resource = ctx->heap, + .instance_count = 1, + .view_mask = info->view_mask, + }; + + /* Wrap the pool allocation in a fake resource for meta-Gallium use */ + struct agx_resource indirect_rsrc = {.bo = batch->geom_indirect_bo}; + struct pipe_draw_indirect_info copy_indirect = { + .draw_count = 1, + .buffer = &indirect_rsrc.base, + .offset = batch->geom_indirect - indirect_rsrc.bo->ptr.gpu, + }; + + ctx->base.draw_vbo(&ctx->base, &draw_info, 0, ©_indirect, NULL, 1); + + /* Restore state */ + ctx->base.bind_vs_state(&ctx->base, vs_cso); + ctx->base.bind_gs_state(&ctx->base, gs_cso); + ctx->prims_generated = prim_query; +} + +static bool +agx_needs_passthrough_gs(struct agx_context *ctx, + const struct pipe_draw_info *info) +{ + /* If there is already a geometry shader in the pipeline, we do not need to + * apply a passthrough GS of our own. + */ + if (ctx->stage[PIPE_SHADER_GEOMETRY].shader) + return false; + + /* Rendering adjacency requires a GS, add a passthrough since we don't have + * one. + */ + if (info->mode == MESA_PRIM_LINES_ADJACENCY || + info->mode == MESA_PRIM_TRIANGLES_ADJACENCY || + info->mode == MESA_PRIM_TRIANGLE_STRIP_ADJACENCY || + info->mode == MESA_PRIM_LINE_STRIP_ADJACENCY) { + perf_debug_ctx(ctx, "Using passthrough GS due to adjacency primitives"); + return true; + } + + /* Transform feedback is layered on geometry shaders, so if transform + * feedback is used, we need a GS. + */ + if (ctx->stage[PIPE_SHADER_VERTEX].shader->has_xfb_info && + ctx->streamout.num_targets) + return true; + + /* Otherwise, we don't need one */ + return false; +} + +static struct agx_uncompiled_shader * +agx_get_passthrough_gs(struct agx_context *ctx, + struct agx_uncompiled_shader *prev_cso, + enum mesa_prim mode) +{ + if (prev_cso->passthrough_progs[mode]) + return prev_cso->passthrough_progs[mode]; + + struct blob_reader reader; + blob_reader_init(&reader, prev_cso->early_serialized_nir.data, + prev_cso->early_serialized_nir.size); + nir_shader *prev = nir_deserialize(NULL, &agx_nir_options, &reader); + + nir_shader *gs = nir_create_passthrough_gs( + &agx_nir_options, prev, mode, mode, false /* emulate edge flags */, + false /* force line strip out */); + + ralloc_free(prev); + + const struct pipe_shader_state templ = { + .type = PIPE_SHADER_IR_NIR, + .ir.nir = gs, + }; + + struct agx_uncompiled_shader *cso = + ctx->base.create_gs_state(&ctx->base, &templ); + + prev_cso->passthrough_progs[mode] = cso; + return cso; +} + +static void +agx_apply_passthrough_gs(struct agx_context *ctx, + const struct pipe_draw_info *info, + unsigned drawid_offset, + const struct pipe_draw_indirect_info *indirect, + const struct pipe_draw_start_count_bias *draws, + unsigned num_draws) +{ + enum pipe_shader_type prev_stage = ctx->stage[PIPE_SHADER_TESS_EVAL].shader + ? PIPE_SHADER_TESS_EVAL + : PIPE_SHADER_VERTEX; + struct agx_uncompiled_shader *prev_cso = ctx->stage[prev_stage].shader; + + assert(ctx->stage[PIPE_SHADER_GEOMETRY].shader == NULL); + + /* We can split XFB from main vertex shader to avoid GS costs, at the expense + * of additional VS invocations. This probably needs tuning for perf... + * slight hit to Manhattan but eliminates spilling in the CTS. + */ + bool split_xfb = (ctx->stage[PIPE_SHADER_VERTEX].shader->has_xfb_info && + ctx->streamout.num_targets); + + struct agx_rasterizer *saved_rast = ctx->rast; + if (split_xfb) { + ctx->base.bind_rasterizer_state( + &ctx->base, util_blitter_get_discard_rasterizer_state(ctx->blitter)); + } + + /* Draw with passthrough */ + ctx->base.bind_gs_state(&ctx->base, + agx_get_passthrough_gs(ctx, prev_cso, info->mode)); + ctx->base.draw_vbo(&ctx->base, info, drawid_offset, indirect, draws, + num_draws); + ctx->base.bind_gs_state(&ctx->base, NULL); + + /* Draw without XFB */ + if (split_xfb) { + unsigned saved_targets = ctx->streamout.num_targets; + struct agx_query *saved_prims_generated = ctx->prims_generated; + + ctx->base.bind_rasterizer_state(&ctx->base, saved_rast); + ctx->streamout.num_targets = 0; + ctx->prims_generated = NULL; + + if (!saved_rast->base.rasterizer_discard) { + ctx->base.draw_vbo(&ctx->base, info, drawid_offset, indirect, draws, + num_draws); + } + + ctx->streamout.num_targets = saved_targets; + ctx->prims_generated = saved_prims_generated; + } +} + static void agx_draw_vbo(struct pipe_context *pctx, const struct pipe_draw_info *info, unsigned drawid_offset, @@ -3129,24 +3665,29 @@ agx_draw_vbo(struct pipe_context *pctx, const struct pipe_draw_info *info, return; } - bool has_xfb_info = ctx->stage[PIPE_SHADER_VERTEX].shader->has_xfb_info; - bool uses_xfb = has_xfb_info && ctx->streamout.num_targets; - bool uses_prims_generated = ctx->active_queries && ctx->prims_generated; + if (agx_needs_passthrough_gs(ctx, info)) { + agx_apply_passthrough_gs(ctx, info, drawid_offset, indirect, draws, + num_draws); + return; + } - if (indirect && (uses_prims_generated || uses_xfb)) { - perf_debug_ctx(ctx, "Emulating indirect draw due to XFB"); + bool uses_prims_generated = ctx->active_queries && ctx->prims_generated; + bool uses_gs = ctx->stage[PIPE_SHADER_GEOMETRY].shader; + + if (indirect && (uses_prims_generated)) { + perf_debug_ctx(ctx, "Emulating indirect draw due to query"); util_draw_indirect(pctx, info, indirect); return; } - if (uses_xfb && info->primitive_restart) { - perf_debug_ctx(ctx, "Emulating primitive restart due to XFB"); + if (uses_gs && info->primitive_restart) { + perf_debug_ctx(ctx, "Emulating primitive restart due to GS"); util_draw_vbo_without_prim_restart(pctx, info, drawid_offset, indirect, draws); return; } - if (!ctx->streamout.key.active && uses_prims_generated) { + if (uses_prims_generated && !ctx->stage[PIPE_SHADER_GEOMETRY].shader) { agx_primitives_update_direct(ctx, info, draws); } @@ -3162,9 +3703,6 @@ agx_draw_vbo(struct pipe_context *pctx, const struct pipe_draw_info *info, ib = agx_index_buffer_direct_ptr(batch, draws, info, &ib_extent); } - if (uses_xfb) - agx_launch_so(pctx, info, draws, ib); - #ifndef NDEBUG if (unlikely(agx_device(pctx->screen)->debug & AGX_DBG_DIRTY)) agx_dirty_all(ctx); @@ -3176,13 +3714,110 @@ agx_draw_vbo(struct pipe_context *pctx, const struct pipe_draw_info *info, /* We don't support side effects in vertex stages (only used internally for * transform feedback lowering), so this is trivial. */ - if (ctx->rast->base.rasterizer_discard && !ctx->streamout.key.active) + if (ctx->rast->base.rasterizer_discard && + !ctx->stage[PIPE_SHADER_GEOMETRY].shader) return; agx_batch_init_state(batch); + /* Update shaders first so we can use them after */ + if (agx_update_vs(ctx)) { + ctx->dirty |= AGX_DIRTY_VS | AGX_DIRTY_VS_PROG; + ctx->stage[PIPE_SHADER_VERTEX].dirty = ~0; + } else if (ctx->stage[PIPE_SHADER_VERTEX].dirty || + (ctx->dirty & AGX_DIRTY_VERTEX)) + ctx->dirty |= AGX_DIRTY_VS; + + agx_update_gs(ctx, info); + + if (ctx->gs) { + batch->geom_indirect = agx_pool_alloc_aligned_with_bo( + &batch->pool, 64, 4, &batch->geom_indirect_bo) + .gpu; + + if (!ctx->heap) { + ctx->heap = pipe_buffer_create(ctx->base.screen, PIPE_BIND_GLOBAL, + PIPE_USAGE_DEFAULT, 1024 * 1024 * 128); + } + + batch->uniforms.geometry_params = + agx_batch_geometry_params(batch, ib, info, draws, indirect); + } + + struct agx_compiled_shader *vs = ctx->vs; + batch->uniforms.layer_id_written = vs->info.writes_layer_viewport ? ~0 : 0; + + if (agx_update_fs(batch)) { + ctx->dirty |= AGX_DIRTY_FS | AGX_DIRTY_FS_PROG; + ctx->stage[PIPE_SHADER_FRAGMENT].dirty = ~0; + } else if ((ctx->stage[PIPE_SHADER_FRAGMENT].dirty) || + (ctx->dirty & (AGX_DIRTY_BLEND_COLOR | AGX_DIRTY_SAMPLE_MASK))) { + ctx->dirty |= AGX_DIRTY_FS; + } + + agx_batch_add_bo(batch, ctx->vs->bo); + agx_batch_add_bo(batch, ctx->fs->bo); + + /* If a GS is active, the mode and index buffer come from the GS output */ + enum mesa_prim mode = info->mode; + + if (indirect) { + struct agx_resource *indirect_rsrc = agx_resource(indirect->buffer); + uint64_t address = indirect_rsrc->bo->ptr.gpu + indirect->offset; + agx_batch_reads(batch, indirect_rsrc); + + /* To implement draw parameters, we use the last 2 words of the indirect + * draw descriptor. Offset by 3 words for indexed draw (5 total) and 2 + * words for non-indexed (4 total). See the layouts of indexed vs + * non-indexed draw descriptors. + * + * This gives us a consistent layout + * + * uint32_t first_vertex; + * uint32_t base_instance; + * + * and we can implement load_first_vertex & load_base_instance without + * checking for indexing. + */ + uint32_t offset = idx_size ? 3 : 2; + batch->uniforms.tables[AGX_SYSVAL_TABLE_PARAMS] = address + offset * 4; + } else { + /* Upload just those two words. */ + uint32_t params[2] = { + idx_size ? draws->index_bias : draws->start, + info->start_instance, + }; + + batch->uniforms.tables[AGX_SYSVAL_TABLE_PARAMS] = + agx_pool_upload_aligned(&batch->pool, params, sizeof(params), 4); + } + + agx_update_descriptors(batch, ctx->vs, PIPE_SHADER_VERTEX); + agx_update_descriptors(batch, ctx->gs, PIPE_SHADER_GEOMETRY); + agx_update_descriptors(batch, ctx->fs, PIPE_SHADER_FRAGMENT); + + if (IS_DIRTY(VERTEX) || true) { + agx_upload_vbos(batch); + } + + if (IS_DIRTY(BLEND_COLOR)) { + memcpy(batch->uniforms.blend_constant, &ctx->blend_color, + sizeof(ctx->blend_color)); + } + + if (IS_DIRTY(VS) || IS_DIRTY(FS) || ctx->gs || IS_DIRTY(VERTEX) || + IS_DIRTY(BLEND_COLOR)) { + + agx_upload_uniforms(batch); + } + + if (ctx->gs) { + agx_launch_gs(batch, info, draws, indirect); + return; + } + /* Dirty track the reduced prim: lines vs points vs triangles */ - enum mesa_prim reduced_prim = u_reduced_prim(info->mode); + enum mesa_prim reduced_prim = u_reduced_prim(mode); if (reduced_prim != batch->reduced_prim) ctx->dirty |= AGX_DIRTY_PRIM; batch->reduced_prim = reduced_prim; @@ -3200,28 +3835,6 @@ agx_draw_vbo(struct pipe_context *pctx, const struct pipe_draw_info *info, batch->resolve |= ctx->zs->store; } - if (agx_update_vs(ctx)) { - ctx->dirty |= AGX_DIRTY_VS | AGX_DIRTY_VS_PROG; - ctx->stage[PIPE_SHADER_VERTEX].dirty = ~0; - } else if (ctx->stage[PIPE_SHADER_VERTEX].dirty || - (ctx->dirty & AGX_DIRTY_VERTEX)) { - ctx->dirty |= AGX_DIRTY_VS; - } - - struct agx_compiled_shader *vs = ctx->vs; - batch->uniforms.layer_id_written = vs->info.writes_layer_viewport ? ~0 : 0; - - if (agx_update_fs(batch)) { - ctx->dirty |= AGX_DIRTY_FS | AGX_DIRTY_FS_PROG; - ctx->stage[PIPE_SHADER_FRAGMENT].dirty = ~0; - } else if (ctx->stage[PIPE_SHADER_FRAGMENT].dirty || - (ctx->dirty & (AGX_DIRTY_BLEND_COLOR | AGX_DIRTY_SAMPLE_MASK))) { - ctx->dirty |= AGX_DIRTY_FS; - } - - agx_batch_add_bo(batch, ctx->vs->bo); - agx_batch_add_bo(batch, ctx->fs->bo); - /* When we approach the end of a command buffer, cycle it out for a new one. * We only need to do this once per draw as long as we conservatively * estimate the maximum bytes of VDM commands that this draw will emit. @@ -3243,14 +3856,14 @@ agx_draw_vbo(struct pipe_context *pctx, const struct pipe_draw_info *info, reduced_prim == MESA_PRIM_LINES, reduced_prim == MESA_PRIM_POINTS); - enum agx_primitive prim = agx_primitive_for_pipe(info->mode); + enum agx_primitive prim = agx_primitive_for_pipe(mode); if (idx_size) { agx_pack(out, VDM_STATE, cfg) cfg.restart_index_present = true; out += AGX_VDM_STATE_LENGTH; agx_pack(out, VDM_STATE_RESTART_INDEX, cfg) { - cfg.value = info->restart_index; + cfg.value = ctx->gs ? BITFIELD_MASK(32) : info->restart_index; } out += AGX_VDM_STATE_RESTART_INDEX_LENGTH; } @@ -3325,29 +3938,6 @@ agx_draw_vbo(struct pipe_context *pctx, const struct pipe_draw_info *info, out += AGX_INDEX_LIST_BUFFER_SIZE_LENGTH; } - /* Insert a memory barrier after transform feedback so the result may be - * consumed by a subsequent vertex shader. - */ - if (ctx->streamout.key.active) { - struct agx_device *dev = agx_device(pctx->screen); - agx_pack(out, VDM_BARRIER, cfg) { - cfg.unk_5 = true; - cfg.unk_6 = true; - cfg.unk_8 = true; - cfg.unk_11 = true; - cfg.unk_20 = true; - if (dev->params.num_clusters_total > 1) { - cfg.unk_24 = true; - if (dev->params.gpu_generation == 13) { - cfg.unk_4 = true; - cfg.unk_26 = true; - } - } - } - - out += AGX_VDM_BARRIER_LENGTH; - } - batch->vdm.current = out; assert((batch->vdm.current + AGX_VDM_STREAM_LINK_LENGTH) <= batch->vdm.end && "Failed to reserve sufficient space in encoder"); @@ -3386,7 +3976,7 @@ agx_texture_barrier(struct pipe_context *pipe, unsigned flags) void agx_launch(struct agx_batch *batch, const struct pipe_grid_info *info, - struct agx_compiled_shader *cs) + struct agx_compiled_shader *cs, enum pipe_shader_type stage) { struct agx_context *ctx = batch->ctx; struct agx_device *dev = agx_device(ctx->base.screen); @@ -3434,11 +4024,11 @@ agx_launch(struct agx_batch *batch, const struct pipe_grid_info *info, cfg.uniform_register_count = cs->info.push_count; cfg.preshader_register_count = cs->info.nr_preamble_gprs; cfg.texture_state_register_count = - agx_nr_tex_descriptors(batch, PIPE_SHADER_COMPUTE, cs); + agx_nr_tex_descriptors(batch, stage, cs); cfg.sampler_state_register_count = - translate_sampler_state_count(ctx, cs, PIPE_SHADER_COMPUTE); - cfg.pipeline = agx_build_pipeline(batch, cs, PIPE_SHADER_COMPUTE, - info->variable_shared_mem); + translate_sampler_state_count(ctx, cs, stage); + cfg.pipeline = + agx_build_pipeline(batch, cs, stage, info->variable_shared_mem); } out += AGX_CDM_HEADER_LENGTH; @@ -3501,7 +4091,7 @@ agx_launch_grid(struct pipe_context *pipe, const struct pipe_grid_info *info) struct agx_compiled_shader *cs = _mesa_hash_table_next_entry(uncompiled->variants, NULL)->data; - agx_launch(batch, info, cs); + agx_launch(batch, info, cs, PIPE_SHADER_COMPUTE); /* TODO: Dirty tracking? */ @@ -3566,15 +4156,17 @@ agx_init_state_functions(struct pipe_context *ctx) ctx->create_surface = agx_create_surface; ctx->create_vertex_elements_state = agx_create_vertex_elements; ctx->create_vs_state = agx_create_shader_state; + ctx->create_gs_state = agx_create_shader_state; ctx->create_compute_state = agx_create_compute_state; ctx->bind_blend_state = agx_bind_blend_state; ctx->bind_depth_stencil_alpha_state = agx_bind_zsa_state; ctx->bind_sampler_states = agx_bind_sampler_states; - ctx->bind_fs_state = agx_bind_shader_state; + ctx->bind_fs_state = agx_bind_fs_state; ctx->bind_rasterizer_state = agx_bind_rasterizer_state; ctx->bind_vertex_elements_state = agx_bind_vertex_elements_state; - ctx->bind_vs_state = agx_bind_shader_state; - ctx->bind_compute_state = agx_bind_shader_state; + ctx->bind_vs_state = agx_bind_vs_state; + ctx->bind_gs_state = agx_bind_gs_state; + ctx->bind_compute_state = agx_bind_cs_state; ctx->delete_blend_state = agx_delete_state; ctx->delete_depth_stencil_alpha_state = agx_delete_state; ctx->delete_fs_state = agx_delete_shader_state; @@ -3583,6 +4175,7 @@ agx_init_state_functions(struct pipe_context *ctx) ctx->delete_sampler_state = agx_delete_sampler_state; ctx->delete_vertex_elements_state = agx_delete_state; ctx->delete_vs_state = agx_delete_shader_state; + ctx->delete_gs_state = agx_delete_shader_state; ctx->set_blend_color = agx_set_blend_color; ctx->set_clip_state = agx_set_clip_state; ctx->set_constant_buffer = agx_set_constant_buffer; diff --git a/src/gallium/drivers/asahi/agx_state.h b/src/gallium/drivers/asahi/agx_state.h index e765c6b8e6c..626510f8687 100644 --- a/src/gallium/drivers/asahi/agx_state.h +++ b/src/gallium/drivers/asahi/agx_state.h @@ -15,6 +15,7 @@ #include "asahi/lib/agx_pack.h" #include "asahi/lib/agx_tilebuffer.h" #include "asahi/lib/pool.h" +#include "asahi/lib/shaders/geometry.h" #include "compiler/nir/nir_lower_blend.h" #include "compiler/shader_enums.h" #include "gallium/auxiliary/util/u_blitter.h" @@ -39,7 +40,7 @@ struct agx_streamout_target { struct pipe_stream_output_target base; - uint32_t offset; + struct pipe_resource *offset; }; static inline struct agx_streamout_target * @@ -48,37 +49,9 @@ agx_so_target(struct pipe_stream_output_target *target) return (struct agx_streamout_target *)target; } -struct agx_xfb_key { - /* If true, compiles a "transform feedback" program instead of a vertex - * shader. This is a kernel that runs on the VDM and writes out the transform - * feedback buffers, with no rasterization. - */ - bool active; - - /* The index size (1, 2, 4) or 0 if drawing without an index buffer. */ - uint8_t index_size; - - /* The primitive mode for unrolling the vertex ID */ - enum mesa_prim mode; - - /* Use first vertex as the provoking vertex for flat shading */ - bool flatshade_first; -}; - -struct agx_xfb_params { - uint64_t base[PIPE_MAX_SO_BUFFERS]; - uint32_t size[PIPE_MAX_SO_BUFFERS]; - uint64_t index_buffer; - uint32_t base_vertex; - uint32_t num_vertices; -}; - struct agx_streamout { struct pipe_stream_output_target *targets[PIPE_MAX_SO_BUFFERS]; unsigned num_targets; - - struct agx_xfb_key key; - struct agx_xfb_params params; }; /* Shaders can access fixed-function state through system values. @@ -92,6 +65,7 @@ struct agx_streamout { */ enum agx_sysval_table { AGX_SYSVAL_TABLE_ROOT, + AGX_SYSVAL_TABLE_PARAMS, AGX_SYSVAL_TABLE_GRID, AGX_SYSVAL_TABLE_VS, AGX_SYSVAL_TABLE_TCS, @@ -125,8 +99,8 @@ struct PACKED agx_draw_uniforms { /* Vertex buffer object bases, if present */ uint64_t vbo_base[PIPE_MAX_ATTRIBS]; - /* Transform feedback info for a transform feedback shader */ - struct agx_xfb_params xfb; + /* Address of geometry param buffer if geometry shaders are used, else 0 */ + uint64_t geometry_params; /* Blend constant if any */ float blend_constant[4]; @@ -195,15 +169,27 @@ struct agx_compiled_shader { /* Uniforms the driver must push */ unsigned push_range_count; struct agx_push_range push[AGX_MAX_PUSH_RANGES]; + + /* Auxiliary programs, or NULL if not used */ + struct agx_compiled_shader *gs_count, *pre_gs; + struct agx_uncompiled_shader *gs_copy; + + /* Output primitive mode for geometry shaders */ + enum mesa_prim gs_output_mode; + + /* Number of words per primitive in the count buffer */ + unsigned gs_count_words; }; struct agx_uncompiled_shader { struct pipe_shader_state base; enum pipe_shader_type type; + struct blob early_serialized_nir; struct blob serialized_nir; uint8_t nir_sha1[20]; struct agx_uncompiled_shader_info info; struct hash_table *variants; + struct agx_uncompiled_shader *passthrough_progs[MESA_PRIM_COUNT]; bool has_xfb_info; /* Whether the shader accesses indexed samplers via the bindless heap */ @@ -293,6 +279,13 @@ struct agx_batch { struct agx_draw_uniforms uniforms; + /* Indirect buffer allocated for geometry shader */ + uint64_t geom_indirect; + struct agx_bo *geom_indirect_bo; + + /* Geometry state buffer if geometry/etc shaders are used */ + uint64_t geometry_state; + /* Uploaded descriptors */ uint64_t textures[PIPE_SHADER_TYPES]; uint32_t texture_count[PIPE_SHADER_TYPES]; @@ -333,6 +326,9 @@ struct agx_batch { /* Result buffer where the kernel places command execution information */ union agx_batch_result *result; size_t result_off; + + /* Actual pointer in a uniform */ + struct agx_bo *geom_params_bo; }; struct agx_zsa { @@ -357,7 +353,6 @@ struct agx_blend { struct asahi_vs_shader_key { struct agx_vbufs vbuf; - struct agx_xfb_key xfb; uint64_t outputs_flat_shaded; uint64_t outputs_linear_shaded; }; @@ -381,8 +376,26 @@ struct asahi_fs_shader_key { enum pipe_format rt_formats[PIPE_MAX_COLOR_BUFS]; }; +struct asahi_gs_shader_key { + /* Input assembly key */ + struct agx_ia_key ia; + + /* Vertex shader key */ + struct agx_vbufs vbuf; + + /* If true, this GS is run only for its side effects (including XFB) */ + bool rasterizer_discard; + + /* Geometry shaders must be linked with a vertex shader. In a monolithic + * pipeline, this is the vertex shader (or tessellation evaluation shader). + * With separate shaders, this needs to be an internal passthrough program. + */ + uint8_t input_nir_sha1[20]; +}; + union asahi_shader_key { struct asahi_vs_shader_key vs; + struct asahi_gs_shader_key gs; struct asahi_fs_shader_key fs; }; @@ -419,9 +432,12 @@ enum agx_dirty { struct agx_context { struct pipe_context base; - struct agx_compiled_shader *vs, *fs; + struct agx_compiled_shader *vs, *fs, *gs; uint32_t dirty; + /* Heap for dynamic memory allocation for geometry/tessellation shaders */ + struct pipe_resource *heap; + /* Acts as a context-level shader key */ bool support_lod_bias; @@ -480,6 +496,8 @@ struct agx_context { /* Bound CL global buffers */ struct util_dynarray global_buffers; + struct agx_compiled_shader *gs_prefix_sums[16]; + struct agx_compiled_shader *gs_setup_indirect[MESA_PRIM_MAX]; struct agx_meta_cache meta; uint32_t syncobj; @@ -541,7 +559,7 @@ agx_context(struct pipe_context *pctx) } void agx_launch(struct agx_batch *batch, const struct pipe_grid_info *info, - struct agx_compiled_shader *cs); + struct agx_compiled_shader *cs, enum pipe_shader_type stage); void agx_init_query_functions(struct pipe_context *ctx); @@ -550,17 +568,11 @@ agx_primitives_update_direct(struct agx_context *ctx, const struct pipe_draw_info *info, const struct pipe_draw_start_count_bias *draw); -void agx_nir_lower_xfb(nir_shader *shader, struct agx_xfb_key *key); - void agx_draw_vbo_from_xfb(struct pipe_context *pctx, const struct pipe_draw_info *info, unsigned drawid_offset, const struct pipe_draw_indirect_info *indirect); -void agx_launch_so(struct pipe_context *pctx, const struct pipe_draw_info *info, - const struct pipe_draw_start_count_bias *draws, - uint64_t index_buffer); - uint64_t agx_batch_get_so_address(struct agx_batch *batch, unsigned buffer, uint32_t *size); @@ -770,6 +782,14 @@ bool agx_nir_layout_uniforms(nir_shader *shader, bool agx_nir_lower_bindings(nir_shader *shader, bool *uses_bindless_samplers); +void agx_nir_lower_gs(nir_shader *gs, nir_shader *input_shader, + const nir_shader *libagx, struct agx_ia_key *ia, + bool rasterizer_discard, nir_shader **gs_count, + nir_shader **gs_copy, nir_shader **pre_gs, + enum mesa_prim *out_mode, unsigned *out_count_words); + +nir_shader *agx_nir_prefix_sum_gs(const nir_shader *libagx, unsigned words); + bool agx_batch_is_active(struct agx_batch *batch); bool agx_batch_is_submitted(struct agx_batch *batch); diff --git a/src/gallium/drivers/asahi/agx_streamout.c b/src/gallium/drivers/asahi/agx_streamout.c index 23b778cc2b9..85452b62f62 100644 --- a/src/gallium/drivers/asahi/agx_streamout.c +++ b/src/gallium/drivers/asahi/agx_streamout.c @@ -6,8 +6,10 @@ #include "compiler/nir/nir_builder.h" #include "compiler/nir/nir_xfb_info.h" +#include "pipe/p_defines.h" #include "util/u_draw.h" #include "util/u_dump.h" +#include "util/u_inlines.h" #include "util/u_prim.h" #include "agx_state.h" @@ -16,21 +18,24 @@ agx_create_stream_output_target(struct pipe_context *pctx, struct pipe_resource *prsc, unsigned buffer_offset, unsigned buffer_size) { - struct pipe_stream_output_target *target; - - target = &rzalloc(pctx, struct agx_streamout_target)->base; + struct agx_streamout_target *target = + rzalloc(pctx, struct agx_streamout_target); if (!target) return NULL; - pipe_reference_init(&target->reference, 1); - pipe_resource_reference(&target->buffer, prsc); + pipe_reference_init(&target->base.reference, 1); + pipe_resource_reference(&target->base.buffer, prsc); - target->context = pctx; - target->buffer_offset = buffer_offset; - target->buffer_size = buffer_size; + target->base.context = pctx; + target->base.buffer_offset = buffer_offset; + target->base.buffer_size = buffer_size; - return target; + uint32_t zero = 0; + target->offset = pipe_buffer_create_with_data(pctx, PIPE_BIND_GLOBAL, + PIPE_USAGE_DEFAULT, 4, &zero); + + return &target->base; } static void @@ -62,8 +67,10 @@ agx_set_stream_output_targets(struct pipe_context *pctx, unsigned num_targets, * Gallium contract and it will work out fine. Probably should be * redefined to be ~0 instead of -1 but it doesn't really matter. */ - if (offsets[i] != -1) - agx_so_target(targets[i])->offset = offsets[i]; + if (offsets[i] != -1) { + pipe_buffer_write(pctx, agx_so_target(targets[i])->offset, 0, 4, + &offsets[i]); + } pipe_so_target_reference(&so->targets[i], targets[i]); } @@ -100,18 +107,11 @@ agx_batch_get_so_address(struct agx_batch *batch, unsigned buffer, } /* Otherwise, write the target */ - struct pipe_stream_output_info *so = - &batch->ctx->stage[PIPE_SHADER_VERTEX].shader->base.stream_output; - struct agx_resource *rsrc = agx_resource(target->buffer); agx_batch_writes(batch, rsrc); - /* The amount of space left depends how much we've already consumed */ - unsigned stride = so->stride[buffer] * 4; - uint32_t offset = agx_so_target(target)->offset * stride; - - *size = offset < target->buffer_size ? (target->buffer_size - offset) : 0; - return rsrc->bo->ptr.gpu + target->buffer_offset + offset; + *size = target->buffer_size; + return rsrc->bo->ptr.gpu + target->buffer_offset; } void @@ -119,9 +119,18 @@ agx_draw_vbo_from_xfb(struct pipe_context *pctx, const struct pipe_draw_info *info, unsigned drawid_offset, const struct pipe_draw_indirect_info *indirect) { + perf_debug_ctx(agx_context(pctx), "draw auto"); + + unsigned count; + pipe_buffer_read(pctx, + agx_so_target(indirect->count_from_stream_output)->offset, + 0, 4, &count); + + /* XXX: Probably need to divide here */ + struct pipe_draw_start_count_bias draw = { .start = 0, - .count = agx_so_target(indirect->count_from_stream_output)->offset, + .count = count, }; pctx->draw_vbo(pctx, info, drawid_offset, NULL, &draw, 1); @@ -142,114 +151,6 @@ xfb_prims_for_vertices(enum mesa_prim mode, unsigned verts) return prims; } -/* - * Launch a streamout pipeline. - */ -void -agx_launch_so(struct pipe_context *pctx, const struct pipe_draw_info *info, - const struct pipe_draw_start_count_bias *draw, - uint64_t index_buffer) -{ - struct agx_context *ctx = agx_context(pctx); - - /* Break recursion from draw_vbo creating draw calls below: Do not do a - * streamout draw for a streamout draw. - */ - if (ctx->streamout.key.active) - return; - - /* Configure the below draw to launch streamout rather than a regular draw */ - ctx->streamout.key.active = true; - ctx->dirty |= AGX_DIRTY_XFB; - - ctx->streamout.key.index_size = info->index_size; - ctx->streamout.key.mode = info->mode; - ctx->streamout.key.flatshade_first = ctx->rast->base.flatshade_first; - ctx->streamout.params.index_buffer = index_buffer; - - /* Ignore provoking vertex for modes that don't depend on the provoking - * vertex, to reduce shader variants. - */ - if (info->mode != MESA_PRIM_TRIANGLE_STRIP) - ctx->streamout.key.flatshade_first = false; - - /* Determine how many vertices are XFB there will be */ - unsigned num_outputs = - u_stream_outputs_for_vertices(info->mode, draw->count); - unsigned count = draw->count; - u_trim_pipe_prim(info->mode, &count); - - ctx->streamout.params.base_vertex = - info->index_size ? draw->index_bias : draw->start; - ctx->streamout.params.num_vertices = count; - - /* Streamout runs as a vertex shader with rasterizer discard */ - void *saved_rast = ctx->rast; - pctx->bind_rasterizer_state( - pctx, util_blitter_get_discard_rasterizer_state(ctx->blitter)); - - /* Dispatch a grid of points, this is compute-like */ - util_draw_arrays_instanced(pctx, MESA_PRIM_POINTS, 0, num_outputs, 0, - info->instance_count); - pctx->bind_rasterizer_state(pctx, saved_rast); - - /* - * Finally, if needed, update the counter of primitives written. The spec - * requires: - * - * If recording the vertices of a primitive to the buffer objects being - * used for transform feedback purposes would result in [overflow]... - * the counter corresponding to the asynchronous query target - * TRANSFORM_FEEDBACK_PRIMITIVES_WRITTEN (see section 13.4) is not - * incremented. - * - * So clamp the number of primitives generated to the number of primitives - * we actually have space to write. - */ - if (ctx->tf_prims_generated) { - uint32_t min_max = ~0; - - for (unsigned i = 0; i < ctx->streamout.num_targets; ++i) { - struct pipe_stream_output_target *target = get_target(ctx, i); - - if (!target) - continue; - - struct pipe_stream_output_info *so = - &ctx->stage[PIPE_SHADER_VERTEX].shader->base.stream_output; - unsigned stride = so->stride[i] * 4; - - /* Ignore spurious targets. I don't see anything in the Gallium - * contract specifically forbidding this. - */ - if (stride == 0) - continue; - - uint32_t offset = agx_so_target(target)->offset * stride; - uint32_t remaining = - offset < target->buffer_size ? (target->buffer_size - offset) : 0; - uint32_t max_vertices = stride ? (remaining / stride) : ~0; - - min_max = MIN2(min_max, max_vertices); - } - - /* We now have the maximum vertices written, round down to primitives */ - uint32_t max_prims = xfb_prims_for_vertices(info->mode, min_max); - uint32_t prims = xfb_prims_for_vertices(info->mode, draw->count); - - ctx->tf_prims_generated->value += MIN2(prims, max_prims); - } - - /* Update the offsets into the streamout buffers */ - for (unsigned i = 0; i < ctx->streamout.num_targets; ++i) { - if (ctx->streamout.targets[i]) - agx_so_target(ctx->streamout.targets[i])->offset += num_outputs; - } - - ctx->dirty |= AGX_DIRTY_XFB; - ctx->streamout.key.active = false; -} - /* * Count generated primitives on the CPU for transform feedback. This only works * in the absence of indirect draws, geometry shaders, or tessellation. @@ -260,301 +161,13 @@ agx_primitives_update_direct(struct agx_context *ctx, const struct pipe_draw_start_count_bias *draw) { assert(ctx->active_queries && ctx->prims_generated && "precondition"); + assert(!ctx->stage[PIPE_SHADER_GEOMETRY].shader && + "Geometry shaders use their own counting"); ctx->prims_generated->value += xfb_prims_for_vertices(info->mode, draw->count); } -/* The OpenGL spec says: - * - * If recording the vertices of a primitive to the buffer objects being - * used for transform feedback purposes would result in either exceeding - * the limits of any buffer object’s size, or in exceeding the end - * position offset + size − 1, as set by BindBufferRange, then no vertices - * of that primitive are recorded in any buffer object. - * - * This function checks for the absence of overflow. - * - * The difficulty is that we are processing a single vertex at a time, so we - * need to do some arithmetic to figure out the bounds for the whole containing - * primitive. - * - * XXX: How do quads get tessellated? - */ -static nir_def * -primitive_fits(nir_builder *b, struct agx_xfb_key *key) -{ - /* Get the number of vertices per primitive in the current mode, usually just - * the base number but quads are tessellated. - */ - uint32_t verts_per_prim = mesa_vertices_per_prim(key->mode); - - if (u_decomposed_prim(key->mode) == MESA_PRIM_QUADS) - verts_per_prim = 6; - - /* Get the ID for this invocation */ - nir_def *id = nir_load_vertex_id_zero_base(b); - - /* Figure out the ID for the first vertex of the next primitive. Since - * transform feedback buffers are tightly packed, that's one byte after the - * end of this primitive, which will make bounds checking convenient. That - * will be: - * - * (id - (id % prim size)) + prim size - */ - nir_def *rem = nir_umod_imm(b, id, verts_per_prim); - nir_def *next_id = nir_iadd_imm(b, nir_isub(b, id, rem), verts_per_prim); - - /* Figure out where that vertex will land */ - nir_def *index = nir_iadd( - b, nir_imul(b, nir_load_instance_id(b), nir_load_num_vertices(b)), - next_id); - - /* Now check for overflow in each written buffer */ - nir_def *all_fits = nir_imm_true(b); - - u_foreach_bit(buffer, b->shader->xfb_info->buffers_written) { - uint16_t stride = b->shader->info.xfb_stride[buffer] * 4; - assert(stride != 0); - - /* For this primitive to fit, the next primitive cannot start after the - * end of the transform feedback buffer. - */ - nir_def *end_offset = nir_imul_imm(b, index, stride); - - /* Check whether that will remain in bounds */ - nir_def *fits = - nir_uge(b, nir_load_xfb_size(b, .base = buffer), end_offset); - - /* Accumulate */ - all_fits = nir_iand(b, all_fits, fits); - } - - return all_fits; -} - -static void -insert_overflow_check(nir_shader *nir, struct agx_xfb_key *key) -{ - nir_function_impl *impl = nir_shader_get_entrypoint(nir); - - /* Extract the current transform feedback shader */ - nir_cf_list list; - nir_cf_extract(&list, nir_before_impl(impl), nir_after_impl(impl)); - - /* Get a builder for the (now empty) shader */ - nir_builder b = nir_builder_at(nir_after_block(nir_start_block(impl))); - - /* Rebuild the shader as - * - * if (!overflow) { - * shader(); - * } - */ - nir_push_if(&b, primitive_fits(&b, key)); - { - b.cursor = nir_cf_reinsert(&list, b.cursor); - } - nir_pop_if(&b, NULL); -} - -static void -lower_xfb_output(nir_builder *b, nir_intrinsic_instr *intr, - unsigned start_component, unsigned num_components, - unsigned buffer, unsigned offset_words) -{ - assert(buffer < MAX_XFB_BUFFERS); - assert(nir_intrinsic_component(intr) == 0); // TODO - - /* Transform feedback info in units of words, convert to bytes. */ - uint16_t stride = b->shader->info.xfb_stride[buffer] * 4; - assert(stride != 0); - - uint16_t offset = offset_words * 4; - - nir_def *index = nir_iadd( - b, nir_imul(b, nir_load_instance_id(b), nir_load_num_vertices(b)), - nir_load_vertex_id_zero_base(b)); - - nir_def *xfb_offset = - nir_iadd_imm(b, nir_imul_imm(b, index, stride), offset); - - nir_def *buf = nir_load_xfb_address(b, 64, .base = buffer); - nir_def *addr = nir_iadd(b, buf, nir_u2u64(b, xfb_offset)); - - nir_def *value = nir_channels( - b, intr->src[0].ssa, BITFIELD_MASK(num_components) << start_component); - nir_store_global(b, addr, 4, value, nir_component_mask(num_components)); -} - -static bool -lower_xfb(nir_builder *b, nir_intrinsic_instr *intr, UNUSED void *data) -{ - if (intr->intrinsic != nir_intrinsic_store_output) - return false; - - /* Assume the inputs are read */ - BITSET_SET(b->shader->info.system_values_read, - SYSTEM_VALUE_VERTEX_ID_ZERO_BASE); - BITSET_SET(b->shader->info.system_values_read, SYSTEM_VALUE_INSTANCE_ID); - - bool progress = false; - - for (unsigned i = 0; i < 2; ++i) { - nir_io_xfb xfb = - i ? nir_intrinsic_io_xfb2(intr) : nir_intrinsic_io_xfb(intr); - - for (unsigned j = 0; j < 2; ++j) { - if (xfb.out[j].num_components > 0) { - b->cursor = nir_before_instr(&intr->instr); - lower_xfb_output(b, intr, i * 2 + j, xfb.out[j].num_components, - xfb.out[j].buffer, xfb.out[j].offset); - progress = true; - } - } - } - - nir_instr_remove(&intr->instr); - return progress; -} - -static bool -lower_xfb_intrinsics(struct nir_builder *b, nir_intrinsic_instr *intr, - void *data) -{ - b->cursor = nir_before_instr(&intr->instr); - - struct agx_xfb_key *key = data; - - switch (intr->intrinsic) { - /* XXX: Rename to "xfb index" to avoid the clash */ - case nir_intrinsic_load_vertex_id_zero_base: { - nir_def *id = nir_load_vertex_id(b); - nir_def_rewrite_uses(&intr->def, id); - return true; - } - - case nir_intrinsic_load_vertex_id: { - /* Get the raw invocation ID */ - nir_def *id = nir_load_vertex_id(b); - - /* Tessellate by primitive mode */ - if (key->mode == MESA_PRIM_LINE_STRIP || - key->mode == MESA_PRIM_LINE_LOOP) { - /* The last vertex is special for a loop. Check if that's we're dealing - * with. - */ - nir_def *num_invocations = - nir_imul_imm(b, nir_load_num_vertices(b), 2); - nir_def *last_vertex = - nir_ieq(b, id, nir_iadd_imm(b, num_invocations, -1)); - - /* (0, 1), (1, 2) */ - id = nir_iadd(b, nir_ushr_imm(b, id, 1), nir_iand_imm(b, id, 1)); - - /* (0, 1), (1, 2), (2, 0) */ - if (key->mode == MESA_PRIM_LINE_LOOP) { - id = nir_bcsel(b, last_vertex, nir_imm_int(b, 0), id); - } - } else if (key->mode == MESA_PRIM_TRIANGLE_STRIP) { - /* Order depends on the provoking vertex. - * - * First: (0, 1, 2), (1, 3, 2), (2, 3, 4). - * Last: (0, 1, 2), (2, 1, 3), (2, 3, 4). - */ - nir_def *prim = nir_udiv_imm(b, id, 3); - nir_def *rem = nir_umod_imm(b, id, 3); - - unsigned pv = key->flatshade_first ? 0 : 2; - - /* Swap the two non-provoking vertices third vertex in odd triangles */ - nir_def *even = nir_ieq_imm(b, nir_iand_imm(b, prim, 1), 0); - nir_def *is_provoking = nir_ieq_imm(b, rem, pv); - nir_def *no_swap = nir_ior(b, is_provoking, even); - nir_def *swapped = nir_isub_imm(b, 3 - pv, rem); - nir_def *off = nir_bcsel(b, no_swap, rem, swapped); - - /* Pull the (maybe swapped) vertex from the corresponding primitive */ - id = nir_iadd(b, prim, off); - } else if (key->mode == MESA_PRIM_TRIANGLE_FAN) { - /* (0, 1, 2), (0, 2, 3) */ - nir_def *prim = nir_udiv_imm(b, id, 3); - nir_def *rem = nir_umod_imm(b, id, 3); - - id = nir_bcsel(b, nir_ieq_imm(b, rem, 0), nir_imm_int(b, 0), - nir_iadd(b, prim, rem)); - } else if (key->mode == MESA_PRIM_QUADS || - key->mode == MESA_PRIM_QUAD_STRIP) { - /* Quads: [(0, 1, 3), (3, 1, 2)], [(4, 5, 7), (7, 5, 6)] - * Quad strips: [(0, 1, 3), (0, 2, 3)], [(2, 3, 5), (2, 4, 5)] - */ - bool strips = key->mode == MESA_PRIM_QUAD_STRIP; - - nir_def *prim = nir_udiv_imm(b, id, 6); - nir_def *rem = nir_umod_imm(b, id, 6); - nir_def *base = nir_imul_imm(b, prim, strips ? 2 : 4); - - /* Quads: [0, 1, 3, 3, 1, 2] - * Quad strips: [0, 1, 3, 0, 2, 3] - */ - uint32_t order_quads = 0x213310; - uint32_t order_strips = 0x230310; - uint32_t order = strips ? order_strips : order_quads; - - /* Index out of the bitpacked array */ - nir_def *offset = nir_iand_imm( - b, nir_ushr(b, nir_imm_int(b, order), nir_imul_imm(b, rem, 4)), - 0xF); - - id = nir_iadd(b, base, offset); - } - - /* Add the "start", either an index bias or a base vertex */ - id = nir_iadd(b, id, nir_load_base_vertex(b)); - - /* If drawing with an index buffer, pull the vertex ID. Otherwise, the - * vertex ID is just the index as-is. - */ - if (key->index_size) { - nir_def *index_buffer = nir_load_xfb_index_buffer(b, 64); - nir_def *offset = nir_imul_imm(b, id, key->index_size); - nir_def *address = nir_iadd(b, index_buffer, nir_u2u64(b, offset)); - nir_def *index = nir_load_global_constant(b, address, key->index_size, - 1, key->index_size * 8); - - id = nir_u2uN(b, index, id->bit_size); - } - - nir_def_rewrite_uses(&intr->def, id); - return true; - } - - default: - return false; - } -} - -void -agx_nir_lower_xfb(nir_shader *nir, struct agx_xfb_key *key) -{ - assert(nir->info.stage == MESA_SHADER_VERTEX); - - NIR_PASS_V(nir, nir_io_add_const_offset_to_base, - nir_var_shader_in | nir_var_shader_out); - NIR_PASS_V(nir, nir_io_add_intrinsic_xfb_info); - - NIR_PASS_V(nir, insert_overflow_check, key); - NIR_PASS_V(nir, nir_shader_intrinsics_pass, lower_xfb, - nir_metadata_block_index | nir_metadata_dominance, key); - NIR_PASS_V(nir, nir_shader_intrinsics_pass, lower_xfb_intrinsics, - nir_metadata_block_index | nir_metadata_dominance, key); - - /* Lowering XFB creates piles of dead code. Eliminate now so we don't - * push unnecessary sysvals. - */ - NIR_PASS_V(nir, nir_opt_dce); -} - void agx_init_streamout_functions(struct pipe_context *ctx) { diff --git a/src/gallium/drivers/asahi/agx_uniforms.c b/src/gallium/drivers/asahi/agx_uniforms.c index 5a389ff928a..c5afc8a046a 100644 --- a/src/gallium/drivers/asahi/agx_uniforms.c +++ b/src/gallium/drivers/asahi/agx_uniforms.c @@ -69,17 +69,6 @@ agx_upload_uniforms(struct agx_batch *batch) batch->uniforms.tables[AGX_SYSVAL_TABLE_ROOT] = root_ptr.gpu; batch->uniforms.sample_mask = ctx->sample_mask; - if (ctx->streamout.key.active) { - batch->uniforms.xfb = ctx->streamout.params; - - for (unsigned i = 0; i < batch->ctx->streamout.num_targets; ++i) { - uint32_t size = 0; - batch->uniforms.xfb.base[i] = - agx_batch_get_so_address(batch, i, &size); - batch->uniforms.xfb.size[i] = size; - } - } - memcpy(root_ptr.cpu, &batch->uniforms, sizeof(batch->uniforms)); }