asahi: Wire up geometry shaders
- Compile GS with linked VS and auxiliary programs - Dispatch GS as compute programs + an indirect draw with the GS copy program - Use passthrough GS to implement XFB, replacing old XFB impl. Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/26056>
This commit is contained in:
Alyssa Rosenzweig
committed by
Marge Bot
Marge Bot
parent
fe7650bcf7
commit
c6a118b654
@@ -2931,7 +2931,7 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
|
||||
if (nir->info.stage == MESA_SHADER_FRAGMENT)
|
||||
out->tag_write_disable = !nir->info.writes_memory;
|
||||
|
||||
bool needs_libagx = false;
|
||||
bool needs_libagx = nir->info.stage == MESA_SHADER_GEOMETRY;
|
||||
|
||||
/* Late tilebuffer lowering creates multisampled image stores */
|
||||
NIR_PASS(needs_libagx, nir, agx_nir_lower_multisampled_image_store);
|
||||
|
||||
@@ -13,6 +13,7 @@ libasahi_lib_files = files(
|
||||
'agx_meta.c',
|
||||
'agx_tilebuffer.c',
|
||||
'agx_nir_lower_alpha.c',
|
||||
'agx_nir_lower_gs.c',
|
||||
'agx_nir_lower_msaa.c',
|
||||
'agx_nir_lower_sample_intrinsics.c',
|
||||
'agx_nir_lower_tilebuffer.c',
|
||||
@@ -28,6 +29,8 @@ libasahi_decode_files = files(
|
||||
|
||||
libagx_shader_files = files(
|
||||
'shaders/libagx.h',
|
||||
'shaders/geometry.cl',
|
||||
'shaders/geometry.h',
|
||||
'shaders/texture.cl',
|
||||
)
|
||||
|
||||
@@ -75,6 +78,7 @@ libagx_shaders = custom_target(
|
||||
prepended_input_args, '-o', '@OUTPUT@', '--',
|
||||
'-cl-std=cl2.0', '-D__OPENCL_VERSION__=200',
|
||||
'-I' + join_paths(meson.current_source_dir(), '.'),
|
||||
'-I' + join_paths(meson.current_source_dir(), '../../'),
|
||||
'-I' + join_paths(meson.current_source_dir(), 'shaders'),
|
||||
'-I' + join_paths(meson.current_build_dir(), '.'),
|
||||
],
|
||||
|
||||
@@ -133,6 +133,7 @@ agx_batch_init(struct agx_context *ctx,
|
||||
batch->clear_depth = 0;
|
||||
batch->clear_stencil = 0;
|
||||
batch->varyings = 0;
|
||||
batch->geometry_state = 0;
|
||||
batch->any_draws = false;
|
||||
batch->initialized = false;
|
||||
batch->draws = 0;
|
||||
|
||||
@@ -19,6 +19,8 @@ agx_blitter_save(struct agx_context *ctx, struct blitter_context *blitter,
|
||||
util_blitter_save_vertex_elements(blitter, ctx->attributes);
|
||||
util_blitter_save_vertex_shader(blitter,
|
||||
ctx->stage[PIPE_SHADER_VERTEX].shader);
|
||||
util_blitter_save_geometry_shader(blitter,
|
||||
ctx->stage[PIPE_SHADER_GEOMETRY].shader);
|
||||
util_blitter_save_rasterizer(blitter, ctx->rast);
|
||||
util_blitter_save_viewport(blitter, &ctx->viewport);
|
||||
util_blitter_save_scissor(blitter, &ctx->scissor);
|
||||
|
||||
@@ -35,6 +35,8 @@ agx_disk_cache_compute_key(struct disk_cache *cache,
|
||||
int key_size;
|
||||
if (uncompiled->type == PIPE_SHADER_VERTEX)
|
||||
key_size = sizeof(shader_key->vs);
|
||||
else if (uncompiled->type == PIPE_SHADER_GEOMETRY)
|
||||
key_size = sizeof(shader_key->gs);
|
||||
else if (uncompiled->type == PIPE_SHADER_FRAGMENT)
|
||||
key_size = sizeof(shader_key->fs);
|
||||
else if (uncompiled->type == PIPE_SHADER_COMPUTE)
|
||||
@@ -66,6 +68,10 @@ agx_disk_cache_store(struct disk_cache *cache,
|
||||
if (!cache)
|
||||
return;
|
||||
|
||||
/* TODO: Support caching GS */
|
||||
if (uncompiled->type == PIPE_SHADER_GEOMETRY)
|
||||
return;
|
||||
|
||||
assert(binary->bo->ptr.cpu != NULL && "shaders must be CPU mapped");
|
||||
|
||||
cache_key cache_key;
|
||||
@@ -100,6 +106,10 @@ agx_disk_cache_retrieve(struct agx_screen *screen,
|
||||
if (!cache)
|
||||
return NULL;
|
||||
|
||||
/* TODO: Support caching GS */
|
||||
if (uncompiled->type == PIPE_SHADER_GEOMETRY)
|
||||
return NULL;
|
||||
|
||||
cache_key cache_key;
|
||||
agx_disk_cache_compute_key(cache, uncompiled, key, cache_key);
|
||||
|
||||
|
||||
@@ -153,18 +153,14 @@ lower_intrinsic(nir_builder *b, nir_intrinsic_instr *intr)
|
||||
intr->src[0].ssa);
|
||||
case nir_intrinsic_load_num_workgroups:
|
||||
return load_sysval(b, 3, 32, AGX_SYSVAL_TABLE_GRID, 0);
|
||||
case nir_intrinsic_load_first_vertex:
|
||||
return load_sysval(b, 1, 32, AGX_SYSVAL_TABLE_PARAMS, 0);
|
||||
case nir_intrinsic_load_base_instance:
|
||||
return load_sysval(b, 1, 32, AGX_SYSVAL_TABLE_PARAMS, 4);
|
||||
case nir_intrinsic_load_layer_id_written_agx:
|
||||
return load_sysval_root(b, 1, 16, &u->layer_id_written);
|
||||
case nir_intrinsic_load_xfb_address:
|
||||
return load_sysval_root(b, 1, 64, &u->xfb.base[nir_intrinsic_base(intr)]);
|
||||
case nir_intrinsic_load_xfb_size:
|
||||
return load_sysval_root(b, 1, 32, &u->xfb.size[nir_intrinsic_base(intr)]);
|
||||
case nir_intrinsic_load_xfb_index_buffer:
|
||||
return load_sysval_root(b, 1, 64, &u->xfb.index_buffer);
|
||||
case nir_intrinsic_load_base_vertex:
|
||||
return load_sysval_root(b, 1, 32, &u->xfb.base_vertex);
|
||||
case nir_intrinsic_load_num_vertices:
|
||||
return load_sysval_root(b, 1, 32, &u->xfb.num_vertices);
|
||||
case nir_intrinsic_load_geometry_param_buffer_agx:
|
||||
return load_sysval_root(b, 1, 64, &u->geometry_params);
|
||||
default:
|
||||
return NULL;
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,6 +15,7 @@
|
||||
#include "asahi/lib/agx_pack.h"
|
||||
#include "asahi/lib/agx_tilebuffer.h"
|
||||
#include "asahi/lib/pool.h"
|
||||
#include "asahi/lib/shaders/geometry.h"
|
||||
#include "compiler/nir/nir_lower_blend.h"
|
||||
#include "compiler/shader_enums.h"
|
||||
#include "gallium/auxiliary/util/u_blitter.h"
|
||||
@@ -39,7 +40,7 @@
|
||||
|
||||
struct agx_streamout_target {
|
||||
struct pipe_stream_output_target base;
|
||||
uint32_t offset;
|
||||
struct pipe_resource *offset;
|
||||
};
|
||||
|
||||
static inline struct agx_streamout_target *
|
||||
@@ -48,37 +49,9 @@ agx_so_target(struct pipe_stream_output_target *target)
|
||||
return (struct agx_streamout_target *)target;
|
||||
}
|
||||
|
||||
struct agx_xfb_key {
|
||||
/* If true, compiles a "transform feedback" program instead of a vertex
|
||||
* shader. This is a kernel that runs on the VDM and writes out the transform
|
||||
* feedback buffers, with no rasterization.
|
||||
*/
|
||||
bool active;
|
||||
|
||||
/* The index size (1, 2, 4) or 0 if drawing without an index buffer. */
|
||||
uint8_t index_size;
|
||||
|
||||
/* The primitive mode for unrolling the vertex ID */
|
||||
enum mesa_prim mode;
|
||||
|
||||
/* Use first vertex as the provoking vertex for flat shading */
|
||||
bool flatshade_first;
|
||||
};
|
||||
|
||||
struct agx_xfb_params {
|
||||
uint64_t base[PIPE_MAX_SO_BUFFERS];
|
||||
uint32_t size[PIPE_MAX_SO_BUFFERS];
|
||||
uint64_t index_buffer;
|
||||
uint32_t base_vertex;
|
||||
uint32_t num_vertices;
|
||||
};
|
||||
|
||||
struct agx_streamout {
|
||||
struct pipe_stream_output_target *targets[PIPE_MAX_SO_BUFFERS];
|
||||
unsigned num_targets;
|
||||
|
||||
struct agx_xfb_key key;
|
||||
struct agx_xfb_params params;
|
||||
};
|
||||
|
||||
/* Shaders can access fixed-function state through system values.
|
||||
@@ -92,6 +65,7 @@ struct agx_streamout {
|
||||
*/
|
||||
enum agx_sysval_table {
|
||||
AGX_SYSVAL_TABLE_ROOT,
|
||||
AGX_SYSVAL_TABLE_PARAMS,
|
||||
AGX_SYSVAL_TABLE_GRID,
|
||||
AGX_SYSVAL_TABLE_VS,
|
||||
AGX_SYSVAL_TABLE_TCS,
|
||||
@@ -125,8 +99,8 @@ struct PACKED agx_draw_uniforms {
|
||||
/* Vertex buffer object bases, if present */
|
||||
uint64_t vbo_base[PIPE_MAX_ATTRIBS];
|
||||
|
||||
/* Transform feedback info for a transform feedback shader */
|
||||
struct agx_xfb_params xfb;
|
||||
/* Address of geometry param buffer if geometry shaders are used, else 0 */
|
||||
uint64_t geometry_params;
|
||||
|
||||
/* Blend constant if any */
|
||||
float blend_constant[4];
|
||||
@@ -195,15 +169,27 @@ struct agx_compiled_shader {
|
||||
/* Uniforms the driver must push */
|
||||
unsigned push_range_count;
|
||||
struct agx_push_range push[AGX_MAX_PUSH_RANGES];
|
||||
|
||||
/* Auxiliary programs, or NULL if not used */
|
||||
struct agx_compiled_shader *gs_count, *pre_gs;
|
||||
struct agx_uncompiled_shader *gs_copy;
|
||||
|
||||
/* Output primitive mode for geometry shaders */
|
||||
enum mesa_prim gs_output_mode;
|
||||
|
||||
/* Number of words per primitive in the count buffer */
|
||||
unsigned gs_count_words;
|
||||
};
|
||||
|
||||
struct agx_uncompiled_shader {
|
||||
struct pipe_shader_state base;
|
||||
enum pipe_shader_type type;
|
||||
struct blob early_serialized_nir;
|
||||
struct blob serialized_nir;
|
||||
uint8_t nir_sha1[20];
|
||||
struct agx_uncompiled_shader_info info;
|
||||
struct hash_table *variants;
|
||||
struct agx_uncompiled_shader *passthrough_progs[MESA_PRIM_COUNT];
|
||||
bool has_xfb_info;
|
||||
|
||||
/* Whether the shader accesses indexed samplers via the bindless heap */
|
||||
@@ -293,6 +279,13 @@ struct agx_batch {
|
||||
|
||||
struct agx_draw_uniforms uniforms;
|
||||
|
||||
/* Indirect buffer allocated for geometry shader */
|
||||
uint64_t geom_indirect;
|
||||
struct agx_bo *geom_indirect_bo;
|
||||
|
||||
/* Geometry state buffer if geometry/etc shaders are used */
|
||||
uint64_t geometry_state;
|
||||
|
||||
/* Uploaded descriptors */
|
||||
uint64_t textures[PIPE_SHADER_TYPES];
|
||||
uint32_t texture_count[PIPE_SHADER_TYPES];
|
||||
@@ -333,6 +326,9 @@ struct agx_batch {
|
||||
/* Result buffer where the kernel places command execution information */
|
||||
union agx_batch_result *result;
|
||||
size_t result_off;
|
||||
|
||||
/* Actual pointer in a uniform */
|
||||
struct agx_bo *geom_params_bo;
|
||||
};
|
||||
|
||||
struct agx_zsa {
|
||||
@@ -357,7 +353,6 @@ struct agx_blend {
|
||||
|
||||
struct asahi_vs_shader_key {
|
||||
struct agx_vbufs vbuf;
|
||||
struct agx_xfb_key xfb;
|
||||
uint64_t outputs_flat_shaded;
|
||||
uint64_t outputs_linear_shaded;
|
||||
};
|
||||
@@ -381,8 +376,26 @@ struct asahi_fs_shader_key {
|
||||
enum pipe_format rt_formats[PIPE_MAX_COLOR_BUFS];
|
||||
};
|
||||
|
||||
struct asahi_gs_shader_key {
|
||||
/* Input assembly key */
|
||||
struct agx_ia_key ia;
|
||||
|
||||
/* Vertex shader key */
|
||||
struct agx_vbufs vbuf;
|
||||
|
||||
/* If true, this GS is run only for its side effects (including XFB) */
|
||||
bool rasterizer_discard;
|
||||
|
||||
/* Geometry shaders must be linked with a vertex shader. In a monolithic
|
||||
* pipeline, this is the vertex shader (or tessellation evaluation shader).
|
||||
* With separate shaders, this needs to be an internal passthrough program.
|
||||
*/
|
||||
uint8_t input_nir_sha1[20];
|
||||
};
|
||||
|
||||
union asahi_shader_key {
|
||||
struct asahi_vs_shader_key vs;
|
||||
struct asahi_gs_shader_key gs;
|
||||
struct asahi_fs_shader_key fs;
|
||||
};
|
||||
|
||||
@@ -419,9 +432,12 @@ enum agx_dirty {
|
||||
|
||||
struct agx_context {
|
||||
struct pipe_context base;
|
||||
struct agx_compiled_shader *vs, *fs;
|
||||
struct agx_compiled_shader *vs, *fs, *gs;
|
||||
uint32_t dirty;
|
||||
|
||||
/* Heap for dynamic memory allocation for geometry/tessellation shaders */
|
||||
struct pipe_resource *heap;
|
||||
|
||||
/* Acts as a context-level shader key */
|
||||
bool support_lod_bias;
|
||||
|
||||
@@ -480,6 +496,8 @@ struct agx_context {
|
||||
/* Bound CL global buffers */
|
||||
struct util_dynarray global_buffers;
|
||||
|
||||
struct agx_compiled_shader *gs_prefix_sums[16];
|
||||
struct agx_compiled_shader *gs_setup_indirect[MESA_PRIM_MAX];
|
||||
struct agx_meta_cache meta;
|
||||
|
||||
uint32_t syncobj;
|
||||
@@ -541,7 +559,7 @@ agx_context(struct pipe_context *pctx)
|
||||
}
|
||||
|
||||
void agx_launch(struct agx_batch *batch, const struct pipe_grid_info *info,
|
||||
struct agx_compiled_shader *cs);
|
||||
struct agx_compiled_shader *cs, enum pipe_shader_type stage);
|
||||
|
||||
void agx_init_query_functions(struct pipe_context *ctx);
|
||||
|
||||
@@ -550,17 +568,11 @@ agx_primitives_update_direct(struct agx_context *ctx,
|
||||
const struct pipe_draw_info *info,
|
||||
const struct pipe_draw_start_count_bias *draw);
|
||||
|
||||
void agx_nir_lower_xfb(nir_shader *shader, struct agx_xfb_key *key);
|
||||
|
||||
void agx_draw_vbo_from_xfb(struct pipe_context *pctx,
|
||||
const struct pipe_draw_info *info,
|
||||
unsigned drawid_offset,
|
||||
const struct pipe_draw_indirect_info *indirect);
|
||||
|
||||
void agx_launch_so(struct pipe_context *pctx, const struct pipe_draw_info *info,
|
||||
const struct pipe_draw_start_count_bias *draws,
|
||||
uint64_t index_buffer);
|
||||
|
||||
uint64_t agx_batch_get_so_address(struct agx_batch *batch, unsigned buffer,
|
||||
uint32_t *size);
|
||||
|
||||
@@ -770,6 +782,14 @@ bool agx_nir_layout_uniforms(nir_shader *shader,
|
||||
|
||||
bool agx_nir_lower_bindings(nir_shader *shader, bool *uses_bindless_samplers);
|
||||
|
||||
void agx_nir_lower_gs(nir_shader *gs, nir_shader *input_shader,
|
||||
const nir_shader *libagx, struct agx_ia_key *ia,
|
||||
bool rasterizer_discard, nir_shader **gs_count,
|
||||
nir_shader **gs_copy, nir_shader **pre_gs,
|
||||
enum mesa_prim *out_mode, unsigned *out_count_words);
|
||||
|
||||
nir_shader *agx_nir_prefix_sum_gs(const nir_shader *libagx, unsigned words);
|
||||
|
||||
bool agx_batch_is_active(struct agx_batch *batch);
|
||||
bool agx_batch_is_submitted(struct agx_batch *batch);
|
||||
|
||||
|
||||
@@ -6,8 +6,10 @@
|
||||
|
||||
#include "compiler/nir/nir_builder.h"
|
||||
#include "compiler/nir/nir_xfb_info.h"
|
||||
#include "pipe/p_defines.h"
|
||||
#include "util/u_draw.h"
|
||||
#include "util/u_dump.h"
|
||||
#include "util/u_inlines.h"
|
||||
#include "util/u_prim.h"
|
||||
#include "agx_state.h"
|
||||
|
||||
@@ -16,21 +18,24 @@ agx_create_stream_output_target(struct pipe_context *pctx,
|
||||
struct pipe_resource *prsc,
|
||||
unsigned buffer_offset, unsigned buffer_size)
|
||||
{
|
||||
struct pipe_stream_output_target *target;
|
||||
|
||||
target = &rzalloc(pctx, struct agx_streamout_target)->base;
|
||||
struct agx_streamout_target *target =
|
||||
rzalloc(pctx, struct agx_streamout_target);
|
||||
|
||||
if (!target)
|
||||
return NULL;
|
||||
|
||||
pipe_reference_init(&target->reference, 1);
|
||||
pipe_resource_reference(&target->buffer, prsc);
|
||||
pipe_reference_init(&target->base.reference, 1);
|
||||
pipe_resource_reference(&target->base.buffer, prsc);
|
||||
|
||||
target->context = pctx;
|
||||
target->buffer_offset = buffer_offset;
|
||||
target->buffer_size = buffer_size;
|
||||
target->base.context = pctx;
|
||||
target->base.buffer_offset = buffer_offset;
|
||||
target->base.buffer_size = buffer_size;
|
||||
|
||||
return target;
|
||||
uint32_t zero = 0;
|
||||
target->offset = pipe_buffer_create_with_data(pctx, PIPE_BIND_GLOBAL,
|
||||
PIPE_USAGE_DEFAULT, 4, &zero);
|
||||
|
||||
return &target->base;
|
||||
}
|
||||
|
||||
static void
|
||||
@@ -62,8 +67,10 @@ agx_set_stream_output_targets(struct pipe_context *pctx, unsigned num_targets,
|
||||
* Gallium contract and it will work out fine. Probably should be
|
||||
* redefined to be ~0 instead of -1 but it doesn't really matter.
|
||||
*/
|
||||
if (offsets[i] != -1)
|
||||
agx_so_target(targets[i])->offset = offsets[i];
|
||||
if (offsets[i] != -1) {
|
||||
pipe_buffer_write(pctx, agx_so_target(targets[i])->offset, 0, 4,
|
||||
&offsets[i]);
|
||||
}
|
||||
|
||||
pipe_so_target_reference(&so->targets[i], targets[i]);
|
||||
}
|
||||
@@ -100,18 +107,11 @@ agx_batch_get_so_address(struct agx_batch *batch, unsigned buffer,
|
||||
}
|
||||
|
||||
/* Otherwise, write the target */
|
||||
struct pipe_stream_output_info *so =
|
||||
&batch->ctx->stage[PIPE_SHADER_VERTEX].shader->base.stream_output;
|
||||
|
||||
struct agx_resource *rsrc = agx_resource(target->buffer);
|
||||
agx_batch_writes(batch, rsrc);
|
||||
|
||||
/* The amount of space left depends how much we've already consumed */
|
||||
unsigned stride = so->stride[buffer] * 4;
|
||||
uint32_t offset = agx_so_target(target)->offset * stride;
|
||||
|
||||
*size = offset < target->buffer_size ? (target->buffer_size - offset) : 0;
|
||||
return rsrc->bo->ptr.gpu + target->buffer_offset + offset;
|
||||
*size = target->buffer_size;
|
||||
return rsrc->bo->ptr.gpu + target->buffer_offset;
|
||||
}
|
||||
|
||||
void
|
||||
@@ -119,9 +119,18 @@ agx_draw_vbo_from_xfb(struct pipe_context *pctx,
|
||||
const struct pipe_draw_info *info, unsigned drawid_offset,
|
||||
const struct pipe_draw_indirect_info *indirect)
|
||||
{
|
||||
perf_debug_ctx(agx_context(pctx), "draw auto");
|
||||
|
||||
unsigned count;
|
||||
pipe_buffer_read(pctx,
|
||||
agx_so_target(indirect->count_from_stream_output)->offset,
|
||||
0, 4, &count);
|
||||
|
||||
/* XXX: Probably need to divide here */
|
||||
|
||||
struct pipe_draw_start_count_bias draw = {
|
||||
.start = 0,
|
||||
.count = agx_so_target(indirect->count_from_stream_output)->offset,
|
||||
.count = count,
|
||||
};
|
||||
|
||||
pctx->draw_vbo(pctx, info, drawid_offset, NULL, &draw, 1);
|
||||
@@ -142,114 +151,6 @@ xfb_prims_for_vertices(enum mesa_prim mode, unsigned verts)
|
||||
return prims;
|
||||
}
|
||||
|
||||
/*
|
||||
* Launch a streamout pipeline.
|
||||
*/
|
||||
void
|
||||
agx_launch_so(struct pipe_context *pctx, const struct pipe_draw_info *info,
|
||||
const struct pipe_draw_start_count_bias *draw,
|
||||
uint64_t index_buffer)
|
||||
{
|
||||
struct agx_context *ctx = agx_context(pctx);
|
||||
|
||||
/* Break recursion from draw_vbo creating draw calls below: Do not do a
|
||||
* streamout draw for a streamout draw.
|
||||
*/
|
||||
if (ctx->streamout.key.active)
|
||||
return;
|
||||
|
||||
/* Configure the below draw to launch streamout rather than a regular draw */
|
||||
ctx->streamout.key.active = true;
|
||||
ctx->dirty |= AGX_DIRTY_XFB;
|
||||
|
||||
ctx->streamout.key.index_size = info->index_size;
|
||||
ctx->streamout.key.mode = info->mode;
|
||||
ctx->streamout.key.flatshade_first = ctx->rast->base.flatshade_first;
|
||||
ctx->streamout.params.index_buffer = index_buffer;
|
||||
|
||||
/* Ignore provoking vertex for modes that don't depend on the provoking
|
||||
* vertex, to reduce shader variants.
|
||||
*/
|
||||
if (info->mode != MESA_PRIM_TRIANGLE_STRIP)
|
||||
ctx->streamout.key.flatshade_first = false;
|
||||
|
||||
/* Determine how many vertices are XFB there will be */
|
||||
unsigned num_outputs =
|
||||
u_stream_outputs_for_vertices(info->mode, draw->count);
|
||||
unsigned count = draw->count;
|
||||
u_trim_pipe_prim(info->mode, &count);
|
||||
|
||||
ctx->streamout.params.base_vertex =
|
||||
info->index_size ? draw->index_bias : draw->start;
|
||||
ctx->streamout.params.num_vertices = count;
|
||||
|
||||
/* Streamout runs as a vertex shader with rasterizer discard */
|
||||
void *saved_rast = ctx->rast;
|
||||
pctx->bind_rasterizer_state(
|
||||
pctx, util_blitter_get_discard_rasterizer_state(ctx->blitter));
|
||||
|
||||
/* Dispatch a grid of points, this is compute-like */
|
||||
util_draw_arrays_instanced(pctx, MESA_PRIM_POINTS, 0, num_outputs, 0,
|
||||
info->instance_count);
|
||||
pctx->bind_rasterizer_state(pctx, saved_rast);
|
||||
|
||||
/*
|
||||
* Finally, if needed, update the counter of primitives written. The spec
|
||||
* requires:
|
||||
*
|
||||
* If recording the vertices of a primitive to the buffer objects being
|
||||
* used for transform feedback purposes would result in [overflow]...
|
||||
* the counter corresponding to the asynchronous query target
|
||||
* TRANSFORM_FEEDBACK_PRIMITIVES_WRITTEN (see section 13.4) is not
|
||||
* incremented.
|
||||
*
|
||||
* So clamp the number of primitives generated to the number of primitives
|
||||
* we actually have space to write.
|
||||
*/
|
||||
if (ctx->tf_prims_generated) {
|
||||
uint32_t min_max = ~0;
|
||||
|
||||
for (unsigned i = 0; i < ctx->streamout.num_targets; ++i) {
|
||||
struct pipe_stream_output_target *target = get_target(ctx, i);
|
||||
|
||||
if (!target)
|
||||
continue;
|
||||
|
||||
struct pipe_stream_output_info *so =
|
||||
&ctx->stage[PIPE_SHADER_VERTEX].shader->base.stream_output;
|
||||
unsigned stride = so->stride[i] * 4;
|
||||
|
||||
/* Ignore spurious targets. I don't see anything in the Gallium
|
||||
* contract specifically forbidding this.
|
||||
*/
|
||||
if (stride == 0)
|
||||
continue;
|
||||
|
||||
uint32_t offset = agx_so_target(target)->offset * stride;
|
||||
uint32_t remaining =
|
||||
offset < target->buffer_size ? (target->buffer_size - offset) : 0;
|
||||
uint32_t max_vertices = stride ? (remaining / stride) : ~0;
|
||||
|
||||
min_max = MIN2(min_max, max_vertices);
|
||||
}
|
||||
|
||||
/* We now have the maximum vertices written, round down to primitives */
|
||||
uint32_t max_prims = xfb_prims_for_vertices(info->mode, min_max);
|
||||
uint32_t prims = xfb_prims_for_vertices(info->mode, draw->count);
|
||||
|
||||
ctx->tf_prims_generated->value += MIN2(prims, max_prims);
|
||||
}
|
||||
|
||||
/* Update the offsets into the streamout buffers */
|
||||
for (unsigned i = 0; i < ctx->streamout.num_targets; ++i) {
|
||||
if (ctx->streamout.targets[i])
|
||||
agx_so_target(ctx->streamout.targets[i])->offset += num_outputs;
|
||||
}
|
||||
|
||||
ctx->dirty |= AGX_DIRTY_XFB;
|
||||
ctx->streamout.key.active = false;
|
||||
}
|
||||
|
||||
/*
|
||||
* Count generated primitives on the CPU for transform feedback. This only works
|
||||
* in the absence of indirect draws, geometry shaders, or tessellation.
|
||||
@@ -260,301 +161,13 @@ agx_primitives_update_direct(struct agx_context *ctx,
|
||||
const struct pipe_draw_start_count_bias *draw)
|
||||
{
|
||||
assert(ctx->active_queries && ctx->prims_generated && "precondition");
|
||||
assert(!ctx->stage[PIPE_SHADER_GEOMETRY].shader &&
|
||||
"Geometry shaders use their own counting");
|
||||
|
||||
ctx->prims_generated->value +=
|
||||
xfb_prims_for_vertices(info->mode, draw->count);
|
||||
}
|
||||
|
||||
/* The OpenGL spec says:
|
||||
*
|
||||
* If recording the vertices of a primitive to the buffer objects being
|
||||
* used for transform feedback purposes would result in either exceeding
|
||||
* the limits of any buffer object’s size, or in exceeding the end
|
||||
* position offset + size − 1, as set by BindBufferRange, then no vertices
|
||||
* of that primitive are recorded in any buffer object.
|
||||
*
|
||||
* This function checks for the absence of overflow.
|
||||
*
|
||||
* The difficulty is that we are processing a single vertex at a time, so we
|
||||
* need to do some arithmetic to figure out the bounds for the whole containing
|
||||
* primitive.
|
||||
*
|
||||
* XXX: How do quads get tessellated?
|
||||
*/
|
||||
static nir_def *
|
||||
primitive_fits(nir_builder *b, struct agx_xfb_key *key)
|
||||
{
|
||||
/* Get the number of vertices per primitive in the current mode, usually just
|
||||
* the base number but quads are tessellated.
|
||||
*/
|
||||
uint32_t verts_per_prim = mesa_vertices_per_prim(key->mode);
|
||||
|
||||
if (u_decomposed_prim(key->mode) == MESA_PRIM_QUADS)
|
||||
verts_per_prim = 6;
|
||||
|
||||
/* Get the ID for this invocation */
|
||||
nir_def *id = nir_load_vertex_id_zero_base(b);
|
||||
|
||||
/* Figure out the ID for the first vertex of the next primitive. Since
|
||||
* transform feedback buffers are tightly packed, that's one byte after the
|
||||
* end of this primitive, which will make bounds checking convenient. That
|
||||
* will be:
|
||||
*
|
||||
* (id - (id % prim size)) + prim size
|
||||
*/
|
||||
nir_def *rem = nir_umod_imm(b, id, verts_per_prim);
|
||||
nir_def *next_id = nir_iadd_imm(b, nir_isub(b, id, rem), verts_per_prim);
|
||||
|
||||
/* Figure out where that vertex will land */
|
||||
nir_def *index = nir_iadd(
|
||||
b, nir_imul(b, nir_load_instance_id(b), nir_load_num_vertices(b)),
|
||||
next_id);
|
||||
|
||||
/* Now check for overflow in each written buffer */
|
||||
nir_def *all_fits = nir_imm_true(b);
|
||||
|
||||
u_foreach_bit(buffer, b->shader->xfb_info->buffers_written) {
|
||||
uint16_t stride = b->shader->info.xfb_stride[buffer] * 4;
|
||||
assert(stride != 0);
|
||||
|
||||
/* For this primitive to fit, the next primitive cannot start after the
|
||||
* end of the transform feedback buffer.
|
||||
*/
|
||||
nir_def *end_offset = nir_imul_imm(b, index, stride);
|
||||
|
||||
/* Check whether that will remain in bounds */
|
||||
nir_def *fits =
|
||||
nir_uge(b, nir_load_xfb_size(b, .base = buffer), end_offset);
|
||||
|
||||
/* Accumulate */
|
||||
all_fits = nir_iand(b, all_fits, fits);
|
||||
}
|
||||
|
||||
return all_fits;
|
||||
}
|
||||
|
||||
static void
|
||||
insert_overflow_check(nir_shader *nir, struct agx_xfb_key *key)
|
||||
{
|
||||
nir_function_impl *impl = nir_shader_get_entrypoint(nir);
|
||||
|
||||
/* Extract the current transform feedback shader */
|
||||
nir_cf_list list;
|
||||
nir_cf_extract(&list, nir_before_impl(impl), nir_after_impl(impl));
|
||||
|
||||
/* Get a builder for the (now empty) shader */
|
||||
nir_builder b = nir_builder_at(nir_after_block(nir_start_block(impl)));
|
||||
|
||||
/* Rebuild the shader as
|
||||
*
|
||||
* if (!overflow) {
|
||||
* shader();
|
||||
* }
|
||||
*/
|
||||
nir_push_if(&b, primitive_fits(&b, key));
|
||||
{
|
||||
b.cursor = nir_cf_reinsert(&list, b.cursor);
|
||||
}
|
||||
nir_pop_if(&b, NULL);
|
||||
}
|
||||
|
||||
static void
|
||||
lower_xfb_output(nir_builder *b, nir_intrinsic_instr *intr,
|
||||
unsigned start_component, unsigned num_components,
|
||||
unsigned buffer, unsigned offset_words)
|
||||
{
|
||||
assert(buffer < MAX_XFB_BUFFERS);
|
||||
assert(nir_intrinsic_component(intr) == 0); // TODO
|
||||
|
||||
/* Transform feedback info in units of words, convert to bytes. */
|
||||
uint16_t stride = b->shader->info.xfb_stride[buffer] * 4;
|
||||
assert(stride != 0);
|
||||
|
||||
uint16_t offset = offset_words * 4;
|
||||
|
||||
nir_def *index = nir_iadd(
|
||||
b, nir_imul(b, nir_load_instance_id(b), nir_load_num_vertices(b)),
|
||||
nir_load_vertex_id_zero_base(b));
|
||||
|
||||
nir_def *xfb_offset =
|
||||
nir_iadd_imm(b, nir_imul_imm(b, index, stride), offset);
|
||||
|
||||
nir_def *buf = nir_load_xfb_address(b, 64, .base = buffer);
|
||||
nir_def *addr = nir_iadd(b, buf, nir_u2u64(b, xfb_offset));
|
||||
|
||||
nir_def *value = nir_channels(
|
||||
b, intr->src[0].ssa, BITFIELD_MASK(num_components) << start_component);
|
||||
nir_store_global(b, addr, 4, value, nir_component_mask(num_components));
|
||||
}
|
||||
|
||||
static bool
|
||||
lower_xfb(nir_builder *b, nir_intrinsic_instr *intr, UNUSED void *data)
|
||||
{
|
||||
if (intr->intrinsic != nir_intrinsic_store_output)
|
||||
return false;
|
||||
|
||||
/* Assume the inputs are read */
|
||||
BITSET_SET(b->shader->info.system_values_read,
|
||||
SYSTEM_VALUE_VERTEX_ID_ZERO_BASE);
|
||||
BITSET_SET(b->shader->info.system_values_read, SYSTEM_VALUE_INSTANCE_ID);
|
||||
|
||||
bool progress = false;
|
||||
|
||||
for (unsigned i = 0; i < 2; ++i) {
|
||||
nir_io_xfb xfb =
|
||||
i ? nir_intrinsic_io_xfb2(intr) : nir_intrinsic_io_xfb(intr);
|
||||
|
||||
for (unsigned j = 0; j < 2; ++j) {
|
||||
if (xfb.out[j].num_components > 0) {
|
||||
b->cursor = nir_before_instr(&intr->instr);
|
||||
lower_xfb_output(b, intr, i * 2 + j, xfb.out[j].num_components,
|
||||
xfb.out[j].buffer, xfb.out[j].offset);
|
||||
progress = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
nir_instr_remove(&intr->instr);
|
||||
return progress;
|
||||
}
|
||||
|
||||
static bool
|
||||
lower_xfb_intrinsics(struct nir_builder *b, nir_intrinsic_instr *intr,
|
||||
void *data)
|
||||
{
|
||||
b->cursor = nir_before_instr(&intr->instr);
|
||||
|
||||
struct agx_xfb_key *key = data;
|
||||
|
||||
switch (intr->intrinsic) {
|
||||
/* XXX: Rename to "xfb index" to avoid the clash */
|
||||
case nir_intrinsic_load_vertex_id_zero_base: {
|
||||
nir_def *id = nir_load_vertex_id(b);
|
||||
nir_def_rewrite_uses(&intr->def, id);
|
||||
return true;
|
||||
}
|
||||
|
||||
case nir_intrinsic_load_vertex_id: {
|
||||
/* Get the raw invocation ID */
|
||||
nir_def *id = nir_load_vertex_id(b);
|
||||
|
||||
/* Tessellate by primitive mode */
|
||||
if (key->mode == MESA_PRIM_LINE_STRIP ||
|
||||
key->mode == MESA_PRIM_LINE_LOOP) {
|
||||
/* The last vertex is special for a loop. Check if that's we're dealing
|
||||
* with.
|
||||
*/
|
||||
nir_def *num_invocations =
|
||||
nir_imul_imm(b, nir_load_num_vertices(b), 2);
|
||||
nir_def *last_vertex =
|
||||
nir_ieq(b, id, nir_iadd_imm(b, num_invocations, -1));
|
||||
|
||||
/* (0, 1), (1, 2) */
|
||||
id = nir_iadd(b, nir_ushr_imm(b, id, 1), nir_iand_imm(b, id, 1));
|
||||
|
||||
/* (0, 1), (1, 2), (2, 0) */
|
||||
if (key->mode == MESA_PRIM_LINE_LOOP) {
|
||||
id = nir_bcsel(b, last_vertex, nir_imm_int(b, 0), id);
|
||||
}
|
||||
} else if (key->mode == MESA_PRIM_TRIANGLE_STRIP) {
|
||||
/* Order depends on the provoking vertex.
|
||||
*
|
||||
* First: (0, 1, 2), (1, 3, 2), (2, 3, 4).
|
||||
* Last: (0, 1, 2), (2, 1, 3), (2, 3, 4).
|
||||
*/
|
||||
nir_def *prim = nir_udiv_imm(b, id, 3);
|
||||
nir_def *rem = nir_umod_imm(b, id, 3);
|
||||
|
||||
unsigned pv = key->flatshade_first ? 0 : 2;
|
||||
|
||||
/* Swap the two non-provoking vertices third vertex in odd triangles */
|
||||
nir_def *even = nir_ieq_imm(b, nir_iand_imm(b, prim, 1), 0);
|
||||
nir_def *is_provoking = nir_ieq_imm(b, rem, pv);
|
||||
nir_def *no_swap = nir_ior(b, is_provoking, even);
|
||||
nir_def *swapped = nir_isub_imm(b, 3 - pv, rem);
|
||||
nir_def *off = nir_bcsel(b, no_swap, rem, swapped);
|
||||
|
||||
/* Pull the (maybe swapped) vertex from the corresponding primitive */
|
||||
id = nir_iadd(b, prim, off);
|
||||
} else if (key->mode == MESA_PRIM_TRIANGLE_FAN) {
|
||||
/* (0, 1, 2), (0, 2, 3) */
|
||||
nir_def *prim = nir_udiv_imm(b, id, 3);
|
||||
nir_def *rem = nir_umod_imm(b, id, 3);
|
||||
|
||||
id = nir_bcsel(b, nir_ieq_imm(b, rem, 0), nir_imm_int(b, 0),
|
||||
nir_iadd(b, prim, rem));
|
||||
} else if (key->mode == MESA_PRIM_QUADS ||
|
||||
key->mode == MESA_PRIM_QUAD_STRIP) {
|
||||
/* Quads: [(0, 1, 3), (3, 1, 2)], [(4, 5, 7), (7, 5, 6)]
|
||||
* Quad strips: [(0, 1, 3), (0, 2, 3)], [(2, 3, 5), (2, 4, 5)]
|
||||
*/
|
||||
bool strips = key->mode == MESA_PRIM_QUAD_STRIP;
|
||||
|
||||
nir_def *prim = nir_udiv_imm(b, id, 6);
|
||||
nir_def *rem = nir_umod_imm(b, id, 6);
|
||||
nir_def *base = nir_imul_imm(b, prim, strips ? 2 : 4);
|
||||
|
||||
/* Quads: [0, 1, 3, 3, 1, 2]
|
||||
* Quad strips: [0, 1, 3, 0, 2, 3]
|
||||
*/
|
||||
uint32_t order_quads = 0x213310;
|
||||
uint32_t order_strips = 0x230310;
|
||||
uint32_t order = strips ? order_strips : order_quads;
|
||||
|
||||
/* Index out of the bitpacked array */
|
||||
nir_def *offset = nir_iand_imm(
|
||||
b, nir_ushr(b, nir_imm_int(b, order), nir_imul_imm(b, rem, 4)),
|
||||
0xF);
|
||||
|
||||
id = nir_iadd(b, base, offset);
|
||||
}
|
||||
|
||||
/* Add the "start", either an index bias or a base vertex */
|
||||
id = nir_iadd(b, id, nir_load_base_vertex(b));
|
||||
|
||||
/* If drawing with an index buffer, pull the vertex ID. Otherwise, the
|
||||
* vertex ID is just the index as-is.
|
||||
*/
|
||||
if (key->index_size) {
|
||||
nir_def *index_buffer = nir_load_xfb_index_buffer(b, 64);
|
||||
nir_def *offset = nir_imul_imm(b, id, key->index_size);
|
||||
nir_def *address = nir_iadd(b, index_buffer, nir_u2u64(b, offset));
|
||||
nir_def *index = nir_load_global_constant(b, address, key->index_size,
|
||||
1, key->index_size * 8);
|
||||
|
||||
id = nir_u2uN(b, index, id->bit_size);
|
||||
}
|
||||
|
||||
nir_def_rewrite_uses(&intr->def, id);
|
||||
return true;
|
||||
}
|
||||
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
agx_nir_lower_xfb(nir_shader *nir, struct agx_xfb_key *key)
|
||||
{
|
||||
assert(nir->info.stage == MESA_SHADER_VERTEX);
|
||||
|
||||
NIR_PASS_V(nir, nir_io_add_const_offset_to_base,
|
||||
nir_var_shader_in | nir_var_shader_out);
|
||||
NIR_PASS_V(nir, nir_io_add_intrinsic_xfb_info);
|
||||
|
||||
NIR_PASS_V(nir, insert_overflow_check, key);
|
||||
NIR_PASS_V(nir, nir_shader_intrinsics_pass, lower_xfb,
|
||||
nir_metadata_block_index | nir_metadata_dominance, key);
|
||||
NIR_PASS_V(nir, nir_shader_intrinsics_pass, lower_xfb_intrinsics,
|
||||
nir_metadata_block_index | nir_metadata_dominance, key);
|
||||
|
||||
/* Lowering XFB creates piles of dead code. Eliminate now so we don't
|
||||
* push unnecessary sysvals.
|
||||
*/
|
||||
NIR_PASS_V(nir, nir_opt_dce);
|
||||
}
|
||||
|
||||
void
|
||||
agx_init_streamout_functions(struct pipe_context *ctx)
|
||||
{
|
||||
|
||||
@@ -69,17 +69,6 @@ agx_upload_uniforms(struct agx_batch *batch)
|
||||
batch->uniforms.tables[AGX_SYSVAL_TABLE_ROOT] = root_ptr.gpu;
|
||||
batch->uniforms.sample_mask = ctx->sample_mask;
|
||||
|
||||
if (ctx->streamout.key.active) {
|
||||
batch->uniforms.xfb = ctx->streamout.params;
|
||||
|
||||
for (unsigned i = 0; i < batch->ctx->streamout.num_targets; ++i) {
|
||||
uint32_t size = 0;
|
||||
batch->uniforms.xfb.base[i] =
|
||||
agx_batch_get_so_address(batch, i, &size);
|
||||
batch->uniforms.xfb.size[i] = size;
|
||||
}
|
||||
}
|
||||
|
||||
memcpy(root_ptr.cpu, &batch->uniforms, sizeof(batch->uniforms));
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user