libagx: accelerate restart unroll across a subgroup
before implementing hard stream compaction algorithms, let's do the easy accelleration. Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/28483>
This commit is contained in:
Alyssa Rosenzweig
committed by
Marge Bot
Marge Bot
parent
b51282500d
commit
890a96e2a7
@@ -1397,14 +1397,15 @@ agx_nir_unroll_restart(nir_builder *b, const void *data)
|
||||
const struct agx_unroll_restart_key *key = data;
|
||||
nir_def *ia = nir_load_input_assembly_buffer_agx(b);
|
||||
nir_def *draw = nir_channel(b, nir_load_workgroup_id(b), 0);
|
||||
nir_def *lane = nir_load_subgroup_invocation(b);
|
||||
nir_def *mode = nir_imm_int(b, key->prim);
|
||||
|
||||
if (key->index_size_B == 1)
|
||||
libagx_unroll_restart_u8(b, ia, mode, draw);
|
||||
libagx_unroll_restart_u8(b, ia, mode, draw, lane);
|
||||
else if (key->index_size_B == 2)
|
||||
libagx_unroll_restart_u16(b, ia, mode, draw);
|
||||
libagx_unroll_restart_u16(b, ia, mode, draw, lane);
|
||||
else if (key->index_size_B == 4)
|
||||
libagx_unroll_restart_u32(b, ia, mode, draw);
|
||||
libagx_unroll_restart_u32(b, ia, mode, draw, lane);
|
||||
else
|
||||
unreachable("invalid index size");
|
||||
}
|
||||
|
||||
@@ -234,6 +234,15 @@ libagx_vertex_id_for_topology(enum mesa_prim mode, bool flatshade_first,
|
||||
}
|
||||
}
|
||||
|
||||
/* Return the ID of the first thread where cond is true. cond must be true for
|
||||
* some thread.
|
||||
*/
|
||||
static uint
|
||||
first_true_thread(bool cond)
|
||||
{
|
||||
return ctz(ballot(cond));
|
||||
}
|
||||
|
||||
/*
|
||||
* When unrolling the index buffer for a draw, we translate the old indirect
|
||||
* draws to new indirect draws. This routine allocates the new index buffer and
|
||||
@@ -273,7 +282,8 @@ setup_unroll_for_draw(global struct agx_ia_state *ia, constant uint *in_draw,
|
||||
|
||||
#define UNROLL(INDEX, suffix) \
|
||||
void libagx_unroll_restart_##suffix(global struct agx_ia_state *ia, \
|
||||
enum mesa_prim mode, uint draw) \
|
||||
enum mesa_prim mode, uint draw, \
|
||||
uint tid) \
|
||||
{ \
|
||||
/* For an indirect multidraw, we are dispatched maxDraws times and \
|
||||
* terminate trailing invocations. \
|
||||
@@ -288,8 +298,10 @@ setup_unroll_for_draw(global struct agx_ia_state *ia, constant uint *in_draw,
|
||||
constant INDEX *in = (constant INDEX *)ia->index_buffer; \
|
||||
in += in_draw[2]; \
|
||||
\
|
||||
global INDEX *out = \
|
||||
setup_unroll_for_draw(ia, in_draw, draw, mode, sizeof(INDEX)); \
|
||||
global INDEX *out; \
|
||||
if (tid == 0) \
|
||||
out = setup_unroll_for_draw(ia, in_draw, draw, mode, sizeof(INDEX)); \
|
||||
out = (global INDEX *)sub_group_broadcast((uintptr_t)out, 0); \
|
||||
\
|
||||
uint out_prims = 0; \
|
||||
INDEX restart_idx = ia->restart_index; \
|
||||
@@ -299,31 +311,42 @@ setup_unroll_for_draw(global struct agx_ia_state *ia, constant uint *in_draw,
|
||||
uint needle = 0; \
|
||||
uint per_prim = mesa_vertices_per_prim(mode); \
|
||||
while (needle < count) { \
|
||||
/* Search for next restart or the end */ \
|
||||
/* Search for next restart or the end. Lanes load in parallel. */ \
|
||||
uint next_restart = needle; \
|
||||
while ((next_restart < count) && in[next_restart] != restart_idx) \
|
||||
++next_restart; \
|
||||
for (;;) { \
|
||||
uint idx = next_restart + tid; \
|
||||
/* Relies on shortcircuiting */ \
|
||||
bool restart = idx >= count || in[idx] == restart_idx; \
|
||||
if (sub_group_any(restart)) { \
|
||||
uint first_restart_tid = first_true_thread(restart); \
|
||||
next_restart += first_restart_tid; \
|
||||
break; \
|
||||
} else { \
|
||||
/* No restart in this group, try the next */ \
|
||||
next_restart += get_sub_group_size(); \
|
||||
} \
|
||||
} \
|
||||
\
|
||||
/* Emit up to the next restart */ \
|
||||
/* Emit up to the next restart. Lanes output in parallel */ \
|
||||
uint subcount = next_restart - needle; \
|
||||
uint subprims = u_decomposed_prims_for_vertices(mode, subcount); \
|
||||
for (uint i = 0; i < subprims; ++i) { \
|
||||
uint out_prims_base = out_prims; \
|
||||
for (uint i = tid; i < subprims; i += get_sub_group_size()) { \
|
||||
for (uint vtx = 0; vtx < per_prim; ++vtx) { \
|
||||
uint id = libagx_vertex_id_for_topology(mode, flatshade_first, \
|
||||
i, vtx, subprims); \
|
||||
uint offset = needle + id; \
|
||||
\
|
||||
out[(out_prims * per_prim) + vtx] = \
|
||||
out[((out_prims_base + i) * per_prim) + vtx] = \
|
||||
offset < in_size_el ? in[offset] : 0; \
|
||||
} \
|
||||
\
|
||||
out_prims++; \
|
||||
} \
|
||||
\
|
||||
out_prims += subprims; \
|
||||
needle = next_restart + 1; \
|
||||
} \
|
||||
\
|
||||
ia->out_draws[(5 * draw) + 0] = out_prims * per_prim; \
|
||||
if (tid == 0) \
|
||||
ia->out_draws[(5 * draw) + 0] = out_prims * per_prim; \
|
||||
}
|
||||
|
||||
UNROLL(uchar, u8)
|
||||
|
||||
@@ -4356,7 +4356,7 @@ agx_draw_without_restart(struct agx_batch *batch,
|
||||
|
||||
/* Unroll the index buffer for each draw */
|
||||
const struct pipe_grid_info grid_setup = {
|
||||
.block = {1, 1, 1},
|
||||
.block = {32, 1, 1},
|
||||
.grid = {indirect->draw_count, 1, 1},
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user