From 2963cd900fa132b9ae438e3d0e662df226f72815 Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Fri, 8 Nov 2024 22:00:21 -0400 Subject: [PATCH] libagx: don't key unroll to index size Probably a premature optimization, it's annoying for precomp and for DGC. Signed-off-by: Alyssa Rosenzweig Part-of: --- src/asahi/lib/agx_nir_lower_gs.c | 9 +- src/asahi/lib/agx_nir_lower_gs.h | 1 - src/asahi/lib/shaders/geometry.cl | 163 +++++++++++++------------- src/asahi/lib/shaders/geometry.h | 5 +- src/asahi/vulkan/hk_cmd_draw.c | 6 +- src/gallium/drivers/asahi/agx_state.c | 2 +- 6 files changed, 93 insertions(+), 93 deletions(-) diff --git a/src/asahi/lib/agx_nir_lower_gs.c b/src/asahi/lib/agx_nir_lower_gs.c index dc06506cc21..6f2276029aa 100644 --- a/src/asahi/lib/agx_nir_lower_gs.c +++ b/src/asahi/lib/agx_nir_lower_gs.c @@ -1548,14 +1548,7 @@ agx_nir_unroll_restart(nir_builder *b, const void *data) nir_def *lane = nir_channel(b, nir_load_local_invocation_id(b), 0); nir_def *mode = nir_imm_int(b, key->prim); - if (key->index_size_B == 1) - libagx_unroll_restart_u8(b, ia, mode, draw, lane); - else if (key->index_size_B == 2) - libagx_unroll_restart_u16(b, ia, mode, draw, lane); - else if (key->index_size_B == 4) - libagx_unroll_restart_u32(b, ia, mode, draw, lane); - else - unreachable("invalid index size"); + libagx_unroll_restart(b, ia, mode, draw, lane); } void diff --git a/src/asahi/lib/agx_nir_lower_gs.h b/src/asahi/lib/agx_nir_lower_gs.h index 51dc1f13b27..de0c50f13fb 100644 --- a/src/asahi/lib/agx_nir_lower_gs.h +++ b/src/asahi/lib/agx_nir_lower_gs.h @@ -46,7 +46,6 @@ void agx_nir_gs_setup_indirect(struct nir_builder *b, const void *key); struct agx_unroll_restart_key { enum mesa_prim prim; - unsigned index_size_B; }; void agx_nir_unroll_restart(struct nir_builder *b, const void *key); diff --git a/src/asahi/lib/shaders/geometry.cl b/src/asahi/lib/shaders/geometry.cl index 4522f2c9355..200e1749612 100644 --- a/src/asahi/lib/shaders/geometry.cl +++ b/src/asahi/lib/shaders/geometry.cl @@ -332,88 +332,91 @@ setup_unroll_for_draw(global struct agx_restart_unroll_params *p, return (global uchar *)heap->heap + old_heap_bottom_B; } -#define UNROLL(INDEX, suffix) \ - kernel void libagx_unroll_restart_##suffix( \ - global struct agx_restart_unroll_params *p, enum mesa_prim mode, \ - uint draw, uint tid) \ - { \ - /* For an indirect multidraw, we are dispatched maxDraws times and \ - * terminate trailing invocations. \ - */ \ - if (p->count && draw >= *(p->count)) \ - return; \ - \ - constant uint *in_draw = \ - (constant uint *)(p->draws + (draw * p->draw_stride)); \ - \ - uint count = in_draw[0]; \ - \ - local uintptr_t out_ptr, in_ptr; \ - if (tid == 0) { \ - out_ptr = (uintptr_t)setup_unroll_for_draw(p, in_draw, draw, mode, \ - sizeof(INDEX)); \ - \ - /* Accessed thru local mem because NIR deref is too aggressive */ \ - in_ptr = (uintptr_t)(libagx_index_buffer( \ - p->index_buffer, p->index_buffer_size_el, in_draw[2], \ - sizeof(INDEX), p->zero_sink)); \ - } \ - \ - barrier(CLK_LOCAL_MEM_FENCE); \ - global INDEX *out = (global INDEX *)out_ptr; \ - \ - local uint scratch[32]; \ - \ - uint out_prims = 0; \ - INDEX restart_idx = p->restart_index; \ - bool flatshade_first = p->flatshade_first; \ - \ - uint needle = 0; \ - uint per_prim = mesa_vertices_per_prim(mode); \ - while (needle < count) { \ - /* Search for next restart or the end. Lanes load in parallel. */ \ - uint next_restart = needle; \ - for (;;) { \ - uint idx = next_restart + tid; \ - bool restart = \ - idx >= count || libagx_load_index_buffer_internal( \ - in_ptr, p->index_buffer_size_el, idx, \ - sizeof(INDEX)) == restart_idx; \ - \ - uint next_offs = first_true_thread_in_workgroup(restart, scratch); \ - \ - next_restart += next_offs; \ - if (next_offs < 1024) \ - break; \ - } \ - \ - /* Emit up to the next restart. Lanes output in parallel */ \ - uint subcount = next_restart - needle; \ - uint subprims = u_decomposed_prims_for_vertices(mode, subcount); \ - uint out_prims_base = out_prims; \ - for (uint i = tid; i < subprims; i += 1024) { \ - for (uint vtx = 0; vtx < per_prim; ++vtx) { \ - uint id = libagx_vertex_id_for_topology(mode, flatshade_first, \ - i, vtx, subprims); \ - uint offset = needle + id; \ - \ - out[((out_prims_base + i) * per_prim) + vtx] = \ - libagx_load_index_buffer_internal( \ - in_ptr, p->index_buffer_size_el, offset, sizeof(INDEX)); \ - } \ - } \ - \ - out_prims += subprims; \ - needle = next_restart + 1; \ - } \ - \ - if (tid == 0) \ - p->out_draws[(5 * draw) + 0] = out_prims * per_prim; \ +kernel void +libagx_unroll_restart(global struct agx_restart_unroll_params *p, + enum mesa_prim mode, uint draw, uint tid) +{ + /* For an indirect multidraw, we are dispatched maxDraws times and + * terminate trailing invocations. + */ + if (p->count && draw >= *(p->count)) + return; + + constant uint *in_draw = + (constant uint *)(p->draws + (draw * p->draw_stride)); + + uint count = in_draw[0]; + + local uintptr_t out_ptr, in_ptr; + if (tid == 0) { + out_ptr = (uintptr_t)setup_unroll_for_draw(p, in_draw, draw, mode, + p->index_size_B); + + /* Accessed thru local mem because NIR deref is too aggressive */ + in_ptr = (uintptr_t)(libagx_index_buffer( + p->index_buffer, p->index_buffer_size_el, in_draw[2], p->index_size_B, + p->zero_sink)); } -UNROLL(uchar, u8) -UNROLL(ushort, u16) -UNROLL(uint, u32) + barrier(CLK_LOCAL_MEM_FENCE); + global uint32_t *out_32 = (global uint32_t *)out_ptr; + global uint16_t *out_16 = (global uint16_t *)out_ptr; + global uint8_t *out_8 = (global uint8_t *)out_ptr; + + local uint scratch[32]; + + uint out_prims = 0; + uint32_t restart_idx = p->restart_index; + bool flatshade_first = p->flatshade_first; + + uint needle = 0; + uint per_prim = mesa_vertices_per_prim(mode); + while (needle < count) { + /* Search for next restart or the end. Lanes load in parallel. */ + uint next_restart = needle; + for (;;) { + uint idx = next_restart + tid; + bool restart = idx >= count || libagx_load_index_buffer_internal( + in_ptr, p->index_buffer_size_el, idx, + p->index_size_B) == restart_idx; + + uint next_offs = first_true_thread_in_workgroup(restart, scratch); + + next_restart += next_offs; + if (next_offs < 1024) + break; + } + + /* Emit up to the next restart. Lanes output in parallel */ + uint subcount = next_restart - needle; + uint subprims = u_decomposed_prims_for_vertices(mode, subcount); + uint out_prims_base = out_prims; + for (uint i = tid; i < subprims; i += 1024) { + for (uint vtx = 0; vtx < per_prim; ++vtx) { + uint id = libagx_vertex_id_for_topology(mode, flatshade_first, i, + vtx, subprims); + uint offset = needle + id; + + uint x = ((out_prims_base + i) * per_prim) + vtx; + uint y = libagx_load_index_buffer_internal( + in_ptr, p->index_buffer_size_el, offset, p->index_size_B); + + if (p->index_size_B == 4) + out_32[x] = y; + else if (p->index_size_B == 2) + out_16[x] = y; + else + out_8[x] = y; + } + } + + out_prims += subprims; + needle = next_restart + 1; + } + + if (tid == 0) + p->out_draws[(5 * draw) + 0] = out_prims * per_prim; +} uint libagx_setup_xfb_buffer(global struct agx_geometry_params *p, uint i) diff --git a/src/asahi/lib/shaders/geometry.h b/src/asahi/lib/shaders/geometry.h index fb432fe16bd..553a65924a0 100644 --- a/src/asahi/lib/shaders/geometry.h +++ b/src/asahi/lib/shaders/geometry.h @@ -60,6 +60,9 @@ struct agx_restart_unroll_params { /* Input index buffer size in elements */ uint32_t index_buffer_size_el; + /* Size of an index in bytes */ + uint32_t index_size_B; + /* Stride for the draw descriptor array */ uint32_t draw_stride; @@ -69,7 +72,7 @@ struct agx_restart_unroll_params { */ uint32_t flatshade_first; } PACKED; -AGX_STATIC_ASSERT(sizeof(struct agx_restart_unroll_params) == 17 * 4); +AGX_STATIC_ASSERT(sizeof(struct agx_restart_unroll_params) == 18 * 4); struct agx_gs_setup_indirect_params { /* Index buffer if present. */ diff --git a/src/asahi/vulkan/hk_cmd_draw.c b/src/asahi/vulkan/hk_cmd_draw.c index 44e9b9928ec..1666a7f5ac0 100644 --- a/src/asahi/vulkan/hk_cmd_draw.c +++ b/src/asahi/vulkan/hk_cmd_draw.c @@ -1456,9 +1456,10 @@ hk_draw_without_restart(struct hk_cmd_buffer *cmd, struct hk_cs *cs, /* Next, we unroll the index buffer used by the indirect draw */ struct agx_unroll_restart_key key = { .prim = vk_conv_topology(dyn->ia.primitive_topology), - .index_size_B = agx_index_size_to_B(draw.index_size), }; + uint32_t index_size_B = agx_index_size_to_B(draw.index_size); + struct agx_restart_unroll_params ia = { .heap = hk_geometry_state(cmd), .index_buffer = draw.index.addr, @@ -1467,10 +1468,11 @@ hk_draw_without_restart(struct hk_cmd_buffer *cmd, struct hk_cs *cs, .out_draws = hk_pool_alloc(cmd, 5 * sizeof(uint32_t) * draw_count, 4).gpu, .max_draws = 1 /* TODO: MDI */, .restart_index = gfx->index.restart, - .index_buffer_size_el = draw.index.range / key.index_size_B, + .index_buffer_size_el = draw.index.range / index_size_B, .flatshade_first = dyn->rs.provoking_vertex == VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT, .zero_sink = dev->rodata.zero_sink, + .index_size_B = index_size_B, }; struct hk_shader *s = diff --git a/src/gallium/drivers/asahi/agx_state.c b/src/gallium/drivers/asahi/agx_state.c index 62d166d46f2..3312d432f3d 100644 --- a/src/gallium/drivers/asahi/agx_state.c +++ b/src/gallium/drivers/asahi/agx_state.c @@ -4269,7 +4269,6 @@ agx_draw_without_restart(struct agx_batch *batch, struct agx_unroll_restart_key key = { .prim = info->mode, - .index_size_B = info->index_size, }; /* Allocate output indirect draw descriptors. This is exact. */ @@ -4286,6 +4285,7 @@ agx_draw_without_restart(struct agx_batch *batch, .index_buffer_size_el = ib_extent / info->index_size, .flatshade_first = batch->ctx->rast->base.flatshade_first, .draws = agx_indirect_buffer_ptr(batch, indirect), + .index_size_B = info->index_size, }; /* Unroll the index buffer for each draw */