poly: Migrate AGX's GS/TESS emulation to common code
This moves most of the code to a new home: src/poly. Most precomp kernels logic that could be moved are provided by poly now. Signed-off-by: Mary Guillemard <mary.guillemard@collabora.com> Acked-by: Alyssa Rosenzweig <alyssa.rosenzweig@intel.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/37914>
This commit is contained in:
Mary Guillemard
committed by
Marge Bot
Marge Bot
parent
8048004238
commit
b2accf86d1
@@ -0,0 +1,8 @@
|
||||
|
||||
BasedOnStyle: InheritParentConfig
|
||||
DisableFormat: false
|
||||
|
||||
AlignConsecutiveBitFields: Consecutive
|
||||
ColumnLimit: 80
|
||||
BreakStringLiterals: false
|
||||
SpaceBeforeParens: ControlStatementsExceptControlMacros
|
||||
@@ -0,0 +1,501 @@
|
||||
/*
|
||||
* Copyright 2023 Alyssa Rosenzweig
|
||||
* Copyright 2023 Valve Corporation
|
||||
* Copyright 2025 Collabora Ltd.
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#include "compiler/libcl/libcl_vk.h"
|
||||
#include "poly/geometry.h"
|
||||
#include "poly/tessellator.h"
|
||||
#include "util/macros.h"
|
||||
#include "util/u_math.h"
|
||||
|
||||
uint64_t nir_ro_to_rw_poly(uint64_t address);
|
||||
|
||||
/* Swap the two non-provoking vertices in odd triangles. This generates a vertex
|
||||
* ID list with a consistent winding order.
|
||||
*
|
||||
* Holding prim and flatshade_first constant, the map : [0, 1, 2] -> [0, 1, 2]
|
||||
* is its own inverse. It is hence used both vertex fetch and transform
|
||||
* feedback.
|
||||
*/
|
||||
static uint
|
||||
map_vertex_in_tri_strip(uint prim, uint vert, bool flatshade_first)
|
||||
{
|
||||
unsigned pv = flatshade_first ? 0 : 2;
|
||||
|
||||
bool even = (prim & 1) == 0;
|
||||
bool provoking = vert == pv;
|
||||
|
||||
return (provoking || even) ? vert : ((3 - pv) - vert);
|
||||
}
|
||||
|
||||
static inline uint
|
||||
xfb_prim(uint id, uint n, uint copy)
|
||||
{
|
||||
return sub_sat(id, n - 1u) + copy;
|
||||
}
|
||||
|
||||
/*
|
||||
* Determine whether an output vertex has an n'th copy in the transform feedback
|
||||
* buffer. This is written weirdly to let constant folding remove unnecessary
|
||||
* stores when length is known statically.
|
||||
*/
|
||||
bool
|
||||
poly_xfb_vertex_copy_in_strip(uint n, uint id, uint length, uint copy)
|
||||
{
|
||||
uint prim = xfb_prim(id, n, copy);
|
||||
|
||||
int num_prims = length - (n - 1);
|
||||
return copy == 0 || (prim < num_prims && id >= copy && copy < num_prims);
|
||||
}
|
||||
|
||||
uint
|
||||
poly_xfb_vertex_offset(uint n, uint invocation_base_prim, uint strip_base_prim,
|
||||
uint id_in_strip, uint copy, bool flatshade_first)
|
||||
{
|
||||
uint prim = xfb_prim(id_in_strip, n, copy);
|
||||
uint vert_0 = min(id_in_strip, n - 1);
|
||||
uint vert = vert_0 - copy;
|
||||
|
||||
if (n == 3) {
|
||||
vert = map_vertex_in_tri_strip(prim, vert, flatshade_first);
|
||||
}
|
||||
|
||||
/* Tally up in the whole buffer */
|
||||
uint base_prim = invocation_base_prim + strip_base_prim;
|
||||
uint base_vertex = base_prim * n;
|
||||
return base_vertex + (prim * n) + vert;
|
||||
}
|
||||
|
||||
uint64_t
|
||||
poly_xfb_vertex_address(constant struct poly_geometry_params *p, uint index,
|
||||
uint buffer, uint stride, uint output_offset)
|
||||
{
|
||||
uint xfb_offset = (index * stride) + output_offset;
|
||||
|
||||
return (uintptr_t)(p->xfb_base[buffer]) + xfb_offset;
|
||||
}
|
||||
|
||||
static uint
|
||||
vertex_id_for_line_loop(uint prim, uint vert, uint num_prims)
|
||||
{
|
||||
/* (0, 1), (1, 2), (2, 0) */
|
||||
if (prim == (num_prims - 1) && vert == 1)
|
||||
return 0;
|
||||
else
|
||||
return prim + vert;
|
||||
}
|
||||
|
||||
uint
|
||||
poly_vertex_id_for_line_class(enum mesa_prim mode, uint prim, uint vert,
|
||||
uint num_prims)
|
||||
{
|
||||
/* Line list, line strip, or line loop */
|
||||
if (mode == MESA_PRIM_LINE_LOOP && prim == (num_prims - 1) && vert == 1)
|
||||
return 0;
|
||||
|
||||
if (mode == MESA_PRIM_LINES)
|
||||
prim *= 2;
|
||||
|
||||
return prim + vert;
|
||||
}
|
||||
|
||||
static uint
|
||||
vertex_id_for_tri_fan(uint prim, uint vert, bool flatshade_first)
|
||||
{
|
||||
/* Vulkan spec section 20.1.7 gives (i + 1, i + 2, 0) for a provoking
|
||||
* first. OpenGL instead wants (0, i + 1, i + 2) with a provoking last.
|
||||
* Piglit clipflat expects us to switch between these orders depending on
|
||||
* provoking vertex, to avoid trivializing the fan.
|
||||
*
|
||||
* Rotate accordingly.
|
||||
*/
|
||||
if (flatshade_first) {
|
||||
vert = (vert == 2) ? 0 : (vert + 1);
|
||||
}
|
||||
|
||||
/* The simpler form assuming last is provoking. */
|
||||
return (vert == 0) ? 0 : prim + vert;
|
||||
}
|
||||
|
||||
uint
|
||||
poly_vertex_id_for_tri_class(enum mesa_prim mode, uint prim, uint vert,
|
||||
bool flatshade_first)
|
||||
{
|
||||
if (flatshade_first && mode == MESA_PRIM_TRIANGLE_FAN) {
|
||||
vert = vert + 1;
|
||||
vert = (vert == 3) ? 0 : vert;
|
||||
}
|
||||
|
||||
if (mode == MESA_PRIM_TRIANGLE_FAN && vert == 0)
|
||||
return 0;
|
||||
|
||||
if (mode == MESA_PRIM_TRIANGLES)
|
||||
prim *= 3;
|
||||
|
||||
/* Triangle list, triangle strip, or triangle fan */
|
||||
if (mode == MESA_PRIM_TRIANGLE_STRIP) {
|
||||
unsigned pv = flatshade_first ? 0 : 2;
|
||||
|
||||
bool even = (prim & 1) == 0;
|
||||
bool provoking = vert == pv;
|
||||
|
||||
vert = ((provoking || even) ? vert : ((3 - pv) - vert));
|
||||
}
|
||||
|
||||
return prim + vert;
|
||||
}
|
||||
|
||||
uint
|
||||
poly_vertex_id_for_line_adj_class(enum mesa_prim mode, uint prim, uint vert)
|
||||
{
|
||||
/* Line list adj or line strip adj */
|
||||
if (mode == MESA_PRIM_LINES_ADJACENCY)
|
||||
prim *= 4;
|
||||
|
||||
return prim + vert;
|
||||
}
|
||||
|
||||
static uint
|
||||
vertex_id_for_tri_strip_adj(uint prim, uint vert, uint num_prims,
|
||||
bool flatshade_first)
|
||||
{
|
||||
/* See Vulkan spec section 20.1.11 "Triangle Strips With Adjancency".
|
||||
*
|
||||
* There are different cases for first/middle/last/only primitives and for
|
||||
* odd/even primitives. Determine which case we're in.
|
||||
*/
|
||||
bool last = prim == (num_prims - 1);
|
||||
bool first = prim == 0;
|
||||
bool even = (prim & 1) == 0;
|
||||
bool even_or_first = even || first;
|
||||
|
||||
/* When the last vertex is provoking, we rotate the primitives
|
||||
* accordingly. This seems required for OpenGL.
|
||||
*/
|
||||
if (!flatshade_first && !even_or_first) {
|
||||
vert = (vert + 4u) % 6u;
|
||||
}
|
||||
|
||||
/* Offsets per the spec. The spec lists 6 cases with 6 offsets. Luckily,
|
||||
* there are lots of patterns we can exploit, avoiding a full 6x6 LUT.
|
||||
*
|
||||
* Here we assume the first vertex is provoking, the Vulkan default.
|
||||
*/
|
||||
uint offsets[6] = {
|
||||
0,
|
||||
first ? 1 : (even ? -2 : 3),
|
||||
even_or_first ? 2 : 4,
|
||||
last ? 5 : 6,
|
||||
even_or_first ? 4 : 2,
|
||||
even_or_first ? 3 : -2,
|
||||
};
|
||||
|
||||
/* Ensure NIR can see thru the local array */
|
||||
uint offset = 0;
|
||||
for (uint i = 1; i < 6; ++i) {
|
||||
if (i == vert)
|
||||
offset = offsets[i];
|
||||
}
|
||||
|
||||
/* Finally add to the base of the primitive */
|
||||
return (prim * 2) + offset;
|
||||
}
|
||||
|
||||
uint
|
||||
poly_vertex_id_for_tri_adj_class(enum mesa_prim mode, uint prim, uint vert,
|
||||
uint nr, bool flatshade_first)
|
||||
{
|
||||
/* Tri adj list or tri adj strip */
|
||||
if (mode == MESA_PRIM_TRIANGLE_STRIP_ADJACENCY) {
|
||||
return vertex_id_for_tri_strip_adj(prim, vert, nr, flatshade_first);
|
||||
} else {
|
||||
return (6 * prim) + vert;
|
||||
}
|
||||
}
|
||||
|
||||
static uint
|
||||
vertex_id_for_topology(enum mesa_prim mode, bool flatshade_first, uint prim,
|
||||
uint vert, uint num_prims)
|
||||
{
|
||||
switch (mode) {
|
||||
case MESA_PRIM_POINTS:
|
||||
case MESA_PRIM_LINES:
|
||||
case MESA_PRIM_TRIANGLES:
|
||||
case MESA_PRIM_LINES_ADJACENCY:
|
||||
case MESA_PRIM_TRIANGLES_ADJACENCY:
|
||||
/* Regular primitive: every N vertices defines a primitive */
|
||||
return (prim * mesa_vertices_per_prim(mode)) + vert;
|
||||
|
||||
case MESA_PRIM_LINE_LOOP:
|
||||
return vertex_id_for_line_loop(prim, vert, num_prims);
|
||||
|
||||
case MESA_PRIM_LINE_STRIP:
|
||||
case MESA_PRIM_LINE_STRIP_ADJACENCY:
|
||||
/* (i, i + 1) or (i, ..., i + 3) */
|
||||
return prim + vert;
|
||||
|
||||
case MESA_PRIM_TRIANGLE_STRIP: {
|
||||
/* Order depends on the provoking vert.
|
||||
*
|
||||
* First: (0, 1, 2), (1, 3, 2), (2, 3, 4).
|
||||
* Last: (0, 1, 2), (2, 1, 3), (2, 3, 4).
|
||||
*
|
||||
* Pull the (maybe swapped) vert from the corresponding primitive
|
||||
*/
|
||||
return prim + map_vertex_in_tri_strip(prim, vert, flatshade_first);
|
||||
}
|
||||
|
||||
case MESA_PRIM_TRIANGLE_FAN:
|
||||
return vertex_id_for_tri_fan(prim, vert, flatshade_first);
|
||||
|
||||
case MESA_PRIM_TRIANGLE_STRIP_ADJACENCY:
|
||||
return vertex_id_for_tri_strip_adj(prim, vert, num_prims,
|
||||
flatshade_first);
|
||||
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
uint
|
||||
poly_map_to_line_adj(uint id)
|
||||
{
|
||||
/* Sequence (1, 2), (5, 6), (9, 10), ... */
|
||||
return ((id & ~1) * 2) + (id & 1) + 1;
|
||||
}
|
||||
|
||||
uint
|
||||
poly_map_to_line_strip_adj(uint id)
|
||||
{
|
||||
/* Sequence (1, 2), (2, 3), (4, 5), .. */
|
||||
uint prim = id / 2;
|
||||
uint vert = id & 1;
|
||||
return prim + vert + 1;
|
||||
}
|
||||
|
||||
uint
|
||||
poly_map_to_tri_strip_adj(uint id)
|
||||
{
|
||||
/* Sequence (0, 2, 4), (2, 6, 4), (4, 6, 8), (6, 10, 8)
|
||||
*
|
||||
* Although tri strips with adjacency have 6 cases in general, after
|
||||
* disregarding the vertices only available in a geometry shader, there are
|
||||
* only even/odd cases. In other words, it's just a triangle strip subject to
|
||||
* extra padding.
|
||||
*
|
||||
* Dividing through by two, the sequence is:
|
||||
*
|
||||
* (0, 1, 2), (1, 3, 2), (2, 3, 4), (3, 5, 4)
|
||||
*/
|
||||
uint prim = id / 3;
|
||||
uint vtx = id % 3;
|
||||
|
||||
/* Flip the winding order of odd triangles */
|
||||
if ((prim % 2) == 1) {
|
||||
if (vtx == 1)
|
||||
vtx = 2;
|
||||
else if (vtx == 2)
|
||||
vtx = 1;
|
||||
}
|
||||
|
||||
return 2 * (prim + vtx);
|
||||
}
|
||||
|
||||
uint
|
||||
poly_load_index_buffer(constant struct poly_ia_state *p, uint id,
|
||||
uint index_size)
|
||||
{
|
||||
return poly_load_index(p->index_buffer, p->index_buffer_range_el, id,
|
||||
index_size);
|
||||
}
|
||||
|
||||
static uint
|
||||
setup_xfb_buffer(global struct poly_geometry_params *p, uint i, uint stride,
|
||||
uint max_output_end, uint vertices_per_prim)
|
||||
{
|
||||
uint xfb_offset = *(p->xfb_offs_ptrs[i]);
|
||||
p->xfb_base[i] = p->xfb_base_original[i] + xfb_offset;
|
||||
|
||||
/* Let output_end = output_offset + output_size.
|
||||
*
|
||||
* Primitive P will write up to (but not including) offset:
|
||||
*
|
||||
* xfb_offset + ((P - 1) * (verts_per_prim * stride))
|
||||
* + ((verts_per_prim - 1) * stride)
|
||||
* + output_end
|
||||
*
|
||||
* To fit all outputs for P, that value must be less than the XFB
|
||||
* buffer size for the output with maximal output_end, as everything
|
||||
* else is constant here across outputs within a buffer/primitive:
|
||||
*
|
||||
* floor(P) <= (stride + size - xfb_offset - output_end)
|
||||
* // (stride * verts_per_prim)
|
||||
*/
|
||||
int numer_s = p->xfb_size[i] + (stride - max_output_end) - xfb_offset;
|
||||
uint numer = max(numer_s, 0);
|
||||
return numer / (stride * vertices_per_prim);
|
||||
}
|
||||
|
||||
void
|
||||
poly_write_strip(GLOBAL uint32_t *index_buffer, uint32_t inv_index_offset,
|
||||
uint32_t prim_index_offset, uint32_t vertex_offset,
|
||||
uint32_t verts_in_prim, uint3 info)
|
||||
{
|
||||
_poly_write_strip(index_buffer, inv_index_offset + prim_index_offset,
|
||||
vertex_offset, verts_in_prim, info.x, info.y, info.z);
|
||||
}
|
||||
|
||||
void
|
||||
poly_pad_index_gs(global int *index_buffer, uint inv_index_offset,
|
||||
uint nr_indices, uint alloc)
|
||||
{
|
||||
for (uint i = nr_indices; i < alloc; ++i) {
|
||||
index_buffer[inv_index_offset + i] = -1;
|
||||
}
|
||||
}
|
||||
|
||||
uintptr_t
|
||||
poly_vertex_output_address(uintptr_t buffer, uint64_t mask, uint vtx,
|
||||
gl_varying_slot location)
|
||||
{
|
||||
/* Written like this to let address arithmetic work */
|
||||
return buffer + ((uintptr_t)poly_tcs_in_offs_el(vtx, location, mask)) * 16;
|
||||
}
|
||||
|
||||
uintptr_t
|
||||
poly_geometry_input_address(constant struct poly_geometry_params *p, uint vtx,
|
||||
gl_varying_slot location)
|
||||
{
|
||||
return poly_vertex_output_address(p->input_buffer, p->input_mask, vtx,
|
||||
location);
|
||||
}
|
||||
|
||||
unsigned
|
||||
poly_input_vertices(constant struct poly_ia_state *ia)
|
||||
{
|
||||
return ia->verts_per_instance;
|
||||
}
|
||||
|
||||
global uint *
|
||||
poly_load_xfb_count_address(constant struct poly_geometry_params *p, int index,
|
||||
int count_words, uint unrolled_id)
|
||||
{
|
||||
return &p->count_buffer[(unrolled_id * count_words) + index];
|
||||
}
|
||||
|
||||
uint
|
||||
poly_previous_xfb_primitives(global struct poly_geometry_params *p,
|
||||
int static_count, int count_index, int count_words,
|
||||
bool prefix_sum, uint unrolled_id)
|
||||
{
|
||||
if (static_count >= 0) {
|
||||
/* If the number of outputted vertices per invocation is known statically,
|
||||
* we can calculate the base.
|
||||
*/
|
||||
return unrolled_id * static_count;
|
||||
} else {
|
||||
/* Otherwise, load from the count buffer buffer. Note that the sums are
|
||||
* inclusive, so index 0 is nonzero. This requires a little fixup here. We
|
||||
* use a saturating unsigned subtraction so we don't read out-of-bounds.
|
||||
*
|
||||
* If we didn't prefix sum, there's only one element.
|
||||
*/
|
||||
uint prim_minus_1 = prefix_sum ? sub_sat(unrolled_id, 1u) : 0;
|
||||
uint count = p->count_buffer[(prim_minus_1 * count_words) + count_index];
|
||||
|
||||
return unrolled_id == 0 ? 0 : count;
|
||||
}
|
||||
}
|
||||
|
||||
/* Like u_foreach_bit, specialized for XFB to enable loop unrolling */
|
||||
#define poly_foreach_xfb(word, index) \
|
||||
for (uint i = 0; i < 4; ++i) \
|
||||
if (word & BITFIELD_BIT(i))
|
||||
|
||||
void
|
||||
poly_pre_gs(global struct poly_geometry_params *p, uint streams,
|
||||
uint buffers_written, uint4 buffer_to_stream, int4 count_index,
|
||||
uint4 stride, uint4 output_end, int4 static_count, uint invocations,
|
||||
uint vertices_per_prim, global uint *gs_invocations,
|
||||
global uint *gs_primitives, global uint *c_primitives,
|
||||
global uint *c_invocations)
|
||||
{
|
||||
unsigned count_words = !!(count_index[0] >= 0) + !!(count_index[1] >= 0) +
|
||||
!!(count_index[2] >= 0) + !!(count_index[3] >= 0);
|
||||
bool prefix_sum = count_words && buffers_written;
|
||||
uint unrolled_in_prims = p->input_primitives;
|
||||
|
||||
/* Determine the number of primitives generated in each stream */
|
||||
uint4 in_prims = 0;
|
||||
poly_foreach_xfb(streams, i) {
|
||||
in_prims[i] = poly_previous_xfb_primitives(p, static_count[i],
|
||||
count_index[i], count_words,
|
||||
prefix_sum, unrolled_in_prims);
|
||||
|
||||
*(p->prims_generated_counter[i]) += in_prims[i];
|
||||
}
|
||||
|
||||
uint4 prims = in_prims;
|
||||
uint emitted_prims = prims[0] + prims[1] + prims[2] + prims[3];
|
||||
|
||||
if (buffers_written) {
|
||||
poly_foreach_xfb(buffers_written, i) {
|
||||
uint max_prims =
|
||||
setup_xfb_buffer(p, i, stride[i], output_end[i], vertices_per_prim);
|
||||
|
||||
unsigned stream = buffer_to_stream[i];
|
||||
prims[stream] = min(prims[stream], max_prims);
|
||||
}
|
||||
|
||||
int4 overflow = prims < in_prims;
|
||||
|
||||
poly_foreach_xfb(streams, i) {
|
||||
p->xfb_verts[i] = prims[i] * vertices_per_prim;
|
||||
|
||||
*(p->xfb_overflow[i]) += (bool)overflow[i];
|
||||
*(p->xfb_prims_generated_counter[i]) += prims[i];
|
||||
}
|
||||
|
||||
*(p->xfb_any_overflow) += any(overflow);
|
||||
|
||||
/* Update XFB counters */
|
||||
poly_foreach_xfb(buffers_written, i) {
|
||||
uint32_t prim_stride_B = stride[i] * vertices_per_prim;
|
||||
unsigned stream = buffer_to_stream[i];
|
||||
|
||||
global uint *ptr = p->xfb_offs_ptrs[i];
|
||||
|
||||
ptr = (global uint *)nir_ro_to_rw_poly((uint64_t)ptr);
|
||||
*ptr += prims[stream] * prim_stride_B;
|
||||
}
|
||||
}
|
||||
|
||||
/* The geometry shader is invoked once per primitive (after unrolling
|
||||
* primitive restart). From the spec:
|
||||
*
|
||||
* In case of instanced geometry shaders (see section 11.3.4.2) the
|
||||
* geometry shader invocations count is incremented for each separate
|
||||
* instanced invocation.
|
||||
*/
|
||||
*gs_invocations += unrolled_in_prims * invocations;
|
||||
*gs_primitives += emitted_prims;
|
||||
|
||||
/* Clipper queries are not well-defined, so we can emulate them in lots of
|
||||
* silly ways. We need the hardware counters to implement them properly. For
|
||||
* now, just consider all primitives emitted as passing through the clipper.
|
||||
* This satisfies spec text:
|
||||
*
|
||||
* The number of primitives that reach the primitive clipping stage.
|
||||
*
|
||||
* and
|
||||
*
|
||||
* If at least one vertex of the primitive lies inside the clipping
|
||||
* volume, the counter is incremented by one or more. Otherwise, the
|
||||
* counter is incremented by zero or more.
|
||||
*/
|
||||
*c_primitives += emitted_prims;
|
||||
*c_invocations += emitted_prims;
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
# Copyright 2024 Valve Corporation
|
||||
# Copyright © 2025 Collabora Ltd.
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
libpoly_shader_files = files(
|
||||
'geometry.cl',
|
||||
'tessellation.cl',
|
||||
)
|
||||
|
||||
libpoly_shaders_spv = custom_target(
|
||||
input : libpoly_shader_files,
|
||||
output : 'libpoly.spv',
|
||||
command : [
|
||||
prog_mesa_clc, '-o', '@OUTPUT@', '--depfile', '@DEPFILE@',
|
||||
libpoly_shader_files, '--',
|
||||
'-I' + join_paths(meson.project_source_root(), 'include'),
|
||||
'-I' + join_paths(meson.project_source_root(), 'src/compiler/libcl'),
|
||||
'-I' + join_paths(meson.current_source_dir(), '.'),
|
||||
'-I' + join_paths(meson.current_source_dir(), '../../'),
|
||||
cl_args,
|
||||
],
|
||||
depends : [],
|
||||
depfile : 'libpoly_shaders.h.d',
|
||||
)
|
||||
|
||||
libpoly_shaders = custom_target(
|
||||
input : libpoly_shaders_spv,
|
||||
output : ['libpoly.cpp', 'libpoly.h'],
|
||||
command : [prog_vtn_bindgen2, libpoly_shaders_spv, '@OUTPUT0@', '@OUTPUT1@'],
|
||||
)
|
||||
|
||||
idep_libpoly = declare_dependency(
|
||||
sources : [libpoly_shaders],
|
||||
include_directories : include_directories('.'),
|
||||
)
|
||||
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* Copyright 2023 Alyssa Rosenzweig
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#include "poly/geometry.h"
|
||||
#include "poly/tessellator.h"
|
||||
|
||||
uint
|
||||
poly_tcs_patch_vertices_in(constant struct poly_tess_args *p)
|
||||
{
|
||||
return p->input_patch_size;
|
||||
}
|
||||
|
||||
uint
|
||||
poly_tes_patch_vertices_in(constant struct poly_tess_args *p)
|
||||
{
|
||||
return p->output_patch_size;
|
||||
}
|
||||
|
||||
uint
|
||||
poly_tcs_unrolled_id(constant struct poly_tess_args *p, uint3 wg_id)
|
||||
{
|
||||
return (wg_id.y * p->patches_per_instance) + wg_id.x;
|
||||
}
|
||||
|
||||
uint64_t
|
||||
poly_tes_buffer(constant struct poly_tess_args *p)
|
||||
{
|
||||
return p->tes_buffer;
|
||||
}
|
||||
|
||||
/*
|
||||
* Helper to lower indexing for a tess eval shader ran as a compute shader. This
|
||||
* handles the tess+geom case. This is simpler than the general input assembly
|
||||
* lowering, as we know:
|
||||
*
|
||||
* 1. the index buffer is U32
|
||||
* 2. the index is in bounds
|
||||
*
|
||||
* Therefore we do a simple load. No bounds checking needed.
|
||||
*/
|
||||
uint32_t
|
||||
poly_load_tes_index(constant struct poly_tess_args *p, uint32_t index)
|
||||
{
|
||||
/* Swap second and third vertices of each triangle to flip winding order
|
||||
* dynamically if needed.
|
||||
*/
|
||||
if (p->ccw) {
|
||||
uint id = index % 3;
|
||||
|
||||
if (id == 1)
|
||||
index++;
|
||||
else if (id == 2)
|
||||
index--;
|
||||
}
|
||||
|
||||
return p->index_buffer[index];
|
||||
}
|
||||
|
||||
uintptr_t
|
||||
poly_tcs_out_address(constant struct poly_tess_args *p, uint patch_id,
|
||||
uint vtx_id, gl_varying_slot location, uint nr_patch_out,
|
||||
uint out_patch_size, uint64_t vtx_out_mask)
|
||||
{
|
||||
uint stride_el =
|
||||
poly_tcs_out_stride_el(nr_patch_out, out_patch_size, vtx_out_mask);
|
||||
|
||||
uint offs_el =
|
||||
poly_tcs_out_offs_el(vtx_id, location, nr_patch_out, vtx_out_mask);
|
||||
|
||||
offs_el += patch_id * stride_el;
|
||||
|
||||
/* Written to match the AGX addressing mode */
|
||||
return (uintptr_t)(p->tcs_buffer) + (((uintptr_t)offs_el) << 2);
|
||||
}
|
||||
|
||||
static uint
|
||||
tes_unrolled_patch_id(uint raw_id)
|
||||
{
|
||||
return raw_id / POLY_TES_PATCH_ID_STRIDE;
|
||||
}
|
||||
|
||||
uint
|
||||
poly_tes_patch_id(constant struct poly_tess_args *p, uint raw_id)
|
||||
{
|
||||
return tes_unrolled_patch_id(raw_id) % p->patches_per_instance;
|
||||
}
|
||||
|
||||
static uint
|
||||
tes_vertex_id_in_patch(uint raw_id)
|
||||
{
|
||||
return raw_id % POLY_TES_PATCH_ID_STRIDE;
|
||||
}
|
||||
|
||||
float2
|
||||
poly_load_tess_coord(constant struct poly_tess_args *p, uint raw_id)
|
||||
{
|
||||
uint patch = tes_unrolled_patch_id(raw_id);
|
||||
uint vtx = tes_vertex_id_in_patch(raw_id);
|
||||
|
||||
global struct poly_tess_point *t =
|
||||
&p->patch_coord_buffer[p->coord_allocs[patch] + vtx];
|
||||
|
||||
/* Written weirdly because NIR struggles with loads of structs */
|
||||
uint2 fixed = *((global uint2 *)t);
|
||||
|
||||
/* Convert fixed point to float */
|
||||
return convert_float2(fixed) / (1u << 16);
|
||||
}
|
||||
|
||||
uintptr_t
|
||||
poly_tes_in_address(constant struct poly_tess_args *p, uint raw_id, uint vtx_id,
|
||||
gl_varying_slot location)
|
||||
{
|
||||
uint patch = tes_unrolled_patch_id(raw_id);
|
||||
|
||||
return poly_tcs_out_address(p, patch, vtx_id, location,
|
||||
p->tcs_patch_constants, p->output_patch_size,
|
||||
p->tcs_per_vertex_outputs);
|
||||
}
|
||||
|
||||
float4
|
||||
poly_tess_level_outer_default(constant struct poly_tess_args *p)
|
||||
{
|
||||
return vload4(0, p->tess_level_outer_default);
|
||||
}
|
||||
|
||||
float2
|
||||
poly_tess_level_inner_default(constant struct poly_tess_args *p)
|
||||
{
|
||||
return vload2(0, p->tess_level_inner_default);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,641 @@
|
||||
/*
|
||||
* Copyright 2023 Alyssa Rosenzweig
|
||||
* Copyright 2023 Valve Corporation
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#include "compiler/libcl/libcl.h"
|
||||
#include "compiler/shader_enums.h"
|
||||
|
||||
#include "util/bitscan.h"
|
||||
#include "util/u_math.h"
|
||||
|
||||
#ifdef __OPENCL_VERSION__
|
||||
#include "compiler/libcl/libcl_vk.h"
|
||||
#endif
|
||||
|
||||
#pragma once
|
||||
|
||||
#define POLY_MAX_SO_BUFFERS 4
|
||||
#define POLY_MAX_VERTEX_STREAMS 4
|
||||
|
||||
enum poly_gs_shape {
|
||||
/* Indexed, where indices are encoded as:
|
||||
*
|
||||
* round_to_pot(max_indices) * round_to_pot(input_primitives) *
|
||||
* * instance_count
|
||||
*
|
||||
* invoked for max_indices * input_primitives * instance_count indices.
|
||||
*
|
||||
* This is used with any dynamic topology. No hardware instancing used.
|
||||
*/
|
||||
POLY_GS_SHAPE_DYNAMIC_INDEXED,
|
||||
|
||||
/* Indexed with a static index buffer. Indices ranges up to max_indices.
|
||||
* Hardware instance count = input_primitives * software instance count.
|
||||
*/
|
||||
POLY_GS_SHAPE_STATIC_INDEXED,
|
||||
|
||||
/* Non-indexed. Dispatched as:
|
||||
*
|
||||
* (max_indices, input_primitives * instance count).
|
||||
*/
|
||||
POLY_GS_SHAPE_STATIC_PER_PRIM,
|
||||
|
||||
/* Non-indexed. Dispatched as:
|
||||
*
|
||||
* (max_indices * input_primitives, instance count).
|
||||
*/
|
||||
POLY_GS_SHAPE_STATIC_PER_INSTANCE,
|
||||
};
|
||||
|
||||
static inline unsigned
|
||||
poly_gs_rast_vertices(enum poly_gs_shape shape, unsigned max_indices,
|
||||
unsigned input_primitives, unsigned instance_count)
|
||||
{
|
||||
switch (shape) {
|
||||
case POLY_GS_SHAPE_DYNAMIC_INDEXED:
|
||||
return max_indices * input_primitives * instance_count;
|
||||
|
||||
case POLY_GS_SHAPE_STATIC_INDEXED:
|
||||
case POLY_GS_SHAPE_STATIC_PER_PRIM:
|
||||
return max_indices;
|
||||
|
||||
case POLY_GS_SHAPE_STATIC_PER_INSTANCE:
|
||||
return max_indices * input_primitives;
|
||||
}
|
||||
|
||||
UNREACHABLE("invalid shape");
|
||||
}
|
||||
|
||||
static inline unsigned
|
||||
poly_gs_rast_instances(enum poly_gs_shape shape, unsigned max_indices,
|
||||
unsigned input_primitives, unsigned instance_count)
|
||||
{
|
||||
switch (shape) {
|
||||
case POLY_GS_SHAPE_DYNAMIC_INDEXED:
|
||||
return 1;
|
||||
|
||||
case POLY_GS_SHAPE_STATIC_INDEXED:
|
||||
case POLY_GS_SHAPE_STATIC_PER_PRIM:
|
||||
return input_primitives * instance_count;
|
||||
|
||||
case POLY_GS_SHAPE_STATIC_PER_INSTANCE:
|
||||
return instance_count;
|
||||
}
|
||||
|
||||
UNREACHABLE("invalid shape");
|
||||
}
|
||||
|
||||
static inline bool
|
||||
poly_gs_indexed(enum poly_gs_shape shape)
|
||||
{
|
||||
return shape == POLY_GS_SHAPE_DYNAMIC_INDEXED ||
|
||||
shape == POLY_GS_SHAPE_STATIC_INDEXED;
|
||||
}
|
||||
|
||||
static inline unsigned
|
||||
poly_gs_index_size(enum poly_gs_shape shape)
|
||||
{
|
||||
switch (shape) {
|
||||
case POLY_GS_SHAPE_DYNAMIC_INDEXED:
|
||||
return 4;
|
||||
case POLY_GS_SHAPE_STATIC_INDEXED:
|
||||
return 1;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* Heap to allocate from. */
|
||||
struct poly_heap {
|
||||
DEVICE(uchar) base;
|
||||
uint32_t bottom, size;
|
||||
} PACKED;
|
||||
static_assert(sizeof(struct poly_heap) == 4 * 4);
|
||||
|
||||
#ifdef __OPENCL_VERSION__
|
||||
static inline uint
|
||||
_poly_heap_alloc_offs(global struct poly_heap *heap, uint size_B, bool atomic)
|
||||
{
|
||||
size_B = align(size_B, 16);
|
||||
|
||||
uint offs;
|
||||
if (atomic) {
|
||||
offs = atomic_fetch_add((volatile atomic_uint *)(&heap->bottom), size_B);
|
||||
} else {
|
||||
offs = heap->bottom;
|
||||
heap->bottom = offs + size_B;
|
||||
}
|
||||
|
||||
/* Use printf+abort because assert is stripped from release builds. */
|
||||
if (heap->bottom >= heap->size) {
|
||||
printf(
|
||||
"FATAL: GPU heap overflow, allocating size %u, at offset %u, heap size %u!",
|
||||
size_B, offs, heap->size);
|
||||
|
||||
abort();
|
||||
}
|
||||
|
||||
return offs;
|
||||
}
|
||||
|
||||
static inline uint
|
||||
poly_heap_alloc_nonatomic_offs(global struct poly_heap *heap, uint size_B)
|
||||
{
|
||||
return _poly_heap_alloc_offs(heap, size_B, false);
|
||||
}
|
||||
|
||||
static inline uint
|
||||
poly_heap_alloc_atomic_offs(global struct poly_heap *heap, uint size_B)
|
||||
{
|
||||
return _poly_heap_alloc_offs(heap, size_B, true);
|
||||
}
|
||||
|
||||
static inline global void *
|
||||
poly_heap_alloc_nonatomic(global struct poly_heap *heap, uint size_B)
|
||||
{
|
||||
return heap->base + poly_heap_alloc_nonatomic_offs(heap, size_B);
|
||||
}
|
||||
|
||||
uint64_t nir_load_ro_sink_address_poly(void);
|
||||
|
||||
static inline uint64_t
|
||||
poly_index_buffer(uint64_t index_buffer, uint size_el, uint offset_el,
|
||||
uint elsize_B)
|
||||
{
|
||||
if (offset_el < size_el)
|
||||
return index_buffer + (offset_el * elsize_B);
|
||||
else
|
||||
return nir_load_ro_sink_address_poly();
|
||||
}
|
||||
#endif
|
||||
|
||||
struct poly_ia_state {
|
||||
/* Index buffer if present. */
|
||||
uint64_t index_buffer;
|
||||
|
||||
/* Size of the bound index buffer for bounds checking */
|
||||
uint32_t index_buffer_range_el;
|
||||
|
||||
/* Number of vertices per instance. Written by CPU for direct draw, indirect
|
||||
* setup kernel for indirect. This is used for VS->GS and VS->TCS indexing.
|
||||
*/
|
||||
uint32_t verts_per_instance;
|
||||
} PACKED;
|
||||
static_assert(sizeof(struct poly_ia_state) == 4 * 4);
|
||||
|
||||
static inline uint
|
||||
poly_index_buffer_range_el(uint size_el, uint offset_el)
|
||||
{
|
||||
return offset_el < size_el ? (size_el - offset_el) : 0;
|
||||
}
|
||||
|
||||
struct poly_geometry_params {
|
||||
/* Address of associated indirect draw buffer */
|
||||
DEVICE(uint) indirect_desc;
|
||||
|
||||
/* Address of count buffer. For an indirect draw, this will be written by the
|
||||
* indirect setup kernel.
|
||||
*/
|
||||
DEVICE(uint) count_buffer;
|
||||
|
||||
/* Address of the primitives generated counters */
|
||||
DEVICE(uint) prims_generated_counter[POLY_MAX_VERTEX_STREAMS];
|
||||
DEVICE(uint) xfb_prims_generated_counter[POLY_MAX_VERTEX_STREAMS];
|
||||
DEVICE(uint) xfb_overflow[POLY_MAX_VERTEX_STREAMS];
|
||||
DEVICE(uint) xfb_any_overflow;
|
||||
|
||||
/* Pointers to transform feedback buffer offsets in bytes */
|
||||
DEVICE(uint) xfb_offs_ptrs[POLY_MAX_SO_BUFFERS];
|
||||
|
||||
/* Output index buffer, allocated by pre-GS. */
|
||||
DEVICE(uint) output_index_buffer;
|
||||
|
||||
/* Address of transform feedback buffer in general, supplied by the CPU. */
|
||||
DEVICE(uchar) xfb_base_original[POLY_MAX_SO_BUFFERS];
|
||||
|
||||
/* Address of transform feedback for the current primitive. Written by pre-GS
|
||||
* program.
|
||||
*/
|
||||
DEVICE(uchar) xfb_base[POLY_MAX_SO_BUFFERS];
|
||||
|
||||
/* Address and present mask for the input to the geometry shader. These will
|
||||
* reflect the vertex shader for VS->GS or instead the tessellation
|
||||
* evaluation shader for TES->GS.
|
||||
*/
|
||||
uint64_t input_buffer;
|
||||
uint64_t input_mask;
|
||||
|
||||
/* Location-indexed mask of flat outputs, used for lowering GL edge flags. */
|
||||
uint64_t flat_outputs;
|
||||
|
||||
uint32_t xfb_size[POLY_MAX_SO_BUFFERS];
|
||||
|
||||
/* Number of vertices emitted by transform feedback per stream. Written by
|
||||
* the pre-GS program.
|
||||
*/
|
||||
uint32_t xfb_verts[POLY_MAX_VERTEX_STREAMS];
|
||||
|
||||
/* Within an indirect GS draw, the grids used to dispatch the VS/GS written
|
||||
* out by the GS indirect setup kernel or the CPU for a direct draw. This is
|
||||
* the "indirect local" format: first 3 is in threads, second 3 is in grid
|
||||
* blocks. This lets us use nontrivial workgroups with indirect draws without
|
||||
* needing any predication.
|
||||
*/
|
||||
uint32_t vs_grid[6];
|
||||
uint32_t gs_grid[6];
|
||||
|
||||
/* Number of input primitives across all instances, calculated by the CPU for
|
||||
* a direct draw or the GS indirect setup kernel for an indirect draw.
|
||||
*/
|
||||
uint32_t input_primitives;
|
||||
|
||||
/* Number of input primitives per instance, rounded up to a power-of-two and
|
||||
* with the base-2 log taken. This is used to partition the output vertex IDs
|
||||
* efficiently.
|
||||
*/
|
||||
uint32_t primitives_log2;
|
||||
|
||||
/* Number of bytes output by the GS count shader per input primitive (may be
|
||||
* 0), written by CPU and consumed by indirect draw setup shader for
|
||||
* allocating counts.
|
||||
*/
|
||||
uint32_t count_buffer_stride;
|
||||
|
||||
/* Dynamic input topology. Must be compatible with the geometry shader's
|
||||
* layout() declared input class.
|
||||
*/
|
||||
uint32_t input_topology;
|
||||
} PACKED;
|
||||
static_assert(sizeof(struct poly_geometry_params) == 86 * 4);
|
||||
|
||||
/* TCS shared memory layout:
|
||||
*
|
||||
* vec4 vs_outputs[VERTICES_IN_INPUT_PATCH][TOTAL_VERTEX_OUTPUTS];
|
||||
*
|
||||
* TODO: compact.
|
||||
*/
|
||||
static inline uint
|
||||
poly_tcs_in_offs_el(uint vtx, gl_varying_slot location,
|
||||
uint64_t crosslane_vs_out_mask)
|
||||
{
|
||||
uint base = vtx * util_bitcount64(crosslane_vs_out_mask);
|
||||
uint offs = util_bitcount64(crosslane_vs_out_mask &
|
||||
(((uint64_t)(1) << location) - 1));
|
||||
|
||||
return base + offs;
|
||||
}
|
||||
|
||||
static inline uint
|
||||
poly_tcs_in_size(uint32_t vertices_in_patch, uint64_t crosslane_vs_out_mask)
|
||||
{
|
||||
return vertices_in_patch * util_bitcount64(crosslane_vs_out_mask) * 16;
|
||||
}
|
||||
|
||||
/*
|
||||
* TCS out buffer layout, per-patch:
|
||||
*
|
||||
* float tess_level_outer[4];
|
||||
* float tess_level_inner[2];
|
||||
* vec4 patch_out[MAX_PATCH_OUTPUTS];
|
||||
* vec4 vtx_out[OUT_PATCH_SIZE][TOTAL_VERTEX_OUTPUTS];
|
||||
*
|
||||
* Vertex out are compacted based on the mask of written out. Patch
|
||||
* out are used as-is.
|
||||
*
|
||||
* Bounding boxes are ignored.
|
||||
*/
|
||||
static inline uint
|
||||
poly_tcs_out_offs_el(uint vtx_id, gl_varying_slot location, uint nr_patch_out,
|
||||
uint64_t vtx_out_mask)
|
||||
{
|
||||
uint off = 0;
|
||||
if (location == VARYING_SLOT_TESS_LEVEL_OUTER)
|
||||
return off;
|
||||
|
||||
off += 4;
|
||||
if (location == VARYING_SLOT_TESS_LEVEL_INNER)
|
||||
return off;
|
||||
|
||||
off += 2;
|
||||
if (location >= VARYING_SLOT_PATCH0)
|
||||
return off + (4 * (location - VARYING_SLOT_PATCH0));
|
||||
|
||||
/* Anything else is a per-vtx output */
|
||||
off += 4 * nr_patch_out;
|
||||
off += 4 * vtx_id * util_bitcount64(vtx_out_mask);
|
||||
|
||||
uint idx = util_bitcount64(vtx_out_mask & (((uint64_t)(1) << location) - 1));
|
||||
return off + (4 * idx);
|
||||
}
|
||||
|
||||
static inline uint
|
||||
poly_tcs_out_stride_el(uint nr_patch_out, uint out_patch_size,
|
||||
uint64_t vtx_out_mask)
|
||||
{
|
||||
return poly_tcs_out_offs_el(out_patch_size, VARYING_SLOT_POS, nr_patch_out,
|
||||
vtx_out_mask);
|
||||
}
|
||||
|
||||
static inline uint
|
||||
poly_tcs_out_stride(uint nr_patch_out, uint out_patch_size,
|
||||
uint64_t vtx_out_mask)
|
||||
{
|
||||
return poly_tcs_out_stride_el(nr_patch_out, out_patch_size, vtx_out_mask) *
|
||||
4;
|
||||
}
|
||||
|
||||
/* In a tess eval shader, stride for hw vertex ID */
|
||||
#define POLY_TES_PATCH_ID_STRIDE 8192
|
||||
|
||||
static inline uint
|
||||
poly_compact_prim(enum mesa_prim prim)
|
||||
{
|
||||
static_assert(MESA_PRIM_QUAD_STRIP == MESA_PRIM_QUADS + 1);
|
||||
static_assert(MESA_PRIM_POLYGON == MESA_PRIM_QUADS + 2);
|
||||
|
||||
#ifndef __OPENCL_VERSION__
|
||||
assert(prim != MESA_PRIM_QUADS);
|
||||
assert(prim != MESA_PRIM_QUAD_STRIP);
|
||||
assert(prim != MESA_PRIM_POLYGON);
|
||||
assert(prim != MESA_PRIM_PATCHES);
|
||||
#endif
|
||||
|
||||
return (prim >= MESA_PRIM_QUADS) ? (prim - 3) : prim;
|
||||
}
|
||||
|
||||
static inline enum mesa_prim
|
||||
poly_uncompact_prim(uint packed)
|
||||
{
|
||||
if (packed >= MESA_PRIM_QUADS)
|
||||
return (enum mesa_prim)(packed + 3);
|
||||
|
||||
return (enum mesa_prim)packed;
|
||||
}
|
||||
|
||||
/*
|
||||
* Write a strip into a 32-bit index buffer. This is the sequence:
|
||||
*
|
||||
* (b, b + 1, b + 2, ..., b + n - 1, -1) where -1 is the restart index
|
||||
*
|
||||
* For points, we write index buffers without restart just for remapping.
|
||||
*/
|
||||
static inline void
|
||||
_poly_write_strip(GLOBAL uint32_t *index_buffer, uint32_t index_offset,
|
||||
uint32_t vertex_offset, uint32_t verts_in_prim,
|
||||
uint32_t stream, uint32_t stream_multiplier, uint32_t n)
|
||||
{
|
||||
bool restart = n > 1;
|
||||
if (verts_in_prim < n)
|
||||
return;
|
||||
|
||||
GLOBAL uint32_t *out = &index_buffer[index_offset];
|
||||
|
||||
/* Write out indices for the strip */
|
||||
for (uint32_t i = 0; i < verts_in_prim; ++i) {
|
||||
out[i] = (vertex_offset + i) * stream_multiplier + stream;
|
||||
}
|
||||
|
||||
if (restart)
|
||||
out[verts_in_prim] = -1;
|
||||
}
|
||||
|
||||
static inline unsigned
|
||||
poly_decomposed_prims_for_vertices_with_tess(enum mesa_prim prim, int vertices,
|
||||
unsigned verts_per_patch)
|
||||
{
|
||||
if (prim >= MESA_PRIM_PATCHES) {
|
||||
return vertices / verts_per_patch;
|
||||
} else {
|
||||
return u_decomposed_prims_for_vertices(prim, vertices);
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef __OPENCL_VERSION__
|
||||
/*
|
||||
* Returns (work_group_scan_inclusive_add(x), work_group_sum(x)). Implemented
|
||||
* manually with subgroup ops and local memory since Mesa doesn't do those
|
||||
* lowerings yet.
|
||||
*/
|
||||
static inline uint2
|
||||
poly_work_group_scan_inclusive_add(uint x, local uint *scratch)
|
||||
{
|
||||
uint sg_id = get_sub_group_id();
|
||||
|
||||
/* Partial prefix sum of the subgroup */
|
||||
uint sg = sub_group_scan_inclusive_add(x);
|
||||
|
||||
/* Reduction (sum) for the subgroup */
|
||||
uint sg_sum = sub_group_broadcast(sg, 31);
|
||||
|
||||
/* Write out all the subgroups sums */
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
scratch[sg_id] = sg_sum;
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
|
||||
/* Read all the subgroup sums. Thread T in subgroup G reads the sum of all
|
||||
* threads in subgroup T.
|
||||
*/
|
||||
uint other_sum = scratch[get_sub_group_local_id()];
|
||||
|
||||
/* Exclusive sum the subgroup sums to get the total before the current group,
|
||||
* which can be added to the total for the current group.
|
||||
*/
|
||||
uint other_sums = sub_group_scan_exclusive_add(other_sum);
|
||||
uint base = sub_group_broadcast(other_sums, sg_id);
|
||||
uint prefix = base + sg;
|
||||
|
||||
/* Reduce the workgroup using the prefix sum we already did */
|
||||
uint reduction = sub_group_broadcast(other_sums + other_sum, 31);
|
||||
|
||||
return (uint2)(prefix, reduction);
|
||||
}
|
||||
|
||||
static inline void
|
||||
poly_prefix_sum(local uint *scratch, global uint *buffer, uint len, uint words,
|
||||
uint word, uint wg_count)
|
||||
{
|
||||
uint tid = cl_local_id.x;
|
||||
|
||||
/* Main loop: complete workgroups processing multiple values at once */
|
||||
uint i, count = 0;
|
||||
uint len_remainder = len % wg_count;
|
||||
uint len_rounded_down = len - len_remainder;
|
||||
|
||||
for (i = tid; i < len_rounded_down; i += wg_count) {
|
||||
global uint *ptr = &buffer[(i * words) + word];
|
||||
uint value = *ptr;
|
||||
uint2 sums = poly_work_group_scan_inclusive_add(value, scratch);
|
||||
|
||||
*ptr = count + sums[0];
|
||||
count += sums[1];
|
||||
}
|
||||
|
||||
/* The last iteration is special since we won't have a full subgroup unless
|
||||
* the length is divisible by the subgroup size, and we don't advance count.
|
||||
*/
|
||||
global uint *ptr = &buffer[(i * words) + word];
|
||||
uint value = (tid < len_remainder) ? *ptr : 0;
|
||||
uint scan = poly_work_group_scan_inclusive_add(value, scratch)[0];
|
||||
|
||||
if (tid < len_remainder) {
|
||||
*ptr = count + scan;
|
||||
}
|
||||
}
|
||||
|
||||
static inline void
|
||||
poly_increment_counters(global uint32_t *a, global uint32_t *b,
|
||||
global uint32_t *c, uint count)
|
||||
{
|
||||
global uint32_t *ptr[] = {a, b, c};
|
||||
|
||||
for (uint i = 0; i < 3; ++i) {
|
||||
if (ptr[i]) {
|
||||
*(ptr[i]) += count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static inline void
|
||||
poly_increment_ia(global uint32_t *ia_vertices, global uint32_t *ia_primitives,
|
||||
global uint32_t *vs_invocations, global uint32_t *c_prims,
|
||||
global uint32_t *c_invs, constant uint32_t *draw,
|
||||
enum mesa_prim prim, unsigned verts_per_patch)
|
||||
{
|
||||
poly_increment_counters(ia_vertices, vs_invocations, NULL,
|
||||
draw[0] * draw[1]);
|
||||
|
||||
uint prims = poly_decomposed_prims_for_vertices_with_tess(prim, draw[0],
|
||||
verts_per_patch) *
|
||||
draw[1];
|
||||
|
||||
poly_increment_counters(ia_primitives, c_prims, c_invs, prims);
|
||||
}
|
||||
|
||||
static inline void
|
||||
poly_gs_setup_indirect(uint64_t index_buffer, constant uint *draw,
|
||||
global uintptr_t *vertex_buffer /* output */,
|
||||
global struct poly_ia_state *ia /* output */,
|
||||
global struct poly_geometry_params *p /* output */,
|
||||
global struct poly_heap *heap,
|
||||
uint64_t vs_outputs /* Vertex (TES) output mask */,
|
||||
uint32_t index_size_B /* 0 if no index bffer */,
|
||||
uint32_t index_buffer_range_el,
|
||||
uint32_t prim /* Input primitive type, enum mesa_prim */,
|
||||
int is_prefix_summing, uint max_indices,
|
||||
enum poly_gs_shape shape)
|
||||
{
|
||||
/* Determine the (primitives, instances) grid size. */
|
||||
uint vertex_count = draw[0];
|
||||
uint instance_count = draw[1];
|
||||
|
||||
ia->verts_per_instance = vertex_count;
|
||||
|
||||
/* Calculate number of primitives input into the GS */
|
||||
uint prim_per_instance = u_decomposed_prims_for_vertices(prim, vertex_count);
|
||||
p->input_primitives = prim_per_instance * instance_count;
|
||||
|
||||
/* Invoke VS as (vertices, instances); GS as (primitives, instances) */
|
||||
p->vs_grid[0] = vertex_count;
|
||||
p->vs_grid[1] = instance_count;
|
||||
|
||||
p->gs_grid[0] = prim_per_instance;
|
||||
p->gs_grid[1] = instance_count;
|
||||
|
||||
p->primitives_log2 = util_logbase2_ceil(prim_per_instance);
|
||||
|
||||
/* If indexing is enabled, the third word is the offset into the index buffer
|
||||
* in elements. Apply that offset now that we have it. For a hardware
|
||||
* indirect draw, the hardware would do this for us, but for software input
|
||||
* assembly we need to do it ourselves.
|
||||
*/
|
||||
if (index_size_B) {
|
||||
ia->index_buffer = poly_index_buffer(index_buffer, index_buffer_range_el,
|
||||
draw[2], index_size_B);
|
||||
|
||||
ia->index_buffer_range_el =
|
||||
poly_index_buffer_range_el(index_buffer_range_el, draw[2]);
|
||||
}
|
||||
|
||||
/* We need to allocate VS and GS count buffers, do so now */
|
||||
uint vertex_buffer_size =
|
||||
poly_tcs_in_size(vertex_count * instance_count, vs_outputs);
|
||||
|
||||
if (is_prefix_summing) {
|
||||
p->count_buffer = poly_heap_alloc_nonatomic(
|
||||
heap, p->input_primitives * p->count_buffer_stride);
|
||||
}
|
||||
|
||||
p->input_buffer =
|
||||
(uintptr_t)poly_heap_alloc_nonatomic(heap, vertex_buffer_size);
|
||||
*vertex_buffer = p->input_buffer;
|
||||
|
||||
p->input_mask = vs_outputs;
|
||||
|
||||
/* Allocate the index buffer and write the draw consuming it */
|
||||
global VkDrawIndexedIndirectCommand *cmd = (global void *)p->indirect_desc;
|
||||
|
||||
*cmd = (VkDrawIndexedIndirectCommand){
|
||||
.indexCount = poly_gs_rast_vertices(shape, max_indices, prim_per_instance,
|
||||
instance_count),
|
||||
.instanceCount = poly_gs_rast_instances(
|
||||
shape, max_indices, prim_per_instance, instance_count),
|
||||
};
|
||||
|
||||
if (shape == POLY_GS_SHAPE_DYNAMIC_INDEXED) {
|
||||
cmd->firstIndex =
|
||||
poly_heap_alloc_nonatomic_offs(heap, cmd->indexCount * 4) / 4;
|
||||
|
||||
p->output_index_buffer =
|
||||
(global uint *)(heap->base + (cmd->firstIndex * 4));
|
||||
}
|
||||
}
|
||||
|
||||
static uint
|
||||
poly_load_index(uintptr_t index_buffer, uint32_t index_buffer_range_el, uint id,
|
||||
uint index_size)
|
||||
{
|
||||
bool oob = id >= index_buffer_range_el;
|
||||
|
||||
/* If the load would be out-of-bounds, load the first element which is
|
||||
* assumed valid. If the application index buffer is empty with robustness2,
|
||||
* index_buffer will point to a zero sink where only the first is valid.
|
||||
*/
|
||||
if (oob) {
|
||||
id = 0;
|
||||
}
|
||||
|
||||
uint el;
|
||||
if (index_size == 1) {
|
||||
el = ((constant uint8_t *)index_buffer)[id];
|
||||
} else if (index_size == 2) {
|
||||
el = ((constant uint16_t *)index_buffer)[id];
|
||||
} else {
|
||||
el = ((constant uint32_t *)index_buffer)[id];
|
||||
}
|
||||
|
||||
/* D3D robustness semantics. TODO: Optimize? */
|
||||
if (oob) {
|
||||
el = 0;
|
||||
}
|
||||
|
||||
return el;
|
||||
}
|
||||
|
||||
static void
|
||||
poly_store_index(uintptr_t index_buffer, uint index_size_B, uint id, uint value)
|
||||
{
|
||||
global uint32_t *out_32 = (global uint32_t *)index_buffer;
|
||||
global uint16_t *out_16 = (global uint16_t *)index_buffer;
|
||||
global uint8_t *out_8 = (global uint8_t *)index_buffer;
|
||||
|
||||
if (index_size_B == 4)
|
||||
out_32[id] = value;
|
||||
else if (index_size_B == 2)
|
||||
out_16[id] = value;
|
||||
else
|
||||
out_8[id] = value;
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,9 @@
|
||||
# Copyright © 2025 Collabora Ltd.
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
inc_poly = include_directories([
|
||||
'.', 'nir'
|
||||
])
|
||||
|
||||
subdir('cl')
|
||||
subdir('nir')
|
||||
@@ -0,0 +1,18 @@
|
||||
# Copyright © 2025 Collabora Ltd.
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
libpoly_nir_files = files(
|
||||
'poly_nir_lower_gs.c',
|
||||
'poly_nir_lower_ia.c',
|
||||
'poly_nir_lower_tess.c',
|
||||
)
|
||||
|
||||
libpoly_nir = static_library(
|
||||
'libpoly_nir',
|
||||
[libpoly_nir_files],
|
||||
include_directories : [inc_poly],
|
||||
c_args : [no_override_init_args, '-Wno-c2x-extensions'],
|
||||
gnu_symbol_visibility : 'hidden',
|
||||
dependencies: [idep_nir, idep_mesautil, idep_libpoly],
|
||||
build_by_default : false,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,61 @@
|
||||
/*
|
||||
* Copyright 2023 Alyssa Rosenzweig
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include "poly/geometry.h"
|
||||
#include "nir.h"
|
||||
#include "shader_enums.h"
|
||||
|
||||
struct nir_def *poly_load_per_vertex_input(struct nir_builder *b,
|
||||
nir_intrinsic_instr *intr,
|
||||
struct nir_def *vertex);
|
||||
|
||||
nir_def *poly_nir_load_vertex_id(struct nir_builder *b, nir_def *id,
|
||||
unsigned index_size_B);
|
||||
|
||||
bool poly_nir_lower_sw_vs(struct nir_shader *s, unsigned index_size_B);
|
||||
|
||||
bool poly_nir_lower_vs_before_gs(struct nir_shader *vs);
|
||||
|
||||
struct poly_gs_info {
|
||||
/* Output primitive mode for geometry shaders */
|
||||
enum mesa_prim mode;
|
||||
|
||||
/* Number of words per primitive in the count buffer */
|
||||
unsigned count_words;
|
||||
|
||||
/* Per-input primitive stride of the output index buffer */
|
||||
unsigned max_indices;
|
||||
|
||||
/* Whether the GS includes transform feedback at a compile-time level */
|
||||
bool xfb;
|
||||
|
||||
/* Whether a prefix sum is required on the count outputs. Implies xfb */
|
||||
bool prefix_sum;
|
||||
|
||||
/* Whether the GS writes to a stream other than stream #0 */
|
||||
bool multistream;
|
||||
|
||||
/* Shape of the rasterization draw, named by the instance ID */
|
||||
enum poly_gs_shape shape;
|
||||
|
||||
/* Static topology used if shape = POLY_GS_SHAPE_STATIC_INDEXED */
|
||||
uint8_t topology[64];
|
||||
};
|
||||
|
||||
bool poly_nir_lower_gs(struct nir_shader *gs, struct nir_shader **gs_count,
|
||||
struct nir_shader **gs_copy, struct nir_shader **pre_gs,
|
||||
struct poly_gs_info *info);
|
||||
|
||||
bool poly_nir_lower_tcs(struct nir_shader *tcs);
|
||||
|
||||
bool poly_nir_lower_tes(struct nir_shader *tes, bool to_hw_vs);
|
||||
|
||||
uint64_t poly_tcs_per_vertex_outputs(const struct nir_shader *nir);
|
||||
|
||||
unsigned poly_tcs_output_stride(const struct nir_shader *nir);
|
||||
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
* Copyright 2023 Valve Corporation
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#include "compiler/nir/nir_builder.h"
|
||||
#include "poly/cl/libpoly.h"
|
||||
#include "poly/geometry.h"
|
||||
#include "nir.h"
|
||||
|
||||
/* XXX: Remove me later */
|
||||
nir_def *poly_nir_load_vertex_id(struct nir_builder *b, nir_def *id,
|
||||
unsigned index_size_B);
|
||||
|
||||
bool poly_nir_lower_sw_vs(struct nir_shader *s, unsigned index_size_B);
|
||||
|
||||
/*
|
||||
* This file implements basic input assembly in software. It runs on software
|
||||
* vertex shaders, as part of geometry/tessellation lowering. It does not apply
|
||||
* the topology, which happens in the geometry shader.
|
||||
*/
|
||||
nir_def *
|
||||
poly_nir_load_vertex_id(nir_builder *b, nir_def *id, unsigned index_size_B)
|
||||
{
|
||||
/* If drawing with an index buffer, pull the vertex ID. Otherwise, the
|
||||
* vertex ID is just the index as-is.
|
||||
*/
|
||||
if (index_size_B) {
|
||||
nir_def *ia = nir_load_input_assembly_buffer_poly(b);
|
||||
id = poly_load_index_buffer(b, ia, id, nir_imm_int(b, index_size_B));
|
||||
}
|
||||
|
||||
/* Add the "start", either an index bias or a base vertex. This must happen
|
||||
* after indexing for proper index bias behaviour.
|
||||
*/
|
||||
return nir_iadd(b, id, nir_load_first_vertex(b));
|
||||
}
|
||||
|
||||
static bool
|
||||
lower(nir_builder *b, nir_intrinsic_instr *intr, void *data)
|
||||
{
|
||||
unsigned *index_size_B = data;
|
||||
b->cursor = nir_before_instr(&intr->instr);
|
||||
|
||||
if (intr->intrinsic == nir_intrinsic_load_vertex_id) {
|
||||
nir_def *id = nir_channel(b, nir_load_global_invocation_id(b, 32), 0);
|
||||
nir_def_replace(&intr->def,
|
||||
poly_nir_load_vertex_id(b, id, *index_size_B));
|
||||
return true;
|
||||
} else if (intr->intrinsic == nir_intrinsic_load_instance_id) {
|
||||
nir_def_replace(&intr->def,
|
||||
nir_channel(b, nir_load_global_invocation_id(b, 32), 1));
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool
|
||||
poly_nir_lower_sw_vs(nir_shader *s, unsigned index_size_B)
|
||||
{
|
||||
return nir_shader_intrinsics_pass(s, lower, nir_metadata_control_flow,
|
||||
&index_size_B);
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
/*
|
||||
* Copyright 2023 Alyssa Rosenzweig
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#include "poly/cl/libpoly.h"
|
||||
#include "poly/geometry.h"
|
||||
#include "poly/nir/poly_nir_lower_gs.h"
|
||||
#include "util/bitscan.h"
|
||||
#include "util/macros.h"
|
||||
#include "nir.h"
|
||||
#include "nir_builder.h"
|
||||
#include "nir_builder_opcodes.h"
|
||||
#include "nir_intrinsics.h"
|
||||
#include "nir_intrinsics_indices.h"
|
||||
#include "shader_enums.h"
|
||||
|
||||
static nir_def *
|
||||
tcs_unrolled_id(nir_builder *b)
|
||||
{
|
||||
return poly_tcs_unrolled_id(b, nir_load_tess_param_buffer_poly(b),
|
||||
nir_load_workgroup_id(b));
|
||||
}
|
||||
|
||||
uint64_t
|
||||
poly_tcs_per_vertex_outputs(const nir_shader *nir)
|
||||
{
|
||||
return nir->info.outputs_written &
|
||||
~(VARYING_BIT_TESS_LEVEL_INNER | VARYING_BIT_TESS_LEVEL_OUTER |
|
||||
VARYING_BIT_BOUNDING_BOX0 | VARYING_BIT_BOUNDING_BOX1);
|
||||
}
|
||||
|
||||
unsigned
|
||||
poly_tcs_output_stride(const nir_shader *nir)
|
||||
{
|
||||
return poly_tcs_out_stride(util_last_bit(nir->info.patch_outputs_written),
|
||||
nir->info.tess.tcs_vertices_out,
|
||||
poly_tcs_per_vertex_outputs(nir));
|
||||
}
|
||||
|
||||
static nir_def *
|
||||
tcs_out_addr(nir_builder *b, nir_intrinsic_instr *intr, nir_def *vertex_id)
|
||||
{
|
||||
nir_io_semantics sem = nir_intrinsic_io_semantics(intr);
|
||||
|
||||
nir_def *offset = nir_get_io_offset_src(intr)->ssa;
|
||||
nir_def *addr = poly_tcs_out_address(
|
||||
b, nir_load_tess_param_buffer_poly(b), tcs_unrolled_id(b), vertex_id,
|
||||
nir_iadd_imm(b, offset, sem.location),
|
||||
nir_imm_int(b, util_last_bit(b->shader->info.patch_outputs_written)),
|
||||
nir_imm_int(b, b->shader->info.tess.tcs_vertices_out),
|
||||
nir_imm_int64(b, poly_tcs_per_vertex_outputs(b->shader)));
|
||||
|
||||
addr = nir_iadd_imm(b, addr, nir_intrinsic_component(intr) * 4);
|
||||
|
||||
return addr;
|
||||
}
|
||||
|
||||
static nir_def *
|
||||
lower_tes_load(nir_builder *b, nir_intrinsic_instr *intr)
|
||||
{
|
||||
gl_varying_slot location = nir_intrinsic_io_semantics(intr).location;
|
||||
nir_src *offset_src = nir_get_io_offset_src(intr);
|
||||
|
||||
nir_def *vertex = nir_imm_int(b, 0);
|
||||
nir_def *offset = offset_src ? offset_src->ssa : nir_imm_int(b, 0);
|
||||
|
||||
if (intr->intrinsic == nir_intrinsic_load_per_vertex_input)
|
||||
vertex = intr->src[0].ssa;
|
||||
|
||||
nir_def *addr = poly_tes_in_address(b, nir_load_tess_param_buffer_poly(b),
|
||||
nir_load_vertex_id(b), vertex,
|
||||
nir_iadd_imm(b, offset, location));
|
||||
|
||||
if (nir_intrinsic_has_component(intr))
|
||||
addr = nir_iadd_imm(b, addr, nir_intrinsic_component(intr) * 4);
|
||||
|
||||
return nir_load_global_constant(b, addr, 4, intr->def.num_components,
|
||||
intr->def.bit_size);
|
||||
}
|
||||
|
||||
static nir_def *
|
||||
tcs_load_input(nir_builder *b, nir_intrinsic_instr *intr)
|
||||
{
|
||||
nir_def *base = nir_imul(
|
||||
b, tcs_unrolled_id(b),
|
||||
poly_tcs_patch_vertices_in(b, nir_load_tess_param_buffer_poly(b)));
|
||||
nir_def *vertex = nir_iadd(b, base, intr->src[0].ssa);
|
||||
|
||||
return poly_load_per_vertex_input(b, intr, vertex);
|
||||
}
|
||||
|
||||
static nir_def *
|
||||
lower_tcs_impl(nir_builder *b, nir_intrinsic_instr *intr)
|
||||
{
|
||||
switch (intr->intrinsic) {
|
||||
case nir_intrinsic_barrier:
|
||||
/* A patch fits in a subgroup, so the barrier is unnecessary. */
|
||||
return NIR_LOWER_INSTR_PROGRESS_REPLACE;
|
||||
|
||||
case nir_intrinsic_load_primitive_id:
|
||||
return nir_channel(b, nir_load_workgroup_id(b), 0);
|
||||
|
||||
case nir_intrinsic_load_instance_id:
|
||||
return nir_channel(b, nir_load_workgroup_id(b), 1);
|
||||
|
||||
case nir_intrinsic_load_invocation_id:
|
||||
if (b->shader->info.tess.tcs_vertices_out == 1)
|
||||
return nir_imm_int(b, 0);
|
||||
else
|
||||
return nir_channel(b, nir_load_local_invocation_id(b), 0);
|
||||
|
||||
case nir_intrinsic_load_per_vertex_input:
|
||||
return tcs_load_input(b, intr);
|
||||
|
||||
case nir_intrinsic_load_patch_vertices_in:
|
||||
return poly_tcs_patch_vertices_in(b, nir_load_tess_param_buffer_poly(b));
|
||||
|
||||
case nir_intrinsic_load_tess_level_outer_default:
|
||||
return poly_tess_level_outer_default(b,
|
||||
nir_load_tess_param_buffer_poly(b));
|
||||
|
||||
case nir_intrinsic_load_tess_level_inner_default:
|
||||
return poly_tess_level_inner_default(b,
|
||||
nir_load_tess_param_buffer_poly(b));
|
||||
|
||||
case nir_intrinsic_load_output: {
|
||||
nir_def *addr = tcs_out_addr(b, intr, nir_undef(b, 1, 32));
|
||||
return nir_load_global(b, addr, 4, intr->def.num_components,
|
||||
intr->def.bit_size);
|
||||
}
|
||||
|
||||
case nir_intrinsic_load_per_vertex_output: {
|
||||
nir_def *addr = tcs_out_addr(b, intr, intr->src[0].ssa);
|
||||
return nir_load_global(b, addr, 4, intr->def.num_components,
|
||||
intr->def.bit_size);
|
||||
}
|
||||
|
||||
case nir_intrinsic_store_output: {
|
||||
/* Only vec2, make sure we can't overwrite */
|
||||
assert(intr->src[0].ssa->num_components <= 2 ||
|
||||
nir_intrinsic_io_semantics(intr).location !=
|
||||
VARYING_SLOT_TESS_LEVEL_INNER);
|
||||
|
||||
nir_store_global(b, tcs_out_addr(b, intr, nir_undef(b, 1, 32)), 4,
|
||||
intr->src[0].ssa, nir_intrinsic_write_mask(intr));
|
||||
return NIR_LOWER_INSTR_PROGRESS_REPLACE;
|
||||
}
|
||||
|
||||
case nir_intrinsic_store_per_vertex_output: {
|
||||
nir_store_global(b, tcs_out_addr(b, intr, intr->src[1].ssa), 4,
|
||||
intr->src[0].ssa, nir_intrinsic_write_mask(intr));
|
||||
return NIR_LOWER_INSTR_PROGRESS_REPLACE;
|
||||
}
|
||||
|
||||
default:
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static bool
|
||||
lower_tcs(nir_builder *b, nir_intrinsic_instr *intr, void *data)
|
||||
{
|
||||
b->cursor = nir_before_instr(&intr->instr);
|
||||
|
||||
nir_def *repl = lower_tcs_impl(b, intr);
|
||||
if (!repl)
|
||||
return false;
|
||||
|
||||
if (repl != NIR_LOWER_INSTR_PROGRESS_REPLACE)
|
||||
nir_def_rewrite_uses(&intr->def, repl);
|
||||
|
||||
nir_instr_remove(&intr->instr);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
poly_nir_lower_tcs(nir_shader *tcs)
|
||||
{
|
||||
return nir_shader_intrinsics_pass(tcs, lower_tcs, nir_metadata_control_flow,
|
||||
NULL);
|
||||
}
|
||||
|
||||
static nir_def *
|
||||
lower_tes_impl(nir_builder *b, nir_intrinsic_instr *intr, void *data)
|
||||
{
|
||||
switch (intr->intrinsic) {
|
||||
case nir_intrinsic_load_tess_coord_xy:
|
||||
return poly_load_tess_coord(b, nir_load_tess_param_buffer_poly(b),
|
||||
nir_load_vertex_id(b));
|
||||
|
||||
case nir_intrinsic_load_primitive_id:
|
||||
return poly_tes_patch_id(b, nir_load_tess_param_buffer_poly(b),
|
||||
nir_load_vertex_id(b));
|
||||
|
||||
case nir_intrinsic_load_input:
|
||||
case nir_intrinsic_load_per_vertex_input:
|
||||
case nir_intrinsic_load_tess_level_inner:
|
||||
case nir_intrinsic_load_tess_level_outer:
|
||||
return lower_tes_load(b, intr);
|
||||
|
||||
case nir_intrinsic_load_patch_vertices_in:
|
||||
return poly_tes_patch_vertices_in(b, nir_load_tess_param_buffer_poly(b));
|
||||
|
||||
default:
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static bool
|
||||
lower_tes(nir_builder *b, nir_intrinsic_instr *intr, void *data)
|
||||
{
|
||||
b->cursor = nir_before_instr(&intr->instr);
|
||||
nir_def *repl = lower_tes_impl(b, intr, data);
|
||||
|
||||
if (repl) {
|
||||
nir_def_replace(&intr->def, repl);
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static bool
|
||||
lower_tes_indexing(nir_builder *b, nir_intrinsic_instr *intr, void *data)
|
||||
{
|
||||
if (intr->intrinsic != nir_intrinsic_load_vertex_id)
|
||||
return false;
|
||||
|
||||
b->cursor = nir_before_instr(&intr->instr);
|
||||
nir_def *p = nir_load_tess_param_buffer_poly(b);
|
||||
nir_def *id = nir_channel(b, nir_load_global_invocation_id(b, 32), 0);
|
||||
nir_def_replace(&intr->def, poly_load_tes_index(b, p, id));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
poly_nir_lower_tes(nir_shader *tes, bool to_hw_vs)
|
||||
{
|
||||
nir_lower_tess_coord_z(
|
||||
tes, tes->info.tess._primitive_mode == TESS_PRIMITIVE_TRIANGLES);
|
||||
|
||||
nir_shader_intrinsics_pass(tes, lower_tes, nir_metadata_control_flow, NULL);
|
||||
|
||||
/* Points mode renders as points, make sure we write point size for the HW */
|
||||
if (tes->info.tess.point_mode && to_hw_vs) {
|
||||
nir_lower_default_point_size(tes);
|
||||
}
|
||||
|
||||
if (to_hw_vs) {
|
||||
/* We lower to a HW VS, so update the shader info so the compiler does the
|
||||
* right thing.
|
||||
*/
|
||||
tes->info.stage = MESA_SHADER_VERTEX;
|
||||
memset(&tes->info.vs, 0, sizeof(tes->info.vs));
|
||||
tes->info.vs.tes_poly = true;
|
||||
} else {
|
||||
/* If we're running as a compute shader, we need to load from the index
|
||||
* buffer manually. Fortunately, this doesn't require a shader key:
|
||||
* tess-as-compute always use U32 index buffers.
|
||||
*/
|
||||
nir_shader_intrinsics_pass(tes, lower_tes_indexing,
|
||||
nir_metadata_control_flow, NULL);
|
||||
}
|
||||
|
||||
nir_lower_idiv(tes, &(nir_lower_idiv_options){.allow_fp16 = true});
|
||||
return nir_progress(true, nir_shader_get_entrypoint(tes), nir_metadata_none);
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
* Copyright 2024 Valve Corporation
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "compiler/libcl/libcl.h"
|
||||
|
||||
enum poly_tess_partitioning {
|
||||
POLY_TESS_PARTITIONING_FRACTIONAL_ODD,
|
||||
POLY_TESS_PARTITIONING_FRACTIONAL_EVEN,
|
||||
POLY_TESS_PARTITIONING_INTEGER,
|
||||
};
|
||||
|
||||
enum poly_tess_mode {
|
||||
/* Do not actually tessellate, just write the index counts */
|
||||
POLY_TESS_MODE_COUNT,
|
||||
|
||||
/* Tessellate using the count buffers to allocate indices */
|
||||
POLY_TESS_MODE_WITH_COUNTS,
|
||||
};
|
||||
|
||||
struct poly_tess_point {
|
||||
uint32_t u;
|
||||
uint32_t v;
|
||||
};
|
||||
static_assert(sizeof(struct poly_tess_point) == 8);
|
||||
|
||||
struct poly_tess_args {
|
||||
/* Heap to allocate tessellator outputs in */
|
||||
DEVICE(struct poly_heap) heap;
|
||||
|
||||
/* Patch coordinate buffer, indexed as:
|
||||
*
|
||||
* coord_allocs[patch_ID] + vertex_in_patch
|
||||
*/
|
||||
DEVICE(struct poly_tess_point) patch_coord_buffer;
|
||||
|
||||
/* Per-patch index within the heap for the tess coords, written by the
|
||||
* tessellator based on the allocated memory.
|
||||
*/
|
||||
DEVICE(uint32_t) coord_allocs;
|
||||
|
||||
/* Space for output draws from the tessellator. API draw calls. */
|
||||
DEVICE(uint32_t) out_draws;
|
||||
|
||||
/* Tessellation control shader output buffer. */
|
||||
DEVICE(float) tcs_buffer;
|
||||
|
||||
/* Count buffer. # of indices per patch written here, then prefix summed. */
|
||||
DEVICE(uint32_t) counts;
|
||||
|
||||
/* Allocated index buffer for all patches, if we're prefix summing counts */
|
||||
DEVICE(uint32_t) index_buffer;
|
||||
|
||||
/* Address of the tess eval invocation counter for implementing pipeline
|
||||
* statistics, if active. Zero if inactive. Incremented by tessellator.
|
||||
*/
|
||||
DEVICE(uint32_t) statistic;
|
||||
|
||||
/* When geom+tess used together, the buffer containing TES outputs (executed
|
||||
* as a hardware compute shader).
|
||||
*/
|
||||
uint64_t tes_buffer;
|
||||
|
||||
/* Bitfield of TCS per-vertex outputs */
|
||||
uint64_t tcs_per_vertex_outputs;
|
||||
|
||||
/* Default tess levels used in OpenGL when there is no TCS in the pipeline.
|
||||
* Unused in Vulkan and OpenGL ES.
|
||||
*/
|
||||
float tess_level_outer_default[4];
|
||||
float tess_level_inner_default[2];
|
||||
|
||||
/* Number of vertices in the input patch */
|
||||
uint32_t input_patch_size;
|
||||
|
||||
/* Number of vertices in the TCS output patch */
|
||||
uint32_t output_patch_size;
|
||||
|
||||
/* Number of patch constants written by TCS */
|
||||
uint32_t tcs_patch_constants;
|
||||
|
||||
/* Number of input patches per instance of the VS/TCS */
|
||||
uint32_t patches_per_instance;
|
||||
|
||||
/* Stride between tessellation facotrs in the TCS output buffer. */
|
||||
uint32_t tcs_stride_el;
|
||||
|
||||
/* Number of patches being tessellated */
|
||||
uint32_t nr_patches;
|
||||
|
||||
/* Partitioning and points mode. These affect per-patch setup code but not
|
||||
* the hot tessellation loop so we make them dynamic to reduce tessellator
|
||||
* variants.
|
||||
*/
|
||||
enum poly_tess_partitioning partitioning;
|
||||
uint32_t points_mode;
|
||||
uint32_t isolines;
|
||||
|
||||
/* When fed into a geometry shader, triangles should be counter-clockwise.
|
||||
* The tessellator always produces clockwise triangles, but we can swap
|
||||
* dynamically in the TES.
|
||||
*/
|
||||
uint32_t ccw;
|
||||
} PACKED;
|
||||
static_assert(sizeof(struct poly_tess_args) == 36 * 4);
|
||||
Reference in New Issue
Block a user