radv: Emit compressed primitive nodes on GFX12

Emits two triangles per node whenever possible. The nir code will
revisit the triangle node to handle the second triangle only if both
triangles are interescted by the ray.

Reviewed-by: Natalie Vock <natalie.vock@gmx.de>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/35734>
This commit is contained in:
Konstantin Seurer
2025-04-07 22:19:42 +02:00
committed by Marge Bot
parent fb83a5793a
commit c4b18c689f
8 changed files with 188 additions and 22 deletions
+1
View File
@@ -24,6 +24,7 @@
#define RADV_BUILD_FLAG_NO_INFS (1u << (VK_BUILD_FLAG_COUNT + 3))
#define RADV_BUILD_FLAG_WRITE_LEAF_NODE_OFFSETS (1u << (VK_BUILD_FLAG_COUNT + 4))
#define RADV_BUILD_FLAG_UPDATE_SINGLE_GEOMETRY (1u << (VK_BUILD_FLAG_COUNT + 5))
#define RADV_BUILD_FLAG_PAIR_COMPRESS_TRIANGLES (1u << (VK_BUILD_FLAG_COUNT + 6))
#define RADV_COPY_MODE_COPY 0
#define RADV_COPY_MODE_SERIALIZE 1
+64
View File
@@ -185,6 +185,70 @@ radv_encode_triangle_gfx12(VOID_REF dst, vk_ir_triangle_node src)
bit_writer_finish(child_writer);
}
void
radv_encode_triangle_gfx12(VOID_REF dst, vk_ir_triangle_node src0, vk_ir_triangle_node src1)
{
bit_writer child_writer;
bit_writer_init(child_writer, dst);
bit_writer_write(child_writer, 31, 5); /* x_vertex_bits_minus_one */
bit_writer_write(child_writer, 31, 5); /* y_vertex_bits_minus_one */
bit_writer_write(child_writer, 31, 5); /* z_vertex_bits_minus_one */
bit_writer_write(child_writer, 0, 5); /* trailing_zero_bits */
bit_writer_write(child_writer, 14, 4); /* geometry_index_base_bits_div_2 */
bit_writer_write(child_writer, 14, 4); /* geometry_index_bits_div_2 */
bit_writer_write(child_writer, 0, 3); /* triangle_pair_count_minus_one */
bit_writer_write(child_writer, 0, 1); /* vertex_type */
bit_writer_write(child_writer, 28, 5); /* primitive_index_base_bits */
bit_writer_write(child_writer, 28, 5); /* primitive_index_bits */
/* header + (9 floats + geometry_id) * 2 triangles */
bit_writer_write(child_writer, RADV_GFX12_PRIMITIVE_NODE_HEADER_SIZE + 2 * 9 * 32 + 2 * 28, 10);
bit_writer_write(child_writer, floatBitsToUint(src0.coords[0][0]), 32);
bit_writer_write(child_writer, floatBitsToUint(src0.coords[0][1]), 32);
bit_writer_write(child_writer, floatBitsToUint(src0.coords[0][2]), 32);
bit_writer_write(child_writer, floatBitsToUint(src0.coords[1][0]), 32);
bit_writer_write(child_writer, floatBitsToUint(src0.coords[1][1]), 32);
bit_writer_write(child_writer, floatBitsToUint(src0.coords[1][2]), 32);
bit_writer_write(child_writer, floatBitsToUint(src0.coords[2][0]), 32);
bit_writer_write(child_writer, floatBitsToUint(src0.coords[2][1]), 32);
bit_writer_write(child_writer, floatBitsToUint(src0.coords[2][2]), 32);
bit_writer_write(child_writer, floatBitsToUint(src1.coords[0][0]), 32);
bit_writer_write(child_writer, floatBitsToUint(src1.coords[0][1]), 32);
bit_writer_write(child_writer, floatBitsToUint(src1.coords[0][2]), 32);
bit_writer_write(child_writer, floatBitsToUint(src1.coords[1][0]), 32);
bit_writer_write(child_writer, floatBitsToUint(src1.coords[1][1]), 32);
bit_writer_write(child_writer, floatBitsToUint(src1.coords[1][2]), 32);
bit_writer_write(child_writer, floatBitsToUint(src1.coords[2][0]), 32);
bit_writer_write(child_writer, floatBitsToUint(src1.coords[2][1]), 32);
bit_writer_write(child_writer, floatBitsToUint(src1.coords[2][2]), 32);
bit_writer_write(child_writer, src1.geometry_id_and_flags & 0xfffffff, 28);
bit_writer_write(child_writer, src0.geometry_id_and_flags & 0xfffffff, 28);
bit_writer_write(child_writer, src0.triangle_id, 28);
bit_writer_write(child_writer, src1.triangle_id, 28);
bit_writer_skip_to(child_writer, 32 * 32 - RADV_GFX12_PRIMITIVE_NODE_PAIR_DESC_SIZE);
uint32_t opaque0 = (src0.geometry_id_and_flags & VK_GEOMETRY_OPAQUE) != 0 ? 1 : 0;
uint32_t opaque1 = (src1.geometry_id_and_flags & VK_GEOMETRY_OPAQUE) != 0 ? 1 : 0;
bit_writer_write(child_writer, 1, 1); /* prim_range_stop */
bit_writer_write(child_writer, 0, 1); /* tri1_double_sided */
bit_writer_write(child_writer, opaque1, 1); /* tri1_opaque */
bit_writer_write(child_writer, 3, 4); /* tri1_v0_index */
bit_writer_write(child_writer, 4, 4); /* tri1_v1_index */
bit_writer_write(child_writer, 5, 4); /* tri1_v2_index */
bit_writer_write(child_writer, 0, 1); /* tri0_double_sided */
bit_writer_write(child_writer, opaque0, 1); /* tri0_opaque */
bit_writer_write(child_writer, 0, 4); /* tri0_v0_index */
bit_writer_write(child_writer, 1, 4); /* tri0_v1_index */
bit_writer_write(child_writer, 2, 4); /* tri0_v2_index */
bit_writer_finish(child_writer);
}
void
radv_encode_aabb_gfx12(VOID_REF dst, vk_ir_aabb_node src)
{
+54 -8
View File
@@ -81,6 +81,8 @@ encode_gfx12(uint32_t ir_leaf_node_size, REF(vk_ir_box_node) intermediate_intern
if (cluster.invocation_index < 2)
child = src.children[cluster.invocation_index];
uint32_t second_child = RADV_BVH_INVALID_NODE;
while (true) {
uint32_t valid_children = radv_ballot(cluster, child != RADV_BVH_INVALID_NODE);
if ((valid_children & 0x80) != 0 || valid_children == 0)
@@ -110,12 +112,30 @@ encode_gfx12(uint32_t ir_leaf_node_size, REF(vk_ir_box_node) intermediate_intern
if (left == RADV_BVH_INVALID_NODE) {
left = right;
right = RADV_BVH_INVALID_NODE;
} else if (right != RADV_BVH_INVALID_NODE && ir_id_to_type(left) == vk_ir_node_triangle &&
ir_id_to_type(right) == vk_ir_node_triangle &&
VK_BUILD_FLAG(RADV_BUILD_FLAG_PAIR_COMPRESS_TRIANGLES)) {
second_child = right;
right = RADV_BVH_INVALID_NODE;
}
child = left;
}
right = radv_read_invocation(cluster, collapse_index, right);
if (VK_BUILD_FLAG(RADV_BUILD_FLAG_PAIR_COMPRESS_TRIANGLES)) {
bool is_valid_triangle = child != RADV_BVH_INVALID_NODE && ir_id_to_type(child) == vk_ir_node_triangle;
uint32_t right_pair_mask =
radv_ballot(cluster, is_valid_triangle && second_child == RADV_BVH_INVALID_NODE &&
right != RADV_BVH_INVALID_NODE && ir_id_to_type(right) == vk_ir_node_triangle);
if (right_pair_mask != 0) {
if (cluster.invocation_index == findLSB(right_pair_mask))
second_child = right;
continue;
}
}
if (cluster.invocation_index == findMSB(valid_children) + 1)
child = right;
}
@@ -193,26 +213,46 @@ encode_gfx12(uint32_t ir_leaf_node_size, REF(vk_ir_box_node) intermediate_intern
uint32_t cull_flags = 0;
if (type == vk_ir_node_internal) {
encoded_type = 5;
REF(vk_ir_box_node) child_node = REF(vk_ir_box_node)OFFSET(args.intermediate_bvh, offset);
REF(vk_ir_box_node) child_node = REF(vk_ir_box_node) OFFSET(args.intermediate_bvh, offset);
cull_flags = DEREF(child_node).flags & 0x3;
} else {
if (VK_BUILD_FLAG(RADV_BUILD_FLAG_WRITE_LEAF_NODE_OFFSETS)) {
/* Write leaf node offset. */
uint32_t leaf_index = offset / ir_leaf_node_size;
REF(uint32_t) child_dst_offset = REF(uint32_t)(args.output_base + args.leaf_node_offsets_offset);
child_dst_offset = INDEX(uint32_t, child_dst_offset, leaf_index);
DEREF(child_dst_offset) = dst_offset;
{
uint32_t leaf_index = offset / ir_leaf_node_size;
REF(uint32_t) child_dst_offset = REF(uint32_t)(args.output_base + args.leaf_node_offsets_offset);
child_dst_offset = INDEX(uint32_t, child_dst_offset, leaf_index);
DEREF(child_dst_offset) = dst_offset;
}
if (second_child != RADV_BVH_INVALID_NODE) {
uint32_t leaf_index = ir_id_to_offset(second_child) / ir_leaf_node_size;
REF(uint32_t) child_dst_offset = REF(uint32_t)(args.output_base + args.leaf_node_offsets_offset);
child_dst_offset = INDEX(uint32_t, child_dst_offset, leaf_index);
DEREF(child_dst_offset) = dst_offset;
}
}
VOID_REF dst_leaf_addr = args.output_base + args.output_bvh_offset + dst_offset;
switch (args.geometry_type) {
case VK_GEOMETRY_TYPE_TRIANGLES_KHR: {
vk_ir_triangle_node src_node = DEREF(REF(vk_ir_triangle_node)(OFFSET(args.intermediate_bvh, offset)));
radv_encode_triangle_gfx12(dst_leaf_addr, src_node);
vk_ir_triangle_node src_node0 = DEREF(REF(vk_ir_triangle_node)(OFFSET(args.intermediate_bvh, offset)));
bool opaque = (src_node.geometry_id_and_flags & VK_GEOMETRY_OPAQUE) != 0;
bool opaque = (src_node0.geometry_id_and_flags & VK_GEOMETRY_OPAQUE) != 0;
cull_flags = opaque ? VK_BVH_BOX_FLAG_ONLY_OPAQUE : VK_BVH_BOX_FLAG_NO_OPAQUE;
if (VK_BUILD_FLAG(RADV_BUILD_FLAG_PAIR_COMPRESS_TRIANGLES) && second_child != RADV_BVH_INVALID_NODE) {
vk_ir_triangle_node src_node1 =
DEREF(REF(vk_ir_triangle_node)(OFFSET(args.intermediate_bvh, ir_id_to_offset(second_child))));
opaque = (src_node1.geometry_id_and_flags & VK_GEOMETRY_OPAQUE) != 0;
cull_flags &= opaque ? VK_BVH_BOX_FLAG_ONLY_OPAQUE : VK_BVH_BOX_FLAG_NO_OPAQUE;
radv_encode_triangle_gfx12(dst_leaf_addr, src_node0, src_node1);
} else {
radv_encode_triangle_gfx12(dst_leaf_addr, src_node0);
}
break;
}
case VK_GEOMETRY_TYPE_AABBS_KHR: {
@@ -238,6 +278,12 @@ encode_gfx12(uint32_t ir_leaf_node_size, REF(vk_ir_box_node) intermediate_intern
}
vk_aabb child_aabb = DEREF(REF(vk_ir_node) OFFSET(args.intermediate_bvh, offset)).aabb;
if (second_child != RADV_BVH_INVALID_NODE) {
vk_aabb second_child_aabb =
DEREF(REF(vk_ir_node) OFFSET(args.intermediate_bvh, ir_id_to_offset(second_child))).aabb;
child_aabb.min = min(child_aabb.min, second_child_aabb.min);
child_aabb.max = max(child_aabb.max, second_child_aabb.max);
}
radv_gfx12_box_child box_child;
box_child.dword0 =
@@ -81,6 +81,7 @@ enum radv_ray_query_field {
radv_ray_query_trav_previous_node,
radv_ray_query_trav_instance_top_node,
radv_ray_query_trav_instance_bottom_node,
radv_ray_query_trav_second_iteration,
radv_ray_query_stack,
radv_ray_query_break_flag,
radv_ray_query_field_count,
@@ -118,6 +119,7 @@ radv_get_ray_query_type()
FIELD(trav_previous_node, glsl_uint_type());
FIELD(trav_instance_top_node, glsl_uint_type());
FIELD(trav_instance_bottom_node, glsl_uint_type());
FIELD(trav_second_iteration, glsl_bool_type());
FIELD(stack, glsl_array_type(glsl_uint_type(), MAX_SCRATCH_STACK_ENTRY_COUNT, 0));
FIELD(break_flag, glsl_bool_type());
@@ -319,6 +321,7 @@ lower_rq_initialize(nir_builder *b, nir_intrinsic_instr *instr, struct ray_query
rq_store(b, rq, trav_previous_node, nir_imm_int(b, RADV_BVH_INVALID_NODE));
rq_store(b, rq, trav_instance_top_node, nir_imm_int(b, RADV_BVH_INVALID_NODE));
rq_store(b, rq, trav_instance_bottom_node, nir_imm_int(b, RADV_BVH_NO_INSTANCE_ROOT));
rq_store(b, rq, trav_second_iteration, nir_imm_false(b));
rq_store(b, rq, trav_top_stack, nir_imm_int(b, -1));
@@ -524,6 +527,7 @@ lower_rq_proceed(nir_builder *b, nir_intrinsic_instr *instr, struct ray_query_va
.previous_node = rq_deref(b, rq, trav_previous_node),
.instance_top_node = rq_deref(b, rq, trav_instance_top_node),
.instance_bottom_node = rq_deref(b, rq, trav_instance_bottom_node),
.second_iteration = rq_deref(b, rq, trav_second_iteration),
.instance_addr = isec_deref(b, candidate, instance_addr),
.sbt_offset_and_flags = isec_deref(b, candidate, sbt_offset_and_flags),
.break_flag = rq_deref(b, rq, break_flag),
+49 -14
View File
@@ -655,23 +655,56 @@ insert_traversal_triangle_case(struct radv_device *device, nir_builder *b, const
static void
insert_traversal_triangle_case_gfx12(struct radv_device *device, nir_builder *b,
const struct radv_ray_traversal_args *args, const struct radv_ray_flags *ray_flags,
nir_def *result, nir_def *bvh_node)
nir_variable *intrinsic_result, nir_def *result, nir_def *global_bvh_node,
nir_def *bvh_node)
{
if (!args->triangle_cb)
return;
nir_def *t[2] = {
nir_channel(b, result, 0),
nir_channel(b, result, 4),
};
nir_def *triangle0_first = nir_flt(b, t[0], t[1]);
nir_def *second_t = nir_bcsel(b, triangle0_first, t[1], t[0]);
nir_def *second_iteration = nir_load_deref(b, args->vars.second_iteration);
nir_def *revisit =
nir_iand(b, nir_inot(b, second_iteration), nir_flt(b, second_t, nir_load_deref(b, args->vars.tmax)));
nir_store_deref(b, args->vars.second_iteration, revisit, 0x1);
if (args->use_bvh_stack_rtn) {
nir_def *next_node = nir_bcsel(b, revisit, bvh_node, nir_imm_int(b, RADV_BVH_STACK_SKIP_0_TO_7));
nir_def *comps[8];
for (unsigned i = 0; i < 6; ++i)
comps[i] = nir_channel(b, result, i);
comps[6] = nir_imm_int(b, RADV_BVH_STACK_SKIP_0_TO_7);
comps[7] = next_node;
nir_store_var(b, intrinsic_result, nir_vec(b, comps, 8), 0xff);
} else {
nir_def *next_node = nir_bcsel(b, revisit, bvh_node, nir_imm_int(b, RADV_BVH_INVALID_NODE));
nir_store_deref(b, args->vars.current_node, next_node, 0x1);
}
nir_def *triangle0 = nir_ixor(b, triangle0_first, second_iteration);
struct radv_triangle_intersection intersection;
intersection.t = nir_channel(b, result, 0);
intersection.t = nir_bcsel(b, triangle0, t[0], t[1]);
nir_push_if(b, nir_iand(b, nir_flt(b, intersection.t, nir_load_deref(b, args->vars.tmax)),
nir_flt(b, args->tmin, intersection.t)));
{
intersection.frontface = nir_inot(b, nir_test_mask(b, nir_channel(b, result, 3), 1));
intersection.base.node_addr = build_node_to_addr(device, b, bvh_node, false);
intersection.base.primitive_id = nir_ishr_imm(b, nir_channel(b, result, 3), 1);
intersection.base.geometry_id_and_flags = nir_ishr_imm(b, nir_channel(b, result, 8), 2);
intersection.base.opaque = nir_inot(b, nir_test_mask(b, nir_channel(b, result, 2), 1u << 31));
intersection.barycentrics = nir_fabs(b, nir_channels(b, result, 0x3 << 1));
nir_def *dword1 = nir_bcsel(b, triangle0, nir_channel(b, result, 1), nir_channel(b, result, 5));
nir_def *dword2 = nir_bcsel(b, triangle0, nir_channel(b, result, 2), nir_channel(b, result, 6));
nir_def *dword3 = nir_bcsel(b, triangle0, nir_channel(b, result, 3), nir_channel(b, result, 7));
intersection.frontface = nir_inot(b, nir_test_mask(b, dword3, 1));
intersection.base.node_addr = build_node_to_addr(device, b, global_bvh_node, false);
intersection.base.primitive_id = nir_ishr_imm(b, dword3, 1);
intersection.base.geometry_id_and_flags =
nir_ishr_imm(b, nir_bcsel(b, triangle0, nir_channel(b, result, 8), nir_channel(b, result, 9)), 2);
intersection.base.opaque = nir_inot(b, nir_test_mask(b, dword2, 1u << 31));
intersection.barycentrics = nir_fabs(b, nir_vec2(b, dword1, dword2));
nir_push_if(b, nir_bcsel(b, intersection.base.opaque, ray_flags->no_cull_opaque, ray_flags->no_cull_no_opaque));
{
@@ -1312,6 +1345,12 @@ radv_build_ray_traversal_gfx12(struct radv_device *device, nir_builder *b, const
}
nir_push_else(b, NULL);
{
if (args->use_bvh_stack_rtn) {
nir_def *skip_0_7 = nir_imm_int(b, RADV_BVH_STACK_SKIP_0_TO_7);
nir_store_var(b, intrinsic_result, nir_vector_insert_imm(b, nir_load_var(b, intrinsic_result), skip_0_7, 7),
0xff);
}
nir_push_if(b, nir_test_mask(b, nir_channel(b, result, 1), 1u << 31));
{
nir_push_if(b, ray_flags.no_skip_aabbs);
@@ -1321,15 +1360,11 @@ radv_build_ray_traversal_gfx12(struct radv_device *device, nir_builder *b, const
nir_push_else(b, NULL);
{
nir_push_if(b, ray_flags.no_skip_triangles);
insert_traversal_triangle_case_gfx12(device, b, args, &ray_flags, result, global_bvh_node);
insert_traversal_triangle_case_gfx12(device, b, args, &ray_flags, intrinsic_result, result, global_bvh_node,
bvh_node);
nir_pop_if(b, NULL);
}
nir_pop_if(b, NULL);
if (args->use_bvh_stack_rtn) {
nir_def *skip_0_7 = nir_imm_int(b, RADV_BVH_STACK_SKIP_0_TO_7);
nir_store_var(b, intrinsic_result, nir_vector_insert_imm(b, nir_load_var(b, intrinsic_result), skip_0_7, 7),
0xff);
}
}
nir_pop_if(b, NULL);
+3
View File
@@ -108,6 +108,9 @@ struct radv_ray_traversal_vars {
nir_deref_instr *instance_top_node;
nir_deref_instr *instance_bottom_node;
/* Whether the current iteration revisits the last triangle node to handle the second triangle. */
nir_deref_instr *second_iteration;
/* Information about the current instance used for culling. */
nir_deref_instr *instance_addr;
nir_deref_instr *sbt_offset_and_flags;
+4
View File
@@ -1226,6 +1226,7 @@ struct rt_traversal_vars {
nir_variable *previous_node;
nir_variable *instance_top_node;
nir_variable *instance_bottom_node;
nir_variable *second_iteration;
};
static struct rt_traversal_vars
@@ -1251,6 +1252,7 @@ init_traversal_vars(nir_builder *b)
ret.instance_top_node = nir_variable_create(b->shader, nir_var_shader_temp, glsl_uint_type(), "instance_top_node");
ret.instance_bottom_node =
nir_variable_create(b->shader, nir_var_shader_temp, glsl_uint_type(), "instance_bottom_node");
ret.second_iteration = nir_variable_create(b->shader, nir_var_shader_temp, glsl_bool_type(), "second_iteration");
return ret;
}
@@ -1578,6 +1580,7 @@ radv_build_traversal(struct radv_device *device, struct radv_ray_tracing_pipelin
nir_store_var(b, trav_vars.previous_node, nir_imm_int(b, RADV_BVH_INVALID_NODE), 0x1);
nir_store_var(b, trav_vars.instance_top_node, nir_imm_int(b, RADV_BVH_INVALID_NODE), 0x1);
nir_store_var(b, trav_vars.instance_bottom_node, nir_imm_int(b, RADV_BVH_NO_INSTANCE_ROOT), 0x1);
nir_store_var(b, trav_vars.second_iteration, nir_imm_false(b), 0x1);
nir_store_var(b, trav_vars.top_stack, nir_imm_int(b, -1), 1);
@@ -1594,6 +1597,7 @@ radv_build_traversal(struct radv_device *device, struct radv_ray_tracing_pipelin
.previous_node = nir_build_deref_var(b, trav_vars.previous_node),
.instance_top_node = nir_build_deref_var(b, trav_vars.instance_top_node),
.instance_bottom_node = nir_build_deref_var(b, trav_vars.instance_bottom_node),
.second_iteration = nir_build_deref_var(b, trav_vars.second_iteration),
.instance_addr = nir_build_deref_var(b, trav_vars.instance_addr),
.sbt_offset_and_flags = nir_build_deref_var(b, trav_vars.sbt_offset_and_flags),
};
@@ -69,6 +69,7 @@ struct update_scratch_layout {
enum radv_encode_key_bits {
RADV_ENCODE_KEY_COMPACT = (1 << 0),
RADV_ENCODE_KEY_WRITE_LEAF_NODE_OFFSETS = (1 << 1),
RADV_ENCODE_KEY_PAIR_COMPRESS_GFX12 = (1 << 2),
};
static void
@@ -262,6 +263,12 @@ radv_get_build_config(VkDevice _device, struct vk_acceleration_structure_build_s
if ((state->build_info->flags & VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_DATA_ACCESS_KHR) ||
state->build_info->type != VK_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL_KHR)
encode_key |= RADV_ENCODE_KEY_WRITE_LEAF_NODE_OFFSETS;
VkGeometryTypeKHR geometry_type = vk_get_as_geometry_type(state->build_info);
if (!(state->build_info->flags & (VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_UPDATE_BIT_KHR |
VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_DATA_ACCESS_KHR)) &&
geometry_type == VK_GEOMETRY_TYPE_TRIANGLES_KHR)
encode_key |= RADV_ENCODE_KEY_PAIR_COMPRESS_GFX12;
}
if (state->build_info->flags & VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_COMPACTION_BIT_KHR)
@@ -343,6 +350,8 @@ radv_build_flags(VkCommandBuffer commandBuffer, uint32_t key)
}
if (key & RADV_ENCODE_KEY_WRITE_LEAF_NODE_OFFSETS)
flags |= RADV_BUILD_FLAG_WRITE_LEAF_NODE_OFFSETS;
if (key & RADV_ENCODE_KEY_PAIR_COMPRESS_GFX12)
flags |= RADV_BUILD_FLAG_PAIR_COMPRESS_TRIANGLES;
return flags;
}