radv/rt: Avoid encoding infinities in box node coords

On Navi33, certain box sorting modes combined with infinity/-infinity in
the child AABBs cause image_bvh64_intersect_ray to return garbage node
pointers.

To avoid this, convert infinity to the maximum representable
floating-point value, which will still intersect with any non-inf ray.

Fixes consistent hangs in DOOM: The Dark Ages.

Cc: mesa-stable
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/35254>

src/amd/vulkan/bvh/build_interface.h

@@ -21,6 +21,7 @@
 #define RADV_BUILD_FLAG_COMPACT         (1u << (VK_BUILD_FLAG_COUNT + 0))
 #define RADV_BUILD_FLAG_BVH8            (1u << (VK_BUILD_FLAG_COUNT + 1))
 #define RADV_BUILD_FLAG_UPDATE_IN_PLACE (1u << (VK_BUILD_FLAG_COUNT + 2))
+#define RADV_BUILD_FLAG_NO_INFS         (1u << (VK_BUILD_FLAG_COUNT + 3))
 
 #define RADV_COPY_MODE_COPY        0
 #define RADV_COPY_MODE_SERIALIZE   1

src/amd/vulkan/bvh/encode.comp

@@ -198,6 +198,20 @@ main()
          vk_aabb child_aabb =
             DEREF(REF(vk_ir_node)OFFSET(args.intermediate_bvh, offset)).aabb;
 
+         /* On gfx11, infinities in AABB coords can cause garbage child nodes to be
+          * returned by box intersection tests with non-default box sorting modes.
+          * Subtract 1 from the integer representation of inf/-inf to turn it into
+          * the maximum/minimum representable floating-point value as a workaround.
+          */
+         if (VK_BUILD_FLAG(RADV_BUILD_FLAG_NO_INFS)) {
+            for (uint32_t i = 0; i < 3; ++i) {
+               if (isinf(child_aabb.min[i]))
+                  child_aabb.min[i] = uintBitsToFloat(floatBitsToUint(child_aabb.min[i]) - 1);
+               if (isinf(child_aabb.max[i]))
+                  child_aabb.max[i] = uintBitsToFloat(floatBitsToUint(child_aabb.max[i]) - 1);
+            }
+         }
+
          DEREF(dst_node).coords[i] = child_aabb;
 
          uint32_t child_id = pack_node_id(dst_offset, ir_type_to_bvh_type(type));

src/amd/vulkan/radv_acceleration_structure.c

@@ -452,6 +452,9 @@ radv_build_flags(VkCommandBuffer commandBuffer, uint32_t key)
       flags |= RADV_BUILD_FLAG_COMPACT;
    if (radv_use_bvh8(pdev))
       flags |= RADV_BUILD_FLAG_BVH8;
+   /* gfx11 box intersection tests can return garbage with infs and non-standard box sorting */
+   if (pdev->info.gfx_level == GFX11)
+      flags |= RADV_BUILD_FLAG_NO_INFS;
 
    return flags;
 }
@@ -461,7 +464,7 @@ radv_encode_bind_pipeline(VkCommandBuffer commandBuffer, uint32_t key)
 {
    radv_bvh_build_bind_pipeline(commandBuffer, RADV_META_OBJECT_KEY_BVH_ENCODE, encode_spv, sizeof(encode_spv),
                                 sizeof(struct encode_args),
-                                radv_build_flags(commandBuffer, key) & RADV_BUILD_FLAG_COMPACT);
+                                radv_build_flags(commandBuffer, key));
 
    return VK_SUCCESS;
 }