radv: Add RADV_DEBUG=validatevas for address validation in nir

The option creates a buffer where each bit stores whether the
corresponding 4096 byte memory section has been allocated. The helper
radv_build_is_valid_va allows for querying the validity of addresses
inside a nir shader which can be useful for debugging.

Reviewed-by: Samuel Pitoiset <samuel.pitoiset@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/34392>

docs/envvars.rst

@@ -1481,6 +1481,8 @@ RADV driver environment variables
       Dump shader disassembly for selected shader stages.
    ``bvh4``
       Use bvh4 encoding on GPUs that support bvh8 encoding.
+   ``validatevas``
+      Enable tracking of VA ranges for radv_build_is_valid_va.
 
 .. envvar:: RADV_FORCE_FAMILY
 

src/amd/vulkan/radv_debug.h

@@ -73,6 +73,7 @@ enum {
    RADV_DEBUG_PSO_HISTORY = 1ull << 58,
    RADV_DEBUG_BVH4 = 1ull << 59,
    RADV_DEBUG_NO_VIDEO = 1ull << 60,
+   RADV_DEBUG_VALIDATE_VAS = 1ull << 61,
    RADV_DEBUG_DUMP_SHADERS = RADV_DEBUG_DUMP_VS | RADV_DEBUG_DUMP_TCS | RADV_DEBUG_DUMP_TES | RADV_DEBUG_DUMP_GS |
                              RADV_DEBUG_DUMP_PS | RADV_DEBUG_DUMP_TASK | RADV_DEBUG_DUMP_MESH | RADV_DEBUG_DUMP_CS |
                              RADV_DEBUG_DUMP_NIR | RADV_DEBUG_DUMP_ASM | RADV_DEBUG_DUMP_BACKEND_IR,

src/amd/vulkan/radv_debug_nir.c

@@ -349,10 +349,147 @@ radv_dump_printf_data(struct radv_device *device, FILE *out)
    header->offset = sizeof(struct radv_printf_buffer_header);
 }
 
+#define RADV_VA_VALIDATION_BITS              40
+#define RADV_VA_VALIDATION_BIT_COUNT         (1ull << RADV_VA_VALIDATION_BITS)
+#define RADV_VA_VALIDATION_GRANULARITY_BYTES 4096
+
+VkResult
+radv_init_va_validation(struct radv_device *device)
+{
+   struct radv_physical_device *pdev = radv_device_physical(device);
+
+   uint64_t size = RADV_VA_VALIDATION_BIT_COUNT / RADV_VA_VALIDATION_GRANULARITY_BYTES / 8;
+
+   VkBufferCreateInfo buffer_create_info = {
+      .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
+      .pNext =
+         &(VkBufferUsageFlags2CreateInfo){
+            .sType = VK_STRUCTURE_TYPE_BUFFER_USAGE_FLAGS_2_CREATE_INFO,
+            .usage = VK_BUFFER_USAGE_2_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_2_SHADER_DEVICE_ADDRESS_BIT,
+         },
+      .size = size,
+   };
+
+   VkDevice _device = radv_device_to_handle(device);
+   VkResult result =
+      device->vk.dispatch_table.CreateBuffer(_device, &buffer_create_info, NULL, &device->va_validation_buffer);
+   if (result != VK_SUCCESS)
+      return result;
+
+   VkMemoryRequirements requirements;
+   device->vk.dispatch_table.GetBufferMemoryRequirements(_device, device->va_validation_buffer, &requirements);
+
+   VkMemoryAllocateFlagsInfo alloc_flags_info = {
+      .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO,
+      .flags = VK_MEMORY_ALLOCATE_ZERO_INITIALIZE_BIT_EXT,
+   };
+
+   VkMemoryAllocateInfo alloc_info = {
+      .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
+      .pNext = &alloc_flags_info,
+      .allocationSize = requirements.size,
+      .memoryTypeIndex =
+         radv_find_memory_index(pdev, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
+                                         VK_MEMORY_PROPERTY_HOST_COHERENT_BIT),
+   };
+
+   result = device->vk.dispatch_table.AllocateMemory(_device, &alloc_info, NULL, &device->va_validation_memory);
+   if (result != VK_SUCCESS)
+      return result;
+
+   void *data = NULL;
+   result = device->vk.dispatch_table.MapMemory(_device, device->va_validation_memory, 0, VK_WHOLE_SIZE, 0, &data);
+   if (result != VK_SUCCESS)
+      return result;
+
+   device->valid_vas = data;
+   memset(data, 0, size);
+
+   result = device->vk.dispatch_table.BindBufferMemory(_device, device->va_validation_buffer,
+                                                       device->va_validation_memory, 0);
+   if (result != VK_SUCCESS)
+      return result;
+
+   VkBufferDeviceAddressInfo addr_info = {
+      .sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO,
+      .buffer = device->va_validation_buffer,
+   };
+   device->valid_vas_addr = device->vk.dispatch_table.GetBufferDeviceAddress(_device, &addr_info);
+
+   return VK_SUCCESS;
+}
+
+void
+radv_finish_va_validation(struct radv_device *device)
+{
+   VkDevice _device = radv_device_to_handle(device);
+
+   device->valid_vas = NULL;
+
+   device->vk.dispatch_table.DestroyBuffer(_device, device->va_validation_buffer, NULL);
+   if (device->va_validation_memory)
+      device->vk.dispatch_table.UnmapMemory(_device, device->va_validation_memory);
+   device->vk.dispatch_table.FreeMemory(_device, device->va_validation_memory, NULL);
+}
+
+void
+radv_va_validation_update_page(struct radv_device *device, uint64_t va, uint64_t size, bool valid)
+{
+   if (!device->valid_vas)
+      return;
+
+   struct radv_physical_device *pdev = radv_device_physical(device);
+   assert(!(((va >> 32) & ~pdev->info.address32_hi) >> (RADV_VA_VALIDATION_BITS - 32)));
+
+   uint64_t start = (va & BITFIELD64_MASK(RADV_VA_VALIDATION_BITS)) / RADV_VA_VALIDATION_GRANULARITY_BYTES;
+   uint64_t end = start + size / RADV_VA_VALIDATION_GRANULARITY_BYTES;
+   assert(end > 0);
+   assert(end <= RADV_VA_VALIDATION_BIT_COUNT);
+
+   if (valid)
+      BITSET_SET_RANGE(device->valid_vas, start, end - 1);
+   else
+      BITSET_CLEAR_RANGE(device->valid_vas, start, end - 1);
+}
+
+nir_def *
+radv_build_is_valid_va(nir_builder *b, nir_def *addr)
+{
+   if (!device_ht)
+      return NULL;
+
+   struct radv_device *device = _mesa_hash_table_search(device_ht, b->shader)->data;
+   if (!device->valid_vas_addr)
+      return NULL;
+
+   nir_def *masked_addr = nir_iand_imm(b, addr, BITFIELD64_MASK(RADV_VA_VALIDATION_BITS));
+   nir_def *then_valid;
+   nir_def *else_valid;
+   nir_push_if(b, nir_ult_imm(b, masked_addr, RADV_VA_VALIDATION_BIT_COUNT * RADV_VA_VALIDATION_GRANULARITY_BYTES));
+   {
+      nir_def *index = nir_u2u32(b, nir_udiv_imm(b, masked_addr, RADV_VA_VALIDATION_GRANULARITY_BYTES));
+      nir_def *offset = nir_imul_imm(b, nir_udiv_imm(b, index, 32), 4);
+      nir_def *dword =
+         nir_build_load_global(b, 1, 32, nir_iadd_imm(b, nir_u2u64(b, offset), device->valid_vas_addr), .align_mul = 4);
+      index = nir_umod_imm(b, index, 32);
+      then_valid = nir_bitnz(b, dword, index);
+   }
+   nir_push_else(b, NULL);
+   {
+      else_valid = nir_imm_false(b);
+   }
+   nir_pop_if(b, NULL);
+   nir_def *valid = nir_if_phi(b, then_valid, else_valid);
+
+   radv_build_printf(b, nir_inot(b, valid), "radv: Invalid VA %lx\n", addr);
+
+   return valid;
+}
+
 void
 radv_device_associate_nir(struct radv_device *device, nir_shader *nir)
 {
-   if (!device->printf.buffer_addr)
+   if (!device->printf.buffer_addr && !device->valid_vas_addr)
       return;
 
    if (!device_ht)

src/amd/vulkan/radv_debug_nir.h

@@ -44,6 +44,10 @@ struct radv_printf_buffer_header {
    uint32_t size;
 };
 
+void radv_device_associate_nir(struct radv_device *device, nir_shader *nir);
+
+/* shader printf */
+
 VkResult radv_printf_data_init(struct radv_device *device);
 
 void radv_printf_data_finish(struct radv_device *device);
@@ -54,6 +58,14 @@ void radv_build_printf(nir_builder *b, nir_def *cond, const char *format, ...);
 
 void radv_dump_printf_data(struct radv_device *device, FILE *out);
 
-void radv_device_associate_nir(struct radv_device *device, nir_shader *nir);
+/* shader va validation */
+
+VkResult radv_init_va_validation(struct radv_device *device);
+
+void radv_finish_va_validation(struct radv_device *device);
+
+void radv_va_validation_update_page(struct radv_device *device, uint64_t va, uint64_t size, bool valid);
+
+nir_def *radv_build_is_valid_va(nir_builder *b, nir_def *addr);
 
 #endif /* RADV_PRINTF_H */

src/amd/vulkan/radv_device.c

@@ -709,6 +709,12 @@ radv_device_init_tools(struct radv_device *device)
    if (result != VK_SUCCESS)
       return result;
 
+   if (instance->debug_flags & RADV_DEBUG_VALIDATE_VAS) {
+      result = radv_init_va_validation(device);
+      if (result != VK_SUCCESS)
+         return result;
+   }
+
    result = radv_device_init_rgp(device);
    if (result != VK_SUCCESS)
       return result;
@@ -740,6 +746,7 @@ radv_device_finish_tools(struct radv_device *device)
    radv_trap_handler_finish(device);
    radv_memory_trace_finish(device);
    radv_device_finish_rgp(device);
+   radv_finish_va_validation(device);
    radv_device_finish_device_fault_detection(device);
 }
 

src/amd/vulkan/radv_device.h

@@ -15,6 +15,7 @@
 #include "ac_spm.h"
 #include "ac_sqtt.h"
 
+#include "util/bitset.h"
 #include "util/mesa-blake3.h"
 
 #include "radv_debug_nir.h"
@@ -172,6 +173,11 @@ struct radv_device {
    struct radeon_winsys_bo *trace_bo;
    struct radv_trace_data *trace_data;
 
+   VkDeviceMemory va_validation_memory;
+   VkBuffer va_validation_buffer;
+   BITSET_WORD *valid_vas;
+   uint64_t valid_vas_addr;
+
    /* Whether to keep shader debug info, for debugging. */
    bool keep_shader_info;
 

src/amd/vulkan/radv_device_memory.c

@@ -53,6 +53,8 @@ radv_free_memory(struct radv_device *device, const VkAllocationCallbacks *pAlloc
 #endif
 
    if (mem->bo) {
+      radv_va_validation_update_page(device, mem->bo->va, mem->alloc_size, false);
+
       if (device->overallocation_disallowed) {
          mtx_lock(&device->overallocation_mutex);
          device->allocated_memory_size[mem->heap_index] -= mem->alloc_size;
@@ -292,6 +294,8 @@ radv_alloc_memory(struct radv_device *device, const VkMemoryAllocateInfo *pAlloc
 
       mem->heap_index = heap_index;
       mem->alloc_size = alloc_size;
+
+      radv_va_validation_update_page(device, mem->bo->va, alloc_size, true);
    }
 
    if (!wsi_info) {

src/amd/vulkan/radv_instance.c

@@ -88,6 +88,7 @@ static const struct debug_control radv_debug_options[] = {{"nofastclears", RADV_
                                                           {"pso_history", RADV_DEBUG_PSO_HISTORY},
                                                           {"bvh4", RADV_DEBUG_BVH4},
                                                           {"novideo", RADV_DEBUG_NO_VIDEO},
+                                                          {"validatevas", RADV_DEBUG_VALIDATE_VAS},
                                                           {NULL, 0}};
 
 const char *

src/amd/vulkan/radv_pipeline_cache.c

@@ -138,6 +138,10 @@ radv_is_cache_disabled(const struct radv_device *device, const struct vk_pipelin
    if (device->printf.buffer_addr)
       return true;
 
+   /* The buffer address used for validating VAs is hardcoded. */
+   if (device->valid_vas_addr)
+      return true;
+
    /* Pipeline caches can be disabled with RADV_DEBUG=nocache, with MESA_GLSL_CACHE_DISABLE=1 and
     * when ACO_DEBUG is used. MESA_GLSL_CACHE_DISABLE is done elsewhere.
     */