diff --git a/docs/envvars.rst b/docs/envvars.rst index 678895c8d89..ffd877c529a 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -1481,6 +1481,8 @@ RADV driver environment variables Dump shader disassembly for selected shader stages. ``bvh4`` Use bvh4 encoding on GPUs that support bvh8 encoding. + ``validatevas`` + Enable tracking of VA ranges for radv_build_is_valid_va. .. envvar:: RADV_FORCE_FAMILY diff --git a/src/amd/vulkan/radv_debug.h b/src/amd/vulkan/radv_debug.h index 4390ba7a9bd..512a4378400 100644 --- a/src/amd/vulkan/radv_debug.h +++ b/src/amd/vulkan/radv_debug.h @@ -73,6 +73,7 @@ enum { RADV_DEBUG_PSO_HISTORY = 1ull << 58, RADV_DEBUG_BVH4 = 1ull << 59, RADV_DEBUG_NO_VIDEO = 1ull << 60, + RADV_DEBUG_VALIDATE_VAS = 1ull << 61, RADV_DEBUG_DUMP_SHADERS = RADV_DEBUG_DUMP_VS | RADV_DEBUG_DUMP_TCS | RADV_DEBUG_DUMP_TES | RADV_DEBUG_DUMP_GS | RADV_DEBUG_DUMP_PS | RADV_DEBUG_DUMP_TASK | RADV_DEBUG_DUMP_MESH | RADV_DEBUG_DUMP_CS | RADV_DEBUG_DUMP_NIR | RADV_DEBUG_DUMP_ASM | RADV_DEBUG_DUMP_BACKEND_IR, diff --git a/src/amd/vulkan/radv_debug_nir.c b/src/amd/vulkan/radv_debug_nir.c index a66333f793c..6018585796a 100644 --- a/src/amd/vulkan/radv_debug_nir.c +++ b/src/amd/vulkan/radv_debug_nir.c @@ -349,10 +349,147 @@ radv_dump_printf_data(struct radv_device *device, FILE *out) header->offset = sizeof(struct radv_printf_buffer_header); } +#define RADV_VA_VALIDATION_BITS 40 +#define RADV_VA_VALIDATION_BIT_COUNT (1ull << RADV_VA_VALIDATION_BITS) +#define RADV_VA_VALIDATION_GRANULARITY_BYTES 4096 + +VkResult +radv_init_va_validation(struct radv_device *device) +{ + struct radv_physical_device *pdev = radv_device_physical(device); + + uint64_t size = RADV_VA_VALIDATION_BIT_COUNT / RADV_VA_VALIDATION_GRANULARITY_BYTES / 8; + + VkBufferCreateInfo buffer_create_info = { + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = + &(VkBufferUsageFlags2CreateInfo){ + .sType = VK_STRUCTURE_TYPE_BUFFER_USAGE_FLAGS_2_CREATE_INFO, + .usage = VK_BUFFER_USAGE_2_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_2_SHADER_DEVICE_ADDRESS_BIT, + }, + .size = size, + }; + + VkDevice _device = radv_device_to_handle(device); + VkResult result = + device->vk.dispatch_table.CreateBuffer(_device, &buffer_create_info, NULL, &device->va_validation_buffer); + if (result != VK_SUCCESS) + return result; + + VkMemoryRequirements requirements; + device->vk.dispatch_table.GetBufferMemoryRequirements(_device, device->va_validation_buffer, &requirements); + + VkMemoryAllocateFlagsInfo alloc_flags_info = { + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, + .flags = VK_MEMORY_ALLOCATE_ZERO_INITIALIZE_BIT_EXT, + }; + + VkMemoryAllocateInfo alloc_info = { + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .pNext = &alloc_flags_info, + .allocationSize = requirements.size, + .memoryTypeIndex = + radv_find_memory_index(pdev, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT), + }; + + result = device->vk.dispatch_table.AllocateMemory(_device, &alloc_info, NULL, &device->va_validation_memory); + if (result != VK_SUCCESS) + return result; + + void *data = NULL; + result = device->vk.dispatch_table.MapMemory(_device, device->va_validation_memory, 0, VK_WHOLE_SIZE, 0, &data); + if (result != VK_SUCCESS) + return result; + + device->valid_vas = data; + memset(data, 0, size); + + result = device->vk.dispatch_table.BindBufferMemory(_device, device->va_validation_buffer, + device->va_validation_memory, 0); + if (result != VK_SUCCESS) + return result; + + VkBufferDeviceAddressInfo addr_info = { + .sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, + .buffer = device->va_validation_buffer, + }; + device->valid_vas_addr = device->vk.dispatch_table.GetBufferDeviceAddress(_device, &addr_info); + + return VK_SUCCESS; +} + +void +radv_finish_va_validation(struct radv_device *device) +{ + VkDevice _device = radv_device_to_handle(device); + + device->valid_vas = NULL; + + device->vk.dispatch_table.DestroyBuffer(_device, device->va_validation_buffer, NULL); + if (device->va_validation_memory) + device->vk.dispatch_table.UnmapMemory(_device, device->va_validation_memory); + device->vk.dispatch_table.FreeMemory(_device, device->va_validation_memory, NULL); +} + +void +radv_va_validation_update_page(struct radv_device *device, uint64_t va, uint64_t size, bool valid) +{ + if (!device->valid_vas) + return; + + struct radv_physical_device *pdev = radv_device_physical(device); + assert(!(((va >> 32) & ~pdev->info.address32_hi) >> (RADV_VA_VALIDATION_BITS - 32))); + + uint64_t start = (va & BITFIELD64_MASK(RADV_VA_VALIDATION_BITS)) / RADV_VA_VALIDATION_GRANULARITY_BYTES; + uint64_t end = start + size / RADV_VA_VALIDATION_GRANULARITY_BYTES; + assert(end > 0); + assert(end <= RADV_VA_VALIDATION_BIT_COUNT); + + if (valid) + BITSET_SET_RANGE(device->valid_vas, start, end - 1); + else + BITSET_CLEAR_RANGE(device->valid_vas, start, end - 1); +} + +nir_def * +radv_build_is_valid_va(nir_builder *b, nir_def *addr) +{ + if (!device_ht) + return NULL; + + struct radv_device *device = _mesa_hash_table_search(device_ht, b->shader)->data; + if (!device->valid_vas_addr) + return NULL; + + nir_def *masked_addr = nir_iand_imm(b, addr, BITFIELD64_MASK(RADV_VA_VALIDATION_BITS)); + nir_def *then_valid; + nir_def *else_valid; + nir_push_if(b, nir_ult_imm(b, masked_addr, RADV_VA_VALIDATION_BIT_COUNT * RADV_VA_VALIDATION_GRANULARITY_BYTES)); + { + nir_def *index = nir_u2u32(b, nir_udiv_imm(b, masked_addr, RADV_VA_VALIDATION_GRANULARITY_BYTES)); + nir_def *offset = nir_imul_imm(b, nir_udiv_imm(b, index, 32), 4); + nir_def *dword = + nir_build_load_global(b, 1, 32, nir_iadd_imm(b, nir_u2u64(b, offset), device->valid_vas_addr), .align_mul = 4); + index = nir_umod_imm(b, index, 32); + then_valid = nir_bitnz(b, dword, index); + } + nir_push_else(b, NULL); + { + else_valid = nir_imm_false(b); + } + nir_pop_if(b, NULL); + nir_def *valid = nir_if_phi(b, then_valid, else_valid); + + radv_build_printf(b, nir_inot(b, valid), "radv: Invalid VA %lx\n", addr); + + return valid; +} + void radv_device_associate_nir(struct radv_device *device, nir_shader *nir) { - if (!device->printf.buffer_addr) + if (!device->printf.buffer_addr && !device->valid_vas_addr) return; if (!device_ht) diff --git a/src/amd/vulkan/radv_debug_nir.h b/src/amd/vulkan/radv_debug_nir.h index 67d0427eb10..7df4dce8a49 100644 --- a/src/amd/vulkan/radv_debug_nir.h +++ b/src/amd/vulkan/radv_debug_nir.h @@ -44,6 +44,10 @@ struct radv_printf_buffer_header { uint32_t size; }; +void radv_device_associate_nir(struct radv_device *device, nir_shader *nir); + +/* shader printf */ + VkResult radv_printf_data_init(struct radv_device *device); void radv_printf_data_finish(struct radv_device *device); @@ -54,6 +58,14 @@ void radv_build_printf(nir_builder *b, nir_def *cond, const char *format, ...); void radv_dump_printf_data(struct radv_device *device, FILE *out); -void radv_device_associate_nir(struct radv_device *device, nir_shader *nir); +/* shader va validation */ + +VkResult radv_init_va_validation(struct radv_device *device); + +void radv_finish_va_validation(struct radv_device *device); + +void radv_va_validation_update_page(struct radv_device *device, uint64_t va, uint64_t size, bool valid); + +nir_def *radv_build_is_valid_va(nir_builder *b, nir_def *addr); #endif /* RADV_PRINTF_H */ diff --git a/src/amd/vulkan/radv_device.c b/src/amd/vulkan/radv_device.c index cdd4ad27460..8b8b09a0d70 100644 --- a/src/amd/vulkan/radv_device.c +++ b/src/amd/vulkan/radv_device.c @@ -709,6 +709,12 @@ radv_device_init_tools(struct radv_device *device) if (result != VK_SUCCESS) return result; + if (instance->debug_flags & RADV_DEBUG_VALIDATE_VAS) { + result = radv_init_va_validation(device); + if (result != VK_SUCCESS) + return result; + } + result = radv_device_init_rgp(device); if (result != VK_SUCCESS) return result; @@ -740,6 +746,7 @@ radv_device_finish_tools(struct radv_device *device) radv_trap_handler_finish(device); radv_memory_trace_finish(device); radv_device_finish_rgp(device); + radv_finish_va_validation(device); radv_device_finish_device_fault_detection(device); } diff --git a/src/amd/vulkan/radv_device.h b/src/amd/vulkan/radv_device.h index 30a1c8ddd7c..86914f6265c 100644 --- a/src/amd/vulkan/radv_device.h +++ b/src/amd/vulkan/radv_device.h @@ -15,6 +15,7 @@ #include "ac_spm.h" #include "ac_sqtt.h" +#include "util/bitset.h" #include "util/mesa-blake3.h" #include "radv_debug_nir.h" @@ -172,6 +173,11 @@ struct radv_device { struct radeon_winsys_bo *trace_bo; struct radv_trace_data *trace_data; + VkDeviceMemory va_validation_memory; + VkBuffer va_validation_buffer; + BITSET_WORD *valid_vas; + uint64_t valid_vas_addr; + /* Whether to keep shader debug info, for debugging. */ bool keep_shader_info; diff --git a/src/amd/vulkan/radv_device_memory.c b/src/amd/vulkan/radv_device_memory.c index a172ce60805..41440781cb6 100644 --- a/src/amd/vulkan/radv_device_memory.c +++ b/src/amd/vulkan/radv_device_memory.c @@ -53,6 +53,8 @@ radv_free_memory(struct radv_device *device, const VkAllocationCallbacks *pAlloc #endif if (mem->bo) { + radv_va_validation_update_page(device, mem->bo->va, mem->alloc_size, false); + if (device->overallocation_disallowed) { mtx_lock(&device->overallocation_mutex); device->allocated_memory_size[mem->heap_index] -= mem->alloc_size; @@ -292,6 +294,8 @@ radv_alloc_memory(struct radv_device *device, const VkMemoryAllocateInfo *pAlloc mem->heap_index = heap_index; mem->alloc_size = alloc_size; + + radv_va_validation_update_page(device, mem->bo->va, alloc_size, true); } if (!wsi_info) { diff --git a/src/amd/vulkan/radv_instance.c b/src/amd/vulkan/radv_instance.c index fcecf75b9d4..3af92eeb811 100644 --- a/src/amd/vulkan/radv_instance.c +++ b/src/amd/vulkan/radv_instance.c @@ -88,6 +88,7 @@ static const struct debug_control radv_debug_options[] = {{"nofastclears", RADV_ {"pso_history", RADV_DEBUG_PSO_HISTORY}, {"bvh4", RADV_DEBUG_BVH4}, {"novideo", RADV_DEBUG_NO_VIDEO}, + {"validatevas", RADV_DEBUG_VALIDATE_VAS}, {NULL, 0}}; const char * diff --git a/src/amd/vulkan/radv_pipeline_cache.c b/src/amd/vulkan/radv_pipeline_cache.c index 9d811480541..a1723f2f8d6 100644 --- a/src/amd/vulkan/radv_pipeline_cache.c +++ b/src/amd/vulkan/radv_pipeline_cache.c @@ -138,6 +138,10 @@ radv_is_cache_disabled(const struct radv_device *device, const struct vk_pipelin if (device->printf.buffer_addr) return true; + /* The buffer address used for validating VAs is hardcoded. */ + if (device->valid_vas_addr) + return true; + /* Pipeline caches can be disabled with RADV_DEBUG=nocache, with MESA_GLSL_CACHE_DISABLE=1 and * when ACO_DEBUG is used. MESA_GLSL_CACHE_DISABLE is done elsewhere. */