From ae9c65a55282d780a58c5c1e60b2c06e968491a8 Mon Sep 17 00:00:00 2001 From: Konstantin Seurer Date: Mon, 28 Nov 2022 20:11:17 +0100 Subject: [PATCH] radv/rra: Copy accel structs directly after build This is the second step of decoupling acceleration structure dumping from lifetimes. It also simplifies the logic a bit. Part-of: --- src/amd/vulkan/layers/radv_rra_layer.c | 136 +++++++++++- src/amd/vulkan/radv_private.h | 4 + src/amd/vulkan/radv_rra.c | 281 ++++--------------------- 3 files changed, 173 insertions(+), 248 deletions(-) diff --git a/src/amd/vulkan/layers/radv_rra_layer.c b/src/amd/vulkan/layers/radv_rra_layer.c index b996d5cd153..387841053d1 100644 --- a/src/amd/vulkan/layers/radv_rra_layer.c +++ b/src/amd/vulkan/layers/radv_rra_layer.c @@ -100,6 +100,63 @@ rra_QueuePresentKHR(VkQueue _queue, const VkPresentInfoKHR *pPresentInfo) return VK_SUCCESS; } +static uint32_t +find_memory_index(VkDevice _device, VkMemoryPropertyFlags flags) +{ + RADV_FROM_HANDLE(radv_device, device, _device); + VkPhysicalDeviceMemoryProperties *mem_properties = &device->physical_device->memory_properties; + for (uint32_t i = 0; i < mem_properties->memoryTypeCount; ++i) { + if (mem_properties->memoryTypes[i].propertyFlags == flags) { + return i; + } + } + unreachable("invalid memory properties"); +} + +static VkResult +rra_init_accel_struct_data_buffer(VkDevice vk_device, struct radv_rra_accel_struct_data *data) +{ + VkBufferCreateInfo buffer_create_info = { + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .size = data->size, + }; + + VkResult result = radv_CreateBuffer(vk_device, &buffer_create_info, NULL, &data->buffer); + if (result != VK_SUCCESS) + return result; + + VkMemoryRequirements requirements; + vk_common_GetBufferMemoryRequirements(vk_device, data->buffer, &requirements); + + VkMemoryAllocateFlagsInfo flags_info = { + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, + .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, + }; + + VkMemoryAllocateInfo alloc_info = { + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .pNext = &flags_info, + .allocationSize = requirements.size, + .memoryTypeIndex = find_memory_index(vk_device, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_CACHED_BIT), + }; + result = radv_AllocateMemory(vk_device, &alloc_info, NULL, &data->memory); + if (result != VK_SUCCESS) + goto fail_buffer; + + result = vk_common_BindBufferMemory(vk_device, data->buffer, data->memory, 0); + if (result != VK_SUCCESS) + goto fail_memory; + + return result; +fail_memory: + radv_FreeMemory(vk_device, data->memory, NULL); +fail_buffer: + radv_DestroyBuffer(vk_device, data->buffer, NULL); + return result; +} + VKAPI_ATTR VkResult VKAPI_CALL rra_CreateAccelerationStructureKHR(VkDevice _device, const VkAccelerationStructureCreateInfoKHR *pCreateInfo, @@ -130,16 +187,27 @@ rra_CreateAccelerationStructureKHR(VkDevice _device, data->size = structure->size; data->type = pCreateInfo->type; - radv_CreateEvent(radv_device_to_handle(device), - &(const VkEventCreateInfo){ - .sType = VK_STRUCTURE_TYPE_EVENT_CREATE_INFO, - }, - NULL, &data->build_event); + VkEventCreateInfo eventCreateInfo = { + .sType = VK_STRUCTURE_TYPE_EVENT_CREATE_INFO, + }; + + result = + radv_CreateEvent(radv_device_to_handle(device), &eventCreateInfo, NULL, &data->build_event); + if (result != VK_SUCCESS) + goto fail_data; + + result = rra_init_accel_struct_data_buffer(_device, data); + if (result != VK_SUCCESS) + goto fail_event; _mesa_hash_table_insert(device->rra_trace.accel_structs, structure, data); _mesa_hash_table_u64_insert(device->rra_trace.accel_struct_vas, structure->va, structure); goto exit; +fail_event: + radv_DestroyEvent(_device, data->build_event, NULL); +fail_data: + free(data); fail_as: radv_DestroyAccelerationStructureKHR(_device, *pAccelerationStructure, pAllocator); *pAccelerationStructure = VK_NULL_HANDLE; @@ -148,6 +216,53 @@ exit: return result; } +static void +copy_accel_struct_to_data(VkCommandBuffer commandBuffer, + struct radv_acceleration_structure *accel_struct, + struct radv_rra_accel_struct_data *data) +{ + RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + + VkMemoryBarrier2 barrier = { + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, + .srcStageMask = VK_PIPELINE_STAGE_2_ACCELERATION_STRUCTURE_BUILD_BIT_KHR, + .srcAccessMask = VK_ACCESS_2_ACCELERATION_STRUCTURE_WRITE_BIT_KHR, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, + }; + + VkDependencyInfo dependencyInfo = { + .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, + .memoryBarrierCount = 1, + .pMemoryBarriers = &barrier, + }; + + radv_CmdPipelineBarrier2(commandBuffer, &dependencyInfo); + + vk_common_CmdSetEvent(commandBuffer, data->build_event, 0); + + struct radv_buffer tmp_buffer; + radv_buffer_init(&tmp_buffer, cmd_buffer->device, accel_struct->bo, accel_struct->size, + accel_struct->mem_offset); + + VkBufferCopy2 region = { + .sType = VK_STRUCTURE_TYPE_BUFFER_COPY_2, + .size = accel_struct->size, + }; + + VkCopyBufferInfo2 copyInfo = { + .sType = VK_STRUCTURE_TYPE_COPY_BUFFER_INFO_2, + .srcBuffer = radv_buffer_to_handle(&tmp_buffer), + .dstBuffer = data->buffer, + .regionCount = 1, + .pRegions = ®ion, + }; + + radv_CmdCopyBuffer2(commandBuffer, ©Info); + + radv_buffer_finish(&tmp_buffer); +} + VKAPI_ATTR void VKAPI_CALL rra_CmdBuildAccelerationStructuresKHR( VkCommandBuffer commandBuffer, uint32_t infoCount, @@ -166,7 +281,7 @@ rra_CmdBuildAccelerationStructuresKHR( assert(entry); struct radv_rra_accel_struct_data *data = entry->data; - vk_common_CmdSetEvent(commandBuffer, data->build_event, 0); + copy_accel_struct_to_data(commandBuffer, structure, data); } simple_mtx_unlock(&cmd_buffer->device->rra_trace.data_mtx); } @@ -187,7 +302,8 @@ rra_CmdCopyAccelerationStructureKHR(VkCommandBuffer commandBuffer, assert(entry); struct radv_rra_accel_struct_data *data = entry->data; - vk_common_CmdSetEvent(commandBuffer, data->build_event, 0); + copy_accel_struct_to_data(commandBuffer, structure, data); + simple_mtx_unlock(&cmd_buffer->device->rra_trace.data_mtx); } @@ -207,7 +323,8 @@ rra_CmdCopyMemoryToAccelerationStructureKHR(VkCommandBuffer commandBuffer, assert(entry); struct radv_rra_accel_struct_data *data = entry->data; - vk_common_CmdSetEvent(commandBuffer, data->build_event, 0); + copy_accel_struct_to_data(commandBuffer, structure, data); + simple_mtx_unlock(&cmd_buffer->device->rra_trace.data_mtx); } @@ -229,7 +346,8 @@ rra_DestroyAccelerationStructureKHR(VkDevice _device, VkAccelerationStructureKHR assert(entry); struct radv_rra_accel_struct_data *data = entry->data; - radv_DestroyEvent(_device, data->build_event, NULL); + radv_destroy_rra_accel_struct_data(_device, data); + _mesa_hash_table_remove(device->rra_trace.accel_structs, entry); _mesa_hash_table_u64_remove(device->rra_trace.accel_struct_vas, structure->va); simple_mtx_unlock(&device->rra_trace.data_mtx); diff --git a/src/amd/vulkan/radv_private.h b/src/amd/vulkan/radv_private.h index 117686f5ae6..045855997cf 100644 --- a/src/amd/vulkan/radv_private.h +++ b/src/amd/vulkan/radv_private.h @@ -803,9 +803,13 @@ struct radv_rra_accel_struct_data { VkEvent build_event; uint64_t va; uint64_t size; + VkBuffer buffer; + VkDeviceMemory memory; VkAccelerationStructureTypeKHR type; }; +void radv_destroy_rra_accel_struct_data(VkDevice device, struct radv_rra_accel_struct_data *data); + struct radv_rra_trace_data { int elapsed_frames; int trace_frame; diff --git a/src/amd/vulkan/radv_rra.c b/src/amd/vulkan/radv_rra.c index 54627a420b1..9fba6803248 100644 --- a/src/amd/vulkan/radv_rra.c +++ b/src/amd/vulkan/radv_rra.c @@ -705,19 +705,11 @@ rra_gather_bvh_info(const uint8_t *bvh, uint32_t node_id, struct rra_bvh_info *d } } -struct rra_copied_accel_struct { - struct radv_rra_accel_struct_data *accel_struct; - uint8_t *data; -}; - static VkResult -rra_dump_acceleration_structure(struct rra_copied_accel_struct *copied_struct, +rra_dump_acceleration_structure(struct radv_rra_accel_struct_data *accel_struct, uint8_t *data, struct hash_table_u64 *accel_struct_vas, bool should_validate, FILE *output) { - struct radv_rra_accel_struct_data *accel_struct = copied_struct->accel_struct; - uint8_t *data = copied_struct->data; - struct radv_accel_struct_header *header = (struct radv_accel_struct_header *)data; bool is_tlas = header->instance_count > 0; @@ -887,203 +879,20 @@ radv_rra_trace_finish(VkDevice vk_device, struct radv_rra_trace_data *data) { if (data->accel_structs) hash_table_foreach (data->accel_structs, entry) - free(entry->data); + radv_destroy_rra_accel_struct_data(vk_device, entry->data); simple_mtx_destroy(&data->data_mtx); _mesa_hash_table_destroy(data->accel_structs, NULL); _mesa_hash_table_u64_destroy(data->accel_struct_vas); } -static uint32_t -find_memory_index(VkDevice _device, VkMemoryPropertyFlags flags) +void +radv_destroy_rra_accel_struct_data(VkDevice device, struct radv_rra_accel_struct_data *data) { - RADV_FROM_HANDLE(radv_device, device, _device); - VkPhysicalDeviceMemoryProperties *mem_properties = &device->physical_device->memory_properties; - for (uint32_t i = 0; i < mem_properties->memoryTypeCount; ++i) { - if (mem_properties->memoryTypes[i].propertyFlags == flags) { - return i; - } - } - unreachable("invalid memory properties"); -} - -#define RRA_COPY_BATCH_SIZE 8 - -struct rra_accel_struct_copy { - struct rra_copied_accel_struct copied_structures[RRA_COPY_BATCH_SIZE]; - uint8_t *map_data; - VkDeviceMemory memory; - VkBuffer buffer; - VkCommandPool pool; - VkCommandBuffer cmd_buffer; -}; - -static VkResult -rra_init_acceleration_structure_copy(VkDevice vk_device, uint32_t family_index, - struct rra_accel_struct_copy *dst) -{ - RADV_FROM_HANDLE(radv_device, device, vk_device); - - VkCommandPoolCreateInfo pool_info = { - .sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, - .queueFamilyIndex = family_index, - }; - - VkResult result = vk_common_CreateCommandPool(vk_device, &pool_info, NULL, &dst->pool); - if (result != VK_SUCCESS) - goto fail; - - VkCommandBufferAllocateInfo cmdbuf_alloc_info = { - .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, - .commandPool = dst->pool, - .commandBufferCount = 1, - }; - result = vk_common_AllocateCommandBuffers(vk_device, &cmdbuf_alloc_info, &dst->cmd_buffer); - if (result != VK_SUCCESS) - goto fail_pool; - - size_t max_size = 0; - - hash_table_foreach(device->rra_trace.accel_structs, entry) - { - VkAccelerationStructureKHR structure = radv_acceleration_structure_to_handle((void *)entry->key); - RADV_FROM_HANDLE(radv_acceleration_structure, accel_struct, structure); - max_size = MAX2(max_size, accel_struct->size); - } - - size_t data_size = max_size * RRA_COPY_BATCH_SIZE; - - VkBufferCreateInfo buffer_create_info = { - .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, - .size = data_size, - .usage = VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, - }; - - result = radv_CreateBuffer(vk_device, &buffer_create_info, NULL, &dst->buffer); - if (result != VK_SUCCESS) - goto fail_pool; - VkMemoryRequirements requirements; - vk_common_GetBufferMemoryRequirements(vk_device, dst->buffer, &requirements); - - VkMemoryAllocateFlagsInfo flags_info = { - .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, - .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, - }; - - VkMemoryAllocateInfo alloc_info = { - .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, - .pNext = &flags_info, - .allocationSize = requirements.size, - .memoryTypeIndex = find_memory_index(vk_device, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | - VK_MEMORY_PROPERTY_HOST_CACHED_BIT), - }; - result = radv_AllocateMemory(vk_device, &alloc_info, NULL, &dst->memory); - if (result != VK_SUCCESS) - goto fail_buffer; - - result = radv_MapMemory(vk_device, dst->memory, 0, VK_WHOLE_SIZE, 0, (void **)&dst->map_data); - if (result != VK_SUCCESS) - goto fail_memory; - - result = vk_common_BindBufferMemory(vk_device, dst->buffer, dst->memory, 0); - if (result != VK_SUCCESS) - goto fail_memory; - - return result; -fail_memory: - radv_FreeMemory(vk_device, dst->memory, NULL); -fail_buffer: - radv_DestroyBuffer(vk_device, dst->buffer, NULL); -fail_pool: - vk_common_DestroyCommandPool(vk_device, dst->pool, NULL); -fail: - return result; -} - -static VkResult -rra_copy_acceleration_structures(VkQueue vk_queue, struct rra_accel_struct_copy *dst, - struct hash_entry **entries, uint32_t count, - uint32_t *copied_structure_count) -{ - RADV_FROM_HANDLE(radv_queue, queue, vk_queue); - *copied_structure_count = 0; - - struct radv_device *device = queue->device; - VkDevice vk_device = radv_device_to_handle(device); - - RADV_FROM_HANDLE(radv_cmd_buffer, cmdbuf, dst->cmd_buffer); - - vk_common_ResetCommandPool(vk_device, dst->pool, 0); - - /* - * Wait for possible AS build/trace calls on all queues. - */ - VkResult result = vk_common_DeviceWaitIdle(radv_device_to_handle(device)); - if (result != VK_SUCCESS) - goto fail; - - VkCommandBufferBeginInfo begin_info = { - .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, - }; - - radv_BeginCommandBuffer(dst->cmd_buffer, &begin_info); - - uint64_t dst_offset = 0; - for (uint32_t i = 0; i < count; i++) { - struct hash_entry *entry = entries[i]; - struct radv_rra_accel_struct_data *data = entry->data; - - VkResult event_result = radv_GetEventStatus(vk_device, data->build_event); - if (event_result != VK_EVENT_SET) { - continue; - } - - VkAccelerationStructureKHR structure = radv_acceleration_structure_to_handle((void *)entry->key); - RADV_FROM_HANDLE(radv_acceleration_structure, accel_struct, structure); - - struct radv_buffer tmp_buffer; - radv_buffer_init(&tmp_buffer, cmdbuf->device, accel_struct->bo, accel_struct->size, accel_struct->mem_offset); - - radv_CmdCopyBuffer2(dst->cmd_buffer, &(const VkCopyBufferInfo2){ - .sType = VK_STRUCTURE_TYPE_COPY_BUFFER_INFO_2, - .srcBuffer = radv_buffer_to_handle(&tmp_buffer), - .dstBuffer = dst->buffer, - .regionCount = 1, - .pRegions = &(const VkBufferCopy2){ - .sType = VK_STRUCTURE_TYPE_BUFFER_COPY_2, - .srcOffset = 0, - .dstOffset = dst_offset, - .size = accel_struct->size, - }, - }); - - radv_buffer_finish(&tmp_buffer); - - dst->copied_structures[*copied_structure_count].accel_struct = data; - dst->copied_structures[*copied_structure_count].data = dst->map_data + dst_offset; - - dst_offset += accel_struct->size; - - ++(*copied_structure_count); - } - result = radv_EndCommandBuffer(dst->cmd_buffer); - if (result != VK_SUCCESS) - goto fail; - - VkSubmitInfo submit_info = { - .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, - .commandBufferCount = 1, - .pCommandBuffers = &dst->cmd_buffer, - }; - result = vk_common_QueueSubmit(vk_queue, 1, &submit_info, VK_NULL_HANDLE); - if (result != VK_SUCCESS) - goto fail; - - result = vk_common_QueueWaitIdle(vk_queue); - -fail: - return result; + radv_DestroyEvent(device, data->build_event, NULL); + radv_DestroyBuffer(device, data->buffer, NULL); + radv_FreeMemory(device, data->memory, NULL); + free(data); } static int @@ -1103,20 +912,29 @@ radv_rra_dump_trace(VkQueue vk_queue, char *filename) RADV_FROM_HANDLE(radv_queue, queue, vk_queue); struct radv_device *device = queue->device; VkDevice vk_device = radv_device_to_handle(device); - struct hash_entry **hash_entries = NULL; + + VkResult result = vk_common_DeviceWaitIdle(vk_device); + if (result != VK_SUCCESS) + return result; uint32_t accel_struct_count = _mesa_hash_table_num_entries(device->rra_trace.accel_structs); - VkResult result; - uint64_t *accel_struct_offsets = calloc(accel_struct_count, sizeof(uint64_t)); if (!accel_struct_offsets) return VK_ERROR_OUT_OF_HOST_MEMORY; + uint32_t struct_count = _mesa_hash_table_num_entries(device->rra_trace.accel_structs); + struct hash_entry **hash_entries = malloc(sizeof(*hash_entries) * struct_count); + if (!hash_entries) { + free(accel_struct_offsets); + return VK_ERROR_OUT_OF_HOST_MEMORY; + } + FILE *file = fopen(filename, "w"); if (!file) { - result = VK_ERROR_OUT_OF_HOST_MEMORY; - goto fail; + free(accel_struct_offsets); + free(hash_entries); + return VK_ERROR_OUT_OF_HOST_MEMORY; } /* @@ -1136,18 +954,6 @@ radv_rra_dump_trace(VkQueue vk_queue, char *filename) uint64_t written_accel_struct_count = 0; - struct rra_accel_struct_copy copy = {0}; - result = rra_init_acceleration_structure_copy(vk_device, queue->vk.queue_family_index, ©); - if (result != VK_SUCCESS) - goto init_fail; - - uint32_t struct_count = _mesa_hash_table_num_entries(device->rra_trace.accel_structs); - hash_entries = malloc(sizeof(*hash_entries) * struct_count); - if (!hash_entries) { - result = VK_ERROR_OUT_OF_HOST_MEMORY; - goto fail; - } - struct hash_entry *last_entry = NULL; for (unsigned i = 0; (last_entry = _mesa_hash_table_next_entry(device->rra_trace.accel_structs, last_entry)); @@ -1155,23 +961,26 @@ radv_rra_dump_trace(VkQueue vk_queue, char *filename) hash_entries[i] = last_entry; qsort(hash_entries, struct_count, sizeof(*hash_entries), accel_struct_entry_cmp); - for (unsigned j = 0; j < struct_count; j += RRA_COPY_BATCH_SIZE) { - uint32_t copied_structure_count; - result = rra_copy_acceleration_structures(vk_queue, ©, hash_entries + j, - MIN2(RRA_COPY_BATCH_SIZE, struct_count - j), - &copied_structure_count); - if (result != VK_SUCCESS) - goto copy_fail; - for (uint32_t i = 0; i < copied_structure_count; ++i) { - accel_struct_offsets[written_accel_struct_count] = (uint64_t)ftell(file); - result = rra_dump_acceleration_structure(©.copied_structures[i], - device->rra_trace.accel_struct_vas, - device->rra_trace.validate_as, file); - if (result != VK_SUCCESS) - continue; - ++written_accel_struct_count; - } + for (unsigned i = 0; i < struct_count; i++) { + struct radv_rra_accel_struct_data *data = hash_entries[i]->data; + if (radv_GetEventStatus(vk_device, data->build_event) != VK_EVENT_SET) + continue; + + void *mapped_data; + result = radv_MapMemory(vk_device, data->memory, 0, VK_WHOLE_SIZE, 0, &mapped_data); + if (result != VK_SUCCESS) + continue; + + accel_struct_offsets[written_accel_struct_count] = (uint64_t)ftell(file); + result = + rra_dump_acceleration_structure(data, mapped_data, device->rra_trace.accel_struct_vas, + device->rra_trace.validate_as, file); + + radv_UnmapMemory(vk_device, data->memory); + + if (result == VK_SUCCESS) + written_accel_struct_count++; } uint64_t chunk_info_offset = (uint64_t)ftell(file); @@ -1197,14 +1006,8 @@ radv_rra_dump_trace(VkQueue vk_queue, char *filename) /* All info is available, dump header now */ fseek(file, 0, SEEK_SET); rra_dump_header(file, chunk_info_offset, file_end - chunk_info_offset); -copy_fail: - radv_DestroyBuffer(vk_device, copy.buffer, NULL); - radv_FreeMemory(vk_device, copy.memory, NULL); - vk_common_DestroyCommandPool(vk_device, copy.pool, NULL); -init_fail: fclose(file); -fail: free(hash_entries); free(accel_struct_offsets); - return result; + return VK_SUCCESS; }