From 4b119b36c839415c7c7c12f2fd3aae9e2cd576c5 Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Fri, 11 Apr 2025 12:00:22 -0400 Subject: [PATCH] hk: use ro maps Signed-off-by: Alyssa Rosenzweig Part-of: --- src/asahi/vulkan/hk_buffer.c | 22 +++++++++- src/asahi/vulkan/hk_buffer.h | 17 ++++++-- src/asahi/vulkan/hk_buffer_view.c | 8 ++-- src/asahi/vulkan/hk_cmd_dispatch.c | 2 +- src/asahi/vulkan/hk_cmd_draw.c | 16 ++++---- src/asahi/vulkan/hk_cmd_meta.c | 8 ++-- src/asahi/vulkan/hk_descriptor_set.c | 61 ++++++++++++++++------------ src/asahi/vulkan/hk_query_pool.c | 2 +- 8 files changed, 87 insertions(+), 49 deletions(-) diff --git a/src/asahi/vulkan/hk_buffer.c b/src/asahi/vulkan/hk_buffer.c index ea404da774e..c4672f55c20 100644 --- a/src/asahi/vulkan/hk_buffer.c +++ b/src/asahi/vulkan/hk_buffer.c @@ -296,5 +296,25 @@ hk_GetBufferOpaqueCaptureAddress(UNUSED VkDevice device, { VK_FROM_HANDLE(hk_buffer, buffer, pInfo->buffer); - return hk_buffer_address(buffer, 0); + return hk_buffer_address_rw(buffer, 0); +} + +uint64_t +hk_buffer_address(const struct hk_buffer *buffer, uint64_t offset, + bool read_only) +{ + struct hk_device *dev = (struct hk_device *)buffer->vk.base.device; + uint64_t addr = vk_buffer_address(&buffer->vk, offset); + + /* If we are accessing the buffer read-only, we want to return the read-only + * shadow mapping so non-resident pages return zeroes. That only applies to + * sparse resident buffers, which will have buffer->va != NULL. If buffer->va + * is NULL, the buffer is not sparse resident, so we don't need the fix up... + * and indeed, there may not be a shadow map available. + */ + if (read_only && buffer->va) { + addr = agx_rw_addr_to_ro(&dev->dev, addr); + } + + return addr; } diff --git a/src/asahi/vulkan/hk_buffer.h b/src/asahi/vulkan/hk_buffer.h index 47ee4e9807d..9e65ad121eb 100644 --- a/src/asahi/vulkan/hk_buffer.h +++ b/src/asahi/vulkan/hk_buffer.h @@ -24,15 +24,24 @@ struct hk_buffer { VK_DEFINE_NONDISP_HANDLE_CASTS(hk_buffer, vk.base, VkBuffer, VK_OBJECT_TYPE_BUFFER) +uint64_t hk_buffer_address(const struct hk_buffer *buffer, uint64_t offset, + bool read_only); + static inline uint64_t -hk_buffer_address(const struct hk_buffer *buffer, uint64_t offset) +hk_buffer_address_rw(const struct hk_buffer *buffer, uint64_t offset) { - return vk_buffer_address(&buffer->vk, offset); + return hk_buffer_address(buffer, offset, false); +} + +static inline uint64_t +hk_buffer_address_ro(const struct hk_buffer *buffer, uint64_t offset) +{ + return hk_buffer_address(buffer, offset, true); } static inline struct hk_addr_range hk_buffer_addr_range(const struct hk_buffer *buffer, uint64_t offset, - uint64_t range) + uint64_t range, bool read_only) { /* If range == 0, return a NULL pointer. Thanks to soft fault, that allows * eliding robustness2 bounds checks for index = 0, as the bottom of VA space @@ -42,7 +51,7 @@ hk_buffer_addr_range(const struct hk_buffer *buffer, uint64_t offset, return (struct hk_addr_range){.range = 0}; return (struct hk_addr_range){ - .addr = hk_buffer_address(buffer, offset), + .addr = hk_buffer_address(buffer, offset, read_only), .range = vk_buffer_range(&buffer->vk, offset, range), }; } diff --git a/src/asahi/vulkan/hk_buffer_view.c b/src/asahi/vulkan/hk_buffer_view.c index 37344e672e1..afb21ee0e69 100644 --- a/src/asahi/vulkan/hk_buffer_view.c +++ b/src/asahi/vulkan/hk_buffer_view.c @@ -93,7 +93,9 @@ hk_CreateBufferView(VkDevice _device, const VkBufferViewCreateInfo *pCreateInfo, * This lets us offset partially in the shader instead, getting * around alignment restrictions on the base address pointer. */ - uint64_t base = hk_buffer_address(buffer, 0) + (view->vk.offset & ~0xf); + uint64_t head_offset_B = view->vk.offset & ~0xf; + uint64_t rw_base = hk_buffer_address_rw(buffer, 0) + head_offset_B; + uint64_t ro_base = hk_buffer_address_ro(buffer, 0) + head_offset_B; uint32_t tail_offset_B = view->vk.offset & 0xf; uint32_t tail_offset_el = tail_offset_B / util_format_get_blocksize(format); assert(tail_offset_el * util_format_get_blocksize(format) == tail_offset_B && @@ -114,7 +116,7 @@ hk_CreateBufferView(VkDevice _device, const VkBufferViewCreateInfo *pCreateInfo, cfg.height = DIV_ROUND_UP(view->vk.elements, cfg.width); cfg.first_level = cfg.last_level = 0; - cfg.address = base; + cfg.address = ro_base; cfg.buffer_size_sw = view->vk.elements; cfg.buffer_offset_sw = tail_offset_el; @@ -146,7 +148,7 @@ hk_CreateBufferView(VkDevice _device, const VkBufferViewCreateInfo *pCreateInfo, cfg.swizzle_a = i; } - cfg.buffer = base; + cfg.buffer = rw_base; cfg.buffer_offset_sw = tail_offset_el; cfg.width = AGX_TEXTURE_BUFFER_WIDTH; diff --git a/src/asahi/vulkan/hk_cmd_dispatch.c b/src/asahi/vulkan/hk_cmd_dispatch.c index bdfb2732a55..0fe522aaa70 100644 --- a/src/asahi/vulkan/hk_cmd_dispatch.c +++ b/src/asahi/vulkan/hk_cmd_dispatch.c @@ -160,7 +160,7 @@ hk_CmdDispatchIndirect(VkCommandBuffer commandBuffer, VkBuffer _buffer, desc->root.cs.base_group[1] = 0; desc->root.cs.base_group[2] = 0; - uint64_t dispatch_addr = hk_buffer_address(buffer, offset); + uint64_t dispatch_addr = hk_buffer_address(buffer, offset, true); assert(dispatch_addr != 0); desc->root.cs.group_count_addr = dispatch_addr; diff --git a/src/asahi/vulkan/hk_cmd_draw.c b/src/asahi/vulkan/hk_cmd_draw.c index 5a6fcc87046..2741ff23184 100644 --- a/src/asahi/vulkan/hk_cmd_draw.c +++ b/src/asahi/vulkan/hk_cmd_draw.c @@ -3285,7 +3285,7 @@ hk_CmdBindIndexBuffer2KHR(VkCommandBuffer commandBuffer, VkBuffer _buffer, VK_FROM_HANDLE(hk_buffer, buffer, _buffer); cmd->state.gfx.index = (struct hk_index_buffer_state){ - .buffer = hk_buffer_addr_range(buffer, offset, size), + .buffer = hk_buffer_addr_range(buffer, offset, size, true), .size = agx_translate_index_size(vk_index_type_to_bytes(indexType)), .restart = vk_index_to_restart(indexType), }; @@ -3323,7 +3323,7 @@ hk_CmdBindVertexBuffers2(VkCommandBuffer commandBuffer, uint32_t firstBinding, uint64_t size = pSizes ? pSizes[i] : VK_WHOLE_SIZE; const struct hk_addr_range addr_range = - hk_buffer_addr_range(buffer, pOffsets[i], size); + hk_buffer_addr_range(buffer, pOffsets[i], size, true); hk_cmd_bind_vertex_buffer(cmd, idx, addr_range); } @@ -3678,7 +3678,7 @@ hk_CmdDrawIndirect(VkCommandBuffer commandBuffer, VkBuffer _buffer, { VK_FROM_HANDLE(hk_buffer, buffer, _buffer); - hk_draw_indirect_inner(commandBuffer, hk_buffer_address(buffer, offset), + hk_draw_indirect_inner(commandBuffer, hk_buffer_address_ro(buffer, offset), drawCount, stride); } @@ -3722,7 +3722,7 @@ hk_CmdDrawIndexedIndirect(VkCommandBuffer commandBuffer, VkBuffer _buffer, VK_FROM_HANDLE(hk_buffer, buffer, _buffer); hk_draw_indexed_indirect_inner( - commandBuffer, hk_buffer_address(buffer, offset), drawCount, stride); + commandBuffer, hk_buffer_address_ro(buffer, offset), drawCount, stride); } /* @@ -3746,8 +3746,8 @@ hk_draw_indirect_count(VkCommandBuffer commandBuffer, VkBuffer _buffer, size_t out_stride = sizeof(uint32_t) * (indexed ? 5 : 4); uint64_t patched = hk_pool_alloc(cmd, out_stride * maxDrawCount, 4).gpu; - uint64_t in = hk_buffer_address(buffer, offset); - uint64_t count_addr = hk_buffer_address(count_buffer, countBufferOffset); + uint64_t in = hk_buffer_address_ro(buffer, offset); + uint64_t count_addr = hk_buffer_address_ro(count_buffer, countBufferOffset); libagx_predicate_indirect(cmd, agx_1d(maxDrawCount), AGX_BARRIER_ALL | AGX_PREGFX, patched, in, @@ -3807,7 +3807,7 @@ hk_CmdBindTransformFeedbackBuffersEXT(VkCommandBuffer commandBuffer, uint32_t idx = firstBinding + i; uint64_t size = pSizes ? pSizes[i] : VK_WHOLE_SIZE; - gfx->xfb[idx] = hk_buffer_addr_range(buffer, pOffsets[i], size); + gfx->xfb[idx] = hk_buffer_addr_range(buffer, pOffsets[i], size, false); } } @@ -3844,7 +3844,7 @@ hk_begin_end_xfb(VkCommandBuffer commandBuffer, uint32_t firstCounterBuffer, VK_FROM_HANDLE(hk_buffer, buffer, pCounterBuffers[i]); uint64_t offset = pCounterBufferOffsets ? pCounterBufferOffsets[i] : 0; - uint64_t cb_addr = hk_buffer_address(buffer, offset); + uint64_t cb_addr = hk_buffer_address_rw(buffer, offset); uint32_t cmd_idx = firstCounterBuffer + i; if (begin) { diff --git a/src/asahi/vulkan/hk_cmd_meta.c b/src/asahi/vulkan/hk_cmd_meta.c index 9d000c761cb..2cb8fc83bfb 100644 --- a/src/asahi/vulkan/hk_cmd_meta.c +++ b/src/asahi/vulkan/hk_cmd_meta.c @@ -870,7 +870,7 @@ hk_meta_copy_image_to_buffer2(struct vk_command_buffer *cmd, hk_format_to_pipe_format(src_image->format); struct vk_meta_push_data push = { - .buffer = hk_buffer_address(buffer, region->bufferOffset), + .buffer = hk_buffer_address_rw(buffer, region->bufferOffset), .row_extent = row_extent, .slice_or_layer_extent = is_3d ? slice_extent : layer_extent, @@ -1070,7 +1070,7 @@ hk_meta_copy_buffer_to_image2(struct vk_command_buffer *cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); struct vk_meta_push_data push = { - .buffer = hk_buffer_address(buffer, region->bufferOffset), + .buffer = hk_buffer_address_ro(buffer, region->bufferOffset), .row_extent = row_extent, .slice_or_layer_extent = is_3d ? slice_extent : layer_extent, }; @@ -1396,8 +1396,8 @@ hk_CmdCopyBuffer2(VkCommandBuffer commandBuffer, const VkCopyBufferInfo2 *info) for (unsigned i = 0; i < info->regionCount; i++) { const VkBufferCopy2 *region = &info->pRegions[i]; - uint64_t src = vk_meta_buffer_address(&dev->vk, info->srcBuffer, - region->srcOffset, region->size); + VK_FROM_HANDLE(hk_buffer, src_buffer, info->srcBuffer); + uint64_t src = hk_buffer_address_ro(src_buffer, region->srcOffset); uint64_t dst = vk_meta_buffer_address(&dev->vk, info->dstBuffer, region->dstOffset, region->size); hk_cmd_copy(cmd, dst, src, region->size); diff --git a/src/asahi/vulkan/hk_descriptor_set.c b/src/asahi/vulkan/hk_descriptor_set.c index 3781c06e7a7..42c9728d041 100644 --- a/src/asahi/vulkan/hk_descriptor_set.c +++ b/src/asahi/vulkan/hk_descriptor_set.c @@ -161,43 +161,42 @@ write_storage_image_view_desc(struct hk_descriptor_set *set, write_desc(set, binding, elem, &desc, sizeof(desc)); } -static void -write_buffer_desc(struct hk_descriptor_set *set, - const VkDescriptorBufferInfo *const info, uint32_t binding, - uint32_t elem) +static struct hk_buffer_address +get_buffer_address(const VkDescriptorBufferInfo *const info, bool read_only) { VK_FROM_HANDLE(hk_buffer, buffer, info->buffer); - const struct hk_addr_range addr_range = - hk_buffer_addr_range(buffer, info->offset, info->range); + hk_buffer_addr_range(buffer, info->offset, info->range, read_only); assert(addr_range.range <= UINT32_MAX); - const struct hk_buffer_address desc = { + return (struct hk_buffer_address){ .base_addr = addr_range.addr, .size = addr_range.range, }; +} + +static void +write_buffer_desc(struct hk_descriptor_set *set, + const VkDescriptorBufferInfo *const info, uint32_t binding, + uint32_t elem, bool read_only) +{ + const struct hk_buffer_address desc = get_buffer_address(info, read_only); + write_desc(set, binding, elem, &desc, sizeof(desc)); } static void write_dynamic_buffer_desc(struct hk_descriptor_set *set, const VkDescriptorBufferInfo *const info, - uint32_t binding, uint32_t elem) + uint32_t binding, uint32_t elem, bool read_only) { - VK_FROM_HANDLE(hk_buffer, buffer, info->buffer); const struct hk_descriptor_set_binding_layout *binding_layout = &set->layout->binding[binding]; - const struct hk_addr_range addr_range = - hk_buffer_addr_range(buffer, info->offset, info->range); - assert(addr_range.range <= UINT32_MAX); - struct hk_buffer_address *desc = &set->dynamic_buffers[binding_layout->dynamic_buffer_index + elem]; - *desc = (struct hk_buffer_address){ - .base_addr = addr_range.addr, - .size = addr_range.range, - }; + + *desc = get_buffer_address(info, read_only); } static void @@ -273,17 +272,21 @@ hk_UpdateDescriptorSets(VkDevice device, uint32_t descriptorWriteCount, case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: for (uint32_t j = 0; j < write->descriptorCount; j++) { - write_buffer_desc(set, write->pBufferInfo + j, write->dstBinding, - write->dstArrayElement + j); + write_buffer_desc( + set, write->pBufferInfo + j, write->dstBinding, + write->dstArrayElement + j, + write->descriptorType == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER); } break; case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: for (uint32_t j = 0; j < write->descriptorCount; j++) { - write_dynamic_buffer_desc(set, write->pBufferInfo + j, - write->dstBinding, - write->dstArrayElement + j); + write_dynamic_buffer_desc( + set, write->pBufferInfo + j, write->dstBinding, + write->dstArrayElement + j, + write->descriptorType == + VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC); } break; @@ -387,8 +390,10 @@ hk_push_descriptor_set_update(struct hk_push_descriptor_set *push_set, case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: for (uint32_t j = 0; j < write->descriptorCount; j++) { - write_buffer_desc(&set, write->pBufferInfo + j, write->dstBinding, - write->dstArrayElement + j); + write_buffer_desc( + &set, write->pBufferInfo + j, write->dstBinding, + write->dstArrayElement + j, + write->descriptorType == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER); } break; @@ -749,7 +754,8 @@ hk_descriptor_set_write_template( data + entry->offset + j * entry->stride; write_buffer_desc(set, info, entry->binding, - entry->array_element + j); + entry->array_element + j, + entry->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER); } break; @@ -759,8 +765,9 @@ hk_descriptor_set_write_template( const VkDescriptorBufferInfo *info = data + entry->offset + j * entry->stride; - write_dynamic_buffer_desc(set, info, entry->binding, - entry->array_element + j); + write_dynamic_buffer_desc( + set, info, entry->binding, entry->array_element + j, + entry->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC); } break; diff --git a/src/asahi/vulkan/hk_query_pool.c b/src/asahi/vulkan/hk_query_pool.c index 9e8a98e19cf..a867194f1e4 100644 --- a/src/asahi/vulkan/hk_query_pool.c +++ b/src/asahi/vulkan/hk_query_pool.c @@ -665,7 +665,7 @@ hk_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQueryPool queryPool, .oq_index = pool->oq_queries ? pool->bo->va->addr + pool->query_start : 0, .first_query = firstQuery, - .dst_addr = hk_buffer_address(dst_buffer, dstOffset), + .dst_addr = hk_buffer_address_rw(dst_buffer, dstOffset), .dst_stride = stride, .reports_per_query = hk_reports_per_query(pool), .flags = flags,