From eb394f53162ea447fa427740b66af49e5e7ff946 Mon Sep 17 00:00:00 2001 From: Gert Wollny Date: Mon, 29 Jan 2024 19:51:17 +0100 Subject: [PATCH] zink: redesign the allocation try loop to test all heaps Before the first time an allocation failed the heap was changed, now we only change the heap type when checking all heaps results in allocation failure. Signed-off-by: Gert Wollny Part-of: --- src/gallium/drivers/zink/zink_resource.c | 35 +- .../drivers/zink/zink_synchronization.cpp | 387 +++++++----------- 2 files changed, 173 insertions(+), 249 deletions(-) diff --git a/src/gallium/drivers/zink/zink_resource.c b/src/gallium/drivers/zink/zink_resource.c index 0260fb8fe41..773c269e408 100644 --- a/src/gallium/drivers/zink/zink_resource.c +++ b/src/gallium/drivers/zink/zink_resource.c @@ -1098,25 +1098,26 @@ allocate_bo(struct zink_screen *screen, const struct pipe_resource *templ, assert(zink_mem_type_idx_from_bits(screen, heap, reqs->memoryTypeBits) != UINT32_MAX); } -retry: - /* iterate over all available memory types to reduce chance of oom */ - for (unsigned i = 0; !obj->bo && i < screen->heap_count[heap]; i++) { - if (!(reqs->memoryTypeBits & BITFIELD_BIT(screen->heap_map[heap][i]))) - continue; + while (1) { + /* iterate over all available memory types to reduce chance of oom */ + for (unsigned i = 0; !obj->bo && i < screen->heap_count[heap]; i++) { + if (!(reqs->memoryTypeBits & BITFIELD_BIT(screen->heap_map[heap][i]))) + continue; - mai.memoryTypeIndex = screen->heap_map[heap][i]; - obj->bo = zink_bo(zink_bo_create(screen, reqs->size, alignment, heap, mai.pNext ? ZINK_ALLOC_NO_SUBALLOC : 0, mai.memoryTypeIndex, mai.pNext)); - if (!obj->bo) { - if (heap == ZINK_HEAP_DEVICE_LOCAL_VISIBLE) { - /* demote BAR allocations to a different heap on failure to avoid oom */ - if (templ->flags & PIPE_RESOURCE_FLAG_MAP_COHERENT || templ->usage == PIPE_USAGE_DYNAMIC) - heap = ZINK_HEAP_HOST_VISIBLE_COHERENT; - else - heap = ZINK_HEAP_DEVICE_LOCAL; - goto retry; - } + mai.memoryTypeIndex = screen->heap_map[heap][i]; + obj->bo = zink_bo(zink_bo_create(screen, reqs->size, alignment, heap, mai.pNext ? ZINK_ALLOC_NO_SUBALLOC : 0, mai.memoryTypeIndex, mai.pNext)); } - } + + if (obj->bo || heap != ZINK_HEAP_DEVICE_LOCAL_VISIBLE) { + break; + + /* demote BAR allocations to a different heap on failure to avoid oom */ + if (templ->flags & PIPE_RESOURCE_FLAG_MAP_COHERENT || templ->usage == PIPE_USAGE_DYNAMIC) + heap = ZINK_HEAP_HOST_VISIBLE_COHERENT; + else + heap = ZINK_HEAP_DEVICE_LOCAL; + } + }; return obj->bo ? 0: -2; } diff --git a/src/gallium/drivers/zink/zink_synchronization.cpp b/src/gallium/drivers/zink/zink_synchronization.cpp index 40eef00567e..05a904f3231 100644 --- a/src/gallium/drivers/zink/zink_synchronization.cpp +++ b/src/gallium/drivers/zink/zink_synchronization.cpp @@ -162,6 +162,10 @@ zink_resource_access_is_write(VkAccessFlags flags) bool zink_resource_image_needs_barrier(struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) { + if (!pipeline) + pipeline = pipeline_dst_stage(new_layout); + if (!flags) + flags = access_dst_flags(new_layout); return res->layout != new_layout || (res->obj->access_stage & pipeline) != pipeline || (res->obj->access & flags) != flags || zink_resource_access_is_write(res->obj->access) || @@ -258,36 +262,29 @@ unordered_res_exec(const struct zink_context *ctx, const struct zink_resource *r return res->obj->unordered_write || !zink_batch_usage_matches(res->obj->bo->writes.u, ctx->batch.state); } -static inline bool -check_unordered_exec(struct zink_context *ctx, struct zink_resource *res, bool is_write) -{ - if (res) { - if (!res->obj->is_buffer) { - /* TODO: figure out how to link up unordered layout -> ordered layout and delete this conditionals */ - if (zink_resource_usage_is_unflushed(res) && !res->obj->unordered_read && !res->obj->unordered_write) - return false; - } - return unordered_res_exec(ctx, res, is_write); - } - return true; -} - VkCommandBuffer zink_get_cmdbuf(struct zink_context *ctx, struct zink_resource *src, struct zink_resource *dst) { bool unordered_exec = (zink_debug & ZINK_DEBUG_NOREORDER) == 0; - - unordered_exec &= check_unordered_exec(ctx, src, false) && - check_unordered_exec(ctx, dst, true); - + /* TODO: figure out how to link up unordered layout -> ordered layout and delete these two conditionals */ + if (src && !src->obj->is_buffer) { + if (zink_resource_usage_is_unflushed(src) && !src->obj->unordered_read && !src->obj->unordered_write) + unordered_exec = false; + } + if (dst && !dst->obj->is_buffer) { + if (zink_resource_usage_is_unflushed(dst) && !dst->obj->unordered_read && !dst->obj->unordered_write) + unordered_exec = false; + } + if (src && unordered_exec) + unordered_exec &= unordered_res_exec(ctx, src, false); + if (dst && unordered_exec) + unordered_exec &= unordered_res_exec(ctx, dst, true); if (src) src->obj->unordered_read = unordered_exec; if (dst) dst->obj->unordered_write = unordered_exec; - if (!unordered_exec || ctx->unordered_blitting) zink_batch_no_rp(ctx); - if (unordered_exec) { ctx->batch.state->has_barriers = true; ctx->batch.has_work = true; @@ -323,201 +320,7 @@ resource_check_defer_image_barrier(struct zink_context *ctx, struct zink_resourc _mesa_set_add(ctx->need_barriers[is_compute], res); } -enum barrier_type { - barrier_default, - barrier_KHR_synchronzation2 -}; - -template -struct emit_memory_barrier { - static void for_image(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, - VkAccessFlags flags, VkPipelineStageFlags pipeline, bool completed, VkCommandBuffer cmdbuf, - bool *queue_import) { - VkImageMemoryBarrier imb; - zink_resource_image_barrier_init(&imb, res, new_layout, flags, pipeline); - if (!res->obj->access_stage || completed) - imb.srcAccessMask = 0; - if (res->obj->needs_zs_evaluate) - imb.pNext = &res->obj->zs_evaluate; - res->obj->needs_zs_evaluate = false; - if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) { - imb.srcQueueFamilyIndex = res->queue; - imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue; - res->queue = VK_QUEUE_FAMILY_IGNORED; - *queue_import = true; - } - VKCTX(CmdPipelineBarrier)( - cmdbuf, - res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, - pipeline, - 0, - 0, NULL, - 0, NULL, - 1, &imb - ); - } - - static void for_buffer(struct zink_context *ctx, struct zink_resource *res, - VkPipelineStageFlags pipeline, - VkAccessFlags flags, - bool unordered, - bool usage_matches, - VkPipelineStageFlags stages, - VkCommandBuffer cmdbuf) { - VkMemoryBarrier bmb; - bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; - bmb.pNext = NULL; - if (unordered) { - stages = usage_matches ? res->obj->unordered_access_stage : stages; - bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access; - } else { - bmb.srcAccessMask = res->obj->access; - } - VKCTX(CmdPipelineBarrier)( - cmdbuf, - stages, - pipeline, - 0, - 1, &bmb, - 0, NULL, - 0, NULL); - } -}; - - -template <> -struct emit_memory_barrier { - static void for_image(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, - VkAccessFlags flags, VkPipelineStageFlags pipeline, bool completed, VkCommandBuffer cmdbuf, - bool *queue_import) { - VkImageMemoryBarrier2 imb; - zink_resource_image_barrier2_init(&imb, res, new_layout, flags, pipeline); - if (!res->obj->access_stage || completed) - imb.srcAccessMask = 0; - if (res->obj->needs_zs_evaluate) - imb.pNext = &res->obj->zs_evaluate; - res->obj->needs_zs_evaluate = false; - if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) { - imb.srcQueueFamilyIndex = res->queue; - imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue; - res->queue = VK_QUEUE_FAMILY_IGNORED; - *queue_import = true; - } - VkDependencyInfo dep = { - VK_STRUCTURE_TYPE_DEPENDENCY_INFO, - NULL, - 0, - 0, - NULL, - 0, - NULL, - 1, - &imb - }; - VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep); - } - - static void for_buffer(struct zink_context *ctx, struct zink_resource *res, - VkPipelineStageFlags pipeline, - VkAccessFlags flags, - bool unordered, - bool usage_matches, - VkPipelineStageFlags stages, - VkCommandBuffer cmdbuf) { - VkMemoryBarrier2 bmb; - bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2; - bmb.pNext = NULL; - if (unordered) { - bmb.srcStageMask = usage_matches ? res->obj->unordered_access_stage : stages; - bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access; - } else { - bmb.srcStageMask = stages; - bmb.srcAccessMask = res->obj->access; - } - bmb.dstStageMask = pipeline; - bmb.dstAccessMask = flags; - VkDependencyInfo dep = { - VK_STRUCTURE_TYPE_DEPENDENCY_INFO, - NULL, - 0, - 1, - &bmb, - 0, - NULL, - 0, - NULL - }; - VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep); - } -}; - -template -struct update_unordered_access_and_get_cmdbuf -{ - /* use base template to make the cases for true and false more explicite below */ -}; - -template <> -struct update_unordered_access_and_get_cmdbuf { - static VkCommandBuffer apply(struct zink_context *ctx, struct zink_resource *res, bool usage_matches, bool is_write) { - assert(!usage_matches); - res->obj->unordered_write = true; - res->obj->unordered_read = true; - ctx->batch.state->has_unsync = true; - return ctx->batch.state->unsynchronized_cmdbuf; - } -}; - -template <> -struct update_unordered_access_and_get_cmdbuf { - static VkCommandBuffer apply(struct zink_context *ctx, struct zink_resource *res, bool usage_matches, bool is_write) { - VkCommandBuffer cmdbuf; - if (!usage_matches) { - res->obj->unordered_write = true; - if (is_write || zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, ZINK_RESOURCE_ACCESS_RW)) - res->obj->unordered_read = true; - } - if (zink_resource_usage_matches(res, ctx->batch.state) && !ctx->unordered_blitting && - /* if current batch usage exists with ordered non-transfer access, never promote - * this avoids layout dsync - */ - (!res->obj->unordered_read || !res->obj->unordered_write)) { - cmdbuf = ctx->batch.state->cmdbuf; - res->obj->unordered_write = false; - res->obj->unordered_read = false; - /* it's impossible to detect this from the caller - * there should be no valid case where this barrier can occur inside a renderpass - */ - zink_batch_no_rp(ctx); - } else { - cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL); - /* force subsequent barriers to be ordered to avoid layout desync */ - if (cmdbuf != ctx->batch.state->reordered_cmdbuf) { - res->obj->unordered_write = false; - res->obj->unordered_read = false; - } - } - return cmdbuf; - } -}; - -template -struct check_defer_image_barrier { - static void apply(UNUSED struct zink_context *ctx, UNUSED struct zink_resource *res, UNUSED VkImageLayout new_layout, - UNUSED VkPipelineStageFlags pipeline) { - } -}; - - -template <> -struct check_defer_image_barrier { - static void apply(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, - VkPipelineStageFlags pipeline) { - resource_check_defer_image_barrier(ctx, res, new_layout, pipeline); - } -}; - -template +template void zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) { @@ -533,12 +336,92 @@ zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res, enum zink_resource_access rw = is_write ? ZINK_RESOURCE_ACCESS_RW : ZINK_RESOURCE_ACCESS_WRITE; bool completed = zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, rw); bool usage_matches = !completed && zink_resource_usage_matches(res, ctx->batch.state); - VkCommandBuffer cmdbuf = update_unordered_access_and_get_cmdbuf::apply(ctx, res, usage_matches, is_write); - + VkCommandBuffer cmdbuf; + if (!usage_matches) { + res->obj->unordered_write = true; + if (is_write || zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, ZINK_RESOURCE_ACCESS_RW)) + res->obj->unordered_read = true; + } else { + assert(!UNSYNCHRONIZED); + } + if (UNSYNCHRONIZED) { + cmdbuf = ctx->batch.state->unsynchronized_cmdbuf; + res->obj->unordered_write = true; + res->obj->unordered_read = true; + ctx->batch.state->has_unsync = true; + } else if (zink_resource_usage_matches(res, ctx->batch.state) && !ctx->unordered_blitting && + /* if current batch usage exists with ordered non-transfer access, never promote + * this avoids layout dsync + */ + (!res->obj->unordered_read || !res->obj->unordered_write)) { + cmdbuf = ctx->batch.state->cmdbuf; + res->obj->unordered_write = false; + res->obj->unordered_read = false; + /* it's impossible to detect this from the caller + * there should be no valid case where this barrier can occur inside a renderpass + */ + zink_batch_no_rp(ctx); + } else { + cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL); + /* force subsequent barriers to be ordered to avoid layout desync */ + if (cmdbuf != ctx->batch.state->reordered_cmdbuf) { + res->obj->unordered_write = false; + res->obj->unordered_read = false; + } + } assert(new_layout); bool marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "image_barrier(%s->%s)", vk_ImageLayout_to_str(res->layout), vk_ImageLayout_to_str(new_layout)); bool queue_import = false; - emit_memory_barrier::for_image(ctx, res, new_layout, flags, pipeline, completed, cmdbuf, &queue_import); + if (HAS_SYNC2) { + VkImageMemoryBarrier2 imb; + zink_resource_image_barrier2_init(&imb, res, new_layout, flags, pipeline); + if (!res->obj->access_stage || completed) + imb.srcAccessMask = 0; + if (res->obj->needs_zs_evaluate) + imb.pNext = &res->obj->zs_evaluate; + res->obj->needs_zs_evaluate = false; + if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) { + imb.srcQueueFamilyIndex = res->queue; + imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue; + res->queue = VK_QUEUE_FAMILY_IGNORED; + queue_import = true; + } + VkDependencyInfo dep = { + VK_STRUCTURE_TYPE_DEPENDENCY_INFO, + NULL, + 0, + 0, + NULL, + 0, + NULL, + 1, + &imb + }; + VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep); + } else { + VkImageMemoryBarrier imb; + zink_resource_image_barrier_init(&imb, res, new_layout, flags, pipeline); + if (!res->obj->access_stage || completed) + imb.srcAccessMask = 0; + if (res->obj->needs_zs_evaluate) + imb.pNext = &res->obj->zs_evaluate; + res->obj->needs_zs_evaluate = false; + if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) { + imb.srcQueueFamilyIndex = res->queue; + imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue; + res->queue = VK_QUEUE_FAMILY_IGNORED; + queue_import = true; + } + VKCTX(CmdPipelineBarrier)( + cmdbuf, + res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + pipeline, + 0, + 0, NULL, + 0, NULL, + 1, &imb + ); + } zink_cmd_debug_marker_end(ctx, cmdbuf, marker); if (!UNSYNCHRONIZED) @@ -550,10 +433,6 @@ zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res, res->obj->access = flags; res->obj->access_stage = pipeline; res->layout = new_layout; - - if (new_layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) - zink_resource_copies_reset(res); - if (res->obj->exportable) simple_mtx_lock(&ctx->batch.state->exportable_lock); if (res->obj->dt) { @@ -569,6 +448,8 @@ zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res, pipe_resource_reference(&pres, &res->base.b); } } + if (new_layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) + zink_resource_copies_reset(res); if (res->obj->exportable && queue_import) { for (struct zink_resource *r = res; r; r = zink_resource(r->base.b.next)) { VkSemaphore sem = zink_screen_export_dmabuf_semaphore(zink_screen(ctx->base.screen), r); @@ -694,9 +575,7 @@ buffer_needs_barrier(struct zink_resource *res, VkAccessFlags flags, VkPipelineS ((unordered ? res->obj->unordered_access : res->obj->access) & flags) != flags; } - - -template +template void zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline) { @@ -759,7 +638,51 @@ zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_resource *res } VkPipelineStageFlags stages = res->obj->access_stage ? res->obj->access_stage : pipeline_access_stage(res->obj->access);; - emit_memory_barrier::for_buffer(ctx, res, pipeline, flags, unordered,usage_matches, stages, cmdbuf); + if (HAS_SYNC2) { + VkMemoryBarrier2 bmb; + bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2; + bmb.pNext = NULL; + if (unordered) { + bmb.srcStageMask = usage_matches ? res->obj->unordered_access_stage : stages; + bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access; + } else { + bmb.srcStageMask = stages; + bmb.srcAccessMask = res->obj->access; + } + bmb.dstStageMask = pipeline; + bmb.dstAccessMask = flags; + VkDependencyInfo dep = { + VK_STRUCTURE_TYPE_DEPENDENCY_INFO, + NULL, + 0, + 1, + &bmb, + 0, + NULL, + 0, + NULL + }; + VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep); + } else { + VkMemoryBarrier bmb; + bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; + bmb.pNext = NULL; + if (unordered) { + stages = usage_matches ? res->obj->unordered_access_stage : stages; + bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access; + } else { + bmb.srcAccessMask = res->obj->access; + } + VKCTX(CmdPipelineBarrier)( + cmdbuf, + stages, + pipeline, + 0, + 1, &bmb, + 0, NULL, + 0, NULL + ); + } zink_cmd_debug_marker_end(ctx, cmdbuf, marker); } @@ -790,12 +713,12 @@ void zink_synchronization_init(struct zink_screen *screen) { if (screen->info.have_vulkan13 || screen->info.have_KHR_synchronization2) { - screen->buffer_barrier = zink_resource_buffer_barrier; - screen->image_barrier = zink_resource_image_barrier; - screen->image_barrier_unsync = zink_resource_image_barrier; + screen->buffer_barrier = zink_resource_buffer_barrier; + screen->image_barrier = zink_resource_image_barrier; + screen->image_barrier_unsync = zink_resource_image_barrier; } else { - screen->buffer_barrier = zink_resource_buffer_barrier; - screen->image_barrier = zink_resource_image_barrier; - screen->image_barrier_unsync = zink_resource_image_barrier; + screen->buffer_barrier = zink_resource_buffer_barrier; + screen->image_barrier = zink_resource_image_barrier; + screen->image_barrier_unsync = zink_resource_image_barrier; } }