diff --git a/src/gallium/drivers/zink/zink_resource.c b/src/gallium/drivers/zink/zink_resource.c index 0260fb8fe41..773c269e408 100644 --- a/src/gallium/drivers/zink/zink_resource.c +++ b/src/gallium/drivers/zink/zink_resource.c @@ -1098,25 +1098,26 @@ allocate_bo(struct zink_screen *screen, const struct pipe_resource *templ, assert(zink_mem_type_idx_from_bits(screen, heap, reqs->memoryTypeBits) != UINT32_MAX); } -retry: - /* iterate over all available memory types to reduce chance of oom */ - for (unsigned i = 0; !obj->bo && i < screen->heap_count[heap]; i++) { - if (!(reqs->memoryTypeBits & BITFIELD_BIT(screen->heap_map[heap][i]))) - continue; + while (1) { + /* iterate over all available memory types to reduce chance of oom */ + for (unsigned i = 0; !obj->bo && i < screen->heap_count[heap]; i++) { + if (!(reqs->memoryTypeBits & BITFIELD_BIT(screen->heap_map[heap][i]))) + continue; - mai.memoryTypeIndex = screen->heap_map[heap][i]; - obj->bo = zink_bo(zink_bo_create(screen, reqs->size, alignment, heap, mai.pNext ? ZINK_ALLOC_NO_SUBALLOC : 0, mai.memoryTypeIndex, mai.pNext)); - if (!obj->bo) { - if (heap == ZINK_HEAP_DEVICE_LOCAL_VISIBLE) { - /* demote BAR allocations to a different heap on failure to avoid oom */ - if (templ->flags & PIPE_RESOURCE_FLAG_MAP_COHERENT || templ->usage == PIPE_USAGE_DYNAMIC) - heap = ZINK_HEAP_HOST_VISIBLE_COHERENT; - else - heap = ZINK_HEAP_DEVICE_LOCAL; - goto retry; - } + mai.memoryTypeIndex = screen->heap_map[heap][i]; + obj->bo = zink_bo(zink_bo_create(screen, reqs->size, alignment, heap, mai.pNext ? ZINK_ALLOC_NO_SUBALLOC : 0, mai.memoryTypeIndex, mai.pNext)); } - } + + if (obj->bo || heap != ZINK_HEAP_DEVICE_LOCAL_VISIBLE) { + break; + + /* demote BAR allocations to a different heap on failure to avoid oom */ + if (templ->flags & PIPE_RESOURCE_FLAG_MAP_COHERENT || templ->usage == PIPE_USAGE_DYNAMIC) + heap = ZINK_HEAP_HOST_VISIBLE_COHERENT; + else + heap = ZINK_HEAP_DEVICE_LOCAL; + } + }; return obj->bo ? 0: -2; } diff --git a/src/gallium/drivers/zink/zink_synchronization.cpp b/src/gallium/drivers/zink/zink_synchronization.cpp index 40eef00567e..05a904f3231 100644 --- a/src/gallium/drivers/zink/zink_synchronization.cpp +++ b/src/gallium/drivers/zink/zink_synchronization.cpp @@ -162,6 +162,10 @@ zink_resource_access_is_write(VkAccessFlags flags) bool zink_resource_image_needs_barrier(struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) { + if (!pipeline) + pipeline = pipeline_dst_stage(new_layout); + if (!flags) + flags = access_dst_flags(new_layout); return res->layout != new_layout || (res->obj->access_stage & pipeline) != pipeline || (res->obj->access & flags) != flags || zink_resource_access_is_write(res->obj->access) || @@ -258,36 +262,29 @@ unordered_res_exec(const struct zink_context *ctx, const struct zink_resource *r return res->obj->unordered_write || !zink_batch_usage_matches(res->obj->bo->writes.u, ctx->batch.state); } -static inline bool -check_unordered_exec(struct zink_context *ctx, struct zink_resource *res, bool is_write) -{ - if (res) { - if (!res->obj->is_buffer) { - /* TODO: figure out how to link up unordered layout -> ordered layout and delete this conditionals */ - if (zink_resource_usage_is_unflushed(res) && !res->obj->unordered_read && !res->obj->unordered_write) - return false; - } - return unordered_res_exec(ctx, res, is_write); - } - return true; -} - VkCommandBuffer zink_get_cmdbuf(struct zink_context *ctx, struct zink_resource *src, struct zink_resource *dst) { bool unordered_exec = (zink_debug & ZINK_DEBUG_NOREORDER) == 0; - - unordered_exec &= check_unordered_exec(ctx, src, false) && - check_unordered_exec(ctx, dst, true); - + /* TODO: figure out how to link up unordered layout -> ordered layout and delete these two conditionals */ + if (src && !src->obj->is_buffer) { + if (zink_resource_usage_is_unflushed(src) && !src->obj->unordered_read && !src->obj->unordered_write) + unordered_exec = false; + } + if (dst && !dst->obj->is_buffer) { + if (zink_resource_usage_is_unflushed(dst) && !dst->obj->unordered_read && !dst->obj->unordered_write) + unordered_exec = false; + } + if (src && unordered_exec) + unordered_exec &= unordered_res_exec(ctx, src, false); + if (dst && unordered_exec) + unordered_exec &= unordered_res_exec(ctx, dst, true); if (src) src->obj->unordered_read = unordered_exec; if (dst) dst->obj->unordered_write = unordered_exec; - if (!unordered_exec || ctx->unordered_blitting) zink_batch_no_rp(ctx); - if (unordered_exec) { ctx->batch.state->has_barriers = true; ctx->batch.has_work = true; @@ -323,201 +320,7 @@ resource_check_defer_image_barrier(struct zink_context *ctx, struct zink_resourc _mesa_set_add(ctx->need_barriers[is_compute], res); } -enum barrier_type { - barrier_default, - barrier_KHR_synchronzation2 -}; - -template -struct emit_memory_barrier { - static void for_image(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, - VkAccessFlags flags, VkPipelineStageFlags pipeline, bool completed, VkCommandBuffer cmdbuf, - bool *queue_import) { - VkImageMemoryBarrier imb; - zink_resource_image_barrier_init(&imb, res, new_layout, flags, pipeline); - if (!res->obj->access_stage || completed) - imb.srcAccessMask = 0; - if (res->obj->needs_zs_evaluate) - imb.pNext = &res->obj->zs_evaluate; - res->obj->needs_zs_evaluate = false; - if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) { - imb.srcQueueFamilyIndex = res->queue; - imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue; - res->queue = VK_QUEUE_FAMILY_IGNORED; - *queue_import = true; - } - VKCTX(CmdPipelineBarrier)( - cmdbuf, - res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, - pipeline, - 0, - 0, NULL, - 0, NULL, - 1, &imb - ); - } - - static void for_buffer(struct zink_context *ctx, struct zink_resource *res, - VkPipelineStageFlags pipeline, - VkAccessFlags flags, - bool unordered, - bool usage_matches, - VkPipelineStageFlags stages, - VkCommandBuffer cmdbuf) { - VkMemoryBarrier bmb; - bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; - bmb.pNext = NULL; - if (unordered) { - stages = usage_matches ? res->obj->unordered_access_stage : stages; - bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access; - } else { - bmb.srcAccessMask = res->obj->access; - } - VKCTX(CmdPipelineBarrier)( - cmdbuf, - stages, - pipeline, - 0, - 1, &bmb, - 0, NULL, - 0, NULL); - } -}; - - -template <> -struct emit_memory_barrier { - static void for_image(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, - VkAccessFlags flags, VkPipelineStageFlags pipeline, bool completed, VkCommandBuffer cmdbuf, - bool *queue_import) { - VkImageMemoryBarrier2 imb; - zink_resource_image_barrier2_init(&imb, res, new_layout, flags, pipeline); - if (!res->obj->access_stage || completed) - imb.srcAccessMask = 0; - if (res->obj->needs_zs_evaluate) - imb.pNext = &res->obj->zs_evaluate; - res->obj->needs_zs_evaluate = false; - if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) { - imb.srcQueueFamilyIndex = res->queue; - imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue; - res->queue = VK_QUEUE_FAMILY_IGNORED; - *queue_import = true; - } - VkDependencyInfo dep = { - VK_STRUCTURE_TYPE_DEPENDENCY_INFO, - NULL, - 0, - 0, - NULL, - 0, - NULL, - 1, - &imb - }; - VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep); - } - - static void for_buffer(struct zink_context *ctx, struct zink_resource *res, - VkPipelineStageFlags pipeline, - VkAccessFlags flags, - bool unordered, - bool usage_matches, - VkPipelineStageFlags stages, - VkCommandBuffer cmdbuf) { - VkMemoryBarrier2 bmb; - bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2; - bmb.pNext = NULL; - if (unordered) { - bmb.srcStageMask = usage_matches ? res->obj->unordered_access_stage : stages; - bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access; - } else { - bmb.srcStageMask = stages; - bmb.srcAccessMask = res->obj->access; - } - bmb.dstStageMask = pipeline; - bmb.dstAccessMask = flags; - VkDependencyInfo dep = { - VK_STRUCTURE_TYPE_DEPENDENCY_INFO, - NULL, - 0, - 1, - &bmb, - 0, - NULL, - 0, - NULL - }; - VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep); - } -}; - -template -struct update_unordered_access_and_get_cmdbuf -{ - /* use base template to make the cases for true and false more explicite below */ -}; - -template <> -struct update_unordered_access_and_get_cmdbuf { - static VkCommandBuffer apply(struct zink_context *ctx, struct zink_resource *res, bool usage_matches, bool is_write) { - assert(!usage_matches); - res->obj->unordered_write = true; - res->obj->unordered_read = true; - ctx->batch.state->has_unsync = true; - return ctx->batch.state->unsynchronized_cmdbuf; - } -}; - -template <> -struct update_unordered_access_and_get_cmdbuf { - static VkCommandBuffer apply(struct zink_context *ctx, struct zink_resource *res, bool usage_matches, bool is_write) { - VkCommandBuffer cmdbuf; - if (!usage_matches) { - res->obj->unordered_write = true; - if (is_write || zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, ZINK_RESOURCE_ACCESS_RW)) - res->obj->unordered_read = true; - } - if (zink_resource_usage_matches(res, ctx->batch.state) && !ctx->unordered_blitting && - /* if current batch usage exists with ordered non-transfer access, never promote - * this avoids layout dsync - */ - (!res->obj->unordered_read || !res->obj->unordered_write)) { - cmdbuf = ctx->batch.state->cmdbuf; - res->obj->unordered_write = false; - res->obj->unordered_read = false; - /* it's impossible to detect this from the caller - * there should be no valid case where this barrier can occur inside a renderpass - */ - zink_batch_no_rp(ctx); - } else { - cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL); - /* force subsequent barriers to be ordered to avoid layout desync */ - if (cmdbuf != ctx->batch.state->reordered_cmdbuf) { - res->obj->unordered_write = false; - res->obj->unordered_read = false; - } - } - return cmdbuf; - } -}; - -template -struct check_defer_image_barrier { - static void apply(UNUSED struct zink_context *ctx, UNUSED struct zink_resource *res, UNUSED VkImageLayout new_layout, - UNUSED VkPipelineStageFlags pipeline) { - } -}; - - -template <> -struct check_defer_image_barrier { - static void apply(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, - VkPipelineStageFlags pipeline) { - resource_check_defer_image_barrier(ctx, res, new_layout, pipeline); - } -}; - -template +template void zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) { @@ -533,12 +336,92 @@ zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res, enum zink_resource_access rw = is_write ? ZINK_RESOURCE_ACCESS_RW : ZINK_RESOURCE_ACCESS_WRITE; bool completed = zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, rw); bool usage_matches = !completed && zink_resource_usage_matches(res, ctx->batch.state); - VkCommandBuffer cmdbuf = update_unordered_access_and_get_cmdbuf::apply(ctx, res, usage_matches, is_write); - + VkCommandBuffer cmdbuf; + if (!usage_matches) { + res->obj->unordered_write = true; + if (is_write || zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, ZINK_RESOURCE_ACCESS_RW)) + res->obj->unordered_read = true; + } else { + assert(!UNSYNCHRONIZED); + } + if (UNSYNCHRONIZED) { + cmdbuf = ctx->batch.state->unsynchronized_cmdbuf; + res->obj->unordered_write = true; + res->obj->unordered_read = true; + ctx->batch.state->has_unsync = true; + } else if (zink_resource_usage_matches(res, ctx->batch.state) && !ctx->unordered_blitting && + /* if current batch usage exists with ordered non-transfer access, never promote + * this avoids layout dsync + */ + (!res->obj->unordered_read || !res->obj->unordered_write)) { + cmdbuf = ctx->batch.state->cmdbuf; + res->obj->unordered_write = false; + res->obj->unordered_read = false; + /* it's impossible to detect this from the caller + * there should be no valid case where this barrier can occur inside a renderpass + */ + zink_batch_no_rp(ctx); + } else { + cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL); + /* force subsequent barriers to be ordered to avoid layout desync */ + if (cmdbuf != ctx->batch.state->reordered_cmdbuf) { + res->obj->unordered_write = false; + res->obj->unordered_read = false; + } + } assert(new_layout); bool marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "image_barrier(%s->%s)", vk_ImageLayout_to_str(res->layout), vk_ImageLayout_to_str(new_layout)); bool queue_import = false; - emit_memory_barrier::for_image(ctx, res, new_layout, flags, pipeline, completed, cmdbuf, &queue_import); + if (HAS_SYNC2) { + VkImageMemoryBarrier2 imb; + zink_resource_image_barrier2_init(&imb, res, new_layout, flags, pipeline); + if (!res->obj->access_stage || completed) + imb.srcAccessMask = 0; + if (res->obj->needs_zs_evaluate) + imb.pNext = &res->obj->zs_evaluate; + res->obj->needs_zs_evaluate = false; + if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) { + imb.srcQueueFamilyIndex = res->queue; + imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue; + res->queue = VK_QUEUE_FAMILY_IGNORED; + queue_import = true; + } + VkDependencyInfo dep = { + VK_STRUCTURE_TYPE_DEPENDENCY_INFO, + NULL, + 0, + 0, + NULL, + 0, + NULL, + 1, + &imb + }; + VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep); + } else { + VkImageMemoryBarrier imb; + zink_resource_image_barrier_init(&imb, res, new_layout, flags, pipeline); + if (!res->obj->access_stage || completed) + imb.srcAccessMask = 0; + if (res->obj->needs_zs_evaluate) + imb.pNext = &res->obj->zs_evaluate; + res->obj->needs_zs_evaluate = false; + if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) { + imb.srcQueueFamilyIndex = res->queue; + imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue; + res->queue = VK_QUEUE_FAMILY_IGNORED; + queue_import = true; + } + VKCTX(CmdPipelineBarrier)( + cmdbuf, + res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + pipeline, + 0, + 0, NULL, + 0, NULL, + 1, &imb + ); + } zink_cmd_debug_marker_end(ctx, cmdbuf, marker); if (!UNSYNCHRONIZED) @@ -550,10 +433,6 @@ zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res, res->obj->access = flags; res->obj->access_stage = pipeline; res->layout = new_layout; - - if (new_layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) - zink_resource_copies_reset(res); - if (res->obj->exportable) simple_mtx_lock(&ctx->batch.state->exportable_lock); if (res->obj->dt) { @@ -569,6 +448,8 @@ zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res, pipe_resource_reference(&pres, &res->base.b); } } + if (new_layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) + zink_resource_copies_reset(res); if (res->obj->exportable && queue_import) { for (struct zink_resource *r = res; r; r = zink_resource(r->base.b.next)) { VkSemaphore sem = zink_screen_export_dmabuf_semaphore(zink_screen(ctx->base.screen), r); @@ -694,9 +575,7 @@ buffer_needs_barrier(struct zink_resource *res, VkAccessFlags flags, VkPipelineS ((unordered ? res->obj->unordered_access : res->obj->access) & flags) != flags; } - - -template +template void zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline) { @@ -759,7 +638,51 @@ zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_resource *res } VkPipelineStageFlags stages = res->obj->access_stage ? res->obj->access_stage : pipeline_access_stage(res->obj->access);; - emit_memory_barrier::for_buffer(ctx, res, pipeline, flags, unordered,usage_matches, stages, cmdbuf); + if (HAS_SYNC2) { + VkMemoryBarrier2 bmb; + bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2; + bmb.pNext = NULL; + if (unordered) { + bmb.srcStageMask = usage_matches ? res->obj->unordered_access_stage : stages; + bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access; + } else { + bmb.srcStageMask = stages; + bmb.srcAccessMask = res->obj->access; + } + bmb.dstStageMask = pipeline; + bmb.dstAccessMask = flags; + VkDependencyInfo dep = { + VK_STRUCTURE_TYPE_DEPENDENCY_INFO, + NULL, + 0, + 1, + &bmb, + 0, + NULL, + 0, + NULL + }; + VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep); + } else { + VkMemoryBarrier bmb; + bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; + bmb.pNext = NULL; + if (unordered) { + stages = usage_matches ? res->obj->unordered_access_stage : stages; + bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access; + } else { + bmb.srcAccessMask = res->obj->access; + } + VKCTX(CmdPipelineBarrier)( + cmdbuf, + stages, + pipeline, + 0, + 1, &bmb, + 0, NULL, + 0, NULL + ); + } zink_cmd_debug_marker_end(ctx, cmdbuf, marker); } @@ -790,12 +713,12 @@ void zink_synchronization_init(struct zink_screen *screen) { if (screen->info.have_vulkan13 || screen->info.have_KHR_synchronization2) { - screen->buffer_barrier = zink_resource_buffer_barrier; - screen->image_barrier = zink_resource_image_barrier; - screen->image_barrier_unsync = zink_resource_image_barrier; + screen->buffer_barrier = zink_resource_buffer_barrier; + screen->image_barrier = zink_resource_image_barrier; + screen->image_barrier_unsync = zink_resource_image_barrier; } else { - screen->buffer_barrier = zink_resource_buffer_barrier; - screen->image_barrier = zink_resource_image_barrier; - screen->image_barrier_unsync = zink_resource_image_barrier; + screen->buffer_barrier = zink_resource_buffer_barrier; + screen->image_barrier = zink_resource_image_barrier; + screen->image_barrier_unsync = zink_resource_image_barrier; } }