From c53a4711cb77fdf19b93797106b2ddf846c32d37 Mon Sep 17 00:00:00 2001 From: Lionel Landwerlin Date: Wed, 15 Nov 2023 09:33:46 +0200 Subject: [PATCH] anv: fix incorrect flushing on shader query copy When doing query result copies in 3D mode, we're flushing the render target cache, but the shader writes go through the dataport. Fixes flakes/fails in piglit with shader query copies forced with Zink : $ query_copy_with_shader_threshold=0 ./bin/arb_query_buffer_object-coherency -auto -fbo Signed-off-by: Lionel Landwerlin Fixes: b3b12c2c27 ("anv: enable CmdCopyQueryPoolResults to use shader for copies") Reviewed-by: Kenneth Graunke Part-of: --- src/intel/vulkan/genX_cmd_buffer.c | 36 ++++++++++++++++++++++++++++++ src/intel/vulkan/genX_query.c | 10 ++++----- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/intel/vulkan/genX_cmd_buffer.c b/src/intel/vulkan/genX_cmd_buffer.c index 90b8d0bdda5..6932ebe3e0e 100644 --- a/src/intel/vulkan/genX_cmd_buffer.c +++ b/src/intel/vulkan/genX_cmd_buffer.c @@ -3284,6 +3284,13 @@ mask_is_write(const VkAccessFlags2 access) VK_ACCESS_2_OPTICAL_FLOW_WRITE_BIT_NV); } +static inline bool +mask_is_transfer_write(const VkAccessFlags2 access) +{ + return access & (VK_ACCESS_2_TRANSFER_WRITE_BIT | + VK_ACCESS_2_MEMORY_WRITE_BIT); +} + static void cmd_buffer_barrier_video(struct anv_cmd_buffer *cmd_buffer, const VkDependencyInfo *dep_info) @@ -3447,6 +3454,16 @@ cmd_buffer_barrier_blitter(struct anv_cmd_buffer *cmd_buffer, #endif } +static inline bool +cmd_buffer_has_pending_copy_query(struct anv_cmd_buffer *cmd_buffer) +{ + /* Query copies are only written with dataport, so we only need to check + * that flag. + */ + return (cmd_buffer->state.queries.buffer_write_bits & + ANV_QUERY_WRITES_DATA_FLUSH) != 0; +} + static void cmd_buffer_barrier(struct anv_cmd_buffer *cmd_buffer, const VkDependencyInfo *dep_info, @@ -3472,6 +3489,7 @@ cmd_buffer_barrier(struct anv_cmd_buffer *cmd_buffer, VkAccessFlags2 dst_flags = 0; bool apply_sparse_flushes = false; + bool flush_query_copies = false; for (uint32_t i = 0; i < dep_info->memoryBarrierCount; i++) { src_flags |= dep_info->pMemoryBarriers[i].srcAccessMask; @@ -3487,6 +3505,11 @@ cmd_buffer_barrier(struct anv_cmd_buffer *cmd_buffer, ANV_QUERY_COMPUTE_WRITES_PENDING_BITS; } + if (stage_is_transfer(dep_info->pMemoryBarriers[i].srcStageMask) && + mask_is_transfer_write(dep_info->pMemoryBarriers[i].srcAccessMask) && + cmd_buffer_has_pending_copy_query(cmd_buffer)) + flush_query_copies = true; + /* There's no way of knowing if this memory barrier is related to sparse * buffers! This is pretty horrible. */ @@ -3512,6 +3535,11 @@ cmd_buffer_barrier(struct anv_cmd_buffer *cmd_buffer, ANV_QUERY_COMPUTE_WRITES_PENDING_BITS; } + if (stage_is_transfer(buf_barrier->srcStageMask) && + mask_is_transfer_write(buf_barrier->srcAccessMask) && + cmd_buffer_has_pending_copy_query(cmd_buffer)) + flush_query_copies = true; + if (anv_buffer_is_sparse(buffer) && mask_is_write(src_flags)) apply_sparse_flushes = true; } @@ -3627,6 +3655,14 @@ cmd_buffer_barrier(struct anv_cmd_buffer *cmd_buffer, if (apply_sparse_flushes) bits |= ANV_PIPE_FLUSH_BITS; + /* Copies from query pools are executed with a shader writing through the + * dataport. + */ + if (flush_query_copies) { + bits |= (GFX_VER >= 12 ? + ANV_PIPE_HDC_PIPELINE_FLUSH_BIT : ANV_PIPE_DATA_CACHE_FLUSH_BIT); + } + if (dst_flags & VK_ACCESS_INDIRECT_COMMAND_READ_BIT) genX(cmd_buffer_flush_generated_draws)(cmd_buffer); diff --git a/src/intel/vulkan/genX_query.c b/src/intel/vulkan/genX_query.c index d97dff3d101..92ed95fe697 100644 --- a/src/intel/vulkan/genX_query.c +++ b/src/intel/vulkan/genX_query.c @@ -1817,11 +1817,11 @@ copy_query_results_with_shader(struct anv_cmd_buffer *cmd_buffer, genX(emit_simple_shader_dispatch)(&state, query_count, push_data_state); - anv_add_pending_pipe_bits(cmd_buffer, - cmd_buffer->state.current_pipeline == GPGPU ? - ANV_QUERY_COMPUTE_WRITES_PENDING_BITS : - ANV_QUERY_RENDER_TARGET_WRITES_PENDING_BITS(device->info), - "after query copy results"); + /* The query copy result shader is writing using the dataport, flush + * HDC/Data cache depending on the generation. Also stall at pixel + * scoreboard in case we're doing the copy with a fragment shader. + */ + cmd_buffer->state.queries.buffer_write_bits |= ANV_QUERY_WRITES_DATA_FLUSH; trace_intel_end_query_copy_shader(&cmd_buffer->trace, query_count); }