diff --git a/src/freedreno/vulkan/tu_clear_blit.cc b/src/freedreno/vulkan/tu_clear_blit.cc index 922dd2fd63d..1769db5a991 100644 --- a/src/freedreno/vulkan/tu_clear_blit.cc +++ b/src/freedreno/vulkan/tu_clear_blit.cc @@ -3178,6 +3178,8 @@ tu_emit_clear_gmem_attachment(struct tu_cmd_buffer *cmd, } } + tu_flush_for_access(&cmd->state.renderpass_cache, TU_ACCESS_BLIT_WRITE_GMEM, TU_ACCESS_NONE); + trace_end_gmem_clear(&cmd->trace, cs); } @@ -3463,6 +3465,8 @@ tu_emit_blit(struct tu_cmd_buffer *cmd, tu_emit_event_write(cmd, cs, FD_BLIT); } + + tu_flush_for_access(&cmd->state.cache, TU_ACCESS_BLIT_WRITE_GMEM, TU_ACCESS_NONE); } static bool diff --git a/src/freedreno/vulkan/tu_cmd_buffer.cc b/src/freedreno/vulkan/tu_cmd_buffer.cc index ee709438044..ab0e70cc6d8 100644 --- a/src/freedreno/vulkan/tu_cmd_buffer.cc +++ b/src/freedreno/vulkan/tu_cmd_buffer.cc @@ -198,6 +198,11 @@ tu6_emit_flushes(struct tu_cmd_buffer *cmd_buffer, .gfx_bindless = CHIP == A6XX ? 0x1f : 0xff, )); } + if (CHIP >= A7XX && flushes & TU_CMD_FLAG_BLIT_CACHE_FLUSH) + /* On A7XX, blit cache flushes are required to ensure blit writes are visible + * via UCHE. This isn't necessary on A6XX, all writes should be visible implictly. + */ + tu_emit_event_write(cmd_buffer, cs, FD_CCU_FLUSH_BLIT_CACHE); if (flushes & TU_CMD_FLAG_WAIT_MEM_WRITES) tu_cs_emit_pkt7(cs, CP_WAIT_MEM_WRITES, 0); if (flushes & TU_CMD_FLAG_WAIT_FOR_IDLE) @@ -3256,7 +3261,7 @@ tu_CmdBindPipeline(VkCommandBuffer commandBuffer, } } -static void +void tu_flush_for_access(struct tu_cache_state *cache, enum tu_cmd_access_mask src_mask, enum tu_cmd_access_mask dst_mask) @@ -3335,6 +3340,24 @@ tu_flush_for_access(struct tu_cache_state *cache, flush_bits |= TU_CMD_FLAG_BINDLESS_DESCRIPTOR_INVALIDATE; } + /* The blit cache is a special case dependency between CP_EVENT_WRITE::BLIT + * (from GMEM loads/clears) to any GMEM attachment reads done via the UCHE + * (Eg: Input attachments/CP_BLIT) which needs an explicit BLIT_CACHE_FLUSH + * for the event blit writes to land, it has the following properties: + * - Set on reads rather than on writes, like flushes. + * - Not executed automatically if pending, like invalidates. + * - Pending bits passed through to secondary command buffers, if they're + * continuing the render pass. + */ + if (src_mask & TU_ACCESS_BLIT_WRITE_GMEM) { + cache->pending_flush_bits |= TU_CMD_FLAG_BLIT_CACHE_FLUSH; + } + + if ((dst_mask & TU_ACCESS_UCHE_READ_GMEM) && + (cache->pending_flush_bits & TU_CMD_FLAG_BLIT_CACHE_FLUSH)) { + flush_bits |= TU_CMD_FLAG_BLIT_CACHE_FLUSH; + } + #undef DST_INCOHERENT_FLUSH cache->flush_bits |= flush_bits; @@ -3438,6 +3461,11 @@ vk2tu_access(VkAccessFlags2 flags, VkPipelineStageFlags2 stages, bool image_only SHADER_STAGES)) mask |= TU_ACCESS_UCHE_READ; + if (gfx_read_access(flags, stages, + VK_ACCESS_2_INPUT_ATTACHMENT_READ_BIT, + SHADER_STAGES)) + mask |= TU_ACCESS_UCHE_READ_GMEM; + if (gfx_read_access(flags, stages, VK_ACCESS_2_DESCRIPTOR_BUFFER_READ_BIT_EXT, SHADER_STAGES)) { @@ -3934,7 +3962,11 @@ tu_CmdExecuteCommands(VkCommandBuffer commandBuffer, * to re-initialize the cache with all pending invalidate bits set. */ if (cmd->state.pass) { - tu_cache_init(&cmd->state.renderpass_cache); + struct tu_cache_state *cache = &cmd->state.renderpass_cache; + BITMASK_ENUM(tu_cmd_flush_bits) retained_pending_flush_bits = + cache->pending_flush_bits & TU_CMD_FLAG_BLIT_CACHE_FLUSH; + tu_cache_init(cache); + cache->pending_flush_bits |= retained_pending_flush_bits; } else { tu_cache_init(&cmd->state.cache); } diff --git a/src/freedreno/vulkan/tu_cmd_buffer.h b/src/freedreno/vulkan/tu_cmd_buffer.h index 182b6037a6e..274239db7be 100644 --- a/src/freedreno/vulkan/tu_cmd_buffer.h +++ b/src/freedreno/vulkan/tu_cmd_buffer.h @@ -132,6 +132,12 @@ enum tu_cmd_access_mask { */ TU_ACCESS_BINDLESS_DESCRIPTOR_READ = 1 << 13, + /* A write to a GMEM attachment made by CP_EVENT_WRITE::BLIT. */ + TU_ACCESS_BLIT_WRITE_GMEM = 1 << 14, + + /* Similar to UCHE_READ, but specifically for GMEM attachment reads. */ + TU_ACCESS_UCHE_READ_GMEM = 1 << 15, + TU_ACCESS_READ = TU_ACCESS_UCHE_READ | TU_ACCESS_CCU_COLOR_READ | @@ -190,6 +196,10 @@ enum tu_cmd_flush_bits { TU_CMD_FLAG_WAIT_FOR_IDLE = 1 << 7, TU_CMD_FLAG_WAIT_FOR_ME = 1 << 8, TU_CMD_FLAG_BINDLESS_DESCRIPTOR_INVALIDATE = 1 << 9, + /* This is an unusual flush that isn't automatically executed if pending, + * as it isn't necessary. Therefore, it's not included in ALL_FLUSH. + */ + TU_CMD_FLAG_BLIT_CACHE_FLUSH = 1 << 10, TU_CMD_FLAG_ALL_FLUSH = TU_CMD_FLAG_CCU_FLUSH_DEPTH | @@ -650,6 +660,11 @@ tu_emit_event_write(struct tu_cmd_buffer *cmd, struct tu_cs *cs, enum fd_gpu_event event); +void +tu_flush_for_access(struct tu_cache_state *cache, + enum tu_cmd_access_mask src_mask, + enum tu_cmd_access_mask dst_mask); + static inline struct tu_descriptor_state * tu_get_descriptors_state(struct tu_cmd_buffer *cmd_buffer, VkPipelineBindPoint bind_point)