diff --git a/src/amd/vulkan/meta/radv_meta_buffer.c b/src/amd/vulkan/meta/radv_meta_buffer.c index 25473ff1026..65c0b4502ec 100644 --- a/src/amd/vulkan/meta/radv_meta_buffer.c +++ b/src/amd/vulkan/meta/radv_meta_buffer.c @@ -247,8 +247,9 @@ radv_fill_buffer(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *im } else if (use_compute) { fill_buffer_shader(cmd_buffer, va, size, value); - flush_bits = RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | - radv_src_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_WRITE_BIT, image); + flush_bits = + RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, image); } else if (size) radv_cp_dma_clear_buffer(cmd_buffer, va, size, value); diff --git a/src/amd/vulkan/meta/radv_meta_clear.c b/src/amd/vulkan/meta/radv_meta_clear.c index 6ea0c59ae6e..ade6ded4d26 100644 --- a/src/amd/vulkan/meta/radv_meta_clear.c +++ b/src/amd/vulkan/meta/radv_meta_clear.c @@ -623,7 +623,8 @@ clear_htile_mask(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *im radv_meta_restore(&saved_state, cmd_buffer); - return RADV_CMD_FLAG_CS_PARTIAL_FLUSH | radv_src_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_WRITE_BIT, image); + return RADV_CMD_FLAG_CS_PARTIAL_FLUSH | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + VK_ACCESS_2_SHADER_WRITE_BIT, image); } static uint32_t @@ -761,8 +762,10 @@ radv_fast_clear_depth(struct radv_cmd_buffer *cmd_buffer, const struct radv_imag if (pre_flush) { enum radv_cmd_flush_bits bits = - radv_src_access_flush(cmd_buffer, VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, iview->image) | - radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_READ_BIT, iview->image); + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, iview->image) | + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, + iview->image); cmd_buffer->state.flush_bits |= bits & ~*pre_flush; *pre_flush |= cmd_buffer->state.flush_bits; } @@ -1318,7 +1321,8 @@ radv_clear_dcc_comp_to_single(struct radv_cmd_buffer *cmd_buffer, struct radv_im radv_meta_restore(&saved_state, cmd_buffer); - return RADV_CMD_FLAG_CS_PARTIAL_FLUSH | radv_src_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_WRITE_BIT, image); + return RADV_CMD_FLAG_CS_PARTIAL_FLUSH | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + VK_ACCESS_2_SHADER_WRITE_BIT, image); } uint32_t @@ -1695,8 +1699,8 @@ radv_fast_clear_color(struct radv_cmd_buffer *cmd_buffer, const struct radv_imag }; if (pre_flush) { - enum radv_cmd_flush_bits bits = - radv_src_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, iview->image); + enum radv_cmd_flush_bits bits = radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, iview->image); cmd_buffer->state.flush_bits |= bits & ~*pre_flush; *pre_flush |= cmd_buffer->state.flush_bits; } diff --git a/src/amd/vulkan/meta/radv_meta_copy.c b/src/amd/vulkan/meta/radv_meta_copy.c index fddc53e0aaa..e3dee165383 100644 --- a/src/amd/vulkan/meta/radv_meta_copy.c +++ b/src/amd/vulkan/meta/radv_meta_copy.c @@ -233,8 +233,10 @@ radv_CmdCopyBufferToImage2(VkCommandBuffer commandBuffer, const VkCopyBufferToIm if (radv_is_format_emulated(pdev, dst_image->vk.format)) { cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_PS_PARTIAL_FLUSH | - radv_src_access_flush(cmd_buffer, VK_ACCESS_TRANSFER_WRITE_BIT, dst_image) | - radv_dst_access_flush(cmd_buffer, VK_ACCESS_TRANSFER_READ_BIT, dst_image); + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_TRANSFER_WRITE_BIT, dst_image) | + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_TRANSFER_READ_BIT, dst_image); const enum util_format_layout format_layout = vk_format_description(dst_image->vk.format)->layout; for (unsigned r = 0; r < pCopyBufferToImageInfo->regionCount; r++) { @@ -611,8 +613,10 @@ radv_CmdCopyImage2(VkCommandBuffer commandBuffer, const VkCopyImageInfo2 *pCopyI if (radv_is_format_emulated(pdev, dst_image->vk.format)) { cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_PS_PARTIAL_FLUSH | - radv_src_access_flush(cmd_buffer, VK_ACCESS_TRANSFER_WRITE_BIT, dst_image) | - radv_dst_access_flush(cmd_buffer, VK_ACCESS_TRANSFER_READ_BIT, dst_image); + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_TRANSFER_WRITE_BIT, dst_image) | + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_TRANSFER_READ_BIT, dst_image); const enum util_format_layout format_layout = vk_format_description(dst_image->vk.format)->layout; for (unsigned r = 0; r < pCopyImageInfo->regionCount; r++) { diff --git a/src/amd/vulkan/meta/radv_meta_copy_vrs_htile.c b/src/amd/vulkan/meta/radv_meta_copy_vrs_htile.c index 93ddbc87826..0c55c73e018 100644 --- a/src/amd/vulkan/meta/radv_meta_copy_vrs_htile.c +++ b/src/amd/vulkan/meta/radv_meta_copy_vrs_htile.c @@ -190,8 +190,9 @@ radv_copy_vrs_htile(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view * } cmd_buffer->state.flush_bits |= - radv_src_access_flush(cmd_buffer, VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, NULL) | - radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_READ_BIT, NULL); + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, NULL) | + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, NULL); radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_COMPUTE_PIPELINE | RADV_META_SAVE_CONSTANTS | RADV_META_SAVE_DESCRIPTORS); @@ -242,6 +243,7 @@ radv_copy_vrs_htile(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view * radv_meta_restore(&saved_state, cmd_buffer); - cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | - radv_src_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_WRITE_BIT, NULL); + cmd_buffer->state.flush_bits |= + RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, NULL); } diff --git a/src/amd/vulkan/meta/radv_meta_dcc_retile.c b/src/amd/vulkan/meta/radv_meta_dcc_retile.c index 107a983e556..55e182c8e4e 100644 --- a/src/amd/vulkan/meta/radv_meta_dcc_retile.c +++ b/src/amd/vulkan/meta/radv_meta_dcc_retile.c @@ -162,7 +162,8 @@ radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image) struct radv_cmd_state *state = &cmd_buffer->state; - state->flush_bits |= radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_READ_BIT, image); + state->flush_bits |= + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, image); unsigned swizzle_mode = image->planes[0].surface.u.gfx9.swizzle_mode; @@ -250,5 +251,6 @@ radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image) radv_meta_restore(&saved_state, cmd_buffer); state->flush_bits |= - RADV_CMD_FLAG_CS_PARTIAL_FLUSH | radv_src_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_WRITE_BIT, image); + RADV_CMD_FLAG_CS_PARTIAL_FLUSH | + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, image); } diff --git a/src/amd/vulkan/meta/radv_meta_decompress.c b/src/amd/vulkan/meta/radv_meta_decompress.c index a5b226e6442..12407da73ec 100644 --- a/src/amd/vulkan/meta/radv_meta_decompress.c +++ b/src/amd/vulkan/meta/radv_meta_decompress.c @@ -575,8 +575,9 @@ radv_expand_depth_stencil_compute(struct radv_cmd_buffer *cmd_buffer, struct rad radv_meta_restore(&saved_state, cmd_buffer); - cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | - radv_src_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_WRITE_BIT, image); + cmd_buffer->state.flush_bits |= + RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, image); /* Initialize the HTILE metadata as "fully expanded". */ uint32_t htile_value = radv_get_htile_initial_value(device, image); diff --git a/src/amd/vulkan/meta/radv_meta_fast_clear.c b/src/amd/vulkan/meta/radv_meta_fast_clear.c index 12bc6ba348f..3228c6efd31 100644 --- a/src/amd/vulkan/meta/radv_meta_fast_clear.c +++ b/src/amd/vulkan/meta/radv_meta_fast_clear.c @@ -483,12 +483,14 @@ radv_process_color_image_layer(struct radv_cmd_buffer *cmd_buffer, struct radv_i radv_CmdBeginRendering(radv_cmd_buffer_to_handle(cmd_buffer), &rendering_info); if (flush_cb) - cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, image); + cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, image); radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0); if (flush_cb) - cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, image); + cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, image); radv_CmdEndRendering(radv_cmd_buffer_to_handle(cmd_buffer)); @@ -670,7 +672,8 @@ radv_decompress_dcc_compute(struct radv_cmd_buffer *cmd_buffer, struct radv_imag struct radv_image_view load_iview = {0}; struct radv_image_view store_iview = {0}; - cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_READ_BIT, image); + cmd_buffer->state.flush_bits |= + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT, image); if (!device->meta_state.fast_clear_flush.cmask_eliminate_pipeline) { VkResult ret = radv_device_init_meta_fast_clear_flush_state_internal(device); @@ -765,8 +768,9 @@ radv_decompress_dcc_compute(struct radv_cmd_buffer *cmd_buffer, struct radv_imag radv_meta_restore(&saved_state, cmd_buffer); - cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | - radv_src_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_WRITE_BIT, image); + cmd_buffer->state.flush_bits |= + RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, image); /* Initialize the DCC metadata as "fully expanded". */ cmd_buffer->state.flush_bits |= radv_init_dcc(cmd_buffer, image, subresourceRange, 0xffffffff); diff --git a/src/amd/vulkan/meta/radv_meta_fmask_expand.c b/src/amd/vulkan/meta/radv_meta_fmask_expand.c index dea93422bcf..8045f033412 100644 --- a/src/amd/vulkan/meta/radv_meta_fmask_expand.c +++ b/src/amd/vulkan/meta/radv_meta_fmask_expand.c @@ -74,7 +74,8 @@ radv_expand_fmask_image_inplace(struct radv_cmd_buffer *cmd_buffer, struct radv_ radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); - cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_READ_BIT, image); + cmd_buffer->state.flush_bits |= + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, image); radv_image_view_init(&iview, device, &(VkImageViewCreateInfo){ @@ -125,7 +126,8 @@ radv_expand_fmask_image_inplace(struct radv_cmd_buffer *cmd_buffer, struct radv_ radv_meta_restore(&saved_state, cmd_buffer); cmd_buffer->state.flush_bits |= - RADV_CMD_FLAG_CS_PARTIAL_FLUSH | radv_src_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_WRITE_BIT, image); + RADV_CMD_FLAG_CS_PARTIAL_FLUSH | + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, image); /* Re-initialize FMASK in fully expanded mode. */ cmd_buffer->state.flush_bits |= radv_init_fmask(cmd_buffer, image, subresourceRange); diff --git a/src/amd/vulkan/meta/radv_meta_resolve.c b/src/amd/vulkan/meta/radv_meta_resolve.c index 2a36f64f8b1..09bf395b05c 100644 --- a/src/amd/vulkan/meta/radv_meta_resolve.c +++ b/src/amd/vulkan/meta/radv_meta_resolve.c @@ -216,14 +216,16 @@ emit_resolve(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *src_im VkCommandBuffer cmd_buffer_h = radv_cmd_buffer_to_handle(cmd_buffer); unsigned fs_key = radv_format_meta_fs_key(device, vk_format); - cmd_buffer->state.flush_bits |= - radv_src_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, src_image) | - radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, src_image); + cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, src_image) | + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, src_image); radv_CmdBindPipeline(cmd_buffer_h, VK_PIPELINE_BIND_POINT_GRAPHICS, device->meta_state.resolve.pipeline[fs_key]); radv_CmdDraw(cmd_buffer_h, 3, 1, 0, 0); - cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, dst_image); + cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, dst_image); } enum radv_resolve_method { diff --git a/src/amd/vulkan/meta/radv_meta_resolve_cs.c b/src/amd/vulkan/meta/radv_meta_resolve_cs.c index c46e7ab2c97..f08aeac46d2 100644 --- a/src/amd/vulkan/meta/radv_meta_resolve_cs.c +++ b/src/amd/vulkan/meta/radv_meta_resolve_cs.c @@ -676,8 +676,9 @@ radv_cmd_buffer_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_resolve_compute_image(cmd_buffer, src_iview->image, src_iview->vk.format, src_layout, dst_iview->image, dst_iview->vk.format, dst_layout, region); - cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | - radv_src_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_WRITE_BIT, NULL); + cmd_buffer->state.flush_bits |= + RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, NULL); } void @@ -697,8 +698,9 @@ radv_depth_stencil_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, VkIm * we have to make the attachment shader-readable. */ cmd_buffer->state.flush_bits |= - radv_src_access_flush(cmd_buffer, VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, NULL) | - radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_READ_BIT, NULL); + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, NULL) | + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, NULL); struct radv_image_view *src_iview = render->ds_att.iview; VkImageLayout src_layout = @@ -761,8 +763,9 @@ radv_depth_stencil_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, VkIm &(VkExtent3D){resolve_area.extent.width, resolve_area.extent.height, layer_count}, aspects, resolve_mode); - cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | - radv_src_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_WRITE_BIT, NULL); + cmd_buffer->state.flush_bits |= + RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, NULL); uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, cmd_buffer->qf); diff --git a/src/amd/vulkan/meta/radv_meta_resolve_fs.c b/src/amd/vulkan/meta/radv_meta_resolve_fs.c index 4f506ca9f2a..1374ef127dd 100644 --- a/src/amd/vulkan/meta/radv_meta_resolve_fs.c +++ b/src/amd/vulkan/meta/radv_meta_resolve_fs.c @@ -589,9 +589,10 @@ emit_resolve(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_ivi }}, }); - cmd_buffer->state.flush_bits |= - radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_READ_BIT, src_iview->image) | - radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, dst_iview->image); + cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_SHADER_READ_BIT, src_iview->image) | + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, dst_iview->image); unsigned push_constants[2] = { src_offset->x - dst_offset->x, @@ -605,8 +606,8 @@ emit_resolve(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_ivi radv_CmdBindPipeline(cmd_buffer_h, VK_PIPELINE_BIND_POINT_GRAPHICS, *pipeline); radv_CmdDraw(cmd_buffer_h, 3, 1, 0, 0); - cmd_buffer->state.flush_bits |= - radv_src_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, dst_iview->image); + cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, dst_iview->image); } static void diff --git a/src/amd/vulkan/radv_acceleration_structure.c b/src/amd/vulkan/radv_acceleration_structure.c index e50c1659c57..4e91df81648 100644 --- a/src/amd/vulkan/radv_acceleration_structure.c +++ b/src/amd/vulkan/radv_acceleration_structure.c @@ -1352,9 +1352,10 @@ radv_CmdBuildAccelerationStructuresKHR(VkCommandBuffer commandBuffer, uint32_t i return; } - enum radv_cmd_flush_bits flush_bits = RADV_CMD_FLAG_CS_PARTIAL_FLUSH | - radv_src_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_WRITE_BIT, NULL) | - radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_READ_BIT, NULL); + enum radv_cmd_flush_bits flush_bits = + RADV_CMD_FLAG_CS_PARTIAL_FLUSH | + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, NULL) | + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT, NULL); radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_COMPUTE_PIPELINE | RADV_META_SAVE_DESCRIPTORS | RADV_META_SAVE_CONSTANTS); @@ -1511,7 +1512,8 @@ radv_CmdCopyAccelerationStructureKHR(VkCommandBuffer commandBuffer, const VkCopy device->meta_state.accel_struct_build.copy_p_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(consts), &consts); - cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT, NULL); + cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT, NULL); radv_indirect_dispatch( cmd_buffer, src_buffer->bo, @@ -1616,7 +1618,8 @@ radv_CmdCopyAccelerationStructureToMemoryKHR(VkCommandBuffer commandBuffer, device->meta_state.accel_struct_build.copy_p_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(consts), &consts); - cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT, NULL); + cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT, NULL); radv_indirect_dispatch( cmd_buffer, src_buffer->bo, diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c index f3efd578513..4895d3f6b21 100644 --- a/src/amd/vulkan/radv_cmd_buffer.c +++ b/src/amd/vulkan/radv_cmd_buffer.c @@ -29,6 +29,7 @@ #include "vk_format.h" #include "vk_framebuffer.h" #include "vk_render_pass.h" +#include "vk_synchronization.h" #include "vk_util.h" #include "ac_debug.h" @@ -7009,8 +7010,11 @@ can_skip_buffer_l2_flushes(struct radv_device *device) */ enum radv_cmd_flush_bits -radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer, VkAccessFlags2 src_flags, const struct radv_image *image) +radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer, VkPipelineStageFlags2 src_stages, VkAccessFlags2 src_flags, + const struct radv_image *image) { + src_flags = vk_expand_src_access_flags2(src_stages, src_flags); + bool has_CB_meta = true, has_DB_meta = true; bool image_is_coherent = image ? image->l2_coherent : false; enum radv_cmd_flush_bits flush_bits = 0; @@ -7087,7 +7091,8 @@ radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer, VkAccessFlags2 src_fla } enum radv_cmd_flush_bits -radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer, VkAccessFlags2 dst_flags, const struct radv_image *image) +radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer, VkPipelineStageFlags2 dst_stages, VkAccessFlags2 dst_flags, + const struct radv_image *image) { struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_physical_device *pdev = radv_device_physical(device); @@ -7097,6 +7102,8 @@ radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer, VkAccessFlags2 dst_fla bool image_is_coherent = image ? image->l2_coherent : false; bool flush_L2_metadata = false; + dst_flags = vk_expand_dst_access_flags2(dst_stages, dst_flags); + if (image) { if (!(image->vk.usage & VK_IMAGE_USAGE_STORAGE_BIT)) { flush_CB = false; @@ -7224,11 +7231,12 @@ radv_emit_resolve_barrier(struct radv_cmd_buffer *cmd_buffer, const struct radv_ if (!iview) continue; - cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, barrier->src_access_mask, iview->image); + cmd_buffer->state.flush_bits |= + radv_src_access_flush(cmd_buffer, barrier->src_stage_mask, barrier->src_access_mask, iview->image); } if (render->ds_att.iview) { - cmd_buffer->state.flush_bits |= - radv_src_access_flush(cmd_buffer, barrier->src_access_mask, render->ds_att.iview->image); + cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, barrier->src_stage_mask, + barrier->src_access_mask, render->ds_att.iview->image); } radv_stage_flush(cmd_buffer, barrier->src_stage_mask); @@ -7238,11 +7246,12 @@ radv_emit_resolve_barrier(struct radv_cmd_buffer *cmd_buffer, const struct radv_ if (!iview) continue; - cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, barrier->dst_access_mask, iview->image); + cmd_buffer->state.flush_bits |= + radv_dst_access_flush(cmd_buffer, barrier->dst_stage_mask, barrier->dst_access_mask, iview->image); } if (render->ds_att.iview) { - cmd_buffer->state.flush_bits |= - radv_dst_access_flush(cmd_buffer, barrier->dst_access_mask, render->ds_att.iview->image); + cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, barrier->dst_stage_mask, + barrier->dst_access_mask, render->ds_att.iview->image); } radv_gang_barrier(cmd_buffer, barrier->src_stage_mask, barrier->dst_stage_mask); @@ -7826,7 +7835,8 @@ radv_EndCommandBuffer(VkCommandBuffer commandBuffer) */ if (cmd_buffer->state.rb_noncoherent_dirty && !can_skip_buffer_l2_flushes(device)) cmd_buffer->state.flush_bits |= radv_src_access_flush( - cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, NULL); + cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, NULL); /* Since NGG streamout uses GDS, we need to make GDS idle when * we leave the IB, otherwise another process might overwrite @@ -12184,9 +12194,10 @@ radv_trace_trace_rays(struct radv_cmd_buffer *cmd_buffer, const VkTraceRaysIndir util_dynarray_append(&cmd_buffer->ray_history, struct radv_rra_ray_history_data *, data); - cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_INV_SCACHE | RADV_CMD_FLAG_CS_PARTIAL_FLUSH | - radv_src_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_WRITE_BIT, NULL) | - radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_READ_BIT, NULL); + cmd_buffer->state.flush_bits |= + RADV_CMD_FLAG_INV_SCACHE | RADV_CMD_FLAG_CS_PARTIAL_FLUSH | + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, NULL) | + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, NULL); radv_update_buffer_cp(cmd_buffer, device->rra_trace.ray_history_addr + offsetof(struct radv_ray_history_header, dispatch_index), @@ -12432,14 +12443,16 @@ radv_initialize_htile(struct radv_cmd_buffer *cmd_buffer, struct radv_image *ima /* Transitioning from LAYOUT_UNDEFINED layout not everyone is consistent * in considering previous rendering work for WAW hazards. */ - state->flush_bits |= radv_src_access_flush(cmd_buffer, VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, image); + state->flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, image); if (image->planes[0].surface.has_stencil && !(range->aspectMask == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT))) { /* Flush caches before performing a separate aspect initialization because it's a * read-modify-write operation. */ - state->flush_bits |= radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_READ_BIT, image); + state->flush_bits |= + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, image); } state->flush_bits |= radv_clear_htile(cmd_buffer, image, range, htile_value); @@ -12564,7 +12577,8 @@ radv_init_color_image_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_i /* Transitioning from LAYOUT_UNDEFINED layout not everyone is * consistent in considering previous rendering work for WAW hazards. */ - cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, image); + cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, image); if (radv_image_has_cmask(image)) { static const uint32_t cmask_clear_values[4] = {0xffffffff, 0xdddddddd, 0xeeeeeeee, 0xffffffff}; @@ -12817,17 +12831,17 @@ radv_barrier(struct radv_cmd_buffer *cmd_buffer, uint32_t dep_count, const VkDep for (uint32_t i = 0; i < dep_info->memoryBarrierCount; i++) { const VkMemoryBarrier2 *barrier = &dep_info->pMemoryBarriers[i]; src_stage_mask |= barrier->srcStageMask; - src_flush_bits |= radv_src_access_flush(cmd_buffer, barrier->srcAccessMask, NULL); + src_flush_bits |= radv_src_access_flush(cmd_buffer, barrier->srcStageMask, barrier->srcAccessMask, NULL); dst_stage_mask |= barrier->dstStageMask; - dst_flush_bits |= radv_dst_access_flush(cmd_buffer, barrier->dstAccessMask, NULL); + dst_flush_bits |= radv_dst_access_flush(cmd_buffer, barrier->dstStageMask, barrier->dstAccessMask, NULL); } for (uint32_t i = 0; i < dep_info->bufferMemoryBarrierCount; i++) { const VkBufferMemoryBarrier2 *barrier = &dep_info->pBufferMemoryBarriers[i]; src_stage_mask |= barrier->srcStageMask; - src_flush_bits |= radv_src_access_flush(cmd_buffer, barrier->srcAccessMask, NULL); + src_flush_bits |= radv_src_access_flush(cmd_buffer, barrier->srcStageMask, barrier->srcAccessMask, NULL); dst_stage_mask |= barrier->dstStageMask; - dst_flush_bits |= radv_dst_access_flush(cmd_buffer, barrier->dstAccessMask, NULL); + dst_flush_bits |= radv_dst_access_flush(cmd_buffer, barrier->dstStageMask, barrier->dstAccessMask, NULL); } for (uint32_t i = 0; i < dep_info->imageMemoryBarrierCount; i++) { @@ -12835,9 +12849,9 @@ radv_barrier(struct radv_cmd_buffer *cmd_buffer, uint32_t dep_count, const VkDep VK_FROM_HANDLE(radv_image, image, barrier->image); src_stage_mask |= barrier->srcStageMask; - src_flush_bits |= radv_src_access_flush(cmd_buffer, barrier->srcAccessMask, image); + src_flush_bits |= radv_src_access_flush(cmd_buffer, barrier->srcStageMask, barrier->srcAccessMask, image); dst_stage_mask |= barrier->dstStageMask; - dst_flush_bits |= radv_dst_access_flush(cmd_buffer, barrier->dstAccessMask, image); + dst_flush_bits |= radv_dst_access_flush(cmd_buffer, barrier->dstStageMask, barrier->dstAccessMask, image); } } diff --git a/src/amd/vulkan/radv_cmd_buffer.h b/src/amd/vulkan/radv_cmd_buffer.h index 9e6cf48cd19..4c83d079df9 100644 --- a/src/amd/vulkan/radv_cmd_buffer.h +++ b/src/amd/vulkan/radv_cmd_buffer.h @@ -705,11 +705,11 @@ void radv_write_vertex_descriptors(const struct radv_cmd_buffer *cmd_buffer, const struct radv_graphics_pipeline *pipeline, bool full_null_descriptors, void *vb_ptr); -enum radv_cmd_flush_bits radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer, VkAccessFlags2 src_flags, - const struct radv_image *image); +enum radv_cmd_flush_bits radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer, VkPipelineStageFlags2 src_stages, + VkAccessFlags2 src_flags, const struct radv_image *image); -enum radv_cmd_flush_bits radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer, VkAccessFlags2 dst_flags, - const struct radv_image *image); +enum radv_cmd_flush_bits radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer, VkPipelineStageFlags2 dst_stages, + VkAccessFlags2 dst_flags, const struct radv_image *image); struct radv_resolve_barrier { VkPipelineStageFlags2 src_stage_mask;