From d426ee6a9901b5de5abd09b32fdd1a4c23c2ba50 Mon Sep 17 00:00:00 2001 From: Connor Abbott Date: Wed, 20 Jul 2022 17:33:09 +0200 Subject: [PATCH] tu: Support resolving D24S8 -> S8 This was missed when we added support for VK_KHR_depth_stencil_resolve. There is a similar feature where the stencil aspect of a D24S8 can be copied "tightly" in CopyImageToBuffer, but it used the texture swizzle and so required the 3d path. To get it to work with the 2D path, which is required for resolves, we have to instead use the A8_UNORM format, which works for texture sampling even for tiled images. We also have to reuse the pre-existing image views because subpass resolves work on image views rather than images, whereas before the fixup was applied while creating the image view. This means threading through the corresponding "opposite" format through setup, src, and dst functions, doing the fixup there (through some shared helpers), and then getting every user to specify the right format. As a bonus, we no longer need to force the 3d path for the CopyImageToBuffer and CopyBufferToImage special cases. Part-of: --- src/freedreno/ci/freedreno-a618-fails.txt | 5 - src/freedreno/fdl/fd6_view.c | 2 + src/freedreno/fdl/freedreno_layout.h | 2 + src/freedreno/vulkan/tu_clear_blit.c | 434 ++++++++++++++-------- 4 files changed, 277 insertions(+), 166 deletions(-) diff --git a/src/freedreno/ci/freedreno-a618-fails.txt b/src/freedreno/ci/freedreno-a618-fails.txt index 615df96bd0d..86d36560e48 100644 --- a/src/freedreno/ci/freedreno-a618-fails.txt +++ b/src/freedreno/ci/freedreno-a618-fails.txt @@ -3,19 +3,14 @@ gmem-dEQP-VK.spirv_assembly.instruction.graphics.variable_pointers.graphics.writ gmem-dEQP-VK.spirv_assembly.instruction.graphics.variable_pointers.graphics.writes_two_buffers_vert,Fail # VK-GL-CTS 1.3.2.0 uprev -dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d24_unorm_s8_uint_separate_layouts.compatibility_depth_zero_stencil_zero_testing_stencil,Fail dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d32_sfloat_s8_uint_separate_layouts.compatibility_depth_zero_stencil_zero_testing_stencil,Fail gmem-dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d32_sfloat_s8_uint.compatibility_depth_zero_stencil_zero_testing_stencil,Fail # https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/3590 dEQP-VK.api.info.get_physical_device_properties2.memory_properties,Fail -dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d24_unorm_s8_uint.compatibility_depth_zero_stencil_zero_testing_stencil,Fail dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d32_sfloat_s8_uint.compatibility_depth_zero_stencil_zero_testing_stencil,Fail -bypass-dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d24_unorm_s8_uint.compatibility_depth_zero_stencil_zero_testing_stencil,Fail -bypass-dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d24_unorm_s8_uint_separate_layouts.compatibility_depth_zero_stencil_zero_testing_stencil,Fail - # https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/3759 # deqp-vk: ../src/freedreno/vulkan/tu_pipeline.c:3894: tu_pipeline_builder_init_graphics: Assertion `subpass->color_count == 0 || !create_info->pColorBlendState || subpass->color_count == create_info->pColorBlendState->attachmentCount' failed dEQP-VK.pipeline.monolithic.color_write_enable_maxa.cwe_after_bind.attachments4_more0,Crash diff --git a/src/freedreno/fdl/fd6_view.c b/src/freedreno/fdl/fd6_view.c index 34dd9a8d26c..00b35ea280d 100644 --- a/src/freedreno/fdl/fd6_view.c +++ b/src/freedreno/fdl/fd6_view.c @@ -249,6 +249,8 @@ fdl6_view_init(struct fdl6_view *view, const struct fdl_layout **layouts, storage_format = FMT6_8_8_8_8_UNORM; } + view->format = args->format; + memset(view->descriptor, 0, sizeof(view->descriptor)); view->descriptor[0] = diff --git a/src/freedreno/fdl/freedreno_layout.h b/src/freedreno/fdl/freedreno_layout.h index b6be49d207c..52be661d29e 100644 --- a/src/freedreno/fdl/freedreno_layout.h +++ b/src/freedreno/fdl/freedreno_layout.h @@ -293,6 +293,8 @@ struct fdl6_view { bool ubwc_enabled; + enum pipe_format format; + uint32_t descriptor[FDL6_TEX_CONST_DWORDS]; /* Descriptor for use as a storage image as opposed to a sampled image. diff --git a/src/freedreno/vulkan/tu_clear_blit.c b/src/freedreno/vulkan/tu_clear_blit.c index 9c1bf2efe96..8178cdb8beb 100644 --- a/src/freedreno/vulkan/tu_clear_blit.c +++ b/src/freedreno/vulkan/tu_clear_blit.c @@ -41,6 +41,8 @@ format_to_ifmt(enum pipe_format format) return R2D_FLOAT32; if (format == PIPE_FORMAT_S8_UINT) return R2D_INT8; + if (format == PIPE_FORMAT_A8_UNORM) + return R2D_UNORM8; /* use the size of the red channel to find the corresponding "ifmt" */ bool is_int = util_format_is_pure_integer(format); @@ -141,17 +143,59 @@ r2d_clear_value(struct tu_cs *cs, enum pipe_format format, const VkClearValue *v tu_cs_emit_array(cs, clear_value, 4); } +static void +fixup_src_format(enum pipe_format *src_format, enum pipe_format dst_format, + enum a6xx_format *fmt) +{ + /* When blitting S8 -> D24S8 or vice versa, we have to override S8, which + * is normally R8_UINT for sampling/blitting purposes, to a unorm format. + * We also have to move stencil, which is normally in the .w channel, into + * the right channel. Reintepreting the S8 texture as A8_UNORM solves both + * problems, and avoids using a swap, which seems to sometimes not work + * with a D24S8 source, or a texture swizzle which is only supported with + * the 3d path. Sometimes this blit happens on already-constructed + * fdl6_view's, e.g. for sysmem resolves, so this has to happen as a fixup. + */ + if (*src_format == PIPE_FORMAT_S8_UINT && + (dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT || + dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT_AS_R8G8B8A8)) { + *fmt = FMT6_A8_UNORM; + *src_format = PIPE_FORMAT_A8_UNORM; + } +} + +static void +fixup_dst_format(enum pipe_format src_format, enum pipe_format *dst_format, + enum a6xx_format *fmt) +{ + if (*dst_format == PIPE_FORMAT_S8_UINT && + (src_format == PIPE_FORMAT_Z24_UNORM_S8_UINT || + src_format == PIPE_FORMAT_Z24_UNORM_S8_UINT_AS_R8G8B8A8)) { + *dst_format = PIPE_FORMAT_A8_UNORM; + *fmt = FMT6_A8_UNORM; + } +} + static void r2d_src(struct tu_cmd_buffer *cmd, struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer, - VkFilter filter) + VkFilter filter, + enum pipe_format dst_format) { uint32_t src_info = iview->SP_PS_2D_SRC_INFO; if (filter != VK_FILTER_NEAREST) src_info |= A6XX_SP_PS_2D_SRC_INFO_FILTER; + enum a6xx_format fmt = (src_info & A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK); + enum pipe_format src_format = iview->format; + fixup_src_format(&src_format, dst_format, &fmt); + + src_info = + (src_info & ~A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK) | + A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT(fmt); + tu_cs_emit_pkt4(cs, REG_A6XX_SP_PS_2D_SRC_INFO, 5); tu_cs_emit(cs, src_info); tu_cs_emit(cs, iview->SP_PS_2D_SRC_SIZE); @@ -199,13 +243,16 @@ r2d_src_buffer(struct tu_cmd_buffer *cmd, struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch, - uint32_t width, uint32_t height) + uint32_t width, uint32_t height, + enum pipe_format dst_format) { struct tu_native_format fmt = tu6_format_texture(format, TILE6_LINEAR); + enum a6xx_format color_format = fmt.fmt; + fixup_src_format(&format, dst_format, &color_format); tu_cs_emit_regs(cs, A6XX_SP_PS_2D_SRC_INFO( - .color_format = fmt.fmt, + .color_format = color_format, .color_swap = fmt.swap, .srgb = util_format_is_srgb(format), .unk20 = 1, @@ -216,10 +263,18 @@ r2d_src_buffer(struct tu_cmd_buffer *cmd, } static void -r2d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer) +r2d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer, + enum pipe_format src_format) { + uint32_t dst_info = iview->RB_2D_DST_INFO; + enum a6xx_format fmt = dst_info & A6XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK; + enum pipe_format dst_format = iview->format; + fixup_dst_format(src_format, &dst_format, &fmt); + + dst_info = + (dst_info & ~A6XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK) | fmt; tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_DST_INFO, 4); - tu_cs_emit(cs, iview->RB_2D_DST_INFO); + tu_cs_emit(cs, dst_info); tu_cs_image_ref_2d(cs, iview, layer, false); tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_DST_FLAGS, 3); @@ -248,9 +303,13 @@ r2d_dst_stencil(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t la } static void -r2d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch) +r2d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch, + enum pipe_format src_format) { struct tu_native_format fmt = tu6_format_color(format, TILE6_LINEAR); + enum a6xx_format color_fmt = fmt.fmt; + fixup_dst_format(src_format, &format, &color_fmt); + fmt.fmt = color_fmt; tu_cs_emit_regs(cs, A6XX_RB_2D_DST_INFO( @@ -264,25 +323,27 @@ r2d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t static void r2d_setup_common(struct tu_cmd_buffer *cmd, struct tu_cs *cs, - enum pipe_format format, + enum pipe_format src_format, + enum pipe_format dst_format, VkImageAspectFlags aspect_mask, unsigned blit_param, bool clear, bool ubwc, bool scissor) { - enum a6xx_format fmt = tu6_base_format(format); - enum a6xx_2d_ifmt ifmt = format_to_ifmt(format); + enum a6xx_format fmt = tu6_base_format(dst_format); + fixup_dst_format(src_format, &dst_format, &fmt); + enum a6xx_2d_ifmt ifmt = format_to_ifmt(dst_format); uint32_t unknown_8c01 = 0; - if ((format == PIPE_FORMAT_Z24_UNORM_S8_UINT || - format == PIPE_FORMAT_Z24X8_UNORM) && ubwc) { + if ((dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT || + dst_format == PIPE_FORMAT_Z24X8_UNORM) && ubwc) { fmt = FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8; } /* note: the only format with partial clearing is D24S8 */ - if (format == PIPE_FORMAT_Z24_UNORM_S8_UINT) { + if (dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT) { /* preserve stencil channel */ if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) unknown_8c01 = 0x08000041; @@ -301,7 +362,7 @@ r2d_setup_common(struct tu_cmd_buffer *cmd, .d24s8 = fmt == FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8 && !clear, .color_format = fmt, .mask = 0xf, - .ifmt = util_format_is_srgb(format) ? R2D_UNORM8_SRGB : ifmt, + .ifmt = util_format_is_srgb(dst_format) ? R2D_UNORM8_SRGB : ifmt, ).value; tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_BLIT_CNTL, 1); @@ -314,17 +375,18 @@ r2d_setup_common(struct tu_cmd_buffer *cmd, fmt = FMT6_16_16_16_16_FLOAT; tu_cs_emit_regs(cs, A6XX_SP_2D_DST_FORMAT( - .sint = util_format_is_pure_sint(format), - .uint = util_format_is_pure_uint(format), + .sint = util_format_is_pure_sint(dst_format), + .uint = util_format_is_pure_uint(dst_format), .color_format = fmt, - .srgb = util_format_is_srgb(format), + .srgb = util_format_is_srgb(dst_format), .mask = 0xf)); } static void r2d_setup(struct tu_cmd_buffer *cmd, struct tu_cs *cs, - enum pipe_format format, + enum pipe_format src_format, + enum pipe_format dst_format, VkImageAspectFlags aspect_mask, unsigned blit_param, bool clear, @@ -337,7 +399,7 @@ r2d_setup(struct tu_cmd_buffer *cmd, tu_emit_cache_flush_ccu(cmd, cs, TU_CMD_CCU_SYSMEM); } - r2d_setup_common(cmd, cs, format, aspect_mask, blit_param, clear, ubwc, false); + r2d_setup_common(cmd, cs, src_format, dst_format, aspect_mask, blit_param, clear, ubwc, false); } static void @@ -893,9 +955,20 @@ r3d_src(struct tu_cmd_buffer *cmd, struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer, - VkFilter filter) + VkFilter filter, + enum pipe_format dst_format) { - r3d_src_common(cmd, cs, iview->descriptor, + uint32_t desc[A6XX_TEX_CONST_DWORDS]; + memcpy(desc, iview->descriptor, sizeof(desc)); + + enum a6xx_format fmt = (desc[0] & A6XX_TEX_CONST_0_FMT__MASK) >> + A6XX_TEX_CONST_0_FMT__SHIFT; + enum pipe_format src_format = iview->format; + fixup_src_format(&src_format, dst_format, &fmt); + desc[0] = (desc[0] & ~A6XX_TEX_CONST_0_FMT__MASK) | + A6XX_TEX_CONST_0_FMT(fmt); + + r3d_src_common(cmd, cs, desc, iview->layer_size * layer, iview->ubwc_layer_size * layer, filter); @@ -906,21 +979,23 @@ r3d_src_buffer(struct tu_cmd_buffer *cmd, struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch, - uint32_t width, uint32_t height) + uint32_t width, uint32_t height, + enum pipe_format dst_format) { uint32_t desc[A6XX_TEX_CONST_DWORDS]; struct tu_native_format fmt = tu6_format_texture(format, TILE6_LINEAR); + enum a6xx_format color_format = fmt.fmt; + fixup_src_format(&format, dst_format, &color_format); desc[0] = COND(util_format_is_srgb(format), A6XX_TEX_CONST_0_SRGB) | - A6XX_TEX_CONST_0_FMT(fmt.fmt) | + A6XX_TEX_CONST_0_FMT(color_format) | A6XX_TEX_CONST_0_SWAP(fmt.swap) | A6XX_TEX_CONST_0_SWIZ_X(A6XX_TEX_X) | - // XXX to swizzle into .w for stencil buffer_to_image - A6XX_TEX_CONST_0_SWIZ_Y(format == PIPE_FORMAT_R8_UNORM ? A6XX_TEX_X : A6XX_TEX_Y) | - A6XX_TEX_CONST_0_SWIZ_Z(format == PIPE_FORMAT_R8_UNORM ? A6XX_TEX_X : A6XX_TEX_Z) | - A6XX_TEX_CONST_0_SWIZ_W(format == PIPE_FORMAT_R8_UNORM ? A6XX_TEX_X : A6XX_TEX_W); + A6XX_TEX_CONST_0_SWIZ_Y(A6XX_TEX_Y) | + A6XX_TEX_CONST_0_SWIZ_Z(A6XX_TEX_Z) | + A6XX_TEX_CONST_0_SWIZ_W(A6XX_TEX_W); desc[1] = A6XX_TEX_CONST_1_WIDTH(width) | A6XX_TEX_CONST_1_HEIGHT(height); desc[2] = A6XX_TEX_CONST_2_PITCH(pitch) | @@ -939,17 +1014,21 @@ r3d_src_gmem(struct tu_cmd_buffer *cmd, struct tu_cs *cs, const struct tu_image_view *iview, enum pipe_format format, + enum pipe_format dst_format, uint32_t gmem_offset, uint32_t cpp) { uint32_t desc[A6XX_TEX_CONST_DWORDS]; memcpy(desc, iview->view.descriptor, sizeof(desc)); + enum a6xx_format fmt = tu6_format_texture(format, TILE6_LINEAR).fmt; + fixup_src_format(&format, dst_format, &fmt); + /* patch the format so that depth/stencil get the right format and swizzle */ desc[0] &= ~(A6XX_TEX_CONST_0_FMT__MASK | A6XX_TEX_CONST_0_SWIZ_X__MASK | A6XX_TEX_CONST_0_SWIZ_Y__MASK | A6XX_TEX_CONST_0_SWIZ_Z__MASK | A6XX_TEX_CONST_0_SWIZ_W__MASK); - desc[0] |= A6XX_TEX_CONST_0_FMT(tu6_format_texture(format, TILE6_2).fmt) | + desc[0] |= A6XX_TEX_CONST_0_FMT(fmt) | A6XX_TEX_CONST_0_SWIZ_X(A6XX_TEX_X) | A6XX_TEX_CONST_0_SWIZ_Y(A6XX_TEX_Y) | A6XX_TEX_CONST_0_SWIZ_Z(A6XX_TEX_Z) | @@ -971,10 +1050,19 @@ r3d_src_gmem(struct tu_cmd_buffer *cmd, } static void -r3d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer) +r3d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer, + enum pipe_format src_format) { + uint32_t mrt_buf_info = iview->RB_MRT_BUF_INFO; + + enum a6xx_format fmt = mrt_buf_info & A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK; + enum pipe_format dst_format = iview->format; + fixup_dst_format(src_format, &dst_format, &fmt); + mrt_buf_info = + (mrt_buf_info & ~A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK) | + A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT(fmt); tu_cs_emit_pkt4(cs, REG_A6XX_RB_MRT_BUF_INFO(0), 6); - tu_cs_emit(cs, iview->RB_MRT_BUF_INFO); + tu_cs_emit(cs, mrt_buf_info); tu_cs_image_ref(cs, iview, layer); tu_cs_emit(cs, 0); @@ -984,7 +1072,7 @@ r3d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer) /* Use color format from RB_MRT_BUF_INFO. This register is relevant for * FMT6_NV12_Y. */ - tu_cs_emit_regs(cs, A6XX_GRAS_LRZ_MRT_BUF_INFO_0(.color_format = iview->RB_MRT_BUF_INFO & 0xff)); + tu_cs_emit_regs(cs, A6XX_GRAS_LRZ_MRT_BUF_INFO_0(.color_format = fmt)); tu_cs_emit_regs(cs, A6XX_RB_RENDER_CNTL(.flag_mrts = iview->ubwc_enabled)); } @@ -1015,12 +1103,16 @@ r3d_dst_stencil(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t la } static void -r3d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch) +r3d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch, + enum pipe_format src_format) { struct tu_native_format fmt = tu6_format_color(format, TILE6_LINEAR); + enum a6xx_format color_fmt = fmt.fmt; + fixup_dst_format(src_format, &format, &color_fmt); + tu_cs_emit_regs(cs, - A6XX_RB_MRT_BUF_INFO(0, .color_format = fmt.fmt, .color_swap = fmt.swap), + A6XX_RB_MRT_BUF_INFO(0, .color_format = color_fmt, .color_swap = fmt.swap), A6XX_RB_MRT_PITCH(0, pitch), A6XX_RB_MRT_ARRAY_PITCH(0, 0), A6XX_RB_MRT_BASE(0, .qword = va), @@ -1049,17 +1141,19 @@ aspect_write_mask(enum pipe_format format, VkImageAspectFlags aspect_mask) static void r3d_setup(struct tu_cmd_buffer *cmd, struct tu_cs *cs, - enum pipe_format format, + enum pipe_format src_format, + enum pipe_format dst_format, VkImageAspectFlags aspect_mask, unsigned blit_param, bool clear, bool ubwc, VkSampleCountFlagBits samples) { - enum a6xx_format fmt = tu6_base_format(format); + enum a6xx_format fmt = tu6_base_format(dst_format); + fixup_dst_format(src_format, &dst_format, &fmt); - if ((format == PIPE_FORMAT_Z24_UNORM_S8_UINT || - format == PIPE_FORMAT_Z24X8_UNORM) && ubwc) { + if ((dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT || + dst_format == PIPE_FORMAT_Z24X8_UNORM) && ubwc) { fmt = FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8; } @@ -1099,13 +1193,13 @@ r3d_setup(struct tu_cmd_buffer *cmd, tu_cs_emit_regs(cs, A6XX_SP_FS_MRT_REG(0, .color_format = fmt, - .color_sint = util_format_is_pure_sint(format), - .color_uint = util_format_is_pure_uint(format))); + .color_sint = util_format_is_pure_sint(dst_format), + .color_uint = util_format_is_pure_uint(dst_format))); tu_cs_emit_regs(cs, A6XX_RB_MRT_CONTROL(0, - .component_enable = aspect_write_mask(format, aspect_mask))); - tu_cs_emit_regs(cs, A6XX_RB_SRGB_CNTL(util_format_is_srgb(format))); - tu_cs_emit_regs(cs, A6XX_SP_SRGB_CNTL(util_format_is_srgb(format))); + .component_enable = aspect_write_mask(dst_format, aspect_mask))); + tu_cs_emit_regs(cs, A6XX_RB_SRGB_CNTL(util_format_is_srgb(dst_format))); + tu_cs_emit_regs(cs, A6XX_SP_SRGB_CNTL(util_format_is_srgb(dst_format))); tu_cs_emit_regs(cs, A6XX_GRAS_LRZ_CNTL(0)); tu_cs_emit_regs(cs, A6XX_RB_LRZ_CNTL(0)); @@ -1177,18 +1271,23 @@ struct blit_ops { struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer, - VkFilter filter); + VkFilter filter, + enum pipe_format dst_format); void (*src_buffer)(struct tu_cmd_buffer *cmd, struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch, - uint32_t width, uint32_t height); - void (*dst)(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer); + uint32_t width, uint32_t height, + enum pipe_format dst_format); + void (*dst)(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer, + enum pipe_format src_format); void (*dst_depth)(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer); void (*dst_stencil)(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer); - void (*dst_buffer)(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch); + void (*dst_buffer)(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch, + enum pipe_format src_format); void (*setup)(struct tu_cmd_buffer *cmd, struct tu_cs *cs, - enum pipe_format format, + enum pipe_format src_format, + enum pipe_format dst_format, VkImageAspectFlags aspect_mask, unsigned blit_param, /* CmdBlitImage: rotation in 2D path and z scaling in 3D path */ bool clear, @@ -1244,7 +1343,7 @@ coords(const struct blit_ops *ops, * everything. */ static enum pipe_format -copy_format(VkFormat vk_format, VkImageAspectFlags aspect_mask, bool copy_buffer) +copy_format(VkFormat vk_format, VkImageAspectFlags aspect_mask) { if (vk_format_is_compressed(vk_format)) { switch (vk_format_get_blocksize(vk_format)) { @@ -1279,12 +1378,6 @@ copy_format(VkFormat vk_format, VkImageAspectFlags aspect_mask, bool copy_buffer case PIPE_FORMAT_G8_B8_R8_420_UNORM: return PIPE_FORMAT_R8_UNORM; - case PIPE_FORMAT_Z24_UNORM_S8_UINT: - if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT && copy_buffer) - return PIPE_FORMAT_R8_UNORM; - else - return format; - case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) return PIPE_FORMAT_S8_UINT; @@ -1313,12 +1406,13 @@ tu6_clear_lrz(struct tu_cmd_buffer *cmd, */ tu6_emit_event_write(cmd, &cmd->cs, CACHE_FLUSH_TS); - ops->setup(cmd, cs, PIPE_FORMAT_Z16_UNORM, VK_IMAGE_ASPECT_DEPTH_BIT, 0, true, false, + ops->setup(cmd, cs, PIPE_FORMAT_Z16_UNORM, PIPE_FORMAT_Z16_UNORM, + VK_IMAGE_ASPECT_DEPTH_BIT, 0, true, false, VK_SAMPLE_COUNT_1_BIT); ops->clear_value(cs, PIPE_FORMAT_Z16_UNORM, value); ops->dst_buffer(cs, PIPE_FORMAT_Z16_UNORM, image->iova + image->lrz_offset, - image->lrz_pitch * 2); + image->lrz_pitch * 2, PIPE_FORMAT_Z16_UNORM); ops->coords(cs, &(VkOffset2D) {}, NULL, &(VkExtent2D) {image->lrz_pitch, image->lrz_height}); ops->run(cmd, cs); ops->teardown(cmd, cs); @@ -1340,11 +1434,13 @@ tu6_dirty_lrz_fc(struct tu_cmd_buffer *cmd, VkClearValue clear = { .color = { .uint32[0] = 0xffffffff } }; /* LRZ fast-clear buffer is always allocated with 512 bytes size. */ - ops->setup(cmd, cs, PIPE_FORMAT_R32_UINT, VK_IMAGE_ASPECT_COLOR_BIT, 0, true, false, + ops->setup(cmd, cs, PIPE_FORMAT_R32_UINT, PIPE_FORMAT_R32_UINT, + VK_IMAGE_ASPECT_COLOR_BIT, 0, true, false, VK_SAMPLE_COUNT_1_BIT); ops->clear_value(cs, PIPE_FORMAT_R32_UINT, &clear); ops->dst_buffer(cs, PIPE_FORMAT_R32_UINT, - image->iova + image->lrz_fc_offset, 512); + image->iova + image->lrz_fc_offset, 512, + PIPE_FORMAT_R32_UINT); ops->coords(cs, &(VkOffset2D) {}, NULL, &(VkExtent2D) {128, 1}); ops->run(cmd, cs); ops->teardown(cmd, cs); @@ -1356,7 +1452,6 @@ tu_image_view_copy_blit(struct fdl6_view *iview, enum pipe_format format, const VkImageSubresourceLayers *subres, uint32_t layer, - bool stencil_read, bool z_scale) { VkImageAspectFlags aspect_mask = subres->aspectMask; @@ -1378,9 +1473,7 @@ tu_image_view_copy_blit(struct fdl6_view *iview, .level_count = 1, .format = tu_format_for_aspect(format, aspect_mask), .swiz = { - /* image_to_buffer from d24s8 with stencil aspect mask writes out to r8 */ - stencil_read ? PIPE_SWIZZLE_W : PIPE_SWIZZLE_X, - PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W + PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W }, .type = z_scale ? FDL_VIEW_TYPE_3D : FDL_VIEW_TYPE_2D, }, false); @@ -1391,10 +1484,9 @@ tu_image_view_copy(struct fdl6_view *iview, struct tu_image *image, enum pipe_format format, const VkImageSubresourceLayers *subres, - uint32_t layer, - bool stencil_read) + uint32_t layer) { - tu_image_view_copy_blit(iview, image, format, subres, layer, stencil_read, false); + tu_image_view_copy_blit(iview, image, format, subres, layer, false); } static void @@ -1406,7 +1498,7 @@ tu_image_view_blit(struct fdl6_view *iview, enum pipe_format format = tu6_plane_format(image->vk_format, tu6_plane_index(image->vk_format, subres->aspectMask)); - tu_image_view_copy_blit(iview, image, format, subres, layer, false, false); + tu_image_view_copy_blit(iview, image, format, subres, layer, false); } static void @@ -1473,19 +1565,17 @@ tu6_blit_image(struct tu_cmd_buffer *cmd, /* use the right format in setup() for D32_S8 * TODO: this probably should use a helper */ - VkFormat format = dst_image->vk_format; - if (format == VK_FORMAT_D32_SFLOAT_S8_UINT) { - if (info->dstSubresource.aspectMask == VK_IMAGE_ASPECT_DEPTH_BIT) - format = VK_FORMAT_D32_SFLOAT; - else if (info->dstSubresource.aspectMask == VK_IMAGE_ASPECT_STENCIL_BIT) - format = VK_FORMAT_S8_UINT; - else - unreachable("unexpected D32_S8 aspect mask in blit_image"); - } - + enum pipe_format src_format = + tu6_plane_format(src_image->vk_format, + tu6_plane_index(src_image->vk_format, + info->srcSubresource.aspectMask)); + enum pipe_format dst_format = + tu6_plane_format(dst_image->vk_format, + tu6_plane_index(src_image->vk_format, + info->srcSubresource.aspectMask)); trace_start_blit(&cmd->trace, cs); - ops->setup(cmd, cs, tu_vk_format_to_pipe_format(format), info->dstSubresource.aspectMask, + ops->setup(cmd, cs, src_format, dst_format, info->dstSubresource.aspectMask, blit_param, false, dst_image->layout[0].ubwc, dst_image->layout[0].nr_samples); @@ -1514,13 +1604,9 @@ tu6_blit_image(struct tu_cmd_buffer *cmd, MIN2(info->dstOffsets[0].z, info->dstOffsets[1].z)); if (z_scale) { - enum pipe_format src_format = - tu6_plane_format(src_image->vk_format, - tu6_plane_index(src_image->vk_format, - info->srcSubresource.aspectMask)); tu_image_view_copy_blit(&src, src_image, src_format, - &info->srcSubresource, 0, false, true); - ops->src(cmd, cs, &src, 0, filter); + &info->srcSubresource, 0, true); + ops->src(cmd, cs, &src, 0, filter, dst_format); } else { tu_image_view_blit(&src, src_image, &info->srcSubresource, info->srcOffsets[0].z); } @@ -1530,9 +1616,9 @@ tu6_blit_image(struct tu_cmd_buffer *cmd, float t = ((float) i + 0.5f) / (float) layers; r3d_coord_z(cs, t * (src1_z - src0_z) + src0_z); } else { - ops->src(cmd, cs, &src, i, filter); + ops->src(cmd, cs, &src, i, filter, dst_format); } - ops->dst(cs, &dst, i); + ops->dst(cs, &dst, i, src_format); ops->run(cmd, cs); } @@ -1612,15 +1698,15 @@ tu_copy_buffer_to_image(struct tu_cmd_buffer *cmd, struct tu_cs *cs = &cmd->cs; uint32_t layers = MAX2(info->imageExtent.depth, info->imageSubresource.layerCount); enum pipe_format src_format = - copy_format(dst_image->vk_format, info->imageSubresource.aspectMask, true); + copy_format(dst_image->vk_format, info->imageSubresource.aspectMask); enum pipe_format dst_format = - copy_format(dst_image->vk_format, info->imageSubresource.aspectMask, false); + copy_format(dst_image->vk_format, info->imageSubresource.aspectMask); const struct blit_ops *ops = &r2d_ops; /* special case for buffer to stencil */ if (dst_image->vk_format == VK_FORMAT_D24_UNORM_S8_UINT && info->imageSubresource.aspectMask == VK_IMAGE_ASPECT_STENCIL_BIT) { - ops = &r3d_ops; + src_format = PIPE_FORMAT_S8_UINT; } /* note: could use "R8_UNORM" when no UBWC */ @@ -1637,29 +1723,29 @@ tu_copy_buffer_to_image(struct tu_cmd_buffer *cmd, uint32_t pitch = src_width * util_format_get_blocksize(src_format); uint32_t layer_size = src_height * pitch; - ops->setup(cmd, cs, dst_format, + ops->setup(cmd, cs, src_format, dst_format, info->imageSubresource.aspectMask, 0, false, dst_image->layout[0].ubwc, dst_image->layout[0].nr_samples); struct fdl6_view dst; - tu_image_view_copy(&dst, dst_image, dst_format, &info->imageSubresource, offset.z, false); + tu_image_view_copy(&dst, dst_image, dst_format, &info->imageSubresource, offset.z); for (uint32_t i = 0; i < layers; i++) { - ops->dst(cs, &dst, i); + ops->dst(cs, &dst, i, src_format); uint64_t src_va = src_buffer->iova + info->bufferOffset + layer_size * i; if ((src_va & 63) || (pitch & 63)) { for (uint32_t y = 0; y < extent.height; y++) { uint32_t x = (src_va & 63) / util_format_get_blocksize(src_format); ops->src_buffer(cmd, cs, src_format, src_va & ~63, pitch, - x + extent.width, 1); + x + extent.width, 1, dst_format); ops->coords(cs, &(VkOffset2D){offset.x, offset.y + y}, &(VkOffset2D){x}, &(VkExtent2D) {extent.width, 1}); ops->run(cmd, cs); src_va += pitch; } } else { - ops->src_buffer(cmd, cs, src_format, src_va, pitch, extent.width, extent.height); + ops->src_buffer(cmd, cs, src_format, src_va, pitch, extent.width, extent.height, dst_format); coords(ops, cs, &offset, &(VkOffset3D){}, &extent); ops->run(cmd, cs); } @@ -1694,16 +1780,14 @@ tu_copy_image_to_buffer(struct tu_cmd_buffer *cmd, struct tu_cs *cs = &cmd->cs; uint32_t layers = MAX2(info->imageExtent.depth, info->imageSubresource.layerCount); enum pipe_format dst_format = - copy_format(src_image->vk_format, info->imageSubresource.aspectMask, true); + copy_format(src_image->vk_format, info->imageSubresource.aspectMask); enum pipe_format src_format = - copy_format(src_image->vk_format, info->imageSubresource.aspectMask, false); + copy_format(src_image->vk_format, info->imageSubresource.aspectMask); const struct blit_ops *ops = &r2d_ops; - bool stencil_read = false; if (src_image->vk_format == VK_FORMAT_D24_UNORM_S8_UINT && info->imageSubresource.aspectMask == VK_IMAGE_ASPECT_STENCIL_BIT) { - ops = &r3d_ops; - stencil_read = true; + dst_format = PIPE_FORMAT_S8_UINT; } /* note: could use "R8_UNORM" when no UBWC */ @@ -1720,27 +1804,27 @@ tu_copy_image_to_buffer(struct tu_cmd_buffer *cmd, uint32_t pitch = dst_width * util_format_get_blocksize(dst_format); uint32_t layer_size = pitch * dst_height; - ops->setup(cmd, cs, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false, + ops->setup(cmd, cs, src_format, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false, VK_SAMPLE_COUNT_1_BIT); struct fdl6_view src; - tu_image_view_copy(&src, src_image, src_format, &info->imageSubresource, offset.z, stencil_read); + tu_image_view_copy(&src, src_image, src_format, &info->imageSubresource, offset.z); for (uint32_t i = 0; i < layers; i++) { - ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST); + ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, dst_format); uint64_t dst_va = dst_buffer->iova + info->bufferOffset + layer_size * i; if ((dst_va & 63) || (pitch & 63)) { for (uint32_t y = 0; y < extent.height; y++) { uint32_t x = (dst_va & 63) / util_format_get_blocksize(dst_format); - ops->dst_buffer(cs, dst_format, dst_va & ~63, 0); + ops->dst_buffer(cs, dst_format, dst_va & ~63, 0, src_format); ops->coords(cs, &(VkOffset2D) {x}, &(VkOffset2D){offset.x, offset.y + y}, &(VkExtent2D) {extent.width, 1}); ops->run(cmd, cs); dst_va += pitch; } } else { - ops->dst_buffer(cs, dst_format, dst_va, pitch); + ops->dst_buffer(cs, dst_format, dst_va, pitch, src_format); coords(ops, cs, &(VkOffset3D) {0, 0}, &offset, &extent); ops->run(cmd, cs); } @@ -1828,8 +1912,8 @@ tu_copy_image_to_image(struct tu_cmd_buffer *cmd, copy_compressed(src_image->vk_format, &src_offset, &extent, NULL, NULL); copy_compressed(dst_image->vk_format, &dst_offset, NULL, NULL, NULL); - enum pipe_format dst_format = copy_format(dst_image->vk_format, info->dstSubresource.aspectMask, false); - enum pipe_format src_format = copy_format(src_image->vk_format, info->srcSubresource.aspectMask, false); + enum pipe_format dst_format = copy_format(dst_image->vk_format, info->dstSubresource.aspectMask); + enum pipe_format src_format = copy_format(src_image->vk_format, info->srcSubresource.aspectMask); /* note: could use "R8_UNORM" when no UBWC */ if (dst_format == PIPE_FORMAT_Y8_UNORM || @@ -1876,8 +1960,8 @@ tu_copy_image_to_image(struct tu_cmd_buffer *cmd, struct fdl6_view dst, src; if (use_staging_blit) { - tu_image_view_copy(&dst, dst_image, dst_format, &info->dstSubresource, dst_offset.z, false); - tu_image_view_copy(&src, src_image, src_format, &info->srcSubresource, src_offset.z, false); + tu_image_view_copy(&dst, dst_image, dst_format, &info->dstSubresource, dst_offset.z); + tu_image_view_copy(&src, src_image, src_format, &info->srcSubresource, src_offset.z); struct fdl_layout staging_layout = { 0 }; VkOffset3D staging_offset = { 0 }; @@ -1918,13 +2002,13 @@ tu_copy_image_to_image(struct tu_cmd_buffer *cmd, .type = FDL_VIEW_TYPE_2D, }, false); - ops->setup(cmd, cs, src_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false, + ops->setup(cmd, cs, src_format, src_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false, dst_image->layout[0].nr_samples); coords(ops, cs, &staging_offset, &src_offset, &extent); for (uint32_t i = 0; i < layers_to_copy; i++) { - ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST); - ops->dst(cs, &staging, i); + ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, src_format); + ops->dst(cs, &staging, i, src_format); ops->run(cmd, cs); } @@ -1946,28 +2030,28 @@ tu_copy_image_to_image(struct tu_cmd_buffer *cmd, .type = FDL_VIEW_TYPE_2D, }, false); - ops->setup(cmd, cs, dst_format, info->dstSubresource.aspectMask, + ops->setup(cmd, cs, dst_format, dst_format, info->dstSubresource.aspectMask, 0, false, dst_image->layout[0].ubwc, dst_image->layout[0].nr_samples); coords(ops, cs, &dst_offset, &staging_offset, &extent); for (uint32_t i = 0; i < layers_to_copy; i++) { - ops->src(cmd, cs, &staging, i, VK_FILTER_NEAREST); - ops->dst(cs, &dst, i); + ops->src(cmd, cs, &staging, i, VK_FILTER_NEAREST, dst_format); + ops->dst(cs, &dst, i, dst_format); ops->run(cmd, cs); } } else { - tu_image_view_copy(&dst, dst_image, format, &info->dstSubresource, dst_offset.z, false); - tu_image_view_copy(&src, src_image, format, &info->srcSubresource, src_offset.z, false); + tu_image_view_copy(&dst, dst_image, format, &info->dstSubresource, dst_offset.z); + tu_image_view_copy(&src, src_image, format, &info->srcSubresource, src_offset.z); - ops->setup(cmd, cs, format, info->dstSubresource.aspectMask, + ops->setup(cmd, cs, format, format, info->dstSubresource.aspectMask, 0, false, dst_image->layout[0].ubwc, dst_image->layout[0].nr_samples); coords(ops, cs, &dst_offset, &src_offset, &extent); for (uint32_t i = 0; i < layers_to_copy; i++) { - ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST); - ops->dst(cs, &dst, i); + ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, format); + ops->dst(cs, &dst, i, format); ops->run(cmd, cs); } } @@ -2015,7 +2099,7 @@ copy_buffer(struct tu_cmd_buffer *cmd, enum pipe_format format = block_size == 4 ? PIPE_FORMAT_R32_UINT : PIPE_FORMAT_R8_UNORM; uint64_t blocks = size / block_size; - ops->setup(cmd, cs, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false, + ops->setup(cmd, cs, format, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false, VK_SAMPLE_COUNT_1_BIT); while (blocks) { @@ -2023,8 +2107,8 @@ copy_buffer(struct tu_cmd_buffer *cmd, uint32_t dst_x = (dst_va & 63) / block_size; uint32_t width = MIN2(MIN2(blocks, 0x4000 - src_x), 0x4000 - dst_x); - ops->src_buffer(cmd, cs, format, src_va & ~63, 0, src_x + width, 1); - ops->dst_buffer( cs, format, dst_va & ~63, 0); + ops->src_buffer(cmd, cs, format, src_va & ~63, 0, src_x + width, 1, format); + ops->dst_buffer( cs, format, dst_va & ~63, 0, format); ops->coords(cs, &(VkOffset2D) {dst_x}, &(VkOffset2D) {src_x}, &(VkExtent2D) {width, 1}); ops->run(cmd, cs); @@ -2092,7 +2176,8 @@ tu_CmdFillBuffer(VkCommandBuffer commandBuffer, uint64_t dst_va = buffer->iova + dstOffset; uint32_t blocks = fillSize / 4; - ops->setup(cmd, cs, PIPE_FORMAT_R32_UINT, VK_IMAGE_ASPECT_COLOR_BIT, 0, true, false, + ops->setup(cmd, cs, PIPE_FORMAT_R32_UINT, PIPE_FORMAT_R32_UINT, + VK_IMAGE_ASPECT_COLOR_BIT, 0, true, false, VK_SAMPLE_COUNT_1_BIT); ops->clear_value(cs, PIPE_FORMAT_R32_UINT, &(VkClearValue){.color = {.uint32[0] = data}}); @@ -2100,7 +2185,7 @@ tu_CmdFillBuffer(VkCommandBuffer commandBuffer, uint32_t dst_x = (dst_va & 63) / 4; uint32_t width = MIN2(blocks, 0x4000 - dst_x); - ops->dst_buffer(cs, PIPE_FORMAT_R32_UINT, dst_va & ~63, 0); + ops->dst_buffer(cs, PIPE_FORMAT_R32_UINT, dst_va & ~63, 0, PIPE_FORMAT_R32_UINT); ops->coords(cs, &(VkOffset2D) {dst_x}, NULL, &(VkExtent2D) {width, 1}); ops->run(cmd, cs); @@ -2121,7 +2206,11 @@ tu_CmdResolveImage2KHR(VkCommandBuffer commandBuffer, const struct blit_ops *ops = &r2d_ops; struct tu_cs *cs = &cmd->cs; - ops->setup(cmd, cs, tu_vk_format_to_pipe_format(dst_image->vk_format), + enum pipe_format src_format = + tu_vk_format_to_pipe_format(src_image->vk_format); + enum pipe_format dst_format = + tu_vk_format_to_pipe_format(dst_image->vk_format); + ops->setup(cmd, cs, src_format, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, dst_image->layout[0].ubwc, VK_SAMPLE_COUNT_1_BIT); @@ -2139,8 +2228,8 @@ tu_CmdResolveImage2KHR(VkCommandBuffer commandBuffer, tu_image_view_blit(&src, src_image, &info->srcSubresource, info->srcOffset.z); for (uint32_t i = 0; i < layers; i++) { - ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST); - ops->dst(cs, &dst, i); + ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, dst_format); + ops->dst(cs, &dst, i, src_format); ops->run(cmd, cs); } } @@ -2157,7 +2246,8 @@ tu_CmdResolveImage2KHR(VkCommandBuffer commandBuffer, static void resolve_sysmem(struct tu_cmd_buffer *cmd, struct tu_cs *cs, - VkFormat format, + VkFormat vk_src_format, + VkFormat vk_dst_format, const struct tu_image_view *src, const struct tu_image_view *dst, uint32_t layer_mask, @@ -2170,30 +2260,33 @@ resolve_sysmem(struct tu_cmd_buffer *cmd, trace_start_sysmem_resolve(&cmd->trace, cs); - ops->setup(cmd, cs, tu_vk_format_to_pipe_format(format), + enum pipe_format src_format = tu_vk_format_to_pipe_format(vk_src_format); + enum pipe_format dst_format = tu_vk_format_to_pipe_format(vk_dst_format); + + ops->setup(cmd, cs, src_format, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, dst->view.ubwc_enabled, VK_SAMPLE_COUNT_1_BIT); ops->coords(cs, &rect->offset, &rect->offset, &rect->extent); for_each_layer(i, layer_mask, layers) { if (src_separate_ds) { - if (format == VK_FORMAT_D32_SFLOAT) { + if (vk_src_format == VK_FORMAT_D32_SFLOAT) { r2d_src_depth(cmd, cs, src, i, VK_FILTER_NEAREST); } else { r2d_src_stencil(cmd, cs, src, i, VK_FILTER_NEAREST); } } else { - ops->src(cmd, cs, &src->view, i, VK_FILTER_NEAREST); + ops->src(cmd, cs, &src->view, i, VK_FILTER_NEAREST, dst_format); } if (dst_separate_ds) { - if (format == VK_FORMAT_D32_SFLOAT) { + if (vk_dst_format == VK_FORMAT_D32_SFLOAT) { ops->dst_depth(cs, dst, i); } else { ops->dst_stencil(cs, dst, i); } } else { - ops->dst(cs, &dst->view, i); + ops->dst(cs, &dst->view, i, src_format); } ops->run(cmd, cs); @@ -2201,7 +2294,7 @@ resolve_sysmem(struct tu_cmd_buffer *cmd, ops->teardown(cmd, cs); - trace_end_sysmem_resolve(&cmd->trace, cs, format); + trace_end_sysmem_resolve(&cmd->trace, cs, vk_dst_format); } void @@ -2221,14 +2314,14 @@ tu_resolve_sysmem(struct tu_cmd_buffer *cmd, bool dst_separate_ds = dst->image->vk_format == VK_FORMAT_D32_SFLOAT_S8_UINT; if (dst_separate_ds) { - resolve_sysmem(cmd, cs, VK_FORMAT_D32_SFLOAT, + resolve_sysmem(cmd, cs, VK_FORMAT_D32_SFLOAT, VK_FORMAT_D32_SFLOAT, src, dst, layer_mask, layers, rect, src_separate_ds, dst_separate_ds); - resolve_sysmem(cmd, cs, VK_FORMAT_S8_UINT, + resolve_sysmem(cmd, cs, VK_FORMAT_S8_UINT, VK_FORMAT_S8_UINT, src, dst, layer_mask, layers, rect, src_separate_ds, dst_separate_ds); } else { - resolve_sysmem(cmd, cs, dst->image->vk_format, + resolve_sysmem(cmd, cs, src->image->vk_format, dst->image->vk_format, src, dst, layer_mask, layers, rect, src_separate_ds, dst_separate_ds); } @@ -2260,7 +2353,7 @@ clear_image(struct tu_cmd_buffer *cmd, const struct blit_ops *ops = image->layout[0].nr_samples > 1 ? &r3d_ops : &r2d_ops; - ops->setup(cmd, cs, format, aspect_mask, 0, true, image->layout[0].ubwc, + ops->setup(cmd, cs, format, format, aspect_mask, 0, true, image->layout[0].ubwc, image->layout[0].nr_samples); if (image->vk_format == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32) ops->clear_value(cs, PIPE_FORMAT_R9G9B9E5_FLOAT, clear_value); @@ -2282,10 +2375,10 @@ clear_image(struct tu_cmd_buffer *cmd, .mipLevel = range->baseMipLevel + j, .baseArrayLayer = range->baseArrayLayer, .layerCount = 1, - }, 0, false, false); + }, 0, false); for (uint32_t i = 0; i < layer_count; i++) { - ops->dst(cs, &dst, i); + ops->dst(cs, &dst, i, format); ops->run(cmd, cs); } } @@ -2757,7 +2850,7 @@ clear_sysmem_attachment(struct tu_cmd_buffer *cmd, trace_start_sysmem_clear(&cmd->trace, cs); - ops->setup(cmd, cs, format, clear_mask, 0, true, iview->view.ubwc_enabled, + ops->setup(cmd, cs, format, format, clear_mask, 0, true, iview->view.ubwc_enabled, cmd->state.pass->attachments[a].samples); ops->coords(cs, &info->renderArea.offset, NULL, &info->renderArea.extent); ops->clear_value(cs, format, &info->pClearValues[a]); @@ -2770,7 +2863,7 @@ clear_sysmem_attachment(struct tu_cmd_buffer *cmd, ops->dst_stencil(cs, iview, i); } } else { - ops->dst(cs, &iview->view, i); + ops->dst(cs, &iview->view, i, format); } ops->run(cmd, cs); } @@ -3029,11 +3122,12 @@ store_cp_blit(struct tu_cmd_buffer *cmd, const struct tu_image_view *iview, uint32_t samples, bool separate_stencil, - enum pipe_format format, + enum pipe_format src_format, + enum pipe_format dst_format, uint32_t gmem_offset, uint32_t cpp) { - r2d_setup_common(cmd, cs, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, + r2d_setup_common(cmd, cs, src_format, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, iview->view.ubwc_enabled, true); if (iview->image->vk_format == VK_FORMAT_D32_SFLOAT_S8_UINT) { @@ -3043,17 +3137,21 @@ store_cp_blit(struct tu_cmd_buffer *cmd, r2d_dst_stencil(cs, iview, 0); } } else { - r2d_dst(cs, &iview->view, 0); + r2d_dst(cs, &iview->view, 0, src_format); } + enum a6xx_format fmt = tu6_format_texture(src_format, TILE6_2).fmt; + fixup_src_format(&src_format, dst_format, &fmt); + tu_cs_emit_regs(cs, A6XX_SP_PS_2D_SRC_INFO( - .color_format = tu6_format_texture(format, TILE6_2).fmt, + .color_format = fmt, + .color_swap = WZYX, .tile_mode = TILE6_2, - .srgb = util_format_is_srgb(format), + .srgb = util_format_is_srgb(src_format), .samples = tu_msaa_samples(samples), - .samples_average = !util_format_is_pure_integer(format) && - !util_format_is_depth_or_stencil(format), + .samples_average = !util_format_is_pure_integer(dst_format) && + !util_format_is_depth_or_stencil(dst_format), .unk20 = 1, .unk22 = 1), /* note: src size does not matter when not scaling */ @@ -3083,7 +3181,8 @@ store_3d_blit(struct tu_cmd_buffer *cmd, const struct tu_image_view *iview, uint32_t dst_samples, bool separate_stencil, - enum pipe_format format, + enum pipe_format src_format, + enum pipe_format dst_format, const VkRect2D *render_area, uint32_t gmem_offset, uint32_t cpp) @@ -3099,7 +3198,7 @@ store_3d_blit(struct tu_cmd_buffer *cmd, CP_REG_TO_SCRATCH_0_SCRATCH(0) | CP_REG_TO_SCRATCH_0_CNT(1 - 1)); - r3d_setup(cmd, cs, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, + r3d_setup(cmd, cs, src_format, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, iview->view.ubwc_enabled, dst_samples); r3d_coords(cs, &render_area->offset, &render_area->offset, &render_area->extent); @@ -3111,10 +3210,10 @@ store_3d_blit(struct tu_cmd_buffer *cmd, r3d_dst_stencil(cs, iview, 0); } } else { - r3d_dst(cs, &iview->view, 0); + r3d_dst(cs, &iview->view, 0, src_format); } - r3d_src_gmem(cmd, cs, iview, format, gmem_offset, cpp); + r3d_src_gmem(cmd, cs, iview, src_format, dst_format, gmem_offset, cpp); /* sync GMEM writes with CACHE. */ tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE); @@ -3198,13 +3297,21 @@ tu_store_gmem_attachment(struct tu_cmd_buffer *cmd, src->format == VK_FORMAT_D32_SFLOAT_S8_UINT && dst->format == VK_FORMAT_S8_UINT; + /* The fast path doesn't support picking out the last component of a D24S8 + * texture reinterpreted as RGBA8_UNORM. + */ + bool resolve_d24s8_s8 = + src->format == VK_FORMAT_D24_UNORM_S8_UINT && + dst->format == VK_FORMAT_S8_UINT; + bool store_common = dst->store && !resolve_d32s8_s8; bool store_separate_stencil = dst->store_stencil || resolve_d32s8_s8; trace_start_gmem_store(&cmd->trace, cs); /* use fast path when render area is aligned, except for unsupported resolve cases */ - if (!unaligned && (a == gmem_a || blit_can_resolve(dst->format))) { + if (!unaligned && !resolve_d24s8_s8 && + (a == gmem_a || blit_can_resolve(dst->format))) { if (store_common) tu_emit_blit(cmd, cs, iview, src, true, false); if (store_separate_stencil) @@ -3218,9 +3325,13 @@ tu_store_gmem_attachment(struct tu_cmd_buffer *cmd, return; } - enum pipe_format format = tu_vk_format_to_pipe_format(src->format); - if (format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) - format = PIPE_FORMAT_Z32_FLOAT; + enum pipe_format src_format = tu_vk_format_to_pipe_format(src->format); + if (src_format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) + src_format = PIPE_FORMAT_Z32_FLOAT; + + enum pipe_format dst_format = tu_vk_format_to_pipe_format(dst->format); + if (dst_format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) + dst_format = PIPE_FORMAT_Z32_FLOAT; if (dst->samples > 1) { /* If we hit this path, we have to disable draw states after every tile @@ -3234,23 +3345,24 @@ tu_store_gmem_attachment(struct tu_cmd_buffer *cmd, tu_disable_draw_states(cmd, cs); if (store_common) { - store_3d_blit(cmd, cs, iview, dst->samples, false, format, - render_area, src->gmem_offset, src->cpp); + store_3d_blit(cmd, cs, iview, dst->samples, false, src_format, + dst_format, render_area, src->gmem_offset, src->cpp); } if (store_separate_stencil) { store_3d_blit(cmd, cs, iview, dst->samples, true, PIPE_FORMAT_S8_UINT, - render_area, src->gmem_offset_stencil, src->samples); + PIPE_FORMAT_S8_UINT, render_area, + src->gmem_offset_stencil, src->samples); } } else { r2d_coords(cs, &render_area->offset, &render_area->offset, &render_area->extent); if (store_common) { - store_cp_blit(cmd, cs, iview, src->samples, false, format, - src->gmem_offset, src->cpp); + store_cp_blit(cmd, cs, iview, src->samples, false, src_format, + dst_format, src->gmem_offset, src->cpp); } if (store_separate_stencil) { store_cp_blit(cmd, cs, iview, src->samples, true, PIPE_FORMAT_S8_UINT, - src->gmem_offset_stencil, src->samples); + PIPE_FORMAT_S8_UINT, src->gmem_offset_stencil, src->samples); } }