diff --git a/src/freedreno/ci/freedreno-a618-fails.txt b/src/freedreno/ci/freedreno-a618-fails.txt index 615df96bd0d..86d36560e48 100644 --- a/src/freedreno/ci/freedreno-a618-fails.txt +++ b/src/freedreno/ci/freedreno-a618-fails.txt @@ -3,19 +3,14 @@ gmem-dEQP-VK.spirv_assembly.instruction.graphics.variable_pointers.graphics.writ gmem-dEQP-VK.spirv_assembly.instruction.graphics.variable_pointers.graphics.writes_two_buffers_vert,Fail # VK-GL-CTS 1.3.2.0 uprev -dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d24_unorm_s8_uint_separate_layouts.compatibility_depth_zero_stencil_zero_testing_stencil,Fail dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d32_sfloat_s8_uint_separate_layouts.compatibility_depth_zero_stencil_zero_testing_stencil,Fail gmem-dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d32_sfloat_s8_uint.compatibility_depth_zero_stencil_zero_testing_stencil,Fail # https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/3590 dEQP-VK.api.info.get_physical_device_properties2.memory_properties,Fail -dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d24_unorm_s8_uint.compatibility_depth_zero_stencil_zero_testing_stencil,Fail dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d32_sfloat_s8_uint.compatibility_depth_zero_stencil_zero_testing_stencil,Fail -bypass-dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d24_unorm_s8_uint.compatibility_depth_zero_stencil_zero_testing_stencil,Fail -bypass-dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d24_unorm_s8_uint_separate_layouts.compatibility_depth_zero_stencil_zero_testing_stencil,Fail - # https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/3759 # deqp-vk: ../src/freedreno/vulkan/tu_pipeline.c:3894: tu_pipeline_builder_init_graphics: Assertion `subpass->color_count == 0 || !create_info->pColorBlendState || subpass->color_count == create_info->pColorBlendState->attachmentCount' failed dEQP-VK.pipeline.monolithic.color_write_enable_maxa.cwe_after_bind.attachments4_more0,Crash diff --git a/src/freedreno/fdl/fd6_view.c b/src/freedreno/fdl/fd6_view.c index 34dd9a8d26c..00b35ea280d 100644 --- a/src/freedreno/fdl/fd6_view.c +++ b/src/freedreno/fdl/fd6_view.c @@ -249,6 +249,8 @@ fdl6_view_init(struct fdl6_view *view, const struct fdl_layout **layouts, storage_format = FMT6_8_8_8_8_UNORM; } + view->format = args->format; + memset(view->descriptor, 0, sizeof(view->descriptor)); view->descriptor[0] = diff --git a/src/freedreno/fdl/freedreno_layout.h b/src/freedreno/fdl/freedreno_layout.h index b6be49d207c..52be661d29e 100644 --- a/src/freedreno/fdl/freedreno_layout.h +++ b/src/freedreno/fdl/freedreno_layout.h @@ -293,6 +293,8 @@ struct fdl6_view { bool ubwc_enabled; + enum pipe_format format; + uint32_t descriptor[FDL6_TEX_CONST_DWORDS]; /* Descriptor for use as a storage image as opposed to a sampled image. diff --git a/src/freedreno/vulkan/tu_clear_blit.c b/src/freedreno/vulkan/tu_clear_blit.c index 9c1bf2efe96..8178cdb8beb 100644 --- a/src/freedreno/vulkan/tu_clear_blit.c +++ b/src/freedreno/vulkan/tu_clear_blit.c @@ -41,6 +41,8 @@ format_to_ifmt(enum pipe_format format) return R2D_FLOAT32; if (format == PIPE_FORMAT_S8_UINT) return R2D_INT8; + if (format == PIPE_FORMAT_A8_UNORM) + return R2D_UNORM8; /* use the size of the red channel to find the corresponding "ifmt" */ bool is_int = util_format_is_pure_integer(format); @@ -141,17 +143,59 @@ r2d_clear_value(struct tu_cs *cs, enum pipe_format format, const VkClearValue *v tu_cs_emit_array(cs, clear_value, 4); } +static void +fixup_src_format(enum pipe_format *src_format, enum pipe_format dst_format, + enum a6xx_format *fmt) +{ + /* When blitting S8 -> D24S8 or vice versa, we have to override S8, which + * is normally R8_UINT for sampling/blitting purposes, to a unorm format. + * We also have to move stencil, which is normally in the .w channel, into + * the right channel. Reintepreting the S8 texture as A8_UNORM solves both + * problems, and avoids using a swap, which seems to sometimes not work + * with a D24S8 source, or a texture swizzle which is only supported with + * the 3d path. Sometimes this blit happens on already-constructed + * fdl6_view's, e.g. for sysmem resolves, so this has to happen as a fixup. + */ + if (*src_format == PIPE_FORMAT_S8_UINT && + (dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT || + dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT_AS_R8G8B8A8)) { + *fmt = FMT6_A8_UNORM; + *src_format = PIPE_FORMAT_A8_UNORM; + } +} + +static void +fixup_dst_format(enum pipe_format src_format, enum pipe_format *dst_format, + enum a6xx_format *fmt) +{ + if (*dst_format == PIPE_FORMAT_S8_UINT && + (src_format == PIPE_FORMAT_Z24_UNORM_S8_UINT || + src_format == PIPE_FORMAT_Z24_UNORM_S8_UINT_AS_R8G8B8A8)) { + *dst_format = PIPE_FORMAT_A8_UNORM; + *fmt = FMT6_A8_UNORM; + } +} + static void r2d_src(struct tu_cmd_buffer *cmd, struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer, - VkFilter filter) + VkFilter filter, + enum pipe_format dst_format) { uint32_t src_info = iview->SP_PS_2D_SRC_INFO; if (filter != VK_FILTER_NEAREST) src_info |= A6XX_SP_PS_2D_SRC_INFO_FILTER; + enum a6xx_format fmt = (src_info & A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK); + enum pipe_format src_format = iview->format; + fixup_src_format(&src_format, dst_format, &fmt); + + src_info = + (src_info & ~A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK) | + A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT(fmt); + tu_cs_emit_pkt4(cs, REG_A6XX_SP_PS_2D_SRC_INFO, 5); tu_cs_emit(cs, src_info); tu_cs_emit(cs, iview->SP_PS_2D_SRC_SIZE); @@ -199,13 +243,16 @@ r2d_src_buffer(struct tu_cmd_buffer *cmd, struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch, - uint32_t width, uint32_t height) + uint32_t width, uint32_t height, + enum pipe_format dst_format) { struct tu_native_format fmt = tu6_format_texture(format, TILE6_LINEAR); + enum a6xx_format color_format = fmt.fmt; + fixup_src_format(&format, dst_format, &color_format); tu_cs_emit_regs(cs, A6XX_SP_PS_2D_SRC_INFO( - .color_format = fmt.fmt, + .color_format = color_format, .color_swap = fmt.swap, .srgb = util_format_is_srgb(format), .unk20 = 1, @@ -216,10 +263,18 @@ r2d_src_buffer(struct tu_cmd_buffer *cmd, } static void -r2d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer) +r2d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer, + enum pipe_format src_format) { + uint32_t dst_info = iview->RB_2D_DST_INFO; + enum a6xx_format fmt = dst_info & A6XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK; + enum pipe_format dst_format = iview->format; + fixup_dst_format(src_format, &dst_format, &fmt); + + dst_info = + (dst_info & ~A6XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK) | fmt; tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_DST_INFO, 4); - tu_cs_emit(cs, iview->RB_2D_DST_INFO); + tu_cs_emit(cs, dst_info); tu_cs_image_ref_2d(cs, iview, layer, false); tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_DST_FLAGS, 3); @@ -248,9 +303,13 @@ r2d_dst_stencil(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t la } static void -r2d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch) +r2d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch, + enum pipe_format src_format) { struct tu_native_format fmt = tu6_format_color(format, TILE6_LINEAR); + enum a6xx_format color_fmt = fmt.fmt; + fixup_dst_format(src_format, &format, &color_fmt); + fmt.fmt = color_fmt; tu_cs_emit_regs(cs, A6XX_RB_2D_DST_INFO( @@ -264,25 +323,27 @@ r2d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t static void r2d_setup_common(struct tu_cmd_buffer *cmd, struct tu_cs *cs, - enum pipe_format format, + enum pipe_format src_format, + enum pipe_format dst_format, VkImageAspectFlags aspect_mask, unsigned blit_param, bool clear, bool ubwc, bool scissor) { - enum a6xx_format fmt = tu6_base_format(format); - enum a6xx_2d_ifmt ifmt = format_to_ifmt(format); + enum a6xx_format fmt = tu6_base_format(dst_format); + fixup_dst_format(src_format, &dst_format, &fmt); + enum a6xx_2d_ifmt ifmt = format_to_ifmt(dst_format); uint32_t unknown_8c01 = 0; - if ((format == PIPE_FORMAT_Z24_UNORM_S8_UINT || - format == PIPE_FORMAT_Z24X8_UNORM) && ubwc) { + if ((dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT || + dst_format == PIPE_FORMAT_Z24X8_UNORM) && ubwc) { fmt = FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8; } /* note: the only format with partial clearing is D24S8 */ - if (format == PIPE_FORMAT_Z24_UNORM_S8_UINT) { + if (dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT) { /* preserve stencil channel */ if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) unknown_8c01 = 0x08000041; @@ -301,7 +362,7 @@ r2d_setup_common(struct tu_cmd_buffer *cmd, .d24s8 = fmt == FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8 && !clear, .color_format = fmt, .mask = 0xf, - .ifmt = util_format_is_srgb(format) ? R2D_UNORM8_SRGB : ifmt, + .ifmt = util_format_is_srgb(dst_format) ? R2D_UNORM8_SRGB : ifmt, ).value; tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_BLIT_CNTL, 1); @@ -314,17 +375,18 @@ r2d_setup_common(struct tu_cmd_buffer *cmd, fmt = FMT6_16_16_16_16_FLOAT; tu_cs_emit_regs(cs, A6XX_SP_2D_DST_FORMAT( - .sint = util_format_is_pure_sint(format), - .uint = util_format_is_pure_uint(format), + .sint = util_format_is_pure_sint(dst_format), + .uint = util_format_is_pure_uint(dst_format), .color_format = fmt, - .srgb = util_format_is_srgb(format), + .srgb = util_format_is_srgb(dst_format), .mask = 0xf)); } static void r2d_setup(struct tu_cmd_buffer *cmd, struct tu_cs *cs, - enum pipe_format format, + enum pipe_format src_format, + enum pipe_format dst_format, VkImageAspectFlags aspect_mask, unsigned blit_param, bool clear, @@ -337,7 +399,7 @@ r2d_setup(struct tu_cmd_buffer *cmd, tu_emit_cache_flush_ccu(cmd, cs, TU_CMD_CCU_SYSMEM); } - r2d_setup_common(cmd, cs, format, aspect_mask, blit_param, clear, ubwc, false); + r2d_setup_common(cmd, cs, src_format, dst_format, aspect_mask, blit_param, clear, ubwc, false); } static void @@ -893,9 +955,20 @@ r3d_src(struct tu_cmd_buffer *cmd, struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer, - VkFilter filter) + VkFilter filter, + enum pipe_format dst_format) { - r3d_src_common(cmd, cs, iview->descriptor, + uint32_t desc[A6XX_TEX_CONST_DWORDS]; + memcpy(desc, iview->descriptor, sizeof(desc)); + + enum a6xx_format fmt = (desc[0] & A6XX_TEX_CONST_0_FMT__MASK) >> + A6XX_TEX_CONST_0_FMT__SHIFT; + enum pipe_format src_format = iview->format; + fixup_src_format(&src_format, dst_format, &fmt); + desc[0] = (desc[0] & ~A6XX_TEX_CONST_0_FMT__MASK) | + A6XX_TEX_CONST_0_FMT(fmt); + + r3d_src_common(cmd, cs, desc, iview->layer_size * layer, iview->ubwc_layer_size * layer, filter); @@ -906,21 +979,23 @@ r3d_src_buffer(struct tu_cmd_buffer *cmd, struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch, - uint32_t width, uint32_t height) + uint32_t width, uint32_t height, + enum pipe_format dst_format) { uint32_t desc[A6XX_TEX_CONST_DWORDS]; struct tu_native_format fmt = tu6_format_texture(format, TILE6_LINEAR); + enum a6xx_format color_format = fmt.fmt; + fixup_src_format(&format, dst_format, &color_format); desc[0] = COND(util_format_is_srgb(format), A6XX_TEX_CONST_0_SRGB) | - A6XX_TEX_CONST_0_FMT(fmt.fmt) | + A6XX_TEX_CONST_0_FMT(color_format) | A6XX_TEX_CONST_0_SWAP(fmt.swap) | A6XX_TEX_CONST_0_SWIZ_X(A6XX_TEX_X) | - // XXX to swizzle into .w for stencil buffer_to_image - A6XX_TEX_CONST_0_SWIZ_Y(format == PIPE_FORMAT_R8_UNORM ? A6XX_TEX_X : A6XX_TEX_Y) | - A6XX_TEX_CONST_0_SWIZ_Z(format == PIPE_FORMAT_R8_UNORM ? A6XX_TEX_X : A6XX_TEX_Z) | - A6XX_TEX_CONST_0_SWIZ_W(format == PIPE_FORMAT_R8_UNORM ? A6XX_TEX_X : A6XX_TEX_W); + A6XX_TEX_CONST_0_SWIZ_Y(A6XX_TEX_Y) | + A6XX_TEX_CONST_0_SWIZ_Z(A6XX_TEX_Z) | + A6XX_TEX_CONST_0_SWIZ_W(A6XX_TEX_W); desc[1] = A6XX_TEX_CONST_1_WIDTH(width) | A6XX_TEX_CONST_1_HEIGHT(height); desc[2] = A6XX_TEX_CONST_2_PITCH(pitch) | @@ -939,17 +1014,21 @@ r3d_src_gmem(struct tu_cmd_buffer *cmd, struct tu_cs *cs, const struct tu_image_view *iview, enum pipe_format format, + enum pipe_format dst_format, uint32_t gmem_offset, uint32_t cpp) { uint32_t desc[A6XX_TEX_CONST_DWORDS]; memcpy(desc, iview->view.descriptor, sizeof(desc)); + enum a6xx_format fmt = tu6_format_texture(format, TILE6_LINEAR).fmt; + fixup_src_format(&format, dst_format, &fmt); + /* patch the format so that depth/stencil get the right format and swizzle */ desc[0] &= ~(A6XX_TEX_CONST_0_FMT__MASK | A6XX_TEX_CONST_0_SWIZ_X__MASK | A6XX_TEX_CONST_0_SWIZ_Y__MASK | A6XX_TEX_CONST_0_SWIZ_Z__MASK | A6XX_TEX_CONST_0_SWIZ_W__MASK); - desc[0] |= A6XX_TEX_CONST_0_FMT(tu6_format_texture(format, TILE6_2).fmt) | + desc[0] |= A6XX_TEX_CONST_0_FMT(fmt) | A6XX_TEX_CONST_0_SWIZ_X(A6XX_TEX_X) | A6XX_TEX_CONST_0_SWIZ_Y(A6XX_TEX_Y) | A6XX_TEX_CONST_0_SWIZ_Z(A6XX_TEX_Z) | @@ -971,10 +1050,19 @@ r3d_src_gmem(struct tu_cmd_buffer *cmd, } static void -r3d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer) +r3d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer, + enum pipe_format src_format) { + uint32_t mrt_buf_info = iview->RB_MRT_BUF_INFO; + + enum a6xx_format fmt = mrt_buf_info & A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK; + enum pipe_format dst_format = iview->format; + fixup_dst_format(src_format, &dst_format, &fmt); + mrt_buf_info = + (mrt_buf_info & ~A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK) | + A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT(fmt); tu_cs_emit_pkt4(cs, REG_A6XX_RB_MRT_BUF_INFO(0), 6); - tu_cs_emit(cs, iview->RB_MRT_BUF_INFO); + tu_cs_emit(cs, mrt_buf_info); tu_cs_image_ref(cs, iview, layer); tu_cs_emit(cs, 0); @@ -984,7 +1072,7 @@ r3d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer) /* Use color format from RB_MRT_BUF_INFO. This register is relevant for * FMT6_NV12_Y. */ - tu_cs_emit_regs(cs, A6XX_GRAS_LRZ_MRT_BUF_INFO_0(.color_format = iview->RB_MRT_BUF_INFO & 0xff)); + tu_cs_emit_regs(cs, A6XX_GRAS_LRZ_MRT_BUF_INFO_0(.color_format = fmt)); tu_cs_emit_regs(cs, A6XX_RB_RENDER_CNTL(.flag_mrts = iview->ubwc_enabled)); } @@ -1015,12 +1103,16 @@ r3d_dst_stencil(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t la } static void -r3d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch) +r3d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch, + enum pipe_format src_format) { struct tu_native_format fmt = tu6_format_color(format, TILE6_LINEAR); + enum a6xx_format color_fmt = fmt.fmt; + fixup_dst_format(src_format, &format, &color_fmt); + tu_cs_emit_regs(cs, - A6XX_RB_MRT_BUF_INFO(0, .color_format = fmt.fmt, .color_swap = fmt.swap), + A6XX_RB_MRT_BUF_INFO(0, .color_format = color_fmt, .color_swap = fmt.swap), A6XX_RB_MRT_PITCH(0, pitch), A6XX_RB_MRT_ARRAY_PITCH(0, 0), A6XX_RB_MRT_BASE(0, .qword = va), @@ -1049,17 +1141,19 @@ aspect_write_mask(enum pipe_format format, VkImageAspectFlags aspect_mask) static void r3d_setup(struct tu_cmd_buffer *cmd, struct tu_cs *cs, - enum pipe_format format, + enum pipe_format src_format, + enum pipe_format dst_format, VkImageAspectFlags aspect_mask, unsigned blit_param, bool clear, bool ubwc, VkSampleCountFlagBits samples) { - enum a6xx_format fmt = tu6_base_format(format); + enum a6xx_format fmt = tu6_base_format(dst_format); + fixup_dst_format(src_format, &dst_format, &fmt); - if ((format == PIPE_FORMAT_Z24_UNORM_S8_UINT || - format == PIPE_FORMAT_Z24X8_UNORM) && ubwc) { + if ((dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT || + dst_format == PIPE_FORMAT_Z24X8_UNORM) && ubwc) { fmt = FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8; } @@ -1099,13 +1193,13 @@ r3d_setup(struct tu_cmd_buffer *cmd, tu_cs_emit_regs(cs, A6XX_SP_FS_MRT_REG(0, .color_format = fmt, - .color_sint = util_format_is_pure_sint(format), - .color_uint = util_format_is_pure_uint(format))); + .color_sint = util_format_is_pure_sint(dst_format), + .color_uint = util_format_is_pure_uint(dst_format))); tu_cs_emit_regs(cs, A6XX_RB_MRT_CONTROL(0, - .component_enable = aspect_write_mask(format, aspect_mask))); - tu_cs_emit_regs(cs, A6XX_RB_SRGB_CNTL(util_format_is_srgb(format))); - tu_cs_emit_regs(cs, A6XX_SP_SRGB_CNTL(util_format_is_srgb(format))); + .component_enable = aspect_write_mask(dst_format, aspect_mask))); + tu_cs_emit_regs(cs, A6XX_RB_SRGB_CNTL(util_format_is_srgb(dst_format))); + tu_cs_emit_regs(cs, A6XX_SP_SRGB_CNTL(util_format_is_srgb(dst_format))); tu_cs_emit_regs(cs, A6XX_GRAS_LRZ_CNTL(0)); tu_cs_emit_regs(cs, A6XX_RB_LRZ_CNTL(0)); @@ -1177,18 +1271,23 @@ struct blit_ops { struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer, - VkFilter filter); + VkFilter filter, + enum pipe_format dst_format); void (*src_buffer)(struct tu_cmd_buffer *cmd, struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch, - uint32_t width, uint32_t height); - void (*dst)(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer); + uint32_t width, uint32_t height, + enum pipe_format dst_format); + void (*dst)(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer, + enum pipe_format src_format); void (*dst_depth)(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer); void (*dst_stencil)(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer); - void (*dst_buffer)(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch); + void (*dst_buffer)(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch, + enum pipe_format src_format); void (*setup)(struct tu_cmd_buffer *cmd, struct tu_cs *cs, - enum pipe_format format, + enum pipe_format src_format, + enum pipe_format dst_format, VkImageAspectFlags aspect_mask, unsigned blit_param, /* CmdBlitImage: rotation in 2D path and z scaling in 3D path */ bool clear, @@ -1244,7 +1343,7 @@ coords(const struct blit_ops *ops, * everything. */ static enum pipe_format -copy_format(VkFormat vk_format, VkImageAspectFlags aspect_mask, bool copy_buffer) +copy_format(VkFormat vk_format, VkImageAspectFlags aspect_mask) { if (vk_format_is_compressed(vk_format)) { switch (vk_format_get_blocksize(vk_format)) { @@ -1279,12 +1378,6 @@ copy_format(VkFormat vk_format, VkImageAspectFlags aspect_mask, bool copy_buffer case PIPE_FORMAT_G8_B8_R8_420_UNORM: return PIPE_FORMAT_R8_UNORM; - case PIPE_FORMAT_Z24_UNORM_S8_UINT: - if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT && copy_buffer) - return PIPE_FORMAT_R8_UNORM; - else - return format; - case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) return PIPE_FORMAT_S8_UINT; @@ -1313,12 +1406,13 @@ tu6_clear_lrz(struct tu_cmd_buffer *cmd, */ tu6_emit_event_write(cmd, &cmd->cs, CACHE_FLUSH_TS); - ops->setup(cmd, cs, PIPE_FORMAT_Z16_UNORM, VK_IMAGE_ASPECT_DEPTH_BIT, 0, true, false, + ops->setup(cmd, cs, PIPE_FORMAT_Z16_UNORM, PIPE_FORMAT_Z16_UNORM, + VK_IMAGE_ASPECT_DEPTH_BIT, 0, true, false, VK_SAMPLE_COUNT_1_BIT); ops->clear_value(cs, PIPE_FORMAT_Z16_UNORM, value); ops->dst_buffer(cs, PIPE_FORMAT_Z16_UNORM, image->iova + image->lrz_offset, - image->lrz_pitch * 2); + image->lrz_pitch * 2, PIPE_FORMAT_Z16_UNORM); ops->coords(cs, &(VkOffset2D) {}, NULL, &(VkExtent2D) {image->lrz_pitch, image->lrz_height}); ops->run(cmd, cs); ops->teardown(cmd, cs); @@ -1340,11 +1434,13 @@ tu6_dirty_lrz_fc(struct tu_cmd_buffer *cmd, VkClearValue clear = { .color = { .uint32[0] = 0xffffffff } }; /* LRZ fast-clear buffer is always allocated with 512 bytes size. */ - ops->setup(cmd, cs, PIPE_FORMAT_R32_UINT, VK_IMAGE_ASPECT_COLOR_BIT, 0, true, false, + ops->setup(cmd, cs, PIPE_FORMAT_R32_UINT, PIPE_FORMAT_R32_UINT, + VK_IMAGE_ASPECT_COLOR_BIT, 0, true, false, VK_SAMPLE_COUNT_1_BIT); ops->clear_value(cs, PIPE_FORMAT_R32_UINT, &clear); ops->dst_buffer(cs, PIPE_FORMAT_R32_UINT, - image->iova + image->lrz_fc_offset, 512); + image->iova + image->lrz_fc_offset, 512, + PIPE_FORMAT_R32_UINT); ops->coords(cs, &(VkOffset2D) {}, NULL, &(VkExtent2D) {128, 1}); ops->run(cmd, cs); ops->teardown(cmd, cs); @@ -1356,7 +1452,6 @@ tu_image_view_copy_blit(struct fdl6_view *iview, enum pipe_format format, const VkImageSubresourceLayers *subres, uint32_t layer, - bool stencil_read, bool z_scale) { VkImageAspectFlags aspect_mask = subres->aspectMask; @@ -1378,9 +1473,7 @@ tu_image_view_copy_blit(struct fdl6_view *iview, .level_count = 1, .format = tu_format_for_aspect(format, aspect_mask), .swiz = { - /* image_to_buffer from d24s8 with stencil aspect mask writes out to r8 */ - stencil_read ? PIPE_SWIZZLE_W : PIPE_SWIZZLE_X, - PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W + PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W }, .type = z_scale ? FDL_VIEW_TYPE_3D : FDL_VIEW_TYPE_2D, }, false); @@ -1391,10 +1484,9 @@ tu_image_view_copy(struct fdl6_view *iview, struct tu_image *image, enum pipe_format format, const VkImageSubresourceLayers *subres, - uint32_t layer, - bool stencil_read) + uint32_t layer) { - tu_image_view_copy_blit(iview, image, format, subres, layer, stencil_read, false); + tu_image_view_copy_blit(iview, image, format, subres, layer, false); } static void @@ -1406,7 +1498,7 @@ tu_image_view_blit(struct fdl6_view *iview, enum pipe_format format = tu6_plane_format(image->vk_format, tu6_plane_index(image->vk_format, subres->aspectMask)); - tu_image_view_copy_blit(iview, image, format, subres, layer, false, false); + tu_image_view_copy_blit(iview, image, format, subres, layer, false); } static void @@ -1473,19 +1565,17 @@ tu6_blit_image(struct tu_cmd_buffer *cmd, /* use the right format in setup() for D32_S8 * TODO: this probably should use a helper */ - VkFormat format = dst_image->vk_format; - if (format == VK_FORMAT_D32_SFLOAT_S8_UINT) { - if (info->dstSubresource.aspectMask == VK_IMAGE_ASPECT_DEPTH_BIT) - format = VK_FORMAT_D32_SFLOAT; - else if (info->dstSubresource.aspectMask == VK_IMAGE_ASPECT_STENCIL_BIT) - format = VK_FORMAT_S8_UINT; - else - unreachable("unexpected D32_S8 aspect mask in blit_image"); - } - + enum pipe_format src_format = + tu6_plane_format(src_image->vk_format, + tu6_plane_index(src_image->vk_format, + info->srcSubresource.aspectMask)); + enum pipe_format dst_format = + tu6_plane_format(dst_image->vk_format, + tu6_plane_index(src_image->vk_format, + info->srcSubresource.aspectMask)); trace_start_blit(&cmd->trace, cs); - ops->setup(cmd, cs, tu_vk_format_to_pipe_format(format), info->dstSubresource.aspectMask, + ops->setup(cmd, cs, src_format, dst_format, info->dstSubresource.aspectMask, blit_param, false, dst_image->layout[0].ubwc, dst_image->layout[0].nr_samples); @@ -1514,13 +1604,9 @@ tu6_blit_image(struct tu_cmd_buffer *cmd, MIN2(info->dstOffsets[0].z, info->dstOffsets[1].z)); if (z_scale) { - enum pipe_format src_format = - tu6_plane_format(src_image->vk_format, - tu6_plane_index(src_image->vk_format, - info->srcSubresource.aspectMask)); tu_image_view_copy_blit(&src, src_image, src_format, - &info->srcSubresource, 0, false, true); - ops->src(cmd, cs, &src, 0, filter); + &info->srcSubresource, 0, true); + ops->src(cmd, cs, &src, 0, filter, dst_format); } else { tu_image_view_blit(&src, src_image, &info->srcSubresource, info->srcOffsets[0].z); } @@ -1530,9 +1616,9 @@ tu6_blit_image(struct tu_cmd_buffer *cmd, float t = ((float) i + 0.5f) / (float) layers; r3d_coord_z(cs, t * (src1_z - src0_z) + src0_z); } else { - ops->src(cmd, cs, &src, i, filter); + ops->src(cmd, cs, &src, i, filter, dst_format); } - ops->dst(cs, &dst, i); + ops->dst(cs, &dst, i, src_format); ops->run(cmd, cs); } @@ -1612,15 +1698,15 @@ tu_copy_buffer_to_image(struct tu_cmd_buffer *cmd, struct tu_cs *cs = &cmd->cs; uint32_t layers = MAX2(info->imageExtent.depth, info->imageSubresource.layerCount); enum pipe_format src_format = - copy_format(dst_image->vk_format, info->imageSubresource.aspectMask, true); + copy_format(dst_image->vk_format, info->imageSubresource.aspectMask); enum pipe_format dst_format = - copy_format(dst_image->vk_format, info->imageSubresource.aspectMask, false); + copy_format(dst_image->vk_format, info->imageSubresource.aspectMask); const struct blit_ops *ops = &r2d_ops; /* special case for buffer to stencil */ if (dst_image->vk_format == VK_FORMAT_D24_UNORM_S8_UINT && info->imageSubresource.aspectMask == VK_IMAGE_ASPECT_STENCIL_BIT) { - ops = &r3d_ops; + src_format = PIPE_FORMAT_S8_UINT; } /* note: could use "R8_UNORM" when no UBWC */ @@ -1637,29 +1723,29 @@ tu_copy_buffer_to_image(struct tu_cmd_buffer *cmd, uint32_t pitch = src_width * util_format_get_blocksize(src_format); uint32_t layer_size = src_height * pitch; - ops->setup(cmd, cs, dst_format, + ops->setup(cmd, cs, src_format, dst_format, info->imageSubresource.aspectMask, 0, false, dst_image->layout[0].ubwc, dst_image->layout[0].nr_samples); struct fdl6_view dst; - tu_image_view_copy(&dst, dst_image, dst_format, &info->imageSubresource, offset.z, false); + tu_image_view_copy(&dst, dst_image, dst_format, &info->imageSubresource, offset.z); for (uint32_t i = 0; i < layers; i++) { - ops->dst(cs, &dst, i); + ops->dst(cs, &dst, i, src_format); uint64_t src_va = src_buffer->iova + info->bufferOffset + layer_size * i; if ((src_va & 63) || (pitch & 63)) { for (uint32_t y = 0; y < extent.height; y++) { uint32_t x = (src_va & 63) / util_format_get_blocksize(src_format); ops->src_buffer(cmd, cs, src_format, src_va & ~63, pitch, - x + extent.width, 1); + x + extent.width, 1, dst_format); ops->coords(cs, &(VkOffset2D){offset.x, offset.y + y}, &(VkOffset2D){x}, &(VkExtent2D) {extent.width, 1}); ops->run(cmd, cs); src_va += pitch; } } else { - ops->src_buffer(cmd, cs, src_format, src_va, pitch, extent.width, extent.height); + ops->src_buffer(cmd, cs, src_format, src_va, pitch, extent.width, extent.height, dst_format); coords(ops, cs, &offset, &(VkOffset3D){}, &extent); ops->run(cmd, cs); } @@ -1694,16 +1780,14 @@ tu_copy_image_to_buffer(struct tu_cmd_buffer *cmd, struct tu_cs *cs = &cmd->cs; uint32_t layers = MAX2(info->imageExtent.depth, info->imageSubresource.layerCount); enum pipe_format dst_format = - copy_format(src_image->vk_format, info->imageSubresource.aspectMask, true); + copy_format(src_image->vk_format, info->imageSubresource.aspectMask); enum pipe_format src_format = - copy_format(src_image->vk_format, info->imageSubresource.aspectMask, false); + copy_format(src_image->vk_format, info->imageSubresource.aspectMask); const struct blit_ops *ops = &r2d_ops; - bool stencil_read = false; if (src_image->vk_format == VK_FORMAT_D24_UNORM_S8_UINT && info->imageSubresource.aspectMask == VK_IMAGE_ASPECT_STENCIL_BIT) { - ops = &r3d_ops; - stencil_read = true; + dst_format = PIPE_FORMAT_S8_UINT; } /* note: could use "R8_UNORM" when no UBWC */ @@ -1720,27 +1804,27 @@ tu_copy_image_to_buffer(struct tu_cmd_buffer *cmd, uint32_t pitch = dst_width * util_format_get_blocksize(dst_format); uint32_t layer_size = pitch * dst_height; - ops->setup(cmd, cs, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false, + ops->setup(cmd, cs, src_format, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false, VK_SAMPLE_COUNT_1_BIT); struct fdl6_view src; - tu_image_view_copy(&src, src_image, src_format, &info->imageSubresource, offset.z, stencil_read); + tu_image_view_copy(&src, src_image, src_format, &info->imageSubresource, offset.z); for (uint32_t i = 0; i < layers; i++) { - ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST); + ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, dst_format); uint64_t dst_va = dst_buffer->iova + info->bufferOffset + layer_size * i; if ((dst_va & 63) || (pitch & 63)) { for (uint32_t y = 0; y < extent.height; y++) { uint32_t x = (dst_va & 63) / util_format_get_blocksize(dst_format); - ops->dst_buffer(cs, dst_format, dst_va & ~63, 0); + ops->dst_buffer(cs, dst_format, dst_va & ~63, 0, src_format); ops->coords(cs, &(VkOffset2D) {x}, &(VkOffset2D){offset.x, offset.y + y}, &(VkExtent2D) {extent.width, 1}); ops->run(cmd, cs); dst_va += pitch; } } else { - ops->dst_buffer(cs, dst_format, dst_va, pitch); + ops->dst_buffer(cs, dst_format, dst_va, pitch, src_format); coords(ops, cs, &(VkOffset3D) {0, 0}, &offset, &extent); ops->run(cmd, cs); } @@ -1828,8 +1912,8 @@ tu_copy_image_to_image(struct tu_cmd_buffer *cmd, copy_compressed(src_image->vk_format, &src_offset, &extent, NULL, NULL); copy_compressed(dst_image->vk_format, &dst_offset, NULL, NULL, NULL); - enum pipe_format dst_format = copy_format(dst_image->vk_format, info->dstSubresource.aspectMask, false); - enum pipe_format src_format = copy_format(src_image->vk_format, info->srcSubresource.aspectMask, false); + enum pipe_format dst_format = copy_format(dst_image->vk_format, info->dstSubresource.aspectMask); + enum pipe_format src_format = copy_format(src_image->vk_format, info->srcSubresource.aspectMask); /* note: could use "R8_UNORM" when no UBWC */ if (dst_format == PIPE_FORMAT_Y8_UNORM || @@ -1876,8 +1960,8 @@ tu_copy_image_to_image(struct tu_cmd_buffer *cmd, struct fdl6_view dst, src; if (use_staging_blit) { - tu_image_view_copy(&dst, dst_image, dst_format, &info->dstSubresource, dst_offset.z, false); - tu_image_view_copy(&src, src_image, src_format, &info->srcSubresource, src_offset.z, false); + tu_image_view_copy(&dst, dst_image, dst_format, &info->dstSubresource, dst_offset.z); + tu_image_view_copy(&src, src_image, src_format, &info->srcSubresource, src_offset.z); struct fdl_layout staging_layout = { 0 }; VkOffset3D staging_offset = { 0 }; @@ -1918,13 +2002,13 @@ tu_copy_image_to_image(struct tu_cmd_buffer *cmd, .type = FDL_VIEW_TYPE_2D, }, false); - ops->setup(cmd, cs, src_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false, + ops->setup(cmd, cs, src_format, src_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false, dst_image->layout[0].nr_samples); coords(ops, cs, &staging_offset, &src_offset, &extent); for (uint32_t i = 0; i < layers_to_copy; i++) { - ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST); - ops->dst(cs, &staging, i); + ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, src_format); + ops->dst(cs, &staging, i, src_format); ops->run(cmd, cs); } @@ -1946,28 +2030,28 @@ tu_copy_image_to_image(struct tu_cmd_buffer *cmd, .type = FDL_VIEW_TYPE_2D, }, false); - ops->setup(cmd, cs, dst_format, info->dstSubresource.aspectMask, + ops->setup(cmd, cs, dst_format, dst_format, info->dstSubresource.aspectMask, 0, false, dst_image->layout[0].ubwc, dst_image->layout[0].nr_samples); coords(ops, cs, &dst_offset, &staging_offset, &extent); for (uint32_t i = 0; i < layers_to_copy; i++) { - ops->src(cmd, cs, &staging, i, VK_FILTER_NEAREST); - ops->dst(cs, &dst, i); + ops->src(cmd, cs, &staging, i, VK_FILTER_NEAREST, dst_format); + ops->dst(cs, &dst, i, dst_format); ops->run(cmd, cs); } } else { - tu_image_view_copy(&dst, dst_image, format, &info->dstSubresource, dst_offset.z, false); - tu_image_view_copy(&src, src_image, format, &info->srcSubresource, src_offset.z, false); + tu_image_view_copy(&dst, dst_image, format, &info->dstSubresource, dst_offset.z); + tu_image_view_copy(&src, src_image, format, &info->srcSubresource, src_offset.z); - ops->setup(cmd, cs, format, info->dstSubresource.aspectMask, + ops->setup(cmd, cs, format, format, info->dstSubresource.aspectMask, 0, false, dst_image->layout[0].ubwc, dst_image->layout[0].nr_samples); coords(ops, cs, &dst_offset, &src_offset, &extent); for (uint32_t i = 0; i < layers_to_copy; i++) { - ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST); - ops->dst(cs, &dst, i); + ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, format); + ops->dst(cs, &dst, i, format); ops->run(cmd, cs); } } @@ -2015,7 +2099,7 @@ copy_buffer(struct tu_cmd_buffer *cmd, enum pipe_format format = block_size == 4 ? PIPE_FORMAT_R32_UINT : PIPE_FORMAT_R8_UNORM; uint64_t blocks = size / block_size; - ops->setup(cmd, cs, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false, + ops->setup(cmd, cs, format, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false, VK_SAMPLE_COUNT_1_BIT); while (blocks) { @@ -2023,8 +2107,8 @@ copy_buffer(struct tu_cmd_buffer *cmd, uint32_t dst_x = (dst_va & 63) / block_size; uint32_t width = MIN2(MIN2(blocks, 0x4000 - src_x), 0x4000 - dst_x); - ops->src_buffer(cmd, cs, format, src_va & ~63, 0, src_x + width, 1); - ops->dst_buffer( cs, format, dst_va & ~63, 0); + ops->src_buffer(cmd, cs, format, src_va & ~63, 0, src_x + width, 1, format); + ops->dst_buffer( cs, format, dst_va & ~63, 0, format); ops->coords(cs, &(VkOffset2D) {dst_x}, &(VkOffset2D) {src_x}, &(VkExtent2D) {width, 1}); ops->run(cmd, cs); @@ -2092,7 +2176,8 @@ tu_CmdFillBuffer(VkCommandBuffer commandBuffer, uint64_t dst_va = buffer->iova + dstOffset; uint32_t blocks = fillSize / 4; - ops->setup(cmd, cs, PIPE_FORMAT_R32_UINT, VK_IMAGE_ASPECT_COLOR_BIT, 0, true, false, + ops->setup(cmd, cs, PIPE_FORMAT_R32_UINT, PIPE_FORMAT_R32_UINT, + VK_IMAGE_ASPECT_COLOR_BIT, 0, true, false, VK_SAMPLE_COUNT_1_BIT); ops->clear_value(cs, PIPE_FORMAT_R32_UINT, &(VkClearValue){.color = {.uint32[0] = data}}); @@ -2100,7 +2185,7 @@ tu_CmdFillBuffer(VkCommandBuffer commandBuffer, uint32_t dst_x = (dst_va & 63) / 4; uint32_t width = MIN2(blocks, 0x4000 - dst_x); - ops->dst_buffer(cs, PIPE_FORMAT_R32_UINT, dst_va & ~63, 0); + ops->dst_buffer(cs, PIPE_FORMAT_R32_UINT, dst_va & ~63, 0, PIPE_FORMAT_R32_UINT); ops->coords(cs, &(VkOffset2D) {dst_x}, NULL, &(VkExtent2D) {width, 1}); ops->run(cmd, cs); @@ -2121,7 +2206,11 @@ tu_CmdResolveImage2KHR(VkCommandBuffer commandBuffer, const struct blit_ops *ops = &r2d_ops; struct tu_cs *cs = &cmd->cs; - ops->setup(cmd, cs, tu_vk_format_to_pipe_format(dst_image->vk_format), + enum pipe_format src_format = + tu_vk_format_to_pipe_format(src_image->vk_format); + enum pipe_format dst_format = + tu_vk_format_to_pipe_format(dst_image->vk_format); + ops->setup(cmd, cs, src_format, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, dst_image->layout[0].ubwc, VK_SAMPLE_COUNT_1_BIT); @@ -2139,8 +2228,8 @@ tu_CmdResolveImage2KHR(VkCommandBuffer commandBuffer, tu_image_view_blit(&src, src_image, &info->srcSubresource, info->srcOffset.z); for (uint32_t i = 0; i < layers; i++) { - ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST); - ops->dst(cs, &dst, i); + ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, dst_format); + ops->dst(cs, &dst, i, src_format); ops->run(cmd, cs); } } @@ -2157,7 +2246,8 @@ tu_CmdResolveImage2KHR(VkCommandBuffer commandBuffer, static void resolve_sysmem(struct tu_cmd_buffer *cmd, struct tu_cs *cs, - VkFormat format, + VkFormat vk_src_format, + VkFormat vk_dst_format, const struct tu_image_view *src, const struct tu_image_view *dst, uint32_t layer_mask, @@ -2170,30 +2260,33 @@ resolve_sysmem(struct tu_cmd_buffer *cmd, trace_start_sysmem_resolve(&cmd->trace, cs); - ops->setup(cmd, cs, tu_vk_format_to_pipe_format(format), + enum pipe_format src_format = tu_vk_format_to_pipe_format(vk_src_format); + enum pipe_format dst_format = tu_vk_format_to_pipe_format(vk_dst_format); + + ops->setup(cmd, cs, src_format, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, dst->view.ubwc_enabled, VK_SAMPLE_COUNT_1_BIT); ops->coords(cs, &rect->offset, &rect->offset, &rect->extent); for_each_layer(i, layer_mask, layers) { if (src_separate_ds) { - if (format == VK_FORMAT_D32_SFLOAT) { + if (vk_src_format == VK_FORMAT_D32_SFLOAT) { r2d_src_depth(cmd, cs, src, i, VK_FILTER_NEAREST); } else { r2d_src_stencil(cmd, cs, src, i, VK_FILTER_NEAREST); } } else { - ops->src(cmd, cs, &src->view, i, VK_FILTER_NEAREST); + ops->src(cmd, cs, &src->view, i, VK_FILTER_NEAREST, dst_format); } if (dst_separate_ds) { - if (format == VK_FORMAT_D32_SFLOAT) { + if (vk_dst_format == VK_FORMAT_D32_SFLOAT) { ops->dst_depth(cs, dst, i); } else { ops->dst_stencil(cs, dst, i); } } else { - ops->dst(cs, &dst->view, i); + ops->dst(cs, &dst->view, i, src_format); } ops->run(cmd, cs); @@ -2201,7 +2294,7 @@ resolve_sysmem(struct tu_cmd_buffer *cmd, ops->teardown(cmd, cs); - trace_end_sysmem_resolve(&cmd->trace, cs, format); + trace_end_sysmem_resolve(&cmd->trace, cs, vk_dst_format); } void @@ -2221,14 +2314,14 @@ tu_resolve_sysmem(struct tu_cmd_buffer *cmd, bool dst_separate_ds = dst->image->vk_format == VK_FORMAT_D32_SFLOAT_S8_UINT; if (dst_separate_ds) { - resolve_sysmem(cmd, cs, VK_FORMAT_D32_SFLOAT, + resolve_sysmem(cmd, cs, VK_FORMAT_D32_SFLOAT, VK_FORMAT_D32_SFLOAT, src, dst, layer_mask, layers, rect, src_separate_ds, dst_separate_ds); - resolve_sysmem(cmd, cs, VK_FORMAT_S8_UINT, + resolve_sysmem(cmd, cs, VK_FORMAT_S8_UINT, VK_FORMAT_S8_UINT, src, dst, layer_mask, layers, rect, src_separate_ds, dst_separate_ds); } else { - resolve_sysmem(cmd, cs, dst->image->vk_format, + resolve_sysmem(cmd, cs, src->image->vk_format, dst->image->vk_format, src, dst, layer_mask, layers, rect, src_separate_ds, dst_separate_ds); } @@ -2260,7 +2353,7 @@ clear_image(struct tu_cmd_buffer *cmd, const struct blit_ops *ops = image->layout[0].nr_samples > 1 ? &r3d_ops : &r2d_ops; - ops->setup(cmd, cs, format, aspect_mask, 0, true, image->layout[0].ubwc, + ops->setup(cmd, cs, format, format, aspect_mask, 0, true, image->layout[0].ubwc, image->layout[0].nr_samples); if (image->vk_format == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32) ops->clear_value(cs, PIPE_FORMAT_R9G9B9E5_FLOAT, clear_value); @@ -2282,10 +2375,10 @@ clear_image(struct tu_cmd_buffer *cmd, .mipLevel = range->baseMipLevel + j, .baseArrayLayer = range->baseArrayLayer, .layerCount = 1, - }, 0, false, false); + }, 0, false); for (uint32_t i = 0; i < layer_count; i++) { - ops->dst(cs, &dst, i); + ops->dst(cs, &dst, i, format); ops->run(cmd, cs); } } @@ -2757,7 +2850,7 @@ clear_sysmem_attachment(struct tu_cmd_buffer *cmd, trace_start_sysmem_clear(&cmd->trace, cs); - ops->setup(cmd, cs, format, clear_mask, 0, true, iview->view.ubwc_enabled, + ops->setup(cmd, cs, format, format, clear_mask, 0, true, iview->view.ubwc_enabled, cmd->state.pass->attachments[a].samples); ops->coords(cs, &info->renderArea.offset, NULL, &info->renderArea.extent); ops->clear_value(cs, format, &info->pClearValues[a]); @@ -2770,7 +2863,7 @@ clear_sysmem_attachment(struct tu_cmd_buffer *cmd, ops->dst_stencil(cs, iview, i); } } else { - ops->dst(cs, &iview->view, i); + ops->dst(cs, &iview->view, i, format); } ops->run(cmd, cs); } @@ -3029,11 +3122,12 @@ store_cp_blit(struct tu_cmd_buffer *cmd, const struct tu_image_view *iview, uint32_t samples, bool separate_stencil, - enum pipe_format format, + enum pipe_format src_format, + enum pipe_format dst_format, uint32_t gmem_offset, uint32_t cpp) { - r2d_setup_common(cmd, cs, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, + r2d_setup_common(cmd, cs, src_format, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, iview->view.ubwc_enabled, true); if (iview->image->vk_format == VK_FORMAT_D32_SFLOAT_S8_UINT) { @@ -3043,17 +3137,21 @@ store_cp_blit(struct tu_cmd_buffer *cmd, r2d_dst_stencil(cs, iview, 0); } } else { - r2d_dst(cs, &iview->view, 0); + r2d_dst(cs, &iview->view, 0, src_format); } + enum a6xx_format fmt = tu6_format_texture(src_format, TILE6_2).fmt; + fixup_src_format(&src_format, dst_format, &fmt); + tu_cs_emit_regs(cs, A6XX_SP_PS_2D_SRC_INFO( - .color_format = tu6_format_texture(format, TILE6_2).fmt, + .color_format = fmt, + .color_swap = WZYX, .tile_mode = TILE6_2, - .srgb = util_format_is_srgb(format), + .srgb = util_format_is_srgb(src_format), .samples = tu_msaa_samples(samples), - .samples_average = !util_format_is_pure_integer(format) && - !util_format_is_depth_or_stencil(format), + .samples_average = !util_format_is_pure_integer(dst_format) && + !util_format_is_depth_or_stencil(dst_format), .unk20 = 1, .unk22 = 1), /* note: src size does not matter when not scaling */ @@ -3083,7 +3181,8 @@ store_3d_blit(struct tu_cmd_buffer *cmd, const struct tu_image_view *iview, uint32_t dst_samples, bool separate_stencil, - enum pipe_format format, + enum pipe_format src_format, + enum pipe_format dst_format, const VkRect2D *render_area, uint32_t gmem_offset, uint32_t cpp) @@ -3099,7 +3198,7 @@ store_3d_blit(struct tu_cmd_buffer *cmd, CP_REG_TO_SCRATCH_0_SCRATCH(0) | CP_REG_TO_SCRATCH_0_CNT(1 - 1)); - r3d_setup(cmd, cs, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, + r3d_setup(cmd, cs, src_format, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, iview->view.ubwc_enabled, dst_samples); r3d_coords(cs, &render_area->offset, &render_area->offset, &render_area->extent); @@ -3111,10 +3210,10 @@ store_3d_blit(struct tu_cmd_buffer *cmd, r3d_dst_stencil(cs, iview, 0); } } else { - r3d_dst(cs, &iview->view, 0); + r3d_dst(cs, &iview->view, 0, src_format); } - r3d_src_gmem(cmd, cs, iview, format, gmem_offset, cpp); + r3d_src_gmem(cmd, cs, iview, src_format, dst_format, gmem_offset, cpp); /* sync GMEM writes with CACHE. */ tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE); @@ -3198,13 +3297,21 @@ tu_store_gmem_attachment(struct tu_cmd_buffer *cmd, src->format == VK_FORMAT_D32_SFLOAT_S8_UINT && dst->format == VK_FORMAT_S8_UINT; + /* The fast path doesn't support picking out the last component of a D24S8 + * texture reinterpreted as RGBA8_UNORM. + */ + bool resolve_d24s8_s8 = + src->format == VK_FORMAT_D24_UNORM_S8_UINT && + dst->format == VK_FORMAT_S8_UINT; + bool store_common = dst->store && !resolve_d32s8_s8; bool store_separate_stencil = dst->store_stencil || resolve_d32s8_s8; trace_start_gmem_store(&cmd->trace, cs); /* use fast path when render area is aligned, except for unsupported resolve cases */ - if (!unaligned && (a == gmem_a || blit_can_resolve(dst->format))) { + if (!unaligned && !resolve_d24s8_s8 && + (a == gmem_a || blit_can_resolve(dst->format))) { if (store_common) tu_emit_blit(cmd, cs, iview, src, true, false); if (store_separate_stencil) @@ -3218,9 +3325,13 @@ tu_store_gmem_attachment(struct tu_cmd_buffer *cmd, return; } - enum pipe_format format = tu_vk_format_to_pipe_format(src->format); - if (format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) - format = PIPE_FORMAT_Z32_FLOAT; + enum pipe_format src_format = tu_vk_format_to_pipe_format(src->format); + if (src_format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) + src_format = PIPE_FORMAT_Z32_FLOAT; + + enum pipe_format dst_format = tu_vk_format_to_pipe_format(dst->format); + if (dst_format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) + dst_format = PIPE_FORMAT_Z32_FLOAT; if (dst->samples > 1) { /* If we hit this path, we have to disable draw states after every tile @@ -3234,23 +3345,24 @@ tu_store_gmem_attachment(struct tu_cmd_buffer *cmd, tu_disable_draw_states(cmd, cs); if (store_common) { - store_3d_blit(cmd, cs, iview, dst->samples, false, format, - render_area, src->gmem_offset, src->cpp); + store_3d_blit(cmd, cs, iview, dst->samples, false, src_format, + dst_format, render_area, src->gmem_offset, src->cpp); } if (store_separate_stencil) { store_3d_blit(cmd, cs, iview, dst->samples, true, PIPE_FORMAT_S8_UINT, - render_area, src->gmem_offset_stencil, src->samples); + PIPE_FORMAT_S8_UINT, render_area, + src->gmem_offset_stencil, src->samples); } } else { r2d_coords(cs, &render_area->offset, &render_area->offset, &render_area->extent); if (store_common) { - store_cp_blit(cmd, cs, iview, src->samples, false, format, - src->gmem_offset, src->cpp); + store_cp_blit(cmd, cs, iview, src->samples, false, src_format, + dst_format, src->gmem_offset, src->cpp); } if (store_separate_stencil) { store_cp_blit(cmd, cs, iview, src->samples, true, PIPE_FORMAT_S8_UINT, - src->gmem_offset_stencil, src->samples); + PIPE_FORMAT_S8_UINT, src->gmem_offset_stencil, src->samples); } }