AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tu: Support resolving D24S8 -> S8

Browse Source 
This was missed when we added support for VK_KHR_depth_stencil_resolve.
There is a similar feature where the stencil aspect of a D24S8 can be
copied "tightly" in CopyImageToBuffer, but it used the texture swizzle
and so required the 3d path. To get it to work with the 2D path, which
is required for resolves, we have to instead use the A8_UNORM format,
which works for texture sampling even for tiled images. We also have to
reuse the pre-existing image views because subpass resolves work on
image views rather than images, whereas before the fixup was applied
while creating the image view. This means threading through the
corresponding "opposite" format through setup, src, and dst functions,
doing the fixup there (through some shared helpers), and then getting
every user to specify the right format. As a bonus, we no longer need to
force the 3d path for the CopyImageToBuffer and CopyBufferToImage
special cases.

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17684>
This commit is contained in:
Connor Abbott
2022-07-20 17:33:09 +02:00
committed by Marge Bot
parent e8b22afae9
commit d426ee6a99
4 changed files with 277 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/freedreno/ci/freedreno-a618-fails.txt
-5
View File
@@ -3,19 +3,14 @@ gmem-dEQP-VK.spirv_assembly.instruction.graphics.variable_pointers.graphics.writ
gmem-dEQP-VK.spirv_assembly.instruction.graphics.variable_pointers.graphics.writes_two_buffers_vert,Fail
# VK-GL-CTS 1.3.2.0 uprev
dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d24_unorm_s8_uint_separate_layouts.compatibility_depth_zero_stencil_zero_testing_stencil,Fail
dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d32_sfloat_s8_uint_separate_layouts.compatibility_depth_zero_stencil_zero_testing_stencil,Fail
gmem-dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d32_sfloat_s8_uint.compatibility_depth_zero_stencil_zero_testing_stencil,Fail
# https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/3590
dEQP-VK.api.info.get_physical_device_properties2.memory_properties,Fail
dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d24_unorm_s8_uint.compatibility_depth_zero_stencil_zero_testing_stencil,Fail
dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d32_sfloat_s8_uint.compatibility_depth_zero_stencil_zero_testing_stencil,Fail
bypass-dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d24_unorm_s8_uint.compatibility_depth_zero_stencil_zero_testing_stencil,Fail
bypass-dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_32_32.samples_2.d24_unorm_s8_uint_separate_layouts.compatibility_depth_zero_stencil_zero_testing_stencil,Fail
# https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/3759
# deqp-vk: ../src/freedreno/vulkan/tu_pipeline.c:3894: tu_pipeline_builder_init_graphics: Assertion `subpass->color_count == 0 || !create_info->pColorBlendState || subpass->color_count == create_info->pColorBlendState->attachmentCount' failed
dEQP-VK.pipeline.monolithic.color_write_enable_maxa.cwe_after_bind.attachments4_more0,Crash
src/freedreno/fdl/fd6_view.c
+2
View File
@@ -249,6 +249,8 @@ fdl6_view_init(struct fdl6_view *view, const struct fdl_layout **layouts,
storage_format = FMT6_8_8_8_8_UNORM;
}
view->format = args->format;
memset(view->descriptor, 0, sizeof(view->descriptor));
view->descriptor[0] =
src/freedreno/fdl/freedreno_layout.h
+2
View File
@@ -293,6 +293,8 @@ struct fdl6_view {
bool ubwc_enabled;
enum pipe_format format;
uint32_t descriptor[FDL6_TEX_CONST_DWORDS];
/* Descriptor for use as a storage image as opposed to a sampled image.
src/freedreno/vulkan/tu_clear_blit.c
+273 -161
View File
@@ -41,6 +41,8 @@ format_to_ifmt(enum pipe_format format)
return R2D_FLOAT32;
if (format == PIPE_FORMAT_S8_UINT)
return R2D_INT8;
if (format == PIPE_FORMAT_A8_UNORM)
return R2D_UNORM8;
/* use the size of the red channel to find the corresponding "ifmt" */
bool is_int = util_format_is_pure_integer(format);
@@ -141,17 +143,59 @@ r2d_clear_value(struct tu_cs *cs, enum pipe_format format, const VkClearValue *v
tu_cs_emit_array(cs, clear_value, 4);
}
static void
fixup_src_format(enum pipe_format *src_format, enum pipe_format dst_format,
enum a6xx_format *fmt)
{
/* When blitting S8 -> D24S8 or vice versa, we have to override S8, which
* is normally R8_UINT for sampling/blitting purposes, to a unorm format.
* We also have to move stencil, which is normally in the .w channel, into
* the right channel. Reintepreting the S8 texture as A8_UNORM solves both
* problems, and avoids using a swap, which seems to sometimes not work
* with a D24S8 source, or a texture swizzle which is only supported with
* the 3d path. Sometimes this blit happens on already-constructed
* fdl6_view's, e.g. for sysmem resolves, so this has to happen as a fixup.
*/
if (*src_format == PIPE_FORMAT_S8_UINT &&
(dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT_AS_R8G8B8A8)) {
*fmt = FMT6_A8_UNORM;
*src_format = PIPE_FORMAT_A8_UNORM;
}
}
static void
fixup_dst_format(enum pipe_format src_format, enum pipe_format *dst_format,
enum a6xx_format *fmt)
{
if (*dst_format == PIPE_FORMAT_S8_UINT &&
(src_format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
src_format == PIPE_FORMAT_Z24_UNORM_S8_UINT_AS_R8G8B8A8)) {
*dst_format = PIPE_FORMAT_A8_UNORM;
*fmt = FMT6_A8_UNORM;
}
}
static void
r2d_src(struct tu_cmd_buffer *cmd,
struct tu_cs *cs,
const struct fdl6_view *iview,
uint32_t layer,
VkFilter filter)
VkFilter filter,
enum pipe_format dst_format)
{
uint32_t src_info = iview->SP_PS_2D_SRC_INFO;
if (filter != VK_FILTER_NEAREST)
src_info |= A6XX_SP_PS_2D_SRC_INFO_FILTER;
enum a6xx_format fmt = (src_info & A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK);
enum pipe_format src_format = iview->format;
fixup_src_format(&src_format, dst_format, &fmt);
src_info =
(src_info & ~A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK) |
A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT(fmt);
tu_cs_emit_pkt4(cs, REG_A6XX_SP_PS_2D_SRC_INFO, 5);
tu_cs_emit(cs, src_info);
tu_cs_emit(cs, iview->SP_PS_2D_SRC_SIZE);
@@ -199,13 +243,16 @@ r2d_src_buffer(struct tu_cmd_buffer *cmd,
struct tu_cs *cs,
enum pipe_format format,
uint64_t va, uint32_t pitch,
uint32_t width, uint32_t height)
uint32_t width, uint32_t height,
enum pipe_format dst_format)
{
struct tu_native_format fmt = tu6_format_texture(format, TILE6_LINEAR);
enum a6xx_format color_format = fmt.fmt;
fixup_src_format(&format, dst_format, &color_format);
tu_cs_emit_regs(cs,
A6XX_SP_PS_2D_SRC_INFO(
.color_format = fmt.fmt,
.color_format = color_format,
.color_swap = fmt.swap,
.srgb = util_format_is_srgb(format),
.unk20 = 1,
@@ -216,10 +263,18 @@ r2d_src_buffer(struct tu_cmd_buffer *cmd,
}
static void
r2d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer)
r2d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer,
enum pipe_format src_format)
{
uint32_t dst_info = iview->RB_2D_DST_INFO;
enum a6xx_format fmt = dst_info & A6XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK;
enum pipe_format dst_format = iview->format;
fixup_dst_format(src_format, &dst_format, &fmt);
dst_info =
(dst_info & ~A6XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK) | fmt;
tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_DST_INFO, 4);
tu_cs_emit(cs, iview->RB_2D_DST_INFO);
tu_cs_emit(cs, dst_info);
tu_cs_image_ref_2d(cs, iview, layer, false);
tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_DST_FLAGS, 3);
@@ -248,9 +303,13 @@ r2d_dst_stencil(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t la
}
static void
r2d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch)
r2d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch,
enum pipe_format src_format)
{
struct tu_native_format fmt = tu6_format_color(format, TILE6_LINEAR);
enum a6xx_format color_fmt = fmt.fmt;
fixup_dst_format(src_format, &format, &color_fmt);
fmt.fmt = color_fmt;
tu_cs_emit_regs(cs,
A6XX_RB_2D_DST_INFO(
@@ -264,25 +323,27 @@ r2d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t
static void
r2d_setup_common(struct tu_cmd_buffer *cmd,
struct tu_cs *cs,
enum pipe_format format,
enum pipe_format src_format,
enum pipe_format dst_format,
VkImageAspectFlags aspect_mask,
unsigned blit_param,
bool clear,
bool ubwc,
bool scissor)
{
enum a6xx_format fmt = tu6_base_format(format);
enum a6xx_2d_ifmt ifmt = format_to_ifmt(format);
enum a6xx_format fmt = tu6_base_format(dst_format);
fixup_dst_format(src_format, &dst_format, &fmt);
enum a6xx_2d_ifmt ifmt = format_to_ifmt(dst_format);
uint32_t unknown_8c01 = 0;
if ((format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
format == PIPE_FORMAT_Z24X8_UNORM) && ubwc) {
if ((dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
dst_format == PIPE_FORMAT_Z24X8_UNORM) && ubwc) {
fmt = FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8;
}
/* note: the only format with partial clearing is D24S8 */
if (format == PIPE_FORMAT_Z24_UNORM_S8_UINT) {
if (dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT) {
/* preserve stencil channel */
if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT)
unknown_8c01 = 0x08000041;
@@ -301,7 +362,7 @@ r2d_setup_common(struct tu_cmd_buffer *cmd,
.d24s8 = fmt == FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8 && !clear,
.color_format = fmt,
.mask = 0xf,
.ifmt = util_format_is_srgb(format) ? R2D_UNORM8_SRGB : ifmt,
.ifmt = util_format_is_srgb(dst_format) ? R2D_UNORM8_SRGB : ifmt,
).value;
tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_BLIT_CNTL, 1);
@@ -314,17 +375,18 @@ r2d_setup_common(struct tu_cmd_buffer *cmd,
fmt = FMT6_16_16_16_16_FLOAT;
tu_cs_emit_regs(cs, A6XX_SP_2D_DST_FORMAT(
.sint = util_format_is_pure_sint(format),
.uint = util_format_is_pure_uint(format),
.sint = util_format_is_pure_sint(dst_format),
.uint = util_format_is_pure_uint(dst_format),
.color_format = fmt,
.srgb = util_format_is_srgb(format),
.srgb = util_format_is_srgb(dst_format),
.mask = 0xf));
}
static void
r2d_setup(struct tu_cmd_buffer *cmd,
struct tu_cs *cs,
enum pipe_format format,
enum pipe_format src_format,
enum pipe_format dst_format,
VkImageAspectFlags aspect_mask,
unsigned blit_param,
bool clear,
@@ -337,7 +399,7 @@ r2d_setup(struct tu_cmd_buffer *cmd,
tu_emit_cache_flush_ccu(cmd, cs, TU_CMD_CCU_SYSMEM);
}
r2d_setup_common(cmd, cs, format, aspect_mask, blit_param, clear, ubwc, false);
r2d_setup_common(cmd, cs, src_format, dst_format, aspect_mask, blit_param, clear, ubwc, false);
}
static void
@@ -893,9 +955,20 @@ r3d_src(struct tu_cmd_buffer *cmd,
struct tu_cs *cs,
const struct fdl6_view *iview,
uint32_t layer,
VkFilter filter)
VkFilter filter,
enum pipe_format dst_format)
{
r3d_src_common(cmd, cs, iview->descriptor,
uint32_t desc[A6XX_TEX_CONST_DWORDS];
memcpy(desc, iview->descriptor, sizeof(desc));
enum a6xx_format fmt = (desc[0] & A6XX_TEX_CONST_0_FMT__MASK) >>
A6XX_TEX_CONST_0_FMT__SHIFT;
enum pipe_format src_format = iview->format;
fixup_src_format(&src_format, dst_format, &fmt);
desc[0] = (desc[0] & ~A6XX_TEX_CONST_0_FMT__MASK) |
A6XX_TEX_CONST_0_FMT(fmt);
r3d_src_common(cmd, cs, desc,
iview->layer_size * layer,
iview->ubwc_layer_size * layer,
filter);
@@ -906,21 +979,23 @@ r3d_src_buffer(struct tu_cmd_buffer *cmd,
struct tu_cs *cs,
enum pipe_format format,
uint64_t va, uint32_t pitch,
uint32_t width, uint32_t height)
uint32_t width, uint32_t height,
enum pipe_format dst_format)
{
uint32_t desc[A6XX_TEX_CONST_DWORDS];
struct tu_native_format fmt = tu6_format_texture(format, TILE6_LINEAR);
enum a6xx_format color_format = fmt.fmt;
fixup_src_format(&format, dst_format, &color_format);
desc[0] =
COND(util_format_is_srgb(format), A6XX_TEX_CONST_0_SRGB) |
A6XX_TEX_CONST_0_FMT(fmt.fmt) |
A6XX_TEX_CONST_0_FMT(color_format) |
A6XX_TEX_CONST_0_SWAP(fmt.swap) |
A6XX_TEX_CONST_0_SWIZ_X(A6XX_TEX_X) |
// XXX to swizzle into .w for stencil buffer_to_image
A6XX_TEX_CONST_0_SWIZ_Y(format == PIPE_FORMAT_R8_UNORM ? A6XX_TEX_X : A6XX_TEX_Y) |
A6XX_TEX_CONST_0_SWIZ_Z(format == PIPE_FORMAT_R8_UNORM ? A6XX_TEX_X : A6XX_TEX_Z) |
A6XX_TEX_CONST_0_SWIZ_W(format == PIPE_FORMAT_R8_UNORM ? A6XX_TEX_X : A6XX_TEX_W);
A6XX_TEX_CONST_0_SWIZ_Y(A6XX_TEX_Y) |
A6XX_TEX_CONST_0_SWIZ_Z(A6XX_TEX_Z) |
A6XX_TEX_CONST_0_SWIZ_W(A6XX_TEX_W);
desc[1] = A6XX_TEX_CONST_1_WIDTH(width) | A6XX_TEX_CONST_1_HEIGHT(height);
desc[2] =
A6XX_TEX_CONST_2_PITCH(pitch) |
@@ -939,17 +1014,21 @@ r3d_src_gmem(struct tu_cmd_buffer *cmd,
struct tu_cs *cs,
const struct tu_image_view *iview,
enum pipe_format format,
enum pipe_format dst_format,
uint32_t gmem_offset,
uint32_t cpp)
{
uint32_t desc[A6XX_TEX_CONST_DWORDS];
memcpy(desc, iview->view.descriptor, sizeof(desc));
enum a6xx_format fmt = tu6_format_texture(format, TILE6_LINEAR).fmt;
fixup_src_format(&format, dst_format, &fmt);
/* patch the format so that depth/stencil get the right format and swizzle */
desc[0] &= ~(A6XX_TEX_CONST_0_FMT__MASK |
A6XX_TEX_CONST_0_SWIZ_X__MASK | A6XX_TEX_CONST_0_SWIZ_Y__MASK |
A6XX_TEX_CONST_0_SWIZ_Z__MASK | A6XX_TEX_CONST_0_SWIZ_W__MASK);
desc[0] |= A6XX_TEX_CONST_0_FMT(tu6_format_texture(format, TILE6_2).fmt) |
desc[0] |= A6XX_TEX_CONST_0_FMT(fmt) |
A6XX_TEX_CONST_0_SWIZ_X(A6XX_TEX_X) |
A6XX_TEX_CONST_0_SWIZ_Y(A6XX_TEX_Y) |
A6XX_TEX_CONST_0_SWIZ_Z(A6XX_TEX_Z) |
@@ -971,10 +1050,19 @@ r3d_src_gmem(struct tu_cmd_buffer *cmd,
}
static void
r3d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer)
r3d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer,
enum pipe_format src_format)
{
uint32_t mrt_buf_info = iview->RB_MRT_BUF_INFO;
enum a6xx_format fmt = mrt_buf_info & A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK;
enum pipe_format dst_format = iview->format;
fixup_dst_format(src_format, &dst_format, &fmt);
mrt_buf_info =
(mrt_buf_info & ~A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK) |
A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT(fmt);
tu_cs_emit_pkt4(cs, REG_A6XX_RB_MRT_BUF_INFO(0), 6);
tu_cs_emit(cs, iview->RB_MRT_BUF_INFO);
tu_cs_emit(cs, mrt_buf_info);
tu_cs_image_ref(cs, iview, layer);
tu_cs_emit(cs, 0);
@@ -984,7 +1072,7 @@ r3d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer)
/* Use color format from RB_MRT_BUF_INFO. This register is relevant for
* FMT6_NV12_Y.
*/
tu_cs_emit_regs(cs, A6XX_GRAS_LRZ_MRT_BUF_INFO_0(.color_format = iview->RB_MRT_BUF_INFO & 0xff));
tu_cs_emit_regs(cs, A6XX_GRAS_LRZ_MRT_BUF_INFO_0(.color_format = fmt));
tu_cs_emit_regs(cs, A6XX_RB_RENDER_CNTL(.flag_mrts = iview->ubwc_enabled));
}
@@ -1015,12 +1103,16 @@ r3d_dst_stencil(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t la
}
static void
r3d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch)
r3d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch,
enum pipe_format src_format)
{
struct tu_native_format fmt = tu6_format_color(format, TILE6_LINEAR);
enum a6xx_format color_fmt = fmt.fmt;
fixup_dst_format(src_format, &format, &color_fmt);
tu_cs_emit_regs(cs,
A6XX_RB_MRT_BUF_INFO(0, .color_format = fmt.fmt, .color_swap = fmt.swap),
A6XX_RB_MRT_BUF_INFO(0, .color_format = color_fmt, .color_swap = fmt.swap),
A6XX_RB_MRT_PITCH(0, pitch),
A6XX_RB_MRT_ARRAY_PITCH(0, 0),
A6XX_RB_MRT_BASE(0, .qword = va),
@@ -1049,17 +1141,19 @@ aspect_write_mask(enum pipe_format format, VkImageAspectFlags aspect_mask)
static void
r3d_setup(struct tu_cmd_buffer *cmd,
struct tu_cs *cs,
enum pipe_format format,
enum pipe_format src_format,
enum pipe_format dst_format,
VkImageAspectFlags aspect_mask,
unsigned blit_param,
bool clear,
bool ubwc,
VkSampleCountFlagBits samples)
{
enum a6xx_format fmt = tu6_base_format(format);
enum a6xx_format fmt = tu6_base_format(dst_format);
fixup_dst_format(src_format, &dst_format, &fmt);
if ((format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
format == PIPE_FORMAT_Z24X8_UNORM) && ubwc) {
if ((dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
dst_format == PIPE_FORMAT_Z24X8_UNORM) && ubwc) {
fmt = FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8;
}
@@ -1099,13 +1193,13 @@ r3d_setup(struct tu_cmd_buffer *cmd,
tu_cs_emit_regs(cs, A6XX_SP_FS_MRT_REG(0,
.color_format = fmt,
.color_sint = util_format_is_pure_sint(format),
.color_uint = util_format_is_pure_uint(format)));
.color_sint = util_format_is_pure_sint(dst_format),
.color_uint = util_format_is_pure_uint(dst_format)));
tu_cs_emit_regs(cs, A6XX_RB_MRT_CONTROL(0,
.component_enable = aspect_write_mask(format, aspect_mask)));
tu_cs_emit_regs(cs, A6XX_RB_SRGB_CNTL(util_format_is_srgb(format)));
tu_cs_emit_regs(cs, A6XX_SP_SRGB_CNTL(util_format_is_srgb(format)));
.component_enable = aspect_write_mask(dst_format, aspect_mask)));
tu_cs_emit_regs(cs, A6XX_RB_SRGB_CNTL(util_format_is_srgb(dst_format)));
tu_cs_emit_regs(cs, A6XX_SP_SRGB_CNTL(util_format_is_srgb(dst_format)));
tu_cs_emit_regs(cs, A6XX_GRAS_LRZ_CNTL(0));
tu_cs_emit_regs(cs, A6XX_RB_LRZ_CNTL(0));
@@ -1177,18 +1271,23 @@ struct blit_ops {
struct tu_cs *cs,
const struct fdl6_view *iview,
uint32_t layer,
VkFilter filter);
VkFilter filter,
enum pipe_format dst_format);
void (*src_buffer)(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
enum pipe_format format,
uint64_t va, uint32_t pitch,
uint32_t width, uint32_t height);
void (*dst)(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer);
uint32_t width, uint32_t height,
enum pipe_format dst_format);
void (*dst)(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer,
enum pipe_format src_format);
void (*dst_depth)(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
void (*dst_stencil)(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
void (*dst_buffer)(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch);
void (*dst_buffer)(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch,
enum pipe_format src_format);
void (*setup)(struct tu_cmd_buffer *cmd,
struct tu_cs *cs,
enum pipe_format format,
enum pipe_format src_format,
enum pipe_format dst_format,
VkImageAspectFlags aspect_mask,
unsigned blit_param, /* CmdBlitImage: rotation in 2D path and z scaling in 3D path */
bool clear,
@@ -1244,7 +1343,7 @@ coords(const struct blit_ops *ops,
* everything.
*/
static enum pipe_format
copy_format(VkFormat vk_format, VkImageAspectFlags aspect_mask, bool copy_buffer)
copy_format(VkFormat vk_format, VkImageAspectFlags aspect_mask)
{
if (vk_format_is_compressed(vk_format)) {
switch (vk_format_get_blocksize(vk_format)) {
@@ -1279,12 +1378,6 @@ copy_format(VkFormat vk_format, VkImageAspectFlags aspect_mask, bool copy_buffer
case PIPE_FORMAT_G8_B8_R8_420_UNORM:
return PIPE_FORMAT_R8_UNORM;
case PIPE_FORMAT_Z24_UNORM_S8_UINT:
if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT && copy_buffer)
return PIPE_FORMAT_R8_UNORM;
else
return format;
case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT)
return PIPE_FORMAT_S8_UINT;
@@ -1313,12 +1406,13 @@ tu6_clear_lrz(struct tu_cmd_buffer *cmd,
*/
tu6_emit_event_write(cmd, &cmd->cs, CACHE_FLUSH_TS);
ops->setup(cmd, cs, PIPE_FORMAT_Z16_UNORM, VK_IMAGE_ASPECT_DEPTH_BIT, 0, true, false,
ops->setup(cmd, cs, PIPE_FORMAT_Z16_UNORM, PIPE_FORMAT_Z16_UNORM,
VK_IMAGE_ASPECT_DEPTH_BIT, 0, true, false,
VK_SAMPLE_COUNT_1_BIT);
ops->clear_value(cs, PIPE_FORMAT_Z16_UNORM, value);
ops->dst_buffer(cs, PIPE_FORMAT_Z16_UNORM,
image->iova + image->lrz_offset,
image->lrz_pitch * 2);
image->lrz_pitch * 2, PIPE_FORMAT_Z16_UNORM);
ops->coords(cs, &(VkOffset2D) {}, NULL, &(VkExtent2D) {image->lrz_pitch, image->lrz_height});
ops->run(cmd, cs);
ops->teardown(cmd, cs);
@@ -1340,11 +1434,13 @@ tu6_dirty_lrz_fc(struct tu_cmd_buffer *cmd,
VkClearValue clear = { .color = { .uint32[0] = 0xffffffff } };
/* LRZ fast-clear buffer is always allocated with 512 bytes size. */
ops->setup(cmd, cs, PIPE_FORMAT_R32_UINT, VK_IMAGE_ASPECT_COLOR_BIT, 0, true, false,
ops->setup(cmd, cs, PIPE_FORMAT_R32_UINT, PIPE_FORMAT_R32_UINT,
VK_IMAGE_ASPECT_COLOR_BIT, 0, true, false,
VK_SAMPLE_COUNT_1_BIT);
ops->clear_value(cs, PIPE_FORMAT_R32_UINT, &clear);
ops->dst_buffer(cs, PIPE_FORMAT_R32_UINT,
image->iova + image->lrz_fc_offset, 512);
image->iova + image->lrz_fc_offset, 512,
PIPE_FORMAT_R32_UINT);
ops->coords(cs, &(VkOffset2D) {}, NULL, &(VkExtent2D) {128, 1});
ops->run(cmd, cs);
ops->teardown(cmd, cs);
@@ -1356,7 +1452,6 @@ tu_image_view_copy_blit(struct fdl6_view *iview,
enum pipe_format format,
const VkImageSubresourceLayers *subres,
uint32_t layer,
bool stencil_read,
bool z_scale)
{
VkImageAspectFlags aspect_mask = subres->aspectMask;
@@ -1378,9 +1473,7 @@ tu_image_view_copy_blit(struct fdl6_view *iview,
.level_count = 1,
.format = tu_format_for_aspect(format, aspect_mask),
.swiz = {
/* image_to_buffer from d24s8 with stencil aspect mask writes out to r8 */
stencil_read ? PIPE_SWIZZLE_W : PIPE_SWIZZLE_X,
PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W
PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W
},
.type = z_scale ? FDL_VIEW_TYPE_3D : FDL_VIEW_TYPE_2D,
}, false);
@@ -1391,10 +1484,9 @@ tu_image_view_copy(struct fdl6_view *iview,
struct tu_image *image,
enum pipe_format format,
const VkImageSubresourceLayers *subres,
uint32_t layer,
bool stencil_read)
uint32_t layer)
{
tu_image_view_copy_blit(iview, image, format, subres, layer, stencil_read, false);
tu_image_view_copy_blit(iview, image, format, subres, layer, false);
}
static void
@@ -1406,7 +1498,7 @@ tu_image_view_blit(struct fdl6_view *iview,
enum pipe_format format =
tu6_plane_format(image->vk_format, tu6_plane_index(image->vk_format,
subres->aspectMask));
tu_image_view_copy_blit(iview, image, format, subres, layer, false, false);
tu_image_view_copy_blit(iview, image, format, subres, layer, false);
}
static void
@@ -1473,19 +1565,17 @@ tu6_blit_image(struct tu_cmd_buffer *cmd,
/* use the right format in setup() for D32_S8
* TODO: this probably should use a helper
*/
VkFormat format = dst_image->vk_format;
if (format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
if (info->dstSubresource.aspectMask == VK_IMAGE_ASPECT_DEPTH_BIT)
format = VK_FORMAT_D32_SFLOAT;
else if (info->dstSubresource.aspectMask == VK_IMAGE_ASPECT_STENCIL_BIT)
format = VK_FORMAT_S8_UINT;
else
unreachable("unexpected D32_S8 aspect mask in blit_image");
}
enum pipe_format src_format =
tu6_plane_format(src_image->vk_format,
tu6_plane_index(src_image->vk_format,
info->srcSubresource.aspectMask));
enum pipe_format dst_format =
tu6_plane_format(dst_image->vk_format,
tu6_plane_index(src_image->vk_format,
info->srcSubresource.aspectMask));
trace_start_blit(&cmd->trace, cs);
ops->setup(cmd, cs, tu_vk_format_to_pipe_format(format), info->dstSubresource.aspectMask,
ops->setup(cmd, cs, src_format, dst_format, info->dstSubresource.aspectMask,
blit_param, false, dst_image->layout[0].ubwc,
dst_image->layout[0].nr_samples);
@@ -1514,13 +1604,9 @@ tu6_blit_image(struct tu_cmd_buffer *cmd,
MIN2(info->dstOffsets[0].z, info->dstOffsets[1].z));
if (z_scale) {
enum pipe_format src_format =
tu6_plane_format(src_image->vk_format,
tu6_plane_index(src_image->vk_format,
info->srcSubresource.aspectMask));
tu_image_view_copy_blit(&src, src_image, src_format,
&info->srcSubresource, 0, false, true);
ops->src(cmd, cs, &src, 0, filter);
&info->srcSubresource, 0, true);
ops->src(cmd, cs, &src, 0, filter, dst_format);
} else {
tu_image_view_blit(&src, src_image, &info->srcSubresource, info->srcOffsets[0].z);
}
@@ -1530,9 +1616,9 @@ tu6_blit_image(struct tu_cmd_buffer *cmd,
float t = ((float) i + 0.5f) / (float) layers;
r3d_coord_z(cs, t * (src1_z - src0_z) + src0_z);
} else {
ops->src(cmd, cs, &src, i, filter);
ops->src(cmd, cs, &src, i, filter, dst_format);
}
ops->dst(cs, &dst, i);
ops->dst(cs, &dst, i, src_format);
ops->run(cmd, cs);
}
@@ -1612,15 +1698,15 @@ tu_copy_buffer_to_image(struct tu_cmd_buffer *cmd,
struct tu_cs *cs = &cmd->cs;
uint32_t layers = MAX2(info->imageExtent.depth, info->imageSubresource.layerCount);
enum pipe_format src_format =
copy_format(dst_image->vk_format, info->imageSubresource.aspectMask, true);
copy_format(dst_image->vk_format, info->imageSubresource.aspectMask);
enum pipe_format dst_format =
copy_format(dst_image->vk_format, info->imageSubresource.aspectMask, false);
copy_format(dst_image->vk_format, info->imageSubresource.aspectMask);
const struct blit_ops *ops = &r2d_ops;
/* special case for buffer to stencil */
if (dst_image->vk_format == VK_FORMAT_D24_UNORM_S8_UINT &&
info->imageSubresource.aspectMask == VK_IMAGE_ASPECT_STENCIL_BIT) {
ops = &r3d_ops;
src_format = PIPE_FORMAT_S8_UINT;
}
/* note: could use "R8_UNORM" when no UBWC */
@@ -1637,29 +1723,29 @@ tu_copy_buffer_to_image(struct tu_cmd_buffer *cmd,
uint32_t pitch = src_width * util_format_get_blocksize(src_format);
uint32_t layer_size = src_height * pitch;
ops->setup(cmd, cs, dst_format,
ops->setup(cmd, cs, src_format, dst_format,
info->imageSubresource.aspectMask, 0, false, dst_image->layout[0].ubwc,
dst_image->layout[0].nr_samples);
struct fdl6_view dst;
tu_image_view_copy(&dst, dst_image, dst_format, &info->imageSubresource, offset.z, false);
tu_image_view_copy(&dst, dst_image, dst_format, &info->imageSubresource, offset.z);
for (uint32_t i = 0; i < layers; i++) {
ops->dst(cs, &dst, i);
ops->dst(cs, &dst, i, src_format);
uint64_t src_va = src_buffer->iova + info->bufferOffset + layer_size * i;
if ((src_va & 63) || (pitch & 63)) {
for (uint32_t y = 0; y < extent.height; y++) {
uint32_t x = (src_va & 63) / util_format_get_blocksize(src_format);
ops->src_buffer(cmd, cs, src_format, src_va & ~63, pitch,
x + extent.width, 1);
x + extent.width, 1, dst_format);
ops->coords(cs, &(VkOffset2D){offset.x, offset.y + y}, &(VkOffset2D){x},
&(VkExtent2D) {extent.width, 1});
ops->run(cmd, cs);
src_va += pitch;
}
} else {
ops->src_buffer(cmd, cs, src_format, src_va, pitch, extent.width, extent.height);
ops->src_buffer(cmd, cs, src_format, src_va, pitch, extent.width, extent.height, dst_format);
coords(ops, cs, &offset, &(VkOffset3D){}, &extent);
ops->run(cmd, cs);
}
@@ -1694,16 +1780,14 @@ tu_copy_image_to_buffer(struct tu_cmd_buffer *cmd,
struct tu_cs *cs = &cmd->cs;
uint32_t layers = MAX2(info->imageExtent.depth, info->imageSubresource.layerCount);
enum pipe_format dst_format =
copy_format(src_image->vk_format, info->imageSubresource.aspectMask, true);
copy_format(src_image->vk_format, info->imageSubresource.aspectMask);
enum pipe_format src_format =
copy_format(src_image->vk_format, info->imageSubresource.aspectMask, false);
copy_format(src_image->vk_format, info->imageSubresource.aspectMask);
const struct blit_ops *ops = &r2d_ops;
bool stencil_read = false;
if (src_image->vk_format == VK_FORMAT_D24_UNORM_S8_UINT &&
info->imageSubresource.aspectMask == VK_IMAGE_ASPECT_STENCIL_BIT) {
ops = &r3d_ops;
stencil_read = true;
dst_format = PIPE_FORMAT_S8_UINT;
}
/* note: could use "R8_UNORM" when no UBWC */
@@ -1720,27 +1804,27 @@ tu_copy_image_to_buffer(struct tu_cmd_buffer *cmd,
uint32_t pitch = dst_width * util_format_get_blocksize(dst_format);
uint32_t layer_size = pitch * dst_height;
ops->setup(cmd, cs, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false,
ops->setup(cmd, cs, src_format, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false,
VK_SAMPLE_COUNT_1_BIT);
struct fdl6_view src;
tu_image_view_copy(&src, src_image, src_format, &info->imageSubresource, offset.z, stencil_read);
tu_image_view_copy(&src, src_image, src_format, &info->imageSubresource, offset.z);
for (uint32_t i = 0; i < layers; i++) {
ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST);
ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, dst_format);
uint64_t dst_va = dst_buffer->iova + info->bufferOffset + layer_size * i;
if ((dst_va & 63) || (pitch & 63)) {
for (uint32_t y = 0; y < extent.height; y++) {
uint32_t x = (dst_va & 63) / util_format_get_blocksize(dst_format);
ops->dst_buffer(cs, dst_format, dst_va & ~63, 0);
ops->dst_buffer(cs, dst_format, dst_va & ~63, 0, src_format);
ops->coords(cs, &(VkOffset2D) {x}, &(VkOffset2D){offset.x, offset.y + y},
&(VkExtent2D) {extent.width, 1});
ops->run(cmd, cs);
dst_va += pitch;
}
} else {
ops->dst_buffer(cs, dst_format, dst_va, pitch);
ops->dst_buffer(cs, dst_format, dst_va, pitch, src_format);
coords(ops, cs, &(VkOffset3D) {0, 0}, &offset, &extent);
ops->run(cmd, cs);
}
@@ -1828,8 +1912,8 @@ tu_copy_image_to_image(struct tu_cmd_buffer *cmd,
copy_compressed(src_image->vk_format, &src_offset, &extent, NULL, NULL);
copy_compressed(dst_image->vk_format, &dst_offset, NULL, NULL, NULL);
enum pipe_format dst_format = copy_format(dst_image->vk_format, info->dstSubresource.aspectMask, false);
enum pipe_format src_format = copy_format(src_image->vk_format, info->srcSubresource.aspectMask, false);
enum pipe_format dst_format = copy_format(dst_image->vk_format, info->dstSubresource.aspectMask);
enum pipe_format src_format = copy_format(src_image->vk_format, info->srcSubresource.aspectMask);
/* note: could use "R8_UNORM" when no UBWC */
if (dst_format == PIPE_FORMAT_Y8_UNORM ||
@@ -1876,8 +1960,8 @@ tu_copy_image_to_image(struct tu_cmd_buffer *cmd,
struct fdl6_view dst, src;
if (use_staging_blit) {
tu_image_view_copy(&dst, dst_image, dst_format, &info->dstSubresource, dst_offset.z, false);
tu_image_view_copy(&src, src_image, src_format, &info->srcSubresource, src_offset.z, false);
tu_image_view_copy(&dst, dst_image, dst_format, &info->dstSubresource, dst_offset.z);
tu_image_view_copy(&src, src_image, src_format, &info->srcSubresource, src_offset.z);
struct fdl_layout staging_layout = { 0 };
VkOffset3D staging_offset = { 0 };
@@ -1918,13 +2002,13 @@ tu_copy_image_to_image(struct tu_cmd_buffer *cmd,
.type = FDL_VIEW_TYPE_2D,
}, false);
ops->setup(cmd, cs, src_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false,
ops->setup(cmd, cs, src_format, src_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false,
dst_image->layout[0].nr_samples);
coords(ops, cs, &staging_offset, &src_offset, &extent);
for (uint32_t i = 0; i < layers_to_copy; i++) {
ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST);
ops->dst(cs, &staging, i);
ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, src_format);
ops->dst(cs, &staging, i, src_format);
ops->run(cmd, cs);
}
@@ -1946,28 +2030,28 @@ tu_copy_image_to_image(struct tu_cmd_buffer *cmd,
.type = FDL_VIEW_TYPE_2D,
}, false);
ops->setup(cmd, cs, dst_format, info->dstSubresource.aspectMask,
ops->setup(cmd, cs, dst_format, dst_format, info->dstSubresource.aspectMask,
0, false, dst_image->layout[0].ubwc,
dst_image->layout[0].nr_samples);
coords(ops, cs, &dst_offset, &staging_offset, &extent);
for (uint32_t i = 0; i < layers_to_copy; i++) {
ops->src(cmd, cs, &staging, i, VK_FILTER_NEAREST);
ops->dst(cs, &dst, i);
ops->src(cmd, cs, &staging, i, VK_FILTER_NEAREST, dst_format);
ops->dst(cs, &dst, i, dst_format);
ops->run(cmd, cs);
}
} else {
tu_image_view_copy(&dst, dst_image, format, &info->dstSubresource, dst_offset.z, false);
tu_image_view_copy(&src, src_image, format, &info->srcSubresource, src_offset.z, false);
tu_image_view_copy(&dst, dst_image, format, &info->dstSubresource, dst_offset.z);
tu_image_view_copy(&src, src_image, format, &info->srcSubresource, src_offset.z);
ops->setup(cmd, cs, format, info->dstSubresource.aspectMask,
ops->setup(cmd, cs, format, format, info->dstSubresource.aspectMask,
0, false, dst_image->layout[0].ubwc,
dst_image->layout[0].nr_samples);
coords(ops, cs, &dst_offset, &src_offset, &extent);
for (uint32_t i = 0; i < layers_to_copy; i++) {
ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST);
ops->dst(cs, &dst, i);
ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, format);
ops->dst(cs, &dst, i, format);
ops->run(cmd, cs);
}
}
@@ -2015,7 +2099,7 @@ copy_buffer(struct tu_cmd_buffer *cmd,
enum pipe_format format = block_size == 4 ? PIPE_FORMAT_R32_UINT : PIPE_FORMAT_R8_UNORM;
uint64_t blocks = size / block_size;
ops->setup(cmd, cs, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false,
ops->setup(cmd, cs, format, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false,
VK_SAMPLE_COUNT_1_BIT);
while (blocks) {
@@ -2023,8 +2107,8 @@ copy_buffer(struct tu_cmd_buffer *cmd,
uint32_t dst_x = (dst_va & 63) / block_size;
uint32_t width = MIN2(MIN2(blocks, 0x4000 - src_x), 0x4000 - dst_x);
ops->src_buffer(cmd, cs, format, src_va & ~63, 0, src_x + width, 1);
ops->dst_buffer( cs, format, dst_va & ~63, 0);
ops->src_buffer(cmd, cs, format, src_va & ~63, 0, src_x + width, 1, format);
ops->dst_buffer( cs, format, dst_va & ~63, 0, format);
ops->coords(cs, &(VkOffset2D) {dst_x}, &(VkOffset2D) {src_x}, &(VkExtent2D) {width, 1});
ops->run(cmd, cs);
@@ -2092,7 +2176,8 @@ tu_CmdFillBuffer(VkCommandBuffer commandBuffer,
uint64_t dst_va = buffer->iova + dstOffset;
uint32_t blocks = fillSize / 4;
ops->setup(cmd, cs, PIPE_FORMAT_R32_UINT, VK_IMAGE_ASPECT_COLOR_BIT, 0, true, false,
ops->setup(cmd, cs, PIPE_FORMAT_R32_UINT, PIPE_FORMAT_R32_UINT,
VK_IMAGE_ASPECT_COLOR_BIT, 0, true, false,
VK_SAMPLE_COUNT_1_BIT);
ops->clear_value(cs, PIPE_FORMAT_R32_UINT, &(VkClearValue){.color = {.uint32[0] = data}});
@@ -2100,7 +2185,7 @@ tu_CmdFillBuffer(VkCommandBuffer commandBuffer,
uint32_t dst_x = (dst_va & 63) / 4;
uint32_t width = MIN2(blocks, 0x4000 - dst_x);
ops->dst_buffer(cs, PIPE_FORMAT_R32_UINT, dst_va & ~63, 0);
ops->dst_buffer(cs, PIPE_FORMAT_R32_UINT, dst_va & ~63, 0, PIPE_FORMAT_R32_UINT);
ops->coords(cs, &(VkOffset2D) {dst_x}, NULL, &(VkExtent2D) {width, 1});
ops->run(cmd, cs);
@@ -2121,7 +2206,11 @@ tu_CmdResolveImage2KHR(VkCommandBuffer commandBuffer,
const struct blit_ops *ops = &r2d_ops;
struct tu_cs *cs = &cmd->cs;
ops->setup(cmd, cs, tu_vk_format_to_pipe_format(dst_image->vk_format),
enum pipe_format src_format =
tu_vk_format_to_pipe_format(src_image->vk_format);
enum pipe_format dst_format =
tu_vk_format_to_pipe_format(dst_image->vk_format);
ops->setup(cmd, cs, src_format, dst_format,
VK_IMAGE_ASPECT_COLOR_BIT, 0, false, dst_image->layout[0].ubwc,
VK_SAMPLE_COUNT_1_BIT);
@@ -2139,8 +2228,8 @@ tu_CmdResolveImage2KHR(VkCommandBuffer commandBuffer,
tu_image_view_blit(&src, src_image, &info->srcSubresource, info->srcOffset.z);
for (uint32_t i = 0; i < layers; i++) {
ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST);
ops->dst(cs, &dst, i);
ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, dst_format);
ops->dst(cs, &dst, i, src_format);
ops->run(cmd, cs);
}
}
@@ -2157,7 +2246,8 @@ tu_CmdResolveImage2KHR(VkCommandBuffer commandBuffer,
static void
resolve_sysmem(struct tu_cmd_buffer *cmd,
struct tu_cs *cs,
VkFormat format,
VkFormat vk_src_format,
VkFormat vk_dst_format,
const struct tu_image_view *src,
const struct tu_image_view *dst,
uint32_t layer_mask,
@@ -2170,30 +2260,33 @@ resolve_sysmem(struct tu_cmd_buffer *cmd,
trace_start_sysmem_resolve(&cmd->trace, cs);
ops->setup(cmd, cs, tu_vk_format_to_pipe_format(format),
enum pipe_format src_format = tu_vk_format_to_pipe_format(vk_src_format);
enum pipe_format dst_format = tu_vk_format_to_pipe_format(vk_dst_format);
ops->setup(cmd, cs, src_format, dst_format,
VK_IMAGE_ASPECT_COLOR_BIT, 0, false, dst->view.ubwc_enabled,
VK_SAMPLE_COUNT_1_BIT);
ops->coords(cs, &rect->offset, &rect->offset, &rect->extent);
for_each_layer(i, layer_mask, layers) {
if (src_separate_ds) {
if (format == VK_FORMAT_D32_SFLOAT) {
if (vk_src_format == VK_FORMAT_D32_SFLOAT) {
r2d_src_depth(cmd, cs, src, i, VK_FILTER_NEAREST);
} else {
r2d_src_stencil(cmd, cs, src, i, VK_FILTER_NEAREST);
}
} else {
ops->src(cmd, cs, &src->view, i, VK_FILTER_NEAREST);
ops->src(cmd, cs, &src->view, i, VK_FILTER_NEAREST, dst_format);
}
if (dst_separate_ds) {
if (format == VK_FORMAT_D32_SFLOAT) {
if (vk_dst_format == VK_FORMAT_D32_SFLOAT) {
ops->dst_depth(cs, dst, i);
} else {
ops->dst_stencil(cs, dst, i);
}
} else {
ops->dst(cs, &dst->view, i);
ops->dst(cs, &dst->view, i, src_format);
}
ops->run(cmd, cs);
@@ -2201,7 +2294,7 @@ resolve_sysmem(struct tu_cmd_buffer *cmd,
ops->teardown(cmd, cs);
trace_end_sysmem_resolve(&cmd->trace, cs, format);
trace_end_sysmem_resolve(&cmd->trace, cs, vk_dst_format);
}
void
@@ -2221,14 +2314,14 @@ tu_resolve_sysmem(struct tu_cmd_buffer *cmd,
bool dst_separate_ds = dst->image->vk_format == VK_FORMAT_D32_SFLOAT_S8_UINT;
if (dst_separate_ds) {
resolve_sysmem(cmd, cs, VK_FORMAT_D32_SFLOAT,
resolve_sysmem(cmd, cs, VK_FORMAT_D32_SFLOAT, VK_FORMAT_D32_SFLOAT,
src, dst, layer_mask, layers, rect,
src_separate_ds, dst_separate_ds);
resolve_sysmem(cmd, cs, VK_FORMAT_S8_UINT,
resolve_sysmem(cmd, cs, VK_FORMAT_S8_UINT, VK_FORMAT_S8_UINT,
src, dst, layer_mask, layers, rect,
src_separate_ds, dst_separate_ds);
} else {
resolve_sysmem(cmd, cs, dst->image->vk_format,
resolve_sysmem(cmd, cs, src->image->vk_format, dst->image->vk_format,
src, dst, layer_mask, layers, rect,
src_separate_ds, dst_separate_ds);
}
@@ -2260,7 +2353,7 @@ clear_image(struct tu_cmd_buffer *cmd,
const struct blit_ops *ops = image->layout[0].nr_samples > 1 ? &r3d_ops : &r2d_ops;
ops->setup(cmd, cs, format, aspect_mask, 0, true, image->layout[0].ubwc,
ops->setup(cmd, cs, format, format, aspect_mask, 0, true, image->layout[0].ubwc,
image->layout[0].nr_samples);
if (image->vk_format == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32)
ops->clear_value(cs, PIPE_FORMAT_R9G9B9E5_FLOAT, clear_value);
@@ -2282,10 +2375,10 @@ clear_image(struct tu_cmd_buffer *cmd,
.mipLevel = range->baseMipLevel + j,
.baseArrayLayer = range->baseArrayLayer,
.layerCount = 1,
}, 0, false, false);
}, 0, false);
for (uint32_t i = 0; i < layer_count; i++) {
ops->dst(cs, &dst, i);
ops->dst(cs, &dst, i, format);
ops->run(cmd, cs);
}
}
@@ -2757,7 +2850,7 @@ clear_sysmem_attachment(struct tu_cmd_buffer *cmd,
trace_start_sysmem_clear(&cmd->trace, cs);
ops->setup(cmd, cs, format, clear_mask, 0, true, iview->view.ubwc_enabled,
ops->setup(cmd, cs, format, format, clear_mask, 0, true, iview->view.ubwc_enabled,
cmd->state.pass->attachments[a].samples);
ops->coords(cs, &info->renderArea.offset, NULL, &info->renderArea.extent);
ops->clear_value(cs, format, &info->pClearValues[a]);
@@ -2770,7 +2863,7 @@ clear_sysmem_attachment(struct tu_cmd_buffer *cmd,
ops->dst_stencil(cs, iview, i);
}
} else {
ops->dst(cs, &iview->view, i);
ops->dst(cs, &iview->view, i, format);
}
ops->run(cmd, cs);
}
@@ -3029,11 +3122,12 @@ store_cp_blit(struct tu_cmd_buffer *cmd,
const struct tu_image_view *iview,
uint32_t samples,
bool separate_stencil,
enum pipe_format format,
enum pipe_format src_format,
enum pipe_format dst_format,
uint32_t gmem_offset,
uint32_t cpp)
{
r2d_setup_common(cmd, cs, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false,
r2d_setup_common(cmd, cs, src_format, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false,
iview->view.ubwc_enabled, true);
if (iview->image->vk_format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
@@ -3043,17 +3137,21 @@ store_cp_blit(struct tu_cmd_buffer *cmd,
r2d_dst_stencil(cs, iview, 0);
}
} else {
r2d_dst(cs, &iview->view, 0);
r2d_dst(cs, &iview->view, 0, src_format);
}
enum a6xx_format fmt = tu6_format_texture(src_format, TILE6_2).fmt;
fixup_src_format(&src_format, dst_format, &fmt);
tu_cs_emit_regs(cs,
A6XX_SP_PS_2D_SRC_INFO(
.color_format = tu6_format_texture(format, TILE6_2).fmt,
.color_format = fmt,
.color_swap = WZYX,
.tile_mode = TILE6_2,
.srgb = util_format_is_srgb(format),
.srgb = util_format_is_srgb(src_format),
.samples = tu_msaa_samples(samples),
.samples_average = !util_format_is_pure_integer(format) &&
!util_format_is_depth_or_stencil(format),
.samples_average = !util_format_is_pure_integer(dst_format) &&
!util_format_is_depth_or_stencil(dst_format),
.unk20 = 1,
.unk22 = 1),
/* note: src size does not matter when not scaling */
@@ -3083,7 +3181,8 @@ store_3d_blit(struct tu_cmd_buffer *cmd,
const struct tu_image_view *iview,
uint32_t dst_samples,
bool separate_stencil,
enum pipe_format format,
enum pipe_format src_format,
enum pipe_format dst_format,
const VkRect2D *render_area,
uint32_t gmem_offset,
uint32_t cpp)
@@ -3099,7 +3198,7 @@ store_3d_blit(struct tu_cmd_buffer *cmd,
CP_REG_TO_SCRATCH_0_SCRATCH(0) |
CP_REG_TO_SCRATCH_0_CNT(1 - 1));
r3d_setup(cmd, cs, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false,
r3d_setup(cmd, cs, src_format, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false,
iview->view.ubwc_enabled, dst_samples);
r3d_coords(cs, &render_area->offset, &render_area->offset, &render_area->extent);
@@ -3111,10 +3210,10 @@ store_3d_blit(struct tu_cmd_buffer *cmd,
r3d_dst_stencil(cs, iview, 0);
}
} else {
r3d_dst(cs, &iview->view, 0);
r3d_dst(cs, &iview->view, 0, src_format);
}
r3d_src_gmem(cmd, cs, iview, format, gmem_offset, cpp);
r3d_src_gmem(cmd, cs, iview, src_format, dst_format, gmem_offset, cpp);
/* sync GMEM writes with CACHE. */
tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE);
@@ -3198,13 +3297,21 @@ tu_store_gmem_attachment(struct tu_cmd_buffer *cmd,
src->format == VK_FORMAT_D32_SFLOAT_S8_UINT &&
dst->format == VK_FORMAT_S8_UINT;
/* The fast path doesn't support picking out the last component of a D24S8
* texture reinterpreted as RGBA8_UNORM.
*/
bool resolve_d24s8_s8 =
src->format == VK_FORMAT_D24_UNORM_S8_UINT &&
dst->format == VK_FORMAT_S8_UINT;
bool store_common = dst->store && !resolve_d32s8_s8;
bool store_separate_stencil = dst->store_stencil || resolve_d32s8_s8;
trace_start_gmem_store(&cmd->trace, cs);
/* use fast path when render area is aligned, except for unsupported resolve cases */
if (!unaligned && (a == gmem_a || blit_can_resolve(dst->format))) {
if (!unaligned && !resolve_d24s8_s8 &&
(a == gmem_a || blit_can_resolve(dst->format))) {
if (store_common)
tu_emit_blit(cmd, cs, iview, src, true, false);
if (store_separate_stencil)
@@ -3218,9 +3325,13 @@ tu_store_gmem_attachment(struct tu_cmd_buffer *cmd,
return;
}
enum pipe_format format = tu_vk_format_to_pipe_format(src->format);
if (format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)
format = PIPE_FORMAT_Z32_FLOAT;
enum pipe_format src_format = tu_vk_format_to_pipe_format(src->format);
if (src_format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)
src_format = PIPE_FORMAT_Z32_FLOAT;
enum pipe_format dst_format = tu_vk_format_to_pipe_format(dst->format);
if (dst_format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)
dst_format = PIPE_FORMAT_Z32_FLOAT;
if (dst->samples > 1) {
/* If we hit this path, we have to disable draw states after every tile
@@ -3234,23 +3345,24 @@ tu_store_gmem_attachment(struct tu_cmd_buffer *cmd,
tu_disable_draw_states(cmd, cs);
if (store_common) {
store_3d_blit(cmd, cs, iview, dst->samples, false, format,
render_area, src->gmem_offset, src->cpp);
store_3d_blit(cmd, cs, iview, dst->samples, false, src_format,
dst_format, render_area, src->gmem_offset, src->cpp);
}
if (store_separate_stencil) {
store_3d_blit(cmd, cs, iview, dst->samples, true, PIPE_FORMAT_S8_UINT,
render_area, src->gmem_offset_stencil, src->samples);
PIPE_FORMAT_S8_UINT, render_area,
src->gmem_offset_stencil, src->samples);
}
} else {
r2d_coords(cs, &render_area->offset, &render_area->offset, &render_area->extent);
if (store_common) {
store_cp_blit(cmd, cs, iview, src->samples, false, format,
src->gmem_offset, src->cpp);
store_cp_blit(cmd, cs, iview, src->samples, false, src_format,
dst_format, src->gmem_offset, src->cpp);
}
if (store_separate_stencil) {
store_cp_blit(cmd, cs, iview, src->samples, true, PIPE_FORMAT_S8_UINT,
src->gmem_offset_stencil, src->samples);
PIPE_FORMAT_S8_UINT, src->gmem_offset_stencil, src->samples);
}
}