From d60ae64527dfba55c998e869c743b519fbf90a3c Mon Sep 17 00:00:00 2001
From: Danylo Piliaiev <dpiliaiev@igalia.com>
Date: Thu, 18 Jul 2024 17:43:21 +0200
Subject: [PATCH] tu/a750: Allow mutable images to have UBWC with all
 compatible formats

A750+ added a special flag that allows HW to correctly interpret UBWC, including
UBWC fast-clear when casting image to a different format permitted by Vulkan.
So it's possible to have UBWC enabled for image that has e.g. R32_UINT and
R8G8B8A8_UNORM in the mutable formats list.

Signed-off-by: Danylo Piliaiev <dpiliaiev@igalia.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/30370>
---
 src/freedreno/common/freedreno_dev_info.h     |  7 +++++
 src/freedreno/common/freedreno_devices.py     |  1 +
 src/freedreno/fdl/fd6_view.c                  | 16 ++++++++----
 src/freedreno/fdl/freedreno_layout.h          |  1 +
 src/freedreno/registers/adreno/a6xx.xml       | 26 ++++++++++++++-----
 src/freedreno/vulkan/tu_clear_blit.cc         |  3 +++
 src/freedreno/vulkan/tu_image.cc              | 17 +++++++++---
 src/freedreno/vulkan/tu_image.h               |  1 +
 .../drivers/freedreno/a6xx/fd6_image.cc       |  1 +
 .../drivers/freedreno/a6xx/fd6_texture.cc     |  1 +
 10 files changed, 59 insertions(+), 15 deletions(-)

diff --git a/src/freedreno/common/freedreno_dev_info.h b/src/freedreno/common/freedreno_dev_info.h
index 3a54d63b85d..a178fa2c020 100644
--- a/src/freedreno/common/freedreno_dev_info.h
+++ b/src/freedreno/common/freedreno_dev_info.h
@@ -283,6 +283,13 @@ struct fd_dev_info {
       bool enable_tp_ubwc_flag_hint;
 
       bool storage_8bit;
+
+      /* A750+ added a special flag that allows HW to correctly interpret UBWC, including
+       * UBWC fast-clear when casting image to a different format permitted by Vulkan.
+       * So it's possible to have UBWC enabled for image that has e.g. R32_UINT and
+       * R8G8B8A8_UNORM in the mutable formats list.
+       */
+      bool ubwc_all_formats_compatible;
    } a7xx;
 };
 
diff --git a/src/freedreno/common/freedreno_devices.py b/src/freedreno/common/freedreno_devices.py
index 1dcfc7a43e5..79c69e42c66 100644
--- a/src/freedreno/common/freedreno_devices.py
+++ b/src/freedreno/common/freedreno_devices.py
@@ -871,6 +871,7 @@ a7xx_750 = A7XXProps(
         no_gs_hw_binning_quirk = True,
         gs_vpc_adjacency_quirk = True,
         storage_8bit = True,
+        ubwc_all_formats_compatible = True,
     )
 
 a730_magic_regs = dict(
diff --git a/src/freedreno/fdl/fd6_view.c b/src/freedreno/fdl/fd6_view.c
index 1e25fea5b41..c6c811f7942 100644
--- a/src/freedreno/fdl/fd6_view.c
+++ b/src/freedreno/fdl/fd6_view.c
@@ -253,7 +253,9 @@ fdl6_view_init(struct fdl6_view *view, const struct fdl_layout **layouts,
       A6XX_TEX_CONST_0_SWAP(swap) |
       fdl6_texswiz(args, has_z24uint_s8uint) |
       A6XX_TEX_CONST_0_MIPLVLS(args->level_count - 1);
-   view->descriptor[1] = A6XX_TEX_CONST_1_WIDTH(width) | A6XX_TEX_CONST_1_HEIGHT(height);
+   view->descriptor[1] =
+      A6XX_TEX_CONST_1_WIDTH(width) | A6XX_TEX_CONST_1_HEIGHT(height) |
+      COND(args->ubwc_fc_mutable, A6XX_TEX_CONST_1_MUTABLEEN);
    view->descriptor[2] =
       A6XX_TEX_CONST_2_PITCHALIGN(layout->pitchalign - 6) |
       A6XX_TEX_CONST_2_PITCH(pitch) |
@@ -340,7 +342,8 @@ fdl6_view_init(struct fdl6_view *view, const struct fdl_layout **layouts,
       A6XX_SP_PS_2D_SRC_INFO_SAMPLES(util_logbase2(layout->nr_samples)) |
       COND(samples_average, A6XX_SP_PS_2D_SRC_INFO_SAMPLES_AVERAGE) |
       A6XX_SP_PS_2D_SRC_INFO_UNK20 |
-      A6XX_SP_PS_2D_SRC_INFO_UNK22;
+      A6XX_SP_PS_2D_SRC_INFO_UNK22 |
+      COND(args->ubwc_fc_mutable, A6XX_SP_PS_2D_SRC_INFO_MUTABLEEN);
 
    view->SP_PS_2D_SRC_SIZE =
       A6XX_SP_PS_2D_SRC_SIZE_WIDTH(width) |
@@ -420,7 +423,8 @@ fdl6_view_init(struct fdl6_view *view, const struct fdl_layout **layouts,
       A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE(tile_mode) |
       A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT(color_format) |
       COND(args->chip >= A7XX && ubwc_enabled, A7XX_RB_MRT_BUF_INFO_LOSSLESSCOMPEN) |
-      A6XX_RB_MRT_BUF_INFO_COLOR_SWAP(color_swap);
+      A6XX_RB_MRT_BUF_INFO_COLOR_SWAP(color_swap) |
+      COND(args->ubwc_fc_mutable, A7XX_RB_MRT_BUF_INFO_MUTABLEEN);
 
    view->SP_FS_MRT_REG =
       A6XX_SP_FS_MRT_REG_COLOR_FORMAT(color_format) |
@@ -432,14 +436,16 @@ fdl6_view_init(struct fdl6_view *view, const struct fdl_layout **layouts,
       A6XX_RB_2D_DST_INFO_TILE_MODE(tile_mode) |
       A6XX_RB_2D_DST_INFO_COLOR_SWAP(color_swap) |
       COND(ubwc_enabled, A6XX_RB_2D_DST_INFO_FLAGS) |
-      COND(util_format_is_srgb(args->format), A6XX_RB_2D_DST_INFO_SRGB);
+      COND(util_format_is_srgb(args->format), A6XX_RB_2D_DST_INFO_SRGB) |
+      COND(args->ubwc_fc_mutable, A6XX_RB_2D_DST_INFO_MUTABLEEN);;
 
    view->RB_BLIT_DST_INFO =
       A6XX_RB_BLIT_DST_INFO_TILE_MODE(tile_mode) |
       A6XX_RB_BLIT_DST_INFO_SAMPLES(util_logbase2(layout->nr_samples)) |
       A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT(blit_format) |
       A6XX_RB_BLIT_DST_INFO_COLOR_SWAP(color_swap) |
-      COND(ubwc_enabled, A6XX_RB_BLIT_DST_INFO_FLAGS);
+      COND(ubwc_enabled, A6XX_RB_BLIT_DST_INFO_FLAGS) |
+      COND(args->ubwc_fc_mutable, A6XX_RB_BLIT_DST_INFO_MUTABLEEN);
 }
 
 void
diff --git a/src/freedreno/fdl/freedreno_layout.h b/src/freedreno/fdl/freedreno_layout.h
index 4b45b5d08ce..531046dfe3c 100644
--- a/src/freedreno/fdl/freedreno_layout.h
+++ b/src/freedreno/fdl/freedreno_layout.h
@@ -296,6 +296,7 @@ struct fdl_view_args {
    enum pipe_format format;
    enum fdl_view_type type;
    enum fdl_chroma_location chroma_offsets[2];
+   bool ubwc_fc_mutable;
 };
 
 #define FDL6_TEX_CONST_DWORDS 16
diff --git a/src/freedreno/registers/adreno/a6xx.xml b/src/freedreno/registers/adreno/a6xx.xml
index 53a45322842..f50f17907f4 100644
--- a/src/freedreno/registers/adreno/a6xx.xml
+++ b/src/freedreno/registers/adreno/a6xx.xml
@@ -4172,6 +4172,7 @@ to upconvert to 32b float internally?
 			<bitfield name="UNK10" pos="10"/>
 			<bitfield name="LOSSLESSCOMPEN" pos="11" type="boolean"/>
 			<bitfield name="COLOR_SWAP" low="13" high="14" type="a3xx_color_swap"/>
+			<bitfield name="MUTABLEEN" pos="16" type="boolean" variants="A7XX-"/>
 		</reg32>
 		<!--
 		at least in gmem, things seem to be aligned to pitch of 64..
@@ -4337,6 +4338,7 @@ to upconvert to 32b float internally?
 		<bitfield name="COLOR_SWAP" low="5" high="6" type="a3xx_color_swap"/>
 		<bitfield name="COLOR_FORMAT" low="7" high="14" type="a6xx_format"/>
 		<bitfield name="UNK15" pos="15" type="boolean"/>
+		<bitfield name="MUTABLEEN" pos="16" type="boolean" variants="A7XX-"/>
 	</reg32>
 	<reg64 offset="0x88d8" name="RB_BLIT_DST" type="waddress" align="64" usage="rp_blit"/>
 	<reg32 offset="0x88da" name="RB_BLIT_DST_PITCH" low="0" high="15" shr="6" type="uint" usage="rp_blit"/>
@@ -4453,13 +4455,12 @@ to upconvert to 32b float internally?
 	<reg32 offset="0x8c00" name="RB_2D_BLIT_CNTL" type="a6xx_2d_blit_cntl" usage="rp_blit"/>
 	<reg32 offset="0x8c01" name="RB_2D_UNKNOWN_8C01" low="0" high="31" usage="rp_blit"/>
 
-	<bitset name="a6xx_2d_surf_info" inline="yes">
+	<bitset name="a6xx_2d_src_surf_info" inline="yes">
 		<bitfield name="COLOR_FORMAT" low="0" high="7" type="a6xx_format"/>
 		<bitfield name="TILE_MODE" low="8" high="9" type="a6xx_tile_mode"/>
 		<bitfield name="COLOR_SWAP" low="10" high="11" type="a3xx_color_swap"/>
 		<bitfield name="FLAGS" pos="12" type="boolean"/>
 		<bitfield name="SRGB" pos="13" type="boolean"/>
-		<!-- the rest is only for src -->
 		<bitfield name="SAMPLES" low="14" high="15" type="a3xx_msaa_samples"/>
 		<bitfield name="FILTER" pos="16" type="boolean"/>
 		<bitfield name="UNK17" pos="17" type="boolean"/>
@@ -4470,11 +4471,21 @@ to upconvert to 32b float internally?
 		<bitfield name="UNK22" pos="22" type="boolean"/>
 		<bitfield name="UNK23" low="23" high="26"/>
 		<bitfield name="UNK28" pos="28" type="boolean"/>
+		<bitfield name="MUTABLEEN" pos="29" type="boolean" variants="A7XX-"/>
+	</bitset>
+
+	<bitset name="a6xx_2d_dst_surf_info" inline="yes">
+		<bitfield name="COLOR_FORMAT" low="0" high="7" type="a6xx_format"/>
+		<bitfield name="TILE_MODE" low="8" high="9" type="a6xx_tile_mode"/>
+		<bitfield name="COLOR_SWAP" low="10" high="11" type="a3xx_color_swap"/>
+		<bitfield name="FLAGS" pos="12" type="boolean"/>
+		<bitfield name="SRGB" pos="13" type="boolean"/>
+		<bitfield name="SAMPLES" low="14" high="15" type="a3xx_msaa_samples"/>
+		<bitfield name="MUTABLEEN" pos="17" type="boolean" variants="A7XX-"/>
 	</bitset>
 
 	<!-- 0x8c02-0x8c16 invalid -->
-	<!-- TODO: RB_2D_DST_INFO has 17 valid bits (doesn't match a6xx_2d_surf_info) -->
-	<reg32 offset="0x8c17" name="RB_2D_DST_INFO" type="a6xx_2d_surf_info" usage="rp_blit"/>
+	<reg32 offset="0x8c17" name="RB_2D_DST_INFO" type="a6xx_2d_dst_surf_info" usage="rp_blit"/>
 	<reg64 offset="0x8c18" name="RB_2D_DST" type="waddress" align="64" usage="rp_blit"/>
 	<reg32 offset="0x8c1a" name="RB_2D_DST_PITCH" low="0" high="15" shr="6" type="uint" usage="rp_blit"/>
 	<!-- this is a guess but seems likely (for NV12/IYUV): -->
@@ -5845,7 +5856,7 @@ to upconvert to 32b float internally?
 	badly named or the functionality moved in a6xx.  But downstream kernel
 	calls this "a6xx_sp_ps_tp_2d_cluster"
 	 -->
-	<reg32 offset="0xb4c0" name="SP_PS_2D_SRC_INFO" type="a6xx_2d_surf_info" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0xb4c0" name="SP_PS_2D_SRC_INFO" type="a6xx_2d_src_surf_info" variants="A6XX" usage="rp_blit"/>
 	<reg32 offset="0xb4c1" name="SP_PS_2D_SRC_SIZE" variants="A6XX" usage="rp_blit">
 		<bitfield name="WIDTH" low="0" high="14" type="uint"/>
 		<bitfield name="HEIGHT" low="15" high="29" type="uint"/>
@@ -5856,7 +5867,7 @@ to upconvert to 32b float internally?
 		<bitfield name="PITCH" low="9" high="23" shr="6" type="uint"/>
 	</reg32>
 
-	<reg32 offset="0xb2c0" name="SP_PS_2D_SRC_INFO" type="a6xx_2d_surf_info" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0xb2c0" name="SP_PS_2D_SRC_INFO" type="a6xx_2d_src_surf_info" variants="A7XX-" usage="rp_blit"/>
 	<reg32 offset="0xb2c1" name="SP_PS_2D_SRC_SIZE" variants="A7XX">
 		<bitfield name="WIDTH" low="0" high="14" type="uint"/>
 		<bitfield name="HEIGHT" low="15" high="29" type="uint"/>
@@ -6380,7 +6391,7 @@ to upconvert to 32b float internally?
 	<reg32 offset="3" name="3"/>
 </domain>
 
-<domain name="A6XX_TEX_CONST" width="32">
+<domain name="A6XX_TEX_CONST" width="32" varset="chip">
 	<doc>Texture constant dwords</doc>
 	<enum name="a6xx_tex_swiz"> <!-- same as a4xx? -->
 		<value name="A6XX_TEX_X" value="0"/>
@@ -6420,6 +6431,7 @@ to upconvert to 32b float internally?
 	<reg32 offset="1" name="1">
 		<bitfield name="WIDTH" low="0" high="14" type="uint"/>
 		<bitfield name="HEIGHT" low="15" high="29" type="uint"/>
+		<bitfield name="MUTABLEEN" pos="31" type="boolean" variants="A7XX-"/>
 	</reg32>
 	<reg32 offset="2" name="2">
 		<!--
diff --git a/src/freedreno/vulkan/tu_clear_blit.cc b/src/freedreno/vulkan/tu_clear_blit.cc
index 248e86647ac..c3075214d24 100644
--- a/src/freedreno/vulkan/tu_clear_blit.cc
+++ b/src/freedreno/vulkan/tu_clear_blit.cc
@@ -1894,6 +1894,7 @@ tu_image_view_copy_blit(struct fdl6_view *iview,
       },
       .format = tu_format_for_aspect(format, aspect_mask),
       .type = z_scale ? FDL_VIEW_TYPE_3D : FDL_VIEW_TYPE_2D,
+      .ubwc_fc_mutable = image->ubwc_fc_mutable,
    };
    fdl6_view_init(iview, &layout, &args, false);
 }
@@ -2467,6 +2468,7 @@ tu_copy_image_to_image(struct tu_cmd_buffer *cmd,
          .swiz = { PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W },
          .format = tu_format_for_aspect(src_format, VK_IMAGE_ASPECT_COLOR_BIT),
          .type = FDL_VIEW_TYPE_2D,
+         .ubwc_fc_mutable = false,
       };
       fdl6_view_init(&staging, &staging_layout_ptr, &copy_to_args, false);
 
@@ -2497,6 +2499,7 @@ tu_copy_image_to_image(struct tu_cmd_buffer *cmd,
          .swiz = { PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W },
          .format = tu_format_for_aspect(dst_format, VK_IMAGE_ASPECT_COLOR_BIT),
          .type = FDL_VIEW_TYPE_2D,
+         .ubwc_fc_mutable = false,
       };
       fdl6_view_init(&staging, &staging_layout_ptr, &copy_from_args, false);
 
diff --git a/src/freedreno/vulkan/tu_image.cc b/src/freedreno/vulkan/tu_image.cc
index 8448020047d..c90502993ce 100644
--- a/src/freedreno/vulkan/tu_image.cc
+++ b/src/freedreno/vulkan/tu_image.cc
@@ -239,6 +239,7 @@ tu_image_view_init(struct tu_device *device,
    args.level_count = vk_image_subresource_level_count(&image->vk, range);
    args.min_lod_clamp = iview->vk.min_lod;
    args.format = tu_format_for_aspect(format, aspect_mask);
+   args.ubwc_fc_mutable = image->ubwc_fc_mutable;
    vk_component_mapping_to_pipe_swizzle(pCreateInfo->components, args.swiz);
    if (conversion) {
       unsigned char conversion_swiz[4], create_swiz[4];
@@ -647,6 +648,7 @@ tu_image_init(struct tu_device *device, struct tu_image *image,
                       device->use_z24uint_s8uint))
       image->ubwc_enabled = false;
 
+   bool fmt_list_has_swaps = false;
    /* Mutable images can be reinterpreted as any other compatible format.
     * This is a problem with UBWC (compression for different formats is different),
     * but also tiling ("swap" affects how tiled formats are stored in memory)
@@ -663,9 +665,11 @@ tu_image_init(struct tu_device *device, struct tu_image *image,
        !vk_format_is_depth_or_stencil(image->vk.format)) {
       const VkImageFormatListCreateInfo *fmt_list =
          vk_find_struct_const(pCreateInfo->pNext, IMAGE_FORMAT_LIST_CREATE_INFO);
+      fmt_list_has_swaps = format_list_has_swaps(fmt_list);
       if (!tu6_mutable_format_list_ubwc_compatible(device->physical_device->info,
                                                    fmt_list)) {
-         if (image->ubwc_enabled) {
+         bool mutable_ubwc_fc = device->physical_device->info->a7xx.ubwc_all_formats_compatible;
+         if (image->ubwc_enabled && !mutable_ubwc_fc) {
             if (fmt_list && fmt_list->viewFormatCount == 2) {
                perf_debug(
                   device,
@@ -687,10 +691,17 @@ tu_image_init(struct tu_device *device, struct tu_image *image,
             image->ubwc_enabled = false;
          }
 
-         if (format_list_reinterprets_r8g8_r16(tu_vk_format_to_pipe_format(image->vk.format), fmt_list) ||
-            format_list_has_swaps(fmt_list)) {
+         bool r8g8_r16 = format_list_reinterprets_r8g8_r16(tu_vk_format_to_pipe_format(image->vk.format), fmt_list);
+
+         /* A750+ TODO: Correctly handle swaps when copying mutable images.
+          * We should be able to support UBWC for mutable images with swaps.
+          */
+         if ((r8g8_r16 && !mutable_ubwc_fc) || fmt_list_has_swaps) {
+            image->ubwc_enabled = false;
             image->force_linear_tile = true;
          }
+
+         image->ubwc_fc_mutable = image->ubwc_enabled && mutable_ubwc_fc;
       }
    }
 
diff --git a/src/freedreno/vulkan/tu_image.h b/src/freedreno/vulkan/tu_image.h
index de9504c3a58..fe7c7e1325f 100644
--- a/src/freedreno/vulkan/tu_image.h
+++ b/src/freedreno/vulkan/tu_image.h
@@ -42,6 +42,7 @@ struct tu_image
 
    bool ubwc_enabled;
    bool force_linear_tile;
+   bool ubwc_fc_mutable;
 };
 VK_DEFINE_NONDISP_HANDLE_CASTS(tu_image, vk.base, VkImage, VK_OBJECT_TYPE_IMAGE)
 
diff --git a/src/gallium/drivers/freedreno/a6xx/fd6_image.cc b/src/gallium/drivers/freedreno/a6xx/fd6_image.cc
index da5a058a476..33eba52997a 100644
--- a/src/gallium/drivers/freedreno/a6xx/fd6_image.cc
+++ b/src/gallium/drivers/freedreno/a6xx/fd6_image.cc
@@ -90,6 +90,7 @@ fd6_image_descriptor(struct fd_context *ctx, const struct pipe_image_view *buf,
          .type = fdl_type_from_pipe_target(buf->resource->target),
          .chroma_offsets = {FDL_CHROMA_LOCATION_COSITED_EVEN,
                             FDL_CHROMA_LOCATION_COSITED_EVEN},
+         .ubwc_fc_mutable = false,
       };
 
       /* fdl6_view makes the storage descriptor treat cubes like a 2D array (so
diff --git a/src/gallium/drivers/freedreno/a6xx/fd6_texture.cc b/src/gallium/drivers/freedreno/a6xx/fd6_texture.cc
index 3e7381967c5..b2fb4c6614f 100644
--- a/src/gallium/drivers/freedreno/a6xx/fd6_texture.cc
+++ b/src/gallium/drivers/freedreno/a6xx/fd6_texture.cc
@@ -455,6 +455,7 @@ fd6_sampler_view_update(struct fd_context *ctx,
          .type = fdl_type_from_pipe_target(cso->target),
          .chroma_offsets = {FDL_CHROMA_LOCATION_COSITED_EVEN,
                             FDL_CHROMA_LOCATION_COSITED_EVEN},
+         .ubwc_fc_mutable = false,
       };
 
       if (rsc->b.b.format == PIPE_FORMAT_R8_G8B8_420_UNORM) {