From 7e68fae25f6e5416787edb8dd74844f4c89ff482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Ol=C5=A1=C3=A1k?= Date: Mon, 22 Mar 2021 19:43:53 -0400 Subject: [PATCH] ac,radeonsi: rewrite DCC retiling without the DCC retile map The retile map is removed and replaced by direct DCC address computations in the retile shader using the new function ac_nir_dcc_addr_from_coord. The RADV code is disabled. Acked-by: Pierre-Eric Pelloux-Prayer Reviewed-by: Bas Nieuwenhuizen Part-of: --- src/amd/common/ac_surface.c | 515 +++++------------- src/amd/common/ac_surface.h | 64 ++- src/amd/common/ac_surface_dcc_address_test.c | 14 + src/amd/common/meson.build | 2 +- src/amd/vulkan/radv_cmd_buffer.c | 6 +- src/amd/vulkan/radv_image.c | 36 -- src/amd/vulkan/radv_meta_dcc_retile.c | 19 +- src/amd/vulkan/radv_private.h | 9 - src/gallium/drivers/radeonsi/Makefile.sources | 1 + src/gallium/drivers/radeonsi/meson.build | 1 + .../drivers/radeonsi/si_compute_blit.c | 82 ++- src/gallium/drivers/radeonsi/si_pipe.h | 7 +- .../drivers/radeonsi/si_shaderlib_nir.c | 102 ++++ .../drivers/radeonsi/si_shaderlib_tgsi.c | 70 --- src/gallium/drivers/radeonsi/si_texture.c | 53 +- 15 files changed, 385 insertions(+), 596 deletions(-) create mode 100644 src/gallium/drivers/radeonsi/si_shaderlib_nir.c diff --git a/src/amd/common/ac_surface.c b/src/amd/common/ac_surface.c index 44df8881176..4d4ca491c45 100644 --- a/src/amd/common/ac_surface.c +++ b/src/amd/common/ac_surface.c @@ -25,6 +25,7 @@ * of the Software. */ +#define AC_SURFACE_INCLUDE_NIR #include "ac_surface.h" #include "ac_gpu_info.h" @@ -138,277 +139,8 @@ typedef uint64_t __u64; struct ac_addrlib { ADDR_HANDLE handle; - - /* The cache of DCC retile maps for reuse when allocating images of - * similar sizes. - */ - simple_mtx_t dcc_retile_map_lock; - struct hash_table *dcc_retile_maps; - struct hash_table *dcc_retile_tile_indices; }; -struct dcc_retile_map_key { - enum radeon_family family; - unsigned retile_width; - unsigned retile_height; - bool rb_aligned; - bool pipe_aligned; - unsigned dcc_retile_num_elements; - ADDR2_COMPUTE_DCC_ADDRFROMCOORD_INPUT input; -}; - -static uint32_t dcc_retile_map_hash_key(const void *key) -{ - return _mesa_hash_data(key, sizeof(struct dcc_retile_map_key)); -} - -static bool dcc_retile_map_keys_equal(const void *a, const void *b) -{ - return memcmp(a, b, sizeof(struct dcc_retile_map_key)) == 0; -} - -static void dcc_retile_map_free(struct hash_entry *entry) -{ - free((void *)entry->key); - free(entry->data); -} - -struct dcc_retile_tile_key { - enum radeon_family family; - unsigned bpp; - unsigned swizzle_mode; - bool rb_aligned; - bool pipe_aligned; -}; - -struct dcc_retile_tile_data { - unsigned tile_width_log2; - unsigned tile_height_log2; - uint16_t *data; -}; - -static uint32_t dcc_retile_tile_hash_key(const void *key) -{ - return _mesa_hash_data(key, sizeof(struct dcc_retile_tile_key)); -} - -static bool dcc_retile_tile_keys_equal(const void *a, const void *b) -{ - return memcmp(a, b, sizeof(struct dcc_retile_tile_key)) == 0; -} - -static void dcc_retile_tile_free(struct hash_entry *entry) -{ - free((void *)entry->key); - free(((struct dcc_retile_tile_data *)entry->data)->data); - free(entry->data); -} - -/* Assumes dcc_retile_map_lock is taken. */ -static const struct dcc_retile_tile_data * -ac_compute_dcc_retile_tile_indices(struct ac_addrlib *addrlib, const struct radeon_info *info, - unsigned bpp, unsigned swizzle_mode, bool rb_aligned, - bool pipe_aligned) -{ - struct dcc_retile_tile_key key; - memset(&key, 0, sizeof(key)); - - key.family = info->family; - key.bpp = bpp; - key.swizzle_mode = swizzle_mode; - key.rb_aligned = rb_aligned; - key.pipe_aligned = pipe_aligned; - - struct hash_entry *entry = _mesa_hash_table_search(addrlib->dcc_retile_tile_indices, &key); - if (entry) - return entry->data; - - ADDR2_COMPUTE_DCCINFO_INPUT din = {0}; - ADDR2_COMPUTE_DCCINFO_OUTPUT dout = {0}; - din.size = sizeof(ADDR2_COMPUTE_DCCINFO_INPUT); - dout.size = sizeof(ADDR2_COMPUTE_DCCINFO_OUTPUT); - - din.dccKeyFlags.pipeAligned = pipe_aligned; - din.dccKeyFlags.rbAligned = rb_aligned; - din.resourceType = ADDR_RSRC_TEX_2D; - din.swizzleMode = swizzle_mode; - din.bpp = bpp; - din.unalignedWidth = 1; - din.unalignedHeight = 1; - din.numSlices = 1; - din.numFrags = 1; - din.numMipLevels = 1; - - ADDR_E_RETURNCODE ret = Addr2ComputeDccInfo(addrlib->handle, &din, &dout); - if (ret != ADDR_OK) - return NULL; - - ADDR2_COMPUTE_DCC_ADDRFROMCOORD_INPUT addrin = {0}; - addrin.size = sizeof(addrin); - addrin.swizzleMode = swizzle_mode; - addrin.resourceType = ADDR_RSRC_TEX_2D; - addrin.bpp = bpp; - addrin.numSlices = 1; - addrin.numMipLevels = 1; - addrin.numFrags = 1; - addrin.pitch = dout.pitch; - addrin.height = dout.height; - addrin.compressBlkWidth = dout.compressBlkWidth; - addrin.compressBlkHeight = dout.compressBlkHeight; - addrin.compressBlkDepth = dout.compressBlkDepth; - addrin.metaBlkWidth = dout.metaBlkWidth; - addrin.metaBlkHeight = dout.metaBlkHeight; - addrin.metaBlkDepth = dout.metaBlkDepth; - addrin.dccKeyFlags.pipeAligned = pipe_aligned; - addrin.dccKeyFlags.rbAligned = rb_aligned; - - unsigned w = dout.metaBlkWidth / dout.compressBlkWidth; - unsigned h = dout.metaBlkHeight / dout.compressBlkHeight; - uint16_t *indices = malloc(w * h * sizeof(uint16_t)); - if (!indices) - return NULL; - - ADDR2_COMPUTE_DCC_ADDRFROMCOORD_OUTPUT addrout = {0}; - addrout.size = sizeof(addrout); - - for (unsigned y = 0; y < h; ++y) { - addrin.y = y * dout.compressBlkHeight; - for (unsigned x = 0; x < w; ++x) { - addrin.x = x * dout.compressBlkWidth; - addrout.addr = 0; - - if (Addr2ComputeDccAddrFromCoord(addrlib->handle, &addrin, &addrout) != ADDR_OK) { - free(indices); - return NULL; - } - indices[y * w + x] = addrout.addr; - } - } - - struct dcc_retile_tile_data *data = calloc(1, sizeof(*data)); - if (!data) { - free(indices); - return NULL; - } - - data->tile_width_log2 = util_logbase2(w); - data->tile_height_log2 = util_logbase2(h); - data->data = indices; - - struct dcc_retile_tile_key *heap_key = mem_dup(&key, sizeof(key)); - if (!heap_key) { - free(data); - free(indices); - return NULL; - } - - entry = _mesa_hash_table_insert(addrlib->dcc_retile_tile_indices, heap_key, data); - if (!entry) { - free(heap_key); - free(data); - free(indices); - } - return data; -} - -static uint32_t ac_compute_retile_tile_addr(const struct dcc_retile_tile_data *tile, - unsigned stride, unsigned x, unsigned y) -{ - unsigned x_mask = (1u << tile->tile_width_log2) - 1; - unsigned y_mask = (1u << tile->tile_height_log2) - 1; - unsigned tile_size_log2 = tile->tile_width_log2 + tile->tile_height_log2; - - unsigned base = ((y >> tile->tile_height_log2) * stride + (x >> tile->tile_width_log2)) - << tile_size_log2; - unsigned offset_in_tile = tile->data[((y & y_mask) << tile->tile_width_log2) + (x & x_mask)]; - return base + offset_in_tile; -} - -static uint32_t *ac_compute_dcc_retile_map(struct ac_addrlib *addrlib, - const struct radeon_info *info, unsigned retile_width, - unsigned retile_height, bool rb_aligned, - bool pipe_aligned, bool use_uint16, - unsigned dcc_retile_num_elements, - const ADDR2_COMPUTE_DCC_ADDRFROMCOORD_INPUT *in) -{ - unsigned dcc_retile_map_size = dcc_retile_num_elements * (use_uint16 ? 2 : 4); - struct dcc_retile_map_key key; - - assert(in->numFrags == 1 && in->numSlices == 1 && in->numMipLevels == 1); - - memset(&key, 0, sizeof(key)); - key.family = info->family; - key.retile_width = retile_width; - key.retile_height = retile_height; - key.rb_aligned = rb_aligned; - key.pipe_aligned = pipe_aligned; - key.dcc_retile_num_elements = dcc_retile_num_elements; - memcpy(&key.input, in, sizeof(*in)); - - simple_mtx_lock(&addrlib->dcc_retile_map_lock); - - /* If we have already computed this retile map, get it from the hash table. */ - struct hash_entry *entry = _mesa_hash_table_search(addrlib->dcc_retile_maps, &key); - if (entry) { - uint32_t *map = entry->data; - simple_mtx_unlock(&addrlib->dcc_retile_map_lock); - return map; - } - - const struct dcc_retile_tile_data *src_tile = ac_compute_dcc_retile_tile_indices( - addrlib, info, in->bpp, in->swizzleMode, rb_aligned, pipe_aligned); - const struct dcc_retile_tile_data *dst_tile = - ac_compute_dcc_retile_tile_indices(addrlib, info, in->bpp, in->swizzleMode, false, false); - if (!src_tile || !dst_tile) { - simple_mtx_unlock(&addrlib->dcc_retile_map_lock); - return NULL; - } - - void *dcc_retile_map = malloc(dcc_retile_map_size); - if (!dcc_retile_map) { - simple_mtx_unlock(&addrlib->dcc_retile_map_lock); - return NULL; - } - - unsigned index = 0; - unsigned w = DIV_ROUND_UP(retile_width, in->compressBlkWidth); - unsigned h = DIV_ROUND_UP(retile_height, in->compressBlkHeight); - unsigned src_stride = DIV_ROUND_UP(w, 1u << src_tile->tile_width_log2); - unsigned dst_stride = DIV_ROUND_UP(w, 1u << dst_tile->tile_width_log2); - - for (unsigned y = 0; y < h; ++y) { - for (unsigned x = 0; x < w; ++x) { - unsigned src_addr = ac_compute_retile_tile_addr(src_tile, src_stride, x, y); - unsigned dst_addr = ac_compute_retile_tile_addr(dst_tile, dst_stride, x, y); - - if (use_uint16) { - ((uint16_t *)dcc_retile_map)[2 * index] = src_addr; - ((uint16_t *)dcc_retile_map)[2 * index + 1] = dst_addr; - } else { - ((uint32_t *)dcc_retile_map)[2 * index] = src_addr; - ((uint32_t *)dcc_retile_map)[2 * index + 1] = dst_addr; - } - ++index; - } - } - - /* Fill the remaining pairs with the last one (for the compute shader). */ - for (unsigned i = index * 2; i < dcc_retile_num_elements; i++) { - if (use_uint16) - ((uint16_t *)dcc_retile_map)[i] = ((uint16_t *)dcc_retile_map)[i - 2]; - else - ((uint32_t *)dcc_retile_map)[i] = ((uint32_t *)dcc_retile_map)[i - 2]; - } - - /* Insert the retile map into the hash table, so that it can be reused and - * the computation can be skipped for similar image sizes. - */ - _mesa_hash_table_insert(addrlib->dcc_retile_maps, mem_dup(&key, sizeof(key)), dcc_retile_map); - - simple_mtx_unlock(&addrlib->dcc_retile_map_lock); - return dcc_retile_map; -} - bool ac_modifier_has_dcc(uint64_t modifier) { return IS_AMD_FMT_MOD(modifier) && AMD_FMT_MOD_GET(DCC, modifier); @@ -742,20 +474,12 @@ struct ac_addrlib *ac_addrlib_create(const struct radeon_info *info, } addrlib->handle = addrCreateOutput.hLib; - simple_mtx_init(&addrlib->dcc_retile_map_lock, mtx_plain); - addrlib->dcc_retile_maps = - _mesa_hash_table_create(NULL, dcc_retile_map_hash_key, dcc_retile_map_keys_equal); - addrlib->dcc_retile_tile_indices = - _mesa_hash_table_create(NULL, dcc_retile_tile_hash_key, dcc_retile_tile_keys_equal); return addrlib; } void ac_addrlib_destroy(struct ac_addrlib *addrlib) { AddrDestroy(addrlib->handle); - simple_mtx_destroy(&addrlib->dcc_retile_map_lock); - _mesa_hash_table_destroy(addrlib->dcc_retile_maps, dcc_retile_map_free); - _mesa_hash_table_destroy(addrlib->dcc_retile_tile_indices, dcc_retile_tile_free); free(addrlib); } @@ -1769,6 +1493,38 @@ static bool is_dcc_supported_by_DCN(const struct radeon_info *info, } } +static void ac_copy_dcc_equation(const struct radeon_info *info, + ADDR2_COMPUTE_DCCINFO_OUTPUT *dcc, + struct gfx9_dcc_equation *equation) +{ + equation->meta_block_width = dcc->metaBlkWidth; + equation->meta_block_height = dcc->metaBlkHeight; + equation->meta_block_depth = dcc->metaBlkDepth; + + if (info->chip_class >= GFX10) { + /* gfx9_dcc_equation doesn't store the first 4 and the last 8 elements. They must be 0. */ + for (unsigned i = 0; i < 4; i++) + assert(dcc->equation.gfx10_bits[i] == 0); + + for (unsigned i = ARRAY_SIZE(equation->u.gfx10_bits) + 4; i < 68; i++) + assert(dcc->equation.gfx10_bits[i] == 0); + + memcpy(equation->u.gfx10_bits, dcc->equation.gfx10_bits + 4, + sizeof(equation->u.gfx10_bits)); + } else { + assert(dcc->equation.gfx9.num_bits <= ARRAY_SIZE(equation->u.gfx9.bit)); + + equation->u.gfx9.num_bits = dcc->equation.gfx9.num_bits; + equation->u.gfx9.num_pipe_bits = dcc->equation.gfx9.numPipeBits; + for (unsigned b = 0; b < ARRAY_SIZE(equation->u.gfx9.bit); b++) { + for (unsigned c = 0; c < ARRAY_SIZE(equation->u.gfx9.bit[b].coord); c++) { + equation->u.gfx9.bit[b].coord[c].dim = dcc->equation.gfx9.bit[b].coord[c].dim; + equation->u.gfx9.bit[b].coord[c].ord = dcc->equation.gfx9.bit[b].coord[c].ord; + } + } + } +} + static int gfx9_compute_miptree(struct ac_addrlib *addrlib, const struct radeon_info *info, const struct ac_surf_config *config, struct radeon_surf *surf, bool compressed, ADDR2_COMPUTE_SURFACE_INFO_INPUT *in) @@ -1982,6 +1738,7 @@ static int gfx9_compute_miptree(struct ac_addrlib *addrlib, const struct radeon_ surf->u.gfx9.color.dcc_block_height = dout.compressBlkHeight; surf->u.gfx9.color.dcc_block_depth = dout.compressBlkDepth; surf->u.gfx9.color.dcc_pitch_max = dout.pitch - 1; + surf->u.gfx9.color.dcc_height = dout.height; surf->meta_size = dout.dccRamSize; surf->meta_slice_size = dout.dccRamSliceSize; surf->meta_alignment_log2 = util_logbase2(dout.dccRamBaseAlign); @@ -2034,6 +1791,10 @@ static int gfx9_compute_miptree(struct ac_addrlib *addrlib, const struct radeon_ surf->u.gfx9.color.display_dcc_size = surf->meta_size; surf->u.gfx9.color.display_dcc_alignment_log2 = surf->meta_alignment_log2; surf->u.gfx9.color.display_dcc_pitch_max = surf->u.gfx9.color.dcc_pitch_max; + surf->u.gfx9.color.display_dcc_height = surf->u.gfx9.color.dcc_height; + + if (in->resourceType == ADDR_RSRC_TEX_2D) + ac_copy_dcc_equation(info, &dout, &surf->u.gfx9.color.dcc_equation); /* Compute displayable DCC. */ if (((in->flags.display && info->use_display_dcc_with_retile_blit) || @@ -2055,84 +1816,11 @@ static int gfx9_compute_miptree(struct ac_addrlib *addrlib, const struct radeon_ surf->u.gfx9.color.display_dcc_size = dout.dccRamSize; surf->u.gfx9.color.display_dcc_alignment_log2 = util_logbase2(dout.dccRamBaseAlign); surf->u.gfx9.color.display_dcc_pitch_max = dout.pitch - 1; + surf->u.gfx9.color.display_dcc_height = dout.height; assert(surf->u.gfx9.color.display_dcc_size <= surf->meta_size); - surf->u.gfx9.color.dcc_retile_use_uint16 = - surf->u.gfx9.color.display_dcc_size <= UINT16_MAX + 1 && surf->meta_size <= UINT16_MAX + 1; - - /* Align the retile map size to get more hash table hits and - * decrease the maximum memory footprint when all retile maps - * are cached in the hash table. - */ - unsigned retile_dim[2] = {in->width, in->height}; - - for (unsigned i = 0; i < 2; i++) { - /* Increase the alignment as the size increases. - * Greater alignment increases retile compute work, - * but decreases maximum memory footprint for the cache. - * - * With this alignment, the worst case memory footprint of - * the cache is: - * 1920x1080: 55 MB - * 2560x1440: 99 MB - * 3840x2160: 305 MB - * - * The worst case size in MB can be computed in Haskell as follows: - * (sum (map get_retile_size (map get_dcc_size (deduplicate (map align_pair - * [(i*16,j*16) | i <- [1..maxwidth`div`16], j <- [1..maxheight`div`16]]))))) - * `div` 1024^2 where alignment x = if x <= 512 then 16 else if x <= 1024 then 32 - * else if x <= 2048 then 64 else 128 align x = (x + (alignment x) - 1) `div` - * (alignment x) * (alignment x) align_pair e = (align (fst e), align (snd e)) - * deduplicate = map head . groupBy (\ a b -> ((fst a) == (fst b)) && ((snd a) - * == (snd b))) . sortBy compare get_dcc_size e = ((fst e) * (snd e) * bpp) `div` 256 - * get_retile_size dcc_size = dcc_size * 2 * (if dcc_size <= 2^16 then 2 else - * 4) bpp = 4; maxwidth = 3840; maxheight = 2160 - */ - if (retile_dim[i] <= 512) - retile_dim[i] = align(retile_dim[i], 16); - else if (retile_dim[i] <= 1024) - retile_dim[i] = align(retile_dim[i], 32); - else if (retile_dim[i] <= 2048) - retile_dim[i] = align(retile_dim[i], 64); - else - retile_dim[i] = align(retile_dim[i], 128); - - /* Don't align more than the DCC pixel alignment. */ - assert(dout.metaBlkWidth >= 128 && dout.metaBlkHeight >= 128); - } - - surf->u.gfx9.color.dcc_retile_num_elements = - DIV_ROUND_UP(retile_dim[0], dout.compressBlkWidth) * - DIV_ROUND_UP(retile_dim[1], dout.compressBlkHeight) * 2; - /* Align the size to 4 (for the compute shader). */ - surf->u.gfx9.color.dcc_retile_num_elements = align(surf->u.gfx9.color.dcc_retile_num_elements, 4); - - /* Compute address mapping from non-displayable to displayable DCC. */ - ADDR2_COMPUTE_DCC_ADDRFROMCOORD_INPUT addrin; - memset(&addrin, 0, sizeof(addrin)); - addrin.size = sizeof(addrin); - addrin.swizzleMode = din.swizzleMode; - addrin.resourceType = din.resourceType; - addrin.bpp = din.bpp; - addrin.numSlices = 1; - addrin.numMipLevels = 1; - addrin.numFrags = 1; - addrin.pitch = dout.pitch; - addrin.height = dout.height; - addrin.compressBlkWidth = dout.compressBlkWidth; - addrin.compressBlkHeight = dout.compressBlkHeight; - addrin.compressBlkDepth = dout.compressBlkDepth; - addrin.metaBlkWidth = dout.metaBlkWidth; - addrin.metaBlkHeight = dout.metaBlkHeight; - addrin.metaBlkDepth = dout.metaBlkDepth; - addrin.dccRamSliceSize = 0; /* Don't care for non-layered images. */ - - surf->u.gfx9.color.dcc_retile_map = ac_compute_dcc_retile_map( - addrlib, info, retile_dim[0], retile_dim[1], surf->u.gfx9.color.dcc.rb_aligned, - surf->u.gfx9.color.dcc.pipe_aligned, surf->u.gfx9.color.dcc_retile_use_uint16, - surf->u.gfx9.color.dcc_retile_num_elements, &addrin); - if (!surf->u.gfx9.color.dcc_retile_map) - return ADDR_OUTOFMEMORY; + ac_copy_dcc_equation(info, &dout, &surf->u.gfx9.color.display_dcc_equation); + surf->u.gfx9.color.dcc.display_equation_valid = true; } } @@ -2427,9 +2115,6 @@ static int gfx9_compute_surface(struct ac_addrlib *addrlib, const struct radeon_ if (AddrSurfInfoIn.flags.stencil) surf->u.gfx9.zs.stencil_offset = 0; surf->cmask_size = 0; - surf->u.gfx9.color.dcc_retile_use_uint16 = false; - surf->u.gfx9.color.dcc_retile_num_elements = 0; - surf->u.gfx9.color.dcc_retile_map = NULL; const bool only_stencil = (surf->flags & RADEON_SURF_SBUFFER) && !(surf->flags & RADEON_SURF_ZBUFFER); @@ -2477,7 +2162,7 @@ static int gfx9_compute_surface(struct ac_addrlib *addrlib, const struct radeon_ (!is_dcc_supported_by_DCN(info, config, surf, surf->u.gfx9.color.dcc.rb_aligned, surf->u.gfx9.color.dcc.pipe_aligned) || /* Don't set is_displayable if displayable DCC is missing. */ - (info->use_display_dcc_with_retile_blit && !surf->u.gfx9.color.dcc_retile_num_elements))) + (info->use_display_dcc_with_retile_blit && !surf->u.gfx9.color.dcc.display_equation_valid))) displayable = false; } surf->is_displayable = displayable; @@ -2622,8 +2307,9 @@ int ac_compute_surface(struct ac_addrlib *addrlib, const struct radeon_info *inf /* It's better when displayable DCC is immediately after * the image due to hw-specific reasons. */ - if (!(surf->flags & RADEON_SURF_Z_OR_SBUFFER) && - info->chip_class >= GFX9 && surf->u.gfx9.color.dcc_retile_num_elements) { + if (info->chip_class >= GFX9 && + !(surf->flags & RADEON_SURF_Z_OR_SBUFFER) && + surf->u.gfx9.color.dcc.display_equation_valid) { /* Add space for the displayable DCC buffer. */ surf->display_dcc_offset = align64(surf->total_size, 1 << surf->u.gfx9.color.display_dcc_alignment_log2); surf->total_size = surf->display_dcc_offset + surf->u.gfx9.color.display_dcc_size; @@ -3110,12 +2796,6 @@ uint64_t ac_surface_get_plane_size(const struct radeon_surf *surf, } } -uint32_t ac_surface_get_retile_map_size(const struct radeon_surf *surf) -{ - return surf->u.gfx9.color.dcc_retile_num_elements * - (surf->u.gfx9.color.dcc_retile_use_uint16 ? 2 : 4); -} - void ac_surface_print_info(FILE *out, const struct radeon_info *info, const struct radeon_surf *surf) { @@ -3211,3 +2891,108 @@ void ac_surface_print_info(FILE *out, const struct radeon_info *info, surf->u.legacy.stencil_tile_split); } } + +nir_ssa_def *ac_nir_dcc_addr_from_coord(nir_builder *b, const struct radeon_info *info, + unsigned bpe, struct gfx9_dcc_equation *equation, + nir_ssa_def *dcc_pitch, nir_ssa_def *dcc_height, + nir_ssa_def *dcc_slice_size, + nir_ssa_def *x, nir_ssa_def *y, nir_ssa_def *z, + nir_ssa_def *sample, nir_ssa_def *pipe_xor) +{ + nir_ssa_def *zero = nir_imm_int(b, 0); + nir_ssa_def *one = nir_imm_int(b, 1); + + if (info->chip_class >= GFX10) { + unsigned bpp_log2 = util_logbase2(bpe); + unsigned meta_block_width_log2 = util_logbase2(equation->meta_block_width); + unsigned meta_block_height_log2 = util_logbase2(equation->meta_block_height); + unsigned blkSizeLog2 = meta_block_width_log2 + meta_block_height_log2 + bpp_log2 - 8; + + nir_ssa_def *coord[] = {x, y, z, 0}; + nir_ssa_def *address = zero; + + for (unsigned i = 1; i < blkSizeLog2 + 1; i++) { + nir_ssa_def *v = zero; + + for (unsigned c = 0; c < 4; c++) { + unsigned index = i * 4 + c - 4; + if (equation->u.gfx10_bits[index]) { + unsigned mask = equation->u.gfx10_bits[index]; + nir_ssa_def *bits = coord[c]; + + while (mask) + v = nir_ixor(b, v, nir_iand(b, nir_ushr_imm(b, bits, u_bit_scan(&mask)), one)); + } + } + + address = nir_ior(b, address, nir_ishl(b, v, nir_imm_int(b, i))); + } + + unsigned blkMask = (1 << blkSizeLog2) - 1; + unsigned pipeMask = (1 << G_0098F8_NUM_PIPES(info->gb_addr_config)) - 1; + unsigned m_pipeInterleaveLog2 = 8 + G_0098F8_PIPE_INTERLEAVE_SIZE_GFX9(info->gb_addr_config); + nir_ssa_def *xb = nir_ushr_imm(b, x, meta_block_width_log2); + nir_ssa_def *yb = nir_ushr_imm(b, y, meta_block_height_log2); + nir_ssa_def *pb = nir_ushr_imm(b, dcc_pitch, meta_block_width_log2); + nir_ssa_def *blkIndex = nir_iadd(b, nir_imul(b, yb, pb), xb); + nir_ssa_def *pipeXor = nir_iand_imm(b, nir_ishl(b, nir_iand_imm(b, pipe_xor, pipeMask), + nir_imm_int(b, m_pipeInterleaveLog2)), blkMask); + + return nir_iadd(b, nir_iadd(b, nir_imul(b, dcc_slice_size, z), + nir_imul(b, blkIndex, nir_ishl(b, one, nir_imm_int(b, blkSizeLog2)))), + nir_ixor(b, nir_ushr(b, address, one), pipeXor)); + } else { + assert(info->chip_class == GFX9); + + unsigned meta_block_width_log2 = util_logbase2(equation->meta_block_width); + unsigned meta_block_height_log2 = util_logbase2(equation->meta_block_height); + unsigned meta_block_depth_log2 = util_logbase2(equation->meta_block_depth); + + unsigned m_pipeInterleaveLog2 = 8 + G_0098F8_PIPE_INTERLEAVE_SIZE_GFX9(info->gb_addr_config); + unsigned numPipeBits = equation->u.gfx9.num_pipe_bits; + nir_ssa_def *pitchInBlock = nir_ushr_imm(b, dcc_pitch, meta_block_width_log2); + nir_ssa_def *sliceSizeInBlock = nir_imul(b, nir_ushr_imm(b, dcc_height, meta_block_height_log2), + pitchInBlock); + + nir_ssa_def *xb = nir_ushr_imm(b, x, meta_block_width_log2); + nir_ssa_def *yb = nir_ushr_imm(b, y, meta_block_height_log2); + nir_ssa_def *zb = nir_ushr_imm(b, z, meta_block_depth_log2); + + nir_ssa_def *blockIndex = nir_iadd(b, nir_iadd(b, nir_imul(b, zb, sliceSizeInBlock), + nir_imul(b, yb, pitchInBlock)), xb); + nir_ssa_def *coords[] = {x, y, z, sample, blockIndex}; + + nir_ssa_def *address = zero; + unsigned num_bits = equation->u.gfx9.num_bits; + assert(num_bits <= 32); + + /* Compute the address up until the last bit that doesn't use the block index. */ + for (unsigned i = 0; i < num_bits - 1; i++) { + nir_ssa_def *xor = zero; + + for (unsigned c = 0; c < 5; c++) { + if (equation->u.gfx9.bit[i].coord[c].dim >= 5) + continue; + + assert(equation->u.gfx9.bit[i].coord[c].ord < 32); + nir_ssa_def *ison = + nir_iand(b, nir_ushr_imm(b, coords[equation->u.gfx9.bit[i].coord[c].dim], + equation->u.gfx9.bit[i].coord[c].ord), one); + + xor = nir_ixor(b, xor, ison); + } + address = nir_ior(b, address, nir_ishl(b, xor, nir_imm_int(b, i))); + } + + /* Fill the remaining bits with the block index. */ + unsigned last = num_bits - 1; + address = nir_ior(b, address, + nir_ishl(b, nir_ushr_imm(b, blockIndex, + equation->u.gfx9.bit[last].coord[0].ord), + nir_imm_int(b, last))); + + nir_ssa_def *pipeXor = nir_iand_imm(b, pipe_xor, (1 << numPipeBits) - 1); + return nir_ixor(b, nir_ushr(b, address, one), + nir_ishl(b, pipeXor, nir_imm_int(b, m_pipeInterleaveLog2))); + } +} diff --git a/src/amd/common/ac_surface.h b/src/amd/common/ac_surface.h index 1679ef9401d..13fac489fbc 100644 --- a/src/amd/common/ac_surface.h +++ b/src/amd/common/ac_surface.h @@ -29,6 +29,11 @@ #include "amd_family.h" #include "util/format/u_format.h" +/* NIR is optional. Some components don't want to include NIR with ac_surface.h. */ +#ifdef AC_SURFACE_INCLUDE_NIR +#include "compiler/nir/nir_builder.h" +#endif + #include #include #include @@ -156,6 +161,7 @@ struct gfx9_surf_meta_flags { uint8_t independent_64B_blocks : 1; uint8_t independent_128B_blocks : 1; uint8_t max_compressed_block_size : 2; + uint8_t display_equation_valid : 1; }; struct gfx9_surf_level { @@ -164,6 +170,50 @@ struct gfx9_surf_level { * where each layer has an array of levels) */ }; +/** + * DCC address equation for doing DCC address computations in shaders. + * + * ac_surface_dcc_address_test.c contains the reference implementation. + * ac_nir_dcc_addr_from_coord is the NIR implementation. + * + * The gfx9 equation doesn't support mipmapping. + * The gfx10 equation doesn't support mipmapping and MSAA. + * (those are also limitations of Addr2ComputeDccAddrFromCoord) + */ +struct gfx9_dcc_equation { + uint16_t meta_block_width; + uint16_t meta_block_height; + uint16_t meta_block_depth; + + union { + /* The gfx9 DCC equation is chip-specific, and it varies with: + * - resource type + * - swizzle_mode + * - bpp + * - number of fragments + * - pipe_aligned + * - rb_aligned + */ + struct { + uint8_t num_bits; + uint8_t num_pipe_bits; + + struct { + struct { + uint8_t dim:3; /* 0..4 */ + uint8_t ord:5; /* 0..31 */ + } coord[5]; /* 0..num_coords-1 */ + } bit[20]; /* 0..num_bits-1 */ + } gfx9; + + /* The gfx10 DCC equation is chip-specific, it requires 64KB_R_X, and it varies with: + * - bpp + * - pipe_aligned + */ + uint16_t gfx10_bits[56]; + } u; +}; + struct gfx9_surf_layout { uint16_t epitch; /* gfx9 only, not on gfx10 */ uint8_t swizzle_mode; /* color or depth */ @@ -220,6 +270,10 @@ struct gfx9_surf_layout { /* CMASK level info (only level 0) */ struct gfx9_surf_level cmask_level0; + + /* For DCC retiling. */ + struct gfx9_dcc_equation dcc_equation; /* 2D only */ + struct gfx9_dcc_equation display_dcc_equation; } color; /* Z/S */ @@ -389,11 +443,19 @@ uint64_t ac_surface_get_plane_stride(enum chip_class chip_class, /* Of the whole miplevel, not an individual layer */ uint64_t ac_surface_get_plane_size(const struct radeon_surf *surf, unsigned plane); -uint32_t ac_surface_get_retile_map_size(const struct radeon_surf *surf); void ac_surface_print_info(FILE *out, const struct radeon_info *info, const struct radeon_surf *surf); +#ifdef AC_SURFACE_INCLUDE_NIR +nir_ssa_def *ac_nir_dcc_addr_from_coord(nir_builder *b, const struct radeon_info *info, + unsigned bpe, struct gfx9_dcc_equation *equation, + nir_ssa_def *dcc_pitch, nir_ssa_def *dcc_height, + nir_ssa_def *dcc_slice_size, + nir_ssa_def *x, nir_ssa_def *y, nir_ssa_def *z, + nir_ssa_def *sample, nir_ssa_def *pipe_xor); +#endif + #ifdef __cplusplus } #endif diff --git a/src/amd/common/ac_surface_dcc_address_test.c b/src/amd/common/ac_surface_dcc_address_test.c index 6868f2cb9ae..ecd4b63ef97 100644 --- a/src/amd/common/ac_surface_dcc_address_test.c +++ b/src/amd/common/ac_surface_dcc_address_test.c @@ -226,6 +226,20 @@ static bool one_dcc_address_test(const char *name, const char *test, ADDR_HANDLE in.height = dout.height; in.pipeXor = xout.pipeBankXor; + /* Validate that the packed gfx9_dcc_equation structure can fit all fields. */ + const struct gfx9_dcc_equation eq; + if (info->chip_class == GFX9) { + /* The bit array is smaller in gfx9_dcc_equation than in addrlib. */ + assert(dout.equation.gfx9.num_bits <= ARRAY_SIZE(eq.u.gfx9.bit)); + } else { + /* gfx9_dcc_equation doesn't store the first 4 and the last 8 elements. They must be 0. */ + for (unsigned i = 0; i < 4; i++) + assert(dout.equation.gfx10_bits[i] == 0); + + for (unsigned i = ARRAY_SIZE(eq.u.gfx10_bits) + 4; i < 68; i++) + assert(dout.equation.gfx10_bits[i] == 0); + } + for (in.x = start_x; in.x < in.pitch; in.x += dout.compressBlkWidth) { for (in.y = start_y; in.y < in.height; in.y += dout.compressBlkHeight) { for (in.slice = start_z; in.slice < depth; in.slice += dout.compressBlkDepth) { diff --git a/src/amd/common/meson.build b/src/amd/common/meson.build index b85453f3668..90476ae778b 100644 --- a/src/amd/common/meson.build +++ b/src/amd/common/meson.build @@ -101,7 +101,7 @@ libamd_common = static_library( ], dependencies : [ dep_thread, dep_elf, dep_libdrm_amdgpu, dep_valgrind, - idep_mesautil, idep_nir_headers + idep_mesautil, idep_nir_headers, idep_nir ], gnu_symbol_visibility : 'hidden', ) diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c index 660f5139a32..f7933c85ef6 100644 --- a/src/amd/vulkan/radv_cmd_buffer.c +++ b/src/amd/vulkan/radv_cmd_buffer.c @@ -6003,7 +6003,7 @@ radv_handle_color_image_transition(struct radv_cmd_buffer *cmd_buffer, struct ra radv_init_color_image_metadata(cmd_buffer, image, src_layout, src_render_loop, dst_layout, dst_render_loop, src_queue_mask, dst_queue_mask, range); - if (image->retile_map) + if (0) radv_retile_transition(cmd_buffer, image, src_layout, dst_layout, dst_queue_mask); return; } @@ -6025,8 +6025,8 @@ radv_handle_color_image_transition(struct radv_cmd_buffer *cmd_buffer, struct ra fast_clear_flushed = true; } - if (image->retile_map) - radv_retile_transition(cmd_buffer, image, src_layout, dst_layout, dst_queue_mask); + /*if (image->retile_map) + radv_retile_transition(cmd_buffer, image, src_layout, dst_layout, dst_queue_mask);*/ } else if (radv_image_has_cmask(image) || radv_image_has_fmask(image)) { if (radv_layout_can_fast_clear(cmd_buffer->device, image, src_layout, src_render_loop, src_queue_mask) && diff --git a/src/amd/vulkan/radv_image.c b/src/amd/vulkan/radv_image.c index bcc8f5768fe..44a2c2ffd39 100644 --- a/src/amd/vulkan/radv_image.c +++ b/src/amd/vulkan/radv_image.c @@ -1330,35 +1330,6 @@ radv_image_reset_layout(struct radv_image *image) } } -static VkResult -radv_image_init_retile_map(struct radv_device *device, struct radv_image *image) -{ - /* If we do a relayout we have to free the old buffer. */ - if (image->retile_map) - device->ws->buffer_destroy(device->ws, image->retile_map); - - image->retile_map = NULL; - if (!radv_image_has_dcc(image) || !image->planes[0].surface.display_dcc_offset || - image->planes[0].surface.display_dcc_offset == image->planes[0].surface.meta_offset) - return VK_SUCCESS; - - uint32_t retile_map_size = ac_surface_get_retile_map_size(&image->planes[0].surface); - image->retile_map = device->ws->buffer_create( - device->ws, retile_map_size, 4096, RADEON_DOMAIN_VRAM, - RADEON_FLAG_READ_ONLY | RADEON_FLAG_NO_INTERPROCESS_SHARING, RADV_BO_PRIORITY_METADATA); - if (!image->retile_map) { - return vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY); - } - void *data = device->ws->buffer_map(image->retile_map); - if (!data) { - device->ws->buffer_destroy(device->ws, image->retile_map); - return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); - } - - memcpy(data, image->planes[0].surface.u.gfx9.color.dcc_retile_map, retile_map_size); - return VK_SUCCESS; -} - VkResult radv_image_create_layout(struct radv_device *device, struct radv_image_create_info create_info, const struct VkImageDrmFormatModifierExplicitCreateInfoEXT *mod_info, @@ -1445,10 +1416,6 @@ radv_image_create_layout(struct radv_device *device, struct radv_image_create_in radv_image_alloc_values(device, image); - result = radv_image_init_retile_map(device, image); - if (result != VK_SUCCESS) - return result; - assert(image->planes[0].surface.surf_size); assert(image->planes[0].surface.modifier == DRM_FORMAT_MOD_INVALID || ac_modifier_has_dcc(image->planes[0].surface.modifier) == radv_image_has_dcc(image)); @@ -1462,9 +1429,6 @@ radv_destroy_image(struct radv_device *device, const VkAllocationCallbacks *pAll if ((image->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) && image->bo) device->ws->buffer_destroy(device->ws, image->bo); - if (image->retile_map) - device->ws->buffer_destroy(device->ws, image->retile_map); - if (image->owned_memory != VK_NULL_HANDLE) { RADV_FROM_HANDLE(radv_device_memory, mem, image->owned_memory); radv_free_memory(device, pAllocator, mem); diff --git a/src/amd/vulkan/radv_meta_dcc_retile.c b/src/amd/vulkan/radv_meta_dcc_retile.c index 5240caf0fe4..92d76fce4bf 100644 --- a/src/amd/vulkan/radv_meta_dcc_retile.c +++ b/src/amd/vulkan/radv_meta_dcc_retile.c @@ -187,7 +187,6 @@ radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image) { struct radv_meta_saved_state saved_state; struct radv_device *device = cmd_buffer->device; - uint32_t retile_map_size = ac_surface_get_retile_map_size(&image->planes[0].surface); assert(image->type == VK_IMAGE_TYPE_2D); assert(image->info.array_size == 1 && image->info.levels == 1); @@ -214,22 +213,8 @@ radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image) struct radv_buffer buffer = {.size = image->size, .bo = image->bo, .offset = image->offset}; - struct radv_buffer retile_buffer = {.size = retile_map_size, - .bo = image->retile_map, - .offset = 0}; - struct radv_buffer_view views[3]; VkBufferView view_handles[3]; - radv_buffer_view_init( - views + 0, cmd_buffer->device, - &(VkBufferViewCreateInfo){ - .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO, - .buffer = radv_buffer_to_handle(&retile_buffer), - .offset = 0, - .range = retile_map_size, - .format = image->planes[0].surface.u.gfx9.color.dcc_retile_use_uint16 ? VK_FORMAT_R16G16_UINT - : VK_FORMAT_R32G32_UINT, - }); radv_buffer_view_init(views + 1, cmd_buffer->device, &(VkBufferViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO, @@ -281,8 +266,8 @@ radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image) /* src+dst pairs count double, so the number of DCC bytes we move is * actually half of dcc_retile_num_elements. */ - radv_unaligned_dispatch(cmd_buffer, image->planes[0].surface.u.gfx9.color.dcc_retile_num_elements / 2, - 1, 1); + /*radv_unaligned_dispatch(cmd_buffer, image->planes[0].surface.u.gfx9.color.dcc_retile_num_elements / 2, + 1, 1);*/ radv_meta_restore(&saved_state, cmd_buffer); diff --git a/src/amd/vulkan/radv_private.h b/src/amd/vulkan/radv_private.h index 8682dad2a4c..72f8601800a 100644 --- a/src/amd/vulkan/radv_private.h +++ b/src/amd/vulkan/radv_private.h @@ -1818,15 +1818,6 @@ struct radv_image { uint64_t fce_pred_offset; uint64_t dcc_pred_offset; - /* On some GPUs DCC needs different tiling of the metadata for - * rendering and for display, so we're stuck with having the metadata - * two times and then occasionally copying one into the other. - * - * The retile map is an array of (src index, dst index) pairs to - * determine how it should be copied between the two. - */ - struct radeon_winsys_bo *retile_map; - /* * Metadata for the TC-compat zrange workaround. If the 32-bit value * stored at this offset is UINT_MAX, the driver will emit diff --git a/src/gallium/drivers/radeonsi/Makefile.sources b/src/gallium/drivers/radeonsi/Makefile.sources index 9a3736c6c23..55ef8085644 100644 --- a/src/gallium/drivers/radeonsi/Makefile.sources +++ b/src/gallium/drivers/radeonsi/Makefile.sources @@ -36,6 +36,7 @@ C_SOURCES := \ si_shader_llvm_tess.c \ si_shader_llvm_vs.c \ si_shader_nir.c \ + si_shaderlib_nir.c \ si_shaderlib_tgsi.c \ si_sqtt.c \ si_state.c \ diff --git a/src/gallium/drivers/radeonsi/meson.build b/src/gallium/drivers/radeonsi/meson.build index 8d7538dec28..44fe59810e9 100644 --- a/src/gallium/drivers/radeonsi/meson.build +++ b/src/gallium/drivers/radeonsi/meson.build @@ -56,6 +56,7 @@ files_libradeonsi = files( 'si_shader_llvm_tess.c', 'si_shader_llvm_vs.c', 'si_shader_nir.c', + 'si_shaderlib_nir.c', 'si_shaderlib_tgsi.c', 'si_sqtt.c', 'si_state.c', diff --git a/src/gallium/drivers/radeonsi/si_compute_blit.c b/src/gallium/drivers/radeonsi/si_compute_blit.c index 5154d5b8046..7ea7893b5bd 100644 --- a/src/gallium/drivers/radeonsi/si_compute_blit.c +++ b/src/gallium/drivers/radeonsi/si_compute_blit.c @@ -664,71 +664,65 @@ void si_retile_dcc(struct si_context *sctx, struct si_texture *tex) /* Save states. */ void *saved_cs = sctx->cs_shader_state.program; - struct pipe_image_view saved_img[3] = {}; + struct pipe_shader_buffer saved_sb = {}; + si_get_shader_buffers(sctx, PIPE_SHADER_COMPUTE, 0, 1, &saved_sb); - for (unsigned i = 0; i < 3; i++) { - util_copy_image_view(&saved_img[i], &sctx->images[PIPE_SHADER_COMPUTE].views[i]); - } + unsigned saved_writable_mask = 0; + if (sctx->const_and_shader_buffers[PIPE_SHADER_COMPUTE].writable_mask & + (1u << si_get_shaderbuf_slot(0))) + saved_writable_mask |= 1 << 0; - /* Set images. */ - bool use_uint16 = tex->surface.u.gfx9.color.dcc_retile_use_uint16; - unsigned num_elements = tex->surface.u.gfx9.color.dcc_retile_num_elements; - struct pipe_image_view img[3]; - - assert(tex->dcc_retile_buffer); + /* Set the DCC buffer. */ assert(tex->surface.meta_offset && tex->surface.meta_offset <= UINT_MAX); assert(tex->surface.display_dcc_offset && tex->surface.display_dcc_offset <= UINT_MAX); + assert(tex->surface.display_dcc_offset < tex->surface.meta_offset); + assert(tex->buffer.bo_size <= UINT_MAX); - for (unsigned i = 0; i < 3; i++) { - img[i].resource = i == 0 ? &tex->dcc_retile_buffer->b.b : &tex->buffer.b.b; - img[i].access = i == 2 ? PIPE_IMAGE_ACCESS_WRITE : PIPE_IMAGE_ACCESS_READ; - img[i].shader_access = SI_IMAGE_ACCESS_AS_BUFFER; - } + struct pipe_shader_buffer sb = {}; + sb.buffer = &tex->buffer.b.b; + sb.buffer_offset = tex->surface.display_dcc_offset; + sb.buffer_size = tex->buffer.bo_size - sb.buffer_offset; + ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, 1, &sb, 0x1); - img[0].format = use_uint16 ? PIPE_FORMAT_R16G16B16A16_UINT : PIPE_FORMAT_R32G32B32A32_UINT; - img[0].u.buf.offset = 0; - img[0].u.buf.size = ac_surface_get_retile_map_size(&tex->surface); + sctx->cs_user_data[0] = tex->surface.meta_offset - tex->surface.display_dcc_offset; + sctx->cs_user_data[1] = (tex->surface.u.gfx9.color.dcc_pitch_max + 1) | + (tex->surface.u.gfx9.color.dcc_height << 16); + sctx->cs_user_data[2] = (tex->surface.u.gfx9.color.display_dcc_pitch_max + 1) | + (tex->surface.u.gfx9.color.display_dcc_height << 16); - img[1].format = PIPE_FORMAT_R8_UINT; - img[1].u.buf.offset = tex->surface.meta_offset; - img[1].u.buf.size = tex->surface.meta_size; + /* There is only 1 shader variant because ac_surface only supports displayable DCC + * with one swizzle mode and 32bpp. + */ + assert(tex->surface.bpe == 4); + assert(sctx->chip_class != GFX9 || tex->surface.u.gfx9.swizzle_mode == 25); /* 64KB_S_X */ + assert(sctx->chip_class != GFX10 || tex->surface.u.gfx9.swizzle_mode == 27); /* 64KB_R_X */ + assert(sctx->chip_class != GFX10_3 || tex->surface.u.gfx9.swizzle_mode == 27); /* 64KB_R_X */ - img[2].format = PIPE_FORMAT_R8_UINT; - img[2].u.buf.offset = tex->surface.display_dcc_offset; - img[2].u.buf.size = tex->surface.u.gfx9.color.display_dcc_size; - - ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 3, 0, img); - - /* Bind the compute shader. */ if (!sctx->cs_dcc_retile) - sctx->cs_dcc_retile = si_create_dcc_retile_cs(ctx); + sctx->cs_dcc_retile = si_create_dcc_retile_cs(sctx, &tex->surface); ctx->bind_compute_state(ctx, sctx->cs_dcc_retile); /* Dispatch compute. */ - /* img[0] has 4 channels per element containing 2 pairs of DCC offsets. */ - unsigned num_threads = num_elements / 4; + unsigned width = DIV_ROUND_UP(tex->buffer.b.b.width0, tex->surface.u.gfx9.color.dcc_block_width); + unsigned height = DIV_ROUND_UP(tex->buffer.b.b.height0, tex->surface.u.gfx9.color.dcc_block_height); struct pipe_grid_info info = {}; - info.block[0] = 64; - info.block[1] = 1; + info.block[0] = 8; + info.block[1] = 8; info.block[2] = 1; - info.grid[0] = DIV_ROUND_UP(num_threads, 64); /* includes the partial block */ - info.grid[1] = 1; + info.last_block[0] = width % info.block[0]; + info.last_block[1] = height % info.block[1]; + info.grid[0] = DIV_ROUND_UP(width, info.block[0]); + info.grid[1] = DIV_ROUND_UP(height, info.block[1]); info.grid[2] = 1; - info.last_block[0] = num_threads % 64; si_launch_grid_internal(sctx, &info, saved_cs, SI_OP_SYNC_BEFORE); - /* Don't flush caches or wait. The driver will wait at the end of this IB, - * and L2 will be flushed by the kernel fence. - */ + /* Don't flush caches. L2 will be flushed by the kernel fence. */ /* Restore states. */ - ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 3, 0, saved_img); - - for (unsigned i = 0; i < 3; i++) { - pipe_resource_reference(&saved_img[i].resource, NULL); - } + ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, 1, &saved_sb, saved_writable_mask); + pipe_resource_reference(&saved_sb.buffer, NULL); } /* Expand FMASK to make it identity, so that image stores can ignore it. */ diff --git a/src/gallium/drivers/radeonsi/si_pipe.h b/src/gallium/drivers/radeonsi/si_pipe.h index 3e86e49cef4..b18dd62e486 100644 --- a/src/gallium/drivers/radeonsi/si_pipe.h +++ b/src/gallium/drivers/radeonsi/si_pipe.h @@ -353,7 +353,6 @@ struct si_texture { * - HTILE buffer (Z/S compression and fast Z/S clear) * - DCC buffer (color compression and new fast color clear) * - displayable DCC buffer (if the DCC buffer is not displayable) - * - DCC retile mapping buffer (if the DCC buffer is not displayable) */ uint64_t cmask_base_address_reg; struct si_resource *cmask_buffer; @@ -417,8 +416,6 @@ struct si_texture { unsigned ps_draw_ratio; /* The number of clears since the last DCC usage analysis. */ unsigned num_slow_clears; - - struct si_resource *dcc_retile_buffer; }; struct si_surface { @@ -1539,6 +1536,9 @@ void si_init_query_functions(struct si_context *sctx); void si_suspend_queries(struct si_context *sctx); void si_resume_queries(struct si_context *sctx); +/* si_shaderlib_nir.c */ +void *si_create_dcc_retile_cs(struct si_context *sctx, struct radeon_surf *surf); + /* si_shaderlib_tgsi.c */ void *si_get_blitter_vs(struct si_context *sctx, enum blitter_attrib_type type, unsigned num_layers); @@ -1552,7 +1552,6 @@ void *si_create_dcc_decompress_cs(struct pipe_context *ctx); void *si_clear_render_target_shader(struct pipe_context *ctx); void *si_clear_render_target_shader_1d_array(struct pipe_context *ctx); void *si_clear_12bytes_buffer_shader(struct pipe_context *ctx); -void *si_create_dcc_retile_cs(struct pipe_context *ctx); void *si_create_fmask_expand_cs(struct pipe_context *ctx, unsigned num_samples, bool is_array); void *si_create_query_result_cs(struct si_context *sctx); void *gfx10_create_sh_query_result_cs(struct si_context *sctx); diff --git a/src/gallium/drivers/radeonsi/si_shaderlib_nir.c b/src/gallium/drivers/radeonsi/si_shaderlib_nir.c new file mode 100644 index 00000000000..49d07fa3f11 --- /dev/null +++ b/src/gallium/drivers/radeonsi/si_shaderlib_nir.c @@ -0,0 +1,102 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * on the rights to use, copy, modify, merge, publish, distribute, sub + * license, and/or sell copies of the Software, and to permit persons to whom + * the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#define AC_SURFACE_INCLUDE_NIR +#include "ac_surface.h" +#include "si_pipe.h" + +static void *create_nir_cs(struct si_context *sctx, nir_builder *b) +{ + nir_shader_gather_info(b->shader, nir_shader_get_entrypoint(b->shader)); + + struct pipe_compute_state state = {0}; + state.ir_type = PIPE_SHADER_IR_NIR; + state.prog = b->shader; + sctx->b.screen->finalize_nir(sctx->b.screen, (void*)state.prog, false); + return sctx->b.create_compute_state(&sctx->b, &state); +} + +static nir_ssa_def *get_global_ids(nir_builder *b, unsigned num_components) +{ + unsigned mask = BITFIELD_MASK(num_components); + + nir_ssa_def *local_ids = nir_channels(b, nir_load_local_invocation_id(b), mask); + nir_ssa_def *block_ids = nir_channels(b, nir_load_work_group_id(b, 32), mask); + nir_ssa_def *block_size = nir_channels(b, nir_load_local_group_size(b), mask); + return nir_iadd(b, nir_imul(b, block_ids, block_size), local_ids); +} + +static void unpack_2x16(nir_builder *b, nir_ssa_def *src, nir_ssa_def **x, nir_ssa_def **y) +{ + *x = nir_iand(b, src, nir_imm_int(b, 0xffff)); + *y = nir_ushr(b, src, nir_imm_int(b, 16)); +} + +void *si_create_dcc_retile_cs(struct si_context *sctx, struct radeon_surf *surf) +{ + const nir_shader_compiler_options *options = + sctx->b.screen->get_compiler_options(sctx->b.screen, PIPE_SHADER_IR_NIR, PIPE_SHADER_COMPUTE); + + nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE, options, "dcc_retile"); + b.shader->info.cs.local_size[0] = 8; + b.shader->info.cs.local_size[1] = 8; + b.shader->info.cs.local_size[2] = 1; + b.shader->info.cs.user_data_components_amd = 3; + b.shader->info.num_ssbos = 1; + + /* Get user data SGPRs. */ + nir_ssa_def *user_sgprs = nir_load_user_data_amd(&b); + + /* Relative offset from the displayable DCC to the non-displayable DCC in the same buffer. */ + nir_ssa_def *src_dcc_offset = nir_channel(&b, user_sgprs, 0); + + nir_ssa_def *src_dcc_pitch, *dst_dcc_pitch, *src_dcc_height, *dst_dcc_height; + unpack_2x16(&b, nir_channel(&b, user_sgprs, 1), &src_dcc_pitch, &src_dcc_height); + unpack_2x16(&b, nir_channel(&b, user_sgprs, 2), &dst_dcc_pitch, &dst_dcc_height); + + /* Get the 2D coordinates. */ + nir_ssa_def *coord = get_global_ids(&b, 2); + nir_ssa_def *zero = nir_imm_int(&b, 0); + + /* Multiply the coordinates by the DCC block size (they are DCC block coordinates). */ + coord = nir_imul(&b, coord, nir_imm_ivec2(&b, surf->u.gfx9.color.dcc_block_width, + surf->u.gfx9.color.dcc_block_height)); + + nir_ssa_def *src_offset = + ac_nir_dcc_addr_from_coord(&b, &sctx->screen->info, surf->bpe, &surf->u.gfx9.color.dcc_equation, + src_dcc_pitch, src_dcc_height, zero, /* DCC slice size */ + nir_channel(&b, coord, 0), nir_channel(&b, coord, 1), /* x, y */ + zero, zero, zero); /* z, sample, pipe_xor */ + src_offset = nir_iadd(&b, src_offset, src_dcc_offset); + nir_ssa_def *value = nir_load_ssbo(&b, 1, 8, zero, src_offset, .align_mul=1); + + nir_ssa_def *dst_offset = + ac_nir_dcc_addr_from_coord(&b, &sctx->screen->info, surf->bpe, &surf->u.gfx9.color.display_dcc_equation, + dst_dcc_pitch, dst_dcc_height, zero, /* DCC slice size */ + nir_channel(&b, coord, 0), nir_channel(&b, coord, 1), /* x, y */ + zero, zero, zero); /* z, sample, pipe_xor */ + nir_store_ssbo(&b, value, zero, dst_offset, .write_mask=0x1, .align_mul=1); + + return create_nir_cs(sctx, &b); +} diff --git a/src/gallium/drivers/radeonsi/si_shaderlib_tgsi.c b/src/gallium/drivers/radeonsi/si_shaderlib_tgsi.c index 7c6f73c4742..a6827092fe4 100644 --- a/src/gallium/drivers/radeonsi/si_shaderlib_tgsi.c +++ b/src/gallium/drivers/radeonsi/si_shaderlib_tgsi.c @@ -255,76 +255,6 @@ void *si_create_clear_buffer_rmw_cs(struct pipe_context *ctx) return ctx->create_compute_state(ctx, &state); } -/* Create a compute shader that copies DCC from one buffer to another - * where each DCC buffer has a different layout. - * - * image[0]: offset remap table (pairs of ), - * 2 pairs are read - * image[1]: DCC source buffer, typed r8_uint - * image[2]: DCC destination buffer, typed r8_uint - */ -void *si_create_dcc_retile_cs(struct pipe_context *ctx) -{ - struct ureg_program *ureg = ureg_create(PIPE_SHADER_COMPUTE); - if (!ureg) - return NULL; - - ureg_property(ureg, TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH, 64); - ureg_property(ureg, TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT, 1); - ureg_property(ureg, TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH, 1); - - /* Compute the global thread ID (in idx). */ - struct ureg_src tid = ureg_DECL_system_value(ureg, TGSI_SEMANTIC_THREAD_ID, 0); - struct ureg_src blk = ureg_DECL_system_value(ureg, TGSI_SEMANTIC_BLOCK_ID, 0); - struct ureg_dst idx = ureg_writemask(ureg_DECL_temporary(ureg), TGSI_WRITEMASK_X); - ureg_UMAD(ureg, idx, blk, ureg_imm1u(ureg, 64), tid); - - /* Load 2 pairs of offsets for DCC load & store. */ - struct ureg_src map = ureg_DECL_image(ureg, 0, TGSI_TEXTURE_BUFFER, 0, false, false); - struct ureg_dst offsets = ureg_DECL_temporary(ureg); - struct ureg_src map_load_args[] = {map, ureg_src(idx)}; - - ureg_memory_insn(ureg, TGSI_OPCODE_LOAD, &offsets, 1, map_load_args, 2, TGSI_MEMORY_RESTRICT, - TGSI_TEXTURE_BUFFER, 0); - - struct ureg_src dcc_src = ureg_DECL_image(ureg, 1, TGSI_TEXTURE_BUFFER, 0, false, false); - struct ureg_dst dcc_dst = - ureg_dst(ureg_DECL_image(ureg, 2, TGSI_TEXTURE_BUFFER, 0, true, false)); - struct ureg_dst dcc_value[2]; - - /* Copy DCC values: - * dst[offsets.y] = src[offsets.x]; - * dst[offsets.w] = src[offsets.z]; - */ - for (unsigned i = 0; i < 2; i++) { - dcc_value[i] = ureg_writemask(ureg_DECL_temporary(ureg), TGSI_WRITEMASK_X); - - struct ureg_src load_args[] = {dcc_src, - ureg_scalar(ureg_src(offsets), TGSI_SWIZZLE_X + i * 2)}; - ureg_memory_insn(ureg, TGSI_OPCODE_LOAD, &dcc_value[i], 1, load_args, 2, TGSI_MEMORY_RESTRICT, - TGSI_TEXTURE_BUFFER, 0); - } - - dcc_dst = ureg_writemask(dcc_dst, TGSI_WRITEMASK_X); - - for (unsigned i = 0; i < 2; i++) { - struct ureg_src store_args[] = {ureg_scalar(ureg_src(offsets), TGSI_SWIZZLE_Y + i * 2), - ureg_src(dcc_value[i])}; - ureg_memory_insn(ureg, TGSI_OPCODE_STORE, &dcc_dst, 1, store_args, 2, TGSI_MEMORY_RESTRICT, - TGSI_TEXTURE_BUFFER, 0); - } - ureg_END(ureg); - - struct pipe_compute_state state = {}; - state.ir_type = PIPE_SHADER_IR_TGSI; - state.prog = ureg_get_tokens(ureg, NULL); - - void *cs = ctx->create_compute_state(ctx, &state); - ureg_destroy(ureg); - ureg_free_tokens(state.prog); - return cs; -} - /* Create the compute shader that is used to collect the results. * * One compute grid with a single thread is launched for every query result diff --git a/src/gallium/drivers/radeonsi/si_texture.c b/src/gallium/drivers/radeonsi/si_texture.c index b61f0b15479..a99d55b8670 100644 --- a/src/gallium/drivers/radeonsi/si_texture.c +++ b/src/gallium/drivers/radeonsi/si_texture.c @@ -500,7 +500,6 @@ static void si_reallocate_texture_inplace(struct si_context *sctx, struct si_tex tex->dcc_gather_statistics = new_tex->dcc_gather_statistics; si_resource_reference(&tex->dcc_separate_buffer, new_tex->dcc_separate_buffer); si_resource_reference(&tex->last_dcc_separate_buffer, new_tex->last_dcc_separate_buffer); - si_resource_reference(&tex->dcc_retile_buffer, new_tex->dcc_retile_buffer); if (new_bind_flag == PIPE_BIND_LINEAR) { assert(!tex->surface.meta_offset); @@ -809,7 +808,6 @@ static void si_texture_destroy(struct pipe_screen *screen, struct pipe_resource radeon_bo_reference(((struct si_screen*)screen)->ws, &resource->buf, NULL); si_resource_reference(&tex->dcc_separate_buffer, NULL); si_resource_reference(&tex->last_dcc_separate_buffer, NULL); - si_resource_reference(&tex->dcc_retile_buffer, NULL); FREE(tex); } @@ -1050,7 +1048,6 @@ static struct si_texture *si_texture_create_object(struct pipe_screen *screen, /* Prepare metadata clears. */ struct si_clear_info clears[4]; unsigned num_clears = 0; - bool aux_ctx_locked_with_copy = false; if (tex->cmask_buffer) { /* Initialize the cmask to 0xCC (= compressed state). */ @@ -1124,55 +1121,19 @@ static struct si_texture *si_texture_create_object(struct pipe_screen *screen, } /* Initialize displayable DCC that requires the retile blit. */ - if (tex->surface.display_dcc_offset) { - if (!(surface->flags & RADEON_SURF_IMPORTED)) { - /* Uninitialized DCC can hang the display hw. - * Clear to white to indicate that. */ - assert(num_clears < ARRAY_SIZE(clears)); - si_init_buffer_clear(&clears[num_clears++], &tex->buffer.b.b, tex->surface.display_dcc_offset, - tex->surface.u.gfx9.color.display_dcc_size, DCC_CLEAR_COLOR_1111); - } - - /* Upload the DCC retile map. - * Use a staging buffer for the upload, because - * the buffer backing the texture is unmappable. - */ - uint32_t dcc_retile_map_size = ac_surface_get_retile_map_size(&tex->surface); - - tex->dcc_retile_buffer = si_aligned_buffer_create(screen, - SI_RESOURCE_FLAG_DRIVER_INTERNAL, PIPE_USAGE_DEFAULT, - dcc_retile_map_size, - sscreen->info.tcc_cache_line_size); - struct si_resource *buf = si_aligned_buffer_create(screen, - SI_RESOURCE_FLAG_DRIVER_INTERNAL, PIPE_USAGE_STREAM, - dcc_retile_map_size, - sscreen->info.tcc_cache_line_size); - void *map = sscreen->ws->buffer_map(sscreen->ws, buf->buf, NULL, PIPE_MAP_WRITE); - - /* Upload the retile map into the staging buffer. */ - memcpy(map, tex->surface.u.gfx9.color.dcc_retile_map, dcc_retile_map_size); - - /* Copy the staging buffer to the buffer backing the texture. */ - struct si_context *sctx = (struct si_context *)sscreen->aux_context; - - simple_mtx_lock(&sscreen->aux_context_lock); - si_copy_buffer(sctx, &tex->dcc_retile_buffer->b.b, &buf->b.b, 0, - 0, buf->b.b.width0, SI_OP_SYNC_AFTER); - aux_ctx_locked_with_copy = true; - - si_resource_reference(&buf, NULL); + if (tex->surface.display_dcc_offset && !(surface->flags & RADEON_SURF_IMPORTED)) { + /* Uninitialized DCC can hang the display hw. + * Clear to white to indicate that. */ + assert(num_clears < ARRAY_SIZE(clears)); + si_init_buffer_clear(&clears[num_clears++], &tex->buffer.b.b, tex->surface.display_dcc_offset, + tex->surface.u.gfx9.color.display_dcc_size, DCC_CLEAR_COLOR_1111); } /* Execute the clears. */ if (num_clears) { - if (!aux_ctx_locked_with_copy) - simple_mtx_lock(&sscreen->aux_context_lock); - + simple_mtx_lock(&sscreen->aux_context_lock); si_execute_clears((struct si_context *)sscreen->aux_context, clears, num_clears, 0); - } - - if (num_clears || aux_ctx_locked_with_copy) { sscreen->aux_context->flush(sscreen->aux_context, NULL, 0); simple_mtx_unlock(&sscreen->aux_context_lock); }