From d62ad0da5f449f7307518ce4af0cad7ec98b87e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Ol=C5=A1=C3=A1k?= Date: Fri, 12 Apr 2024 22:37:03 -0400 Subject: [PATCH] radeonsi: use MIMG A16 (16-bit image coordinates) in compute blits This reduces VGPR usage for MSAA blits and blitting multiple pixels per lane. Reviewed-by: Pierre-Eric Pelloux-Prayer Part-of: --- .../drivers/radeonsi/si_compute_blit.c | 3 ++ src/gallium/drivers/radeonsi/si_pipe.h | 1 + .../drivers/radeonsi/si_shaderlib_nir.c | 39 ++++++++++++------- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/gallium/drivers/radeonsi/si_compute_blit.c b/src/gallium/drivers/radeonsi/si_compute_blit.c index 11edf8e72e0..df777a13952 100644 --- a/src/gallium/drivers/radeonsi/si_compute_blit.c +++ b/src/gallium/drivers/radeonsi/si_compute_blit.c @@ -1424,6 +1424,7 @@ bool si_compute_blit(struct si_context *sctx, const struct pipe_blit_info *info, if (is_clear) { assert(dst_samples <= 8); options.log_samples = util_logbase2(dst_samples); + options.a16 = sctx->gfx_level >= GFX9 && util_is_box_sint16(&info->dst.box); options.d16 = has_d16 && max_dst_chan_size <= (util_format_is_float(info->dst.format) || util_format_is_pure_integer(info->dst.format) ? 16 : 11); @@ -1455,6 +1456,8 @@ bool si_compute_blit(struct si_context *sctx, const struct pipe_blit_info *info, options.use_integer_one = util_format_is_pure_integer(info->dst.format) && options.last_src_channel < options.last_dst_channel && options.last_dst_channel == 3; + options.a16 = sctx->gfx_level >= GFX9 && util_is_box_sint16(&info->dst.box) && + util_is_box_sint16(&info->src.box); options.d16 = has_d16 && /* Blitting FP16 using D16 has precision issues. Resolving has precision * issues all the way down to R11G11B10_FLOAT. */ diff --git a/src/gallium/drivers/radeonsi/si_pipe.h b/src/gallium/drivers/radeonsi/si_pipe.h index b575e5f4889..8166993b66d 100644 --- a/src/gallium/drivers/radeonsi/si_pipe.h +++ b/src/gallium/drivers/radeonsi/si_pipe.h @@ -1655,6 +1655,7 @@ union si_compute_blit_shader_key { bool dst_is_msaa:1; bool src_has_z:1; bool dst_has_z:1; + bool a16:1; bool d16:1; uint8_t log_samples:2; bool sample0_only:1; /* src is MSAA, dst is not MSAA, log2_samples is ignored */ diff --git a/src/gallium/drivers/radeonsi/si_shaderlib_nir.c b/src/gallium/drivers/radeonsi/si_shaderlib_nir.c index ec5705f0db4..71895fe1299 100644 --- a/src/gallium/drivers/radeonsi/si_shaderlib_nir.c +++ b/src/gallium/drivers/radeonsi/si_shaderlib_nir.c @@ -364,11 +364,13 @@ void *si_create_blit_cs(struct si_context *sctx, const union si_compute_blit_sha unsigned lane_size = lane_width * lane_height * lane_depth; assert(lane_size <= SI_MAX_COMPUTE_BLIT_LANE_SIZE); - nir_def *zero = nir_imm_int(&b, 0); + nir_def *zero_lod = nir_imm_intN_t(&b, 0, options->a16 ? 16 : 32); /* Instructions. */ /* Let's work with 0-based src and dst coordinates (thread IDs) first. */ - nir_def *dst_xyz = nir_pad_vector_imm_int(&b, get_global_ids(&b, options->wg_dim, 32), 0, 3); + unsigned coord_bit_size = options->a16 ? 16 : 32; + nir_def *dst_xyz = get_global_ids(&b, options->wg_dim, coord_bit_size); + dst_xyz = nir_pad_vector_imm_int(&b, dst_xyz, 0, 3); /* If the blit area is unaligned, we launched extra threads to make it aligned. * Skip those threads here. @@ -376,7 +378,8 @@ void *si_create_blit_cs(struct si_context *sctx, const union si_compute_blit_sha nir_if *if_positive = NULL; if (options->has_start_xyz) { nir_def *start_xyz = nir_channel(&b, nir_load_user_data_amd(&b), 3); - start_xyz = nir_trim_vector(&b, nir_u2u32(&b, nir_unpack_32_4x8(&b, start_xyz)), 3); + start_xyz = nir_u2uN(&b, nir_unpack_32_4x8(&b, start_xyz), coord_bit_size); + start_xyz = nir_trim_vector(&b, start_xyz, 3); dst_xyz = nir_isub(&b, dst_xyz, start_xyz); nir_def *is_positive_xyz = nir_ige_imm(&b, dst_xyz, 0); @@ -386,7 +389,8 @@ void *si_create_blit_cs(struct si_context *sctx, const union si_compute_blit_sha if_positive = nir_push_if(&b, is_positive); } - dst_xyz = nir_imul(&b, dst_xyz, nir_imm_ivec3(&b, lane_width, lane_height, lane_depth)); + dst_xyz = nir_imul(&b, dst_xyz, nir_imm_ivec3_intN(&b, lane_width, lane_height, lane_depth, + coord_bit_size)); nir_def *src_xyz = dst_xyz; /* Flip src coordinates. */ @@ -407,7 +411,7 @@ void *si_create_blit_cs(struct si_context *sctx, const union si_compute_blit_sha /* Add box.xyz. */ nir_def *base_coord_src = NULL, *base_coord_dst = NULL; - unpack_2x16_signed(&b, 32, nir_trim_vector(&b, nir_load_user_data_amd(&b), 3), + unpack_2x16_signed(&b, coord_bit_size, nir_trim_vector(&b, nir_load_user_data_amd(&b), 3), &base_coord_src, &base_coord_dst); base_coord_dst = nir_iadd(&b, base_coord_dst, dst_xyz); base_coord_src = nir_iadd(&b, base_coord_src, src_xyz); @@ -467,12 +471,12 @@ void *si_create_blit_cs(struct si_context *sctx, const union si_compute_blit_sha tmp_y = lane_height - 1 - y; coord_src[i] = nir_iadd(&b, base_coord_src, - nir_imm_ivec4(&b, tmp_x, tmp_y, z, 0)); + nir_imm_ivec4_intN(&b, tmp_x, tmp_y, z, 0, coord_bit_size)); if (options->src_is_1d) coord_src[i] = nir_swizzle(&b, coord_src[i], swizzle_xz, 4); if (options->src_is_msaa) { coord_src[i] = nir_vector_insert_imm(&b, coord_src[i], - nir_imm_int(&b, sample), + nir_imm_intN_t(&b, sample, coord_bit_size), num_src_coords - 1); } @@ -482,8 +486,15 @@ void *si_create_blit_cs(struct si_context *sctx, const union si_compute_blit_sha assert(!options->src_is_1d || chan == 0); if (!src_resinfo) { + /* Always use the 32-bit return type because the image dimensions can be + * > INT16_MAX even if the blit box fits within sint16. + */ src_resinfo = nir_image_deref_size(&b, 4, 32, deref_ssa(&b, img_src), - zero); + zero_lod); + if (coord_bit_size == 16) { + src_resinfo = nir_umin_imm(&b, src_resinfo, INT16_MAX); + src_resinfo = nir_i2i16(&b, src_resinfo); + } } nir_def *tmp = nir_channel(&b, coord_src[i], chan); @@ -524,7 +535,7 @@ void *si_create_blit_cs(struct si_context *sctx, const union si_compute_blit_sha sample0[i] = nir_image_deref_load(&b, options->last_src_channel + 1, bit_size, deref_ssa(&b, img_src), coord_src[i * src_samples], nir_channel(&b, coord_src[i * src_samples], - num_src_coords - 1), zero, + num_src_coords - 1), zero_lod, .image_dim = img_src->type->sampler_dimensionality, .image_array = img_src->type->sampler_array); } @@ -535,7 +546,7 @@ void *si_create_blit_cs(struct si_context *sctx, const union si_compute_blit_sha foreach_pixel_in_lane(src_samples, sample, x, y, z, i) { color[i] = nir_image_deref_load(&b, options->last_src_channel + 1, bit_size, deref_ssa(&b, img_src), coord_src[i], - nir_channel(&b, coord_src[i], num_src_coords - 1), zero, + nir_channel(&b, coord_src[i], num_src_coords - 1), zero_lod, .image_dim = img_src->type->sampler_dimensionality, .image_array = img_src->type->sampler_array); } @@ -577,11 +588,13 @@ void *si_create_blit_cs(struct si_context *sctx, const union si_compute_blit_sha /* Initialize dst coordinates, one vector per pixel. */ foreach_pixel_in_lane(dst_samples, sample, x, y, z, i) { - coord_dst[i] = nir_iadd(&b, base_coord_dst, nir_imm_ivec4(&b, x, y, z, 0)); + coord_dst[i] = nir_iadd(&b, base_coord_dst, + nir_imm_ivec4_intN(&b, x, y, z, 0, coord_bit_size)); if (options->dst_is_1d) coord_dst[i] = nir_swizzle(&b, coord_dst[i], swizzle_xz, 4); if (options->dst_is_msaa) { - coord_dst[i] = nir_vector_insert_imm(&b, coord_dst[i], nir_imm_int(&b, sample), + coord_dst[i] = nir_vector_insert_imm(&b, coord_dst[i], + nir_imm_intN_t(&b, sample, coord_bit_size), num_dst_coords - 1); } } @@ -600,7 +613,7 @@ void *si_create_blit_cs(struct si_context *sctx, const union si_compute_blit_sha foreach_pixel_in_lane(dst_samples, sample, x, y, z, i) { nir_bindless_image_store(&b, img_dst_desc, coord_dst[i], nir_channel(&b, coord_dst[i], num_dst_coords - 1), - src_samples > 1 ? color[i] : color[i / dst_samples], zero, + src_samples > 1 ? color[i] : color[i / dst_samples], zero_lod, .image_dim = glsl_get_sampler_dim(img_type[1]), .image_array = glsl_sampler_type_is_array(img_type[1])); }