diff --git a/src/freedreno/ir3/ir3_nir.c b/src/freedreno/ir3/ir3_nir.c index c3c72d2c63f..ce193b682db 100644 --- a/src/freedreno/ir3/ir3_nir.c +++ b/src/freedreno/ir3/ir3_nir.c @@ -380,14 +380,6 @@ ir3_optimize_loop(struct ir3_compiler *compiler, }; progress |= OPT(s, nir_opt_offsets, &offset_options); - nir_load_store_vectorize_options vectorize_opts = { - .modes = nir_var_mem_ubo | nir_var_mem_ssbo | nir_var_uniform, - .callback = ir3_nir_should_vectorize_mem, - .robust_modes = options->robust_modes, - .cb_data = compiler, - }; - progress |= OPT(s, nir_opt_load_store_vectorize, &vectorize_opts); - if (lower_flrp != 0) { if (OPT(s, nir_lower_flrp, lower_flrp, false /* always_precise */)) { OPT(s, nir_opt_constant_folding); @@ -652,7 +644,20 @@ ir3_finalize_nir(struct ir3_compiler *compiler, bool idiv_progress = OPT(s, nir_opt_idiv_const, 8); idiv_progress |= OPT(s, nir_lower_idiv, &idiv_options); - if (idiv_progress) + /* Do load/store vectorization after the first opt loop to give us a chance + * to optimize lowered SSBO pointers. Without the first opt loop every + * SSBO load/store with a different pointer looks like it has a different + * descriptor, even when it doesn't. + */ + nir_load_store_vectorize_options vectorize_opts = { + .modes = nir_var_mem_ubo | nir_var_mem_ssbo | nir_var_uniform, + .callback = ir3_nir_should_vectorize_mem, + .robust_modes = options->robust_modes, + .cb_data = compiler, + }; + bool vectorize_progress = OPT(s, nir_opt_load_store_vectorize, &vectorize_opts); + + if (idiv_progress || vectorize_progress) ir3_optimize_loop(compiler, options, s); OPT(s, nir_remove_dead_variables, nir_var_function_temp, NULL); diff --git a/src/freedreno/vulkan/tu_shader.cc b/src/freedreno/vulkan/tu_shader.cc index 3d0fc2fe692..189735ad77b 100644 --- a/src/freedreno/vulkan/tu_shader.cc +++ b/src/freedreno/vulkan/tu_shader.cc @@ -310,23 +310,6 @@ lower_ssbo_ubo_intrinsic(struct tu_device *dev, } } - /* Descriptor index has to be adjusted in the following cases: - * - isam loads, when the 16-bit descriptor cannot also be used for 32-bit - * loads -- next-index descriptor will be able to do that; - * - 8-bit SSBO loads and stores -- next-index descriptor is dedicated to - * storage accesses of that size. - */ - if ((dev->physical_device->info->a6xx.storage_16bit && - !dev->physical_device->info->a6xx.has_isam_v && - intrin->intrinsic == nir_intrinsic_load_ssbo && - (nir_intrinsic_access(intrin) & ACCESS_CAN_REORDER) && - intrin->def.bit_size > 16) || - (dev->physical_device->info->a7xx.storage_8bit && - ((intrin->intrinsic == nir_intrinsic_load_ssbo && intrin->def.bit_size == 8) || - (intrin->intrinsic == nir_intrinsic_store_ssbo && intrin->src[0].ssa->bit_size == 8)))) { - descriptor_idx = nir_iadd_imm(b, descriptor_idx, 1); - } - nir_def *results[MAX_SETS] = { NULL }; if (nir_scalar_is_const(scalar_idx)) { @@ -1099,6 +1082,62 @@ tu_nir_lower_fdm(nir_shader *shader, const struct lower_fdm_options *options) lower_fdm_instr, (void *)options); } +static bool +lower_ssbo_descriptor_instr(nir_builder *b, nir_intrinsic_instr *intrin, + void *cb_data) +{ + struct tu_device *dev = (struct tu_device *)cb_data; + + /* Descriptor index has to be adjusted in the following cases: + * - isam loads, when the 16-bit descriptor cannot also be used for 32-bit + * loads -- next-index descriptor will be able to do that; + * - 8-bit SSBO loads and stores -- next-index descriptor is dedicated to + * storage accesses of that size. + */ + if ((dev->physical_device->info->a6xx.storage_16bit && + !dev->physical_device->info->a6xx.has_isam_v && + intrin->intrinsic == nir_intrinsic_load_ssbo && + (nir_intrinsic_access(intrin) & ACCESS_CAN_REORDER) && + intrin->def.bit_size > 16) || + (dev->physical_device->info->a7xx.storage_8bit && + ((intrin->intrinsic == nir_intrinsic_load_ssbo && intrin->def.bit_size == 8) || + (intrin->intrinsic == nir_intrinsic_store_ssbo && intrin->src[0].ssa->bit_size == 8)))) { + unsigned buffer_src; + if (intrin->intrinsic == nir_intrinsic_store_ssbo) { + /* This has the value first */ + buffer_src = 1; + } else { + buffer_src = 0; + } + + b->cursor = nir_before_instr(&intrin->instr); + nir_def *buffer = intrin->src[buffer_src].ssa; + assert(buffer->parent_instr->type == nir_instr_type_intrinsic); + nir_intrinsic_instr *bindless = + nir_instr_as_intrinsic(buffer->parent_instr); + assert(bindless->intrinsic == nir_intrinsic_bindless_resource_ir3); + nir_def *descriptor_idx = bindless->src[0].ssa; + descriptor_idx = nir_iadd_imm(b, descriptor_idx, 1); + nir_def *new_buffer = + nir_bindless_resource_ir3(b, 32, descriptor_idx, + .desc_set = nir_intrinsic_desc_set(bindless)); + nir_src_rewrite(&intrin->src[buffer_src], new_buffer); + + return true; + } + + return false; +} + +static bool +tu_nir_lower_ssbo_descriptor(nir_shader *shader, + struct tu_device *dev) +{ + return nir_shader_intrinsics_pass(shader, lower_ssbo_descriptor_instr, + nir_metadata_control_flow, + (void *)dev); +} + static void shared_type_info(const struct glsl_type *type, unsigned *size, unsigned *align) { @@ -2620,6 +2659,11 @@ tu_shader_create(struct tu_device *dev, ir3_finalize_nir(dev->compiler, &nir_options, nir); + /* This has to happen after finalizing, so that we know the final bitsize + * after vectorizing. + */ + NIR_PASS(_, nir, tu_nir_lower_ssbo_descriptor, dev); + const struct ir3_shader_options options = { .api_wavesize = key->api_wavesize, .real_wavesize = key->real_wavesize,