diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index 51eea72d513..5ccac8810de 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -2830,11 +2830,11 @@ agx_optimize_loop_nir(nir_shader *nir) } while (progress); } -static bool -mem_vectorize_cb(unsigned align_mul, unsigned align_offset, unsigned bit_size, - unsigned num_components, unsigned hole_size, - nir_intrinsic_instr *low, nir_intrinsic_instr *high, - void *data) +bool +agx_mem_vectorize_cb(unsigned align_mul, unsigned align_offset, + unsigned bit_size, unsigned num_components, + unsigned hole_size, nir_intrinsic_instr *low, + nir_intrinsic_instr *high, void *data) { if (hole_size) return false; @@ -3003,7 +3003,7 @@ agx_optimize_nir(nir_shader *nir, bool soft_fault, unsigned *preamble_size) NIR_PASS(_, nir, nir_opt_load_store_vectorize, &(const nir_load_store_vectorize_options){ .modes = nir_var_mem_global | nir_var_mem_constant, - .callback = mem_vectorize_cb, + .callback = agx_mem_vectorize_cb, }); NIR_PASS(_, nir, nir_lower_pack); diff --git a/src/asahi/compiler/agx_compile.h b/src/asahi/compiler/agx_compile.h index 70dd7dabfb8..628a0b25cd6 100644 --- a/src/asahi/compiler/agx_compile.h +++ b/src/asahi/compiler/agx_compile.h @@ -263,6 +263,10 @@ bool agx_nir_lower_interpolation(nir_shader *s); bool agx_nir_lower_cull_distance_vs(struct nir_shader *s); bool agx_nir_lower_cull_distance_fs(struct nir_shader *s, unsigned nr_distances); +bool agx_mem_vectorize_cb(unsigned align_mul, unsigned align_offset, + unsigned bit_size, unsigned num_components, + unsigned hole_size, nir_intrinsic_instr *low, + nir_intrinsic_instr *high, void *data); void agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key, struct util_debug_callback *debug, diff --git a/src/gallium/drivers/asahi/agx_state.c b/src/gallium/drivers/asahi/agx_state.c index bd49ada89f1..e4556bc7f16 100644 --- a/src/gallium/drivers/asahi/agx_state.c +++ b/src/gallium/drivers/asahi/agx_state.c @@ -1868,6 +1868,19 @@ agx_shader_initialize(struct agx_device *dev, struct agx_uncompiled_shader *so, so->info.cull_distance_size = nir->info.cull_distance_array_size; } + /* Vectorize SSBOs before lowering them, since it is significantly harder to + * vectorize the lowered code. + */ + NIR_PASS(_, nir, nir_lower_load_const_to_scalar); + NIR_PASS(_, nir, nir_copy_prop); + + NIR_PASS( + _, nir, nir_opt_load_store_vectorize, + &(const nir_load_store_vectorize_options){ + .modes = nir_var_mem_global | nir_var_mem_constant | nir_var_mem_ssbo, + .callback = agx_mem_vectorize_cb, + }); + NIR_PASS(_, nir, agx_nir_lower_texture, true); NIR_PASS(_, nir, nir_lower_ssbo, NULL);