diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index 0cf5bfde604..89c9c0a831a 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -2717,7 +2717,27 @@ agx_optimize_nir(nir_shader *nir, unsigned *preamble_size) }); NIR_PASS(_, nir, nir_lower_pack); + nir_convert_to_lcssa(nir, true, true); + NIR_PASS_V(nir, nir_divergence_analysis); bool progress = false; + + static const nir_lower_subgroups_options subgroups_options = { + .ballot_bit_size = 32, + .ballot_components = 1, + .lower_elect = true, + .lower_subgroup_masks = true, + }; + + NIR_PASS(progress, nir, nir_opt_uniform_atomics, true); + NIR_PASS(progress, nir, nir_opt_uniform_subgroup, &subgroups_options); + + /* The above create operations that need lowering/optimizing */ + if (progress) { + NIR_PASS(_, nir, agx_nir_lower_subgroups); + NIR_PASS(_, nir, nir_opt_algebraic); + } + + progress = false; NIR_PASS(progress, nir, agx_nir_lower_address); /* If address lowering made progress, clean up before forming preambles. diff --git a/src/asahi/compiler/agx_nir_lower_subgroups.c b/src/asahi/compiler/agx_nir_lower_subgroups.c index 8ed85d90583..a59316cadc6 100644 --- a/src/asahi/compiler/agx_nir_lower_subgroups.c +++ b/src/asahi/compiler/agx_nir_lower_subgroups.c @@ -64,6 +64,12 @@ lower(nir_builder *b, nir_intrinsic_instr *intr, void *data) return true; } + case nir_intrinsic_last_invocation: { + nir_def *active_mask = nir_ballot(b, 1, 32, nir_imm_true(b)); + nir_def_rewrite_uses(&intr->def, nir_ufind_msb(b, active_mask)); + return true; + } + case nir_intrinsic_vote_ieq: case nir_intrinsic_vote_feq: { /* The common lowering does: