From ce33ffd03ac6a19825eb695e430c52d27041c13d Mon Sep 17 00:00:00 2001 From: Rhys Perry Date: Tue, 8 Oct 2024 17:35:44 +0100 Subject: [PATCH] aco: ensure phis uniformized by divergence analysis are SGPR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise, they might not actually be uniform when divergence analysis claimed they are. fossil-db (navi31): Totals from 5118 (6.45% of 79395) affected shaders: MaxWaves: 159520 -> 159560 (+0.03%); split: +0.03%, -0.01% Instrs: 2138456 -> 2165351 (+1.26%); split: -0.02%, +1.28% CodeSize: 11152460 -> 11260340 (+0.97%); split: -0.02%, +0.98% VGPRs: 225144 -> 218124 (-3.12%); split: -3.25%, +0.13% Latency: 11116102 -> 11059208 (-0.51%); split: -0.69%, +0.18% InvThroughput: 1230193 -> 1252148 (+1.78%); split: -0.01%, +1.80% VClause: 39518 -> 39513 (-0.01%); split: -0.49%, +0.48% SClause: 59378 -> 59434 (+0.09%); split: -0.02%, +0.11% Copies: 156172 -> 165997 (+6.29%); split: -0.81%, +7.10% PreSGPRs: 181094 -> 181203 (+0.06%) PreVGPRs: 139731 -> 139393 (-0.24%) VALU: 1220769 -> 1244301 (+1.93%); split: -0.02%, +1.95% SALU: 199567 -> 200240 (+0.34%); split: -0.00%, +0.34% fossil-db (navi21): Totals from 35520 (44.74% of 79395) affected shaders: MaxWaves: 951830 -> 951870 (+0.00%) Instrs: 20227773 -> 20229388 (+0.01%); split: -0.00%, +0.01% CodeSize: 105513724 -> 105379916 (-0.13%); split: -0.13%, +0.01% VGPRs: 1375400 -> 1375232 (-0.01%) Latency: 81013985 -> 81046435 (+0.04%); split: -0.00%, +0.04% InvThroughput: 15273291 -> 15269166 (-0.03%); split: -0.04%, +0.01% VClause: 354310 -> 354314 (+0.00%); split: -0.00%, +0.00% SClause: 417047 -> 417049 (+0.00%); split: -0.00%, +0.00% Copies: 1699486 -> 1699445 (-0.00%); split: -0.01%, +0.01% Branches: 591269 -> 591274 (+0.00%); split: -0.00%, +0.00% PreSGPRs: 1370567 -> 1371062 (+0.04%) PreVGPRs: 1100953 -> 1100716 (-0.02%) VALU: 11075164 -> 11076189 (+0.01%); split: -0.00%, +0.01% SALU: 3647378 -> 3648002 (+0.02%); split: -0.00%, +0.02% Signed-off-by: Rhys Perry Reviewed-by: Daniel Schürmann Part-of: --- .../aco_instruction_selection_setup.cpp | 23 ++++++++++++++++--- src/amd/compiler/aco_lower_phis.cpp | 15 ++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/amd/compiler/aco_instruction_selection_setup.cpp b/src/amd/compiler/aco_instruction_selection_setup.cpp index c1d875049fb..4cc292f98be 100644 --- a/src/amd/compiler/aco_instruction_selection_setup.cpp +++ b/src/amd/compiler/aco_instruction_selection_setup.cpp @@ -585,9 +585,26 @@ init_context(isel_context* ctx, nir_shader* shader) if (phi->def.divergent) { type = RegType::vgpr; } else { - nir_foreach_phi_src (src, phi) { - if (regclasses[src->src.ssa->index].type() == RegType::vgpr) - type = RegType::vgpr; + bool vgpr_src = false; + nir_foreach_phi_src (src, phi) + vgpr_src |= regclasses[src->src.ssa->index].type() == RegType::vgpr; + + if (vgpr_src) { + type = RegType::vgpr; + + /* This might be the case because of nir_divergence_ignore_undef_if_phi_srcs. */ + bool divergent_merge = false; + if (nir_cf_node_prev(&block->cf_node) && + nir_cf_node_prev(&block->cf_node)->type == nir_cf_node_if) { + nir_if* nif = nir_cf_node_as_if(nir_cf_node_prev(&block->cf_node)); + divergent_merge = nir_src_is_divergent(nif->condition); + } + + /* In case of uniform phis after divergent merges, ensure that the dst is an + * SGPR and does not contain undefined values for some invocations. + */ + if (divergent_merge) + type = RegType::sgpr; } } diff --git a/src/amd/compiler/aco_lower_phis.cpp b/src/amd/compiler/aco_lower_phis.cpp index 1aa54f2c00c..a2f4153bd7f 100644 --- a/src/amd/compiler/aco_lower_phis.cpp +++ b/src/amd/compiler/aco_lower_phis.cpp @@ -316,6 +316,21 @@ init_state(Program* program, Block* block, ssa_state* state, aco_ptr& phi) { + if (phi->opcode == aco_opcode::p_phi) { + /* Insert p_as_uniform for VGPR->SGPR phis. */ + Builder bld(program); + for (unsigned i = 0; i < phi->operands.size(); i++) { + if (phi->operands[i].isOfType(RegType::vgpr)) { + Block* pred = &program->blocks[block->logical_preds[i]]; + Temp new_phi_src = bld.tmp(phi->definitions[0].regClass()); + insert_before_logical_end( + pred, bld.pseudo(aco_opcode::p_as_uniform, Definition(new_phi_src), phi->operands[i]) + .get_ptr()); + phi->operands[i].setTemp(new_phi_src); + } + } + } + if (block->linear_preds == block->logical_preds) { phi->opcode = aco_opcode::p_linear_phi; return;