diff --git a/src/amd/compiler/aco_instruction_selection_setup.cpp b/src/amd/compiler/aco_instruction_selection_setup.cpp index c1d875049fb..4cc292f98be 100644 --- a/src/amd/compiler/aco_instruction_selection_setup.cpp +++ b/src/amd/compiler/aco_instruction_selection_setup.cpp @@ -585,9 +585,26 @@ init_context(isel_context* ctx, nir_shader* shader) if (phi->def.divergent) { type = RegType::vgpr; } else { - nir_foreach_phi_src (src, phi) { - if (regclasses[src->src.ssa->index].type() == RegType::vgpr) - type = RegType::vgpr; + bool vgpr_src = false; + nir_foreach_phi_src (src, phi) + vgpr_src |= regclasses[src->src.ssa->index].type() == RegType::vgpr; + + if (vgpr_src) { + type = RegType::vgpr; + + /* This might be the case because of nir_divergence_ignore_undef_if_phi_srcs. */ + bool divergent_merge = false; + if (nir_cf_node_prev(&block->cf_node) && + nir_cf_node_prev(&block->cf_node)->type == nir_cf_node_if) { + nir_if* nif = nir_cf_node_as_if(nir_cf_node_prev(&block->cf_node)); + divergent_merge = nir_src_is_divergent(nif->condition); + } + + /* In case of uniform phis after divergent merges, ensure that the dst is an + * SGPR and does not contain undefined values for some invocations. + */ + if (divergent_merge) + type = RegType::sgpr; } } diff --git a/src/amd/compiler/aco_lower_phis.cpp b/src/amd/compiler/aco_lower_phis.cpp index 1aa54f2c00c..a2f4153bd7f 100644 --- a/src/amd/compiler/aco_lower_phis.cpp +++ b/src/amd/compiler/aco_lower_phis.cpp @@ -316,6 +316,21 @@ init_state(Program* program, Block* block, ssa_state* state, aco_ptr& phi) { + if (phi->opcode == aco_opcode::p_phi) { + /* Insert p_as_uniform for VGPR->SGPR phis. */ + Builder bld(program); + for (unsigned i = 0; i < phi->operands.size(); i++) { + if (phi->operands[i].isOfType(RegType::vgpr)) { + Block* pred = &program->blocks[block->logical_preds[i]]; + Temp new_phi_src = bld.tmp(phi->definitions[0].regClass()); + insert_before_logical_end( + pred, bld.pseudo(aco_opcode::p_as_uniform, Definition(new_phi_src), phi->operands[i]) + .get_ptr()); + phi->operands[i].setTemp(new_phi_src); + } + } + } + if (block->linear_preds == block->logical_preds) { phi->opcode = aco_opcode::p_linear_phi; return;