diff --git a/src/amd/compiler/aco_lower_to_hw_instr.cpp b/src/amd/compiler/aco_lower_to_hw_instr.cpp index 5fd17e28134..6cfcb5f22df 100644 --- a/src/amd/compiler/aco_lower_to_hw_instr.cpp +++ b/src/amd/compiler/aco_lower_to_hw_instr.cpp @@ -1121,6 +1121,35 @@ emit_bpermute_shared_vgpr(Builder& bld, aco_ptr& instr) adjust_bpermute_dst(bld, dst, input_data); } +void +emit_permlane64_shared_vgpr(Builder& bld, aco_ptr& instr) +{ + /* Manually swap the data between the two halves using two shared VGPRs. */ + Operand input_data = instr->operands[0]; + Definition dst = instr->definitions[0]; + assert(input_data.size() == 1 && input_data.physReg().byte() == 0); + assert(dst.size() == 1 && dst.physReg().byte() == 0); + + assert(bld.program->gfx_level >= GFX10 && bld.program->gfx_level <= GFX10_3); + assert(bld.program->wave_size == 64); + + unsigned shared_vgpr_reg_0 = align(bld.program->config->num_vgprs, 4) + 256; + PhysReg shared_vgpr_lo(shared_vgpr_reg_0); + PhysReg shared_vgpr_hi(shared_vgpr_reg_0 + 1); + + /* Copy low and high parts to separate shared vgprs. */ + bld.vop1_dpp(aco_opcode::v_mov_b32, Definition(shared_vgpr_lo, v1), input_data, + dpp_quad_perm(0, 1, 2, 3), 0x3, 0xf, false); + bld.vop1_dpp(aco_opcode::v_mov_b32, Definition(shared_vgpr_hi, v1), input_data, + dpp_quad_perm(0, 1, 2, 3), 0xc, 0xf, false); + + /* Copy data back to the opposite half. */ + bld.vop1_dpp(aco_opcode::v_mov_b32, dst, Operand(shared_vgpr_lo, v1), dpp_quad_perm(0, 1, 2, 3), + 0xc, 0xf, false); + bld.vop1_dpp(aco_opcode::v_mov_b32, dst, Operand(shared_vgpr_hi, v1), dpp_quad_perm(0, 1, 2, 3), + 0x3, 0xf, false); +} + void emit_bpermute_readlane(Builder& bld, aco_ptr& instr) { @@ -2494,6 +2523,10 @@ lower_to_hw_instr(Program* program) emit_bpermute_permlane(bld, instr); break; } + case aco_opcode::p_permlane64_shared_vgpr: { + emit_permlane64_shared_vgpr(bld, instr); + break; + }; case aco_opcode::p_constaddr: { unsigned id = instr->definitions[0].tempId(); PhysReg reg = instr->definitions[0].physReg(); diff --git a/src/amd/compiler/aco_opcodes.py b/src/amd/compiler/aco_opcodes.py index 25b3e7aede8..78002cd12ff 100644 --- a/src/amd/compiler/aco_opcodes.py +++ b/src/amd/compiler/aco_opcodes.py @@ -428,6 +428,11 @@ insn("p_bpermute_shared_vgpr") # operands: linear VGPR, index * 4, input data, same half (bool) insn("p_bpermute_permlane") +# simulates v_permlane64_b32 behavior using shared vgprs (for GFX10/10.3) +# definitions result VGPR +# operands: input data +insn("p_permlane64_shared_vgpr") + # creates a lane mask where only the first active lane is selected insn("p_elect") diff --git a/src/amd/compiler/aco_optimizer.cpp b/src/amd/compiler/aco_optimizer.cpp index f5f9a76dd28..44e5c355689 100644 --- a/src/amd/compiler/aco_optimizer.cpp +++ b/src/amd/compiler/aco_optimizer.cpp @@ -522,6 +522,7 @@ alu_can_accept_constant(const aco_ptr& instr, unsigned operand) case aco_opcode::p_bpermute_readlane: case aco_opcode::p_bpermute_shared_vgpr: case aco_opcode::p_bpermute_permlane: + case aco_opcode::p_permlane64_shared_vgpr: case aco_opcode::p_interp_gfx11: case aco_opcode::p_dual_src_export_gfx11: case aco_opcode::v_interp_p1_f32: diff --git a/src/amd/compiler/aco_register_allocation.cpp b/src/amd/compiler/aco_register_allocation.cpp index 7c6d8835803..db8e6f1f420 100644 --- a/src/amd/compiler/aco_register_allocation.cpp +++ b/src/amd/compiler/aco_register_allocation.cpp @@ -575,7 +575,8 @@ get_subdword_operand_stride(amd_gfx_level gfx_level, const aco_ptr& assert(gfx_level >= GFX8); if (instr->isPseudo()) { /* v_readfirstlane_b32 cannot use SDWA */ - if (instr->opcode == aco_opcode::p_as_uniform) + if (instr->opcode == aco_opcode::p_as_uniform || + instr->opcode == aco_opcode::p_permlane64_shared_vgpr) return 4; else return rc.bytes() % 2 == 0 ? 2 : 1; @@ -684,7 +685,8 @@ DefInfo::get_subdword_definition_info(Program* program, const aco_ptrisPseudo()) { - if (instr->opcode == aco_opcode::p_interp_gfx11) { + if (instr->opcode == aco_opcode::p_interp_gfx11 || + instr->opcode == aco_opcode::p_permlane64_shared_vgpr) { rc = RegClass(RegType::vgpr, rc.size()); stride = 4; } diff --git a/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp b/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp index 982885e3669..ff0c2a4cb94 100644 --- a/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp +++ b/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp @@ -3781,6 +3781,9 @@ emit_rotate_by_constant(isel_context* ctx, Temp& dst, Temp src, unsigned cluster bool has_wf_dpp = ctx->program->gfx_level >= GFX8 && ctx->program->gfx_level < GFX10; if (delta == 32 && ctx->program->gfx_level >= GFX11) { dst = bld.vop1(aco_opcode::v_permlane64_b32, bld.def(rc), src); + } else if (delta == 32 && can_use_shared_vgprs(ctx)) { + enable_shared_vgprs(ctx); + dst = bld.pseudo(aco_opcode::p_permlane64_shared_vgpr, bld.def(rc), src); } else if (delta == 1 && has_wf_dpp) { dst = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(rc), src, dpp_wf_rl1); } else if (delta == 63 && has_wf_dpp) {