aco/gfx10: optimize subgroupRotate(x, 32) and subgroupShuffleXor(x, 32)

We don't have v_permlane64_b32 yet, but we can still optimize it using
shared vgprs. Using the DPP16 row mask, we can even avoid writing exec.

With v0 input/output and v24/v25 as shared vgprs, this results in:
v_mov_b32_dpp v24, v0 quad_perm:[0,1,2,3] row_mask:0x3 bank_mask:0xf
v_mov_b32_dpp v25, v0 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
v_mov_b32_dpp v0, v24 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
v_mov_b32_dpp v0, v25 quad_perm:[0,1,2,3] row_mask:0x3 bank_mask:0xf

Reviewed-by: Daniel Schürmann <daniel@schuermann.dev>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/36390>

src/amd/compiler/aco_lower_to_hw_instr.cpp

@@ -1121,6 +1121,35 @@ emit_bpermute_shared_vgpr(Builder& bld, aco_ptr<Instruction>& instr)
    adjust_bpermute_dst(bld, dst, input_data);
 }
 
+void
+emit_permlane64_shared_vgpr(Builder& bld, aco_ptr<Instruction>& instr)
+{
+   /* Manually swap the data between the two halves using two shared VGPRs. */
+   Operand input_data = instr->operands[0];
+   Definition dst = instr->definitions[0];
+   assert(input_data.size() == 1 && input_data.physReg().byte() == 0);
+   assert(dst.size() == 1 && dst.physReg().byte() == 0);
+
+   assert(bld.program->gfx_level >= GFX10 && bld.program->gfx_level <= GFX10_3);
+   assert(bld.program->wave_size == 64);
+
+   unsigned shared_vgpr_reg_0 = align(bld.program->config->num_vgprs, 4) + 256;
+   PhysReg shared_vgpr_lo(shared_vgpr_reg_0);
+   PhysReg shared_vgpr_hi(shared_vgpr_reg_0 + 1);
+
+   /* Copy low and high parts to separate shared vgprs. */
+   bld.vop1_dpp(aco_opcode::v_mov_b32, Definition(shared_vgpr_lo, v1), input_data,
+                dpp_quad_perm(0, 1, 2, 3), 0x3, 0xf, false);
+   bld.vop1_dpp(aco_opcode::v_mov_b32, Definition(shared_vgpr_hi, v1), input_data,
+                dpp_quad_perm(0, 1, 2, 3), 0xc, 0xf, false);
+
+   /* Copy data back to the opposite half. */
+   bld.vop1_dpp(aco_opcode::v_mov_b32, dst, Operand(shared_vgpr_lo, v1), dpp_quad_perm(0, 1, 2, 3),
+                0xc, 0xf, false);
+   bld.vop1_dpp(aco_opcode::v_mov_b32, dst, Operand(shared_vgpr_hi, v1), dpp_quad_perm(0, 1, 2, 3),
+                0x3, 0xf, false);
+}
+
 void
 emit_bpermute_readlane(Builder& bld, aco_ptr<Instruction>& instr)
 {
@@ -2494,6 +2523,10 @@ lower_to_hw_instr(Program* program)
                emit_bpermute_permlane(bld, instr);
                break;
             }
+            case aco_opcode::p_permlane64_shared_vgpr: {
+               emit_permlane64_shared_vgpr(bld, instr);
+               break;
+            };
             case aco_opcode::p_constaddr: {
                unsigned id = instr->definitions[0].tempId();
                PhysReg reg = instr->definitions[0].physReg();

src/amd/compiler/aco_opcodes.py

@@ -428,6 +428,11 @@ insn("p_bpermute_shared_vgpr")
 # operands: linear VGPR, index * 4, input data, same half (bool)
 insn("p_bpermute_permlane")
 
+# simulates v_permlane64_b32 behavior using shared vgprs (for GFX10/10.3)
+# definitions result VGPR
+# operands: input data
+insn("p_permlane64_shared_vgpr")
+
 # creates a lane mask where only the first active lane is selected
 insn("p_elect")
 

src/amd/compiler/aco_optimizer.cpp

@@ -522,6 +522,7 @@ alu_can_accept_constant(const aco_ptr<Instruction>& instr, unsigned operand)
    case aco_opcode::p_bpermute_readlane:
    case aco_opcode::p_bpermute_shared_vgpr:
    case aco_opcode::p_bpermute_permlane:
+   case aco_opcode::p_permlane64_shared_vgpr:
    case aco_opcode::p_interp_gfx11:
    case aco_opcode::p_dual_src_export_gfx11:
    case aco_opcode::v_interp_p1_f32:

src/amd/compiler/aco_register_allocation.cpp

@@ -575,7 +575,8 @@ get_subdword_operand_stride(amd_gfx_level gfx_level, const aco_ptr<Instruction>&
    assert(gfx_level >= GFX8);
    if (instr->isPseudo()) {
       /* v_readfirstlane_b32 cannot use SDWA */
-      if (instr->opcode == aco_opcode::p_as_uniform)
+      if (instr->opcode == aco_opcode::p_as_uniform ||
+          instr->opcode == aco_opcode::p_permlane64_shared_vgpr)
          return 4;
       else
          return rc.bytes() % 2 == 0 ? 2 : 1;
@@ -684,7 +685,8 @@ DefInfo::get_subdword_definition_info(Program* program, const aco_ptr<Instructio
    stride = rc.bytes() % 2 == 0 ? 2 : 1;
 
    if (instr->isPseudo()) {
-      if (instr->opcode == aco_opcode::p_interp_gfx11) {
+      if (instr->opcode == aco_opcode::p_interp_gfx11 ||
+          instr->opcode == aco_opcode::p_permlane64_shared_vgpr) {
          rc = RegClass(RegType::vgpr, rc.size());
          stride = 4;
       }

src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp

@@ -3781,6 +3781,9 @@ emit_rotate_by_constant(isel_context* ctx, Temp& dst, Temp src, unsigned cluster
       bool has_wf_dpp = ctx->program->gfx_level >= GFX8 && ctx->program->gfx_level < GFX10;
       if (delta == 32 && ctx->program->gfx_level >= GFX11) {
          dst = bld.vop1(aco_opcode::v_permlane64_b32, bld.def(rc), src);
+      } else if (delta == 32 && can_use_shared_vgprs(ctx)) {
+         enable_shared_vgprs(ctx);
+         dst = bld.pseudo(aco_opcode::p_permlane64_shared_vgpr, bld.def(rc), src);
       } else if (delta == 1 && has_wf_dpp) {
          dst = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(rc), src, dpp_wf_rl1);
       } else if (delta == 63 && has_wf_dpp) {