From 4c74077b62da5581d074e45ce9554ef89c236ab7 Mon Sep 17 00:00:00 2001 From: Georg Lehmann Date: Wed, 17 Jan 2024 11:52:10 +0100 Subject: [PATCH] aco: implement rotate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Rhys Perry Reviewed-by: Daniel Schürmann Part-of: --- src/amd/compiler/aco_builder_h.py | 7 + .../compiler/aco_instruction_selection.cpp | 149 ++++++++++++++++++ .../aco_instruction_selection_setup.cpp | 1 + 3 files changed, 157 insertions(+) diff --git a/src/amd/compiler/aco_builder_h.py b/src/amd/compiler/aco_builder_h.py index 4d56d067585..e4518a0f9c5 100644 --- a/src/amd/compiler/aco_builder_h.py +++ b/src/amd/compiler/aco_builder_h.py @@ -97,6 +97,13 @@ ds_pattern_bitmode(unsigned and_mask, unsigned or_mask, unsigned xor_mask) return and_mask | (or_mask << 5) | (xor_mask << 10); } +inline unsigned +ds_pattern_rotate(unsigned delta, unsigned mask) +{ + assert(delta < 32 && mask < 32); + return mask | (delta << 5) | 0xc000; +} + aco_ptr create_s_mov(Definition dst, Operand src); enum sendmsg { diff --git a/src/amd/compiler/aco_instruction_selection.cpp b/src/amd/compiler/aco_instruction_selection.cpp index 6b9a6c740c7..e12bfc2d735 100644 --- a/src/amd/compiler/aco_instruction_selection.cpp +++ b/src/amd/compiler/aco_instruction_selection.cpp @@ -7961,6 +7961,52 @@ inclusive_scan_to_exclusive(isel_context* ctx, ReduceOp op, Definition dst, Temp } } +bool +emit_rotate_by_constant(isel_context* ctx, Temp& dst, Temp src, unsigned cluster_size, + uint64_t delta) +{ + Builder bld(ctx->program, ctx->block); + RegClass rc = src.regClass(); + dst = Temp(0, rc); + delta %= cluster_size; + + if (delta == 0) { + dst = bld.copy(bld.def(rc), src); + } else if (delta * 2 == cluster_size && cluster_size <= 32) { + dst = emit_masked_swizzle(ctx, bld, src, ds_pattern_bitmode(0x1f, 0, delta), true); + } else if (cluster_size == 4) { + unsigned res[4]; + for (unsigned i = 0; i < 4; i++) + res[i] = (i + delta) & 0x3; + uint32_t dpp_ctrl = dpp_quad_perm(res[0], res[1], res[2], res[3]); + if (ctx->program->gfx_level >= GFX8) + dst = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(rc), src, dpp_ctrl); + else + dst = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl); + } else if (cluster_size == 8 && ctx->program->gfx_level >= GFX10) { + uint32_t lane_sel = 0; + for (unsigned i = 0; i < 8; i++) + lane_sel |= ((i + delta) & 0x7) << (i * 3); + dst = bld.vop1_dpp8(aco_opcode::v_mov_b32, bld.def(rc), src, lane_sel); + } else if (cluster_size == 16 && ctx->program->gfx_level >= GFX8) { + dst = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(rc), src, dpp_row_rr(16 - delta)); + } else if (cluster_size <= 32 && ctx->program->gfx_level >= GFX9) { + uint32_t ctrl = ds_pattern_rotate(delta, ~(cluster_size - 1) & 0x1f); + dst = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, ctrl); + } else if (cluster_size == 64) { + bool has_wf_dpp = ctx->program->gfx_level >= GFX8 && ctx->program->gfx_level < GFX10; + if (delta == 32 && ctx->program->gfx_level >= GFX11) { + dst = bld.vop1(aco_opcode::v_permlane64_b32, bld.def(rc), src); + } else if (delta == 1 && has_wf_dpp) { + dst = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(rc), src, dpp_wf_rl1); + } else if (delta == 63 && has_wf_dpp) { + dst = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(rc), src, dpp_wf_rr1); + } + } + + return dst.id() != 0; +} + void emit_interp_center(isel_context* ctx, Temp dst, Temp bary, Temp pos1, Temp pos2) { @@ -8432,6 +8478,109 @@ visit_intrinsic(isel_context* ctx, nir_intrinsic_instr* instr) } break; } + case nir_intrinsic_rotate: { + Temp src = get_ssa_temp(ctx, instr->src[0].ssa); + Temp delta = get_ssa_temp(ctx, instr->src[1].ssa); + Temp dst = get_ssa_temp(ctx, &instr->def); + assert(nir_intrinsic_execution_scope(instr) == SCOPE_SUBGROUP); + assert(instr->def.bit_size > 1 && instr->def.bit_size <= 32); + + if (!nir_src_is_divergent(instr->src[0])) { + emit_uniform_subgroup(ctx, instr, src); + break; + } + + unsigned cluster_size = nir_intrinsic_cluster_size(instr); + cluster_size = util_next_power_of_two( + MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size)); + + if (cluster_size == 1) { + bld.copy(Definition(dst), src); + break; + } + + delta = bld.as_uniform(delta); + src = as_vgpr(ctx, src); + + Temp tmp; + if (nir_src_is_const(instr->src[1]) && + emit_rotate_by_constant(ctx, tmp, src, cluster_size, nir_src_as_uint(instr->src[1]))) { + } else if (cluster_size == 2) { + Temp noswap = + bld.sopc(aco_opcode::s_bitcmp0_b32, bld.def(s1, scc), delta, Operand::c32(0)); + noswap = bool_to_vector_condition(ctx, noswap); + Temp swapped = emit_masked_swizzle(ctx, bld, src, ds_pattern_bitmode(0x1f, 0, 0x1), true); + tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(src.regClass()), swapped, src, noswap); + } else if (ctx->program->gfx_level >= GFX10 && cluster_size <= 16) { + if (cluster_size == 4) /* shift mask already does this for 8/16. */ + delta = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), delta, + Operand::c32(0x3)); + delta = + bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), delta, Operand::c32(2)); + + Temp lo = bld.copy(bld.def(s1), Operand::c32(cluster_size == 4 ? 0x32103210 : 0x76543210)); + Temp hi; + + if (cluster_size <= 8) { + Temp shr = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), lo, delta); + if (cluster_size == 4) { + Temp lotolohi = bld.copy(bld.def(s1), Operand::c32(0x4444)); + Temp lohi = + bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), shr, lotolohi); + lo = bld.sop2(aco_opcode::s_pack_ll_b32_b16, bld.def(s1), shr, lohi); + } else { + delta = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), + Operand::c32(32), delta); + Temp shl = + bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), lo, delta); + lo = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), shr, shl); + } + Temp lotohi = bld.copy(bld.def(s1), Operand::c32(0x88888888)); + hi = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), lo, lotohi); + } else { + hi = bld.copy(bld.def(s1), Operand::c32(0xfedcba98)); + + Temp lohi = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), lo, hi); + + Temp shr = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), lohi, delta); + delta = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand::c32(64), + delta); + Temp shl = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), lohi, delta); + + lohi = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), shr, shl); + lo = bld.tmp(s1); + hi = bld.tmp(s1); + bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), lohi); + } + + Builder::Result ret = + bld.vop3(aco_opcode::v_permlane16_b32, bld.def(src.regClass()), src, lo, hi); + ret->valu().opsel[0] = true; /* set FETCH_INACTIVE */ + ret->valu().opsel[1] = true; /* set BOUND_CTRL */ + tmp = ret; + } else { + /* Fallback to ds_bpermute if we can't find a special instruction. */ + Temp tid = emit_mbcnt(ctx, bld.tmp(v1)); + Temp src_lane = bld.vadd32(bld.def(v1), tid, delta); + + if (ctx->program->gfx_level >= GFX10 && cluster_size == 32) { + /* ds_bpermute is restricted to 32 lanes on GFX10+. */ + Temp index_x4 = + bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand::c32(2u), src_lane); + tmp = bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, src); + } else { + /* Technically, full wave rotate doesn't need this, but it breaks the pseudo ops. */ + src_lane = bld.vop3(aco_opcode::v_bfi_b32, bld.def(v1), Operand::c32(cluster_size - 1), + src_lane, tid); + tmp = emit_bpermute(ctx, bld, src_lane, src); + } + } + + tmp = emit_extract_vector(ctx, tmp, 0, dst.regClass()); + bld.copy(Definition(dst), tmp); + set_wqm(ctx); + break; + } case nir_intrinsic_load_sample_id: { bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->def)), get_arg(ctx, ctx->args->ancillary), Operand::c32(8u), Operand::c32(4u)); diff --git a/src/amd/compiler/aco_instruction_selection_setup.cpp b/src/amd/compiler/aco_instruction_selection_setup.cpp index 846bcf9b143..bcf08075ef2 100644 --- a/src/amd/compiler/aco_instruction_selection_setup.cpp +++ b/src/amd/compiler/aco_instruction_selection_setup.cpp @@ -552,6 +552,7 @@ init_context(isel_context* ctx, nir_shader* shader) case nir_intrinsic_quad_swap_diagonal: case nir_intrinsic_quad_swizzle_amd: case nir_intrinsic_masked_swizzle_amd: + case nir_intrinsic_rotate: case nir_intrinsic_inclusive_scan: case nir_intrinsic_exclusive_scan: case nir_intrinsic_reduce: