From 588b1ce53316b91d3aa37268c82a6898c036e600 Mon Sep 17 00:00:00 2001 From: Rhys Perry Date: Mon, 4 Mar 2024 12:36:00 +0000 Subject: [PATCH] aco: split instruction assembly into functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rhys Perry Reviewed-by: Georg Lehmann Reviewed-by: Daniel Schürmann Part-of: --- src/amd/compiler/aco_assembler.cpp | 1573 ++++++++++++++++------------ 1 file changed, 876 insertions(+), 697 deletions(-) diff --git a/src/amd/compiler/aco_assembler.cpp b/src/amd/compiler/aco_assembler.cpp index 0b9efdb0d21..41f03843f99 100644 --- a/src/amd/compiler/aco_assembler.cpp +++ b/src/amd/compiler/aco_assembler.cpp @@ -138,6 +138,858 @@ needs_vop3_gfx11(asm_context& ctx, Instruction* instr) return false; } +void +emit_sop2_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + + uint32_t encoding = (0b10 << 30); + encoding |= opcode << 23; + encoding |= !instr->definitions.empty() ? reg(ctx, instr->definitions[0]) << 16 : 0; + encoding |= instr->operands.size() >= 2 ? reg(ctx, instr->operands[1]) << 8 : 0; + encoding |= !instr->operands.empty() ? reg(ctx, instr->operands[0]) : 0; + out.push_back(encoding); +} + +void +emit_sopk_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + SOPK_instruction& sopk = instr->sopk(); + + if (instr->opcode == aco_opcode::s_subvector_loop_begin) { + assert(ctx.gfx_level >= GFX10); + assert(ctx.subvector_begin_pos == -1); + ctx.subvector_begin_pos = out.size(); + } else if (instr->opcode == aco_opcode::s_subvector_loop_end) { + assert(ctx.gfx_level >= GFX10); + assert(ctx.subvector_begin_pos != -1); + /* Adjust s_subvector_loop_begin instruction to the address after the end */ + out[ctx.subvector_begin_pos] |= (out.size() - ctx.subvector_begin_pos); + /* Adjust s_subvector_loop_end instruction to the address after the beginning */ + sopk.imm = (uint16_t)(ctx.subvector_begin_pos - (int)out.size()); + ctx.subvector_begin_pos = -1; + } + + uint32_t encoding = (0b1011 << 28); + encoding |= opcode << 23; + encoding |= !instr->definitions.empty() && !(instr->definitions[0].physReg() == scc) + ? reg(ctx, instr->definitions[0]) << 16 + : !instr->operands.empty() && instr->operands[0].physReg() <= 127 + ? reg(ctx, instr->operands[0]) << 16 + : 0; + encoding |= sopk.imm; + out.push_back(encoding); +} + +void +emit_sop1_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + + uint32_t encoding = (0b101111101 << 23); + encoding |= !instr->definitions.empty() ? reg(ctx, instr->definitions[0]) << 16 : 0; + encoding |= opcode << 8; + encoding |= !instr->operands.empty() ? reg(ctx, instr->operands[0]) : 0; + out.push_back(encoding); +} + +void +emit_sopc_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + + uint32_t encoding = (0b101111110 << 23); + encoding |= opcode << 16; + encoding |= instr->operands.size() == 2 ? reg(ctx, instr->operands[1]) << 8 : 0; + encoding |= !instr->operands.empty() ? reg(ctx, instr->operands[0]) : 0; + out.push_back(encoding); +} + +void +emit_sopp_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + SOPP_instruction& sopp = instr->sopp(); + + uint32_t encoding = (0b101111111 << 23); + encoding |= opcode << 16; + encoding |= (uint16_t)sopp.imm; + if (sopp.block != -1) { + sopp.pass_flags = 0; + ctx.branches.emplace_back(out.size(), &sopp); + } + out.push_back(encoding); +} + +void +emit_smem_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + SMEM_instruction& smem = instr->smem(); + + bool soe = instr->operands.size() >= (!instr->definitions.empty() ? 3 : 4); + bool is_load = !instr->definitions.empty(); + uint32_t encoding = 0; + + if (ctx.gfx_level <= GFX7) { + encoding = (0b11000 << 27); + encoding |= opcode << 22; + encoding |= instr->definitions.size() ? reg(ctx, instr->definitions[0]) << 15 : 0; + encoding |= instr->operands.size() ? (reg(ctx, instr->operands[0]) >> 1) << 9 : 0; + if (instr->operands.size() >= 2) { + if (!instr->operands[1].isConstant()) { + encoding |= reg(ctx, instr->operands[1]); + } else if (instr->operands[1].constantValue() >= 1024) { + encoding |= 255; /* SQ_SRC_LITERAL */ + } else { + encoding |= instr->operands[1].constantValue() >> 2; + encoding |= 1 << 8; + } + } + out.push_back(encoding); + /* SMRD instructions can take a literal on GFX7 */ + if (instr->operands.size() >= 2 && instr->operands[1].isConstant() && + instr->operands[1].constantValue() >= 1024) + out.push_back(instr->operands[1].constantValue() >> 2); + return; + } + + if (ctx.gfx_level <= GFX9) { + encoding = (0b110000 << 26); + assert(!smem.dlc); /* Device-level coherent is not supported on GFX9 and lower */ + encoding |= smem.nv ? 1 << 15 : 0; + } else { + encoding = (0b111101 << 26); + assert(!smem.nv); /* Non-volatile is not supported on GFX10 */ + encoding |= smem.dlc ? 1 << (ctx.gfx_level >= GFX11 ? 13 : 14) : 0; + } + + encoding |= opcode << 18; + encoding |= smem.glc ? 1 << (ctx.gfx_level >= GFX11 ? 14 : 16) : 0; + + if (ctx.gfx_level <= GFX9) { + if (instr->operands.size() >= 2) + encoding |= instr->operands[1].isConstant() ? 1 << 17 : 0; /* IMM - immediate enable */ + } + if (ctx.gfx_level == GFX9) { + encoding |= soe ? 1 << 14 : 0; + } + + if (is_load || instr->operands.size() >= 3) { /* SDATA */ + encoding |= (is_load ? reg(ctx, instr->definitions[0]) : reg(ctx, instr->operands[2])) << 6; + } + if (instr->operands.size() >= 1) { /* SBASE */ + encoding |= reg(ctx, instr->operands[0]) >> 1; + } + + out.push_back(encoding); + encoding = 0; + + int32_t offset = 0; + uint32_t soffset = + ctx.gfx_level >= GFX10 + ? reg(ctx, sgpr_null) /* On GFX10 this is disabled by specifying SGPR_NULL */ + : 0; /* On GFX9, it is disabled by the SOE bit (and it's not present on + GFX8 and below) */ + if (instr->operands.size() >= 2) { + const Operand& op_off1 = instr->operands[1]; + if (ctx.gfx_level <= GFX9) { + offset = op_off1.isConstant() ? op_off1.constantValue() : reg(ctx, op_off1); + } else { + /* GFX10 only supports constants in OFFSET, so put the operand in SOFFSET if it's an + * SGPR */ + if (op_off1.isConstant()) { + offset = op_off1.constantValue(); + } else { + soffset = reg(ctx, op_off1); + assert(!soe); /* There is no place to put the other SGPR offset, if any */ + } + } + + if (soe) { + const Operand& op_off2 = instr->operands.back(); + assert(ctx.gfx_level >= GFX9); /* GFX8 and below don't support specifying a constant + and an SGPR at the same time */ + assert(!op_off2.isConstant()); + soffset = reg(ctx, op_off2); + } + } + encoding |= offset; + encoding |= soffset << 25; + + out.push_back(encoding); +} + +void +emit_vop2_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + VALU_instruction& valu = instr->valu(); + + uint32_t encoding = 0; + encoding |= opcode << 25; + encoding |= reg(ctx, instr->definitions[0], 8) << 17; + encoding |= (valu.opsel[3] ? 128 : 0) << 17; + encoding |= reg(ctx, instr->operands[1], 8) << 9; + encoding |= (valu.opsel[1] ? 128 : 0) << 9; + encoding |= reg(ctx, instr->operands[0]); + encoding |= valu.opsel[0] ? 128 : 0; + out.push_back(encoding); +} + +void +emit_vop1_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + VALU_instruction& valu = instr->valu(); + + uint32_t encoding = (0b0111111 << 25); + if (!instr->definitions.empty()) { + encoding |= reg(ctx, instr->definitions[0], 8) << 17; + encoding |= (valu.opsel[3] ? 128 : 0) << 17; + } + encoding |= opcode << 9; + if (!instr->operands.empty()) { + encoding |= reg(ctx, instr->operands[0]); + encoding |= valu.opsel[0] ? 128 : 0; + } + out.push_back(encoding); +} + +void +emit_vopc_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + VALU_instruction& valu = instr->valu(); + + uint32_t encoding = (0b0111110 << 25); + encoding |= opcode << 17; + encoding |= reg(ctx, instr->operands[1], 8) << 9; + encoding |= (valu.opsel[1] ? 128 : 0) << 9; + encoding |= reg(ctx, instr->operands[0]); + encoding |= valu.opsel[0] ? 128 : 0; + out.push_back(encoding); +} + +void +emit_vintrp_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + VINTRP_instruction& interp = instr->vintrp(); + + uint32_t encoding = 0; + if (instr->opcode == aco_opcode::v_interp_p1ll_f16 || + instr->opcode == aco_opcode::v_interp_p1lv_f16 || + instr->opcode == aco_opcode::v_interp_p2_legacy_f16 || + instr->opcode == aco_opcode::v_interp_p2_f16) { + if (ctx.gfx_level == GFX8 || ctx.gfx_level == GFX9) { + encoding = (0b110100 << 26); + } else if (ctx.gfx_level >= GFX10) { + encoding = (0b110101 << 26); + } else { + unreachable("Unknown gfx_level."); + } + + encoding |= opcode << 16; + encoding |= reg(ctx, instr->definitions[0], 8); + out.push_back(encoding); + + encoding = 0; + encoding |= interp.attribute; + encoding |= interp.component << 6; + encoding |= reg(ctx, instr->operands[0]) << 9; + if (instr->opcode == aco_opcode::v_interp_p2_f16 || + instr->opcode == aco_opcode::v_interp_p2_legacy_f16 || + instr->opcode == aco_opcode::v_interp_p1lv_f16) { + encoding |= reg(ctx, instr->operands[2]) << 18; + } + out.push_back(encoding); + } else { + if (ctx.gfx_level == GFX8 || ctx.gfx_level == GFX9) { + encoding = (0b110101 << 26); /* Vega ISA doc says 110010 but it's wrong */ + } else { + encoding = (0b110010 << 26); + } + + assert(encoding); + encoding |= reg(ctx, instr->definitions[0], 8) << 18; + encoding |= opcode << 16; + encoding |= interp.attribute << 10; + encoding |= interp.component << 8; + if (instr->opcode == aco_opcode::v_interp_mov_f32) + encoding |= (0x3 & instr->operands[0].constantValue()); + else + encoding |= reg(ctx, instr->operands[0], 8); + out.push_back(encoding); + } +} + +void +emit_vinterp_inreg_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + VINTERP_inreg_instruction& interp = instr->vinterp_inreg(); + + uint32_t encoding = (0b11001101 << 24); + encoding |= reg(ctx, instr->definitions[0], 8); + encoding |= (uint32_t)interp.wait_exp << 8; + encoding |= (uint32_t)interp.opsel << 11; + encoding |= (uint32_t)interp.clamp << 15; + encoding |= opcode << 16; + out.push_back(encoding); + + encoding = 0; + for (unsigned i = 0; i < instr->operands.size(); i++) + encoding |= reg(ctx, instr->operands[i]) << (i * 9); + for (unsigned i = 0; i < 3; i++) + encoding |= interp.neg[i] << (29 + i); + out.push_back(encoding); +} + +void +emit_vopd_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + VOPD_instruction& vopd = instr->vopd(); + + uint32_t encoding = (0b110010 << 26); + encoding |= reg(ctx, instr->operands[0]); + if (instr->opcode != aco_opcode::v_dual_mov_b32) + encoding |= reg(ctx, instr->operands[1], 8) << 9; + encoding |= (uint32_t)ctx.opcode[(int)vopd.opy] << 17; + encoding |= opcode << 22; + out.push_back(encoding); + + unsigned opy_start = get_vopd_opy_start(instr); + + encoding = reg(ctx, instr->operands[opy_start]); + if (vopd.opy != aco_opcode::v_dual_mov_b32) + encoding |= reg(ctx, instr->operands[opy_start + 1], 8) << 9; + encoding |= (reg(ctx, instr->definitions[1], 8) >> 1) << 17; + encoding |= reg(ctx, instr->definitions[0], 8) << 24; + out.push_back(encoding); +} + +void +emit_ds_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + DS_instruction& ds = instr->ds(); + + uint32_t encoding = (0b110110 << 26); + if (ctx.gfx_level == GFX8 || ctx.gfx_level == GFX9) { + encoding |= opcode << 17; + encoding |= (ds.gds ? 1 : 0) << 16; + } else { + encoding |= opcode << 18; + encoding |= (ds.gds ? 1 : 0) << 17; + } + encoding |= ((0xFF & ds.offset1) << 8); + encoding |= (0xFFFF & ds.offset0); + out.push_back(encoding); + encoding = 0; + if (!instr->definitions.empty()) + encoding |= reg(ctx, instr->definitions[0], 8) << 24; + if (instr->operands.size() >= 3 && instr->operands[2].physReg() != m0) + encoding |= reg(ctx, instr->operands[2], 8) << 16; + if (instr->operands.size() >= 2 && instr->operands[1].physReg() != m0) + encoding |= reg(ctx, instr->operands[1], 8) << 8; + if (!instr->operands[0].isUndefined()) + encoding |= reg(ctx, instr->operands[0], 8); + out.push_back(encoding); +} + +void +emit_ldsdir_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + LDSDIR_instruction& dir = instr->ldsdir(); + + uint32_t encoding = (0b11001110 << 24); + encoding |= opcode << 20; + encoding |= (uint32_t)dir.wait_vdst << 16; + encoding |= (uint32_t)dir.attr << 10; + encoding |= (uint32_t)dir.attr_chan << 8; + encoding |= reg(ctx, instr->definitions[0], 8); + out.push_back(encoding); +} + +void +emit_mubuf_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + MUBUF_instruction& mubuf = instr->mubuf(); + + uint32_t encoding = (0b111000 << 26); + if (ctx.gfx_level >= GFX11 && mubuf.lds) /* GFX11 has separate opcodes for LDS loads */ + opcode = opcode == 0 ? 0x32 : (opcode + 0x1d); + else + encoding |= (mubuf.lds ? 1 : 0) << 16; + encoding |= opcode << 18; + encoding |= (mubuf.glc ? 1 : 0) << 14; + if (ctx.gfx_level <= GFX10_3) + encoding |= (mubuf.idxen ? 1 : 0) << 13; + assert(!mubuf.addr64 || ctx.gfx_level <= GFX7); + if (ctx.gfx_level == GFX6 || ctx.gfx_level == GFX7) + encoding |= (mubuf.addr64 ? 1 : 0) << 15; + if (ctx.gfx_level <= GFX10_3) + encoding |= (mubuf.offen ? 1 : 0) << 12; + if (ctx.gfx_level == GFX8 || ctx.gfx_level == GFX9) { + assert(!mubuf.dlc); /* Device-level coherent is not supported on GFX9 and lower */ + encoding |= (mubuf.slc ? 1 : 0) << 17; + } else if (ctx.gfx_level >= GFX11) { + encoding |= (mubuf.slc ? 1 : 0) << 12; + encoding |= (mubuf.dlc ? 1 : 0) << 13; + } else if (ctx.gfx_level >= GFX10) { + encoding |= (mubuf.dlc ? 1 : 0) << 15; + } + encoding |= 0x0FFF & mubuf.offset; + out.push_back(encoding); + encoding = 0; + if (ctx.gfx_level <= GFX7 || (ctx.gfx_level >= GFX10 && ctx.gfx_level <= GFX10_3)) { + encoding |= (mubuf.slc ? 1 : 0) << 22; + } + encoding |= reg(ctx, instr->operands[2]) << 24; + if (ctx.gfx_level >= GFX11) { + encoding |= (mubuf.tfe ? 1 : 0) << 21; + encoding |= (mubuf.offen ? 1 : 0) << 22; + encoding |= (mubuf.idxen ? 1 : 0) << 23; + } else { + encoding |= (mubuf.tfe ? 1 : 0) << 23; + } + encoding |= (reg(ctx, instr->operands[0]) >> 2) << 16; + if (instr->operands.size() > 3 && !mubuf.lds) + encoding |= reg(ctx, instr->operands[3], 8) << 8; + else if (!mubuf.lds) + encoding |= reg(ctx, instr->definitions[0], 8) << 8; + encoding |= reg(ctx, instr->operands[1], 8); + out.push_back(encoding); +} + +void +emit_mtbuf_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + MTBUF_instruction& mtbuf = instr->mtbuf(); + + uint32_t img_format = ac_get_tbuffer_format(ctx.gfx_level, mtbuf.dfmt, mtbuf.nfmt); + + uint32_t encoding = (0b111010 << 26); + assert(img_format <= 0x7F); + assert(!mtbuf.dlc || ctx.gfx_level >= GFX10); + if (ctx.gfx_level >= GFX11) { + encoding |= (mtbuf.slc ? 1 : 0) << 12; + encoding |= (mtbuf.dlc ? 1 : 0) << 13; + } else { + /* DLC bit replaces one bit of the OPCODE on GFX10 */ + encoding |= (mtbuf.dlc ? 1 : 0) << 15; + } + if (ctx.gfx_level <= GFX10_3) { + encoding |= (mtbuf.idxen ? 1 : 0) << 13; + encoding |= (mtbuf.offen ? 1 : 0) << 12; + } + encoding |= (mtbuf.glc ? 1 : 0) << 14; + encoding |= 0x0FFF & mtbuf.offset; + encoding |= (img_format << 19); /* Handles both the GFX10 FORMAT and the old NFMT+DFMT */ + + if (ctx.gfx_level == GFX8 || ctx.gfx_level == GFX9 || ctx.gfx_level >= GFX11) { + encoding |= opcode << 15; + } else { + encoding |= (opcode & 0x07) << 16; /* 3 LSBs of 4-bit OPCODE */ + } + + out.push_back(encoding); + encoding = 0; + + encoding |= reg(ctx, instr->operands[2]) << 24; + if (ctx.gfx_level >= GFX11) { + encoding |= (mtbuf.tfe ? 1 : 0) << 21; + encoding |= (mtbuf.offen ? 1 : 0) << 22; + encoding |= (mtbuf.idxen ? 1 : 0) << 23; + } else { + encoding |= (mtbuf.tfe ? 1 : 0) << 23; + encoding |= (mtbuf.slc ? 1 : 0) << 22; + } + encoding |= (reg(ctx, instr->operands[0]) >> 2) << 16; + if (instr->operands.size() > 3) + encoding |= reg(ctx, instr->operands[3], 8) << 8; + else + encoding |= reg(ctx, instr->definitions[0], 8) << 8; + encoding |= reg(ctx, instr->operands[1], 8); + + if (ctx.gfx_level >= GFX10) { + encoding |= (((opcode & 0x08) >> 3) << 21); /* MSB of 4-bit OPCODE */ + } + + out.push_back(encoding); +} + +void +emit_mimg_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + MIMG_instruction& mimg = instr->mimg(); + + unsigned nsa_dwords = get_mimg_nsa_dwords(instr); + assert(!nsa_dwords || ctx.gfx_level >= GFX10); + + uint32_t encoding = (0b111100 << 26); + if (ctx.gfx_level >= GFX11) { /* GFX11: rearranges most fields */ + assert(nsa_dwords <= 1); + encoding |= nsa_dwords; + encoding |= mimg.dim << 2; + encoding |= mimg.unrm ? 1 << 7 : 0; + encoding |= (0xF & mimg.dmask) << 8; + encoding |= mimg.slc ? 1 << 12 : 0; + encoding |= mimg.dlc ? 1 << 13 : 0; + encoding |= mimg.glc ? 1 << 14 : 0; + encoding |= mimg.r128 ? 1 << 15 : 0; + encoding |= mimg.a16 ? 1 << 16 : 0; + encoding |= mimg.d16 ? 1 << 17 : 0; + encoding |= (opcode & 0xFF) << 18; + } else { + encoding |= mimg.slc ? 1 << 25 : 0; + encoding |= (opcode & 0x7f) << 18; + encoding |= (opcode >> 7) & 1; + encoding |= mimg.lwe ? 1 << 17 : 0; + encoding |= mimg.tfe ? 1 << 16 : 0; + encoding |= mimg.glc ? 1 << 13 : 0; + encoding |= mimg.unrm ? 1 << 12 : 0; + if (ctx.gfx_level <= GFX9) { + assert(!mimg.dlc); /* Device-level coherent is not supported on GFX9 and lower */ + assert(!mimg.r128); + encoding |= mimg.a16 ? 1 << 15 : 0; + encoding |= mimg.da ? 1 << 14 : 0; + } else { + encoding |= mimg.r128 ? 1 << 15 + : 0; /* GFX10: A16 moved to 2nd word, R128 replaces it in 1st word */ + encoding |= nsa_dwords << 1; + encoding |= mimg.dim << 3; /* GFX10: dimensionality instead of declare array */ + encoding |= mimg.dlc ? 1 << 7 : 0; + } + encoding |= (0xF & mimg.dmask) << 8; + } + out.push_back(encoding); + + encoding = reg(ctx, instr->operands[3], 8); /* VADDR */ + if (!instr->definitions.empty()) { + encoding |= reg(ctx, instr->definitions[0], 8) << 8; /* VDATA */ + } else if (!instr->operands[2].isUndefined()) { + encoding |= reg(ctx, instr->operands[2], 8) << 8; /* VDATA */ + } + encoding |= (0x1F & (reg(ctx, instr->operands[0]) >> 2)) << 16; /* T# (resource) */ + + assert(!mimg.d16 || ctx.gfx_level >= GFX9); + if (ctx.gfx_level >= GFX11) { + if (!instr->operands[1].isUndefined()) + encoding |= (0x1F & (reg(ctx, instr->operands[1]) >> 2)) << 26; /* sampler */ + + encoding |= mimg.tfe ? 1 << 21 : 0; + encoding |= mimg.lwe ? 1 << 22 : 0; + } else { + if (!instr->operands[1].isUndefined()) + encoding |= (0x1F & (reg(ctx, instr->operands[1]) >> 2)) << 21; /* sampler */ + + encoding |= mimg.d16 ? 1 << 31 : 0; + if (ctx.gfx_level >= GFX10) { + /* GFX10: A16 still exists, but is in a different place */ + encoding |= mimg.a16 ? 1 << 30 : 0; + } + } + + out.push_back(encoding); + + if (nsa_dwords) { + out.resize(out.size() + nsa_dwords); + std::vector::iterator nsa = std::prev(out.end(), nsa_dwords); + for (unsigned i = 0; i < instr->operands.size() - 4u; i++) + nsa[i / 4] |= reg(ctx, instr->operands[4 + i], 8) << (i % 4 * 8); + } +} + +void +emit_flatlike_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + FLAT_instruction& flat = instr->flatlike(); + + uint32_t encoding = (0b110111 << 26); + encoding |= opcode << 18; + if (ctx.gfx_level == GFX9 || ctx.gfx_level >= GFX11) { + if (instr->isFlat()) + assert(flat.offset <= 0xfff); + else + assert(flat.offset >= -4096 && flat.offset < 4096); + encoding |= flat.offset & 0x1fff; + } else if (ctx.gfx_level <= GFX8 || instr->isFlat()) { + /* GFX10 has a 12-bit immediate OFFSET field, + * but it has a hw bug: it ignores the offset, called FlatSegmentOffsetBug + */ + assert(flat.offset == 0); + } else { + assert(flat.offset >= -2048 && flat.offset <= 2047); + encoding |= flat.offset & 0xfff; + } + if (instr->isScratch()) + encoding |= 1 << (ctx.gfx_level >= GFX11 ? 16 : 14); + else if (instr->isGlobal()) + encoding |= 2 << (ctx.gfx_level >= GFX11 ? 16 : 14); + encoding |= flat.lds ? 1 << 13 : 0; + encoding |= flat.glc ? 1 << (ctx.gfx_level >= GFX11 ? 14 : 16) : 0; + encoding |= flat.slc ? 1 << (ctx.gfx_level >= GFX11 ? 15 : 17) : 0; + if (ctx.gfx_level >= GFX10) { + assert(!flat.nv); + encoding |= flat.dlc ? 1 << (ctx.gfx_level >= GFX11 ? 13 : 12) : 0; + } else { + assert(!flat.dlc); + } + out.push_back(encoding); + encoding = reg(ctx, instr->operands[0], 8); + if (!instr->definitions.empty()) + encoding |= reg(ctx, instr->definitions[0], 8) << 24; + if (instr->operands.size() >= 3) + encoding |= reg(ctx, instr->operands[2], 8) << 8; + if (!instr->operands[1].isUndefined()) { + assert(ctx.gfx_level >= GFX10 || instr->operands[1].physReg() != 0x7F); + assert(instr->format != Format::FLAT); + encoding |= reg(ctx, instr->operands[1], 8) << 16; + } else if (instr->format != Format::FLAT || + ctx.gfx_level >= GFX10) { /* SADDR is actually used with FLAT on GFX10 */ + /* For GFX10.3 scratch, 0x7F disables both ADDR and SADDR, unlike sgpr_null, which only + * disables SADDR. On GFX11, this was replaced with SVE. + */ + if (ctx.gfx_level <= GFX9 || + (instr->isScratch() && instr->operands[0].isUndefined() && ctx.gfx_level < GFX11)) + encoding |= 0x7F << 16; + else + encoding |= reg(ctx, sgpr_null) << 16; + } + if (ctx.gfx_level >= GFX11 && instr->isScratch()) + encoding |= !instr->operands[0].isUndefined() ? 1 << 23 : 0; + else + encoding |= flat.nv ? 1 << 23 : 0; + out.push_back(encoding); +} + +void +emit_exp_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + Export_instruction& exp = instr->exp(); + uint32_t encoding; + if (ctx.gfx_level == GFX8 || ctx.gfx_level == GFX9) { + encoding = (0b110001 << 26); + } else { + encoding = (0b111110 << 26); + } + + if (ctx.gfx_level >= GFX11) { + encoding |= exp.row_en ? 0b1 << 13 : 0; + } else { + encoding |= exp.valid_mask ? 0b1 << 12 : 0; + encoding |= exp.compressed ? 0b1 << 10 : 0; + } + encoding |= exp.done ? 0b1 << 11 : 0; + encoding |= exp.dest << 4; + encoding |= exp.enabled_mask; + out.push_back(encoding); + encoding = reg(ctx, exp.operands[0], 8); + encoding |= reg(ctx, exp.operands[1], 8) << 8; + encoding |= reg(ctx, exp.operands[2], 8) << 16; + encoding |= reg(ctx, exp.operands[3], 8) << 24; + out.push_back(encoding); +} + +void emit_instruction(asm_context& ctx, std::vector& out, Instruction* instr); + +void +emit_dpp16_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + assert(ctx.gfx_level >= GFX8); + DPP16_instruction& dpp = instr->dpp16(); + + /* first emit the instruction without the DPP operand */ + Operand dpp_op = instr->operands[0]; + instr->operands[0] = Operand(PhysReg{250}, v1); + instr->format = (Format)((uint16_t)instr->format & ~(uint16_t)Format::DPP16); + emit_instruction(ctx, out, instr); + uint32_t encoding = (0xF & dpp.row_mask) << 28; + encoding |= (0xF & dpp.bank_mask) << 24; + encoding |= dpp.abs[1] << 23; + encoding |= dpp.neg[1] << 22; + encoding |= dpp.abs[0] << 21; + encoding |= dpp.neg[0] << 20; + encoding |= dpp.fetch_inactive << 18; + encoding |= dpp.bound_ctrl << 19; + encoding |= dpp.dpp_ctrl << 8; + encoding |= reg(ctx, dpp_op, 8); + encoding |= dpp.opsel[0] && !instr->isVOP3() ? 128 : 0; + out.push_back(encoding); +} + +void +emit_dpp8_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + assert(ctx.gfx_level >= GFX10); + DPP8_instruction& dpp = instr->dpp8(); + + /* first emit the instruction without the DPP operand */ + Operand dpp_op = instr->operands[0]; + instr->operands[0] = Operand(PhysReg{233u + dpp.fetch_inactive}, v1); + instr->format = (Format)((uint16_t)instr->format & ~(uint16_t)Format::DPP8); + emit_instruction(ctx, out, instr); + uint32_t encoding = reg(ctx, dpp_op, 8); + encoding |= dpp.opsel[0] && !instr->isVOP3() ? 128 : 0; + encoding |= dpp.lane_sel << 8; + out.push_back(encoding); +} + +void +emit_vop3_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + VALU_instruction& vop3 = instr->valu(); + + if (instr->isVOP2()) { + opcode = opcode + 0x100; + } else if (instr->isVOP1()) { + if (ctx.gfx_level == GFX8 || ctx.gfx_level == GFX9) + opcode = opcode + 0x140; + else + opcode = opcode + 0x180; + } else if (instr->isVOPC()) { + opcode = opcode + 0x0; + } else if (instr->isVINTRP()) { + opcode = opcode + 0x270; + } + + uint32_t encoding; + if (ctx.gfx_level <= GFX9) { + encoding = (0b110100 << 26); + } else if (ctx.gfx_level >= GFX10) { + encoding = (0b110101 << 26); + } else { + unreachable("Unknown gfx_level."); + } + + if (ctx.gfx_level <= GFX7) { + encoding |= opcode << 17; + encoding |= (vop3.clamp ? 1 : 0) << 11; + } else { + encoding |= opcode << 16; + encoding |= (vop3.clamp ? 1 : 0) << 15; + } + encoding |= vop3.opsel << 11; + for (unsigned i = 0; i < 3; i++) + encoding |= vop3.abs[i] << (8 + i); + /* On GFX9 and older, v_cmpx implicitly writes exec besides writing an SGPR pair. + * On GFX10 and newer, v_cmpx always writes just exec. + */ + if (instr->definitions.size() == 2 && instr->isVOPC()) + assert(ctx.gfx_level <= GFX9 && instr->definitions[1].physReg() == exec); + else if (instr->definitions.size() == 2) + encoding |= reg(ctx, instr->definitions[1]) << 8; + encoding |= reg(ctx, instr->definitions[0], 8); + out.push_back(encoding); + encoding = 0; + if (instr->opcode == aco_opcode::v_interp_mov_f32) { + encoding = 0x3 & instr->operands[0].constantValue(); + } else if (instr->opcode == aco_opcode::v_writelane_b32_e64) { + encoding |= reg(ctx, instr->operands[0]) << 0; + encoding |= reg(ctx, instr->operands[1]) << 9; + /* Encoding src2 works fine with hardware but breaks some disassemblers. */ + } else { + for (unsigned i = 0; i < instr->operands.size(); i++) + encoding |= reg(ctx, instr->operands[i]) << (i * 9); + } + encoding |= vop3.omod << 27; + for (unsigned i = 0; i < 3; i++) + encoding |= vop3.neg[i] << (29 + i); + out.push_back(encoding); +} + +void +emit_vop3p_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + VALU_instruction& vop3 = instr->valu(); + + uint32_t encoding; + if (ctx.gfx_level == GFX9) { + encoding = (0b110100111 << 23); + } else if (ctx.gfx_level >= GFX10) { + encoding = (0b110011 << 26); + } else { + unreachable("Unknown gfx_level."); + } + + encoding |= opcode << 16; + encoding |= (vop3.clamp ? 1 : 0) << 15; + encoding |= vop3.opsel_lo << 11; + encoding |= ((vop3.opsel_hi & 0x4) ? 1 : 0) << 14; + for (unsigned i = 0; i < 3; i++) + encoding |= vop3.neg_hi[i] << (8 + i); + encoding |= reg(ctx, instr->definitions[0], 8); + out.push_back(encoding); + encoding = 0; + for (unsigned i = 0; i < instr->operands.size(); i++) + encoding |= reg(ctx, instr->operands[i]) << (i * 9); + encoding |= (vop3.opsel_hi & 0x3) << 27; + for (unsigned i = 0; i < 3; i++) + encoding |= vop3.neg_lo[i] << (29 + i); + out.push_back(encoding); +} + +void +emit_sdwa_instruction(asm_context& ctx, std::vector& out, Instruction* instr) +{ + assert(ctx.gfx_level >= GFX8 && ctx.gfx_level < GFX11); + SDWA_instruction& sdwa = instr->sdwa(); + + /* first emit the instruction without the SDWA operand */ + Operand sdwa_op = instr->operands[0]; + instr->operands[0] = Operand(PhysReg{249}, v1); + instr->format = (Format)((uint16_t)instr->format & ~(uint16_t)Format::SDWA); + emit_instruction(ctx, out, instr); + + uint32_t encoding = 0; + + if (instr->isVOPC()) { + if (instr->definitions[0].physReg() != + (ctx.gfx_level >= GFX10 && is_cmpx(instr->opcode) ? exec : vcc)) { + encoding |= reg(ctx, instr->definitions[0]) << 8; + encoding |= 1 << 15; + } + encoding |= (sdwa.clamp ? 1 : 0) << 13; + } else { + encoding |= sdwa.dst_sel.to_sdwa_sel(instr->definitions[0].physReg().byte()) << 8; + uint32_t dst_u = sdwa.dst_sel.sign_extend() ? 1 : 0; + if (instr->definitions[0].bytes() < 4) /* dst_preserve */ + dst_u = 2; + encoding |= dst_u << 11; + encoding |= (sdwa.clamp ? 1 : 0) << 13; + encoding |= sdwa.omod << 14; + } + + encoding |= sdwa.sel[0].to_sdwa_sel(sdwa_op.physReg().byte()) << 16; + encoding |= sdwa.sel[0].sign_extend() ? 1 << 19 : 0; + encoding |= sdwa.abs[0] << 21; + encoding |= sdwa.neg[0] << 20; + + if (instr->operands.size() >= 2) { + encoding |= sdwa.sel[1].to_sdwa_sel(instr->operands[1].physReg().byte()) << 24; + encoding |= sdwa.sel[1].sign_extend() ? 1 << 27 : 0; + encoding |= sdwa.abs[1] << 29; + encoding |= sdwa.neg[1] << 28; + } + + encoding |= reg(ctx, sdwa_op, 8); + encoding |= (sdwa_op.physReg() < 256) << 23; + if (instr->operands.size() >= 2) + encoding |= (instr->operands[1].physReg() < 256) << 31; + out.push_back(encoding); +} + void emit_instruction(asm_context& ctx, std::vector& out, Instruction* instr) { @@ -216,597 +1068,81 @@ emit_instruction(asm_context& ctx, std::vector& out, Instruction* inst switch (instr->format) { case Format::SOP2: { - uint32_t encoding = (0b10 << 30); - encoding |= opcode << 23; - encoding |= !instr->definitions.empty() ? reg(ctx, instr->definitions[0]) << 16 : 0; - encoding |= instr->operands.size() >= 2 ? reg(ctx, instr->operands[1]) << 8 : 0; - encoding |= !instr->operands.empty() ? reg(ctx, instr->operands[0]) : 0; - out.push_back(encoding); + emit_sop2_instruction(ctx, out, instr); break; } case Format::SOPK: { - SOPK_instruction& sopk = instr->sopk(); - - if (instr->opcode == aco_opcode::s_subvector_loop_begin) { - assert(ctx.gfx_level >= GFX10); - assert(ctx.subvector_begin_pos == -1); - ctx.subvector_begin_pos = out.size(); - } else if (instr->opcode == aco_opcode::s_subvector_loop_end) { - assert(ctx.gfx_level >= GFX10); - assert(ctx.subvector_begin_pos != -1); - /* Adjust s_subvector_loop_begin instruction to the address after the end */ - out[ctx.subvector_begin_pos] |= (out.size() - ctx.subvector_begin_pos); - /* Adjust s_subvector_loop_end instruction to the address after the beginning */ - sopk.imm = (uint16_t)(ctx.subvector_begin_pos - (int)out.size()); - ctx.subvector_begin_pos = -1; - } - - uint32_t encoding = (0b1011 << 28); - encoding |= opcode << 23; - encoding |= !instr->definitions.empty() && !(instr->definitions[0].physReg() == scc) - ? reg(ctx, instr->definitions[0]) << 16 - : !instr->operands.empty() && instr->operands[0].physReg() <= 127 - ? reg(ctx, instr->operands[0]) << 16 - : 0; - encoding |= sopk.imm; - out.push_back(encoding); + emit_sopk_instruction(ctx, out, instr); break; } case Format::SOP1: { - uint32_t encoding = (0b101111101 << 23); - encoding |= !instr->definitions.empty() ? reg(ctx, instr->definitions[0]) << 16 : 0; - encoding |= opcode << 8; - encoding |= !instr->operands.empty() ? reg(ctx, instr->operands[0]) : 0; - out.push_back(encoding); + emit_sop1_instruction(ctx, out, instr); break; } case Format::SOPC: { - uint32_t encoding = (0b101111110 << 23); - encoding |= opcode << 16; - encoding |= instr->operands.size() == 2 ? reg(ctx, instr->operands[1]) << 8 : 0; - encoding |= !instr->operands.empty() ? reg(ctx, instr->operands[0]) : 0; - out.push_back(encoding); + emit_sopc_instruction(ctx, out, instr); break; } case Format::SOPP: { - SOPP_instruction& sopp = instr->sopp(); - uint32_t encoding = (0b101111111 << 23); - encoding |= opcode << 16; - encoding |= (uint16_t)sopp.imm; - if (sopp.block != -1) { - sopp.pass_flags = 0; - ctx.branches.emplace_back(out.size(), &sopp); - } - out.push_back(encoding); + emit_sopp_instruction(ctx, out, instr); break; } case Format::SMEM: { - SMEM_instruction& smem = instr->smem(); - bool soe = instr->operands.size() >= (!instr->definitions.empty() ? 3 : 4); - bool is_load = !instr->definitions.empty(); - uint32_t encoding = 0; - - if (ctx.gfx_level <= GFX7) { - encoding = (0b11000 << 27); - encoding |= opcode << 22; - encoding |= instr->definitions.size() ? reg(ctx, instr->definitions[0]) << 15 : 0; - encoding |= instr->operands.size() ? (reg(ctx, instr->operands[0]) >> 1) << 9 : 0; - if (instr->operands.size() >= 2) { - if (!instr->operands[1].isConstant()) { - encoding |= reg(ctx, instr->operands[1]); - } else if (instr->operands[1].constantValue() >= 1024) { - encoding |= 255; /* SQ_SRC_LITERAL */ - } else { - encoding |= instr->operands[1].constantValue() >> 2; - encoding |= 1 << 8; - } - } - out.push_back(encoding); - /* SMRD instructions can take a literal on GFX7 */ - if (instr->operands.size() >= 2 && instr->operands[1].isConstant() && - instr->operands[1].constantValue() >= 1024) - out.push_back(instr->operands[1].constantValue() >> 2); - return; - } - - if (ctx.gfx_level <= GFX9) { - encoding = (0b110000 << 26); - assert(!smem.dlc); /* Device-level coherent is not supported on GFX9 and lower */ - encoding |= smem.nv ? 1 << 15 : 0; - } else { - encoding = (0b111101 << 26); - assert(!smem.nv); /* Non-volatile is not supported on GFX10 */ - encoding |= smem.dlc ? 1 << (ctx.gfx_level >= GFX11 ? 13 : 14) : 0; - } - - encoding |= opcode << 18; - encoding |= smem.glc ? 1 << (ctx.gfx_level >= GFX11 ? 14 : 16) : 0; - - if (ctx.gfx_level <= GFX9) { - if (instr->operands.size() >= 2) - encoding |= instr->operands[1].isConstant() ? 1 << 17 : 0; /* IMM - immediate enable */ - } - if (ctx.gfx_level == GFX9) { - encoding |= soe ? 1 << 14 : 0; - } - - if (is_load || instr->operands.size() >= 3) { /* SDATA */ - encoding |= (is_load ? reg(ctx, instr->definitions[0]) : reg(ctx, instr->operands[2])) - << 6; - } - if (instr->operands.size() >= 1) { /* SBASE */ - encoding |= reg(ctx, instr->operands[0]) >> 1; - } - - out.push_back(encoding); - encoding = 0; - - int32_t offset = 0; - uint32_t soffset = - ctx.gfx_level >= GFX10 - ? reg(ctx, sgpr_null) /* On GFX10 this is disabled by specifying SGPR_NULL */ - : 0; /* On GFX9, it is disabled by the SOE bit (and it's not present on - GFX8 and below) */ - if (instr->operands.size() >= 2) { - const Operand& op_off1 = instr->operands[1]; - if (ctx.gfx_level <= GFX9) { - offset = op_off1.isConstant() ? op_off1.constantValue() : reg(ctx, op_off1); - } else { - /* GFX10 only supports constants in OFFSET, so put the operand in SOFFSET if it's an - * SGPR */ - if (op_off1.isConstant()) { - offset = op_off1.constantValue(); - } else { - soffset = reg(ctx, op_off1); - assert(!soe); /* There is no place to put the other SGPR offset, if any */ - } - } - - if (soe) { - const Operand& op_off2 = instr->operands.back(); - assert(ctx.gfx_level >= GFX9); /* GFX8 and below don't support specifying a constant - and an SGPR at the same time */ - assert(!op_off2.isConstant()); - soffset = reg(ctx, op_off2); - } - } - encoding |= offset; - encoding |= soffset << 25; - - out.push_back(encoding); + emit_smem_instruction(ctx, out, instr); return; } case Format::VOP2: { - VALU_instruction& valu = instr->valu(); - uint32_t encoding = 0; - encoding |= opcode << 25; - encoding |= reg(ctx, instr->definitions[0], 8) << 17; - encoding |= (valu.opsel[3] ? 128 : 0) << 17; - encoding |= reg(ctx, instr->operands[1], 8) << 9; - encoding |= (valu.opsel[1] ? 128 : 0) << 9; - encoding |= reg(ctx, instr->operands[0]); - encoding |= valu.opsel[0] ? 128 : 0; - out.push_back(encoding); + emit_vop2_instruction(ctx, out, instr); break; } case Format::VOP1: { - VALU_instruction& valu = instr->valu(); - uint32_t encoding = (0b0111111 << 25); - if (!instr->definitions.empty()) { - encoding |= reg(ctx, instr->definitions[0], 8) << 17; - encoding |= (valu.opsel[3] ? 128 : 0) << 17; - } - encoding |= opcode << 9; - if (!instr->operands.empty()) { - encoding |= reg(ctx, instr->operands[0]); - encoding |= valu.opsel[0] ? 128 : 0; - } - out.push_back(encoding); + emit_vop1_instruction(ctx, out, instr); break; } case Format::VOPC: { - VALU_instruction& valu = instr->valu(); - uint32_t encoding = (0b0111110 << 25); - encoding |= opcode << 17; - encoding |= reg(ctx, instr->operands[1], 8) << 9; - encoding |= (valu.opsel[1] ? 128 : 0) << 9; - encoding |= reg(ctx, instr->operands[0]); - encoding |= valu.opsel[0] ? 128 : 0; - out.push_back(encoding); + emit_vopc_instruction(ctx, out, instr); break; } case Format::VINTRP: { - VINTRP_instruction& interp = instr->vintrp(); - uint32_t encoding = 0; - - if (instr->opcode == aco_opcode::v_interp_p1ll_f16 || - instr->opcode == aco_opcode::v_interp_p1lv_f16 || - instr->opcode == aco_opcode::v_interp_p2_legacy_f16 || - instr->opcode == aco_opcode::v_interp_p2_f16) { - if (ctx.gfx_level == GFX8 || ctx.gfx_level == GFX9) { - encoding = (0b110100 << 26); - } else if (ctx.gfx_level >= GFX10) { - encoding = (0b110101 << 26); - } else { - unreachable("Unknown gfx_level."); - } - - encoding |= opcode << 16; - encoding |= reg(ctx, instr->definitions[0], 8); - out.push_back(encoding); - - encoding = 0; - encoding |= interp.attribute; - encoding |= interp.component << 6; - encoding |= reg(ctx, instr->operands[0]) << 9; - if (instr->opcode == aco_opcode::v_interp_p2_f16 || - instr->opcode == aco_opcode::v_interp_p2_legacy_f16 || - instr->opcode == aco_opcode::v_interp_p1lv_f16) { - encoding |= reg(ctx, instr->operands[2]) << 18; - } - out.push_back(encoding); - } else { - if (ctx.gfx_level == GFX8 || ctx.gfx_level == GFX9) { - encoding = (0b110101 << 26); /* Vega ISA doc says 110010 but it's wrong */ - } else { - encoding = (0b110010 << 26); - } - - assert(encoding); - encoding |= reg(ctx, instr->definitions[0], 8) << 18; - encoding |= opcode << 16; - encoding |= interp.attribute << 10; - encoding |= interp.component << 8; - if (instr->opcode == aco_opcode::v_interp_mov_f32) - encoding |= (0x3 & instr->operands[0].constantValue()); - else - encoding |= reg(ctx, instr->operands[0], 8); - out.push_back(encoding); - } + emit_vintrp_instruction(ctx, out, instr); break; } case Format::VINTERP_INREG: { - VINTERP_inreg_instruction& interp = instr->vinterp_inreg(); - uint32_t encoding = (0b11001101 << 24); - encoding |= reg(ctx, instr->definitions[0], 8); - encoding |= (uint32_t)interp.wait_exp << 8; - encoding |= (uint32_t)interp.opsel << 11; - encoding |= (uint32_t)interp.clamp << 15; - encoding |= opcode << 16; - out.push_back(encoding); - - encoding = 0; - for (unsigned i = 0; i < instr->operands.size(); i++) - encoding |= reg(ctx, instr->operands[i]) << (i * 9); - for (unsigned i = 0; i < 3; i++) - encoding |= interp.neg[i] << (29 + i); - out.push_back(encoding); + emit_vinterp_inreg_instruction(ctx, out, instr); break; } case Format::VOPD: { - VOPD_instruction& vopd = instr->vopd(); - uint32_t encoding = (0b110010 << 26); - encoding |= reg(ctx, instr->operands[0]); - if (instr->opcode != aco_opcode::v_dual_mov_b32) - encoding |= reg(ctx, instr->operands[1], 8) << 9; - encoding |= (uint32_t)ctx.opcode[(int)vopd.opy] << 17; - encoding |= opcode << 22; - out.push_back(encoding); - - unsigned opy_start = get_vopd_opy_start(instr); - - encoding = reg(ctx, instr->operands[opy_start]); - if (vopd.opy != aco_opcode::v_dual_mov_b32) - encoding |= reg(ctx, instr->operands[opy_start + 1], 8) << 9; - encoding |= (reg(ctx, instr->definitions[1], 8) >> 1) << 17; - encoding |= reg(ctx, instr->definitions[0], 8) << 24; - out.push_back(encoding); + emit_vopd_instruction(ctx, out, instr); break; } case Format::DS: { - DS_instruction& ds = instr->ds(); - uint32_t encoding = (0b110110 << 26); - if (ctx.gfx_level == GFX8 || ctx.gfx_level == GFX9) { - encoding |= opcode << 17; - encoding |= (ds.gds ? 1 : 0) << 16; - } else { - encoding |= opcode << 18; - encoding |= (ds.gds ? 1 : 0) << 17; - } - encoding |= ((0xFF & ds.offset1) << 8); - encoding |= (0xFFFF & ds.offset0); - out.push_back(encoding); - encoding = 0; - if (!instr->definitions.empty()) - encoding |= reg(ctx, instr->definitions[0], 8) << 24; - if (instr->operands.size() >= 3 && instr->operands[2].physReg() != m0) - encoding |= reg(ctx, instr->operands[2], 8) << 16; - if (instr->operands.size() >= 2 && instr->operands[1].physReg() != m0) - encoding |= reg(ctx, instr->operands[1], 8) << 8; - if (!instr->operands[0].isUndefined()) - encoding |= reg(ctx, instr->operands[0], 8); - out.push_back(encoding); + emit_ds_instruction(ctx, out, instr); break; } case Format::LDSDIR: { - LDSDIR_instruction& dir = instr->ldsdir(); - uint32_t encoding = (0b11001110 << 24); - encoding |= opcode << 20; - encoding |= (uint32_t)dir.wait_vdst << 16; - encoding |= (uint32_t)dir.attr << 10; - encoding |= (uint32_t)dir.attr_chan << 8; - encoding |= reg(ctx, instr->definitions[0], 8); - out.push_back(encoding); + emit_ldsdir_instruction(ctx, out, instr); break; } case Format::MUBUF: { - MUBUF_instruction& mubuf = instr->mubuf(); - uint32_t encoding = (0b111000 << 26); - if (ctx.gfx_level >= GFX11 && mubuf.lds) /* GFX11 has separate opcodes for LDS loads */ - opcode = opcode == 0 ? 0x32 : (opcode + 0x1d); - else - encoding |= (mubuf.lds ? 1 : 0) << 16; - encoding |= opcode << 18; - encoding |= (mubuf.glc ? 1 : 0) << 14; - if (ctx.gfx_level <= GFX10_3) - encoding |= (mubuf.idxen ? 1 : 0) << 13; - assert(!mubuf.addr64 || ctx.gfx_level <= GFX7); - if (ctx.gfx_level == GFX6 || ctx.gfx_level == GFX7) - encoding |= (mubuf.addr64 ? 1 : 0) << 15; - if (ctx.gfx_level <= GFX10_3) - encoding |= (mubuf.offen ? 1 : 0) << 12; - if (ctx.gfx_level == GFX8 || ctx.gfx_level == GFX9) { - assert(!mubuf.dlc); /* Device-level coherent is not supported on GFX9 and lower */ - encoding |= (mubuf.slc ? 1 : 0) << 17; - } else if (ctx.gfx_level >= GFX11) { - encoding |= (mubuf.slc ? 1 : 0) << 12; - encoding |= (mubuf.dlc ? 1 : 0) << 13; - } else if (ctx.gfx_level >= GFX10) { - encoding |= (mubuf.dlc ? 1 : 0) << 15; - } - encoding |= 0x0FFF & mubuf.offset; - out.push_back(encoding); - encoding = 0; - if (ctx.gfx_level <= GFX7 || (ctx.gfx_level >= GFX10 && ctx.gfx_level <= GFX10_3)) { - encoding |= (mubuf.slc ? 1 : 0) << 22; - } - encoding |= reg(ctx, instr->operands[2]) << 24; - if (ctx.gfx_level >= GFX11) { - encoding |= (mubuf.tfe ? 1 : 0) << 21; - encoding |= (mubuf.offen ? 1 : 0) << 22; - encoding |= (mubuf.idxen ? 1 : 0) << 23; - } else { - encoding |= (mubuf.tfe ? 1 : 0) << 23; - } - encoding |= (reg(ctx, instr->operands[0]) >> 2) << 16; - if (instr->operands.size() > 3 && !mubuf.lds) - encoding |= reg(ctx, instr->operands[3], 8) << 8; - else if (!mubuf.lds) - encoding |= reg(ctx, instr->definitions[0], 8) << 8; - encoding |= reg(ctx, instr->operands[1], 8); - out.push_back(encoding); + emit_mubuf_instruction(ctx, out, instr); break; } case Format::MTBUF: { - MTBUF_instruction& mtbuf = instr->mtbuf(); - - uint32_t img_format = ac_get_tbuffer_format(ctx.gfx_level, mtbuf.dfmt, mtbuf.nfmt); - uint32_t encoding = (0b111010 << 26); - assert(img_format <= 0x7F); - assert(!mtbuf.dlc || ctx.gfx_level >= GFX10); - if (ctx.gfx_level >= GFX11) { - encoding |= (mtbuf.slc ? 1 : 0) << 12; - encoding |= (mtbuf.dlc ? 1 : 0) << 13; - } else { - /* DLC bit replaces one bit of the OPCODE on GFX10 */ - encoding |= (mtbuf.dlc ? 1 : 0) << 15; - } - if (ctx.gfx_level <= GFX10_3) { - encoding |= (mtbuf.idxen ? 1 : 0) << 13; - encoding |= (mtbuf.offen ? 1 : 0) << 12; - } - encoding |= (mtbuf.glc ? 1 : 0) << 14; - encoding |= 0x0FFF & mtbuf.offset; - encoding |= (img_format << 19); /* Handles both the GFX10 FORMAT and the old NFMT+DFMT */ - - if (ctx.gfx_level == GFX8 || ctx.gfx_level == GFX9 || ctx.gfx_level >= GFX11) { - encoding |= opcode << 15; - } else { - encoding |= (opcode & 0x07) << 16; /* 3 LSBs of 4-bit OPCODE */ - } - - out.push_back(encoding); - encoding = 0; - - encoding |= reg(ctx, instr->operands[2]) << 24; - if (ctx.gfx_level >= GFX11) { - encoding |= (mtbuf.tfe ? 1 : 0) << 21; - encoding |= (mtbuf.offen ? 1 : 0) << 22; - encoding |= (mtbuf.idxen ? 1 : 0) << 23; - } else { - encoding |= (mtbuf.tfe ? 1 : 0) << 23; - encoding |= (mtbuf.slc ? 1 : 0) << 22; - } - encoding |= (reg(ctx, instr->operands[0]) >> 2) << 16; - if (instr->operands.size() > 3) - encoding |= reg(ctx, instr->operands[3], 8) << 8; - else - encoding |= reg(ctx, instr->definitions[0], 8) << 8; - encoding |= reg(ctx, instr->operands[1], 8); - - if (ctx.gfx_level >= GFX10) { - encoding |= (((opcode & 0x08) >> 3) << 21); /* MSB of 4-bit OPCODE */ - } - - out.push_back(encoding); + emit_mtbuf_instruction(ctx, out, instr); break; } case Format::MIMG: { - unsigned nsa_dwords = get_mimg_nsa_dwords(instr); - assert(!nsa_dwords || ctx.gfx_level >= GFX10); - - MIMG_instruction& mimg = instr->mimg(); - uint32_t encoding = (0b111100 << 26); - if (ctx.gfx_level >= GFX11) { /* GFX11: rearranges most fields */ - assert(nsa_dwords <= 1); - encoding |= nsa_dwords; - encoding |= mimg.dim << 2; - encoding |= mimg.unrm ? 1 << 7 : 0; - encoding |= (0xF & mimg.dmask) << 8; - encoding |= mimg.slc ? 1 << 12 : 0; - encoding |= mimg.dlc ? 1 << 13 : 0; - encoding |= mimg.glc ? 1 << 14 : 0; - encoding |= mimg.r128 ? 1 << 15 : 0; - encoding |= mimg.a16 ? 1 << 16 : 0; - encoding |= mimg.d16 ? 1 << 17 : 0; - encoding |= (opcode & 0xFF) << 18; - } else { - encoding |= mimg.slc ? 1 << 25 : 0; - encoding |= (opcode & 0x7f) << 18; - encoding |= (opcode >> 7) & 1; - encoding |= mimg.lwe ? 1 << 17 : 0; - encoding |= mimg.tfe ? 1 << 16 : 0; - encoding |= mimg.glc ? 1 << 13 : 0; - encoding |= mimg.unrm ? 1 << 12 : 0; - if (ctx.gfx_level <= GFX9) { - assert(!mimg.dlc); /* Device-level coherent is not supported on GFX9 and lower */ - assert(!mimg.r128); - encoding |= mimg.a16 ? 1 << 15 : 0; - encoding |= mimg.da ? 1 << 14 : 0; - } else { - encoding |= mimg.r128 - ? 1 << 15 - : 0; /* GFX10: A16 moved to 2nd word, R128 replaces it in 1st word */ - encoding |= nsa_dwords << 1; - encoding |= mimg.dim << 3; /* GFX10: dimensionality instead of declare array */ - encoding |= mimg.dlc ? 1 << 7 : 0; - } - encoding |= (0xF & mimg.dmask) << 8; - } - out.push_back(encoding); - - encoding = reg(ctx, instr->operands[3], 8); /* VADDR */ - if (!instr->definitions.empty()) { - encoding |= reg(ctx, instr->definitions[0], 8) << 8; /* VDATA */ - } else if (!instr->operands[2].isUndefined()) { - encoding |= reg(ctx, instr->operands[2], 8) << 8; /* VDATA */ - } - encoding |= (0x1F & (reg(ctx, instr->operands[0]) >> 2)) << 16; /* T# (resource) */ - - assert(!mimg.d16 || ctx.gfx_level >= GFX9); - if (ctx.gfx_level >= GFX11) { - if (!instr->operands[1].isUndefined()) - encoding |= (0x1F & (reg(ctx, instr->operands[1]) >> 2)) << 26; /* sampler */ - - encoding |= mimg.tfe ? 1 << 21 : 0; - encoding |= mimg.lwe ? 1 << 22 : 0; - } else { - if (!instr->operands[1].isUndefined()) - encoding |= (0x1F & (reg(ctx, instr->operands[1]) >> 2)) << 21; /* sampler */ - - encoding |= mimg.d16 ? 1 << 31 : 0; - if (ctx.gfx_level >= GFX10) { - /* GFX10: A16 still exists, but is in a different place */ - encoding |= mimg.a16 ? 1 << 30 : 0; - } - } - - out.push_back(encoding); - - if (nsa_dwords) { - out.resize(out.size() + nsa_dwords); - std::vector::iterator nsa = std::prev(out.end(), nsa_dwords); - for (unsigned i = 0; i < instr->operands.size() - 4u; i++) - nsa[i / 4] |= reg(ctx, instr->operands[4 + i], 8) << (i % 4 * 8); - } + emit_mimg_instruction(ctx, out, instr); break; } case Format::FLAT: case Format::SCRATCH: case Format::GLOBAL: { - FLAT_instruction& flat = instr->flatlike(); - uint32_t encoding = (0b110111 << 26); - encoding |= opcode << 18; - if (ctx.gfx_level == GFX9 || ctx.gfx_level >= GFX11) { - if (instr->isFlat()) - assert(flat.offset <= 0xfff); - else - assert(flat.offset >= -4096 && flat.offset < 4096); - encoding |= flat.offset & 0x1fff; - } else if (ctx.gfx_level <= GFX8 || instr->isFlat()) { - /* GFX10 has a 12-bit immediate OFFSET field, - * but it has a hw bug: it ignores the offset, called FlatSegmentOffsetBug - */ - assert(flat.offset == 0); - } else { - assert(flat.offset >= -2048 && flat.offset <= 2047); - encoding |= flat.offset & 0xfff; - } - if (instr->isScratch()) - encoding |= 1 << (ctx.gfx_level >= GFX11 ? 16 : 14); - else if (instr->isGlobal()) - encoding |= 2 << (ctx.gfx_level >= GFX11 ? 16 : 14); - encoding |= flat.lds ? 1 << 13 : 0; - encoding |= flat.glc ? 1 << (ctx.gfx_level >= GFX11 ? 14 : 16) : 0; - encoding |= flat.slc ? 1 << (ctx.gfx_level >= GFX11 ? 15 : 17) : 0; - if (ctx.gfx_level >= GFX10) { - assert(!flat.nv); - encoding |= flat.dlc ? 1 << (ctx.gfx_level >= GFX11 ? 13 : 12) : 0; - } else { - assert(!flat.dlc); - } - out.push_back(encoding); - encoding = reg(ctx, instr->operands[0], 8); - if (!instr->definitions.empty()) - encoding |= reg(ctx, instr->definitions[0], 8) << 24; - if (instr->operands.size() >= 3) - encoding |= reg(ctx, instr->operands[2], 8) << 8; - if (!instr->operands[1].isUndefined()) { - assert(ctx.gfx_level >= GFX10 || instr->operands[1].physReg() != 0x7F); - assert(instr->format != Format::FLAT); - encoding |= reg(ctx, instr->operands[1], 8) << 16; - } else if (instr->format != Format::FLAT || - ctx.gfx_level >= GFX10) { /* SADDR is actually used with FLAT on GFX10 */ - /* For GFX10.3 scratch, 0x7F disables both ADDR and SADDR, unlike sgpr_null, which only - * disables SADDR. On GFX11, this was replaced with SVE. - */ - if (ctx.gfx_level <= GFX9 || - (instr->isScratch() && instr->operands[0].isUndefined() && ctx.gfx_level < GFX11)) - encoding |= 0x7F << 16; - else - encoding |= reg(ctx, sgpr_null) << 16; - } - if (ctx.gfx_level >= GFX11 && instr->isScratch()) - encoding |= !instr->operands[0].isUndefined() ? 1 << 23 : 0; - else - encoding |= flat.nv ? 1 << 23 : 0; - out.push_back(encoding); + emit_flatlike_instruction(ctx, out, instr); break; } case Format::EXP: { - Export_instruction& exp = instr->exp(); - uint32_t encoding; - if (ctx.gfx_level == GFX8 || ctx.gfx_level == GFX9) { - encoding = (0b110001 << 26); - } else { - encoding = (0b111110 << 26); - } - - if (ctx.gfx_level >= GFX11) { - encoding |= exp.row_en ? 0b1 << 13 : 0; - } else { - encoding |= exp.valid_mask ? 0b1 << 12 : 0; - encoding |= exp.compressed ? 0b1 << 10 : 0; - } - encoding |= exp.done ? 0b1 << 11 : 0; - encoding |= exp.dest << 4; - encoding |= exp.enabled_mask; - out.push_back(encoding); - encoding = reg(ctx, exp.operands[0], 8); - encoding |= reg(ctx, exp.operands[1], 8) << 8; - encoding |= reg(ctx, exp.operands[2], 8) << 16; - encoding |= reg(ctx, exp.operands[3], 8) << 24; - out.push_back(encoding); + emit_exp_instruction(ctx, out, instr); break; } case Format::PSEUDO: @@ -816,174 +1152,17 @@ emit_instruction(asm_context& ctx, std::vector& out, Instruction* inst break; default: if (instr->isDPP16()) { - assert(ctx.gfx_level >= GFX8); - DPP16_instruction& dpp = instr->dpp16(); - - /* first emit the instruction without the DPP operand */ - Operand dpp_op = instr->operands[0]; - instr->operands[0] = Operand(PhysReg{250}, v1); - instr->format = (Format)((uint16_t)instr->format & ~(uint16_t)Format::DPP16); - emit_instruction(ctx, out, instr); - uint32_t encoding = (0xF & dpp.row_mask) << 28; - encoding |= (0xF & dpp.bank_mask) << 24; - encoding |= dpp.abs[1] << 23; - encoding |= dpp.neg[1] << 22; - encoding |= dpp.abs[0] << 21; - encoding |= dpp.neg[0] << 20; - encoding |= dpp.fetch_inactive << 18; - encoding |= dpp.bound_ctrl << 19; - encoding |= dpp.dpp_ctrl << 8; - encoding |= reg(ctx, dpp_op, 8); - encoding |= dpp.opsel[0] && !instr->isVOP3() ? 128 : 0; - out.push_back(encoding); + emit_dpp16_instruction(ctx, out, instr); return; } else if (instr->isDPP8()) { - assert(ctx.gfx_level >= GFX10); - DPP8_instruction& dpp = instr->dpp8(); - - /* first emit the instruction without the DPP operand */ - Operand dpp_op = instr->operands[0]; - instr->operands[0] = Operand(PhysReg{233u + dpp.fetch_inactive}, v1); - instr->format = (Format)((uint16_t)instr->format & ~(uint16_t)Format::DPP8); - emit_instruction(ctx, out, instr); - uint32_t encoding = reg(ctx, dpp_op, 8); - encoding |= dpp.opsel[0] && !instr->isVOP3() ? 128 : 0; - encoding |= dpp.lane_sel << 8; - out.push_back(encoding); + emit_dpp8_instruction(ctx, out, instr); return; } else if (instr->isVOP3()) { - VALU_instruction& vop3 = instr->valu(); - - if (instr->isVOP2()) { - opcode = opcode + 0x100; - } else if (instr->isVOP1()) { - if (ctx.gfx_level == GFX8 || ctx.gfx_level == GFX9) - opcode = opcode + 0x140; - else - opcode = opcode + 0x180; - } else if (instr->isVOPC()) { - opcode = opcode + 0x0; - } else if (instr->isVINTRP()) { - opcode = opcode + 0x270; - } - - uint32_t encoding; - if (ctx.gfx_level <= GFX9) { - encoding = (0b110100 << 26); - } else if (ctx.gfx_level >= GFX10) { - encoding = (0b110101 << 26); - } else { - unreachable("Unknown gfx_level."); - } - - if (ctx.gfx_level <= GFX7) { - encoding |= opcode << 17; - encoding |= (vop3.clamp ? 1 : 0) << 11; - } else { - encoding |= opcode << 16; - encoding |= (vop3.clamp ? 1 : 0) << 15; - } - encoding |= vop3.opsel << 11; - for (unsigned i = 0; i < 3; i++) - encoding |= vop3.abs[i] << (8 + i); - /* On GFX9 and older, v_cmpx implicitly writes exec besides writing an SGPR pair. - * On GFX10 and newer, v_cmpx always writes just exec. - */ - if (instr->definitions.size() == 2 && instr->isVOPC()) - assert(ctx.gfx_level <= GFX9 && instr->definitions[1].physReg() == exec); - else if (instr->definitions.size() == 2) - encoding |= reg(ctx, instr->definitions[1]) << 8; - encoding |= reg(ctx, instr->definitions[0], 8); - out.push_back(encoding); - encoding = 0; - if (instr->opcode == aco_opcode::v_interp_mov_f32) { - encoding = 0x3 & instr->operands[0].constantValue(); - } else if (instr->opcode == aco_opcode::v_writelane_b32_e64) { - encoding |= reg(ctx, instr->operands[0]) << 0; - encoding |= reg(ctx, instr->operands[1]) << 9; - /* Encoding src2 works fine with hardware but breaks some disassemblers. */ - } else { - for (unsigned i = 0; i < instr->operands.size(); i++) - encoding |= reg(ctx, instr->operands[i]) << (i * 9); - } - encoding |= vop3.omod << 27; - for (unsigned i = 0; i < 3; i++) - encoding |= vop3.neg[i] << (29 + i); - out.push_back(encoding); - + emit_vop3_instruction(ctx, out, instr); } else if (instr->isVOP3P()) { - VALU_instruction& vop3 = instr->valu(); - - uint32_t encoding; - if (ctx.gfx_level == GFX9) { - encoding = (0b110100111 << 23); - } else if (ctx.gfx_level >= GFX10) { - encoding = (0b110011 << 26); - } else { - unreachable("Unknown gfx_level."); - } - - encoding |= opcode << 16; - encoding |= (vop3.clamp ? 1 : 0) << 15; - encoding |= vop3.opsel_lo << 11; - encoding |= ((vop3.opsel_hi & 0x4) ? 1 : 0) << 14; - for (unsigned i = 0; i < 3; i++) - encoding |= vop3.neg_hi[i] << (8 + i); - encoding |= reg(ctx, instr->definitions[0], 8); - out.push_back(encoding); - encoding = 0; - for (unsigned i = 0; i < instr->operands.size(); i++) - encoding |= reg(ctx, instr->operands[i]) << (i * 9); - encoding |= (vop3.opsel_hi & 0x3) << 27; - for (unsigned i = 0; i < 3; i++) - encoding |= vop3.neg_lo[i] << (29 + i); - out.push_back(encoding); + emit_vop3p_instruction(ctx, out, instr); } else if (instr->isSDWA()) { - assert(ctx.gfx_level >= GFX8 && ctx.gfx_level < GFX11); - SDWA_instruction& sdwa = instr->sdwa(); - - /* first emit the instruction without the SDWA operand */ - Operand sdwa_op = instr->operands[0]; - instr->operands[0] = Operand(PhysReg{249}, v1); - instr->format = (Format)((uint16_t)instr->format & ~(uint16_t)Format::SDWA); - emit_instruction(ctx, out, instr); - - uint32_t encoding = 0; - - if (instr->isVOPC()) { - if (instr->definitions[0].physReg() != - (ctx.gfx_level >= GFX10 && is_cmpx(instr->opcode) ? exec : vcc)) { - encoding |= reg(ctx, instr->definitions[0]) << 8; - encoding |= 1 << 15; - } - encoding |= (sdwa.clamp ? 1 : 0) << 13; - } else { - encoding |= sdwa.dst_sel.to_sdwa_sel(instr->definitions[0].physReg().byte()) << 8; - uint32_t dst_u = sdwa.dst_sel.sign_extend() ? 1 : 0; - if (instr->definitions[0].bytes() < 4) /* dst_preserve */ - dst_u = 2; - encoding |= dst_u << 11; - encoding |= (sdwa.clamp ? 1 : 0) << 13; - encoding |= sdwa.omod << 14; - } - - encoding |= sdwa.sel[0].to_sdwa_sel(sdwa_op.physReg().byte()) << 16; - encoding |= sdwa.sel[0].sign_extend() ? 1 << 19 : 0; - encoding |= sdwa.abs[0] << 21; - encoding |= sdwa.neg[0] << 20; - - if (instr->operands.size() >= 2) { - encoding |= sdwa.sel[1].to_sdwa_sel(instr->operands[1].physReg().byte()) << 24; - encoding |= sdwa.sel[1].sign_extend() ? 1 << 27 : 0; - encoding |= sdwa.abs[1] << 29; - encoding |= sdwa.neg[1] << 28; - } - - encoding |= reg(ctx, sdwa_op, 8); - encoding |= (sdwa_op.physReg() < 256) << 23; - if (instr->operands.size() >= 2) - encoding |= (instr->operands[1].physReg() < 256) << 31; - out.push_back(encoding); + emit_sdwa_instruction(ctx, out, instr); } else { unreachable("unimplemented instruction format"); }