From e1e5bc0dd051dc2392f3ba9415d678b4a59eb734 Mon Sep 17 00:00:00 2001 From: Rhys Perry Date: Wed, 1 May 2024 19:43:01 +0100 Subject: [PATCH] aco: support GFX12 in assembler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rhys Perry Reviewed-by: Georg Lehmann Acked-by: Daniel Schürmann Part-of: --- src/amd/compiler/aco_assembler.cpp | 216 ++++++++++++++++++++++++++-- src/amd/compiler/aco_ir.h | 4 +- src/amd/compiler/aco_opcodes.py | 3 +- src/amd/compiler/aco_opcodes_cpp.py | 5 + src/amd/compiler/aco_print_ir.cpp | 2 + 5 files changed, 220 insertions(+), 10 deletions(-) diff --git a/src/amd/compiler/aco_assembler.cpp b/src/amd/compiler/aco_assembler.cpp index 56e4004a827..4a85a8dab15 100644 --- a/src/amd/compiler/aco_assembler.cpp +++ b/src/amd/compiler/aco_assembler.cpp @@ -43,8 +43,10 @@ struct asm_context { opcode = &instr_info.opcode_gfx9[0]; else if (gfx_level <= GFX10_3) opcode = &instr_info.opcode_gfx10[0]; - else if (gfx_level >= GFX11) + else if (gfx_level <= GFX11_5) opcode = &instr_info.opcode_gfx11[0]; + else + opcode = &instr_info.opcode_gfx12[0]; } int subvector_begin_pos = -1; @@ -120,6 +122,18 @@ needs_vop3_gfx11(asm_context& ctx, Instruction* instr) return false; } +template +uint32_t +get_gfx12_cpol(const T& instr) +{ + if (instr_info.is_atomic[(int)instr.opcode]) { + return (instr.glc ? 1 /*TH_ATOMIC_RETURN*/ : 0) << 2; + } else { + return (instr.definitions.empty() || instr.glc || instr.slc || instr.dlc) ? 3 /*SCOPE_SYS*/ + : 0 /*SCOPE_CU*/; + } +} + void emit_sop2_instruction(asm_context& ctx, std::vector& out, Instruction* instr) { @@ -249,11 +263,18 @@ emit_smem_instruction(asm_context& ctx, std::vector& out, Instruction* } else { encoding = (0b111101 << 26); assert(!smem.nv); /* Non-volatile is not supported on GFX10 */ - encoding |= smem.dlc ? 1 << (ctx.gfx_level >= GFX11 ? 13 : 14) : 0; + if (ctx.gfx_level <= GFX11_5) + encoding |= smem.dlc ? 1 << (ctx.gfx_level >= GFX11 ? 13 : 14) : 0; } - encoding |= opcode << 18; - encoding |= smem.glc ? 1 << (ctx.gfx_level >= GFX11 ? 14 : 16) : 0; + if (ctx.gfx_level <= GFX11_5) { + encoding |= opcode << 18; + encoding |= smem.glc ? 1 << (ctx.gfx_level >= GFX11 ? 14 : 16) : 0; + } else { + encoding |= opcode << 13; + if (is_load) + encoding |= ((smem.glc || smem.dlc) ? 3 /*SCOPE_SYS*/ : 0 /*SCOPE_CU*/) << 21; + } if (ctx.gfx_level <= GFX9) { if (instr->operands.size() >= 2) @@ -502,6 +523,8 @@ emit_ldsdir_instruction(asm_context& ctx, std::vector& out, Instructio uint32_t encoding = (0b11001110 << 24); encoding |= opcode << 20; encoding |= (uint32_t)dir.wait_vdst << 16; + if (ctx.gfx_level >= GFX12) + encoding |= (uint32_t)dir.wait_vsrc << 23; encoding |= (uint32_t)dir.attr << 10; encoding |= (uint32_t)dir.attr_chan << 8; encoding |= reg(ctx, instr->definitions[0], 8); @@ -560,6 +583,43 @@ emit_mubuf_instruction(asm_context& ctx, std::vector& out, Instruction out.push_back(encoding); } +void +emit_mubuf_instruction_gfx12(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + MUBUF_instruction& mubuf = instr->mubuf(); + assert(!mubuf.lds); + + uint32_t encoding = 0b110001 << 26; + encoding |= opcode << 14; + if (instr->operands[2].isConstant()) { + assert(instr->operands[2].constantValue() == 0); + encoding |= reg(ctx, sgpr_null); + } else { + encoding |= reg(ctx, instr->operands[2]); + } + encoding |= (mubuf.tfe ? 1 : 0) << 22; + out.push_back(encoding); + + encoding = 0; + if (instr->operands.size() > 3) + encoding |= reg(ctx, instr->operands[3], 8); + else + encoding |= reg(ctx, instr->definitions[0], 8); + encoding |= reg(ctx, instr->operands[0]) << 9; + encoding |= (mubuf.offen ? 1 : 0) << 30; + encoding |= (mubuf.idxen ? 1 : 0) << 31; + encoding |= get_gfx12_cpol(mubuf) << 18; + encoding |= 1 << 23; + out.push_back(encoding); + + encoding = 0; + if (!instr->operands[1].isUndefined()) + encoding |= reg(ctx, instr->operands[1], 8); + encoding |= (mubuf.offset & 0x00ffffff) << 8; + out.push_back(encoding); +} + void emit_mtbuf_instruction(asm_context& ctx, std::vector& out, Instruction* instr) { @@ -618,6 +678,44 @@ emit_mtbuf_instruction(asm_context& ctx, std::vector& out, Instruction out.push_back(encoding); } +void +emit_mtbuf_instruction_gfx12(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + MTBUF_instruction& mtbuf = instr->mtbuf(); + + uint32_t img_format = ac_get_tbuffer_format(ctx.gfx_level, mtbuf.dfmt, mtbuf.nfmt); + + uint32_t encoding = 0b110001 << 26; + encoding |= 0b1000 << 18; + encoding |= opcode << 14; + if (instr->operands[2].isConstant()) { + assert(instr->operands[2].constantValue() == 0); + encoding |= reg(ctx, sgpr_null); + } else { + encoding |= reg(ctx, instr->operands[2]); + } + encoding |= (mtbuf.tfe ? 1 : 0) << 22; + out.push_back(encoding); + + encoding = 0; + if (instr->operands.size() > 3) + encoding |= reg(ctx, instr->operands[3], 8); + else + encoding |= reg(ctx, instr->definitions[0], 8); + encoding |= reg(ctx, instr->operands[0]) << 9; + encoding |= (mtbuf.offen ? 1 : 0) << 30; + encoding |= (mtbuf.idxen ? 1 : 0) << 31; + encoding |= get_gfx12_cpol(mtbuf) << 18; + encoding |= img_format << 23; + out.push_back(encoding); + + encoding = 0; + encoding |= reg(ctx, instr->operands[1], 8); + encoding |= (mtbuf.offset & 0x00ffffff) << 8; + out.push_back(encoding); +} + void emit_mimg_instruction(asm_context& ctx, std::vector& out, Instruction* instr) { @@ -701,6 +799,58 @@ emit_mimg_instruction(asm_context& ctx, std::vector& out, Instruction* } } +void +emit_mimg_instruction_gfx12(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + MIMG_instruction& mimg = instr->mimg(); + + bool vsample = !instr->operands[1].isUndefined() || instr->opcode == aco_opcode::image_msaa_load; + uint32_t encoding = opcode << 14; + if (vsample) { + encoding |= 0b111001 << 26; + encoding |= mimg.tfe << 3; + encoding |= mimg.unrm << 13; + } else { + encoding |= 0b110100 << 26; + } + encoding |= mimg.dim; + encoding |= mimg.r128 << 4; + encoding |= mimg.d16 << 5; + encoding |= mimg.a16 << 6; + encoding |= (mimg.dmask & 0xf) << 22; + out.push_back(encoding); + + uint8_t vaddr[5] = {0, 0, 0, 0, 0}; + for (unsigned i = 3; i < instr->operands.size(); i++) + vaddr[i - 3] = reg(ctx, instr->operands[i], 8); + unsigned num_vaddr = instr->operands.size() - 3; + for (unsigned i = 0; i < MIN2(instr->operands.back().size() - 1, 5 - num_vaddr); i++) + vaddr[num_vaddr + i] = reg(ctx, instr->operands.back(), 8) + i + 1; + + encoding = 0; + if (!instr->definitions.empty()) + encoding |= reg(ctx, instr->definitions[0], 8); /* VDATA */ + else if (!instr->operands[2].isUndefined()) + encoding |= reg(ctx, instr->operands[2], 8); /* VDATA */ + encoding |= reg(ctx, instr->operands[0]) << 9; /* T# (resource) */ + if (vsample) { + encoding |= mimg.lwe << 8; + if (instr->opcode != aco_opcode::image_msaa_load) + encoding |= reg(ctx, instr->operands[1]) << 23; /* sampler */ + } else { + encoding |= mimg.tfe << 23; + encoding |= vaddr[4] << 24; + } + encoding |= get_gfx12_cpol(mimg) << 18; + out.push_back(encoding); + + encoding = 0; + for (unsigned i = 0; i < 4; i++) + encoding |= vaddr[i] << (i * 8); + out.push_back(encoding); +} + void emit_flatlike_instruction(asm_context& ctx, std::vector& out, Instruction* instr) { @@ -765,6 +915,44 @@ emit_flatlike_instruction(asm_context& ctx, std::vector& out, Instruct out.push_back(encoding); } +void +emit_flatlike_instruction_gfx12(asm_context& ctx, std::vector& out, Instruction* instr) +{ + uint32_t opcode = ctx.opcode[(int)instr->opcode]; + FLAT_instruction& flat = instr->flatlike(); + assert(!flat.lds); + + uint32_t encoding = opcode << 14; + encoding |= 0b111011 << 26; + if (!instr->operands[1].isUndefined()) { + assert(!instr->isFlat()); + encoding |= reg(ctx, instr->operands[1]); + } else { + encoding |= reg(ctx, sgpr_null); + } + if (instr->isScratch()) + encoding |= 1 << 24; + else if (instr->isGlobal()) + encoding |= 2 << 24; + out.push_back(encoding); + + encoding = 0; + if (!instr->definitions.empty()) + encoding |= reg(ctx, instr->definitions[0], 8); + if (instr->isScratch()) + encoding |= !instr->operands[0].isUndefined() ? 1 << 17 : 0; + encoding |= get_gfx12_cpol(flat) << 18; + if (instr->operands.size() >= 3) + encoding |= reg(ctx, instr->operands[2], 8) << 23; + out.push_back(encoding); + + encoding = 0; + if (!instr->operands[0].isUndefined()) + encoding |= reg(ctx, instr->operands[0], 8); + encoding |= (flat.offset & 0x00ffffff) << 8; + out.push_back(encoding); +} + void emit_exp_instruction(asm_context& ctx, std::vector& out, Instruction* instr) { @@ -1117,21 +1305,33 @@ emit_instruction(asm_context& ctx, std::vector& out, Instruction* inst break; } case Format::MUBUF: { - emit_mubuf_instruction(ctx, out, instr); + if (ctx.gfx_level >= GFX12) + emit_mubuf_instruction_gfx12(ctx, out, instr); + else + emit_mubuf_instruction(ctx, out, instr); break; } case Format::MTBUF: { - emit_mtbuf_instruction(ctx, out, instr); + if (ctx.gfx_level >= GFX12) + emit_mtbuf_instruction_gfx12(ctx, out, instr); + else + emit_mtbuf_instruction(ctx, out, instr); break; } case Format::MIMG: { - emit_mimg_instruction(ctx, out, instr); + if (ctx.gfx_level >= GFX12) + emit_mimg_instruction_gfx12(ctx, out, instr); + else + emit_mimg_instruction(ctx, out, instr); break; } case Format::FLAT: case Format::SCRATCH: case Format::GLOBAL: { - emit_flatlike_instruction(ctx, out, instr); + if (ctx.gfx_level >= GFX12) + emit_flatlike_instruction_gfx12(ctx, out, instr); + else + emit_flatlike_instruction(ctx, out, instr); break; } case Format::EXP: { diff --git a/src/amd/compiler/aco_ir.h b/src/amd/compiler/aco_ir.h index 2ced8b25b37..171d0ec7c48 100644 --- a/src/amd/compiler/aco_ir.h +++ b/src/amd/compiler/aco_ir.h @@ -1467,7 +1467,8 @@ struct LDSDIR_instruction : public Instruction { uint8_t attr : 6; uint8_t attr_chan : 2; uint32_t wait_vdst : 4; - uint32_t padding : 28; + uint32_t wait_vsrc : 1; + uint32_t padding : 27; }; static_assert(sizeof(LDSDIR_instruction) == sizeof(Instruction) + 8, "Unexpected padding"); @@ -2250,6 +2251,7 @@ typedef struct { const int16_t opcode_gfx9[static_cast(aco_opcode::num_opcodes)]; const int16_t opcode_gfx10[static_cast(aco_opcode::num_opcodes)]; const int16_t opcode_gfx11[static_cast(aco_opcode::num_opcodes)]; + const int16_t opcode_gfx12[static_cast(aco_opcode::num_opcodes)]; const std::bitset(aco_opcode::num_opcodes)> can_use_input_modifiers; const std::bitset(aco_opcode::num_opcodes)> can_use_output_modifiers; const std::bitset(aco_opcode::num_opcodes)> is_atomic; diff --git a/src/amd/compiler/aco_opcodes.py b/src/amd/compiler/aco_opcodes.py index 5fa8d3bf10d..2087783b33b 100644 --- a/src/amd/compiler/aco_opcodes.py +++ b/src/amd/compiler/aco_opcodes.py @@ -116,7 +116,8 @@ class Format(IntEnum): return [('uint8_t', 'attr', 0), ('uint8_t', 'attr_chan', 0), ('memory_sync_info', 'sync', 'memory_sync_info()'), - ('uint8_t', 'wait_vdst', 15)] + ('uint8_t', 'wait_vdst', 15), + ('uint8_t', 'wait_vsrc', 1)] elif self == Format.MTBUF: return [('unsigned', 'dfmt', None), ('unsigned', 'nfmt', None), diff --git a/src/amd/compiler/aco_opcodes_cpp.py b/src/amd/compiler/aco_opcodes_cpp.py index 46dc89e3c07..e972288f840 100644 --- a/src/amd/compiler/aco_opcodes_cpp.py +++ b/src/amd/compiler/aco_opcodes_cpp.py @@ -40,6 +40,11 @@ extern const aco::Info instr_info = { ${instructions[name].op.gfx11}, % endfor }, + { + % for name in opcode_names: + ${instructions[name].op.gfx12}, + % endfor + }, std::bitset<${len(opcode_names)}>("${can_use_input_modifiers}"), std::bitset<${len(opcode_names)}>("${can_use_output_modifiers}"), std::bitset<${len(opcode_names)}>("${is_atomic}"), diff --git a/src/amd/compiler/aco_print_ir.cpp b/src/amd/compiler/aco_print_ir.cpp index 3fe100d8e8d..0009f01640c 100644 --- a/src/amd/compiler/aco_print_ir.cpp +++ b/src/amd/compiler/aco_print_ir.cpp @@ -452,6 +452,8 @@ print_instr_format_specific(enum amd_gfx_level gfx_level, const Instruction* ins fprintf(output, " attr%u.%c", ldsdir.attr, "xyzw"[ldsdir.attr_chan]); if (ldsdir.wait_vdst != 15) fprintf(output, " wait_vdst:%u", ldsdir.wait_vdst); + if (ldsdir.wait_vsrc != 1) + fprintf(output, " wait_vsrc:%u", ldsdir.wait_vsrc); print_sync(ldsdir.sync, output); break; }