From a4c537c5b3a0b883265a561817e88bb443f50b8d Mon Sep 17 00:00:00 2001 From: Georg Lehmann Date: Sun, 6 Jul 2025 20:54:49 +0200 Subject: [PATCH] aco: use new disable_wqm for mubuf/mtbuf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foz-DB GFX1201: Totals from 66 (0.08% of 80251) affected shaders: Instrs: 45373 -> 45663 (+0.64%); split: -0.01%, +0.65% CodeSize: 251708 -> 252900 (+0.47%); split: -0.00%, +0.48% Latency: 278977 -> 278652 (-0.12%); split: -0.14%, +0.02% InvThroughput: 38259 -> 38245 (-0.04%); split: -0.05%, +0.02% VClause: 982 -> 962 (-2.04%) Copies: 2882 -> 2808 (-2.57%) PreSGPRs: 2564 -> 2599 (+1.37%) SALU: 4748 -> 5010 (+5.52%) Reviewed-by: Daniel Schürmann Part-of: --- src/amd/compiler/aco_insert_exec_mask.cpp | 18 ++++++++++---- src/amd/compiler/aco_ir.cpp | 3 ++- .../aco_select_nir_intrinsics.cpp | 24 ++++++++++++++----- src/amd/compiler/tests/test_isel.cpp | 2 +- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/amd/compiler/aco_insert_exec_mask.cpp b/src/amd/compiler/aco_insert_exec_mask.cpp index 86e86b2847b..e57d68a2577 100644 --- a/src/amd/compiler/aco_insert_exec_mask.cpp +++ b/src/amd/compiler/aco_insert_exec_mask.cpp @@ -62,11 +62,7 @@ struct exec_ctx { bool needs_exact(aco_ptr& instr) { - if (instr->isMUBUF()) { - return instr->mubuf().disable_wqm; - } else if (instr->isMTBUF()) { - return instr->mtbuf().disable_wqm; - } else if (instr->isMIMG()) { + if (instr->isMIMG()) { return instr->mimg().disable_wqm; } else if (instr->isFlatLike()) { return instr->flatlike().disable_wqm; @@ -420,6 +416,12 @@ remove_disable_wqm(Instruction* instr) { assert(instr_disables_wqm(instr)); + if (instr->isMUBUF()) { + instr->mubuf().disable_wqm = false; + } else if (instr->isMTBUF()) { + instr->mtbuf().disable_wqm = false; + } + /* Remove the two masks so that the assembler doesn't need to handle them. */ instr->operands.pop_back(); instr->operands.pop_back(); @@ -835,6 +837,12 @@ insert_exec_mask(Program* program) bool instr_disables_wqm(Instruction* instr) { + if (instr->isMUBUF()) { + return instr->mubuf().disable_wqm; + } else if (instr->isMTBUF()) { + return instr->mtbuf().disable_wqm; + } + return false; } diff --git a/src/amd/compiler/aco_ir.cpp b/src/amd/compiler/aco_ir.cpp index 772e0bc023c..5d65d9e8d42 100644 --- a/src/amd/compiler/aco_ir.cpp +++ b/src/amd/compiler/aco_ir.cpp @@ -1449,7 +1449,8 @@ get_tied_defs(Instruction* instr) instr->opcode == aco_opcode::ds_bvh_stack_push8_pop1_rtn_b32 || instr->opcode == aco_opcode::ds_bvh_stack_push8_pop2_rtn_b64) { ops.push_back(0); - } else if (instr->isMUBUF() && instr->definitions.size() == 1 && instr->operands.size() == 4) { + } else if (instr->isMUBUF() && instr->definitions.size() == 1 && + (instr_info.is_atomic[(int)instr->opcode] || instr->mubuf().tfe)) { ops.push_back(3); } else if (instr->isMIMG() && instr->definitions.size() == 1 && !instr->operands[2].isUndefined()) { diff --git a/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp b/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp index 1671eff9325..c20e8f8de88 100644 --- a/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp +++ b/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp @@ -2206,11 +2206,13 @@ visit_image_store(isel_context* ctx, nir_intrinsic_instr* instr) default: UNREACHABLE(">4 channel buffer image store"); } } - aco_ptr store{create_instruction(opcode, Format::MUBUF, 4, 0)}; + aco_ptr store{create_instruction(opcode, Format::MUBUF, 6, 0)}; store->operands[0] = Operand(rsrc); store->operands[1] = Operand(vindex); store->operands[2] = Operand::c32(0); store->operands[3] = Operand(data); + store->operands[4] = Operand(); + store->operands[5] = Operand(); store->mubuf().idxen = true; store->mubuf().cache = cache; store->mubuf().disable_wqm = true; @@ -2357,12 +2359,14 @@ visit_image_atomic(isel_context* ctx, nir_intrinsic_instr* instr) Temp resource = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa)); // assert(ctx->options->gfx_level < GFX9 && "GFX9 stride size workaround not yet // implemented."); - aco_ptr mubuf{create_instruction(is_64bit ? buf_op64 : buf_op, Format::MUBUF, 4, + aco_ptr mubuf{create_instruction(is_64bit ? buf_op64 : buf_op, Format::MUBUF, 6, return_previous ? 1 : 0)}; mubuf->operands[0] = Operand(resource); mubuf->operands[1] = Operand(vindex); mubuf->operands[2] = Operand::c32(0); mubuf->operands[3] = Operand(data); + mubuf->operands[4] = Operand(); + mubuf->operands[5] = Operand(); Definition def = return_previous ? (cmpswap ? bld.def(data.regClass()) : Definition(dst)) : Definition(); if (return_previous) @@ -2449,11 +2453,13 @@ visit_store_ssbo(isel_context* ctx, nir_intrinsic_instr* instr) if (write_datas[i].bytes() < 4) access |= ACCESS_MAY_STORE_SUBDWORD; - aco_ptr store{create_instruction(op, Format::MUBUF, 4, 0)}; + aco_ptr store{create_instruction(op, Format::MUBUF, 6, 0)}; store->operands[0] = Operand(rsrc); store->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1); store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand::c32(0); store->operands[3] = Operand(write_datas[i]); + store->operands[4] = Operand(); + store->operands[5] = Operand(); store->mubuf().offset = offsets[i]; store->mubuf().offen = (offset.type() == RegType::vgpr); store->mubuf().cache = get_cache_flags(ctx, access); @@ -2486,11 +2492,13 @@ visit_atomic_ssbo(isel_context* ctx, nir_intrinsic_instr* instr) Temp dst = get_ssa_temp(ctx, &instr->def); aco_opcode op = instr->def.bit_size == 32 ? op32 : op64; - aco_ptr mubuf{create_instruction(op, Format::MUBUF, 4, return_previous ? 1 : 0)}; + aco_ptr mubuf{create_instruction(op, Format::MUBUF, 6, return_previous ? 1 : 0)}; mubuf->operands[0] = Operand(rsrc); mubuf->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1); mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand::c32(0); mubuf->operands[3] = Operand(data); + mubuf->operands[4] = Operand(); + mubuf->operands[5] = Operand(); Definition def = return_previous ? (cmpswap ? bld.def(data.regClass()) : Definition(dst)) : Definition(); if (return_previous) @@ -2639,7 +2647,7 @@ visit_store_global(isel_context* ctx, nir_intrinsic_instr* instr) Temp rsrc = get_mubuf_global_rsrc(bld, write_address); - aco_ptr mubuf{create_instruction(op, Format::MUBUF, 4, 0)}; + aco_ptr mubuf{create_instruction(op, Format::MUBUF, 6, 0)}; mubuf->operands[0] = Operand(rsrc); if (write_address.type() == RegType::vgpr) mubuf->operands[1] = Operand(write_address); @@ -2650,6 +2658,8 @@ visit_store_global(isel_context* ctx, nir_intrinsic_instr* instr) mubuf->operands[2] = write_offset.type() == RegType::sgpr ? Operand(write_offset) : Operand::c32(0); mubuf->operands[3] = Operand(write_datas[i]); + mubuf->operands[4] = Operand(); + mubuf->operands[5] = Operand(); mubuf->mubuf().offen = write_offset.type() == RegType::vgpr; mubuf->mubuf().cache = get_cache_flags(ctx, access); mubuf->mubuf().offset = write_const_offset; @@ -2779,7 +2789,7 @@ visit_global_atomic(isel_context* ctx, nir_intrinsic_instr* instr) aco_opcode op = instr->def.bit_size == 32 ? op32 : op64; - aco_ptr mubuf{create_instruction(op, Format::MUBUF, 4, return_previous ? 1 : 0)}; + aco_ptr mubuf{create_instruction(op, Format::MUBUF, 6, return_previous ? 1 : 0)}; mubuf->operands[0] = Operand(rsrc); if (addr.type() == RegType::vgpr) mubuf->operands[1] = Operand(addr); @@ -2789,6 +2799,8 @@ visit_global_atomic(isel_context* ctx, nir_intrinsic_instr* instr) mubuf->operands[1] = Operand(v1); mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand::c32(0); mubuf->operands[3] = Operand(data); + mubuf->operands[4] = Operand(); + mubuf->operands[5] = Operand(); Definition def = return_previous ? (cmpswap ? bld.def(data.regClass()) : Definition(dst)) : Definition(); if (return_previous) diff --git a/src/amd/compiler/tests/test_isel.cpp b/src/amd/compiler/tests/test_isel.cpp index b6ad3803e43..36b49c267f8 100644 --- a/src/amd/compiler/tests/test_isel.cpp +++ b/src/amd/compiler/tests/test_isel.cpp @@ -53,7 +53,7 @@ BEGIN_TEST(isel.compute.simple) }; void main() { //>> v1: %data = p_parallelcopy 42 - //! buffer_store_dword (kill)%_, v1: undef, 0, (kill)%data disable_wqm storage:buffer + //! buffer_store_dword (kill)%_, v1: undef, 0, (kill)%data storage:buffer res = 42; } );