aco/isel: add init_disable_wqm helper

Reviewed-by: Rhys Perry <pendingchaos02@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/36785>

src/amd/compiler/instruction_selection/aco_instruction_selection.h

@@ -7,6 +7,7 @@
 #ifndef ACO_INSTRUCTION_SELECTION_H
 #define ACO_INSTRUCTION_SELECTION_H
 
+#include "aco_builder.h"
 #include "aco_ir.h"
 
 #include "nir.h"
@@ -178,6 +179,18 @@ should_declare_array(ac_image_dim dim)
           dim == ac_image_2darraymsaa;
 }
 
+template <typename T>
+inline void
+init_disable_wqm(Builder& bld, T& instr, bool disable_wqm)
+{
+   if (disable_wqm) {
+      instr_exact_mask(&instr) = Operand();
+      instr_wqm_mask(&instr) = Operand();
+      instr.disable_wqm = true;
+      bld.program->needs_exact = true;
+   }
+}
+
 /* aco_isel_setup.cpp */
 void init_context(isel_context* ctx, nir_shader* shader);
 void cleanup_context(isel_context* ctx);

src/amd/compiler/instruction_selection/aco_isel_helpers.cpp

@@ -551,13 +551,7 @@ emit_mimg(Builder& bld, aco_opcode op, std::vector<Temp> dsts, Temp rsrc, Operan
    for (unsigned i = 0; i < coords.size(); i++)
       mimg->operands[3 + i] = Operand(coords[i]);
 
-   if (disable_wqm) {
-      instr_exact_mask(mimg.get()) = Operand();
-      instr_wqm_mask(mimg.get()) = Operand();
-      mimg->mimg().disable_wqm = true;
-      bld.program->needs_exact = true;
-   }
-
+   init_disable_wqm(bld, mimg->mimg(), disable_wqm);
    mimg->mimg().strict_wqm = strict_wqm;
 
    return &bld.insert(std::move(mimg))->mimg();

src/amd/compiler/instruction_selection/aco_select_nir.cpp

@@ -465,12 +465,7 @@ visit_tex(isel_context* ctx, nir_tex_instr* instr)
       mubuf->mubuf().tfe = instr->is_sparse;
       if (mubuf->mubuf().tfe)
          mubuf->operands[3] = emit_tfe_init(bld, tmp_dst);
-      if (disable_wqm) {
-         instr_exact_mask(mubuf.get()) = Operand();
-         instr_wqm_mask(mubuf.get()) = Operand();
-         mubuf->mubuf().disable_wqm = true;
-         bld.program->needs_exact = true;
-      }
+      init_disable_wqm(bld, mubuf->mubuf(), disable_wqm);
       ctx->block->instructions.emplace_back(std::move(mubuf));
 
       expand_vector(ctx, tmp_dst, dst, instr->def.num_components, dmask);

src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp

@@ -1990,13 +1990,10 @@ visit_image_store(isel_context* ctx, nir_intrinsic_instr* instr)
       store->operands[1] = Operand(vindex);
       store->operands[2] = Operand::c32(0);
       store->operands[3] = Operand(data);
-      store->operands[4] = Operand();
-      store->operands[5] = Operand();
       store->mubuf().idxen = true;
       store->mubuf().cache = cache;
-      store->mubuf().disable_wqm = true;
       store->mubuf().sync = sync;
-      ctx->program->needs_exact = true;
+      init_disable_wqm(bld, store->mubuf(), true);
       ctx->block->instructions.emplace_back(std::move(store));
       return;
    }
@@ -2142,8 +2139,6 @@ visit_image_atomic(isel_context* ctx, nir_intrinsic_instr* instr)
       mubuf->operands[1] = Operand(vindex);
       mubuf->operands[2] = Operand::c32(0);
       mubuf->operands[3] = Operand(data);
-      mubuf->operands[4] = Operand();
-      mubuf->operands[5] = Operand();
       Definition def =
          return_previous ? (cmpswap ? bld.def(data.regClass()) : Definition(dst)) : Definition();
       if (return_previous)
@@ -2151,9 +2146,8 @@ visit_image_atomic(isel_context* ctx, nir_intrinsic_instr* instr)
       mubuf->mubuf().offset = 0;
       mubuf->mubuf().idxen = true;
       mubuf->mubuf().cache = get_atomic_cache_flags(ctx, return_previous);
-      mubuf->mubuf().disable_wqm = true;
       mubuf->mubuf().sync = sync;
-      ctx->program->needs_exact = true;
+      init_disable_wqm(bld, mubuf->mubuf(), true);
       ctx->block->instructions.emplace_back(std::move(mubuf));
       if (return_previous && cmpswap)
          bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), def.getTemp(), Operand::zero());
@@ -2234,14 +2228,11 @@ visit_store_ssbo(isel_context* ctx, nir_intrinsic_instr* instr)
       store->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
       store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand::c32(0);
       store->operands[3] = Operand(write_datas[i]);
-      store->operands[4] = Operand();
-      store->operands[5] = Operand();
       store->mubuf().offset = offsets[i];
       store->mubuf().offen = (offset.type() == RegType::vgpr);
       store->mubuf().cache = get_cache_flags(ctx, access, type);
-      store->mubuf().disable_wqm = true;
       store->mubuf().sync = sync;
-      ctx->program->needs_exact = true;
+      init_disable_wqm(bld, store->mubuf(), true);
       ctx->block->instructions.emplace_back(std::move(store));
    }
 }
@@ -2273,8 +2264,6 @@ visit_atomic_ssbo(isel_context* ctx, nir_intrinsic_instr* instr)
    mubuf->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand::c32(0);
    mubuf->operands[3] = Operand(data);
-   mubuf->operands[4] = Operand();
-   mubuf->operands[5] = Operand();
    Definition def =
       return_previous ? (cmpswap ? bld.def(data.regClass()) : Definition(dst)) : Definition();
    if (return_previous)
@@ -2282,9 +2271,8 @@ visit_atomic_ssbo(isel_context* ctx, nir_intrinsic_instr* instr)
    mubuf->mubuf().offset = 0;
    mubuf->mubuf().offen = (offset.type() == RegType::vgpr);
    mubuf->mubuf().cache = get_atomic_cache_flags(ctx, return_previous);
-   mubuf->mubuf().disable_wqm = true;
    mubuf->mubuf().sync = get_memory_sync_info(instr, storage_buffer, semantic_atomicrmw);
-   ctx->program->needs_exact = true;
+   init_disable_wqm(bld, mubuf->mubuf(), true);
    ctx->block->instructions.emplace_back(std::move(mubuf));
    if (return_previous && cmpswap)
       bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), def.getTemp(), Operand::zero());
@@ -2409,14 +2397,11 @@ visit_store_global(isel_context* ctx, nir_intrinsic_instr* instr)
             flat->operands[1] = Operand(s1);
          }
          flat->operands[2] = Operand(write_datas[i]);
-         flat->operands[3] = Operand();
-         flat->operands[4] = Operand();
          flat->flatlike().cache = get_cache_flags(ctx, access, type);
          assert(global || !write_const_offset);
          flat->flatlike().offset = write_const_offset;
-         flat->flatlike().disable_wqm = true;
          flat->flatlike().sync = sync;
-         ctx->program->needs_exact = true;
+         init_disable_wqm(bld, flat->flatlike(), true);
          ctx->block->instructions.emplace_back(std::move(flat));
       } else {
          assert(ctx->options->gfx_level == GFX6 || write_address.type() != RegType::vgpr);
@@ -2436,15 +2421,12 @@ visit_store_global(isel_context* ctx, nir_intrinsic_instr* instr)
          mubuf->operands[2] =
             write_offset.type() == RegType::sgpr ? Operand(write_offset) : Operand::c32(0);
          mubuf->operands[3] = Operand(write_datas[i]);
-         mubuf->operands[4] = Operand();
-         mubuf->operands[5] = Operand();
          mubuf->mubuf().offen = write_offset.type() == RegType::vgpr;
          mubuf->mubuf().cache = get_cache_flags(ctx, access, type);
          mubuf->mubuf().offset = write_const_offset;
          mubuf->mubuf().addr64 = write_address.type() == RegType::vgpr;
-         mubuf->mubuf().disable_wqm = true;
          mubuf->mubuf().sync = sync;
-         ctx->program->needs_exact = true;
+         init_disable_wqm(bld, mubuf->mubuf(), true);
          ctx->block->instructions.emplace_back(std::move(mubuf));
       }
    }
@@ -2548,16 +2530,13 @@ visit_global_atomic(isel_context* ctx, nir_intrinsic_instr* instr)
          flat->operands[1] = Operand(s1);
       }
       flat->operands[2] = Operand(data);
-      flat->operands[3] = Operand();
-      flat->operands[4] = Operand();
       if (return_previous)
          flat->definitions[0] = Definition(dst);
       flat->flatlike().cache = get_atomic_cache_flags(ctx, return_previous);
       assert(global || !const_offset);
       flat->flatlike().offset = const_offset;
-      flat->flatlike().disable_wqm = true;
       flat->flatlike().sync = get_memory_sync_info(instr, storage_buffer, semantic_atomicrmw);
-      ctx->program->needs_exact = true;
+      init_disable_wqm(bld, flat->flatlike(), true);
       ctx->block->instructions.emplace_back(std::move(flat));
    } else {
       assert(ctx->options->gfx_level == GFX6 || addr.type() != RegType::vgpr);
@@ -2579,8 +2558,6 @@ visit_global_atomic(isel_context* ctx, nir_intrinsic_instr* instr)
          mubuf->operands[1] = Operand(v1);
       mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand::c32(0);
       mubuf->operands[3] = Operand(data);
-      mubuf->operands[4] = Operand();
-      mubuf->operands[5] = Operand();
       Definition def =
          return_previous ? (cmpswap ? bld.def(data.regClass()) : Definition(dst)) : Definition();
       if (return_previous)
@@ -2589,9 +2566,8 @@ visit_global_atomic(isel_context* ctx, nir_intrinsic_instr* instr)
       mubuf->mubuf().cache = get_atomic_cache_flags(ctx, return_previous);
       mubuf->mubuf().offset = const_offset;
       mubuf->mubuf().addr64 = addr.type() == RegType::vgpr;
-      mubuf->mubuf().disable_wqm = true;
       mubuf->mubuf().sync = get_memory_sync_info(instr, storage_buffer, semantic_atomicrmw);
-      ctx->program->needs_exact = true;
+      init_disable_wqm(bld, mubuf->mubuf(), true);
       ctx->block->instructions.emplace_back(std::move(mubuf));
       if (return_previous && cmpswap)
          bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), def.getTemp(), Operand::zero());