From 6de68c5dcad60c354856a0a574c711e5fc2f5d0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Sch=C3=BCrmann?= <daniel@schuermann.dev>
Date: Fri, 11 Mar 2022 19:12:50 +0100
Subject: [PATCH] aco: Avoid live-range splits in Exact mode

Because the data register of atomic VMEM instructions
is shared between src and dst, it might be necessary
to create live-range splits during RA.
Make the live-range splits explicit in WQM mode.

Totals from 7 (0.01% of 134913) affected shaders: (GFX10.3)
Latency: 17209 -> 17210 (+0.01%)

Reviewed-by: Rhys Perry <pendingchaos02@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/15347>
---
 src/amd/compiler/aco_insert_exec_mask.cpp | 31 ++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/src/amd/compiler/aco_insert_exec_mask.cpp b/src/amd/compiler/aco_insert_exec_mask.cpp
index c96ee88c92e..81bb71f4160 100644
--- a/src/amd/compiler/aco_insert_exec_mask.cpp
+++ b/src/amd/compiler/aco_insert_exec_mask.cpp
@@ -482,6 +482,33 @@ add_coupling_code(exec_ctx& ctx, Block* block, std::vector<aco_ptr<Instruction>>
    return i;
 }
 
+/* Avoid live-range splits in Exact mode:
+ * Because the data register of atomic VMEM instructions
+ * is shared between src and dst, it might be necessary
+ * to create live-range splits during RA.
+ * Make the live-range splits explicit in WQM mode.
+ */
+void
+handle_atomic_data(exec_ctx& ctx, Builder& bld, unsigned block_idx, aco_ptr<Instruction>& instr)
+{
+   /* check if this is an atomic VMEM instruction */
+   int idx = -1;
+   if (!instr->isVMEM() || instr->definitions.empty())
+      return;
+   else if (instr->isMIMG())
+      idx = instr->operands[2].isTemp() ? 2 : -1;
+   else if (instr->operands.size() == 4)
+      idx = 3;
+
+   if (idx != -1) {
+      /* insert explicit copy of atomic data in WQM-mode */
+      transition_to_WQM(ctx, bld, block_idx);
+      Temp data = instr->operands[idx].getTemp();
+      data = bld.copy(bld.def(data.regClass()), data);
+      instr->operands[idx].setTemp(data);
+   }
+}
+
 void
 process_instructions(exec_ctx& ctx, Block* block, std::vector<aco_ptr<Instruction>>& instructions,
                      unsigned idx)
@@ -517,7 +544,9 @@ process_instructions(exec_ctx& ctx, Block* block, std::vector<aco_ptr<Instructio
       if (needs == WQM && state != WQM) {
          transition_to_WQM(ctx, bld, block->index);
          state = WQM;
-      } else if (needs == Exact && state != Exact) {
+      } else if (needs == Exact) {
+         if (ctx.info[block->index].block_needs & WQM)
+            handle_atomic_data(ctx, bld, block->index, instr);
          transition_to_Exact(ctx, bld, block->index);
          state = Exact;
       }