From 053b975ca19bacdd7e007a0e68e9b65c90559215 Mon Sep 17 00:00:00 2001
From: Karol Herbst <kherbst@redhat.com>
Date: Mon, 5 May 2025 23:06:12 +0200
Subject: [PATCH] nak: fix MMA latencies for Ampere

Acked-by: Faith Ekstrand <faith.ekstrand@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/32777>
---
 .../compiler/nak/sm80_instr_latencies.rs      | 57 ++++++++++---------
 1 file changed, 30 insertions(+), 27 deletions(-)

diff --git a/src/nouveau/compiler/nak/sm80_instr_latencies.rs b/src/nouveau/compiler/nak/sm80_instr_latencies.rs
index d1c8b32413b..590decbe353 100644
--- a/src/nouveau/compiler/nak/sm80_instr_latencies.rs
+++ b/src/nouveau/compiler/nak/sm80_instr_latencies.rs
@@ -257,8 +257,8 @@ impl RegLatencySM80 {
                 RedirectedFP64 => 10,
                 Clmad => 12,
                 IMMA_88 => 13,
-                MMA_1x_collect => 13,
-                MMA_2x_collect => 17,
+                MMA_1x_collect => 16,
+                MMA_2x_collect => 24,
                 DMMA => 25,
                 Cbu => 1,
                 Decoupled => 1,
@@ -280,8 +280,8 @@ impl RegLatencySM80 {
                 RedirectedFP64 => 10,
                 Clmad => 12,
                 IMMA_88 => 13,
-                MMA_1x_collect => 13,
-                MMA_2x_collect => 17,
+                MMA_1x_collect => 16,
+                MMA_2x_collect => 24,
                 DMMA => 25,
                 Cbu => 1,
                 Decoupled => 1,
@@ -303,8 +303,8 @@ impl RegLatencySM80 {
                 RedirectedFP64 => 10,
                 Clmad => 12,
                 IMMA_88 => 13,
-                MMA_1x_collect => 13,
-                MMA_2x_collect => 17,
+                MMA_1x_collect => 16,
+                MMA_2x_collect => 24,
                 DMMA => 25,
                 Cbu => 1,
                 Decoupled => 1,
@@ -326,8 +326,8 @@ impl RegLatencySM80 {
                 RedirectedFP64 => 8,
                 Clmad => 10,
                 IMMA_88 => 11,
-                MMA_1x_collect => 11,
-                MMA_2x_collect => 15,
+                MMA_1x_collect => 14,
+                MMA_2x_collect => 22,
                 DMMA => 23,
                 Cbu => 1,
                 Decoupled => 1,
@@ -353,8 +353,8 @@ impl RegLatencySM80 {
                     RedirectedFP64 => 10,
                     Clmad => 12,
                     IMMA_88 => 13,
-                    MMA_1x_collect => 13,
-                    MMA_2x_collect => 17,
+                    MMA_1x_collect => 16,
+                    MMA_2x_collect => 24,
                     DMMA => 25,
                     Cbu => 1,
                     Decoupled => 1,
@@ -377,8 +377,8 @@ impl RegLatencySM80 {
                 RedirectedFP64 => 10,
                 Clmad => 12,
                 IMMA_88 => 13,
-                MMA_1x_collect => 13,
-                MMA_2x_collect => 17,
+                MMA_1x_collect => 16,
+                MMA_2x_collect => 24,
                 DMMA => 25,
                 Cbu => 1,
                 Decoupled => 1,
@@ -400,8 +400,8 @@ impl RegLatencySM80 {
                 RedirectedFP64 => 6,
                 Clmad => 12,
                 IMMA_88 => 13,
-                MMA_1x_collect => 13,
-                MMA_2x_collect => 17,
+                MMA_1x_collect => 16,
+                MMA_2x_collect => 24,
                 DMMA => 25,
                 Cbu => 1,
                 Decoupled => 1,
@@ -423,8 +423,8 @@ impl RegLatencySM80 {
                 RedirectedFP64 => 10,
                 Clmad => 8,
                 IMMA_88 => 13,
-                MMA_1x_collect => 13,
-                MMA_2x_collect => 17,
+                MMA_1x_collect => 16,
+                MMA_2x_collect => 24,
                 DMMA => 25,
                 Cbu => 1,
                 Decoupled => 1,
@@ -447,8 +447,8 @@ impl RegLatencySM80 {
                     RedirectedFP64 => 11,
                     Clmad => 13,
                     IMMA_88 => 14,        //6??
-                    MMA_1x_collect => 14, //6??
-                    MMA_2x_collect => 18,
+                    MMA_1x_collect => 16, //6??
+                    MMA_2x_collect => 24,
                     DMMA => 26,
                     Cbu => 1,
                     Decoupled => 1,
@@ -472,8 +472,8 @@ impl RegLatencySM80 {
                     RedirectedFP64 => 11,
                     Clmad => 13,
                     IMMA_88 => 14,
-                    MMA_1x_collect => 14,
-                    MMA_2x_collect => 18, //10??
+                    MMA_1x_collect => 16,
+                    MMA_2x_collect => 24, //10??
                     DMMA => 26,
                     Cbu => 1,
                     Decoupled => 1,
@@ -497,8 +497,8 @@ impl RegLatencySM80 {
                     RedirectedFP64 => 11,
                     Clmad => 13,
                     IMMA_88 => 14,
-                    MMA_1x_collect => 14,
-                    MMA_2x_collect => 18,
+                    MMA_1x_collect => 16,
+                    MMA_2x_collect => 24,
                     DMMA => 26, //18??
                     Cbu => 1,
                     Decoupled => 1,
@@ -521,8 +521,8 @@ impl RegLatencySM80 {
                 RedirectedFP64 => 6,
                 Clmad => 8,
                 IMMA_88 => 11,
-                MMA_1x_collect => 11,
-                MMA_2x_collect => 15,
+                MMA_1x_collect => 14,
+                MMA_2x_collect => 22,
                 DMMA => 23,
                 Cbu => 1,
                 Decoupled => 1,
@@ -544,8 +544,8 @@ impl RegLatencySM80 {
                 RedirectedFP64 => 7,
                 Clmad => 9,
                 IMMA_88 => 12,
-                MMA_1x_collect => 12,
-                MMA_2x_collect => 16,
+                MMA_1x_collect => 15,
+                MMA_2x_collect => 23,
                 DMMA => 24,
                 Cbu => 1,
                 Decoupled => 1,
@@ -787,7 +787,10 @@ impl RegLatencySM80 {
         match writer {
             CoupledAlu | CoupledDisp64 | CoupledFMA | IMADWideWriteDL
             | IMADWideWriteDH | FP16 | FP16_Alu | FP16_F32 | HFMA2_MMA
-            | RedirectedFP64 => 1,
+            | RedirectedFP64 => match reader {
+                MMA_2x_collect => 7,
+                _ => 1,
+            },
             Clmad | IMMA_88 | MMA_1x_collect | MMA_2x_collect | DMMA | Cbu
             | Decoupled | DecoupledAgu => match reader {
                 CoupledAlu | CoupledDisp64 | CoupledFMA | IMADWideReadAB