nak: Add OpLdSharedLock and OpStSCheckUnlock

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/35028>

src/nouveau/compiler/nak/from_nir.rs

@@ -3008,6 +3008,31 @@ impl<'a> ShaderFromNir<'a> {
                 });
                 self.set_dst(&intrin.def, dst);
             }
+            nir_intrinsic_load_shared_lock_nv => {
+                let size_B = intrin.def.bit_size() / 8;
+                let mem_type = MemType::from_size(size_B, false);
+
+                let (addr, offset) = self.get_io_addr_offset(&srcs[0], 24);
+                let dst = b.alloc_ssa_vec(RegFile::GPR, size_B.div_ceil(4));
+                let locked = b.alloc_ssa(RegFile::Pred);
+
+                b.push_op(OpLdSharedLock {
+                    dst: dst.clone().into(),
+                    locked: locked.clone().into(),
+                    addr,
+                    offset,
+                    mem_type,
+                });
+                let locked_gpr = b.sel(locked.into(), 1.into(), 0.into());
+
+                // for 32-bit we have 2x32 return type,
+                // for 64-bit we need 2x64, so is_locked must be a 64-bit val.
+                // we can fill the remaining SSAValue with a copy of is_locked
+                let locked_dst = std::iter::repeat(locked_gpr).take(dst.len());
+                let nir_dst: Vec<_> =
+                    dst.iter().copied().chain(locked_dst).collect();
+                self.set_ssa(intrin.def.as_def(), nir_dst);
+            }
             nir_intrinsic_load_sysval_nv => {
                 let idx = u8::try_from(intrin.base()).unwrap();
                 debug_assert!(intrin.def.num_components == 1);
@@ -3370,6 +3395,25 @@ impl<'a> ShaderFromNir<'a> {
                     access: access,
                 });
             }
+            nir_intrinsic_store_shared_unlock_nv => {
+                let data = self.get_src(&srcs[0]);
+                let size_B =
+                    (srcs[0].bit_size() / 8) * srcs[0].num_components();
+                let mem_type = MemType::from_size(size_B, false);
+
+                let (addr, offset) = self.get_io_addr_offset(&srcs[1], 24);
+                let locked = b.alloc_ssa(RegFile::Pred);
+
+                b.push_op(OpStSCheckUnlock {
+                    locked: locked.clone().into(),
+                    addr,
+                    data,
+                    offset,
+                    mem_type,
+                });
+                let locked_gpr = b.sel(locked.into(), 1.into(), 0.into());
+                self.set_dst(intrin.def.as_def(), locked_gpr.into());
+            }
             nir_intrinsic_emit_vertex_nv | nir_intrinsic_end_primitive_nv => {
                 assert!(intrin.def.bit_size() == 32);
                 assert!(intrin.def.num_components() == 1);

src/nouveau/compiler/nak/ir.rs

@@ -5259,6 +5259,34 @@ impl DisplayOp for OpLdc {
 }
 impl_display_for_op!(OpLdc);
 
+/// Used for Kepler to implement shared atomics.
+/// In addition to the load, it tries to lock the address,
+/// Kepler hardware has (1024?) hardware mutex locks.
+#[repr(C)]
+#[derive(SrcsAsSlice, DstsAsSlice)]
+pub struct OpLdSharedLock {
+    pub dst: Dst,
+    #[dst_type(Pred)]
+    pub locked: Dst,
+
+    #[src_type(GPR)]
+    pub addr: Src,
+
+    pub offset: i32,
+    pub mem_type: MemType,
+}
+
+impl DisplayOp for OpLdSharedLock {
+    fn fmt_op(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "ldslk{} [{}", self.mem_type, self.addr)?;
+        if self.offset > 0 {
+            write!(f, "+{:#x}", self.offset)?;
+        }
+        write!(f, "]")
+    }
+}
+impl_display_for_op!(OpLdSharedLock);
+
 #[repr(C)]
 #[derive(SrcsAsSlice, DstsAsSlice)]
 pub struct OpSt {
@@ -5283,6 +5311,35 @@ impl DisplayOp for OpSt {
 }
 impl_display_for_op!(OpSt);
 
+/// Used for Kepler to implement shared atomics.
+/// It checks that the address is still properly locked, performs the
+/// store operation and unlocks the previously unlocked address.
+#[repr(C)]
+#[derive(SrcsAsSlice, DstsAsSlice)]
+pub struct OpStSCheckUnlock {
+    #[dst_type(Pred)]
+    pub locked: Dst,
+
+    #[src_type(GPR)]
+    pub addr: Src,
+    #[src_type(SSA)]
+    pub data: Src,
+
+    pub offset: i32,
+    pub mem_type: MemType,
+}
+
+impl DisplayOp for OpStSCheckUnlock {
+    fn fmt_op(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "stscul{} [{}", self.mem_type, self.addr)?;
+        if self.offset > 0 {
+            write!(f, "+{:#x}", self.offset)?;
+        }
+        write!(f, "] {}", self.data)
+    }
+}
+impl_display_for_op!(OpStSCheckUnlock);
+
 #[repr(C)]
 #[derive(SrcsAsSlice, DstsAsSlice)]
 pub struct OpAtom {
@@ -6621,7 +6678,9 @@ pub enum Op {
     SuAtom(OpSuAtom),
     Ld(OpLd),
     Ldc(OpLdc),
+    LdSharedLock(OpLdSharedLock),
     St(OpSt),
+    StSCheckUnlock(OpStSCheckUnlock),
     Atom(OpAtom),
     AL2P(OpAL2P),
     ALd(OpALd),
@@ -6780,7 +6839,9 @@ impl Op {
             // Memory ops
             Op::Ld(_)
             | Op::Ldc(_)
+            | Op::LdSharedLock(_)
             | Op::St(_)
+            | Op::StSCheckUnlock(_)
             | Op::Atom(_)
             | Op::AL2P(_)
             | Op::ALd(_)
@@ -7190,7 +7251,9 @@ impl Instr {
             Op::ASt(_)
             | Op::SuSt(_)
             | Op::SuAtom(_)
+            | Op::LdSharedLock(_)
             | Op::St(_)
+            | Op::StSCheckUnlock(_)
             | Op::Atom(_)
             | Op::CCtl(_)
             | Op::MemBar(_)

src/nouveau/compiler/nak/opt_instr_sched_common.rs

@@ -163,7 +163,9 @@ pub fn side_effect_type(op: &Op) -> SideEffect {
         // Memory ops
         Op::Ipa(_) | Op::Ldc(_) => SideEffect::None,
         Op::Ld(_)
+        | Op::LdSharedLock(_)
         | Op::St(_)
+        | Op::StSCheckUnlock(_)
         | Op::Atom(_)
         | Op::AL2P(_)
         | Op::ALd(_)
@@ -266,7 +268,9 @@ pub fn estimate_variable_latency(sm: u8, op: &Op) -> u32 {
         Op::Ldc(_) => 4,
 
         Op::Ld(_)
+        | Op::LdSharedLock(_)
         | Op::St(_)
+        | Op::StSCheckUnlock(_)
         | Op::Atom(_)
         | Op::AL2P(_)
         | Op::ALd(_)