From 71d8126e1bb8d838137440d7c2509c6ce4925fc0 Mon Sep 17 00:00:00 2001 From: Faith Ekstrand Date: Thu, 11 Jul 2024 18:20:20 -0500 Subject: [PATCH] nak/sm50: Split IAdd2 into IAdd2 and IAdd2X Part-of: --- src/nouveau/compiler/nak/builder.rs | 5 +- src/nouveau/compiler/nak/ir.rs | 25 ++++++- src/nouveau/compiler/nak/opt_copy_prop.rs | 10 +-- src/nouveau/compiler/nak/sm50.rs | 90 +++++++++++++++++++++-- 4 files changed, 110 insertions(+), 20 deletions(-) diff --git a/src/nouveau/compiler/nak/builder.rs b/src/nouveau/compiler/nak/builder.rs index cab4e2e4874..d792787b0d5 100644 --- a/src/nouveau/compiler/nak/builder.rs +++ b/src/nouveau/compiler/nak/builder.rs @@ -288,7 +288,6 @@ pub trait SSABuilder: Builder { self.push_op(OpIAdd2 { dst: dst.into(), srcs: [x, y], - carry_in: 0.into(), carry_out: Dst::None, }); } @@ -338,9 +337,8 @@ pub trait SSABuilder: Builder { dst: dst[0].into(), srcs: [x[0].into(), y[0].into()], carry_out: carry.into(), - carry_in: 0.into(), }); - self.push_op(OpIAdd2 { + self.push_op(OpIAdd2X { dst: dst[1].into(), srcs: [x[1].into(), y[1].into()], carry_out: Dst::None, @@ -417,7 +415,6 @@ pub trait SSABuilder: Builder { self.push_op(OpIAdd2 { dst: dst.into(), srcs: [0.into(), i.ineg()], - carry_in: 0.into(), carry_out: Dst::None, }); } diff --git a/src/nouveau/compiler/nak/ir.rs b/src/nouveau/compiler/nak/ir.rs index e1aa87f7dd7..70f6076101c 100644 --- a/src/nouveau/compiler/nak/ir.rs +++ b/src/nouveau/compiler/nak/ir.rs @@ -3050,14 +3050,31 @@ pub struct OpIAdd2 { pub dst: Dst, pub carry_out: Dst, - #[src_type(ALU)] + #[src_type(I32)] pub srcs: [Src; 2], - pub carry_in: Src, } impl DisplayOp for OpIAdd2 { fn fmt_op(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "iadd2 {} {}", self.srcs[0], self.srcs[1])?; + write!(f, "iadd2 {} {}", self.srcs[0], self.srcs[1]) + } +} + +/// Only used on SM50 +#[repr(C)] +#[derive(SrcsAsSlice, DstsAsSlice)] +pub struct OpIAdd2X { + pub dst: Dst, + pub carry_out: Dst, + + #[src_type(B32)] + pub srcs: [Src; 2], + pub carry_in: Src, +} + +impl DisplayOp for OpIAdd2X { + fn fmt_op(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "iadd2.x {} {}", self.srcs[0], self.srcs[1])?; if !self.carry_in.is_zero() { write!(f, " {}", self.carry_in)?; } @@ -5359,6 +5376,7 @@ pub enum Op { Flo(OpFlo), IAbs(OpIAbs), IAdd2(OpIAdd2), + IAdd2X(OpIAdd2X), IAdd3(OpIAdd3), IAdd3X(OpIAdd3X), IDp4(OpIDp4), @@ -5813,6 +5831,7 @@ impl Instr { Op::BMsk(_) | Op::IAbs(_) | Op::IAdd2(_) + | Op::IAdd2X(_) | Op::IAdd3(_) | Op::IAdd3X(_) | Op::IDp4(_) diff --git a/src/nouveau/compiler/nak/opt_copy_prop.rs b/src/nouveau/compiler/nak/opt_copy_prop.rs index aaae7bd8581..a17c46222a1 100644 --- a/src/nouveau/compiler/nak/opt_copy_prop.rs +++ b/src/nouveau/compiler/nak/opt_copy_prop.rs @@ -562,12 +562,10 @@ impl CopyPropPass { assert!(dst.comps() == 1); let dst = dst[0]; - if add.carry_in.is_zero() { - if add.srcs[0].is_zero() { - self.add_copy(bi, dst, SrcType::I32, add.srcs[1]); - } else if add.srcs[1].is_zero() { - self.add_copy(bi, dst, SrcType::I32, add.srcs[0]); - } + if add.srcs[0].is_zero() { + self.add_copy(bi, dst, SrcType::I32, add.srcs[1]); + } else if add.srcs[1].is_zero() { + self.add_copy(bi, dst, SrcType::I32, add.srcs[0]); } } Op::IAdd3(add) => { diff --git a/src/nouveau/compiler/nak/sm50.rs b/src/nouveau/compiler/nak/sm50.rs index a7c511790f0..e63285c7a42 100644 --- a/src/nouveau/compiler/nak/sm50.rs +++ b/src/nouveau/compiler/nak/sm50.rs @@ -170,6 +170,16 @@ impl SM50Encoder<'_> { self.set_bit(neg_bit, src.src_mod.is_ineg()); } + fn set_reg_bnot_src( + &mut self, + range: Range, + not_bit: usize, + src: Src, + ) { + self.set_reg_src_ref(range, src.src_ref); + self.set_bit(not_bit, src.src_mod.is_bnot()); + } + fn set_pred_dst(&mut self, range: Range, dst: Dst) { match dst { Dst::None => { @@ -278,6 +288,21 @@ impl SM50Encoder<'_> { self.set_bit(neg_bit, src.src_mod.is_ineg()); } + + fn set_cb_bnot_src( + &mut self, + range: Range, + not_bit: usize, + src: Src, + ) { + if let SrcRef::CBuf(cb) = &src.src_ref { + self.set_src_cb(range, cb); + } else { + panic!("Not a CBuf source"); + } + + self.set_bit(not_bit, src.src_mod.is_bnot()); + } } // @@ -1074,11 +1099,6 @@ impl SM50Op for OpIAdd2 { } fn encode(&self, e: &mut SM50Encoder<'_>) { - let carry_in = match self.carry_in.src_ref { - SrcRef::Reg(reg) if reg.file() == RegFile::Carry => true, - SrcRef::Zero => false, - other => panic!("invalid carry_in src for IADD2 {other}"), - }; let carry_out = match self.carry_out { Dst::Reg(reg) if reg.file() == RegFile::Carry => true, Dst::None => false, @@ -1092,7 +1112,7 @@ impl SM50Op for OpIAdd2 { e.set_reg_ineg_src(8..16, 56, self.srcs[0]); e.set_src_imm32(20..52, imm32); - e.set_bit(53, carry_in); + e.set_bit(43, false); // .X e.set_bit(52, carry_out); } else { match &self.srcs[1].src_ref { @@ -1114,7 +1134,62 @@ impl SM50Op for OpIAdd2 { e.set_dst(self.dst); e.set_reg_ineg_src(8..16, 49, self.srcs[0]); - e.set_bit(43, carry_in); + e.set_bit(43, false); // .X + e.set_bit(47, carry_out); + } + } +} + +impl SM50Op for OpIAdd2X { + fn legalize(&mut self, b: &mut LegalizeBuilder) { + use RegFile::GPR; + let [src0, src1] = &mut self.srcs; + swap_srcs_if_not_reg(src0, src1, GPR); + b.copy_alu_src_if_not_reg(src0, GPR, SrcType::I32); + } + + fn encode(&self, e: &mut SM50Encoder<'_>) { + match self.carry_in.src_ref { + SrcRef::Reg(reg) if reg.file() == RegFile::Carry => (), + other => panic!("invalid carry_out dst for iadd2.x {other}"), + } + + let carry_out = match self.carry_out { + Dst::Reg(reg) if reg.file() == RegFile::Carry => true, + Dst::None => false, + other => panic!("invalid carry_out dst for IADD2 {other}"), + }; + + if let Some(imm32) = self.srcs[1].as_imm_not_i20() { + e.set_opcode(0x1c00); + + e.set_dst(self.dst); + e.set_reg_bnot_src(8..16, 56, self.srcs[0]); + e.set_src_imm32(20..52, imm32); + + e.set_bit(43, true); // .X + e.set_bit(52, carry_out); + } else { + match &self.srcs[1].src_ref { + SrcRef::Zero | SrcRef::Reg(_) => { + e.set_opcode(0x5c10); + e.set_reg_bnot_src(20..28, 48, self.srcs[1]); + } + SrcRef::Imm32(imm) => { + e.set_opcode(0x3810); + e.set_src_imm_i20(20..39, 56, *imm); + } + SrcRef::CBuf(_) => { + e.set_opcode(0x4c10); + e.set_cb_bnot_src(20..39, 48, self.srcs[1]); + } + src => panic!("Unsupported src type for IADD: {src}"), + } + + e.set_dst(self.dst); + e.set_reg_bnot_src(8..16, 49, self.srcs[0]); + + e.set_bit(43, true); // .X e.set_bit(47, carry_out); } } @@ -2643,6 +2718,7 @@ macro_rules! as_sm50_op_match { Op::DMul(op) => op, Op::DSetP(op) => op, Op::IAdd2(op) => op, + Op::IAdd2X(op) => op, Op::Mov(op) => op, Op::Sel(op) => op, Op::Shfl(op) => op,