From a9d27892372f9ed67ba86bc16018733249abd813 Mon Sep 17 00:00:00 2001 From: Mel Henning Date: Wed, 30 Apr 2025 19:17:28 -0400 Subject: [PATCH] nak: Src is no longer Copy Reviewed-by: Faith Ekstrand Part-of: --- src/nouveau/compiler/nak/assign_regs.rs | 2 +- src/nouveau/compiler/nak/builder.rs | 35 +-- src/nouveau/compiler/nak/from_nir.rs | 11 +- src/nouveau/compiler/nak/ir.rs | 4 +- src/nouveau/compiler/nak/legalize.rs | 20 +- src/nouveau/compiler/nak/opt_copy_prop.rs | 99 +++++-- src/nouveau/compiler/nak/opt_lop.rs | 6 +- src/nouveau/compiler/nak/sm20.rs | 114 ++++---- src/nouveau/compiler/nak/sm50.rs | 332 +++++++++++----------- src/nouveau/compiler/nak/sm70_encode.rs | 199 ++++++------- src/nouveau/compiler/nak/to_cssa.rs | 9 +- 11 files changed, 443 insertions(+), 388 deletions(-) diff --git a/src/nouveau/compiler/nak/assign_regs.rs b/src/nouveau/compiler/nak/assign_regs.rs index 794e79dabb0..3168be470a6 100644 --- a/src/nouveau/compiler/nak/assign_regs.rs +++ b/src/nouveau/compiler/nak/assign_regs.rs @@ -1240,7 +1240,7 @@ impl AssignRegsBlock { for (i, src) in out.srcs.iter().enumerate() { let reg = u32::try_from(i).unwrap(); let dst = RegRef::new(RegFile::GPR, reg, 1); - pcopy.push(dst.into(), *src); + pcopy.push(dst.into(), src.clone()); } None diff --git a/src/nouveau/compiler/nak/builder.rs b/src/nouveau/compiler/nak/builder.rs index 65176f29552..926e8835b8d 100644 --- a/src/nouveau/compiler/nak/builder.rs +++ b/src/nouveau/compiler/nak/builder.rs @@ -154,7 +154,7 @@ pub trait SSABuilder: Builder { dst: dst[0].into(), low: x[0].into(), high: 0.into(), - shift, + shift: shift.clone(), right: false, wrap: true, data_type: IntType::U64, @@ -168,7 +168,7 @@ pub trait SSABuilder: Builder { dst: dst[0].into(), low: 0.into(), high: x[0].into(), - shift, + shift: shift.clone(), right: false, wrap: true, data_type: IntType::U64, @@ -222,7 +222,7 @@ pub trait SSABuilder: Builder { dst: dst[0].into(), low: x[0].into(), high: x[1].into(), - shift, + shift: shift.clone(), right: true, wrap: true, data_type: if signed { IntType::I64 } else { IntType::U64 }, @@ -384,8 +384,8 @@ pub trait SSABuilder: Builder { let is_3src = !x.is_zero() && !y.is_zero() && !z.is_zero(); - let x = split_iadd64_src(x); - let y = split_iadd64_src(y); + let [x0, x1] = split_iadd64_src(x); + let [y0, y1] = split_iadd64_src(y); let dst = self.alloc_ssa_vec(RegFile::GPR, 2); if self.sm() >= 70 { let carry1 = self.alloc_ssa(RegFile::Pred); @@ -398,16 +398,16 @@ pub trait SSABuilder: Builder { (Dst::None, false.into()) }; - let z = split_iadd64_src(z); + let [z0, z1] = split_iadd64_src(z); self.push_op(OpIAdd3 { dst: dst[0].into(), overflow: [carry1.into(), carry2_dst], - srcs: [x[0], y[0], z[0]], + srcs: [x0, y0, z0], }); self.push_op(OpIAdd3X { dst: dst[1].into(), overflow: [Dst::None, Dst::None], - srcs: [x[1], y[1], z[1]], + srcs: [x1, y1, z1], carry: [carry1.into(), carry2_src], }); } else { @@ -415,12 +415,12 @@ pub trait SSABuilder: Builder { let carry = self.alloc_ssa(RegFile::Carry); self.push_op(OpIAdd2 { dst: dst[0].into(), - srcs: [x[0], y[0]], + srcs: [x0, y0], carry_out: carry.into(), }); self.push_op(OpIAdd2X { dst: dst[1].into(), - srcs: [x[1], y[1]], + srcs: [x1, y1], carry_out: Dst::None, carry_in: carry.into(), }); @@ -469,7 +469,7 @@ pub trait SSABuilder: Builder { } else { self.push_op(OpIMul { dst: dst[0].into(), - srcs: [x, y], + srcs: [x.clone(), y.clone()], signed: [signed; 2], high: false, }); @@ -772,8 +772,9 @@ pub trait SSABuilder: Builder { fn prmt4(&mut self, src: [Src; 4], sel: [u8; 4]) -> SSAValue { let max_sel = *sel.iter().max().unwrap(); + let [src0, src1, src2, src3] = src; if max_sel < 8 { - self.prmt(src[0], src[1], sel) + self.prmt(src0, src1, sel) } else if max_sel < 12 { let mut sel_a = [0_u8; 4]; let mut sel_b = [0_u8; 4]; @@ -785,8 +786,8 @@ pub trait SSABuilder: Builder { sel_b[usize::from(i)] = (sel[usize::from(i)] - 8) + 4; } } - let a = self.prmt(src[0], src[1], sel_a); - self.prmt(a.into(), src[2], sel_b) + let a = self.prmt(src0, src1, sel_a); + self.prmt(a.into(), src2, sel_b) } else if max_sel < 16 { let mut sel_a = [0_u8; 4]; let mut sel_b = [0_u8; 4]; @@ -800,8 +801,8 @@ pub trait SSABuilder: Builder { sel_c[usize::from(i)] = 4 + i; } } - let a = self.prmt(src[0], src[1], sel_a); - let b = self.prmt(src[2], src[3], sel_b); + let a = self.prmt(src0, src1, sel_a); + let b = self.prmt(src2, src3, sel_b); self.prmt(a.into(), b.into(), sel_c) } else { panic!("Invalid permute value: {max_sel}"); @@ -827,7 +828,7 @@ pub trait SSABuilder: Builder { self.push_op(OpPSetP { dsts: [tmp.into(), Dst::None], ops: [PredSetOp::And, PredSetOp::And], - srcs: [cond, x, true.into()], + srcs: [cond.clone(), x, true.into()], }); self.push_op(OpPSetP { dsts: [dst.into(), Dst::None], diff --git a/src/nouveau/compiler/nak/from_nir.rs b/src/nouveau/compiler/nak/from_nir.rs index bf7fcfed942..d990af909f6 100644 --- a/src/nouveau/compiler/nak/from_nir.rs +++ b/src/nouveau/compiler/nak/from_nir.rs @@ -568,7 +568,7 @@ impl<'a> ShaderFromNir<'a> { } 8 => { for dc in 0..bits.div_ceil(32) { - let mut psrc = [None; 4]; + let mut psrc = [None, None, None, None]; let mut psel = [0_u8; 4]; for b in 0..4 { @@ -594,9 +594,9 @@ impl<'a> ShaderFromNir<'a> { let psrc = { let mut res = [Src::ZERO; 4]; - for (idx, src) in psrc.iter().enumerate() { + for (idx, src) in psrc.into_iter().enumerate() { if let Some(src) = src { - res[idx] = *src; + res[idx] = src; } } @@ -620,7 +620,8 @@ impl<'a> ShaderFromNir<'a> { psel[w * 2 + 1] = (w as u8 * 4) + byte + 1; } } - comps.push(b.prmt(psrc[0], psrc[1], psel)); + let [psrc0, psrc1] = psrc; + comps.push(b.prmt(psrc0, psrc1, psel)); } } _ => panic!("Unknown bit size: {src_bit_size}"), @@ -1290,7 +1291,7 @@ impl<'a> ShaderFromNir<'a> { let x = restrict_f16v2_src(src0); let lz = restrict_f16v2_src( - b.hset2(FloatCmpOp::OrdLt, x, 0.into()).into(), + b.hset2(FloatCmpOp::OrdLt, x.clone(), 0.into()).into(), ); let gz = restrict_f16v2_src( b.hset2(FloatCmpOp::OrdGt, x, 0.into()).into(), diff --git a/src/nouveau/compiler/nak/ir.rs b/src/nouveau/compiler/nak/ir.rs index 0048d619f25..a98b0ecd885 100644 --- a/src/nouveau/compiler/nak/ir.rs +++ b/src/nouveau/compiler/nak/ir.rs @@ -1103,7 +1103,7 @@ impl fmt::Display for SrcSwizzle { } } -#[derive(Clone, Copy, PartialEq)] +#[derive(Clone, PartialEq)] pub struct Src { pub src_ref: SrcRef, pub src_mod: SrcMod, @@ -1236,7 +1236,7 @@ impl Src { } } - pub fn to_ssa(&self) -> SSARef { + pub fn to_ssa(self) -> SSARef { if self.src_mod.is_none() { self.src_ref.to_ssa() } else { diff --git a/src/nouveau/compiler/nak/legalize.rs b/src/nouveau/compiler/nak/legalize.rs index 0d6dbf2f222..facdca22452 100644 --- a/src/nouveau/compiler/nak/legalize.rs +++ b/src/nouveau/compiler/nak/legalize.rs @@ -237,34 +237,34 @@ pub trait LegalizeBuildHelpers: SSABuilder { match src_type { SrcType::F16 | SrcType::F16v2 => { let val = self.alloc_ssa(reg_file); + let old_src = std::mem::replace(src, val.into()); self.push_op(OpHAdd2 { dst: val.into(), - srcs: [Src::ZERO.fneg(), *src], + srcs: [Src::ZERO.fneg(), old_src], saturate: false, ftz: false, f32: false, }); - *src = val.into(); } SrcType::F32 => { let val = self.alloc_ssa(reg_file); + let old_src = std::mem::replace(src, val.into()); self.push_op(OpFAdd { dst: val.into(), - srcs: [Src::ZERO.fneg(), *src], + srcs: [Src::ZERO.fneg(), old_src], saturate: false, rnd_mode: FRndMode::NearestEven, ftz: false, }); - *src = val.into(); } SrcType::F64 => { let val = self.alloc_ssa_vec(reg_file, 2); + let old_src = std::mem::replace(src, val.into()); self.push_op(OpDAdd { dst: val.into(), - srcs: [Src::ZERO.fneg(), *src], + srcs: [Src::ZERO.fneg(), old_src], rnd_mode: FRndMode::NearestEven, }); - *src = val.into(); } _ => panic!("Invalid ffabs srouce type"), } @@ -278,9 +278,10 @@ pub trait LegalizeBuildHelpers: SSABuilder { ) { assert!(src_type == SrcType::I32); let val = self.alloc_ssa(reg_file); + let old_src = std::mem::replace(src, val.into()); if self.sm() >= 70 { self.push_op(OpIAdd3 { - srcs: [Src::ZERO, *src, Src::ZERO], + srcs: [Src::ZERO, old_src, Src::ZERO], overflow: [Dst::None, Dst::None], dst: val.into(), }); @@ -288,10 +289,9 @@ pub trait LegalizeBuildHelpers: SSABuilder { self.push_op(OpIAdd2 { dst: val.into(), carry_out: Dst::None, - srcs: [Src::ZERO, *src], + srcs: [Src::ZERO, old_src], }); } - *src = val.into(); } fn copy_alu_src_if_fabs( @@ -429,7 +429,7 @@ fn legalize_instr( }) => { let bar_in_ssa = bar_in.src_ref.as_ssa().unwrap(); if !bar_out.is_none() && bl.is_live_after_ip(&bar_in_ssa[0], ip) { - let gpr = b.bmov_to_gpr(*bar_in); + let gpr = b.bmov_to_gpr(bar_in.clone()); let tmp = b.bmov_to_bar(gpr.into()); *bar_in = tmp.into(); } diff --git a/src/nouveau/compiler/nak/opt_copy_prop.rs b/src/nouveau/compiler/nak/opt_copy_prop.rs index 53a69be966c..9c9662165f2 100644 --- a/src/nouveau/compiler/nak/opt_copy_prop.rs +++ b/src/nouveau/compiler/nak/opt_copy_prop.rs @@ -319,7 +319,7 @@ impl CopyPropPass { } let entry_src_idx = usize::from(entry_src_idx.unwrap()); - let entry_src = entry.srcs[entry_src_idx]; + let entry_src = &entry.srcs[entry_src_idx]; if !cbuf_rule.allows_src(entry.bi, &entry_src) { return; @@ -514,9 +514,19 @@ impl CopyPropPass { if !add.saturate { if add.srcs[0].is_fneg_zero(SrcType::F16v2) { - self.add_copy(bi, dst, SrcType::F16v2, add.srcs[1]); + self.add_copy( + bi, + dst, + SrcType::F16v2, + add.srcs[1].clone(), + ); } else if add.srcs[1].is_fneg_zero(SrcType::F16v2) { - self.add_copy(bi, dst, SrcType::F16v2, add.srcs[0]); + self.add_copy( + bi, + dst, + SrcType::F16v2, + add.srcs[0].clone(), + ); } } } @@ -527,18 +537,28 @@ impl CopyPropPass { if !add.saturate { if add.srcs[0].is_fneg_zero(SrcType::F32) { - self.add_copy(bi, dst, SrcType::F32, add.srcs[1]); + self.add_copy( + bi, + dst, + SrcType::F32, + add.srcs[1].clone(), + ); } else if add.srcs[1].is_fneg_zero(SrcType::F32) { - self.add_copy(bi, dst, SrcType::F32, add.srcs[0]); + self.add_copy( + bi, + dst, + SrcType::F32, + add.srcs[0].clone(), + ); } } } Op::DAdd(add) => { let dst = add.dst.as_ssa().unwrap(); if add.srcs[0].is_fneg_zero(SrcType::F64) { - self.add_fp64_copy(bi, dst, add.srcs[1]); + self.add_fp64_copy(bi, dst, add.srcs[1].clone()); } else if add.srcs[1].is_fneg_zero(SrcType::F64) { - self.add_fp64_copy(bi, dst, add.srcs[0]); + self.add_fp64_copy(bi, dst, add.srcs[0].clone()); } } Op::Lop3(lop) => { @@ -559,7 +579,12 @@ impl CopyPropPass { } else { for s in 0..3 { if op.lut == LogicOp3::SRC_MASKS[s] { - self.add_copy(bi, dst, SrcType::ALU, lop.srcs[s]); + self.add_copy( + bi, + dst, + SrcType::ALU, + lop.srcs[s].clone(), + ); } } } @@ -596,14 +621,14 @@ impl CopyPropPass { bi, dst, SrcType::Pred, - lop.srcs[s], + lop.srcs[s].clone(), ); } else if op.lut == !LogicOp3::SRC_MASKS[s] { self.add_copy( bi, dst, SrcType::Pred, - lop.srcs[s].bnot(), + lop.srcs[s].clone().bnot(), ); } } @@ -615,9 +640,11 @@ impl CopyPropPass { assert!(dst.comps() == 1); let dst = dst[0]; - let src = match (sel.srcs[0], sel.srcs[1]) { - (z, u) if z.is_zero() && u.is_nonzero() => sel.cond.bnot(), - (u, z) if z.is_zero() && u.is_nonzero() => sel.cond, + let src = match &sel.srcs { + [z, u] if z.is_zero() && u.is_nonzero() => { + sel.cond.clone().bnot() + } + [u, z] if z.is_zero() && u.is_nonzero() => sel.cond.clone(), _ => return, }; @@ -628,7 +655,7 @@ impl CopyPropPass { assert!(dst.comps() == 1); let dst = dst[0]; - let src = match (isetp.srcs[0], isetp.srcs[1]) { + let src = match (&isetp.srcs[0], &isetp.srcs[1]) { (z, x) | (x, z) if z.is_zero() => x, _ => return, }; @@ -645,7 +672,7 @@ impl CopyPropPass { IntCmpOp::Ne => false, _ => return, }; - self.add_i2b(dst, src, inverted); + self.add_i2b(dst, src.clone(), inverted); } Op::IAdd2(add) => { let dst = add.dst.as_ssa().unwrap(); @@ -653,9 +680,9 @@ impl CopyPropPass { let dst = dst[0]; if add.srcs[0].is_zero() { - self.add_copy(bi, dst, SrcType::I32, add.srcs[1]); + self.add_copy(bi, dst, SrcType::I32, add.srcs[1].clone()); } else if add.srcs[1].is_zero() { - self.add_copy(bi, dst, SrcType::I32, add.srcs[0]); + self.add_copy(bi, dst, SrcType::I32, add.srcs[0].clone()); } } Op::IAdd3(add) => { @@ -665,12 +692,22 @@ impl CopyPropPass { if add.srcs[0].is_zero() { if add.srcs[1].is_zero() { - self.add_copy(bi, dst, SrcType::I32, add.srcs[2]); + self.add_copy( + bi, + dst, + SrcType::I32, + add.srcs[2].clone(), + ); } else if add.srcs[2].is_zero() { - self.add_copy(bi, dst, SrcType::I32, add.srcs[1]); + self.add_copy( + bi, + dst, + SrcType::I32, + add.srcs[1].clone(), + ); } } else if add.srcs[1].is_zero() && add.srcs[2].is_zero() { - self.add_copy(bi, dst, SrcType::I32, add.srcs[0]); + self.add_copy(bi, dst, SrcType::I32, add.srcs[0].clone()); } } Op::Prmt(prmt) => { @@ -680,11 +717,21 @@ impl CopyPropPass { if let Some(imm) = prmt.as_u32() { self.add_copy(bi, dst[0], SrcType::GPR, imm.into()); } else if sel == PrmtSel(0x3210) { - self.add_copy(bi, dst[0], SrcType::GPR, prmt.srcs[0]); + self.add_copy( + bi, + dst[0], + SrcType::GPR, + prmt.srcs[0].clone(), + ); } else if sel == PrmtSel(0x7654) { - self.add_copy(bi, dst[0], SrcType::GPR, prmt.srcs[1]); + self.add_copy( + bi, + dst[0], + SrcType::GPR, + prmt.srcs[1].clone(), + ); } else { - self.add_prmt(bi, dst[0], sel, prmt.srcs); + self.add_prmt(bi, dst[0], sel, prmt.srcs.clone()); } } } @@ -693,19 +740,19 @@ impl CopyPropPass { if r2ur.src.is_uniform() { let dst = r2ur.dst.as_ssa().unwrap(); assert!(dst.comps() == 1); - self.add_copy(bi, dst[0], SrcType::GPR, r2ur.src); + self.add_copy(bi, dst[0], SrcType::GPR, r2ur.src.clone()); } } Op::Copy(copy) => { let dst = copy.dst.as_ssa().unwrap(); assert!(dst.comps() == 1); - self.add_copy(bi, dst[0], SrcType::GPR, copy.src); + self.add_copy(bi, dst[0], SrcType::GPR, copy.src.clone()); } Op::ParCopy(pcopy) => { for (dst, src) in pcopy.dsts_srcs.iter() { let dst = dst.as_ssa().unwrap(); assert!(dst.comps() == 1); - self.add_copy(bi, dst[0], SrcType::GPR, *src); + self.add_copy(bi, dst[0], SrcType::GPR, src.clone()); } } _ => (), diff --git a/src/nouveau/compiler/nak/opt_lop.rs b/src/nouveau/compiler/nak/opt_lop.rs index 9ace08bd3bd..422ca9242db 100644 --- a/src/nouveau/compiler/nak/opt_lop.rs +++ b/src/nouveau/compiler/nak/opt_lop.rs @@ -173,7 +173,7 @@ impl LopPass { for i in 0..3 { if entry_srcs[i] != usize::MAX { - srcs[entry_srcs[i]] = entry.srcs[i]; + srcs[entry_srcs[i]] = entry.srcs[i].clone(); } } for op in ops.iter_mut() { @@ -215,7 +215,7 @@ impl LopPass { if let Dst::SSA(ssa) = op.dst { assert!(ssa.comps() == 1); - self.add_lop(ssa[0], op.op, op.srcs); + self.add_lop(ssa[0], op.op, op.srcs.clone()); } } @@ -248,7 +248,7 @@ impl LopPass { for i in 0..2 { if let Dst::SSA(ssa) = op.dsts[i] { assert!(ssa.comps() == 1); - self.add_lop(ssa[0], op.ops[i], op.srcs); + self.add_lop(ssa[0], op.ops[i], op.srcs.clone()); } } } diff --git a/src/nouveau/compiler/nak/sm20.rs b/src/nouveau/compiler/nak/sm20.rs index 2637353202d..e2997c67749 100644 --- a/src/nouveau/compiler/nak/sm20.rs +++ b/src/nouveau/compiler/nak/sm20.rs @@ -231,7 +231,7 @@ impl SM20Encoder<'_> { self.set_field(range, reg.base_idx()); } - fn set_pred_src(&mut self, range: Range, src: Src) { + fn set_pred_src(&mut self, range: Range, src: &Src) { let (not, reg) = match src.src_ref { SrcRef::True => (false, true_reg()), SrcRef::False => (true, true_reg()), @@ -299,7 +299,7 @@ impl SM20Encoder<'_> { } } - fn set_reg_src(&mut self, range: Range, src: Src) { + fn set_reg_src(&mut self, range: Range, src: &Src) { assert!(src.src_swizzle.is_none()); self.set_reg_src_ref(range, &src.src_ref); } @@ -313,7 +313,7 @@ impl SM20Encoder<'_> { self.set_reg(range, reg); } - fn set_carry_in(&mut self, bit: usize, src: Src) { + fn set_carry_in(&mut self, bit: usize, src: &Src) { assert!(src.src_mod.is_none()); match src.src_ref { SrcRef::Zero => self.set_bit(bit, false), @@ -672,7 +672,7 @@ impl SM20Op for OpFMnMx { e.set_bit(7, self.srcs[0].src_mod.has_fabs()); e.set_bit(8, self.srcs[1].src_mod.has_fneg()); e.set_bit(9, self.srcs[0].src_mod.has_fneg()); - e.set_pred_src(49..53, self.min); + e.set_pred_src(49..53, &self.min); } } @@ -842,7 +842,7 @@ impl SM20Op for OpFSetP { e.set_bit(9, self.srcs[0].src_mod.has_fneg()); e.set_pred_dst(14..17, &Dst::None); e.set_pred_dst(17..20, &self.dst); - e.set_pred_src(49..53, self.accum); + e.set_pred_src(49..53, &self.accum); e.set_pred_set_op(53..55, self.set_op); e.set_float_cmp_op(55..59, self.cmp_op); e.set_bit(59, self.ftz); @@ -860,8 +860,8 @@ impl SM20Op for OpFSwz { e.set_opcode(SM20Unit::Float, 0x12); e.set_dst(14..20, &self.dst); - e.set_reg_src(20..26, self.srcs[0]); - e.set_reg_src(26..32, self.srcs[1]); + e.set_reg_src(20..26, &self.srcs[0]); + e.set_reg_src(26..32, &self.srcs[1]); e.set_bit(5, self.ftz); e.set_field( @@ -979,7 +979,7 @@ impl SM20Op for OpDMnMx { e.set_bit(7, self.srcs[0].src_mod.has_fabs()); e.set_bit(8, self.srcs[1].src_mod.has_fneg()); e.set_bit(9, self.srcs[0].src_mod.has_fneg()); - e.set_pred_src(49..53, self.min); + e.set_pred_src(49..53, &self.min); } } @@ -1035,7 +1035,7 @@ impl SM20Op for OpDSetP { e.set_bit(9, self.srcs[0].src_mod.has_fneg()); e.set_pred_dst(14..17, &Dst::None); e.set_pred_dst(17..20, &self.dst); - e.set_pred_src(49..53, self.accum); + e.set_pred_src(49..53, &self.accum); e.set_pred_set_op(53..55, self.set_op); e.set_float_cmp_op(55..59, self.cmp_op); } @@ -1159,7 +1159,7 @@ impl SM20Op for OpIAdd2X { } e.set_bit(5, false); // saturate - e.set_carry_in(6, self.carry_in); + e.set_carry_in(6, &self.carry_in); e.set_bit(8, self.srcs[1].src_mod.is_bnot()); e.set_bit(9, self.srcs[0].src_mod.is_bnot()); } @@ -1306,7 +1306,7 @@ impl SM20Op for OpIMnMx { IntCmpType::I32 => 1_u8, }, ); - e.set_pred_src(49..53, self.min); + e.set_pred_src(49..53, &self.min); } } @@ -1338,7 +1338,7 @@ impl SM20Op for OpISetP { e.set_bit(6, self.ex); e.set_pred_dst(14..17, &Dst::None); e.set_pred_dst(17..20, &self.dst); - e.set_pred_src(49..53, self.accum); + e.set_pred_src(49..53, &self.accum); e.set_pred_set_op(53..55, self.set_op); e.set_int_cmp_op(55..58, self.cmp_op); } @@ -1618,7 +1618,7 @@ impl SM20Op for OpSel { Some(&self.srcs[1]), None, ); - e.set_pred_src(49..53, self.cond); + e.set_pred_src(49..53, &self.cond); } } @@ -1637,14 +1637,14 @@ impl SM20Op for OpShfl { e.set_opcode(SM20Unit::Mem, 0x22); e.set_pred_dst2(8..10, 58..59, &self.in_bounds); e.set_dst(14..20, &self.dst); - e.set_reg_src(20..26, self.src); + e.set_reg_src(20..26, &self.src); assert!(self.lane.src_mod.is_none()); if let Some(u) = self.lane.src_ref.as_u32() { e.set_field(26..32, u & 0x1f); e.set_bit(5, true); } else { - e.set_reg_src(26..32, self.lane); + e.set_reg_src(26..32, &self.lane); e.set_bit(5, false); } @@ -1653,7 +1653,7 @@ impl SM20Op for OpShfl { e.set_field(42..55, u & 0x1fff); e.set_bit(6, true); } else { - e.set_reg_src(49..55, self.c); + e.set_reg_src(49..55, &self.c); e.set_bit(6, false); } @@ -1679,10 +1679,10 @@ impl SM20Op for OpPSetP { e.set_pred_dst(14..17, &self.dsts[1]); e.set_pred_dst(17..20, &self.dsts[0]); - e.set_pred_src(20..24, self.srcs[0]); - e.set_pred_src(26..30, self.srcs[1]); + e.set_pred_src(20..24, &self.srcs[0]); + e.set_pred_src(26..30, &self.srcs[1]); e.set_pred_set_op(30..32, self.ops[0]); - e.set_pred_src(49..53, self.srcs[2]); + e.set_pred_src(49..53, &self.srcs[2]); e.set_pred_set_op(53..55, self.ops[1]); } } @@ -1766,8 +1766,8 @@ impl SM20Op for OpTex { e.set_dst(14..20, &self.dsts[0]); assert!(self.dsts[1].is_none()); assert!(self.fault.is_none()); - e.set_reg_src(20..26, self.srcs[0]); - e.set_reg_src(26..32, self.srcs[1]); + e.set_reg_src(20..26, &self.srcs[0]); + e.set_reg_src(26..32, &self.srcs[1]); e.set_tex_channel_mask(46..50, self.channel_mask); e.set_tex_dim(51..54, self.dim); e.set_bit(54, self.offset); @@ -1804,8 +1804,8 @@ impl SM20Op for OpTld { e.set_dst(14..20, &self.dsts[0]); assert!(self.dsts[1].is_none()); assert!(self.fault.is_none()); - e.set_reg_src(20..26, self.srcs[0]); - e.set_reg_src(26..32, self.srcs[1]); + e.set_reg_src(20..26, &self.srcs[0]); + e.set_reg_src(26..32, &self.srcs[1]); e.set_tex_channel_mask(46..50, self.channel_mask); e.set_tex_dim(51..54, self.dim); e.set_bit(54, self.offset); @@ -1851,8 +1851,8 @@ impl SM20Op for OpTld4 { e.set_dst(14..20, &self.dsts[0]); assert!(self.dsts[1].is_none()); assert!(self.fault.is_none()); - e.set_reg_src(20..26, self.srcs[0]); - e.set_reg_src(26..32, self.srcs[1]); + e.set_reg_src(20..26, &self.srcs[0]); + e.set_reg_src(26..32, &self.srcs[1]); e.set_bit(45, false); // .ndv e.set_tex_channel_mask(46..50, self.channel_mask); e.set_tex_dim(51..54, self.dim); @@ -1895,8 +1895,8 @@ impl SM20Op for OpTmml { e.set_bit(9, self.nodep); e.set_dst(14..20, &self.dsts[0]); assert!(self.dsts[1].is_none()); - e.set_reg_src(20..26, self.srcs[0]); - e.set_reg_src(26..32, self.srcs[1]); + e.set_reg_src(20..26, &self.srcs[0]); + e.set_reg_src(26..32, &self.srcs[1]); e.set_tex_channel_mask(46..50, self.channel_mask); e.set_tex_dim(51..54, self.dim); } @@ -1929,8 +1929,8 @@ impl SM20Op for OpTxd { e.set_bit(9, self.nodep); e.set_dst(14..20, &self.dsts[0]); assert!(self.dsts[1].is_none()); - e.set_reg_src(20..26, self.srcs[0]); - e.set_reg_src(26..32, self.srcs[1]); + e.set_reg_src(20..26, &self.srcs[0]); + e.set_reg_src(26..32, &self.srcs[1]); e.set_tex_channel_mask(46..50, self.channel_mask); e.set_tex_dim(51..54, self.dim); e.set_bit(54, self.offset); @@ -1964,8 +1964,8 @@ impl SM20Op for OpTxq { e.set_bit(9, self.nodep); e.set_dst(14..20, &self.dsts[0]); assert!(self.dsts[1].is_none()); - e.set_reg_src(20..26, self.src); - e.set_reg_src(26..32, 0.into()); + e.set_reg_src(20..26, &self.src); + e.set_reg_src(26..32, &0.into()); e.set_tex_channel_mask(46..50, self.channel_mask); e.set_field( 54..57, @@ -2061,7 +2061,7 @@ impl SM20Op for OpLd { e.set_mem_type(5..8, self.access.mem_type); // 8..9: cache hints (.ca, .cg, .lu, .cv) e.set_dst(14..20, &self.dst); - e.set_reg_src(20..26, self.addr); + e.set_reg_src(20..26, &self.addr); } } @@ -2093,7 +2093,7 @@ impl SM20Op for OpLdc { }, ); e.set_dst(14..20, &self.dst); - e.set_reg_src(20..26, self.offset); + e.set_reg_src(20..26, &self.offset); e.set_field(26..42, cb.offset); e.set_field(42..47, cb_idx); } @@ -2124,14 +2124,14 @@ impl SM20Op for OpSt { } e.set_mem_type(5..8, self.access.mem_type); // 8..9: cache hints (.ca, .cg, .lu, .cv) - e.set_reg_src(14..20, self.data); - e.set_reg_src(20..26, self.addr); + e.set_reg_src(14..20, &self.data); + e.set_reg_src(20..26, &self.addr); } } fn atom_src_as_ssa( b: &mut LegalizeBuilder, - src: Src, + src: &Src, atom_type: AtomType, ) -> SSARef { if let Some(ssa) = src.as_ssa() { @@ -2154,8 +2154,8 @@ fn atom_src_as_ssa( impl SM20Op for OpAtom { fn legalize(&mut self, b: &mut LegalizeBuilder) { if self.atom_op == AtomOp::CmpExch(AtomCmpSrc::Separate) { - let cmpr = atom_src_as_ssa(b, self.cmpr, self.atom_type); - let data = atom_src_as_ssa(b, self.data, self.atom_type); + let cmpr = atom_src_as_ssa(b, &self.cmpr, self.atom_type); + let data = atom_src_as_ssa(b, &self.data, self.atom_type); let mut cmpr_data = Vec::new(); cmpr_data.extend_from_slice(&cmpr); @@ -2214,8 +2214,8 @@ impl SM20Op for OpAtom { e.set_field(9..10, typ & 0x1); e.set_field(59..62, typ >> 1); - e.set_reg_src(20..26, self.addr); - e.set_reg_src(14..20, self.data); + e.set_reg_src(20..26, &self.addr); + e.set_reg_src(14..20, &self.data); if self.dst.is_none() { e.set_field(26..58, self.addr_offset); @@ -2235,9 +2235,9 @@ impl SM20Op for OpAtom { let data_idx = cmpr_data.base_idx() + u32::from(data_comps); let data = RegRef::new(cmpr_data.file(), data_idx, data_comps); - e.set_reg_src(49..55, data.into()); + e.set_reg_src(49..55, &data.into()); } else if !self.dst.is_none() { - e.set_reg_src(49..55, 0.into()); + e.set_reg_src(49..55, &0.into()); } } } @@ -2261,8 +2261,8 @@ impl SM20Op for OpALd { e.set_bit(9, self.output); e.set_dst(14..20, &self.dst); - e.set_reg_src(20..26, self.offset); - e.set_reg_src(26..32, self.vtx); + e.set_reg_src(20..26, &self.offset); + e.set_reg_src(26..32, &self.vtx); e.set_field(32..42, self.addr); } } @@ -2279,10 +2279,10 @@ impl SM20Op for OpASt { e.set_bit(8, self.patch); assert!(!self.phys); - e.set_reg_src(20..26, self.offset); - e.set_reg_src(26..32, self.data); + e.set_reg_src(20..26, &self.offset); + e.set_reg_src(26..32, &self.data); e.set_field(32..42, self.addr); - e.set_reg_src(49..55, self.vtx); + e.set_reg_src(49..55, &self.vtx); } } @@ -2313,9 +2313,9 @@ impl SM20Op for OpIpa { }, ); e.set_dst(14..20, &self.dst); - e.set_reg_src(20..26, 0.into()); // indirect - e.set_reg_src(26..32, self.inv_w); - e.set_reg_src(49..55, self.offset); + e.set_reg_src(20..26, &0.into()); // indirect + e.set_reg_src(26..32, &self.inv_w); + e.set_reg_src(49..55, &self.offset); e.set_field(32..42, self.addr); } } @@ -2353,7 +2353,7 @@ impl SM20Op for OpCCtl { }, ); e.set_dst(14..20, &Dst::None); - e.set_reg_src(20..26, self.addr); + e.set_reg_src(20..26, &self.addr); e.set_field(26..28, 0); // 1: .u, 2: .c: 3: .i assert!(self.addr_offset % 4 == 0); @@ -2501,11 +2501,11 @@ impl SM20Op for OpBar { e.set_field(5..7, 0_u8); // 0: .popc, 1: .and, 2: .or e.set_field(7..9, 0_u8); // 0: .sync, 1: .arv, 2: .red - e.set_reg_src(20..26, 0.into()); - e.set_reg_src(26..32, 0.into()); + e.set_reg_src(20..26, &0.into()); + e.set_reg_src(26..32, &0.into()); e.set_bit(46, false); // src1_is_imm e.set_bit(47, false); // src0_is_imm - e.set_pred_src(49..53, true.into()); + e.set_pred_src(49..53, &true.into()); e.set_pred_dst(53..56, &Dst::None); } } @@ -2530,7 +2530,7 @@ impl SM20Op for OpIsberd { fn encode(&self, e: &mut SM20Encoder<'_>) { e.set_opcode(SM20Unit::Tex, 0x0); e.set_dst(14..20, &self.dst); - e.set_reg_src(20..26, self.idx); + e.set_reg_src(20..26, &self.idx); e.set_field(26..42, 0_u16); // offset } } @@ -2576,7 +2576,7 @@ impl SM20Op for OpPixLd { }, ); e.set_dst(14..20, &self.dst); - e.set_reg_src(20..26, 0.into()); + e.set_reg_src(20..26, &0.into()); e.set_field(26..34, 0_u16); // offset e.set_pred_dst(53..56, &Dst::None); } @@ -2610,7 +2610,7 @@ impl SM20Op for OpVote { }, ); e.set_dst(14..20, &self.ballot); - e.set_pred_src(20..24, self.pred); + e.set_pred_src(20..24, &self.pred); e.set_pred_dst(54..57, &self.vote); } } diff --git a/src/nouveau/compiler/nak/sm50.rs b/src/nouveau/compiler/nak/sm50.rs index fafdf5e2089..9cb58e88077 100644 --- a/src/nouveau/compiler/nak/sm50.rs +++ b/src/nouveau/compiler/nak/sm50.rs @@ -257,17 +257,17 @@ impl SM50Encoder<'_> { self.set_field(range, reg.base_idx()); } - fn set_reg_src_ref(&mut self, range: Range, src_ref: SrcRef) { + fn set_reg_src_ref(&mut self, range: Range, src_ref: &SrcRef) { match src_ref { SrcRef::Zero => self.set_reg(range, zero_reg()), - SrcRef::Reg(reg) => self.set_reg(range, reg), + SrcRef::Reg(reg) => self.set_reg(range, *reg), _ => panic!("Not a register"), } } - fn set_reg_src(&mut self, range: Range, src: Src) { + fn set_reg_src(&mut self, range: Range, src: &Src) { assert!(src.src_mod.is_none()); - self.set_reg_src_ref(range, src.src_ref); + self.set_reg_src_ref(range, &src.src_ref); } fn set_reg_fmod_src( @@ -275,9 +275,9 @@ impl SM50Encoder<'_> { range: Range, abs_bit: usize, neg_bit: usize, - src: Src, + src: &Src, ) { - self.set_reg_src_ref(range, src.src_ref); + self.set_reg_src_ref(range, &src.src_ref); self.set_bit(abs_bit, src.src_mod.has_fabs()); self.set_bit(neg_bit, src.src_mod.has_fneg()); } @@ -286,9 +286,9 @@ impl SM50Encoder<'_> { &mut self, range: Range, neg_bit: usize, - src: Src, + src: &Src, ) { - self.set_reg_src_ref(range, src.src_ref); + self.set_reg_src_ref(range, &src.src_ref); self.set_bit(neg_bit, src.src_mod.is_ineg()); } @@ -296,9 +296,9 @@ impl SM50Encoder<'_> { &mut self, range: Range, not_bit: usize, - src: Src, + src: &Src, ) { - self.set_reg_src_ref(range, src.src_ref); + self.set_reg_src_ref(range, &src.src_ref); self.set_bit(not_bit, src.src_mod.is_bnot()); } @@ -312,7 +312,7 @@ impl SM50Encoder<'_> { } } - fn set_pred_src(&mut self, range: Range, not_bit: usize, src: Src) { + fn set_pred_src(&mut self, range: Range, not_bit: usize, src: &Src) { let (not, reg) = match src.src_ref { SrcRef::True => (false, true_reg()), SrcRef::False => (true, true_reg()), @@ -381,7 +381,7 @@ impl SM50Encoder<'_> { range: Range, abs_bit: usize, neg_bit: usize, - src: Src, + src: &Src, ) { if let SrcRef::CBuf(cb) = &src.src_ref { self.set_src_cb(range, cb); @@ -397,7 +397,7 @@ impl SM50Encoder<'_> { &mut self, range: Range, neg_bit: usize, - src: Src, + src: &Src, ) { if let SrcRef::CBuf(cb) = &src.src_ref { self.set_src_cb(range, cb); @@ -412,7 +412,7 @@ impl SM50Encoder<'_> { &mut self, range: Range, not_bit: usize, - src: Src, + src: &Src, ) { if let SrcRef::CBuf(cb) = &src.src_ref { self.set_src_cb(range, cb); @@ -502,7 +502,7 @@ impl SM50Op for OpFAdd { if let Some(imm32) = self.srcs[1].as_imm_not_f20() { e.set_opcode(0x0800); e.set_dst(&self.dst); - e.set_reg_fmod_src(8..16, 54, 56, self.srcs[0]); + e.set_reg_fmod_src(8..16, 54, 56, &self.srcs[0]); e.set_src_imm32(20..52, imm32); assert!(self.rnd_mode == FRndMode::NearestEven); e.set_bit(55, self.ftz); @@ -510,7 +510,7 @@ impl SM50Op for OpFAdd { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c58); - e.set_reg_fmod_src(20..28, 49, 45, self.srcs[1]); + e.set_reg_fmod_src(20..28, 49, 45, &self.srcs[1]); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3858); @@ -519,13 +519,13 @@ impl SM50Op for OpFAdd { } SrcRef::CBuf(_) => { e.set_opcode(0x4c58); - e.set_cb_fmod_src(20..39, 49, 45, self.srcs[1]); + e.set_cb_fmod_src(20..39, 49, 45, &self.srcs[1]); } src => panic!("Invalid fadd src1: {src}"), } e.set_dst(&self.dst); - e.set_reg_fmod_src(8..16, 46, 48, self.srcs[0]); + e.set_reg_fmod_src(8..16, 46, 48, &self.srcs[0]); e.set_rnd_mode(39..41, self.rnd_mode); e.set_bit(44, self.ftz); @@ -567,7 +567,7 @@ impl SM50Op for OpFFma { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5980); - e.set_reg_src_ref(20..28, self.srcs[1].src_ref); + e.set_reg_src_ref(20..28, &self.srcs[1].src_ref); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3280); @@ -584,18 +584,18 @@ impl SM50Op for OpFFma { src => panic!("Invalid ffma src1: {src}"), } - e.set_reg_src_ref(39..47, self.srcs[2].src_ref); + e.set_reg_src_ref(39..47, &self.srcs[2].src_ref); } SrcRef::CBuf(cb) => { e.set_opcode(0x5180); e.set_src_cb(20..39, cb); - e.set_reg_src_ref(39..47, self.srcs[1].src_ref); + e.set_reg_src_ref(39..47, &self.srcs[1].src_ref); } src => panic!("Invalid ffma src2: {src}"), } e.set_dst(&self.dst); - e.set_reg_src_ref(8..16, self.srcs[0].src_ref); + e.set_reg_src_ref(8..16, &self.srcs[0].src_ref); e.set_bit(48, fneg_fmul); e.set_bit(49, fneg_src2); @@ -620,7 +620,7 @@ impl SM50Op for OpFMnMx { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c60); - e.set_reg_fmod_src(20..28, 49, 45, self.srcs[1]); + e.set_reg_fmod_src(20..28, 49, 45, &self.srcs[1]); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3860); @@ -629,14 +629,14 @@ impl SM50Op for OpFMnMx { } SrcRef::CBuf(_) => { e.set_opcode(0x4c60); - e.set_cb_fmod_src(20..39, 49, 45, self.srcs[1]); + e.set_cb_fmod_src(20..39, 49, 45, &self.srcs[1]); } src => panic!("Invalid fmnmx src2: {src}"), } - e.set_reg_fmod_src(8..16, 46, 48, self.srcs[0]); + e.set_reg_fmod_src(8..16, 46, 48, &self.srcs[0]); e.set_dst(&self.dst); - e.set_pred_src(39..42, 42, self.min); + e.set_pred_src(39..42, 42, &self.min); e.set_bit(44, self.ftz); } } @@ -684,7 +684,7 @@ impl SM50Op for OpFMul { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c68); - e.set_reg_src(20..28, self.srcs[1]); + e.set_reg_src(20..28, &self.srcs[1]); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3868); @@ -705,7 +705,7 @@ impl SM50Op for OpFMul { e.set_bit(50, self.saturate); } - e.set_reg_src_ref(8..16, self.srcs[0].src_ref); + e.set_reg_src_ref(8..16, &self.srcs[0].src_ref); e.set_dst(&self.dst); } } @@ -720,7 +720,7 @@ impl SM50Op for OpRro { match &self.src.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c90); - e.set_reg_fmod_src(20..28, 49, 45, self.src); + e.set_reg_fmod_src(20..28, 49, 45, &self.src); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3890); @@ -729,7 +729,7 @@ impl SM50Op for OpRro { } SrcRef::CBuf(_) => { e.set_opcode(0x4c90); - e.set_cb_fmod_src(20..39, 49, 45, self.src); + e.set_cb_fmod_src(20..39, 49, 45, &self.src); } src => panic!("Invalid rro src: {src}"), } @@ -754,7 +754,7 @@ impl SM50Op for OpMuFu { e.set_opcode(0x5080); e.set_dst(&self.dst); - e.set_reg_fmod_src(8..16, 46, 48, self.src); + e.set_reg_fmod_src(8..16, 46, 48, &self.src); e.set_field( 20..24, @@ -845,7 +845,7 @@ impl SM50Op for OpFSet { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5800); - e.set_reg_fmod_src(20..28, 44, 53, self.srcs[1]); + e.set_reg_fmod_src(20..28, 44, 53, &self.srcs[1]); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3000); @@ -854,13 +854,13 @@ impl SM50Op for OpFSet { } SrcRef::CBuf(_) => { e.set_opcode(0x4800); - e.set_cb_fmod_src(20..39, 44, 6, self.srcs[1]); + e.set_cb_fmod_src(20..39, 44, 6, &self.srcs[1]); } src => panic!("Invalid fset src1: {src}"), } - e.set_reg_fmod_src(8..16, 54, 43, self.srcs[0]); - e.set_pred_src(39..42, 42, SrcRef::True.into()); + e.set_reg_fmod_src(8..16, 54, 43, &self.srcs[0]); + e.set_pred_src(39..42, 42, &SrcRef::True.into()); e.set_float_cmp_op(48..52, self.cmp_op); e.set_bit(52, true); // bool float e.set_bit(55, self.ftz); @@ -883,7 +883,7 @@ impl SM50Op for OpFSetP { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5bb0); - e.set_reg_fmod_src(20..28, 44, 6, self.srcs[1]); + e.set_reg_fmod_src(20..28, 44, 6, &self.srcs[1]); } SrcRef::Imm32(imm32) => { e.set_opcode(0x36b0); @@ -892,15 +892,15 @@ impl SM50Op for OpFSetP { } SrcRef::CBuf(_) => { e.set_opcode(0x4bb0); - e.set_cb_fmod_src(20..39, 44, 6, self.srcs[1]); + e.set_cb_fmod_src(20..39, 44, 6, &self.srcs[1]); } src => panic!("Invalid fsetp src1: {src}"), } e.set_pred_dst(3..6, &self.dst); e.set_pred_dst(0..3, &Dst::None); // dst1 - e.set_reg_fmod_src(8..16, 7, 43, self.srcs[0]); - e.set_pred_src(39..42, 42, self.accum); + e.set_reg_fmod_src(8..16, 7, 43, &self.srcs[0]); + e.set_pred_src(39..42, 42, &self.accum); e.set_pred_set_op(45..47, self.set_op); e.set_bit(47, self.ftz); e.set_float_cmp_op(48..52, self.cmp_op); @@ -918,8 +918,8 @@ impl SM50Op for OpFSwzAdd { e.set_opcode(0x50f8); e.set_dst(&self.dst); - e.set_reg_src(8..16, self.srcs[0]); - e.set_reg_src(20..28, self.srcs[1]); + e.set_reg_src(8..16, &self.srcs[0]); + e.set_reg_src(20..28, &self.srcs[1]); e.set_field( 39..41, @@ -962,7 +962,7 @@ impl SM50Op for OpDAdd { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c70); - e.set_reg_fmod_src(20..28, 49, 45, self.srcs[1]); + e.set_reg_fmod_src(20..28, 49, 45, &self.srcs[1]); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3870); @@ -971,13 +971,13 @@ impl SM50Op for OpDAdd { } SrcRef::CBuf(_) => { e.set_opcode(0x4c70); - e.set_cb_fmod_src(20..39, 49, 45, self.srcs[1]); + e.set_cb_fmod_src(20..39, 49, 45, &self.srcs[1]); } src => panic!("Invalid dadd src1: {src}"), } e.set_dst(&self.dst); - e.set_reg_fmod_src(8..16, 46, 48, self.srcs[0]); + e.set_reg_fmod_src(8..16, 46, 48, &self.srcs[0]); e.set_rnd_mode(39..41, self.rnd_mode); } } @@ -1015,7 +1015,7 @@ impl SM50Op for OpDFma { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5b70); - e.set_reg_src_ref(20..28, self.srcs[1].src_ref); + e.set_reg_src_ref(20..28, &self.srcs[1].src_ref); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3670); @@ -1028,18 +1028,18 @@ impl SM50Op for OpDFma { src => panic!("Invalid dfma src1: {src}"), } - e.set_reg_src_ref(39..47, self.srcs[2].src_ref); + e.set_reg_src_ref(39..47, &self.srcs[2].src_ref); } SrcRef::CBuf(cb) => { e.set_opcode(0x5370); e.set_src_cb(20..39, cb); - e.set_reg_src_ref(39..47, self.srcs[1].src_ref); + e.set_reg_src_ref(39..47, &self.srcs[1].src_ref); } src => panic!("Invalid dfma src2: {src}"), } e.set_dst(&self.dst); - e.set_reg_src_ref(8..16, self.srcs[0].src_ref); + e.set_reg_src_ref(8..16, &self.srcs[0].src_ref); e.set_bit(48, fneg_fmul); e.set_bit(49, fneg_src2); @@ -1061,7 +1061,7 @@ impl SM50Op for OpDMnMx { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c50); - e.set_reg_fmod_src(20..28, 49, 45, self.srcs[1]); + e.set_reg_fmod_src(20..28, 49, 45, &self.srcs[1]); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3850); @@ -1070,14 +1070,14 @@ impl SM50Op for OpDMnMx { } SrcRef::CBuf(_) => { e.set_opcode(0x4c50); - e.set_cb_fmod_src(20..39, 49, 45, self.srcs[1]); + e.set_cb_fmod_src(20..39, 49, 45, &self.srcs[1]); } src => panic!("Invalid dmnmx src1: {src}"), } - e.set_reg_fmod_src(8..16, 46, 48, self.srcs[0]); + e.set_reg_fmod_src(8..16, 46, 48, &self.srcs[0]); e.set_dst(&self.dst); - e.set_pred_src(39..42, 42, self.min); + e.set_pred_src(39..42, 42, &self.min); } } @@ -1103,7 +1103,7 @@ impl SM50Op for OpDMul { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c80); - e.set_reg_src_ref(20..28, self.srcs[1].src_ref); + e.set_reg_src_ref(20..28, &self.srcs[1].src_ref); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3880); @@ -1117,7 +1117,7 @@ impl SM50Op for OpDMul { } e.set_dst(&self.dst); - e.set_reg_src_ref(8..16, self.srcs[0].src_ref); + e.set_reg_src_ref(8..16, &self.srcs[0].src_ref); e.set_rnd_mode(39..41, self.rnd_mode); e.set_bit(48, fneg); @@ -1139,7 +1139,7 @@ impl SM50Op for OpDSetP { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5b80); - e.set_reg_fmod_src(20..28, 44, 6, self.srcs[1]); + e.set_reg_fmod_src(20..28, 44, 6, &self.srcs[1]); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3680); @@ -1148,17 +1148,17 @@ impl SM50Op for OpDSetP { } SrcRef::CBuf(_) => { e.set_opcode(0x4b80); - e.set_reg_fmod_src(20..39, 44, 6, self.srcs[1]); + e.set_reg_fmod_src(20..39, 44, 6, &self.srcs[1]); } src => panic!("Invalid dsetp src1: {src}"), } e.set_pred_dst(3..6, &self.dst); e.set_pred_dst(0..3, &Dst::None); // dst1 - e.set_pred_src(39..42, 42, self.accum); + e.set_pred_src(39..42, 42, &self.accum); e.set_pred_set_op(45..47, self.set_op); e.set_float_cmp_op(48..52, self.cmp_op); - e.set_reg_fmod_src(8..16, 7, 43, self.srcs[0]); + e.set_reg_fmod_src(8..16, 7, 43, &self.srcs[0]); } } @@ -1172,7 +1172,7 @@ impl SM50Op for OpBfe { match &self.range.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c00); - e.set_reg_src(20..28, self.range); + e.set_reg_src(20..28, &self.range); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3800); @@ -1194,7 +1194,7 @@ impl SM50Op for OpBfe { e.set_bit(40, true); } - e.set_reg_src(8..16, self.base); + e.set_reg_src(8..16, &self.base); e.set_dst(&self.dst); } } @@ -1209,7 +1209,7 @@ impl SM50Op for OpFlo { match &self.src.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c30); - e.set_reg_src_ref(20..28, self.src.src_ref); + e.set_reg_src_ref(20..28, &self.src.src_ref); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3830); @@ -1263,7 +1263,7 @@ impl SM50Op for OpIAdd2 { e.set_opcode(0x1c00); e.set_dst(&self.dst); - e.set_reg_ineg_src(8..16, 56, self.srcs[0]); + e.set_reg_ineg_src(8..16, 56, &self.srcs[0]); e.set_src_imm32(20..52, imm32); e.set_bit(52, carry_out); @@ -1272,7 +1272,7 @@ impl SM50Op for OpIAdd2 { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c10); - e.set_reg_ineg_src(20..28, 48, self.srcs[1]); + e.set_reg_ineg_src(20..28, 48, &self.srcs[1]); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3810); @@ -1281,13 +1281,13 @@ impl SM50Op for OpIAdd2 { } SrcRef::CBuf(_) => { e.set_opcode(0x4c10); - e.set_cb_ineg_src(20..39, 48, self.srcs[1]); + e.set_cb_ineg_src(20..39, 48, &self.srcs[1]); } src => panic!("Invalid iadd src1: {src}"), } e.set_dst(&self.dst); - e.set_reg_ineg_src(8..16, 49, self.srcs[0]); + e.set_reg_ineg_src(8..16, 49, &self.srcs[0]); e.set_bit(43, false); // .X e.set_bit(47, carry_out); @@ -1319,7 +1319,7 @@ impl SM50Op for OpIAdd2X { e.set_opcode(0x1c00); e.set_dst(&self.dst); - e.set_reg_bnot_src(8..16, 56, self.srcs[0]); + e.set_reg_bnot_src(8..16, 56, &self.srcs[0]); e.set_src_imm32(20..52, imm32); e.set_bit(52, carry_out); @@ -1328,7 +1328,7 @@ impl SM50Op for OpIAdd2X { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c10); - e.set_reg_bnot_src(20..28, 48, self.srcs[1]); + e.set_reg_bnot_src(20..28, 48, &self.srcs[1]); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3810); @@ -1337,13 +1337,13 @@ impl SM50Op for OpIAdd2X { } SrcRef::CBuf(_) => { e.set_opcode(0x4c10); - e.set_cb_bnot_src(20..39, 48, self.srcs[1]); + e.set_cb_bnot_src(20..39, 48, &self.srcs[1]); } src => panic!("Invalid iadd.x src1: {src}"), } e.set_dst(&self.dst); - e.set_reg_bnot_src(8..16, 49, self.srcs[0]); + e.set_reg_bnot_src(8..16, 49, &self.srcs[0]); e.set_bit(43, true); // .X e.set_bit(47, carry_out); @@ -1376,7 +1376,7 @@ impl SM50Op for OpIMad { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5a00); - e.set_reg_src_ref(20..28, self.srcs[1].src_ref); + e.set_reg_src_ref(20..28, &self.srcs[1].src_ref); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3400); @@ -1389,18 +1389,18 @@ impl SM50Op for OpIMad { src => panic!("Invalid imad src1: {src}"), } - e.set_reg_src_ref(39..47, self.srcs[2].src_ref); + e.set_reg_src_ref(39..47, &self.srcs[2].src_ref); } SrcRef::CBuf(cb) => { e.set_opcode(0x5200); e.set_src_cb(20..39, cb); - e.set_reg_src_ref(39..47, self.srcs[1].src_ref); + e.set_reg_src_ref(39..47, &self.srcs[1].src_ref); } src => panic!("Invalid imad src2: {src}"), } e.set_dst(&self.dst); - e.set_reg_src(8..16, self.srcs[0]); + e.set_reg_src(8..16, &self.srcs[0]); e.set_bit(48, self.signed); // src0 signed e.set_bit(51, ineg_imul); @@ -1434,7 +1434,7 @@ impl SM50Op for OpIMul { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c38); - e.set_reg_src(20..28, self.srcs[1]); + e.set_reg_src(20..28, &self.srcs[1]); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3838); @@ -1453,7 +1453,7 @@ impl SM50Op for OpIMul { } e.set_dst(&self.dst); - e.set_reg_src(8..16, self.srcs[0]); + e.set_reg_src(8..16, &self.srcs[0]); } } @@ -1470,7 +1470,7 @@ impl SM50Op for OpIMnMx { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c20); - e.set_reg_src(20..28, self.srcs[1]); + e.set_reg_src(20..28, &self.srcs[1]); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3820); @@ -1485,8 +1485,8 @@ impl SM50Op for OpIMnMx { } e.set_dst(&self.dst); - e.set_reg_src(8..16, self.srcs[0]); - e.set_pred_src(39..42, 42, self.min); + e.set_reg_src(8..16, &self.srcs[0]); + e.set_pred_src(39..42, 42, &self.min); e.set_bit(47, false); // .CC e.set_bit( 48, @@ -1513,7 +1513,7 @@ impl SM50Op for OpISetP { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5b60); - e.set_reg_src(20..28, self.srcs[1]); + e.set_reg_src(20..28, &self.srcs[1]); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3660); @@ -1529,8 +1529,8 @@ impl SM50Op for OpISetP { e.set_pred_dst(0..3, &Dst::None); // dst1 e.set_pred_dst(3..6, &self.dst); - e.set_reg_src(8..16, self.srcs[0]); - e.set_pred_src(39..42, 42, self.accum); + e.set_reg_src(8..16, &self.srcs[0]); + e.set_pred_src(39..42, 42, &self.accum); // isetp.x seems to take the accumulator into account and we don't fully // understand how. Until we do, disallow it. @@ -1570,7 +1570,7 @@ impl SM50Op for OpLop2 { e.set_opcode(0x0400); e.set_dst(&self.dst); - e.set_reg_bnot_src(8..16, 55, self.srcs[0]); + e.set_reg_bnot_src(8..16, 55, &self.srcs[0]); e.set_src_imm32(20..52, imm32); e.set_field( 53..55, @@ -1588,7 +1588,7 @@ impl SM50Op for OpLop2 { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c40); - e.set_reg_bnot_src(20..28, 40, self.srcs[1]); + e.set_reg_bnot_src(20..28, 40, &self.srcs[1]); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3840); @@ -1597,13 +1597,13 @@ impl SM50Op for OpLop2 { } SrcRef::CBuf(_) => { e.set_opcode(0x4c40); - e.set_cb_bnot_src(20..39, 40, self.srcs[1]); + e.set_cb_bnot_src(20..39, 40, &self.srcs[1]); } src => panic!("Invalid lop2 src1: {src}"), } e.set_dst(&self.dst); - e.set_reg_bnot_src(8..16, 39, self.srcs[0]); + e.set_reg_bnot_src(8..16, 39, &self.srcs[0]); e.set_field( 41..43, @@ -1630,7 +1630,7 @@ impl SM50Op for OpPopC { match &self.src.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c08); - e.set_reg_bnot_src(20..28, 40, self.src); + e.set_reg_bnot_src(20..28, 40, &self.src); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3808); @@ -1639,7 +1639,7 @@ impl SM50Op for OpPopC { } SrcRef::CBuf(_) => { e.set_opcode(0x4c08); - e.set_cb_bnot_src(20..39, 40, self.src); + e.set_cb_bnot_src(20..39, 40, &self.src); } src => panic!("Invalid popc src1: {src}"), } @@ -1661,7 +1661,7 @@ impl SM50Op for OpShf { match &self.shift.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(if self.right { 0x5cf8 } else { 0x5bf8 }); - e.set_reg_src(20..28, self.shift); + e.set_reg_src(20..28, &self.shift); } SrcRef::Imm32(imm32) => { e.set_opcode(if self.right { 0x38f8 } else { 0x36f8 }); @@ -1683,8 +1683,8 @@ impl SM50Op for OpShf { ); e.set_dst(&self.dst); - e.set_reg_src(8..16, self.low); - e.set_reg_src(39..47, self.high); + e.set_reg_src(8..16, &self.low); + e.set_reg_src(39..47, &self.high); e.set_bit(47, false); // .CC @@ -1708,11 +1708,11 @@ impl SM50Op for OpShl { fn encode(&self, e: &mut SM50Encoder<'_>) { e.set_dst(&self.dst); - e.set_reg_src(8..16, self.src); + e.set_reg_src(8..16, &self.src); match &self.shift.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c48); - e.set_reg_src(20..28, self.shift); + e.set_reg_src(20..28, &self.shift); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3848); @@ -1738,11 +1738,11 @@ impl SM50Op for OpShr { fn encode(&self, e: &mut SM50Encoder<'_>) { e.set_dst(&self.dst); - e.set_reg_src(8..16, self.src); + e.set_reg_src(8..16, &self.src); match &self.shift.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c28); - e.set_reg_src(20..28, self.shift); + e.set_reg_src(20..28, &self.shift); } SrcRef::Imm32(imm32) => { e.set_opcode(0x3828); @@ -1770,7 +1770,7 @@ impl SM50Op for OpF2F { match &self.src.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5ca8); - e.set_reg_fmod_src(20..28, 49, 45, self.src); + e.set_reg_fmod_src(20..28, 49, 45, &self.src); } SrcRef::Imm32(imm32) => { e.set_opcode(0x38a8); @@ -1779,7 +1779,7 @@ impl SM50Op for OpF2F { } SrcRef::CBuf(_) => { e.set_opcode(0x4ca8); - e.set_cb_fmod_src(20..39, 49, 45, self.src); + e.set_cb_fmod_src(20..39, 49, 45, &self.src); } src => panic!("Invalid f2f src: {src}"), } @@ -1812,7 +1812,7 @@ impl SM50Op for OpF2I { match &self.src.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5cb0); - e.set_reg_fmod_src(20..28, 49, 45, self.src); + e.set_reg_fmod_src(20..28, 49, 45, &self.src); } SrcRef::Imm32(imm32) => { e.set_opcode(0x38b0); @@ -1821,7 +1821,7 @@ impl SM50Op for OpF2I { } SrcRef::CBuf(_) => { e.set_opcode(0x4cb0); - e.set_cb_fmod_src(20..39, 49, 45, self.src); + e.set_cb_fmod_src(20..39, 49, 45, &self.src); } src => panic!("Invalid f2i src: {src}"), } @@ -1853,7 +1853,7 @@ impl SM50Op for OpI2F { match &self.src.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5cb8); - e.set_reg_ineg_src(20..28, 45, self.src); + e.set_reg_ineg_src(20..28, 45, &self.src); } SrcRef::Imm32(imm32) => { e.set_opcode(0x38b8); @@ -1862,7 +1862,7 @@ impl SM50Op for OpI2F { } SrcRef::CBuf(_) => { e.set_opcode(0x4cb8); - e.set_cb_ineg_src(20..39, 45, self.src); + e.set_cb_ineg_src(20..39, 45, &self.src); } src => panic!("Invalid i2f src: {src}"), } @@ -1894,7 +1894,7 @@ impl SM50Op for OpI2I { match &self.src.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5ce0); - e.set_reg_src(20..28, self.src); + e.set_reg_src(20..28, &self.src); } SrcRef::Imm32(imm32) => { e.set_opcode(0x38e0); @@ -1936,7 +1936,7 @@ impl SM50Op for OpMov { match &self.src.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5c98); - e.set_reg_src(20..28, self.src); + e.set_reg_src(20..28, &self.src); e.set_field(39..43, self.quad_lanes); } SrcRef::Imm32(imm32) => { @@ -1967,7 +1967,7 @@ impl SM50Op for OpPrmt { match &self.sel.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5bc0); - e.set_reg_src(20..28, self.sel); + e.set_reg_src(20..28, &self.sel); } SrcRef::Imm32(imm32) => { e.set_opcode(0x36c0); @@ -1982,8 +1982,8 @@ impl SM50Op for OpPrmt { } e.set_dst(&self.dst); - e.set_reg_src(8..16, self.srcs[0]); - e.set_reg_src(39..47, self.srcs[1]); + e.set_reg_src(8..16, &self.srcs[0]); + e.set_reg_src(39..47, &self.srcs[1]); e.set_field( 48..51, match self.mode { @@ -2014,7 +2014,7 @@ impl SM50Op for OpSel { match &self.srcs[1].src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x5ca0); - e.set_reg_src_ref(20..28, self.srcs[1].src_ref); + e.set_reg_src_ref(20..28, &self.srcs[1].src_ref); } SrcRef::Imm32(imm32) => { e.set_opcode(0x38a0); @@ -2028,8 +2028,8 @@ impl SM50Op for OpSel { } e.set_dst(&self.dst); - e.set_reg_src(8..16, self.srcs[0]); - e.set_pred_src(39..42, 42, self.cond); + e.set_reg_src(8..16, &self.srcs[0]); + e.set_pred_src(39..42, 42, &self.cond); } } @@ -2046,12 +2046,12 @@ impl SM50Op for OpShfl { e.set_dst(&self.dst); e.set_pred_dst(48..51, &self.in_bounds); - e.set_reg_src(8..16, self.src); + e.set_reg_src(8..16, &self.src); match &self.lane.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_bit(28, false); - e.set_reg_src(20..28, self.lane); + e.set_reg_src(20..28, &self.lane); } SrcRef::Imm32(imm32) => { e.set_bit(28, true); @@ -2062,7 +2062,7 @@ impl SM50Op for OpShfl { match &self.c.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_bit(29, false); - e.set_reg_src(39..47, self.c); + e.set_reg_src(39..47, &self.c); } SrcRef::Imm32(imm32) => { e.set_bit(29, true); @@ -2094,9 +2094,9 @@ impl SM50Op for OpPSetP { e.set_pred_dst(3..6, &self.dsts[0]); e.set_pred_dst(0..3, &self.dsts[1]); - e.set_pred_src(12..15, 15, self.srcs[0]); - e.set_pred_src(29..32, 32, self.srcs[1]); - e.set_pred_src(39..42, 42, self.srcs[2]); + e.set_pred_src(12..15, 15, &self.srcs[0]); + e.set_pred_src(29..32, 32, &self.srcs[1]); + e.set_pred_src(39..42, 42, &self.srcs[2]); e.set_pred_set_op(24..26, self.ops[0]); e.set_pred_set_op(45..47, self.ops[1]); @@ -2180,8 +2180,8 @@ impl SM50Op for OpTex { e.set_dst(&self.dsts[0]); assert!(self.dsts[1].is_none()); assert!(self.fault.is_none()); - e.set_reg_src(8..16, self.srcs[0]); - e.set_reg_src(20..28, self.srcs[1]); + e.set_reg_src(8..16, &self.srcs[0]); + e.set_reg_src(20..28, &self.srcs[1]); e.set_tex_dim(28..31, self.dim); e.set_tex_channel_mask(31..35, self.channel_mask); @@ -2213,8 +2213,8 @@ impl SM50Op for OpTld { e.set_dst(&self.dsts[0]); assert!(self.dsts[1].is_none()); assert!(self.fault.is_none()); - e.set_reg_src(8..16, self.srcs[0]); - e.set_reg_src(20..28, self.srcs[1]); + e.set_reg_src(8..16, &self.srcs[0]); + e.set_reg_src(20..28, &self.srcs[1]); e.set_tex_dim(28..31, self.dim); e.set_tex_channel_mask(31..35, self.channel_mask); @@ -2261,8 +2261,8 @@ impl SM50Op for OpTld4 { e.set_dst(&self.dsts[0]); assert!(self.dsts[1].is_none()); assert!(self.fault.is_none()); - e.set_reg_src(8..16, self.srcs[0]); - e.set_reg_src(20..28, self.srcs[1]); + e.set_reg_src(8..16, &self.srcs[0]); + e.set_reg_src(20..28, &self.srcs[1]); e.set_tex_dim(28..31, self.dim); e.set_tex_channel_mask(31..35, self.channel_mask); @@ -2293,8 +2293,8 @@ impl SM50Op for OpTmml { e.set_dst(&self.dsts[0]); assert!(self.dsts[1].is_none()); - e.set_reg_src(8..16, self.srcs[0]); - e.set_reg_src(20..28, self.srcs[1]); + e.set_reg_src(8..16, &self.srcs[0]); + e.set_reg_src(20..28, &self.srcs[1]); e.set_tex_dim(28..31, self.dim); e.set_tex_channel_mask(31..35, self.channel_mask); @@ -2325,8 +2325,8 @@ impl SM50Op for OpTxd { e.set_dst(&self.dsts[0]); assert!(self.dsts[1].is_none()); assert!(self.fault.is_none()); - e.set_reg_src(8..16, self.srcs[0]); - e.set_reg_src(20..28, self.srcs[1]); + e.set_reg_src(8..16, &self.srcs[0]); + e.set_reg_src(20..28, &self.srcs[1]); e.set_tex_dim(28..31, self.dim); e.set_tex_channel_mask(31..35, self.channel_mask); @@ -2356,7 +2356,7 @@ impl SM50Op for OpTxq { e.set_dst(&self.dsts[0]); assert!(self.dsts[1].is_none()); - e.set_reg_src(8..16, self.src); + e.set_reg_src(8..16, &self.src); e.set_field( 22..28, @@ -2477,8 +2477,8 @@ impl SM50Op for OpSuLd { e.set_dst(&self.dst); - e.set_reg_src(8..16, self.coord); - e.set_reg_src(39..47, self.handle); + e.set_reg_src(8..16, &self.coord); + e.set_reg_src(39..47, &self.handle); } } @@ -2501,9 +2501,9 @@ impl SM50Op for OpSuSt { } } - e.set_reg_src(8..16, self.coord); - e.set_reg_src(0..8, self.data); - e.set_reg_src(39..47, self.handle); + e.set_reg_src(8..16, &self.coord); + e.set_reg_src(0..8, &self.data); + e.set_reg_src(39..47, &self.handle); e.set_image_dim(33..36, self.image_dim); e.set_mem_order(&self.mem_order); @@ -2566,9 +2566,9 @@ impl SM50Op for OpSuAtom { e.set_dst(&self.dst); - e.set_reg_src(20..28, self.data); - e.set_reg_src(8..16, self.coord); - e.set_reg_src(39..47, self.handle); + e.set_reg_src(20..28, &self.data); + e.set_reg_src(8..16, &self.coord); + e.set_reg_src(39..47, &self.handle); } } @@ -2585,7 +2585,7 @@ impl SM50Op for OpLd { }); e.set_dst(&self.dst); - e.set_reg_src(8..16, self.addr); + e.set_reg_src(8..16, &self.addr); e.set_field(20..44, self.offset); e.set_mem_access(&self.access); @@ -2610,7 +2610,7 @@ impl SM50Op for OpLdc { e.set_opcode(0xef90); e.set_dst(&self.dst); - e.set_reg_src(8..16, self.offset); + e.set_reg_src(8..16, &self.offset); e.set_field(20..36, cb.offset); e.set_field(36..41, cb_idx); e.set_field( @@ -2638,8 +2638,8 @@ impl SM50Op for OpSt { MemSpace::Shared => 0xef58, }); - e.set_reg_src(0..8, self.data); - e.set_reg_src(8..16, self.addr); + e.set_reg_src(0..8, &self.data); + e.set_reg_src(8..16, &self.addr); e.set_field(20..44, self.offset); e.set_mem_access(&self.access); } @@ -2647,7 +2647,7 @@ impl SM50Op for OpSt { fn atom_src_as_ssa( b: &mut LegalizeBuilder, - src: Src, + src: &Src, atom_type: AtomType, ) -> SSARef { if let Some(ssa) = src.as_ssa() { @@ -2670,8 +2670,8 @@ fn atom_src_as_ssa( impl SM50Op for OpAtom { fn legalize(&mut self, b: &mut LegalizeBuilder) { if self.atom_op == AtomOp::CmpExch(AtomCmpSrc::Separate) { - let cmpr = atom_src_as_ssa(b, self.cmpr, self.atom_type); - let data = atom_src_as_ssa(b, self.data, self.atom_type); + let cmpr = atom_src_as_ssa(b, &self.cmpr, self.atom_type); + let data = atom_src_as_ssa(b, &self.data, self.atom_type); let mut cmpr_data = Vec::new(); cmpr_data.extend_from_slice(&cmpr); @@ -2691,7 +2691,7 @@ impl SM50Op for OpAtom { if self.dst.is_none() { e.set_opcode(0xebf8); - e.set_reg_src(0..8, self.data); + e.set_reg_src(0..8, &self.data); let data_type = match self.atom_type { AtomType::U32 => 0_u8, @@ -2715,13 +2715,13 @@ impl SM50Op for OpAtom { let (data_src, data_layout) = match cmp_src { AtomCmpSrc::Separate => { if self.data.is_zero() { - (self.cmpr, 1_u8) + (&self.cmpr, 1_u8) } else { assert!(self.cmpr.is_zero()); - (self.data, 2_u8) + (&self.data, 2_u8) } } - AtomCmpSrc::Packed => (self.data, 0_u8), + AtomCmpSrc::Packed => (&self.data, 0_u8), }; e.set_reg_src(20..28, data_src); @@ -2737,7 +2737,7 @@ impl SM50Op for OpAtom { e.set_opcode(0xed00); e.set_dst(&self.dst); - e.set_reg_src(20..28, self.data); + e.set_reg_src(20..28, &self.data); let data_type = match self.atom_type { AtomType::U32 => 0_u8, @@ -2754,7 +2754,7 @@ impl SM50Op for OpAtom { e.set_mem_order(&self.mem_order); - e.set_reg_src(8..16, self.addr); + e.set_reg_src(8..16, &self.addr); e.set_field(28..48, self.addr_offset); e.set_field( 48..49, @@ -2771,7 +2771,7 @@ impl SM50Op for OpAtom { assert!(cmp_src == AtomCmpSrc::Packed); assert!(self.cmpr.is_zero()); - e.set_reg_src(20..28, self.data); + e.set_reg_src(20..28, &self.data); let subop = match self.atom_type { AtomType::U32 => 4_u8, @@ -2782,7 +2782,7 @@ impl SM50Op for OpAtom { } else { e.set_opcode(0xec00); - e.set_reg_src(20..28, self.data); + e.set_reg_src(20..28, &self.data); let data_type = match self.atom_type { AtomType::U32 => 0_u8, @@ -2801,7 +2801,7 @@ impl SM50Op for OpAtom { } e.set_dst(&self.dst); - e.set_reg_src(8..16, self.addr); + e.set_reg_src(8..16, &self.addr); assert_eq!(self.addr_offset % 4, 0); e.set_field(30..52, self.addr_offset / 4); } @@ -2818,7 +2818,7 @@ impl SM50Op for OpAL2P { e.set_opcode(0xefa0); e.set_dst(&self.dst); - e.set_reg_src(8..16, self.offset); + e.set_reg_src(8..16, &self.offset); e.set_field(20..31, self.addr); e.set_bit(32, self.output); @@ -2843,8 +2843,8 @@ impl SM50Op for OpALd { } else if !self.patch { assert!(self.offset.is_zero()); } - e.set_reg_src(8..16, self.offset); - e.set_reg_src(39..47, self.vtx); + e.set_reg_src(8..16, &self.offset); + e.set_reg_src(39..47, &self.vtx); e.set_field(20..30, self.addr); e.set_bit(31, self.patch); @@ -2861,9 +2861,9 @@ impl SM50Op for OpASt { fn encode(&self, e: &mut SM50Encoder<'_>) { e.set_opcode(0xeff0); - e.set_reg_src(0..8, self.data); - e.set_reg_src(8..16, self.offset); - e.set_reg_src(39..47, self.vtx); + e.set_reg_src(0..8, &self.data); + e.set_reg_src(8..16, &self.offset); + e.set_reg_src(39..47, &self.vtx); assert!(!self.phys); e.set_field(20..30, self.addr); @@ -2882,9 +2882,9 @@ impl SM50Op for OpIpa { e.set_opcode(0xe000); e.set_dst(&self.dst); - e.set_reg_src(8..16, 0.into()); // addr - e.set_reg_src(20..28, self.inv_w); - e.set_reg_src(39..47, self.offset); + e.set_reg_src(8..16, &0.into()); // addr + e.set_reg_src(20..28, &self.inv_w); + e.set_reg_src(39..47, &self.offset); assert!(self.addr % 4 == 0); e.set_field(28..38, self.addr); @@ -2955,7 +2955,7 @@ impl SM50Op for OpCCtl { op => panic!("Unsupported cache control {op:?}"), }, ); - e.set_reg_src(8..16, self.addr); + e.set_reg_src(8..16, &self.addr); } } @@ -3095,7 +3095,7 @@ impl SM50Op for OpBar { fn encode(&self, e: &mut SM50Encoder<'_>) { e.set_opcode(0xf0a8); - e.set_reg_src(8..16, SrcRef::Zero.into()); + e.set_reg_src(8..16, &SrcRef::Zero.into()); // 00: RED.POPC // 01: RED.AND @@ -3108,7 +3108,7 @@ impl SM50Op for OpBar { // 03: SCAN e.set_field(32..35, 0_u8); - e.set_pred_src(39..42, 42, SrcRef::True.into()); + e.set_pred_src(39..42, 42, &SrcRef::True.into()); } } @@ -3132,7 +3132,7 @@ impl SM50Op for OpIsberd { fn encode(&self, e: &mut SM50Encoder<'_>) { e.set_opcode(0xefd0); e.set_dst(&self.dst); - e.set_reg_src(8..16, self.idx); + e.set_reg_src(8..16, &self.idx); } } @@ -3168,7 +3168,7 @@ impl SM50Op for OpPixLd { fn encode(&self, e: &mut SM50Encoder<'_>) { e.set_opcode(0xefe8); e.set_dst(&self.dst); - e.set_reg_src(8..16, 0.into()); + e.set_reg_src(8..16, &0.into()); e.set_field( 31..34, match &self.val { @@ -3206,7 +3206,7 @@ impl SM50Op for OpVote { e.set_dst(&self.ballot); e.set_pred_dst(45..48, &self.vote); - e.set_pred_src(39..42, 42, self.pred); + e.set_pred_src(39..42, 42, &self.pred); e.set_field( 48..50, @@ -3230,7 +3230,7 @@ impl SM50Op for OpOut { match &self.stream.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0xfbe0); - e.set_reg_src(20..28, self.stream); + e.set_reg_src(20..28, &self.stream); } SrcRef::Imm32(imm32) => { e.set_opcode(0xf6e0); @@ -3252,7 +3252,7 @@ impl SM50Op for OpOut { }, ); - e.set_reg_src(8..16, self.handle); + e.set_reg_src(8..16, &self.handle); e.set_dst(&self.dst); } } diff --git a/src/nouveau/compiler/nak/sm70_encode.rs b/src/nouveau/compiler/nak/sm70_encode.rs index e48b4149cdf..f624656fd35 100644 --- a/src/nouveau/compiler/nak/sm70_encode.rs +++ b/src/nouveau/compiler/nak/sm70_encode.rs @@ -85,7 +85,7 @@ impl SM70Encoder<'_> { self.set_field(range, reg.base_idx()); } - fn set_reg_src(&mut self, range: Range, src: Src) { + fn set_reg_src(&mut self, range: Range, src: &Src) { assert!(src.src_mod.is_none()); match src.src_ref { SrcRef::Zero => self.set_reg(range, self.zero_reg(RegFile::GPR)), @@ -106,7 +106,7 @@ impl SM70Encoder<'_> { &mut self, range: Range, not_bit: usize, - src: Src, + src: &Src, file: RegFile, ) { let (not, reg) = match src.src_ref { @@ -122,11 +122,16 @@ impl SM70Encoder<'_> { self.set_bit(not_bit, not ^ src_mod_is_bnot(src.src_mod)); } - fn set_pred_src(&mut self, range: Range, not_bit: usize, src: Src) { + fn set_pred_src(&mut self, range: Range, not_bit: usize, src: &Src) { self.set_pred_src_file(range, not_bit, src, RegFile::Pred); } - fn set_upred_src(&mut self, range: Range, not_bit: usize, src: Src) { + fn set_upred_src( + &mut self, + range: Range, + not_bit: usize, + src: &Src, + ) { self.set_pred_src_file(range, not_bit, src, RegFile::UPred); } @@ -188,7 +193,7 @@ impl SM70Encoder<'_> { self.set_bar_reg(range, *dst.as_reg().unwrap()); } - fn set_bar_src(&mut self, range: Range, src: Src) { + fn set_bar_src(&mut self, range: Range, src: &Src) { assert!(src.src_mod.is_none()); self.set_bar_reg(range, *src.src_ref.as_reg().unwrap()); } @@ -724,7 +729,7 @@ impl SM70Op for OpFMnMx { Some(&self.srcs[1]), Some(&Src::ZERO), ); - e.set_pred_src(87..90, 90, self.min); + e.set_pred_src(87..90, 90, &self.min); e.set_bit(80, self.ftz); } } @@ -859,7 +864,7 @@ impl SM70Op for OpFSetP { e.set_pred_dst(81..84, &self.dst); e.set_pred_dst(84..87, &Dst::None); // dst1 - e.set_pred_src(87..90, 90, self.accum); + e.set_pred_src(87..90, 90, &self.accum); } } @@ -875,8 +880,8 @@ impl SM70Op for OpFSwzAdd { e.set_opcode(0x822); e.set_dst(&self.dst); - e.set_reg_src(24..32, self.srcs[0]); - e.set_reg_src(64..72, self.srcs[1]); + e.set_reg_src(24..32, &self.srcs[0]); + e.set_reg_src(64..72, &self.srcs[1]); let mut subop = 0x0_u8; @@ -1021,7 +1026,7 @@ impl SM70Op for OpDSetP { e.set_pred_dst(81..84, &self.dst); e.set_pred_dst(84..87, &Dst::None); /* dst1 */ - e.set_pred_src(87..90, 90, self.accum); + e.set_pred_src(87..90, 90, &self.accum); } } @@ -1151,7 +1156,7 @@ impl SM70Op for OpHSet2 { e.set_float_cmp_op(76..80, self.cmp_op); e.set_bit(80, self.ftz); - e.set_pred_src(87..90, 90, self.accum); + e.set_pred_src(87..90, 90, &self.accum); } } @@ -1194,7 +1199,7 @@ impl SM70Op for OpHSetP2 { e.set_pred_dst(81..84, &self.dsts[0]); e.set_pred_dst(84..87, &self.dsts[1]); - e.set_pred_src(87..90, 90, self.accum); + e.set_pred_src(87..90, 90, &self.accum); } } @@ -1224,7 +1229,7 @@ impl SM70Op for OpHMnMx2 { e.set_bit(82, false); // .XORSIGN e.set_bit(85, false); // .BF16_V2 - e.set_pred_src(87..90, 90, self.min); + e.set_pred_src(87..90, 90, &self.min); } } @@ -1343,8 +1348,8 @@ impl SM70Op for OpIAdd3 { ) }; - e.set_pred_src(87..90, 90, false.into()); - e.set_pred_src(77..80, 80, false.into()); + e.set_pred_src(87..90, 90, &false.into()); + e.set_pred_src(77..80, 80, &false.into()); e.set_pred_dst(81..84, &self.overflow[0]); e.set_pred_dst(84..87, &self.overflow[1]); @@ -1359,13 +1364,13 @@ impl SM70Op for OpIAdd3X { swap_srcs_if_not_reg(src2, src1, gpr); if !src0.src_mod.is_none() && !src1.src_mod.is_none() { let val = b.alloc_ssa(gpr); + let old_src0 = std::mem::replace(src0, val.into()); b.push_op(OpIAdd3X { - srcs: [Src::ZERO, *src0, Src::ZERO], + srcs: [Src::ZERO, old_src0, Src::ZERO], overflow: [Dst::None, Dst::None], dst: val.into(), - carry: [false.into(); 2], + carry: [false.into(), false.into()], }); - *src0 = val.into(); } b.copy_alu_src_if_not_reg(src0, gpr, SrcType::B32); b.copy_alu_src_if_both_not_reg(src1, src2, gpr, SrcType::B32); @@ -1390,8 +1395,8 @@ impl SM70Op for OpIAdd3X { Some(&self.srcs[2]), ); - e.set_upred_src(87..90, 90, self.carry[0]); - e.set_upred_src(77..80, 80, self.carry[1]); + e.set_upred_src(87..90, 90, &self.carry[0]); + e.set_upred_src(77..80, 80, &self.carry[1]); } else { e.encode_alu( 0x010, @@ -1401,8 +1406,8 @@ impl SM70Op for OpIAdd3X { Some(&self.srcs[2]), ); - e.set_pred_src(87..90, 90, self.carry[0]); - e.set_pred_src(77..80, 80, self.carry[1]); + e.set_pred_src(87..90, 90, &self.carry[0]); + e.set_pred_src(77..80, 80, &self.carry[1]); } e.set_bit(74, true); // .X @@ -1533,7 +1538,7 @@ impl SM70Op for OpIMnMx { Some(&self.srcs[1]), None, ); - e.set_pred_src(87..90, 90, self.min); + e.set_pred_src(87..90, 90, &self.min); e.set_bit( 73, match self.cmp_type { @@ -1569,8 +1574,8 @@ impl SM70Op for OpISetP { None, ); - e.set_upred_src(68..71, 71, self.low_cmp); - e.set_upred_src(87..90, 90, self.accum); + e.set_upred_src(68..71, 71, &self.low_cmp); + e.set_upred_src(87..90, 90, &self.accum); } else { e.encode_alu( 0x00c, @@ -1580,8 +1585,8 @@ impl SM70Op for OpISetP { None, ); - e.set_pred_src(68..71, 71, self.low_cmp); - e.set_pred_src(87..90, 90, self.accum); + e.set_pred_src(68..71, 71, &self.low_cmp); + e.set_pred_src(87..90, 90, &self.accum); } e.set_bit(72, self.ex); @@ -1689,7 +1694,7 @@ impl SM70Op for OpLeaX { Some(&self.b), c, ); - e.set_upred_src(87..90, 90, self.carry); + e.set_upred_src(87..90, 90, &self.carry); } else { e.encode_alu( 0x011, @@ -1698,7 +1703,7 @@ impl SM70Op for OpLeaX { Some(&self.b), c, ); - e.set_pred_src(87..90, 90, self.carry); + e.set_pred_src(87..90, 90, &self.carry); } e.set_bit(72, self.intermediate_mod.is_bnot()); @@ -1778,7 +1783,7 @@ impl SM70Op for OpLop3 { Some(&self.srcs[2]), ); - e.set_upred_src(87..90, 90, SrcRef::False.into()); + e.set_upred_src(87..90, 90, &SrcRef::False.into()); } else { e.encode_alu( 0x012, @@ -1788,7 +1793,7 @@ impl SM70Op for OpLop3 { Some(&self.srcs[2]), ); - e.set_pred_src(87..90, 90, SrcRef::False.into()); + e.set_pred_src(87..90, 90, &SrcRef::False.into()); } e.set_field(72..80, self.op.lut); @@ -2069,7 +2074,7 @@ impl SM70Op for OpSel { None, ); - e.set_upred_src(87..90, 90, self.cond); + e.set_upred_src(87..90, 90, &self.cond); } else { e.encode_alu( 0x007, @@ -2079,7 +2084,7 @@ impl SM70Op for OpSel { None, ); - e.set_pred_src(87..90, 90, self.cond); + e.set_pred_src(87..90, 90, &self.cond); } } } @@ -2100,12 +2105,12 @@ impl SM70Op for OpShfl { SrcRef::Zero | SrcRef::Reg(_) => match &self.c.src_ref { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x389); - e.set_reg_src(32..40, self.lane); - e.set_reg_src(64..72, self.c); + e.set_reg_src(32..40, &self.lane); + e.set_reg_src(64..72, &self.c); } SrcRef::Imm32(imm_c) => { e.set_opcode(0x589); - e.set_reg_src(32..40, self.lane); + e.set_reg_src(32..40, &self.lane); e.set_field(40..53, *imm_c & 0x1f1f); } _ => panic!("Invalid instruction form"), @@ -2114,7 +2119,7 @@ impl SM70Op for OpShfl { SrcRef::Zero | SrcRef::Reg(_) => { e.set_opcode(0x989); e.set_field(53..58, *imm_lane & 0x1f); - e.set_reg_src(64..72, self.c); + e.set_reg_src(64..72, &self.c); } SrcRef::Imm32(imm_c) => { e.set_opcode(0xf89); @@ -2128,7 +2133,7 @@ impl SM70Op for OpShfl { e.set_dst(&self.dst); e.set_pred_dst(81..84, &self.in_bounds); - e.set_reg_src(24..32, self.src); + e.set_reg_src(24..32, &self.src); e.set_field( 58..60, match self.op { @@ -2185,9 +2190,9 @@ impl SM70Op for OpPLop3 { if self.is_uniform() { e.set_opcode(0x89c); - e.set_upred_src(68..71, 71, self.srcs[2]); - e.set_upred_src(77..80, 80, self.srcs[1]); - e.set_upred_src(87..90, 90, self.srcs[0]); + e.set_upred_src(68..71, 71, &self.srcs[2]); + e.set_upred_src(77..80, 80, &self.srcs[1]); + e.set_upred_src(87..90, 90, &self.srcs[0]); } else { e.set_opcode(0x81c); @@ -2196,13 +2201,13 @@ impl SM70Op for OpPLop3 { .as_reg() .is_some_and(|r| r.is_uniform()) { - e.set_upred_src(68..71, 71, self.srcs[2]); + e.set_upred_src(68..71, 71, &self.srcs[2]); e.set_bit(67, true); } else { - e.set_pred_src(68..71, 71, self.srcs[2]); + e.set_pred_src(68..71, 71, &self.srcs[2]); } - e.set_pred_src(77..80, 80, self.srcs[1]); - e.set_pred_src(87..90, 90, self.srcs[0]); + e.set_pred_src(77..80, 80, &self.srcs[1]); + e.set_pred_src(87..90, 90, &self.srcs[0]); } e.set_field(16..24, self.ops[1].lut); e.set_field(64..67, self.ops[0].lut & 0x7); @@ -2221,7 +2226,7 @@ impl SM70Op for OpR2UR { fn encode(&self, e: &mut SM70Encoder<'_>) { e.set_opcode(0x3c2); e.set_udst(&self.dst); - e.set_reg_src(24..32, self.src); + e.set_reg_src(24..32, &self.src); e.set_pred_dst(81..84, &Dst::None); } } @@ -2373,8 +2378,8 @@ impl SM70Op for OpTex { } e.set_pred_dst(81..84, &self.fault); - e.set_reg_src(24..32, self.srcs[0]); - e.set_reg_src(32..40, self.srcs[1]); + e.set_reg_src(24..32, &self.srcs[0]); + e.set_reg_src(32..40, &self.srcs[1]); if e.sm >= 100 { e.set_field(48..56, 0xff_u8); // ureg @@ -2426,8 +2431,8 @@ impl SM70Op for OpTld { } e.set_pred_dst(81..84, &self.fault); - e.set_reg_src(24..32, self.srcs[0]); - e.set_reg_src(32..40, self.srcs[1]); + e.set_reg_src(24..32, &self.srcs[0]); + e.set_reg_src(32..40, &self.srcs[1]); if e.sm >= 100 { e.set_field(48..56, 0xff_u8); // ureg @@ -2480,8 +2485,8 @@ impl SM70Op for OpTld4 { } e.set_pred_dst(81..84, &self.fault); - e.set_reg_src(24..32, self.srcs[0]); - e.set_reg_src(32..40, self.srcs[1]); + e.set_reg_src(24..32, &self.srcs[0]); + e.set_reg_src(32..40, &self.srcs[1]); if e.sm >= 100 { e.set_field(48..56, 0xff_u8); // ureg @@ -2533,8 +2538,8 @@ impl SM70Op for OpTmml { e.set_field(64..72, 255_u8); } - e.set_reg_src(24..32, self.srcs[0]); - e.set_reg_src(32..40, self.srcs[1]); + e.set_reg_src(24..32, &self.srcs[0]); + e.set_reg_src(32..40, &self.srcs[1]); e.set_tex_dim(61..64, self.dim); e.set_tex_channel_mask(72..76, self.channel_mask); @@ -2577,8 +2582,8 @@ impl SM70Op for OpTxd { } e.set_pred_dst(81..84, &self.fault); - e.set_reg_src(24..32, self.srcs[0]); - e.set_reg_src(32..40, self.srcs[1]); + e.set_reg_src(24..32, &self.srcs[0]); + e.set_reg_src(32..40, &self.srcs[1]); if e.sm >= 100 { e.set_field(48..56, 0xff_u8); // ureg @@ -2621,7 +2626,7 @@ impl SM70Op for OpTxq { e.set_field(64..72, 255_u8); } - e.set_reg_src(24..32, self.src); + e.set_reg_src(24..32, &self.src); e.set_field( 62..64, match self.query { @@ -2737,8 +2742,8 @@ impl SM70Op for OpSuLd { } e.set_dst(&self.dst); - e.set_reg_src(24..32, self.coord); - e.set_reg_src(64..72, self.handle); + e.set_reg_src(24..32, &self.coord); + e.set_reg_src(64..72, &self.handle); e.set_pred_dst(81..84, &self.fault); e.set_image_dim(61..64, self.image_dim); @@ -2764,9 +2769,9 @@ impl SM70Op for OpSuSt { } } - e.set_reg_src(24..32, self.coord); - e.set_reg_src(32..40, self.data); - e.set_reg_src(64..72, self.handle); + e.set_reg_src(24..32, &self.coord); + e.set_reg_src(32..40, &self.data); + e.set_reg_src(64..72, &self.handle); e.set_image_dim(61..64, self.image_dim); e.set_mem_order(&self.mem_order); @@ -2792,9 +2797,9 @@ impl SM70Op for OpSuAtom { }; e.set_dst(&self.dst); - e.set_reg_src(24..32, self.coord); - e.set_reg_src(32..40, self.data); - e.set_reg_src(64..72, self.handle); + e.set_reg_src(24..32, &self.coord); + e.set_reg_src(32..40, &self.data); + e.set_reg_src(64..72, &self.handle); e.set_pred_dst(81..84, &self.fault); e.set_image_dim(61..64, self.image_dim); @@ -2844,7 +2849,7 @@ impl SM70Op for OpLd { } e.set_dst(&self.dst); - e.set_reg_src(24..32, self.addr); + e.set_reg_src(24..32, &self.addr); e.set_field(40..64, self.offset); } } @@ -2872,7 +2877,7 @@ impl SM70Op for OpLdc { e.set_opcode(0xb82); e.set_dst(&self.dst); - e.set_reg_src(24..32, self.offset); + e.set_reg_src(24..32, &self.offset); e.set_field( 78..80, match self.mode { @@ -2896,11 +2901,11 @@ impl SM70Op for OpLdc { e.set_opcode(0x582); e.set_dst(&self.dst); - e.set_reg_src(64..72, self.offset); + e.set_reg_src(64..72, &self.offset); } e.set_ureg(24..32, handle); - e.set_reg_src(64..72, self.offset); + e.set_reg_src(64..72, &self.offset); assert!(self.mode == LdcMode::Indexed); e.set_bit(91, true); // Bindless } @@ -2946,8 +2951,8 @@ impl SM70Op for OpSt { } } - e.set_reg_src(24..32, self.addr); - e.set_reg_src(32..40, self.data); + e.set_reg_src(24..32, &self.addr); + e.set_reg_src(32..40, &self.data); e.set_field(40..64, self.offset); } } @@ -3030,14 +3035,14 @@ impl SM70Op for OpAtom { e.set_opcode(0x98e); } - e.set_reg_src(32..40, self.data); + e.set_reg_src(32..40, &self.data); e.set_atom_op(87..90, self.atom_op); } else if let AtomOp::CmpExch(cmp_src) = self.atom_op { e.set_opcode(0x3a9); assert!(cmp_src == AtomCmpSrc::Separate); - e.set_reg_src(32..40, self.cmpr); - e.set_reg_src(64..72, self.data); + e.set_reg_src(32..40, &self.cmpr); + e.set_reg_src(64..72, &self.data); } else { if e.sm >= 90 && self.atom_type.is_float() { e.set_opcode(0x3a3); @@ -3045,7 +3050,7 @@ impl SM70Op for OpAtom { e.set_opcode(0x3a8); } - e.set_reg_src(32..40, self.data); + e.set_reg_src(32..40, &self.data); e.set_atom_op(87..91, self.atom_op); } @@ -3068,12 +3073,12 @@ impl SM70Op for OpAtom { e.set_opcode(0x38d); assert!(cmp_src == AtomCmpSrc::Separate); - e.set_reg_src(32..40, self.cmpr); - e.set_reg_src(64..72, self.data); + e.set_reg_src(32..40, &self.cmpr); + e.set_reg_src(64..72, &self.data); } else { e.set_opcode(0x38c); - e.set_reg_src(32..40, self.data); + e.set_reg_src(32..40, &self.data); assert!( self.atom_type != AtomType::U64 || self.atom_op == AtomOp::Exch, @@ -3094,7 +3099,7 @@ impl SM70Op for OpAtom { } e.set_dst(&self.dst); - e.set_reg_src(24..32, self.addr); + e.set_reg_src(24..32, &self.addr); e.set_field(40..64, self.addr_offset); e.set_atom_type(self.atom_type); } @@ -3109,7 +3114,7 @@ impl SM70Op for OpAL2P { e.set_opcode(0x920); e.set_dst(&self.dst); - e.set_reg_src(24..32, self.offset); + e.set_reg_src(24..32, &self.offset); e.set_field(40..50, self.addr); e.set_field(74..76, 0_u8); // comps @@ -3126,8 +3131,8 @@ impl SM70Op for OpALd { e.set_opcode(0x321); e.set_dst(&self.dst); - e.set_reg_src(32..40, self.vtx); - e.set_reg_src(24..32, self.offset); + e.set_reg_src(32..40, &self.vtx); + e.set_reg_src(24..32, &self.offset); e.set_field(40..50, self.addr); e.set_field(74..76, self.comps - 1); @@ -3145,9 +3150,9 @@ impl SM70Op for OpASt { fn encode(&self, e: &mut SM70Encoder<'_>) { e.set_opcode(0x322); - e.set_reg_src(32..40, self.data); - e.set_reg_src(64..72, self.vtx); - e.set_reg_src(24..32, self.offset); + e.set_reg_src(32..40, &self.data); + e.set_reg_src(64..72, &self.vtx); + e.set_reg_src(24..32, &self.offset); e.set_field(40..50, self.addr); e.set_field(74..76, self.comps - 1); @@ -3190,7 +3195,7 @@ impl SM70Op for OpIpa { ); assert!(self.inv_w.is_zero()); - e.set_reg_src(32..40, self.offset); + e.set_reg_src(32..40, &self.offset); // TODO: What is this for? e.set_pred_dst(81..84, &Dst::None); @@ -3226,7 +3231,7 @@ impl SM70Op for OpCCtl { assert!(matches!(self.mem_space, MemSpace::Global(_))); e.set_opcode(0x98f); - e.set_reg_src(24..32, self.addr); + e.set_reg_src(24..32, &self.addr); e.set_field(32..64, self.addr_offset); e.set_field( @@ -3309,14 +3314,14 @@ impl SM70Op for OpBMov { e.set_opcode(0x356); e.set_bar_dst(24..28, &self.dst); - e.set_reg_src(32..40, self.src); + e.set_reg_src(32..40, &self.src); e.set_bit(84, self.clear); } else { e.set_opcode(0x355); e.set_dst(&self.dst); - e.set_bar_src(24..28, self.src); + e.set_bar_src(24..28, &self.src); e.set_bit(84, self.clear); } @@ -3332,7 +3337,7 @@ impl SM70Op for OpBreak { e.set_opcode(0x942); assert!(self.bar_in.src_ref.as_reg() == self.bar_out.as_reg()); e.set_bar_dst(16..20, &self.bar_out); - e.set_pred_src(87..90, 90, self.cond); + e.set_pred_src(87..90, 90, &self.cond); } } @@ -3346,7 +3351,7 @@ impl SM70Op for OpBSSy { assert!(self.bar_in.src_ref.as_reg() == self.bar_out.as_reg()); e.set_bar_dst(16..20, &self.bar_out); e.set_rel_offset(34..64, &self.target); - e.set_pred_src(87..90, 90, self.cond); + e.set_pred_src(87..90, 90, &self.cond); } } @@ -3357,8 +3362,8 @@ impl SM70Op for OpBSync { fn encode(&self, e: &mut SM70Encoder<'_>) { e.set_opcode(0x941); - e.set_bar_src(16..20, self.bar); - e.set_pred_src(87..90, 90, self.cond); + e.set_bar_src(16..20, &self.bar); + e.set_pred_src(87..90, 90, &self.cond); } } @@ -3397,7 +3402,7 @@ impl SM70Op for OpWarpSync { fn encode(&self, e: &mut SM70Encoder<'_>) { e.encode_alu(0x148, None, None, Some(&Src::from(self.mask)), None); - e.set_pred_src(87..90, 90, SrcRef::True.into()); + e.set_pred_src(87..90, 90, &SrcRef::True.into()); } } @@ -3450,7 +3455,7 @@ impl SM70Op for OpIsberd { fn encode(&self, e: &mut SM70Encoder<'_>) { e.set_opcode(0x923); e.set_dst(&self.dst); - e.set_reg_src(24..32, self.idx); + e.set_reg_src(24..32, &self.idx); } } @@ -3461,7 +3466,7 @@ impl SM70Op for OpKill { fn encode(&self, e: &mut SM70Encoder<'_>) { e.set_opcode(0x95b); - e.set_pred_src(87..90, 90, SrcRef::True.into()); + e.set_pred_src(87..90, 90, &SrcRef::True.into()); } } @@ -3579,7 +3584,7 @@ impl SM70Op for OpVote { ); e.set_pred_dst(81..84, &self.vote); - e.set_pred_src(87..90, 90, self.pred); + e.set_pred_src(87..90, 90, &self.pred); } } diff --git a/src/nouveau/compiler/nak/to_cssa.rs b/src/nouveau/compiler/nak/to_cssa.rs index 7a63d82eb67..262f98c664b 100644 --- a/src/nouveau/compiler/nak/to_cssa.rs +++ b/src/nouveau/compiler/nak/to_cssa.rs @@ -359,8 +359,9 @@ impl Function { let ss = cg.ssa_set(&vec[0]); if cg.sets_interfere(ps, ss, &self.blocks) { let tmp = self.ssa_alloc.alloc(file); - pcopy.push(tmp.into(), *src); - *src = tmp.into(); + let old_src = + std::mem::replace(src, tmp.into()); + pcopy.push(tmp.into(), old_src); } else { cg.sets_merge(ps, ss); } @@ -378,11 +379,11 @@ impl Function { annotation: "generated by to_cssa".into(), })); } + let old_src = std::mem::replace(src, tmp.into()); instrs.push(Instr::new_boxed(OpCopy { dst: tmp.into(), - src: *src, + src: old_src, })); - *src = tmp.into(); } if !pcopy.is_empty() {