AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nak: Src is no longer Copy

Browse Source 
Reviewed-by: Faith Ekstrand <faith.ekstrand@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/34794>
This commit is contained in:
Mel Henning
2025-04-30 19:17:28 -04:00
committed by Marge Bot
parent 30f6ca6391
commit a9d2789237
11 changed files with 443 additions and 388 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/nouveau/compiler/nak/assign_regs.rs
+1 -1
View File
@@ -1240,7 +1240,7 @@ impl AssignRegsBlock {
for (i, src) in out.srcs.iter().enumerate() {
let reg = u32::try_from(i).unwrap();
let dst = RegRef::new(RegFile::GPR, reg, 1);
pcopy.push(dst.into(), *src);
pcopy.push(dst.into(), src.clone());
}
None
src/nouveau/compiler/nak/builder.rs
+18 -17
View File
@@ -154,7 +154,7 @@ pub trait SSABuilder: Builder {
dst: dst[0].into(),
low: x[0].into(),
high: 0.into(),
shift,
shift: shift.clone(),
right: false,
wrap: true,
data_type: IntType::U64,
@@ -168,7 +168,7 @@ pub trait SSABuilder: Builder {
dst: dst[0].into(),
low: 0.into(),
high: x[0].into(),
shift,
shift: shift.clone(),
right: false,
wrap: true,
data_type: IntType::U64,
@@ -222,7 +222,7 @@ pub trait SSABuilder: Builder {
dst: dst[0].into(),
low: x[0].into(),
high: x[1].into(),
shift,
shift: shift.clone(),
right: true,
wrap: true,
data_type: if signed { IntType::I64 } else { IntType::U64 },
@@ -384,8 +384,8 @@ pub trait SSABuilder: Builder {
let is_3src = !x.is_zero() && !y.is_zero() && !z.is_zero();
let x = split_iadd64_src(x);
let y = split_iadd64_src(y);
let [x0, x1] = split_iadd64_src(x);
let [y0, y1] = split_iadd64_src(y);
let dst = self.alloc_ssa_vec(RegFile::GPR, 2);
if self.sm() >= 70 {
let carry1 = self.alloc_ssa(RegFile::Pred);
@@ -398,16 +398,16 @@ pub trait SSABuilder: Builder {
(Dst::None, false.into())
};
let z = split_iadd64_src(z);
let [z0, z1] = split_iadd64_src(z);
self.push_op(OpIAdd3 {
dst: dst[0].into(),
overflow: [carry1.into(), carry2_dst],
srcs: [x[0], y[0], z[0]],
srcs: [x0, y0, z0],
});
self.push_op(OpIAdd3X {
dst: dst[1].into(),
overflow: [Dst::None, Dst::None],
srcs: [x[1], y[1], z[1]],
srcs: [x1, y1, z1],
carry: [carry1.into(), carry2_src],
});
} else {
@@ -415,12 +415,12 @@ pub trait SSABuilder: Builder {
let carry = self.alloc_ssa(RegFile::Carry);
self.push_op(OpIAdd2 {
dst: dst[0].into(),
srcs: [x[0], y[0]],
srcs: [x0, y0],
carry_out: carry.into(),
});
self.push_op(OpIAdd2X {
dst: dst[1].into(),
srcs: [x[1], y[1]],
srcs: [x1, y1],
carry_out: Dst::None,
carry_in: carry.into(),
});
@@ -469,7 +469,7 @@ pub trait SSABuilder: Builder {
} else {
self.push_op(OpIMul {
dst: dst[0].into(),
srcs: [x, y],
srcs: [x.clone(), y.clone()],
signed: [signed; 2],
high: false,
});
@@ -772,8 +772,9 @@ pub trait SSABuilder: Builder {
fn prmt4(&mut self, src: [Src; 4], sel: [u8; 4]) -> SSAValue {
let max_sel = *sel.iter().max().unwrap();
let [src0, src1, src2, src3] = src;
if max_sel < 8 {
self.prmt(src[0], src[1], sel)
self.prmt(src0, src1, sel)
} else if max_sel < 12 {
let mut sel_a = [0_u8; 4];
let mut sel_b = [0_u8; 4];
@@ -785,8 +786,8 @@ pub trait SSABuilder: Builder {
sel_b[usize::from(i)] = (sel[usize::from(i)] - 8) + 4;
}
}
let a = self.prmt(src[0], src[1], sel_a);
self.prmt(a.into(), src[2], sel_b)
let a = self.prmt(src0, src1, sel_a);
self.prmt(a.into(), src2, sel_b)
} else if max_sel < 16 {
let mut sel_a = [0_u8; 4];
let mut sel_b = [0_u8; 4];
@@ -800,8 +801,8 @@ pub trait SSABuilder: Builder {
sel_c[usize::from(i)] = 4 + i;
}
}
let a = self.prmt(src[0], src[1], sel_a);
let b = self.prmt(src[2], src[3], sel_b);
let a = self.prmt(src0, src1, sel_a);
let b = self.prmt(src2, src3, sel_b);
self.prmt(a.into(), b.into(), sel_c)
} else {
panic!("Invalid permute value: {max_sel}");
@@ -827,7 +828,7 @@ pub trait SSABuilder: Builder {
self.push_op(OpPSetP {
dsts: [tmp.into(), Dst::None],
ops: [PredSetOp::And, PredSetOp::And],
srcs: [cond, x, true.into()],
srcs: [cond.clone(), x, true.into()],
});
self.push_op(OpPSetP {
dsts: [dst.into(), Dst::None],
src/nouveau/compiler/nak/from_nir.rs
+6 -5
View File
@@ -568,7 +568,7 @@ impl<'a> ShaderFromNir<'a> {
}
8 => {
for dc in 0..bits.div_ceil(32) {
let mut psrc = [None; 4];
let mut psrc = [None, None, None, None];
let mut psel = [0_u8; 4];
for b in 0..4 {
@@ -594,9 +594,9 @@ impl<'a> ShaderFromNir<'a> {
let psrc = {
let mut res = [Src::ZERO; 4];
for (idx, src) in psrc.iter().enumerate() {
for (idx, src) in psrc.into_iter().enumerate() {
if let Some(src) = src {
res[idx] = *src;
res[idx] = src;
}
}
@@ -620,7 +620,8 @@ impl<'a> ShaderFromNir<'a> {
psel[w * 2 + 1] = (w as u8 * 4) + byte + 1;
}
}
comps.push(b.prmt(psrc[0], psrc[1], psel));
let [psrc0, psrc1] = psrc;
comps.push(b.prmt(psrc0, psrc1, psel));
}
}
_ => panic!("Unknown bit size: {src_bit_size}"),
@@ -1290,7 +1291,7 @@ impl<'a> ShaderFromNir<'a> {
let x = restrict_f16v2_src(src0);
let lz = restrict_f16v2_src(
b.hset2(FloatCmpOp::OrdLt, x, 0.into()).into(),
b.hset2(FloatCmpOp::OrdLt, x.clone(), 0.into()).into(),
);
let gz = restrict_f16v2_src(
b.hset2(FloatCmpOp::OrdGt, x, 0.into()).into(),
src/nouveau/compiler/nak/ir.rs
+2 -2
View File
@@ -1103,7 +1103,7 @@ impl fmt::Display for SrcSwizzle {
}
}
#[derive(Clone, Copy, PartialEq)]
#[derive(Clone, PartialEq)]
pub struct Src {
pub src_ref: SrcRef,
pub src_mod: SrcMod,
@@ -1236,7 +1236,7 @@ impl Src {
}
}
pub fn to_ssa(&self) -> SSARef {
pub fn to_ssa(self) -> SSARef {
if self.src_mod.is_none() {
self.src_ref.to_ssa()
} else {
src/nouveau/compiler/nak/legalize.rs
+10 -10
View File
@@ -237,34 +237,34 @@ pub trait LegalizeBuildHelpers: SSABuilder {
match src_type {
SrcType::F16 | SrcType::F16v2 => {
let val = self.alloc_ssa(reg_file);
let old_src = std::mem::replace(src, val.into());
self.push_op(OpHAdd2 {
dst: val.into(),
srcs: [Src::ZERO.fneg(), *src],
srcs: [Src::ZERO.fneg(), old_src],
saturate: false,
ftz: false,
f32: false,
});
*src = val.into();
}
SrcType::F32 => {
let val = self.alloc_ssa(reg_file);
let old_src = std::mem::replace(src, val.into());
self.push_op(OpFAdd {
dst: val.into(),
srcs: [Src::ZERO.fneg(), *src],
srcs: [Src::ZERO.fneg(), old_src],
saturate: false,
rnd_mode: FRndMode::NearestEven,
ftz: false,
});
*src = val.into();
}
SrcType::F64 => {
let val = self.alloc_ssa_vec(reg_file, 2);
let old_src = std::mem::replace(src, val.into());
self.push_op(OpDAdd {
dst: val.into(),
srcs: [Src::ZERO.fneg(), *src],
srcs: [Src::ZERO.fneg(), old_src],
rnd_mode: FRndMode::NearestEven,
});
*src = val.into();
}
_ => panic!("Invalid ffabs srouce type"),
}
@@ -278,9 +278,10 @@ pub trait LegalizeBuildHelpers: SSABuilder {
) {
assert!(src_type == SrcType::I32);
let val = self.alloc_ssa(reg_file);
let old_src = std::mem::replace(src, val.into());
if self.sm() >= 70 {
self.push_op(OpIAdd3 {
srcs: [Src::ZERO, *src, Src::ZERO],
srcs: [Src::ZERO, old_src, Src::ZERO],
overflow: [Dst::None, Dst::None],
dst: val.into(),
});
@@ -288,10 +289,9 @@ pub trait LegalizeBuildHelpers: SSABuilder {
self.push_op(OpIAdd2 {
dst: val.into(),
carry_out: Dst::None,
srcs: [Src::ZERO, *src],
srcs: [Src::ZERO, old_src],
});
}
*src = val.into();
}
fn copy_alu_src_if_fabs(
@@ -429,7 +429,7 @@ fn legalize_instr(
}) => {
let bar_in_ssa = bar_in.src_ref.as_ssa().unwrap();
if !bar_out.is_none() && bl.is_live_after_ip(&bar_in_ssa[0], ip) {
let gpr = b.bmov_to_gpr(*bar_in);
let gpr = b.bmov_to_gpr(bar_in.clone());
let tmp = b.bmov_to_bar(gpr.into());
*bar_in = tmp.into();
}
src/nouveau/compiler/nak/opt_copy_prop.rs
+73 -26
View File
@@ -319,7 +319,7 @@ impl CopyPropPass {
}
let entry_src_idx = usize::from(entry_src_idx.unwrap());
let entry_src = entry.srcs[entry_src_idx];
let entry_src = &entry.srcs[entry_src_idx];
if !cbuf_rule.allows_src(entry.bi, &entry_src) {
return;
@@ -514,9 +514,19 @@ impl CopyPropPass {
if !add.saturate {
if add.srcs[0].is_fneg_zero(SrcType::F16v2) {
self.add_copy(bi, dst, SrcType::F16v2, add.srcs[1]);
self.add_copy(
bi,
dst,
SrcType::F16v2,
add.srcs[1].clone(),
);
} else if add.srcs[1].is_fneg_zero(SrcType::F16v2) {
self.add_copy(bi, dst, SrcType::F16v2, add.srcs[0]);
self.add_copy(
bi,
dst,
SrcType::F16v2,
add.srcs[0].clone(),
);
}
}
}
@@ -527,18 +537,28 @@ impl CopyPropPass {
if !add.saturate {
if add.srcs[0].is_fneg_zero(SrcType::F32) {
self.add_copy(bi, dst, SrcType::F32, add.srcs[1]);
self.add_copy(
bi,
dst,
SrcType::F32,
add.srcs[1].clone(),
);
} else if add.srcs[1].is_fneg_zero(SrcType::F32) {
self.add_copy(bi, dst, SrcType::F32, add.srcs[0]);
self.add_copy(
bi,
dst,
SrcType::F32,
add.srcs[0].clone(),
);
}
}
}
Op::DAdd(add) => {
let dst = add.dst.as_ssa().unwrap();
if add.srcs[0].is_fneg_zero(SrcType::F64) {
self.add_fp64_copy(bi, dst, add.srcs[1]);
self.add_fp64_copy(bi, dst, add.srcs[1].clone());
} else if add.srcs[1].is_fneg_zero(SrcType::F64) {
self.add_fp64_copy(bi, dst, add.srcs[0]);
self.add_fp64_copy(bi, dst, add.srcs[0].clone());
}
}
Op::Lop3(lop) => {
@@ -559,7 +579,12 @@ impl CopyPropPass {
} else {
for s in 0..3 {
if op.lut == LogicOp3::SRC_MASKS[s] {
self.add_copy(bi, dst, SrcType::ALU, lop.srcs[s]);
self.add_copy(
bi,
dst,
SrcType::ALU,
lop.srcs[s].clone(),
);
}
}
}
@@ -596,14 +621,14 @@ impl CopyPropPass {
bi,
dst,
SrcType::Pred,
lop.srcs[s],
lop.srcs[s].clone(),
);
} else if op.lut == !LogicOp3::SRC_MASKS[s] {
self.add_copy(
bi,
dst,
SrcType::Pred,
lop.srcs[s].bnot(),
lop.srcs[s].clone().bnot(),
);
}
}
@@ -615,9 +640,11 @@ impl CopyPropPass {
assert!(dst.comps() == 1);
let dst = dst[0];
let src = match (sel.srcs[0], sel.srcs[1]) {
(z, u) if z.is_zero() && u.is_nonzero() => sel.cond.bnot(),
(u, z) if z.is_zero() && u.is_nonzero() => sel.cond,
let src = match &sel.srcs {
[z, u] if z.is_zero() && u.is_nonzero() => {
sel.cond.clone().bnot()
}
[u, z] if z.is_zero() && u.is_nonzero() => sel.cond.clone(),
_ => return,
};
@@ -628,7 +655,7 @@ impl CopyPropPass {
assert!(dst.comps() == 1);
let dst = dst[0];
let src = match (isetp.srcs[0], isetp.srcs[1]) {
let src = match (&isetp.srcs[0], &isetp.srcs[1]) {
(z, x) | (x, z) if z.is_zero() => x,
_ => return,
};
@@ -645,7 +672,7 @@ impl CopyPropPass {
IntCmpOp::Ne => false,
_ => return,
};
self.add_i2b(dst, src, inverted);
self.add_i2b(dst, src.clone(), inverted);
}
Op::IAdd2(add) => {
let dst = add.dst.as_ssa().unwrap();
@@ -653,9 +680,9 @@ impl CopyPropPass {
let dst = dst[0];
if add.srcs[0].is_zero() {
self.add_copy(bi, dst, SrcType::I32, add.srcs[1]);
self.add_copy(bi, dst, SrcType::I32, add.srcs[1].clone());
} else if add.srcs[1].is_zero() {
self.add_copy(bi, dst, SrcType::I32, add.srcs[0]);
self.add_copy(bi, dst, SrcType::I32, add.srcs[0].clone());
}
}
Op::IAdd3(add) => {
@@ -665,12 +692,22 @@ impl CopyPropPass {
if add.srcs[0].is_zero() {
if add.srcs[1].is_zero() {
self.add_copy(bi, dst, SrcType::I32, add.srcs[2]);
self.add_copy(
bi,
dst,
SrcType::I32,
add.srcs[2].clone(),
);
} else if add.srcs[2].is_zero() {
self.add_copy(bi, dst, SrcType::I32, add.srcs[1]);
self.add_copy(
bi,
dst,
SrcType::I32,
add.srcs[1].clone(),
);
}
} else if add.srcs[1].is_zero() && add.srcs[2].is_zero() {
self.add_copy(bi, dst, SrcType::I32, add.srcs[0]);
self.add_copy(bi, dst, SrcType::I32, add.srcs[0].clone());
}
}
Op::Prmt(prmt) => {
@@ -680,11 +717,21 @@ impl CopyPropPass {
if let Some(imm) = prmt.as_u32() {
self.add_copy(bi, dst[0], SrcType::GPR, imm.into());
} else if sel == PrmtSel(0x3210) {
self.add_copy(bi, dst[0], SrcType::GPR, prmt.srcs[0]);
self.add_copy(
bi,
dst[0],
SrcType::GPR,
prmt.srcs[0].clone(),
);
} else if sel == PrmtSel(0x7654) {
self.add_copy(bi, dst[0], SrcType::GPR, prmt.srcs[1]);
self.add_copy(
bi,
dst[0],
SrcType::GPR,
prmt.srcs[1].clone(),
);
} else {
self.add_prmt(bi, dst[0], sel, prmt.srcs);
self.add_prmt(bi, dst[0], sel, prmt.srcs.clone());
}
}
}
@@ -693,19 +740,19 @@ impl CopyPropPass {
if r2ur.src.is_uniform() {
let dst = r2ur.dst.as_ssa().unwrap();
assert!(dst.comps() == 1);
self.add_copy(bi, dst[0], SrcType::GPR, r2ur.src);
self.add_copy(bi, dst[0], SrcType::GPR, r2ur.src.clone());
}
}
Op::Copy(copy) => {
let dst = copy.dst.as_ssa().unwrap();
assert!(dst.comps() == 1);
self.add_copy(bi, dst[0], SrcType::GPR, copy.src);
self.add_copy(bi, dst[0], SrcType::GPR, copy.src.clone());
}
Op::ParCopy(pcopy) => {
for (dst, src) in pcopy.dsts_srcs.iter() {
let dst = dst.as_ssa().unwrap();
assert!(dst.comps() == 1);
self.add_copy(bi, dst[0], SrcType::GPR, *src);
self.add_copy(bi, dst[0], SrcType::GPR, src.clone());
}
}
_ => (),
src/nouveau/compiler/nak/opt_lop.rs
+3 -3
View File
@@ -173,7 +173,7 @@ impl LopPass {
for i in 0..3 {
if entry_srcs[i] != usize::MAX {
srcs[entry_srcs[i]] = entry.srcs[i];
srcs[entry_srcs[i]] = entry.srcs[i].clone();
}
}
for op in ops.iter_mut() {
@@ -215,7 +215,7 @@ impl LopPass {
if let Dst::SSA(ssa) = op.dst {
assert!(ssa.comps() == 1);
self.add_lop(ssa[0], op.op, op.srcs);
self.add_lop(ssa[0], op.op, op.srcs.clone());
}
}
@@ -248,7 +248,7 @@ impl LopPass {
for i in 0..2 {
if let Dst::SSA(ssa) = op.dsts[i] {
assert!(ssa.comps() == 1);
self.add_lop(ssa[0], op.ops[i], op.srcs);
self.add_lop(ssa[0], op.ops[i], op.srcs.clone());
}
}
}
src/nouveau/compiler/nak/sm20.rs
+57 -57
View File
@@ -231,7 +231,7 @@ impl SM20Encoder<'_> {
self.set_field(range, reg.base_idx());
}
fn set_pred_src(&mut self, range: Range<usize>, src: Src) {
fn set_pred_src(&mut self, range: Range<usize>, src: &Src) {
let (not, reg) = match src.src_ref {
SrcRef::True => (false, true_reg()),
SrcRef::False => (true, true_reg()),
@@ -299,7 +299,7 @@ impl SM20Encoder<'_> {
}
}
fn set_reg_src(&mut self, range: Range<usize>, src: Src) {
fn set_reg_src(&mut self, range: Range<usize>, src: &Src) {
assert!(src.src_swizzle.is_none());
self.set_reg_src_ref(range, &src.src_ref);
}
@@ -313,7 +313,7 @@ impl SM20Encoder<'_> {
self.set_reg(range, reg);
}
fn set_carry_in(&mut self, bit: usize, src: Src) {
fn set_carry_in(&mut self, bit: usize, src: &Src) {
assert!(src.src_mod.is_none());
match src.src_ref {
SrcRef::Zero => self.set_bit(bit, false),
@@ -672,7 +672,7 @@ impl SM20Op for OpFMnMx {
e.set_bit(7, self.srcs[0].src_mod.has_fabs());
e.set_bit(8, self.srcs[1].src_mod.has_fneg());
e.set_bit(9, self.srcs[0].src_mod.has_fneg());
e.set_pred_src(49..53, self.min);
e.set_pred_src(49..53, &self.min);
}
}
@@ -842,7 +842,7 @@ impl SM20Op for OpFSetP {
e.set_bit(9, self.srcs[0].src_mod.has_fneg());
e.set_pred_dst(14..17, &Dst::None);
e.set_pred_dst(17..20, &self.dst);
e.set_pred_src(49..53, self.accum);
e.set_pred_src(49..53, &self.accum);
e.set_pred_set_op(53..55, self.set_op);
e.set_float_cmp_op(55..59, self.cmp_op);
e.set_bit(59, self.ftz);
@@ -860,8 +860,8 @@ impl SM20Op for OpFSwz {
e.set_opcode(SM20Unit::Float, 0x12);
e.set_dst(14..20, &self.dst);
e.set_reg_src(20..26, self.srcs[0]);
e.set_reg_src(26..32, self.srcs[1]);
e.set_reg_src(20..26, &self.srcs[0]);
e.set_reg_src(26..32, &self.srcs[1]);
e.set_bit(5, self.ftz);
e.set_field(
@@ -979,7 +979,7 @@ impl SM20Op for OpDMnMx {
e.set_bit(7, self.srcs[0].src_mod.has_fabs());
e.set_bit(8, self.srcs[1].src_mod.has_fneg());
e.set_bit(9, self.srcs[0].src_mod.has_fneg());
e.set_pred_src(49..53, self.min);
e.set_pred_src(49..53, &self.min);
}
}
@@ -1035,7 +1035,7 @@ impl SM20Op for OpDSetP {
e.set_bit(9, self.srcs[0].src_mod.has_fneg());
e.set_pred_dst(14..17, &Dst::None);
e.set_pred_dst(17..20, &self.dst);
e.set_pred_src(49..53, self.accum);
e.set_pred_src(49..53, &self.accum);
e.set_pred_set_op(53..55, self.set_op);
e.set_float_cmp_op(55..59, self.cmp_op);
}
@@ -1159,7 +1159,7 @@ impl SM20Op for OpIAdd2X {
}
e.set_bit(5, false); // saturate
e.set_carry_in(6, self.carry_in);
e.set_carry_in(6, &self.carry_in);
e.set_bit(8, self.srcs[1].src_mod.is_bnot());
e.set_bit(9, self.srcs[0].src_mod.is_bnot());
}
@@ -1306,7 +1306,7 @@ impl SM20Op for OpIMnMx {
IntCmpType::I32 => 1_u8,
},
);
e.set_pred_src(49..53, self.min);
e.set_pred_src(49..53, &self.min);
}
}
@@ -1338,7 +1338,7 @@ impl SM20Op for OpISetP {
e.set_bit(6, self.ex);
e.set_pred_dst(14..17, &Dst::None);
e.set_pred_dst(17..20, &self.dst);
e.set_pred_src(49..53, self.accum);
e.set_pred_src(49..53, &self.accum);
e.set_pred_set_op(53..55, self.set_op);
e.set_int_cmp_op(55..58, self.cmp_op);
}
@@ -1618,7 +1618,7 @@ impl SM20Op for OpSel {
Some(&self.srcs[1]),
None,
);
e.set_pred_src(49..53, self.cond);
e.set_pred_src(49..53, &self.cond);
}
}
@@ -1637,14 +1637,14 @@ impl SM20Op for OpShfl {
e.set_opcode(SM20Unit::Mem, 0x22);
e.set_pred_dst2(8..10, 58..59, &self.in_bounds);
e.set_dst(14..20, &self.dst);
e.set_reg_src(20..26, self.src);
e.set_reg_src(20..26, &self.src);
assert!(self.lane.src_mod.is_none());
if let Some(u) = self.lane.src_ref.as_u32() {
e.set_field(26..32, u & 0x1f);
e.set_bit(5, true);
} else {
e.set_reg_src(26..32, self.lane);
e.set_reg_src(26..32, &self.lane);
e.set_bit(5, false);
}
@@ -1653,7 +1653,7 @@ impl SM20Op for OpShfl {
e.set_field(42..55, u & 0x1fff);
e.set_bit(6, true);
} else {
e.set_reg_src(49..55, self.c);
e.set_reg_src(49..55, &self.c);
e.set_bit(6, false);
}
@@ -1679,10 +1679,10 @@ impl SM20Op for OpPSetP {
e.set_pred_dst(14..17, &self.dsts[1]);
e.set_pred_dst(17..20, &self.dsts[0]);
e.set_pred_src(20..24, self.srcs[0]);
e.set_pred_src(26..30, self.srcs[1]);
e.set_pred_src(20..24, &self.srcs[0]);
e.set_pred_src(26..30, &self.srcs[1]);
e.set_pred_set_op(30..32, self.ops[0]);
e.set_pred_src(49..53, self.srcs[2]);
e.set_pred_src(49..53, &self.srcs[2]);
e.set_pred_set_op(53..55, self.ops[1]);
}
}
@@ -1766,8 +1766,8 @@ impl SM20Op for OpTex {
e.set_dst(14..20, &self.dsts[0]);
assert!(self.dsts[1].is_none());
assert!(self.fault.is_none());
e.set_reg_src(20..26, self.srcs[0]);
e.set_reg_src(26..32, self.srcs[1]);
e.set_reg_src(20..26, &self.srcs[0]);
e.set_reg_src(26..32, &self.srcs[1]);
e.set_tex_channel_mask(46..50, self.channel_mask);
e.set_tex_dim(51..54, self.dim);
e.set_bit(54, self.offset);
@@ -1804,8 +1804,8 @@ impl SM20Op for OpTld {
e.set_dst(14..20, &self.dsts[0]);
assert!(self.dsts[1].is_none());
assert!(self.fault.is_none());
e.set_reg_src(20..26, self.srcs[0]);
e.set_reg_src(26..32, self.srcs[1]);
e.set_reg_src(20..26, &self.srcs[0]);
e.set_reg_src(26..32, &self.srcs[1]);
e.set_tex_channel_mask(46..50, self.channel_mask);
e.set_tex_dim(51..54, self.dim);
e.set_bit(54, self.offset);
@@ -1851,8 +1851,8 @@ impl SM20Op for OpTld4 {
e.set_dst(14..20, &self.dsts[0]);
assert!(self.dsts[1].is_none());
assert!(self.fault.is_none());
e.set_reg_src(20..26, self.srcs[0]);
e.set_reg_src(26..32, self.srcs[1]);
e.set_reg_src(20..26, &self.srcs[0]);
e.set_reg_src(26..32, &self.srcs[1]);
e.set_bit(45, false); // .ndv
e.set_tex_channel_mask(46..50, self.channel_mask);
e.set_tex_dim(51..54, self.dim);
@@ -1895,8 +1895,8 @@ impl SM20Op for OpTmml {
e.set_bit(9, self.nodep);
e.set_dst(14..20, &self.dsts[0]);
assert!(self.dsts[1].is_none());
e.set_reg_src(20..26, self.srcs[0]);
e.set_reg_src(26..32, self.srcs[1]);
e.set_reg_src(20..26, &self.srcs[0]);
e.set_reg_src(26..32, &self.srcs[1]);
e.set_tex_channel_mask(46..50, self.channel_mask);
e.set_tex_dim(51..54, self.dim);
}
@@ -1929,8 +1929,8 @@ impl SM20Op for OpTxd {
e.set_bit(9, self.nodep);
e.set_dst(14..20, &self.dsts[0]);
assert!(self.dsts[1].is_none());
e.set_reg_src(20..26, self.srcs[0]);
e.set_reg_src(26..32, self.srcs[1]);
e.set_reg_src(20..26, &self.srcs[0]);
e.set_reg_src(26..32, &self.srcs[1]);
e.set_tex_channel_mask(46..50, self.channel_mask);
e.set_tex_dim(51..54, self.dim);
e.set_bit(54, self.offset);
@@ -1964,8 +1964,8 @@ impl SM20Op for OpTxq {
e.set_bit(9, self.nodep);
e.set_dst(14..20, &self.dsts[0]);
assert!(self.dsts[1].is_none());
e.set_reg_src(20..26, self.src);
e.set_reg_src(26..32, 0.into());
e.set_reg_src(20..26, &self.src);
e.set_reg_src(26..32, &0.into());
e.set_tex_channel_mask(46..50, self.channel_mask);
e.set_field(
54..57,
@@ -2061,7 +2061,7 @@ impl SM20Op for OpLd {
e.set_mem_type(5..8, self.access.mem_type);
// 8..9: cache hints (.ca, .cg, .lu, .cv)
e.set_dst(14..20, &self.dst);
e.set_reg_src(20..26, self.addr);
e.set_reg_src(20..26, &self.addr);
}
}
@@ -2093,7 +2093,7 @@ impl SM20Op for OpLdc {
},
);
e.set_dst(14..20, &self.dst);
e.set_reg_src(20..26, self.offset);
e.set_reg_src(20..26, &self.offset);
e.set_field(26..42, cb.offset);
e.set_field(42..47, cb_idx);
}
@@ -2124,14 +2124,14 @@ impl SM20Op for OpSt {
}
e.set_mem_type(5..8, self.access.mem_type);
// 8..9: cache hints (.ca, .cg, .lu, .cv)
e.set_reg_src(14..20, self.data);
e.set_reg_src(20..26, self.addr);
e.set_reg_src(14..20, &self.data);
e.set_reg_src(20..26, &self.addr);
}
}
fn atom_src_as_ssa(
b: &mut LegalizeBuilder,
src: Src,
src: &Src,
atom_type: AtomType,
) -> SSARef {
if let Some(ssa) = src.as_ssa() {
@@ -2154,8 +2154,8 @@ fn atom_src_as_ssa(
impl SM20Op for OpAtom {
fn legalize(&mut self, b: &mut LegalizeBuilder) {
if self.atom_op == AtomOp::CmpExch(AtomCmpSrc::Separate) {
let cmpr = atom_src_as_ssa(b, self.cmpr, self.atom_type);
let data = atom_src_as_ssa(b, self.data, self.atom_type);
let cmpr = atom_src_as_ssa(b, &self.cmpr, self.atom_type);
let data = atom_src_as_ssa(b, &self.data, self.atom_type);
let mut cmpr_data = Vec::new();
cmpr_data.extend_from_slice(&cmpr);
@@ -2214,8 +2214,8 @@ impl SM20Op for OpAtom {
e.set_field(9..10, typ & 0x1);
e.set_field(59..62, typ >> 1);
e.set_reg_src(20..26, self.addr);
e.set_reg_src(14..20, self.data);
e.set_reg_src(20..26, &self.addr);
e.set_reg_src(14..20, &self.data);
if self.dst.is_none() {
e.set_field(26..58, self.addr_offset);
@@ -2235,9 +2235,9 @@ impl SM20Op for OpAtom {
let data_idx = cmpr_data.base_idx() + u32::from(data_comps);
let data = RegRef::new(cmpr_data.file(), data_idx, data_comps);
e.set_reg_src(49..55, data.into());
e.set_reg_src(49..55, &data.into());
} else if !self.dst.is_none() {
e.set_reg_src(49..55, 0.into());
e.set_reg_src(49..55, &0.into());
}
}
}
@@ -2261,8 +2261,8 @@ impl SM20Op for OpALd {
e.set_bit(9, self.output);
e.set_dst(14..20, &self.dst);
e.set_reg_src(20..26, self.offset);
e.set_reg_src(26..32, self.vtx);
e.set_reg_src(20..26, &self.offset);
e.set_reg_src(26..32, &self.vtx);
e.set_field(32..42, self.addr);
}
}
@@ -2279,10 +2279,10 @@ impl SM20Op for OpASt {
e.set_bit(8, self.patch);
assert!(!self.phys);
e.set_reg_src(20..26, self.offset);
e.set_reg_src(26..32, self.data);
e.set_reg_src(20..26, &self.offset);
e.set_reg_src(26..32, &self.data);
e.set_field(32..42, self.addr);
e.set_reg_src(49..55, self.vtx);
e.set_reg_src(49..55, &self.vtx);
}
}
@@ -2313,9 +2313,9 @@ impl SM20Op for OpIpa {
},
);
e.set_dst(14..20, &self.dst);
e.set_reg_src(20..26, 0.into()); // indirect
e.set_reg_src(26..32, self.inv_w);
e.set_reg_src(49..55, self.offset);
e.set_reg_src(20..26, &0.into()); // indirect
e.set_reg_src(26..32, &self.inv_w);
e.set_reg_src(49..55, &self.offset);
e.set_field(32..42, self.addr);
}
}
@@ -2353,7 +2353,7 @@ impl SM20Op for OpCCtl {
},
);
e.set_dst(14..20, &Dst::None);
e.set_reg_src(20..26, self.addr);
e.set_reg_src(20..26, &self.addr);
e.set_field(26..28, 0); // 1: .u, 2: .c: 3: .i
assert!(self.addr_offset % 4 == 0);
@@ -2501,11 +2501,11 @@ impl SM20Op for OpBar {
e.set_field(5..7, 0_u8); // 0: .popc, 1: .and, 2: .or
e.set_field(7..9, 0_u8); // 0: .sync, 1: .arv, 2: .red
e.set_reg_src(20..26, 0.into());
e.set_reg_src(26..32, 0.into());
e.set_reg_src(20..26, &0.into());
e.set_reg_src(26..32, &0.into());
e.set_bit(46, false); // src1_is_imm
e.set_bit(47, false); // src0_is_imm
e.set_pred_src(49..53, true.into());
e.set_pred_src(49..53, &true.into());
e.set_pred_dst(53..56, &Dst::None);
}
}
@@ -2530,7 +2530,7 @@ impl SM20Op for OpIsberd {
fn encode(&self, e: &mut SM20Encoder<'_>) {
e.set_opcode(SM20Unit::Tex, 0x0);
e.set_dst(14..20, &self.dst);
e.set_reg_src(20..26, self.idx);
e.set_reg_src(20..26, &self.idx);
e.set_field(26..42, 0_u16); // offset
}
}
@@ -2576,7 +2576,7 @@ impl SM20Op for OpPixLd {
},
);
e.set_dst(14..20, &self.dst);
e.set_reg_src(20..26, 0.into());
e.set_reg_src(20..26, &0.into());
e.set_field(26..34, 0_u16); // offset
e.set_pred_dst(53..56, &Dst::None);
}
@@ -2610,7 +2610,7 @@ impl SM20Op for OpVote {
},
);
e.set_dst(14..20, &self.ballot);
e.set_pred_src(20..24, self.pred);
e.set_pred_src(20..24, &self.pred);
e.set_pred_dst(54..57, &self.vote);
}
}
src/nouveau/compiler/nak/sm50.rs
+166 -166
View File
File diff suppressed because it is too large Load Diff
src/nouveau/compiler/nak/sm70_encode.rs
+102 -97
View File
@@ -85,7 +85,7 @@ impl SM70Encoder<'_> {
self.set_field(range, reg.base_idx());
}
fn set_reg_src(&mut self, range: Range<usize>, src: Src) {
fn set_reg_src(&mut self, range: Range<usize>, src: &Src) {
assert!(src.src_mod.is_none());
match src.src_ref {
SrcRef::Zero => self.set_reg(range, self.zero_reg(RegFile::GPR)),
@@ -106,7 +106,7 @@ impl SM70Encoder<'_> {
&mut self,
range: Range<usize>,
not_bit: usize,
src: Src,
src: &Src,
file: RegFile,
) {
let (not, reg) = match src.src_ref {
@@ -122,11 +122,16 @@ impl SM70Encoder<'_> {
self.set_bit(not_bit, not ^ src_mod_is_bnot(src.src_mod));
}
fn set_pred_src(&mut self, range: Range<usize>, not_bit: usize, src: Src) {
fn set_pred_src(&mut self, range: Range<usize>, not_bit: usize, src: &Src) {
self.set_pred_src_file(range, not_bit, src, RegFile::Pred);
}
fn set_upred_src(&mut self, range: Range<usize>, not_bit: usize, src: Src) {
fn set_upred_src(
&mut self,
range: Range<usize>,
not_bit: usize,
src: &Src,
) {
self.set_pred_src_file(range, not_bit, src, RegFile::UPred);
}
@@ -188,7 +193,7 @@ impl SM70Encoder<'_> {
self.set_bar_reg(range, *dst.as_reg().unwrap());
}
fn set_bar_src(&mut self, range: Range<usize>, src: Src) {
fn set_bar_src(&mut self, range: Range<usize>, src: &Src) {
assert!(src.src_mod.is_none());
self.set_bar_reg(range, *src.src_ref.as_reg().unwrap());
}
@@ -724,7 +729,7 @@ impl SM70Op for OpFMnMx {
Some(&self.srcs[1]),
Some(&Src::ZERO),
);
e.set_pred_src(87..90, 90, self.min);
e.set_pred_src(87..90, 90, &self.min);
e.set_bit(80, self.ftz);
}
}
@@ -859,7 +864,7 @@ impl SM70Op for OpFSetP {
e.set_pred_dst(81..84, &self.dst);
e.set_pred_dst(84..87, &Dst::None); // dst1
e.set_pred_src(87..90, 90, self.accum);
e.set_pred_src(87..90, 90, &self.accum);
}
}
@@ -875,8 +880,8 @@ impl SM70Op for OpFSwzAdd {
e.set_opcode(0x822);
e.set_dst(&self.dst);
e.set_reg_src(24..32, self.srcs[0]);
e.set_reg_src(64..72, self.srcs[1]);
e.set_reg_src(24..32, &self.srcs[0]);
e.set_reg_src(64..72, &self.srcs[1]);
let mut subop = 0x0_u8;
@@ -1021,7 +1026,7 @@ impl SM70Op for OpDSetP {
e.set_pred_dst(81..84, &self.dst);
e.set_pred_dst(84..87, &Dst::None); /* dst1 */
e.set_pred_src(87..90, 90, self.accum);
e.set_pred_src(87..90, 90, &self.accum);
}
}
@@ -1151,7 +1156,7 @@ impl SM70Op for OpHSet2 {
e.set_float_cmp_op(76..80, self.cmp_op);
e.set_bit(80, self.ftz);
e.set_pred_src(87..90, 90, self.accum);
e.set_pred_src(87..90, 90, &self.accum);
}
}
@@ -1194,7 +1199,7 @@ impl SM70Op for OpHSetP2 {
e.set_pred_dst(81..84, &self.dsts[0]);
e.set_pred_dst(84..87, &self.dsts[1]);
e.set_pred_src(87..90, 90, self.accum);
e.set_pred_src(87..90, 90, &self.accum);
}
}
@@ -1224,7 +1229,7 @@ impl SM70Op for OpHMnMx2 {
e.set_bit(82, false); // .XORSIGN
e.set_bit(85, false); // .BF16_V2
e.set_pred_src(87..90, 90, self.min);
e.set_pred_src(87..90, 90, &self.min);
}
}
@@ -1343,8 +1348,8 @@ impl SM70Op for OpIAdd3 {
)
};
e.set_pred_src(87..90, 90, false.into());
e.set_pred_src(77..80, 80, false.into());
e.set_pred_src(87..90, 90, &false.into());
e.set_pred_src(77..80, 80, &false.into());
e.set_pred_dst(81..84, &self.overflow[0]);
e.set_pred_dst(84..87, &self.overflow[1]);
@@ -1359,13 +1364,13 @@ impl SM70Op for OpIAdd3X {
swap_srcs_if_not_reg(src2, src1, gpr);
if !src0.src_mod.is_none() && !src1.src_mod.is_none() {
let val = b.alloc_ssa(gpr);
let old_src0 = std::mem::replace(src0, val.into());
b.push_op(OpIAdd3X {
srcs: [Src::ZERO, *src0, Src::ZERO],
srcs: [Src::ZERO, old_src0, Src::ZERO],
overflow: [Dst::None, Dst::None],
dst: val.into(),
carry: [false.into(); 2],
carry: [false.into(), false.into()],
});
*src0 = val.into();
}
b.copy_alu_src_if_not_reg(src0, gpr, SrcType::B32);
b.copy_alu_src_if_both_not_reg(src1, src2, gpr, SrcType::B32);
@@ -1390,8 +1395,8 @@ impl SM70Op for OpIAdd3X {
Some(&self.srcs[2]),
);
e.set_upred_src(87..90, 90, self.carry[0]);
e.set_upred_src(77..80, 80, self.carry[1]);
e.set_upred_src(87..90, 90, &self.carry[0]);
e.set_upred_src(77..80, 80, &self.carry[1]);
} else {
e.encode_alu(
0x010,
@@ -1401,8 +1406,8 @@ impl SM70Op for OpIAdd3X {
Some(&self.srcs[2]),
);
e.set_pred_src(87..90, 90, self.carry[0]);
e.set_pred_src(77..80, 80, self.carry[1]);
e.set_pred_src(87..90, 90, &self.carry[0]);
e.set_pred_src(77..80, 80, &self.carry[1]);
}
e.set_bit(74, true); // .X
@@ -1533,7 +1538,7 @@ impl SM70Op for OpIMnMx {
Some(&self.srcs[1]),
None,
);
e.set_pred_src(87..90, 90, self.min);
e.set_pred_src(87..90, 90, &self.min);
e.set_bit(
73,
match self.cmp_type {
@@ -1569,8 +1574,8 @@ impl SM70Op for OpISetP {
None,
);
e.set_upred_src(68..71, 71, self.low_cmp);
e.set_upred_src(87..90, 90, self.accum);
e.set_upred_src(68..71, 71, &self.low_cmp);
e.set_upred_src(87..90, 90, &self.accum);
} else {
e.encode_alu(
0x00c,
@@ -1580,8 +1585,8 @@ impl SM70Op for OpISetP {
None,
);
e.set_pred_src(68..71, 71, self.low_cmp);
e.set_pred_src(87..90, 90, self.accum);
e.set_pred_src(68..71, 71, &self.low_cmp);
e.set_pred_src(87..90, 90, &self.accum);
}
e.set_bit(72, self.ex);
@@ -1689,7 +1694,7 @@ impl SM70Op for OpLeaX {
Some(&self.b),
c,
);
e.set_upred_src(87..90, 90, self.carry);
e.set_upred_src(87..90, 90, &self.carry);
} else {
e.encode_alu(
0x011,
@@ -1698,7 +1703,7 @@ impl SM70Op for OpLeaX {
Some(&self.b),
c,
);
e.set_pred_src(87..90, 90, self.carry);
e.set_pred_src(87..90, 90, &self.carry);
}
e.set_bit(72, self.intermediate_mod.is_bnot());
@@ -1778,7 +1783,7 @@ impl SM70Op for OpLop3 {
Some(&self.srcs[2]),
);
e.set_upred_src(87..90, 90, SrcRef::False.into());
e.set_upred_src(87..90, 90, &SrcRef::False.into());
} else {
e.encode_alu(
0x012,
@@ -1788,7 +1793,7 @@ impl SM70Op for OpLop3 {
Some(&self.srcs[2]),
);
e.set_pred_src(87..90, 90, SrcRef::False.into());
e.set_pred_src(87..90, 90, &SrcRef::False.into());
}
e.set_field(72..80, self.op.lut);
@@ -2069,7 +2074,7 @@ impl SM70Op for OpSel {
None,
);
e.set_upred_src(87..90, 90, self.cond);
e.set_upred_src(87..90, 90, &self.cond);
} else {
e.encode_alu(
0x007,
@@ -2079,7 +2084,7 @@ impl SM70Op for OpSel {
None,
);
e.set_pred_src(87..90, 90, self.cond);
e.set_pred_src(87..90, 90, &self.cond);
}
}
}
@@ -2100,12 +2105,12 @@ impl SM70Op for OpShfl {
SrcRef::Zero | SrcRef::Reg(_) => match &self.c.src_ref {
SrcRef::Zero | SrcRef::Reg(_) => {
e.set_opcode(0x389);
e.set_reg_src(32..40, self.lane);
e.set_reg_src(64..72, self.c);
e.set_reg_src(32..40, &self.lane);
e.set_reg_src(64..72, &self.c);
}
SrcRef::Imm32(imm_c) => {
e.set_opcode(0x589);
e.set_reg_src(32..40, self.lane);
e.set_reg_src(32..40, &self.lane);
e.set_field(40..53, *imm_c & 0x1f1f);
}
_ => panic!("Invalid instruction form"),
@@ -2114,7 +2119,7 @@ impl SM70Op for OpShfl {
SrcRef::Zero | SrcRef::Reg(_) => {
e.set_opcode(0x989);
e.set_field(53..58, *imm_lane & 0x1f);
e.set_reg_src(64..72, self.c);
e.set_reg_src(64..72, &self.c);
}
SrcRef::Imm32(imm_c) => {
e.set_opcode(0xf89);
@@ -2128,7 +2133,7 @@ impl SM70Op for OpShfl {
e.set_dst(&self.dst);
e.set_pred_dst(81..84, &self.in_bounds);
e.set_reg_src(24..32, self.src);
e.set_reg_src(24..32, &self.src);
e.set_field(
58..60,
match self.op {
@@ -2185,9 +2190,9 @@ impl SM70Op for OpPLop3 {
if self.is_uniform() {
e.set_opcode(0x89c);
e.set_upred_src(68..71, 71, self.srcs[2]);
e.set_upred_src(77..80, 80, self.srcs[1]);
e.set_upred_src(87..90, 90, self.srcs[0]);
e.set_upred_src(68..71, 71, &self.srcs[2]);
e.set_upred_src(77..80, 80, &self.srcs[1]);
e.set_upred_src(87..90, 90, &self.srcs[0]);
} else {
e.set_opcode(0x81c);
@@ -2196,13 +2201,13 @@ impl SM70Op for OpPLop3 {
.as_reg()
.is_some_and(|r| r.is_uniform())
{
e.set_upred_src(68..71, 71, self.srcs[2]);
e.set_upred_src(68..71, 71, &self.srcs[2]);
e.set_bit(67, true);
} else {
e.set_pred_src(68..71, 71, self.srcs[2]);
e.set_pred_src(68..71, 71, &self.srcs[2]);
}
e.set_pred_src(77..80, 80, self.srcs[1]);
e.set_pred_src(87..90, 90, self.srcs[0]);
e.set_pred_src(77..80, 80, &self.srcs[1]);
e.set_pred_src(87..90, 90, &self.srcs[0]);
}
e.set_field(16..24, self.ops[1].lut);
e.set_field(64..67, self.ops[0].lut & 0x7);
@@ -2221,7 +2226,7 @@ impl SM70Op for OpR2UR {
fn encode(&self, e: &mut SM70Encoder<'_>) {
e.set_opcode(0x3c2);
e.set_udst(&self.dst);
e.set_reg_src(24..32, self.src);
e.set_reg_src(24..32, &self.src);
e.set_pred_dst(81..84, &Dst::None);
}
}
@@ -2373,8 +2378,8 @@ impl SM70Op for OpTex {
}
e.set_pred_dst(81..84, &self.fault);
e.set_reg_src(24..32, self.srcs[0]);
e.set_reg_src(32..40, self.srcs[1]);
e.set_reg_src(24..32, &self.srcs[0]);
e.set_reg_src(32..40, &self.srcs[1]);
if e.sm >= 100 {
e.set_field(48..56, 0xff_u8); // ureg
@@ -2426,8 +2431,8 @@ impl SM70Op for OpTld {
}
e.set_pred_dst(81..84, &self.fault);
e.set_reg_src(24..32, self.srcs[0]);
e.set_reg_src(32..40, self.srcs[1]);
e.set_reg_src(24..32, &self.srcs[0]);
e.set_reg_src(32..40, &self.srcs[1]);
if e.sm >= 100 {
e.set_field(48..56, 0xff_u8); // ureg
@@ -2480,8 +2485,8 @@ impl SM70Op for OpTld4 {
}
e.set_pred_dst(81..84, &self.fault);
e.set_reg_src(24..32, self.srcs[0]);
e.set_reg_src(32..40, self.srcs[1]);
e.set_reg_src(24..32, &self.srcs[0]);
e.set_reg_src(32..40, &self.srcs[1]);
if e.sm >= 100 {
e.set_field(48..56, 0xff_u8); // ureg
@@ -2533,8 +2538,8 @@ impl SM70Op for OpTmml {
e.set_field(64..72, 255_u8);
}
e.set_reg_src(24..32, self.srcs[0]);
e.set_reg_src(32..40, self.srcs[1]);
e.set_reg_src(24..32, &self.srcs[0]);
e.set_reg_src(32..40, &self.srcs[1]);
e.set_tex_dim(61..64, self.dim);
e.set_tex_channel_mask(72..76, self.channel_mask);
@@ -2577,8 +2582,8 @@ impl SM70Op for OpTxd {
}
e.set_pred_dst(81..84, &self.fault);
e.set_reg_src(24..32, self.srcs[0]);
e.set_reg_src(32..40, self.srcs[1]);
e.set_reg_src(24..32, &self.srcs[0]);
e.set_reg_src(32..40, &self.srcs[1]);
if e.sm >= 100 {
e.set_field(48..56, 0xff_u8); // ureg
@@ -2621,7 +2626,7 @@ impl SM70Op for OpTxq {
e.set_field(64..72, 255_u8);
}
e.set_reg_src(24..32, self.src);
e.set_reg_src(24..32, &self.src);
e.set_field(
62..64,
match self.query {
@@ -2737,8 +2742,8 @@ impl SM70Op for OpSuLd {
}
e.set_dst(&self.dst);
e.set_reg_src(24..32, self.coord);
e.set_reg_src(64..72, self.handle);
e.set_reg_src(24..32, &self.coord);
e.set_reg_src(64..72, &self.handle);
e.set_pred_dst(81..84, &self.fault);
e.set_image_dim(61..64, self.image_dim);
@@ -2764,9 +2769,9 @@ impl SM70Op for OpSuSt {
}
}
e.set_reg_src(24..32, self.coord);
e.set_reg_src(32..40, self.data);
e.set_reg_src(64..72, self.handle);
e.set_reg_src(24..32, &self.coord);
e.set_reg_src(32..40, &self.data);
e.set_reg_src(64..72, &self.handle);
e.set_image_dim(61..64, self.image_dim);
e.set_mem_order(&self.mem_order);
@@ -2792,9 +2797,9 @@ impl SM70Op for OpSuAtom {
};
e.set_dst(&self.dst);
e.set_reg_src(24..32, self.coord);
e.set_reg_src(32..40, self.data);
e.set_reg_src(64..72, self.handle);
e.set_reg_src(24..32, &self.coord);
e.set_reg_src(32..40, &self.data);
e.set_reg_src(64..72, &self.handle);
e.set_pred_dst(81..84, &self.fault);
e.set_image_dim(61..64, self.image_dim);
@@ -2844,7 +2849,7 @@ impl SM70Op for OpLd {
}
e.set_dst(&self.dst);
e.set_reg_src(24..32, self.addr);
e.set_reg_src(24..32, &self.addr);
e.set_field(40..64, self.offset);
}
}
@@ -2872,7 +2877,7 @@ impl SM70Op for OpLdc {
e.set_opcode(0xb82);
e.set_dst(&self.dst);
e.set_reg_src(24..32, self.offset);
e.set_reg_src(24..32, &self.offset);
e.set_field(
78..80,
match self.mode {
@@ -2896,11 +2901,11 @@ impl SM70Op for OpLdc {
e.set_opcode(0x582);
e.set_dst(&self.dst);
e.set_reg_src(64..72, self.offset);
e.set_reg_src(64..72, &self.offset);
}
e.set_ureg(24..32, handle);
e.set_reg_src(64..72, self.offset);
e.set_reg_src(64..72, &self.offset);
assert!(self.mode == LdcMode::Indexed);
e.set_bit(91, true); // Bindless
}
@@ -2946,8 +2951,8 @@ impl SM70Op for OpSt {
}
}
e.set_reg_src(24..32, self.addr);
e.set_reg_src(32..40, self.data);
e.set_reg_src(24..32, &self.addr);
e.set_reg_src(32..40, &self.data);
e.set_field(40..64, self.offset);
}
}
@@ -3030,14 +3035,14 @@ impl SM70Op for OpAtom {
e.set_opcode(0x98e);
}
e.set_reg_src(32..40, self.data);
e.set_reg_src(32..40, &self.data);
e.set_atom_op(87..90, self.atom_op);
} else if let AtomOp::CmpExch(cmp_src) = self.atom_op {
e.set_opcode(0x3a9);
assert!(cmp_src == AtomCmpSrc::Separate);
e.set_reg_src(32..40, self.cmpr);
e.set_reg_src(64..72, self.data);
e.set_reg_src(32..40, &self.cmpr);
e.set_reg_src(64..72, &self.data);
} else {
if e.sm >= 90 && self.atom_type.is_float() {
e.set_opcode(0x3a3);
@@ -3045,7 +3050,7 @@ impl SM70Op for OpAtom {
e.set_opcode(0x3a8);
}
e.set_reg_src(32..40, self.data);
e.set_reg_src(32..40, &self.data);
e.set_atom_op(87..91, self.atom_op);
}
@@ -3068,12 +3073,12 @@ impl SM70Op for OpAtom {
e.set_opcode(0x38d);
assert!(cmp_src == AtomCmpSrc::Separate);
e.set_reg_src(32..40, self.cmpr);
e.set_reg_src(64..72, self.data);
e.set_reg_src(32..40, &self.cmpr);
e.set_reg_src(64..72, &self.data);
} else {
e.set_opcode(0x38c);
e.set_reg_src(32..40, self.data);
e.set_reg_src(32..40, &self.data);
assert!(
self.atom_type != AtomType::U64
|| self.atom_op == AtomOp::Exch,
@@ -3094,7 +3099,7 @@ impl SM70Op for OpAtom {
}
e.set_dst(&self.dst);
e.set_reg_src(24..32, self.addr);
e.set_reg_src(24..32, &self.addr);
e.set_field(40..64, self.addr_offset);
e.set_atom_type(self.atom_type);
}
@@ -3109,7 +3114,7 @@ impl SM70Op for OpAL2P {
e.set_opcode(0x920);
e.set_dst(&self.dst);
e.set_reg_src(24..32, self.offset);
e.set_reg_src(24..32, &self.offset);
e.set_field(40..50, self.addr);
e.set_field(74..76, 0_u8); // comps
@@ -3126,8 +3131,8 @@ impl SM70Op for OpALd {
e.set_opcode(0x321);
e.set_dst(&self.dst);
e.set_reg_src(32..40, self.vtx);
e.set_reg_src(24..32, self.offset);
e.set_reg_src(32..40, &self.vtx);
e.set_reg_src(24..32, &self.offset);
e.set_field(40..50, self.addr);
e.set_field(74..76, self.comps - 1);
@@ -3145,9 +3150,9 @@ impl SM70Op for OpASt {
fn encode(&self, e: &mut SM70Encoder<'_>) {
e.set_opcode(0x322);
e.set_reg_src(32..40, self.data);
e.set_reg_src(64..72, self.vtx);
e.set_reg_src(24..32, self.offset);
e.set_reg_src(32..40, &self.data);
e.set_reg_src(64..72, &self.vtx);
e.set_reg_src(24..32, &self.offset);
e.set_field(40..50, self.addr);
e.set_field(74..76, self.comps - 1);
@@ -3190,7 +3195,7 @@ impl SM70Op for OpIpa {
);
assert!(self.inv_w.is_zero());
e.set_reg_src(32..40, self.offset);
e.set_reg_src(32..40, &self.offset);
// TODO: What is this for?
e.set_pred_dst(81..84, &Dst::None);
@@ -3226,7 +3231,7 @@ impl SM70Op for OpCCtl {
assert!(matches!(self.mem_space, MemSpace::Global(_)));
e.set_opcode(0x98f);
e.set_reg_src(24..32, self.addr);
e.set_reg_src(24..32, &self.addr);
e.set_field(32..64, self.addr_offset);
e.set_field(
@@ -3309,14 +3314,14 @@ impl SM70Op for OpBMov {
e.set_opcode(0x356);
e.set_bar_dst(24..28, &self.dst);
e.set_reg_src(32..40, self.src);
e.set_reg_src(32..40, &self.src);
e.set_bit(84, self.clear);
} else {
e.set_opcode(0x355);
e.set_dst(&self.dst);
e.set_bar_src(24..28, self.src);
e.set_bar_src(24..28, &self.src);
e.set_bit(84, self.clear);
}
@@ -3332,7 +3337,7 @@ impl SM70Op for OpBreak {
e.set_opcode(0x942);
assert!(self.bar_in.src_ref.as_reg() == self.bar_out.as_reg());
e.set_bar_dst(16..20, &self.bar_out);
e.set_pred_src(87..90, 90, self.cond);
e.set_pred_src(87..90, 90, &self.cond);
}
}
@@ -3346,7 +3351,7 @@ impl SM70Op for OpBSSy {
assert!(self.bar_in.src_ref.as_reg() == self.bar_out.as_reg());
e.set_bar_dst(16..20, &self.bar_out);
e.set_rel_offset(34..64, &self.target);
e.set_pred_src(87..90, 90, self.cond);
e.set_pred_src(87..90, 90, &self.cond);
}
}
@@ -3357,8 +3362,8 @@ impl SM70Op for OpBSync {
fn encode(&self, e: &mut SM70Encoder<'_>) {
e.set_opcode(0x941);
e.set_bar_src(16..20, self.bar);
e.set_pred_src(87..90, 90, self.cond);
e.set_bar_src(16..20, &self.bar);
e.set_pred_src(87..90, 90, &self.cond);
}
}
@@ -3397,7 +3402,7 @@ impl SM70Op for OpWarpSync {
fn encode(&self, e: &mut SM70Encoder<'_>) {
e.encode_alu(0x148, None, None, Some(&Src::from(self.mask)), None);
e.set_pred_src(87..90, 90, SrcRef::True.into());
e.set_pred_src(87..90, 90, &SrcRef::True.into());
}
}
@@ -3450,7 +3455,7 @@ impl SM70Op for OpIsberd {
fn encode(&self, e: &mut SM70Encoder<'_>) {
e.set_opcode(0x923);
e.set_dst(&self.dst);
e.set_reg_src(24..32, self.idx);
e.set_reg_src(24..32, &self.idx);
}
}
@@ -3461,7 +3466,7 @@ impl SM70Op for OpKill {
fn encode(&self, e: &mut SM70Encoder<'_>) {
e.set_opcode(0x95b);
e.set_pred_src(87..90, 90, SrcRef::True.into());
e.set_pred_src(87..90, 90, &SrcRef::True.into());
}
}
@@ -3579,7 +3584,7 @@ impl SM70Op for OpVote {
);
e.set_pred_dst(81..84, &self.vote);
e.set_pred_src(87..90, 90, self.pred);
e.set_pred_src(87..90, 90, &self.pred);
}
}
src/nouveau/compiler/nak/to_cssa.rs
+5 -4
View File
@@ -359,8 +359,9 @@ impl Function {
let ss = cg.ssa_set(&vec[0]);
if cg.sets_interfere(ps, ss, &self.blocks) {
let tmp = self.ssa_alloc.alloc(file);
pcopy.push(tmp.into(), *src);
*src = tmp.into();
let old_src =
std::mem::replace(src, tmp.into());
pcopy.push(tmp.into(), old_src);
} else {
cg.sets_merge(ps, ss);
}
@@ -378,11 +379,11 @@ impl Function {
annotation: "generated by to_cssa".into(),
}));
}
let old_src = std::mem::replace(src, tmp.into());
instrs.push(Instr::new_boxed(OpCopy {
dst: tmp.into(),
src: *src,
src: old_src,
}));
*src = tmp.into();
}
if !pcopy.is_empty() {