From 840c0281ced493ca7dba5bacdf3ce265ec954dd9 Mon Sep 17 00:00:00 2001 From: Faith Ekstrand Date: Thu, 20 Apr 2023 16:14:04 -0500 Subject: [PATCH] nak: Implement image load/store Part-of: --- src/nouveau/compiler/nak_encode_sm75.rs | 98 +++++++++++++++------ src/nouveau/compiler/nak_from_nir.rs | 75 ++++++++++++++++ src/nouveau/compiler/nak_ir.rs | 108 ++++++++++++++++++++++++ 3 files changed, 257 insertions(+), 24 deletions(-) diff --git a/src/nouveau/compiler/nak_encode_sm75.rs b/src/nouveau/compiler/nak_encode_sm75.rs index 5ccff7991b3..7a8607ae081 100644 --- a/src/nouveau/compiler/nak_encode_sm75.rs +++ b/src/nouveau/compiler/nak_encode_sm75.rs @@ -916,6 +916,77 @@ impl SM75Instr { self.set_field(72..76, op.mask); } + fn set_image_dim(&mut self, range: Range, dim: ImageDim) { + assert!(range.len() == 3); + self.set_field( + range, + match dim { + ImageDim::_1D => 0_u8, + ImageDim::_1DBuffer => 1_u8, + ImageDim::_1DArray => 2_u8, + ImageDim::_2D => 3_u8, + ImageDim::_2DArray => 4_u8, + ImageDim::_3D => 5_u8, + }, + ); + } + + fn set_mem_order_scope(&mut self, order: &MemOrder, scope: &MemScope) { + if self.sm <= 75 { + self.set_field( + 77..79, + match scope { + MemScope::CTA => 0_u8, + MemScope::Cluster => 1_u8, + MemScope::GPU => 2_u8, + MemScope::System => 3_u8, + }, + ); + self.set_field( + 79..81, + match order { + /* Constant => 0_u8, */ + MemOrder::Weak => 1_u8, + MemOrder::Strong => 2_u8, + /* MMIO => 3_u8, */ + }, + ); + } else { + assert!(*scope == MemScope::System); + assert!(*order == MemOrder::Strong); + self.set_field(77..81, 0xa); + } + } + + fn encode_suld(&mut self, op: &OpSuLd) { + self.set_opcode(0x998); + + self.set_dst(op.dst); + self.set_reg_src(24..32, op.coord); + self.set_reg_src(64..72, op.handle); + self.set_pred_dst(81..84, op.resident); + + self.set_image_dim(61..64, op.image_dim); + self.set_mem_order_scope(&op.mem_order, &op.mem_scope); + + assert!(op.mask == 0x1 || op.mask == 0x3 || op.mask == 0xf); + self.set_field(72..76, op.mask); + } + + fn encode_sust(&mut self, op: &OpSuSt) { + self.set_opcode(0x99c); + + self.set_reg_src(24..32, op.coord); + self.set_reg_src(32..40, op.data); + self.set_reg_src(64..72, op.handle); + + self.set_image_dim(61..64, op.image_dim); + self.set_mem_order_scope(&op.mem_order, &op.mem_scope); + + assert!(op.mask == 0x1 || op.mask == 0x3 || op.mask == 0xf); + self.set_field(72..76, op.mask); + } + fn set_mem_access(&mut self, access: &MemAccess) { self.set_field( 72..73, @@ -936,30 +1007,7 @@ impl SM75Instr { MemType::B128 => 6_u8, }, ); - if self.sm <= 75 { - self.set_field( - 77..79, - match access.scope { - MemScope::CTA => 0_u8, - MemScope::Cluster => 1_u8, - MemScope::GPU => 2_u8, - MemScope::System => 3_u8, - }, - ); - self.set_field( - 79..81, - match access.order { - /* Constant => 0_u8, */ - /* Weak? => 1_u8, */ - MemOrder::Strong => 2_u8, - /* MMIO => 3_u8, */ - }, - ); - } else { - assert!(access.scope == MemScope::System); - assert!(access.order == MemOrder::Strong); - self.set_field(77..81, 0xa); - } + self.set_mem_order_scope(&access.order, &access.scope); } fn encode_ld(&mut self, op: &OpLd) { @@ -1118,6 +1166,8 @@ impl SM75Instr { Op::Tmml(op) => si.encode_tmml(&op), Op::Txd(op) => si.encode_txd(&op), Op::Txq(op) => si.encode_txq(&op), + Op::SuLd(op) => si.encode_suld(&op), + Op::SuSt(op) => si.encode_sust(&op), Op::Ld(op) => si.encode_ld(&op), Op::St(op) => si.encode_st(&op), Op::ALd(op) => si.encode_ald(&op), diff --git a/src/nouveau/compiler/nak_from_nir.rs b/src/nouveau/compiler/nak_from_nir.rs index e61073a0699..be4c95366e2 100644 --- a/src/nouveau/compiler/nak_from_nir.rs +++ b/src/nouveau/compiler/nak_from_nir.rs @@ -778,9 +778,84 @@ impl<'a> ShaderFromNir<'a> { } } + fn get_image_dim(&mut self, intrin: &nir_intrinsic_instr) -> ImageDim { + let is_array = intrin.image_array(); + let image_dim = intrin.image_dim(); + match intrin.image_dim() { + GLSL_SAMPLER_DIM_1D => { + if is_array { + ImageDim::_1DArray + } else { + ImageDim::_1D + } + } + GLSL_SAMPLER_DIM_2D => { + if is_array { + ImageDim::_2DArray + } else { + ImageDim::_2D + } + } + GLSL_SAMPLER_DIM_3D => { + assert!(!is_array); + ImageDim::_3D + } + GLSL_SAMPLER_DIM_CUBE => ImageDim::_2DArray, + GLSL_SAMPLER_DIM_BUF => { + assert!(!is_array); + ImageDim::_1DBuffer + } + _ => panic!("Unsupported image dimension: {}", image_dim), + } + } + + fn get_image_coord( + &mut self, + intrin: &nir_intrinsic_instr, + dim: ImageDim, + ) -> Src { + let vec = self.get_ssa(intrin.get_src(1).as_def()); + /* let sample = self.get_src(&srcs[2]); */ + let comps = usize::from(dim.coord_comps()); + SSARef::try_from(&vec[0..comps]).unwrap().into() + } + fn parse_intrinsic(&mut self, intrin: &nir_intrinsic_instr) { let srcs = intrin.srcs_as_slice(); match intrin.intrinsic { + nir_intrinsic_bindless_image_load => { + let handle = self.get_src(&srcs[0]); + let dim = self.get_image_dim(intrin); + let coord = self.get_image_coord(intrin, dim); + /* let sample = self.get_src(&srcs[2]); */ + let dst = self.get_dst(&intrin.def); + self.instrs.push(Instr::new(OpSuLd { + dst: dst, + resident: Dst::None, + image_dim: dim, + mem_order: MemOrder::Weak, + mem_scope: MemScope::CTA, + mask: 0xf, + handle: handle, + coord: coord, + })); + } + nir_intrinsic_bindless_image_store => { + let handle = self.get_src(&srcs[0]); + let dim = self.get_image_dim(intrin); + let coord = self.get_image_coord(intrin, dim); + /* let sample = self.get_src(&srcs[2]); */ + let data = self.get_src(&srcs[3]); + self.instrs.push(Instr::new(OpSuSt { + image_dim: dim, + mem_order: MemOrder::Weak, + mem_scope: MemScope::CTA, + mask: 0xf, + handle: handle, + coord: coord, + data: data, + })); + } nir_intrinsic_load_barycentric_centroid => (), nir_intrinsic_load_barycentric_pixel => (), nir_intrinsic_load_barycentric_sample => (), diff --git a/src/nouveau/compiler/nak_ir.rs b/src/nouveau/compiler/nak_ir.rs index ebe95afb2e7..09ad30d6cb8 100644 --- a/src/nouveau/compiler/nak_ir.rs +++ b/src/nouveau/compiler/nak_ir.rs @@ -1015,6 +1015,42 @@ impl fmt::Display for TexQuery { } } +#[derive(Clone, Copy, Eq, PartialEq)] +pub enum ImageDim { + _1D, + _1DBuffer, + _1DArray, + _2D, + _2DArray, + _3D, +} + +impl ImageDim { + pub fn coord_comps(&self) -> u8 { + match self { + ImageDim::_1D => 1, + ImageDim::_1DBuffer => 1, + ImageDim::_1DArray => 2, + ImageDim::_2D => 2, + ImageDim::_2DArray => 3, + ImageDim::_3D => 3, + } + } +} + +impl fmt::Display for ImageDim { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ImageDim::_1D => write!(f, "1D"), + ImageDim::_1DBuffer => write!(f, "1D_BUFFER"), + ImageDim::_1DArray => write!(f, "1D_ARRAY"), + ImageDim::_2D => write!(f, "2D"), + ImageDim::_2DArray => write!(f, "2D_ARRAY"), + ImageDim::_3D => write!(f, "3D"), + } + } +} + pub enum IntType { U8, I8, @@ -1158,9 +1194,19 @@ impl fmt::Display for MemType { #[derive(Clone, Copy, Eq, Hash, PartialEq)] pub enum MemOrder { + Weak, Strong, } +impl fmt::Display for MemOrder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + MemOrder::Weak => write!(f, "WEAK"), + MemOrder::Strong => write!(f, "STRONG"), + } + } +} + #[derive(Clone, Copy, Eq, Hash, PartialEq)] pub enum MemScope { CTA, @@ -1833,6 +1879,63 @@ impl fmt::Display for OpTxq { } } +#[repr(C)] +#[derive(SrcsAsSlice, DstsAsSlice)] +pub struct OpSuLd { + pub dst: Dst, + pub resident: Dst, + + pub image_dim: ImageDim, + pub mem_order: MemOrder, + pub mem_scope: MemScope, + pub mask: u8, + pub handle: Src, + pub coord: Src, +} + +impl fmt::Display for OpSuLd { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "SULD.P.{}.{}.{} {{ {} {} }} [{}] {}", + self.image_dim, + self.mem_order, + self.mem_scope, + self.dst, + self.resident, + self.coord, + self.handle, + ) + } +} + +#[repr(C)] +#[derive(SrcsAsSlice, DstsAsSlice)] +pub struct OpSuSt { + pub image_dim: ImageDim, + pub mem_order: MemOrder, + pub mem_scope: MemScope, + pub mask: u8, + pub handle: Src, + pub coord: Src, + pub data: Src, +} + +impl fmt::Display for OpSuSt { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "SUST.P.{}.{}.{} [{}] {} {}", + self.image_dim, + self.mem_order, + self.mem_scope, + self.coord, + self.data, + self.handle, + ) + } +} + #[repr(C)] #[derive(SrcsAsSlice, DstsAsSlice)] pub struct OpLd { @@ -2292,6 +2395,8 @@ pub enum Op { Tmml(OpTmml), Txd(OpTxd), Txq(OpTxq), + SuLd(OpSuLd), + SuSt(OpSuSt), Ld(OpLd), St(OpSt), ALd(OpALd), @@ -2707,6 +2812,7 @@ impl Instr { pub fn can_eliminate(&self) -> bool { match self.op { Op::ASt(_) + | Op::SuSt(_) | Op::St(_) | Op::Bra(_) | Op::Exit(_) @@ -2745,6 +2851,8 @@ impl Instr { Op::Tmml(_) => None, Op::Txd(_) => None, Op::Txq(_) => None, + Op::SuLd(_) => None, + Op::SuSt(_) => None, Op::Ld(_) => None, Op::St(_) => None, Op::Bra(_) | Op::Exit(_) => Some(15),