From 787d3b0ffc15fd1177d8d8c45ae718454af9cc6e Mon Sep 17 00:00:00 2001 From: Karol Herbst Date: Tue, 3 Dec 2024 16:27:19 +0100 Subject: [PATCH] rusticl/mem: add Allocation type This will handle the raw memory operations in the future, mostly focused on guaranteeing consistency across devices. It also handles sub allocation because it is the better place of tracking this. Part-of: --- src/gallium/frontends/rusticl/api/memory.rs | 14 +- src/gallium/frontends/rusticl/core/kernel.rs | 2 +- src/gallium/frontends/rusticl/core/memory.rs | 479 ++++++++++++++++--- 3 files changed, 425 insertions(+), 70 deletions(-) diff --git a/src/gallium/frontends/rusticl/api/memory.rs b/src/gallium/frontends/rusticl/api/memory.rs index 6ded7f3e83a..fa2d471bd05 100644 --- a/src/gallium/frontends/rusticl/api/memory.rs +++ b/src/gallium/frontends/rusticl/api/memory.rs @@ -219,7 +219,7 @@ unsafe impl CLInfo for cl_mem { let mem = MemBase::ref_from_raw(*self)?; match *q { CL_MEM_ASSOCIATED_MEMOBJECT => { - let ptr = match mem.parent.as_ref() { + let ptr = match mem.parent() { // Note we use as_ptr here which doesn't increase the reference count. Some(Mem::Buffer(buffer)) => cl_mem::from_ptr(Arc::as_ptr(buffer)), Some(Mem::Image(image)) => cl_mem::from_ptr(Arc::as_ptr(image)), @@ -237,7 +237,7 @@ unsafe impl CLInfo for cl_mem { CL_MEM_MAP_COUNT => v.write::(0), CL_MEM_HOST_PTR => v.write::<*mut c_void>(mem.host_ptr()), CL_MEM_OFFSET => v.write::(if mem.is_buffer() { - Buffer::ref_from_raw(*self)?.offset + Buffer::ref_from_raw(*self)?.offset() } else { 0 }), @@ -329,7 +329,7 @@ fn create_sub_buffer( let b = Buffer::arc_from_raw(buffer)?; // CL_INVALID_MEM_OBJECT if buffer ... is a sub-buffer object. - if b.parent.is_some() { + if b.parent().is_some() { return Err(CL_INVALID_MEM_OBJECT); } @@ -1166,8 +1166,8 @@ fn enqueue_copy_buffer( // sub-buffers of the same associated buffer object and they overlap. The regions overlap if // src_offset ≤ dst_offset ≤ src_offset + size - 1 or if dst_offset ≤ src_offset ≤ dst_offset + size - 1. if src.has_same_parent(&dst) { - let src_offset = src_offset + src.offset; - let dst_offset = dst_offset + dst.offset; + let src_offset = src_offset + src.offset(); + let dst_offset = dst_offset + dst.offset(); if (src_offset <= dst_offset && dst_offset < src_offset + size) || (dst_offset <= src_offset && src_offset < dst_offset + size) @@ -1523,9 +1523,9 @@ fn enqueue_copy_buffer_rect( if src.has_same_parent(&dst) && check_copy_overlap( &src_ori, - src.offset, + src.offset(), &dst_ori, - dst.offset, + dst.offset(), &r, src_row_pitch, src_slice_pitch, diff --git a/src/gallium/frontends/rusticl/core/kernel.rs b/src/gallium/frontends/rusticl/core/kernel.rs index ca335005169..41f904787c5 100644 --- a/src/gallium/frontends/rusticl/core/kernel.rs +++ b/src/gallium/frontends/rusticl/core/kernel.rs @@ -1431,7 +1431,7 @@ impl Kernel { KernelArgValue::Buffer(buffer) => { let buffer = &buffer_arcs[&(buffer.as_ptr() as usize)]; let res = buffer.get_res_of_dev(q.device)?; - add_global(q, &mut input, &mut resource_info, res, buffer.offset); + add_global(q, &mut input, &mut resource_info, res, buffer.offset()); } KernelArgValue::Image(image) => { let image = &image_arcs[&(image.as_ptr() as usize)]; diff --git a/src/gallium/frontends/rusticl/core/memory.rs b/src/gallium/frontends/rusticl/core/memory.rs index b069cf36f83..1019d749a29 100644 --- a/src/gallium/frontends/rusticl/core/memory.rs +++ b/src/gallium/frontends/rusticl/core/memory.rs @@ -34,6 +34,7 @@ use std::os::raw::c_void; use std::ptr; use std::sync::Arc; use std::sync::Mutex; +use std::sync::MutexGuard; struct Mapping { layout: Layout, @@ -130,6 +131,372 @@ impl MutMemoryPtr { } } +#[derive(Copy, Clone, PartialEq)] +pub enum ResourceValidityEntity { + Host, + Device(&'static Device), +} + +/// Allocation with real GPU backing storage. Tracks on which device the content is valid on. +pub struct ResourceAllocation { + pub res: HashMap<&'static Device, Arc>, + valid_on: Mutex>, + // it's a bit hacky, but storing the pointer as `usize` gives us `Send` and `Sync`. The + // application is required to ensure no data races exist on the memory anyway. + host_ptr: usize, + hostptr_devs: Vec, + // this might be non zero for dma-buf imported resources + offset: usize, +} + +impl ResourceAllocation { + /// # Panics + /// + /// valid_on needs to be a Vec with at least one element, will panic otherwise. + fn get_best_valid_entity_for_transfer( + valid_on: &MutexGuard>, + ) -> ResourceValidityEntity { + // We want to avoid having to copy over the PCIe bus, so we prefer an entity which is either + // the host itself or a device using host memory. + let res = valid_on.iter().min_by_key(|entity| match entity { + ResourceValidityEntity::Host => 0, + ResourceValidityEntity::Device(dev) => { + if dev.unified_memory() { + 1 + } else { + 2 + } + } + }); + + *res.unwrap() + } + + /// Small helper function to indicate when transparent migration is never required, e.g. if it's + /// a single device allocation with no hostptr. + fn can_skip_migration(&self) -> bool { + match self.hostptr_devs.len() { + // If storage isn't shared between devices, we only need to migrate when there is more + // than one device. + 0 => self.res.len() == 1, + + // If all devices use a host_ptr allocation, the content is automatically synchronized + // as they share the same storage. The - 1 is required as the Host is also part of + // `hostptr_devs`. + len => len - 1 == self.res.len(), + } + } + + /// Returns the GPU resource for the device `ctx` is associated with. It will transparently + /// migrate the data to the GPU. + /// TODO: add a map function to return a mapping to the resource of one device the data is valid + /// on instead of migrating if the user would simply map the resource anyway. + fn get_res_for_access(&self, ctx: &QueueContext, rw: RWFlags) -> CLResult<&Arc> { + let dev = ctx.dev; + let dev_entity = ResourceValidityEntity::Device(dev); + let to_res = self.res.get(dev).ok_or(CL_OUT_OF_HOST_MEMORY)?; + + // in most cases we can skip most of the work below. + if self.can_skip_migration() { + return Ok(to_res); + } + + let Ok(mut valid_on) = self.valid_on.lock() else { + return Err(CL_OUT_OF_HOST_MEMORY); + }; + + // If the content isn't valid on dev we need to migrate it to it. + if matches!(rw, RWFlags::RD | RWFlags::RW) && !valid_on.contains(&dev_entity) { + // valid_on is a vec with at least one element, so this call won't panic. + let entity = Self::get_best_valid_entity_for_transfer(&valid_on); + + let helper_ctx; + let map; + let flush; + + if to_res.is_buffer() { + let ptr; + match entity { + ResourceValidityEntity::Host => { + flush = false; + ptr = self.host_ptr as *mut c_void; + } + ResourceValidityEntity::Device(dev) => { + flush = true; + + let from_res = &self.res[dev]; + helper_ctx = dev.helper_ctx(); + + // update the resource and wait for the operation to finish. We also map the resources + // unsynchronized as we can't block or flush any other contexts here as this might cause + // deadlocks. + map = helper_ctx + .map_buffer_unsynchronized( + from_res, + 0, + from_res.width() as i32, + RWFlags::RD, + ) + .ok_or(CL_OUT_OF_HOST_MEMORY)?; + + ptr = map.ptr(); + } + } + + ctx.buffer_subdata(to_res, 0, ptr, to_res.width()); + } else { + let ResourceValidityEntity::Device(dev) = entity else { + // we don't support migrating from host_ptr for images yet. It's also not needed + // because the Image struct has a more optimized way of doing things there. + unimplemented!(); + }; + + flush = true; + let from_res = &self.res[dev]; + helper_ctx = dev.helper_ctx(); + + // update the resource and wait for the operation to finish. We also map the resources + // unsynchronized as we can't block or flush any other contexts here as this might cause + // deadlocks. + let bx = pipe_box { + width: from_res.width() as i32, + height: from_res.height() as i32, + depth: from_res.depth() as i16, + ..Default::default() + }; + + map = helper_ctx + .map_texture_unsynchronized(from_res, &bx, RWFlags::RD) + .ok_or(CL_OUT_OF_HOST_MEMORY)?; + + let row_pitch: u32 = map.row_pitch(); + let slice_pitch: usize = map.slice_pitch(); + + let bx = pipe_box { + width: to_res.width() as i32, + height: to_res.height() as i32, + depth: to_res.depth() as i16, + ..Default::default() + }; + + ctx.texture_subdata(to_res, &bx, map.ptr(), row_pitch, slice_pitch); + } + + // TODO: we really kinda need to figure out how we can make the compiler scream, that + // temporarily mapped memory might be accessed at some random point in the future + // by a GPU unless it's queues are flushed and processed. + if flush { + ctx.flush().wait(); + } + } + + if matches!(rw, RWFlags::WR | RWFlags::RW) { + // If the user writes to it it's not valid on any other device anymore. + valid_on.clear(); + } + + if !valid_on.contains(&dev_entity) { + // if we update one hostptr resource, we update them all. + if self.hostptr_devs.contains(&dev_entity) { + valid_on.extend_from_slice(&self.hostptr_devs); + } else { + valid_on.push(ResourceValidityEntity::Device(dev)); + } + } + + Ok(to_res) + } + + pub fn migrate_to_hostptr(&self, ctx: &QueueContext, rw: RWFlags) -> CLResult<()> { + let host_entity = ResourceValidityEntity::Host; + let host_ptr = self.host_ptr as *mut c_void; + + // in most cases we can skip most of the work below. + if self.can_skip_migration() || host_ptr.is_null() { + return Ok(()); + } + + let Ok(mut valid_on) = self.valid_on.lock() else { + return Err(CL_OUT_OF_HOST_MEMORY); + }; + + // If the content isn't valid on the host we need to migrate it to it. + if matches!(rw, RWFlags::RD | RWFlags::RW) && !valid_on.contains(&host_entity) { + let ctx_dev_entity = ResourceValidityEntity::Device(ctx.dev); + let mut entity = ctx_dev_entity; + + if !valid_on.contains(&entity) { + // valid_on is a vec with at least one element, so this call won't panic. + entity = Self::get_best_valid_entity_for_transfer(&valid_on); + } + + debug_assert!(entity != ResourceValidityEntity::Host); + + let ResourceValidityEntity::Device(from_dev) = entity else { + // we check if `valid_on` contains a host entity above, so this should never happen. + unreachable!(); + }; + + let helper_ctx; + let map; + let from_res = &self.res[from_dev]; + + assert!( + from_res.is_buffer(), + "Transparent resource migration only supported on buffers." + ); + + if from_dev == ctx.dev { + map = ctx + .buffer_map(from_res, 0, from_res.width() as i32, RWFlags::RD) + .ok_or(CL_OUT_OF_HOST_MEMORY)?; + } else { + helper_ctx = from_dev.helper_ctx(); + // update the resource and wait for the operation to finish. We also map the resources + // unsynchronized as we can't block or flush any other contexts here as this might cause + // deadlocks. + map = helper_ctx + .map_buffer_unsynchronized(from_res, 0, from_res.width() as i32, RWFlags::RD) + .ok_or(CL_OUT_OF_HOST_MEMORY)?; + } + + let ptr = map.ptr(); + // SAFETY: The application promises, that host_ptr is big enough to hold the entire + // content of the buffer, also `ptr` is the mapped resource containing at least + // `from_res.width()` bytes. Also both pointers do not overlap. + unsafe { + ptr::copy_nonoverlapping(ptr, host_ptr, from_res.width() as usize); + } + } + + if matches!(rw, RWFlags::WR | RWFlags::RW) { + // If the user writes to it it's not valid on any other device anymore. + valid_on.clear(); + } + + if !valid_on.contains(&host_entity) { + // if we update the hostptr, we update all devices having a hostptr allocation. + valid_on.extend_from_slice(&self.hostptr_devs); + } + + Ok(()) + } +} + +pub struct SubAllocation { + mem: Mem, + // offset relative to the actual resource, not relative to `mem`. This saves us a few + // calculations and we only need the total amount anyway. + offset: usize, +} + +/// Abstraction over the memory allocation. It might be a real GPU backing storage or simply a sub +/// allocation over an existing memory object. +enum Allocation { + Resource(ResourceAllocation), + SubAlloc(SubAllocation), +} + +// TODO: - Once it's used for more stuff might make sense to split it into an Image and Buffer +// variant. +// - Instead of doing full migration every time, it could also do it for only parts of the +// allocation. +impl Allocation { + /// Creates a new allocation object assuming the initial data is valid on every device. + pub fn new( + res: HashMap<&'static Device, Arc>, + offset: usize, + host_ptr: *mut c_void, + ) -> Self { + let hostptr_devs = if !host_ptr.is_null() { + res.iter() + // we only add devices we actually have a host ptr resource for + .filter_map(|(&dev, res)| { + res.is_user().then_some(ResourceValidityEntity::Device(dev)) + }) + // and the host itself + .chain([ResourceValidityEntity::Host]) + .collect() + } else { + Vec::new() + }; + + let mut valid_on: Vec<_> = res + .keys() + .copied() + .map(ResourceValidityEntity::Device) + .collect(); + if !host_ptr.is_null() { + valid_on.push(ResourceValidityEntity::Host); + } + + Self::Resource(ResourceAllocation { + valid_on: Mutex::new(valid_on), + res: res, + host_ptr: host_ptr as usize, + hostptr_devs: hostptr_devs, + offset: offset, + }) + } + + fn new_sub(mem: Mem, offset: usize) -> Self { + Self::SubAlloc(SubAllocation { + // we precalculate the entire offset here. + offset: offset + mem.alloc.offset(), + mem: mem, + }) + } + + /// Returns true if the backing storage of the two objects is equal. + fn backing_resource_eq(&self, other: &Self) -> bool { + ptr::eq(self.get_real_resource(), other.get_real_resource()) + } + + /// Follows the sub-allocation chain until it hits a real GPU allocation. + fn get_real_resource(&self) -> &ResourceAllocation { + match self { + Allocation::SubAlloc(sub) => sub.mem.alloc.get_real_resource(), + Allocation::Resource(res) => res, + } + } + + /// Returns the resource associated with `dev` without any data migration. + fn get_res_of_dev(&self, dev: &Device) -> CLResult<&Arc> { + self.get_real_resource() + .res + .get(dev) + .ok_or(CL_OUT_OF_HOST_MEMORY) + } + + /// Returns the resource associated with `ctx.dev` and transparently migrate the data. + fn get_res_for_access(&self, ctx: &QueueContext, rw: RWFlags) -> CLResult<&Arc> { + self.get_real_resource().get_res_for_access(ctx, rw) + } + + /// Migrates the content to the host. Fails if there is no host ptr. + pub fn _migrate_to_hostptr(&self, ctx: &QueueContext, rw: RWFlags) -> CLResult<()> { + self.get_real_resource().migrate_to_hostptr(ctx, rw) + } + + pub fn host_ptr(&self) -> *mut c_void { + let mut host_ptr = self.get_real_resource().host_ptr; + + // we can only apply the offset as long the host_ptr isn't null. + if host_ptr != 0 { + host_ptr += self.offset(); + } + + host_ptr as _ + } + + fn offset(&self) -> usize { + match self { + Allocation::Resource(res) => res.offset, + Allocation::SubAlloc(sub) => sub.offset, + } + } +} + pub enum Mem { Buffer(Arc), Image(Arc), @@ -197,22 +564,17 @@ impl Mem { pub struct MemBase { pub base: CLObjectBase, pub context: Arc, - pub parent: Option, pub mem_type: cl_mem_object_type, pub flags: cl_mem_flags, pub size: usize, - // it's a bit hacky, but storing the pointer as `usize` gives us `Send` and `Sync`. The - // application is required to ensure no data races exist on the memory anyway. - pub host_ptr: usize, pub props: Properties, pub cbs: Mutex>, pub gl_obj: Option, - res: Option>>, + alloc: Allocation, } pub struct Buffer { base: MemBase, - pub offset: usize, maps: Mutex>>, } @@ -395,7 +757,7 @@ impl MemBase { context: Arc, flags: cl_mem_flags, size: usize, - host_ptr: *mut c_void, + mut host_ptr: *mut c_void, props: Properties, ) -> CLResult> { let res_type = if bit_check(flags, CL_MEM_ALLOC_HOST_PTR) { @@ -411,27 +773,24 @@ impl MemBase { res_type, )?; - let host_ptr = if bit_check(flags, CL_MEM_USE_HOST_PTR) { - host_ptr as usize - } else { - 0 - }; + // We can only keep the host_ptr when `CL_MEM_USE_HOST_PTR` is set. + if !bit_check(flags, CL_MEM_USE_HOST_PTR) { + host_ptr = ptr::null_mut() + } + let alloc = Allocation::new(buffer, 0, host_ptr); Ok(Arc::new(Buffer { base: Self { base: CLObjectBase::new(RusticlTypes::Buffer), context: context, - parent: None, mem_type: CL_MEM_OBJECT_BUFFER, flags: flags, size: size, - host_ptr: host_ptr, props: props, gl_obj: None, cbs: Mutex::new(Vec::new()), - res: Some(buffer), + alloc: alloc, }, - offset: 0, maps: Mutex::new(TrackedPointers::new()), })) } @@ -442,27 +801,18 @@ impl MemBase { offset: usize, size: usize, ) -> Arc { - let host_ptr = if parent.host_ptr().is_null() { - 0 - } else { - unsafe { parent.host_ptr().byte_add(offset) as usize } - }; - Arc::new(Buffer { base: Self { base: CLObjectBase::new(RusticlTypes::Buffer), context: parent.context.clone(), - parent: Some(Mem::Buffer(parent)), mem_type: CL_MEM_OBJECT_BUFFER, flags: flags, size: size, - host_ptr: host_ptr, props: Properties::default(), gl_obj: None, cbs: Mutex::new(Vec::new()), - res: None, + alloc: Allocation::new_sub(Mem::Buffer(parent), offset), }, - offset: offset, maps: Mutex::new(TrackedPointers::new()), }) } @@ -474,7 +824,7 @@ impl MemBase { image_format: &cl_image_format, mut image_desc: cl_image_desc, image_elem_size: u8, - host_ptr: *mut c_void, + mut host_ptr: *mut c_void, props: Properties, ) -> CLResult> { // we have to sanitize the image_desc a little for internal use @@ -497,7 +847,9 @@ impl MemBase { ResourceType::Normal }; - let texture = if parent.is_none() { + let alloc = if let Some(parent) = parent { + Allocation::new_sub(parent, 0) + } else { let mut texture = context.create_texture( &image_desc, image_format, @@ -518,15 +870,12 @@ impl MemBase { ) } - Some(texture?) - } else { - None - }; + // We can only keep the host_ptr when `CL_MEM_USE_HOST_PTR` is set. + if !bit_check(flags, CL_MEM_USE_HOST_PTR) { + host_ptr = ptr::null_mut() + } - let host_ptr = if bit_check(flags, CL_MEM_USE_HOST_PTR) { - host_ptr as usize - } else { - 0 + Allocation::new(texture?, 0, host_ptr) }; let pipe_format = image_format.to_pipe_format().unwrap(); @@ -534,15 +883,13 @@ impl MemBase { base: Self { base: CLObjectBase::new(RusticlTypes::Image), context: context, - parent: parent, mem_type: image_desc.image_type, flags: flags, size: image_desc.pixels() * image_format.pixel_size().unwrap() as usize, - host_ptr: host_ptr, props: props, gl_obj: None, cbs: Mutex::new(Vec::new()), - res: texture, + alloc: alloc, }, image_format: *image_format, pipe_format: pipe_format, @@ -635,11 +982,9 @@ impl MemBase { let base = Self { base: CLObjectBase::new(rusticl_type), context: context, - parent: None, mem_type: mem_type, flags: flags, size: gl_mem_props.size(), - host_ptr: 0, props: Properties::default(), gl_obj: Some(GLObject { gl_object_target: gl_export_manager.export_in.target, @@ -648,13 +993,12 @@ impl MemBase { shadow_map: shadow_map, }), cbs: Mutex::new(Vec::new()), - res: Some(texture), + alloc: Allocation::new(texture, gl_mem_props.offset as usize, ptr::null_mut()), }; Ok(if rusticl_type == RusticlTypes::Buffer { Arc::new(Buffer { base: base, - offset: gl_mem_props.offset as usize, maps: Mutex::new(TrackedPointers::new()), }) .into_cl() @@ -693,28 +1037,33 @@ impl MemBase { // this is kinda bogus, because that won't work with system SVM, but the spec wants us to // implement this. pub fn is_svm(&self) -> bool { - self.context.find_svm_alloc(self.host_ptr).is_some() + self.context + .find_svm_alloc(self.host_ptr() as usize) + .is_some() && bit_check(self.flags, CL_MEM_USE_HOST_PTR) } pub fn get_res_of_dev(&self, dev: &Device) -> CLResult<&Arc> { - self.get_parent() - .res - .as_ref() - .and_then(|resources| resources.get(dev)) - .ok_or(CL_OUT_OF_HOST_MEMORY) + self.alloc.get_res_of_dev(dev) } fn get_parent(&self) -> &Self { - if let Some(parent) = &self.parent { - parent - } else { - self + match self.parent() { + Some(parent) => parent, + None => self, + } + } + + /// Returns the parent memory object or None if self isn't a sub allocated memory object. + pub fn parent(&self) -> Option<&Mem> { + match &self.alloc { + Allocation::SubAlloc(sub) => Some(&sub.mem), + Allocation::Resource(_) => None, } } pub fn host_ptr(&self) -> *mut c_void { - self.host_ptr as *mut c_void + self.alloc.host_ptr() } fn is_pure_user_memory(&self, d: &Device) -> CLResult { @@ -774,7 +1123,9 @@ impl Drop for MemBase { impl Buffer { fn apply_offset(&self, offset: usize) -> CLResult { - self.offset.checked_add(offset).ok_or(CL_OUT_OF_HOST_MEMORY) + self.offset() + .checked_add(offset) + .ok_or(CL_OUT_OF_HOST_MEMORY) } pub fn copy_rect( @@ -861,7 +1212,7 @@ impl Buffer { // If image is created from a buffer, use image's slice and row pitch instead let tx_dst; let dst_pitch; - if let Some(Mem::Buffer(buffer)) = &dst.parent { + if let Some(Mem::Buffer(buffer)) = dst.parent() { dst_pitch = [ bpp, dst.image_desc.row_pitch()? as usize, @@ -937,6 +1288,10 @@ impl Buffer { ) } + pub fn offset(&self) -> usize { + self.alloc.offset() + } + pub fn read( &self, ctx: &QueueContext, @@ -1133,7 +1488,7 @@ impl Image { let src_pitch; let tx_src; - if let Some(Mem::Buffer(buffer)) = &self.parent { + if let Some(Mem::Buffer(buffer)) = self.parent() { src_pitch = [ bpp, self.image_desc.row_pitch()? as usize, @@ -1200,7 +1555,7 @@ impl Image { let tx_dst; let dst_pitch; let src_pitch; - if let Some(Mem::Buffer(buffer)) = &self.parent { + if let Some(Mem::Buffer(buffer)) = self.parent() { src_pitch = [ bpp, self.image_desc.row_pitch()? as usize, @@ -1219,7 +1574,7 @@ impl Image { src_pitch = [1, tx_src.row_pitch() as usize, tx_src.slice_pitch()]; } - if let Some(Mem::Buffer(buffer)) = &dst.parent { + if let Some(Mem::Buffer(buffer)) = dst.parent() { // If image is created from a buffer, use image's slice and row pitch instead dst_pitch = [ bpp, @@ -1322,7 +1677,7 @@ impl Image { } pub fn is_parent_buffer(&self) -> bool { - matches!(self.parent, Some(Mem::Buffer(_))) + matches!(self.parent(), Some(Mem::Buffer(_))) } pub fn map( @@ -1411,7 +1766,7 @@ impl Image { let tx; let src_row_pitch; let src_slice_pitch; - if let Some(Mem::Buffer(buffer)) = &self.parent { + if let Some(Mem::Buffer(buffer)) = self.parent() { src_row_pitch = self.image_desc.image_row_pitch; src_slice_pitch = self.image_desc.image_slice_pitch; @@ -1543,7 +1898,7 @@ impl Image { // texture_subdata most likely maps the resource anyway perf_warning!("clEnqueueWriteImage and clEnqueueUnmapMemObject stall the GPU"); - if let Some(Mem::Buffer(buffer)) = &self.parent { + if let Some(Mem::Buffer(buffer)) = self.parent() { let pixel_size = self.image_format.pixel_size().unwrap(); let (offset, size) = CLVec::calc_offset_size( dst_origin,