diff --git a/src/gallium/frontends/rusticl/api/icd.rs b/src/gallium/frontends/rusticl/api/icd.rs index 9d71eec8099..f1d87f366d4 100644 --- a/src/gallium/frontends/rusticl/api/icd.rs +++ b/src/gallium/frontends/rusticl/api/icd.rs @@ -177,11 +177,12 @@ pub type CLResult = Result; #[repr(u32)] pub enum RusticlTypes { // random number - Context = 0xec4cf9a9, + Buffer = 0xec4cf9a9, + Context, Device, Event, + Image, Kernel, - Mem, Program, Queue, Sampler, @@ -194,14 +195,15 @@ impl RusticlTypes { pub const fn from_u32(val: u32) -> Option { let result = match val { - 0xec4cf9a9 => Self::Context, - 0xec4cf9aa => Self::Device, - 0xec4cf9ab => Self::Event, - 0xec4cf9ac => Self::Kernel, - 0xec4cf9ad => Self::Mem, - 0xec4cf9ae => Self::Program, - 0xec4cf9af => Self::Queue, - 0xec4cf9b0 => Self::Sampler, + 0xec4cf9a9 => Self::Buffer, + 0xec4cf9aa => Self::Context, + 0xec4cf9ab => Self::Device, + 0xec4cf9ac => Self::Event, + 0xec4cf9ad => Self::Image, + 0xec4cf9ae => Self::Kernel, + 0xec4cf9af => Self::Program, + 0xec4cf9b0 => Self::Queue, + 0xec4cf9b1 => Self::Sampler, _ => return None, }; debug_assert!(result.u32() == val); diff --git a/src/gallium/frontends/rusticl/api/kernel.rs b/src/gallium/frontends/rusticl/api/kernel.rs index 48aca75b224..6fd1288ea07 100644 --- a/src/gallium/frontends/rusticl/api/kernel.rs +++ b/src/gallium/frontends/rusticl/api/kernel.rs @@ -393,13 +393,13 @@ fn set_kernel_arg( if ptr.is_null() || (*ptr).is_null() { KernelArgValue::None } else { - KernelArgValue::MemObject(Mem::arc_from_raw(*ptr)?) + KernelArgValue::Buffer(Buffer::arc_from_raw(*ptr)?) } } KernelArgType::MemLocal => KernelArgValue::LocalMem(arg_size), KernelArgType::Image | KernelArgType::RWImage | KernelArgType::Texture => { let img: *const cl_mem = arg_value.cast(); - KernelArgValue::MemObject(Mem::arc_from_raw(*img)?) + KernelArgValue::Image(Image::arc_from_raw(*img)?) } KernelArgType::Sampler => { let ptr: *const cl_sampler = arg_value.cast(); diff --git a/src/gallium/frontends/rusticl/api/memory.rs b/src/gallium/frontends/rusticl/api/memory.rs index 0c8e10ab224..96fc2b0d786 100644 --- a/src/gallium/frontends/rusticl/api/memory.rs +++ b/src/gallium/frontends/rusticl/api/memory.rs @@ -81,7 +81,7 @@ fn validate_map_flags_common(map_flags: cl_mem_flags) -> CLResult<()> { Ok(()) } -fn validate_map_flags(m: &Mem, map_flags: cl_mem_flags) -> CLResult<()> { +fn validate_map_flags(m: &MemBase, map_flags: cl_mem_flags) -> CLResult<()> { validate_map_flags_common(map_flags)?; // CL_INVALID_OPERATION if buffer has been created with CL_MEM_HOST_WRITE_ONLY or @@ -105,7 +105,7 @@ fn filter_image_access_flags(flags: cl_mem_flags) -> cl_mem_flags { as cl_mem_flags } -fn inherit_mem_flags(mut flags: cl_mem_flags, mem: &Mem) -> cl_mem_flags { +fn inherit_mem_flags(mut flags: cl_mem_flags, mem: &MemBase) -> cl_mem_flags { let read_write_mask = cl_bitfield::from( CL_MEM_READ_WRITE | CL_MEM_WRITE_ONLY | @@ -186,7 +186,7 @@ fn validate_host_ptr(host_ptr: *mut ::std::os::raw::c_void, flags: cl_mem_flags) Ok(()) } -fn validate_matching_buffer_flags(mem: &Mem, flags: cl_mem_flags) -> CLResult<()> { +fn validate_matching_buffer_flags(mem: &MemBase, flags: cl_mem_flags) -> CLResult<()> { // CL_INVALID_VALUE if an image is being created from another memory object (buffer or image) // under one of the following circumstances: // @@ -215,15 +215,16 @@ fn validate_matching_buffer_flags(mem: &Mem, flags: cl_mem_flags) -> CLResult<() #[cl_info_entrypoint(cl_get_mem_object_info)] impl CLInfo for cl_mem { fn query(&self, q: cl_mem_info, _: &[u8]) -> CLResult>> { - let mem = Mem::ref_from_raw(*self)?; + let mem = MemBase::ref_from_raw(*self)?; Ok(match *q { CL_MEM_ASSOCIATED_MEMOBJECT => { let ptr = match mem.parent.as_ref() { // Note we use as_ptr here which doesn't increase the reference count. - Some(parent) => Arc::as_ptr(parent), - None => ptr::null(), + Some(Mem::Buffer(buffer)) => cl_mem::from_ptr(Arc::as_ptr(buffer)), + Some(Mem::Image(image)) => cl_mem::from_ptr(Arc::as_ptr(image)), + None => ptr::null_mut(), }; - cl_prop::(cl_mem::from_ptr(ptr)) + cl_prop::(ptr.cast()) } CL_MEM_CONTEXT => { // Note we use as_ptr here which doesn't increase the reference count. @@ -236,7 +237,11 @@ impl CLInfo for cl_mem { CL_MEM_HOST_PTR => cl_prop::<*mut c_void>(mem.host_ptr), CL_MEM_OFFSET => cl_prop::(mem.offset), CL_MEM_PROPERTIES => cl_prop::<&Vec>(&mem.props), - CL_MEM_REFERENCE_COUNT => cl_prop::(Mem::refcnt(*self)?), + CL_MEM_REFERENCE_COUNT => cl_prop::(if mem.is_buffer() { + Buffer::refcnt(*self)? + } else { + Image::refcnt(*self)? + }), CL_MEM_SIZE => cl_prop::(mem.size), CL_MEM_TYPE => cl_prop::(mem.mem_type), CL_MEM_USES_SVM_POINTER | CL_MEM_USES_SVM_POINTER_ARM => { @@ -281,7 +286,7 @@ fn create_buffer_with_properties( return Err(CL_INVALID_PROPERTY); } - Ok(Mem::new_buffer(c, flags, size, host_ptr, props)?.into_cl()) + Ok(MemBase::new_buffer(c, flags, size, host_ptr, props)?.into_cl()) } #[cl_entrypoint] @@ -301,7 +306,7 @@ fn create_sub_buffer( buffer_create_type: cl_buffer_create_type, buffer_create_info: *const ::std::os::raw::c_void, ) -> CLResult { - let b = Mem::arc_from_raw(buffer)?; + let b = Buffer::arc_from_raw(buffer)?; // CL_INVALID_MEM_OBJECT if buffer ... is a sub-buffer object. if b.parent.is_some() { @@ -340,7 +345,7 @@ fn create_sub_buffer( _ => return Err(CL_INVALID_VALUE), }; - Ok(Mem::new_sub_buffer(b, flags, offset, size).into_cl()) + Ok(MemBase::new_sub_buffer(b, flags, offset, size).into_cl()) // TODO // CL_MISALIGNED_SUB_BUFFER_OFFSET if there are no devices in context associated with buffer for which the origin field of the cl_buffer_region structure passed in buffer_create_info is aligned to the CL_DEVICE_MEM_BASE_ADDR_ALIGN value. @@ -352,7 +357,7 @@ fn set_mem_object_destructor_callback( pfn_notify: Option, user_data: *mut ::std::os::raw::c_void, ) -> CLResult<()> { - let m = Mem::ref_from_raw(memobj)?; + let m = MemBase::ref_from_raw(memobj)?; // SAFETY: The requirements on `MemCB::new` match the requirements // imposed by the OpenCL specification. It is the caller's duty to uphold them. @@ -391,7 +396,7 @@ fn validate_image_desc( host_ptr: *mut ::std::os::raw::c_void, elem_size: usize, devs: &[&Device], -) -> CLResult<(cl_image_desc, Option>)> { +) -> CLResult<(cl_image_desc, Option)> { // CL_INVALID_IMAGE_DESCRIPTOR if values specified in image_desc are not valid const err: cl_int = CL_INVALID_IMAGE_DESCRIPTOR; @@ -461,7 +466,7 @@ fn validate_image_desc( // TODO: cl_khr_image2d_from_buffer is an optional feature let p = unsafe { &desc.anon_1.mem_object }; let parent = if !p.is_null() { - let p = Mem::arc_from_raw(*p)?; + let p = MemBase::arc_from_raw(*p)?; if !match desc.image_type { CL_MEM_OBJECT_IMAGE1D_BUFFER => p.is_buffer(), CL_MEM_OBJECT_IMAGE2D => { @@ -539,7 +544,7 @@ fn validate_image_desc( Ok((desc, parent)) } -fn validate_image_bounds(i: &Mem, origin: CLVec, region: CLVec) -> CLResult<()> { +fn validate_image_bounds(i: &MemBase, origin: CLVec, region: CLVec) -> CLResult<()> { let dims = i.image_desc.dims_with_array(); let bound = region + origin; if bound > i.image_desc.size() { @@ -593,7 +598,7 @@ fn validate_buffer( // the specified memory objects data store are modified, those changes are reflected in the // contents of the image object and vice-versa at corresponding synchronization points. if !mem_object.is_null() { - let mem = Mem::ref_from_raw(mem_object)?; + let mem = MemBase::ref_from_raw(mem_object)?; match mem.mem_type { CL_MEM_OBJECT_BUFFER => { @@ -700,7 +705,7 @@ fn validate_buffer( #[cl_info_entrypoint(cl_get_image_info)] impl CLInfo for cl_mem { fn query(&self, q: cl_image_info, _: &[u8]) -> CLResult>> { - let mem = Mem::ref_from_raw(*self)?; + let mem = MemBase::ref_from_raw(*self)?; Ok(match *q { CL_IMAGE_ARRAY_SIZE => cl_prop::(mem.image_desc.image_array_size), CL_IMAGE_BUFFER => cl_prop::(unsafe { mem.image_desc.anon_1.buffer }), @@ -770,7 +775,7 @@ fn create_image_with_properties( return Err(CL_INVALID_PROPERTY); } - Ok(Mem::new_image( + Ok(MemBase::new_image( c, parent, desc.image_type, @@ -1023,7 +1028,7 @@ fn enqueue_read_buffer( event: *mut cl_event, ) -> CLResult<()> { let q = Queue::arc_from_raw(command_queue)?; - let b = Mem::arc_from_raw(buffer)?; + let b = Buffer::arc_from_raw(buffer)?; let block = check_cl_bool(blocking_read).ok_or(CL_INVALID_VALUE)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; @@ -1076,7 +1081,7 @@ fn enqueue_write_buffer( event: *mut cl_event, ) -> CLResult<()> { let q = Queue::arc_from_raw(command_queue)?; - let b = Mem::arc_from_raw(buffer)?; + let b = Buffer::arc_from_raw(buffer)?; let block = check_cl_bool(blocking_write).ok_or(CL_INVALID_VALUE)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; @@ -1129,8 +1134,8 @@ fn enqueue_copy_buffer( event: *mut cl_event, ) -> CLResult<()> { let q = Queue::arc_from_raw(command_queue)?; - let src = Mem::arc_from_raw(src_buffer)?; - let dst = Mem::arc_from_raw(dst_buffer)?; + let src = Buffer::arc_from_raw(src_buffer)?; + let dst = Buffer::arc_from_raw(dst_buffer)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; // CL_INVALID_CONTEXT if the context associated with command_queue, src_buffer and dst_buffer @@ -1203,7 +1208,7 @@ fn enqueue_read_buffer_rect( ) -> CLResult<()> { let block = check_cl_bool(blocking_read).ok_or(CL_INVALID_VALUE)?; let q = Queue::arc_from_raw(command_queue)?; - let buf = Mem::arc_from_raw(buffer)?; + let buf = Buffer::arc_from_raw(buffer)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; // CL_INVALID_OPERATION if clEnqueueReadBufferRect is called on buffer which has been created @@ -1326,7 +1331,7 @@ fn enqueue_write_buffer_rect( ) -> CLResult<()> { let block = check_cl_bool(blocking_write).ok_or(CL_INVALID_VALUE)?; let q = Queue::arc_from_raw(command_queue)?; - let buf = Mem::arc_from_raw(buffer)?; + let buf = Buffer::arc_from_raw(buffer)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; // CL_INVALID_OPERATION if clEnqueueWriteBufferRect is called on buffer which has been created @@ -1447,8 +1452,8 @@ fn enqueue_copy_buffer_rect( event: *mut cl_event, ) -> CLResult<()> { let q = Queue::arc_from_raw(command_queue)?; - let src = Mem::arc_from_raw(src_buffer)?; - let dst = Mem::arc_from_raw(dst_buffer)?; + let src = Buffer::arc_from_raw(src_buffer)?; + let dst = Buffer::arc_from_raw(dst_buffer)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; // CL_INVALID_VALUE if src_origin, dst_origin, or region is NULL. @@ -1582,7 +1587,7 @@ fn enqueue_fill_buffer( event: *mut cl_event, ) -> CLResult<()> { let q = Queue::arc_from_raw(command_queue)?; - let b = Mem::arc_from_raw(buffer)?; + let b = Buffer::arc_from_raw(buffer)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; // CL_INVALID_VALUE if offset or offset + size require accessing elements outside the buffer @@ -1636,7 +1641,7 @@ fn enqueue_map_buffer( event: *mut cl_event, ) -> CLResult<*mut c_void> { let q = Queue::arc_from_raw(command_queue)?; - let b = Mem::arc_from_raw(buffer)?; + let b = Buffer::arc_from_raw(buffer)?; let block = check_cl_bool(blocking_map).ok_or(CL_INVALID_VALUE)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; @@ -1692,7 +1697,7 @@ fn enqueue_read_image( event: *mut cl_event, ) -> CLResult<()> { let q = Queue::arc_from_raw(command_queue)?; - let i = Mem::arc_from_raw(image)?; + let i = Image::arc_from_raw(image)?; let block = check_cl_bool(blocking_read).ok_or(CL_INVALID_VALUE)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; let pixel_size = i.image_format.pixel_size().unwrap() as usize; @@ -1783,7 +1788,7 @@ fn enqueue_write_image( event: *mut cl_event, ) -> CLResult<()> { let q = Queue::arc_from_raw(command_queue)?; - let i = Mem::arc_from_raw(image)?; + let i = Image::arc_from_raw(image)?; let block = check_cl_bool(blocking_write).ok_or(CL_INVALID_VALUE)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; let pixel_size = i.image_format.pixel_size().unwrap() as usize; @@ -1872,8 +1877,8 @@ fn enqueue_copy_image( event: *mut cl_event, ) -> CLResult<()> { let q = Queue::arc_from_raw(command_queue)?; - let src_image = Mem::arc_from_raw(src_image)?; - let dst_image = Mem::arc_from_raw(dst_image)?; + let src_image = Image::arc_from_raw(src_image)?; + let dst_image = Image::arc_from_raw(dst_image)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; // CL_INVALID_CONTEXT if the context associated with command_queue, src_image and dst_image are not the same @@ -1932,7 +1937,7 @@ fn enqueue_fill_image( event: *mut cl_event, ) -> CLResult<()> { let q = Queue::arc_from_raw(command_queue)?; - let i = Mem::arc_from_raw(image)?; + let i = Image::arc_from_raw(image)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; // CL_INVALID_CONTEXT if the context associated with command_queue and image are not the same @@ -1985,8 +1990,8 @@ fn enqueue_copy_buffer_to_image( event: *mut cl_event, ) -> CLResult<()> { let q = Queue::arc_from_raw(command_queue)?; - let src = Mem::arc_from_raw(src_buffer)?; - let dst = Mem::arc_from_raw(dst_image)?; + let src = Buffer::arc_from_raw(src_buffer)?; + let dst = Image::arc_from_raw(dst_image)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; // CL_INVALID_CONTEXT if the context associated with command_queue, src_buffer and dst_image @@ -2041,8 +2046,8 @@ fn enqueue_copy_image_to_buffer( event: *mut cl_event, ) -> CLResult<()> { let q = Queue::arc_from_raw(command_queue)?; - let src = Mem::arc_from_raw(src_image)?; - let dst = Mem::arc_from_raw(dst_buffer)?; + let src = Image::arc_from_raw(src_image)?; + let dst = Buffer::arc_from_raw(dst_buffer)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; // CL_INVALID_CONTEXT if the context associated with command_queue, src_image and dst_buffer @@ -2100,7 +2105,7 @@ fn enqueue_map_image( event: *mut cl_event, ) -> CLResult<*mut ::std::os::raw::c_void> { let q = Queue::arc_from_raw(command_queue)?; - let i = Mem::arc_from_raw(image)?; + let i = Image::arc_from_raw(image)?; let block = check_cl_bool(blocking_map).ok_or(CL_INVALID_VALUE)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; @@ -2167,12 +2172,22 @@ fn enqueue_map_image( #[cl_entrypoint] fn retain_mem_object(mem: cl_mem) -> CLResult<()> { - Mem::retain(mem) + let m = MemBase::ref_from_raw(mem)?; + match m.base.get_type()? { + RusticlTypes::Buffer => Buffer::retain(mem), + RusticlTypes::Image => Image::retain(mem), + _ => Err(CL_INVALID_MEM_OBJECT), + } } #[cl_entrypoint] fn release_mem_object(mem: cl_mem) -> CLResult<()> { - Mem::release(mem) + let m = MemBase::ref_from_raw(mem)?; + match m.base.get_type()? { + RusticlTypes::Buffer => Buffer::release(mem), + RusticlTypes::Image => Image::release(mem), + _ => Err(CL_INVALID_MEM_OBJECT), + } } #[cl_entrypoint] @@ -2185,7 +2200,7 @@ fn enqueue_unmap_mem_object( event: *mut cl_event, ) -> CLResult<()> { let q = Queue::arc_from_raw(command_queue)?; - let m = Mem::arc_from_raw(memobj)?; + let m = MemBase::arc_from_raw(memobj)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; // CL_INVALID_CONTEXT if context associated with command_queue and memobj are not the same @@ -2221,7 +2236,7 @@ fn enqueue_migrate_mem_objects( ) -> CLResult<()> { let q = Queue::arc_from_raw(command_queue)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; - let bufs = Mem::refs_from_arr(mem_objects, num_mem_objects)?; + let bufs = MemBase::refs_from_arr(mem_objects, num_mem_objects)?; // CL_INVALID_VALUE if num_mem_objects is zero or if mem_objects is NULL. if bufs.is_empty() { @@ -2962,7 +2977,7 @@ fn create_pipe( #[cl_info_entrypoint(cl_get_gl_texture_info)] impl CLInfo for cl_mem { fn query(&self, q: cl_gl_texture_info, _: &[u8]) -> CLResult>> { - let mem = Mem::ref_from_raw(*self)?; + let mem = MemBase::ref_from_raw(*self)?; Ok(match *q { CL_GL_MIPMAP_LEVEL => cl_prop::(0), CL_GL_TEXTURE_TARGET => cl_prop::( @@ -3006,7 +3021,7 @@ fn create_from_gl( let gl_export_manager = gl_ctx_manager.export_object(&c, target, flags as u32, miplevel, texture)?; - Ok(Mem::from_gl(c, flags, &gl_export_manager)?.into_cl()) + Ok(MemBase::from_gl(c, flags, &gl_export_manager)?) } else { Err(CL_INVALID_CONTEXT) } @@ -3087,7 +3102,7 @@ fn get_gl_object_info( gl_object_type: *mut cl_gl_object_type, gl_object_name: *mut cl_GLuint, ) -> CLResult<()> { - let m = Mem::ref_from_raw(memobj)?; + let m = MemBase::ref_from_raw(memobj)?; match &m.gl_obj { Some(gl_obj) => { @@ -3114,7 +3129,7 @@ fn enqueue_acquire_gl_objects( ) -> CLResult<()> { let q = Queue::arc_from_raw(command_queue)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; - let objs = Mem::arcs_from_arr(mem_objects, num_objects)?; + let objs = MemBase::arcs_from_arr(mem_objects, num_objects)?; let gl_ctx_manager = &q.context.gl_ctx_manager; // CL_INVALID_CONTEXT if context associated with command_queue was not created from an OpenGL context @@ -3148,7 +3163,7 @@ fn enqueue_release_gl_objects( ) -> CLResult<()> { let q = Queue::arc_from_raw(command_queue)?; let evs = event_list_from_cl(&q, num_events_in_wait_list, event_wait_list)?; - let objs = Mem::arcs_from_arr(mem_objects, num_objects)?; + let objs = MemBase::arcs_from_arr(mem_objects, num_objects)?; let gl_ctx_manager = &q.context.gl_ctx_manager; // CL_INVALID_CONTEXT if context associated with command_queue was not created from an OpenGL context diff --git a/src/gallium/frontends/rusticl/api/types.rs b/src/gallium/frontends/rusticl/api/types.rs index 81e10bd686a..64f82970da1 100644 --- a/src/gallium/frontends/rusticl/api/types.rs +++ b/src/gallium/frontends/rusticl/api/types.rs @@ -2,7 +2,7 @@ use crate::api::icd::CLResult; use crate::api::icd::ReferenceCountedAPIPointer; use crate::core::context::Context; use crate::core::event::Event; -use crate::core::memory::Mem; +use crate::core::memory::MemBase; use crate::core::program::Program; use crate::core::queue::Queue; @@ -153,7 +153,7 @@ cl_callback!( ); impl MemCB { - pub fn call(self, mem: &Mem) { + pub fn call(self, mem: &MemBase) { let cl = cl_mem::from_ptr(mem); // SAFETY: `cl` must have pointed to an OpenCL context, which is where we just got it from. // All other requirements are covered by this callback's type invariants. diff --git a/src/gallium/frontends/rusticl/core/gl.rs b/src/gallium/frontends/rusticl/core/gl.rs index c7d38248645..7f2cc500164 100644 --- a/src/gallium/frontends/rusticl/core/gl.rs +++ b/src/gallium/frontends/rusticl/core/gl.rs @@ -456,18 +456,17 @@ pub fn create_shadow_slice( Ok(slice) } -pub fn copy_cube_to_slice( - q: &Arc, - ctx: &PipeContext, - mem_objects: &[Arc], -) -> CLResult<()> { +pub fn copy_cube_to_slice(q: &Arc, ctx: &PipeContext, mem_objects: &[Mem]) -> CLResult<()> { for mem in mem_objects { - let gl_obj = mem.gl_obj.as_ref().unwrap(); + let Mem::Image(image) = mem else { + continue; + }; + let gl_obj = image.gl_obj.as_ref().unwrap(); if !is_cube_map_face(gl_obj.gl_object_target) { continue; } - let width = mem.image_desc.image_width; - let height = mem.image_desc.image_height; + let width = image.image_desc.image_width; + let height = image.image_desc.image_height; // Fill in values for doing the copy let idx = get_array_slice_idx(gl_obj.gl_object_target); @@ -476,7 +475,7 @@ pub fn copy_cube_to_slice( let region = CLVec::::new([width, height, 1]); let src_bx = create_pipe_box(src_origin, region, CL_MEM_OBJECT_IMAGE2D_ARRAY)?; - let cl_res = mem.get_res_of_dev(q.device)?; + let cl_res = image.get_res_of_dev(q.device)?; let gl_res = gl_obj.shadow_map.as_ref().unwrap().get(cl_res).unwrap(); ctx.resource_copy_region(gl_res.as_ref(), cl_res.as_ref(), &dst_offset, &src_bx); @@ -485,18 +484,17 @@ pub fn copy_cube_to_slice( Ok(()) } -pub fn copy_slice_to_cube( - q: &Arc, - ctx: &PipeContext, - mem_objects: &[Arc], -) -> CLResult<()> { +pub fn copy_slice_to_cube(q: &Arc, ctx: &PipeContext, mem_objects: &[Mem]) -> CLResult<()> { for mem in mem_objects { - let gl_obj = mem.gl_obj.as_ref().unwrap(); + let Mem::Image(image) = mem else { + continue; + }; + let gl_obj = image.gl_obj.as_ref().unwrap(); if !is_cube_map_face(gl_obj.gl_object_target) { continue; } - let width = mem.image_desc.image_width; - let height = mem.image_desc.image_height; + let width = image.image_desc.image_width; + let height = image.image_desc.image_height; // Fill in values for doing the copy let idx = get_array_slice_idx(gl_obj.gl_object_target) as u32; @@ -505,7 +503,7 @@ pub fn copy_slice_to_cube( let region = CLVec::::new([width, height, 1]); let src_bx = create_pipe_box(src_origin, region, CL_MEM_OBJECT_IMAGE2D_ARRAY)?; - let cl_res = mem.get_res_of_dev(q.device)?; + let cl_res = image.get_res_of_dev(q.device)?; let gl_res = gl_obj.shadow_map.as_ref().unwrap().get(cl_res).unwrap(); ctx.resource_copy_region(cl_res.as_ref(), gl_res.as_ref(), &dst_offset, &src_bx); diff --git a/src/gallium/frontends/rusticl/core/kernel.rs b/src/gallium/frontends/rusticl/core/kernel.rs index 8f4a5798a31..fc4c387dce0 100644 --- a/src/gallium/frontends/rusticl/core/kernel.rs +++ b/src/gallium/frontends/rusticl/core/kernel.rs @@ -31,10 +31,11 @@ use std::sync::Arc; #[derive(Clone)] pub enum KernelArgValue { None, + Buffer(Arc), Constant(Vec), - MemObject(Arc), - Sampler(Arc), + Image(Arc), LocalMem(usize), + Sampler(Arc), } #[derive(Hash, PartialEq, Eq, Clone, Copy)] @@ -903,59 +904,61 @@ impl Kernel { } match val.borrow().as_ref().unwrap() { KernelArgValue::Constant(c) => input.extend_from_slice(c), - KernelArgValue::MemObject(mem) => { - let res = mem.get_res_of_dev(q.device)?; - // If resource is a buffer and mem a 2D image, the 2d image was created from a - // buffer. Use strides and dimensions of 2d image + KernelArgValue::Buffer(buffer) => { + let res = buffer.get_res_of_dev(q.device)?; + if q.device.address_bits() == 64 { + input.extend_from_slice(&buffer.offset.to_ne_bytes()); + } else { + input.extend_from_slice(&(buffer.offset as u32).to_ne_bytes()); + } + resource_info.push((res.clone(), arg.offset)); + } + KernelArgValue::Image(image) => { + let res = image.get_res_of_dev(q.device)?; + + // If resource is a buffer, the image was created from a buffer. Use strides and + // dimensions of the image then. let app_img_info = - if res.as_ref().is_buffer() && mem.mem_type == CL_MEM_OBJECT_IMAGE2D { + if res.as_ref().is_buffer() && image.mem_type == CL_MEM_OBJECT_IMAGE2D { Some(AppImgInfo::new( - mem.image_desc.row_pitch()? / mem.image_elem_size as u32, - mem.image_desc.width()?, - mem.image_desc.height()?, + image.image_desc.row_pitch()? / image.image_elem_size as u32, + image.image_desc.width()?, + image.image_desc.height()?, )) } else { None }; - if mem.is_buffer() { - if q.device.address_bits() == 64 { - input.extend_from_slice(&mem.offset.to_ne_bytes()); - } else { - input.extend_from_slice(&(mem.offset as u32).to_ne_bytes()); - } - resource_info.push((res.clone(), arg.offset)); + + let format = image.pipe_format; + let (formats, orders) = if arg.kind == KernelArgType::Image { + iviews.push(res.pipe_image_view( + format, + false, + image.pipe_image_host_access(), + app_img_info.as_ref(), + )); + (&mut img_formats, &mut img_orders) + } else if arg.kind == KernelArgType::RWImage { + iviews.push(res.pipe_image_view( + format, + true, + image.pipe_image_host_access(), + app_img_info.as_ref(), + )); + (&mut img_formats, &mut img_orders) } else { - let format = mem.pipe_format; - let (formats, orders) = if arg.kind == KernelArgType::Image { - iviews.push(res.pipe_image_view( - format, - false, - mem.pipe_image_host_access(), - app_img_info.as_ref(), - )); - (&mut img_formats, &mut img_orders) - } else if arg.kind == KernelArgType::RWImage { - iviews.push(res.pipe_image_view( - format, - true, - mem.pipe_image_host_access(), - app_img_info.as_ref(), - )); - (&mut img_formats, &mut img_orders) - } else { - sviews.push((res.clone(), format, app_img_info)); - (&mut tex_formats, &mut tex_orders) - }; + sviews.push((res.clone(), format, app_img_info)); + (&mut tex_formats, &mut tex_orders) + }; - let binding = arg.binding as usize; - assert!(binding >= formats.len()); + let binding = arg.binding as usize; + assert!(binding >= formats.len()); - formats.resize(binding, 0); - orders.resize(binding, 0); + formats.resize(binding, 0); + orders.resize(binding, 0); - formats.push(mem.image_format.image_channel_data_type as u16); - orders.push(mem.image_format.image_channel_order as u16); - } + formats.push(image.image_format.image_channel_data_type as u16); + orders.push(image.image_format.image_channel_order as u16); } KernelArgValue::LocalMem(size) => { // TODO 32 bit diff --git a/src/gallium/frontends/rusticl/core/memory.rs b/src/gallium/frontends/rusticl/core/memory.rs index 4a3176b667c..ef0f6bd036c 100644 --- a/src/gallium/frontends/rusticl/core/memory.rs +++ b/src/gallium/frontends/rusticl/core/memory.rs @@ -8,6 +8,7 @@ use crate::core::gl::*; use crate::core::queue::*; use crate::core::util::*; use crate::impl_cl_type_trait; +use crate::impl_cl_type_trait_base; use mesa_rust::pipe::context::*; use mesa_rust::pipe::resource::*; @@ -25,6 +26,7 @@ use std::convert::TryInto; use std::mem; use std::mem::size_of; use std::ops::AddAssign; +use std::ops::Deref; use std::os::raw::c_void; use std::ptr; use std::sync::Arc; @@ -111,10 +113,26 @@ impl Mappings { } } -pub struct Mem { +pub enum Mem { + Buffer(Arc), + Image(Arc), +} + +impl Deref for Mem { + type Target = MemBase; + + fn deref(&self) -> &Self::Target { + match self { + Self::Buffer(b) => &b.base, + Self::Image(i) => &i.base, + } + } +} + +pub struct MemBase { pub base: CLObjectBase, pub context: Arc, - pub parent: Option>, + pub parent: Option, pub mem_type: cl_mem_object_type, pub flags: cl_mem_flags, pub size: usize, @@ -131,7 +149,33 @@ pub struct Mem { maps: Mutex, } -impl_cl_type_trait!(cl_mem, Mem, CL_INVALID_MEM_OBJECT); +pub struct Buffer { + base: MemBase, +} + +pub struct Image { + base: MemBase, +} + +impl Deref for Buffer { + type Target = MemBase; + + fn deref(&self) -> &Self::Target { + &self.base + } +} + +impl Deref for Image { + type Target = MemBase; + + fn deref(&self) -> &Self::Target { + &self.base + } +} + +impl_cl_type_trait_base!(cl_mem, MemBase, [Buffer, Image], CL_INVALID_MEM_OBJECT); +impl_cl_type_trait!(cl_mem, Buffer, CL_INVALID_MEM_OBJECT, base.base); +impl_cl_type_trait!(cl_mem, Image, CL_INVALID_MEM_OBJECT, base.base); pub trait CLImageDescInfo { fn type_info(&self) -> (u8, bool); @@ -274,14 +318,14 @@ fn can_map_directly(dev: &Device, res: &PipeResource) -> bool { && (res.is_buffer() || res.is_linear()) } -impl Mem { +impl MemBase { pub fn new_buffer( context: Arc, flags: cl_mem_flags, size: usize, host_ptr: *mut c_void, props: Vec, - ) -> CLResult> { + ) -> CLResult> { let res_type = if bit_check(flags, CL_MEM_ALLOC_HOST_PTR) { ResourceType::Staging } else { @@ -301,63 +345,67 @@ impl Mem { ptr::null_mut() }; - Ok(Arc::new(Self { - base: CLObjectBase::new(RusticlTypes::Mem), - context: context, - parent: None, - mem_type: CL_MEM_OBJECT_BUFFER, - flags: flags, - size: size, - offset: 0, - host_ptr: host_ptr, - image_format: cl_image_format::default(), - pipe_format: pipe_format::PIPE_FORMAT_NONE, - image_desc: cl_image_desc::default(), - image_elem_size: 0, - props: props, - gl_obj: None, - cbs: Mutex::new(Vec::new()), - res: Some(buffer), - maps: Mappings::new(), + Ok(Arc::new(Buffer { + base: Self { + base: CLObjectBase::new(RusticlTypes::Buffer), + context: context, + parent: None, + mem_type: CL_MEM_OBJECT_BUFFER, + flags: flags, + size: size, + offset: 0, + host_ptr: host_ptr, + image_format: cl_image_format::default(), + pipe_format: pipe_format::PIPE_FORMAT_NONE, + image_desc: cl_image_desc::default(), + image_elem_size: 0, + props: props, + gl_obj: None, + cbs: Mutex::new(Vec::new()), + res: Some(buffer), + maps: Mappings::new(), + }, })) } pub fn new_sub_buffer( - parent: Arc, + parent: Arc, flags: cl_mem_flags, offset: usize, size: usize, - ) -> Arc { + ) -> Arc { let host_ptr = if parent.host_ptr.is_null() { ptr::null_mut() } else { unsafe { parent.host_ptr.add(offset) } }; - Arc::new(Self { - base: CLObjectBase::new(RusticlTypes::Mem), - context: parent.context.clone(), - parent: Some(parent), - mem_type: CL_MEM_OBJECT_BUFFER, - flags: flags, - size: size, - offset: offset, - host_ptr: host_ptr, - image_format: cl_image_format::default(), - pipe_format: pipe_format::PIPE_FORMAT_NONE, - image_desc: cl_image_desc::default(), - image_elem_size: 0, - props: Vec::new(), - gl_obj: None, - cbs: Mutex::new(Vec::new()), - res: None, - maps: Mappings::new(), + Arc::new(Buffer { + base: Self { + base: CLObjectBase::new(RusticlTypes::Buffer), + context: parent.context.clone(), + parent: Some(Mem::Buffer(parent)), + mem_type: CL_MEM_OBJECT_BUFFER, + flags: flags, + size: size, + offset: offset, + host_ptr: host_ptr, + image_format: cl_image_format::default(), + pipe_format: pipe_format::PIPE_FORMAT_NONE, + image_desc: cl_image_desc::default(), + image_elem_size: 0, + props: Vec::new(), + gl_obj: None, + cbs: Mutex::new(Vec::new()), + res: None, + maps: Mappings::new(), + }, }) } pub fn new_image( context: Arc, - parent: Option>, + parent: Option, mem_type: cl_mem_object_type, flags: cl_mem_flags, image_format: &cl_image_format, @@ -365,7 +413,7 @@ impl Mem { image_elem_size: u8, host_ptr: *mut c_void, props: Vec, - ) -> CLResult> { + ) -> CLResult> { // we have to sanitize the image_desc a little for internal use let api_image_desc = image_desc; let dims = image_desc.dims(); @@ -419,32 +467,52 @@ impl Mem { }; let pipe_format = image_format.to_pipe_format().unwrap(); - Ok(Arc::new(Self { - base: CLObjectBase::new(RusticlTypes::Mem), - context: context, - parent: parent, - mem_type: mem_type, - flags: flags, - size: image_desc.pixels() * image_format.pixel_size().unwrap() as usize, - offset: 0, - host_ptr: host_ptr, - image_format: *image_format, - pipe_format: pipe_format, - image_desc: api_image_desc, - image_elem_size: image_elem_size, - props: props, - gl_obj: None, - cbs: Mutex::new(Vec::new()), - res: texture, - maps: Mappings::new(), + Ok(Arc::new(Image { + base: Self { + base: CLObjectBase::new(RusticlTypes::Image), + context: context, + parent: parent, + mem_type: mem_type, + flags: flags, + size: image_desc.pixels() * image_format.pixel_size().unwrap() as usize, + offset: 0, + host_ptr: host_ptr, + image_format: *image_format, + pipe_format: pipe_format, + image_desc: api_image_desc, + image_elem_size: image_elem_size, + props: props, + gl_obj: None, + cbs: Mutex::new(Vec::new()), + res: texture, + maps: Mappings::new(), + }, })) } + pub fn arc_from_raw(ptr: cl_mem) -> CLResult { + let mem = Self::ref_from_raw(ptr)?; + match mem.base.get_type()? { + RusticlTypes::Buffer => Ok(Mem::Buffer(Buffer::arc_from_raw(ptr)?)), + RusticlTypes::Image => Ok(Mem::Image(Image::arc_from_raw(ptr)?)), + _ => Err(CL_INVALID_MEM_OBJECT), + } + } + + pub fn arcs_from_arr(objs: *const cl_mem, count: u32) -> CLResult> { + let count = count as usize; + let mut res = Vec::with_capacity(count); + for i in 0..count { + res.push(Self::arc_from_raw(unsafe { *objs.add(i) })?); + } + Ok(res) + } + pub fn from_gl( context: Arc, flags: cl_mem_flags, gl_export_manager: &GLExportManager, - ) -> CLResult> { + ) -> CLResult { let export_in = &gl_export_manager.export_in; let export_out = &gl_export_manager.export_out; @@ -452,12 +520,20 @@ impl Mem { let gl_mem_props = gl_export_manager.get_gl_mem_props()?; // Handle Buffers - let (image_format, pipe_format) = if gl_export_manager.is_gl_buffer() { - (cl_image_format::default(), pipe_format::PIPE_FORMAT_NONE) + let (image_format, pipe_format, rusticl_type) = if gl_export_manager.is_gl_buffer() { + ( + cl_image_format::default(), + pipe_format::PIPE_FORMAT_NONE, + RusticlTypes::Buffer, + ) } else { let image_format = format_from_gl(export_out.internal_format).ok_or(CL_OUT_OF_HOST_MEMORY)?; - (image_format, image_format.to_pipe_format().unwrap()) + ( + image_format, + image_format.to_pipe_format().unwrap(), + RusticlTypes::Image, + ) }; let imported_gl_tex = context.import_gl_buffer( @@ -513,8 +589,9 @@ impl Mem { if mem_type != CL_MEM_OBJECT_BUFFER { assert_eq!(gl_mem_props.offset, 0); } - Ok(Arc::new(Self { - base: CLObjectBase::new(RusticlTypes::Mem), + + let base = Self { + base: CLObjectBase::new(rusticl_type), context: context, parent: None, mem_type: mem_type, @@ -531,7 +608,12 @@ impl Mem { cbs: Mutex::new(Vec::new()), res: Some(texture), maps: Mappings::new(), - })) + }; + Ok(if rusticl_type == RusticlTypes::Buffer { + Arc::new(Buffer { base: base }).into_cl() + } else { + Arc::new(Image { base: base }).into_cl() + }) } pub fn pixel_size(&self) -> Option { @@ -774,7 +856,7 @@ impl Mem { &self, q: &Arc, ctx: &PipeContext, - dst: &Arc, + dst: &MemBase, mut src_origin: CLVec, mut dst_origin: CLVec, region: &CLVec, @@ -1361,7 +1443,7 @@ impl Mem { } } -impl Drop for Mem { +impl Drop for MemBase { fn drop(&mut self) { let cbs = mem::take(self.cbs.get_mut().unwrap()); for cb in cbs.into_iter().rev() {