From 13e47cde01b01a0f866c8d1e62668857821bcd18 Mon Sep 17 00:00:00 2001 From: Karol Herbst Date: Mon, 16 Dec 2024 00:30:54 +0100 Subject: [PATCH] iris: implement SVM interfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Acked-by: Adam Jackson Reviewed-by: José Roberto de Souza Part-of: --- docs/features.txt | 2 +- docs/relnotes/new_features.txt | 2 +- src/gallium/drivers/iris/iris_bufmgr.c | 71 ++++++++++++++++++------ src/gallium/drivers/iris/iris_bufmgr.h | 8 ++- src/gallium/drivers/iris/iris_resource.c | 56 ++++++++++++++++++- src/gallium/drivers/iris/iris_screen.c | 7 +++ 6 files changed, 125 insertions(+), 21 deletions(-) diff --git a/docs/features.txt b/docs/features.txt index b03dadb670a..45224751600 100644 --- a/docs/features.txt +++ b/docs/features.txt @@ -756,7 +756,7 @@ Rusticl OpenCL 1.2 -- all DONE: Rusticl OpenCL 2.0 -- all DONE: - Shared virtual memory DONE (nvc0, llvmpipe) + Shared virtual memory DONE (iris, nvc0, llvmpipe) Device queues not started - cl_khr_create_command_queue DONE - Additional queries for clGetDeviceInfo DONE diff --git a/docs/relnotes/new_features.txt b/docs/relnotes/new_features.txt index 73ecf0a68ed..d4ad80b0a5e 100644 --- a/docs/relnotes/new_features.txt +++ b/docs/relnotes/new_features.txt @@ -28,7 +28,7 @@ VK_EXT_texel_buffer_alignment on panvk cl_khr_kernel_clock on freedreno, iris, llvmpipe, nvc0, panfrost, radeonsi and zink with llvm-19 or newer GL_KHR_texture_compression_astc_hdr on panfrost and asahi cl_ext_buffer_device_address on iris, llvmpipe and zink -Completed OpenCL 2.0 coarse grain buffer SVM support +Completed OpenCL 2.0 coarse grain buffer SVM support for iris VK_EXT_shader_subgroup_ballot on panvk VK_EXT_shader_subgroup_vote on panvk Vulkan video support on GFX12 (RDNA4) for RADV diff --git a/src/gallium/drivers/iris/iris_bufmgr.c b/src/gallium/drivers/iris/iris_bufmgr.c index 08ea27161c9..e28cb27ee1c 100644 --- a/src/gallium/drivers/iris/iris_bufmgr.c +++ b/src/gallium/drivers/iris/iris_bufmgr.c @@ -288,7 +288,7 @@ static struct bo_cache_bucket * bucket_for_size(struct iris_bufmgr *bufmgr, uint64_t size, enum iris_heap heap, enum bo_alloc_flags flags) { - if (flags & BO_ALLOC_PROTECTED) + if (flags & (BO_ALLOC_PROTECTED | BO_ALLOC_NO_VMA)) return NULL; const struct intel_device_info *devinfo = &bufmgr->devinfo; @@ -394,6 +394,37 @@ vma_alloc(struct iris_bufmgr *bufmgr, return intel_canonical_address(addr); } +bool +iris_bufmgr_alloc_heap(struct iris_bufmgr *bufmgr, uint64_t start, uint64_t size) +{ + simple_mtx_lock(&bufmgr->lock); + bool res = util_vma_heap_alloc_addr(&bufmgr->vma_allocator[IRIS_MEMZONE_OTHER], start, size); + simple_mtx_unlock(&bufmgr->lock); + return res; +} + +void +iris_bufmgr_free_heap(struct iris_bufmgr *bufmgr, uint64_t start, uint64_t size) +{ + simple_mtx_lock(&bufmgr->lock); + util_vma_heap_free(&bufmgr->vma_allocator[IRIS_MEMZONE_OTHER], start, size); + simple_mtx_unlock(&bufmgr->lock); +} + +bool +iris_bufmgr_assign_vma(struct iris_bufmgr *bufmgr, struct iris_bo *bo, uint64_t address) +{ + assert(bo->address == 0 || address == 0); + bo->address = intel_canonical_address(address); + + /* we disallow addresses to be assigned which need to be made canonical */ + assert(bo->address == address); + if (address) + return bufmgr->kmd_backend->gem_vm_bind(bo, 0); + else + return bufmgr->kmd_backend->gem_vm_unbind(bo); +} + static void vma_free(struct iris_bufmgr *bufmgr, uint64_t address, @@ -1280,7 +1311,7 @@ iris_bo_alloc(struct iris_bufmgr *bufmgr, return NULL; } - if (bo->address == 0ull) { + if (bo->address == 0ull && !(flags & BO_ALLOC_NO_VMA)) { simple_mtx_lock(&bufmgr->lock); bo->address = vma_alloc(bufmgr, memzone, bo->size, alignment); simple_mtx_unlock(&bufmgr->lock); @@ -1339,7 +1370,7 @@ iris_bo_close(int fd, uint32_t gem_handle) struct iris_bo * iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name, - void *ptr, size_t size, + void *ptr, size_t size, unsigned flags, enum iris_memory_zone memzone) { struct iris_bo *bo; @@ -1362,13 +1393,6 @@ iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name, if (INTEL_DEBUG(DEBUG_CAPTURE_ALL)) bo->real.capture = true; - simple_mtx_lock(&bufmgr->lock); - bo->address = vma_alloc(bufmgr, memzone, size, 1); - simple_mtx_unlock(&bufmgr->lock); - - if (bo->address == 0ull) - goto err_close; - p_atomic_set(&bo->refcount, 1); bo->index = -1; bo->idle = true; @@ -1376,8 +1400,17 @@ iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name, bo->real.mmap_mode = heap_to_mmap_mode(bufmgr, bo->real.heap); bo->real.prime_fd = -1; - if (!bufmgr->kmd_backend->gem_vm_bind(bo, 0)) - goto err_vma_free; + if (!(flags & BO_ALLOC_NO_VMA)) { + simple_mtx_lock(&bufmgr->lock); + bo->address = vma_alloc(bufmgr, memzone, size, 1); + simple_mtx_unlock(&bufmgr->lock); + + if (bo->address == 0ull) + goto err_close; + + if (!bufmgr->kmd_backend->gem_vm_bind(bo, 0)) + goto err_vma_free; + } return bo; @@ -1428,6 +1461,8 @@ iris_bo_gem_create_from_name(struct iris_bufmgr *bufmgr, { struct iris_bo *bo; + assert(!(flags & BO_ALLOC_NO_VMA)); + /* At the moment most applications only have a few named bo. * For instance, in a DRI client only the render buffers passed * between X and the client are named. And since X returns the @@ -1538,10 +1573,12 @@ bo_close(struct iris_bo *bo) } /* Unbind and return the VMA for reuse */ - if (bufmgr->kmd_backend->gem_vm_unbind(bo)) - vma_free(bo->bufmgr, bo->address, bo->size); - else - DBG("Unable to unbind vm of buf %u\n", bo->gem_handle); + if (bo->address) { + if (bufmgr->kmd_backend->gem_vm_unbind(bo)) + vma_free(bo->bufmgr, bo->address, bo->size); + else + DBG("Unable to unbind vm of buf %u\n", bo->gem_handle); + } if (bo->real.prime_fd != -1) close(bo->real.prime_fd); @@ -1951,6 +1988,8 @@ iris_bo_import_dmabuf(struct iris_bufmgr *bufmgr, int prime_fd, uint32_t handle; struct iris_bo *bo; + assert(!(flags & BO_ALLOC_NO_VMA)); + simple_mtx_lock(&bufmgr->lock); int ret = drmPrimeFDToHandle(bufmgr->fd, prime_fd, &handle); if (ret) { diff --git a/src/gallium/drivers/iris/iris_bufmgr.h b/src/gallium/drivers/iris/iris_bufmgr.h index ed9507d4e99..1c020280468 100644 --- a/src/gallium/drivers/iris/iris_bufmgr.h +++ b/src/gallium/drivers/iris/iris_bufmgr.h @@ -411,6 +411,8 @@ enum bo_alloc_flags { BO_ALLOC_CPU_VISIBLE = (1<<9), /* BO content is compressed. */ BO_ALLOC_COMPRESSED = (1<<10), + /* Do not allocate or bind a vma */ + BO_ALLOC_NO_VMA = (1<<11), }; /** @@ -429,7 +431,7 @@ struct iris_bo *iris_bo_alloc(struct iris_bufmgr *bufmgr, struct iris_bo * iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name, - void *ptr, size_t size, + void *ptr, size_t size, unsigned flags, enum iris_memory_zone memzone); /** Takes a reference on a buffer object */ @@ -696,6 +698,10 @@ bool iris_bufmgr_compute_engine_supported(struct iris_bufmgr *bufmgr); uint64_t iris_bufmgr_get_dummy_aux_address(struct iris_bufmgr *bufmgr); struct iris_bo *iris_bufmgr_get_mem_fence_bo(struct iris_bufmgr *bufmgr); +bool iris_bufmgr_alloc_heap(struct iris_bufmgr *bufmgr, uint64_t start, uint64_t size); +void iris_bufmgr_free_heap(struct iris_bufmgr *bufmgr, uint64_t start, uint64_t size); +bool iris_bufmgr_assign_vma(struct iris_bufmgr *bufmgr, struct iris_bo *bo, uint64_t address); + enum iris_madvice { IRIS_MADVICE_WILL_NEED = 0, IRIS_MADVICE_DONT_NEED = 1, diff --git a/src/gallium/drivers/iris/iris_resource.c b/src/gallium/drivers/iris/iris_resource.c index 1493c58a917..8126457ca26 100644 --- a/src/gallium/drivers/iris/iris_resource.c +++ b/src/gallium/drivers/iris/iris_resource.c @@ -48,6 +48,7 @@ #include "util/ralloc.h" #include "i915/iris_bufmgr.h" #include "iris_batch.h" +#include "iris_bufmgr.h" #include "iris_context.h" #include "iris_resource.h" #include "iris_screen.h" @@ -489,6 +490,9 @@ iris_resource_alloc_flags(const struct iris_screen *screen, if (templ->bind & PIPE_BIND_SCANOUT) flags |= BO_ALLOC_SCANOUT; + if (templ->flags & PIPE_RESOURCE_FLAG_FRONTEND_VM) + flags |= BO_ALLOC_NO_SUBALLOC | BO_ALLOC_NO_VMA; + if (templ->flags & (PIPE_RESOURCE_FLAG_MAP_COHERENT | PIPE_RESOURCE_FLAG_MAP_PERSISTENT)) flags |= BO_ALLOC_SMEM | BO_ALLOC_CACHED_COHERENT; @@ -533,6 +537,9 @@ iris_resource_destroy(struct pipe_screen *screen, if (p_res->target == PIPE_BUFFER) util_range_destroy(&res->valid_buffer_range); + if (p_res->flags & PIPE_RESOURCE_FLAG_FRONTEND_VM) + assert(res->bo->address == 0); + iris_resource_disable_aux(res); threaded_resource_deinit(p_res); @@ -1241,6 +1248,7 @@ iris_resource_from_user_memory(struct pipe_screen *pscreen, struct iris_screen *screen = (struct iris_screen *)pscreen; struct iris_bufmgr *bufmgr = screen->bufmgr; struct iris_resource *res = iris_alloc_resource(pscreen, templ); + unsigned flags = 0; if (!res) return NULL; @@ -1272,10 +1280,13 @@ iris_resource_from_user_memory(struct pipe_screen *pscreen, size_t mem_size = offset + res_size; mem_size = ALIGN_NPOT(mem_size, page_size); + if (templ->flags & PIPE_RESOURCE_FLAG_FRONTEND_VM) + flags |= BO_ALLOC_NO_VMA; + res->internal_format = templ->format; res->base.is_user_ptr = true; res->bo = iris_bo_create_userptr(bufmgr, "user", mem_start, mem_size, - IRIS_MEMZONE_OTHER); + flags, IRIS_MEMZONE_OTHER); res->offset = offset; if (!res->bo) { iris_resource_destroy(pscreen, &res->base.b); @@ -1977,7 +1988,8 @@ iris_invalidate_buffer(struct iris_context *ice, struct iris_resource *res) struct iris_screen *screen = (void *) ice->ctx.screen; if (res->base.b.target != PIPE_BUFFER || - res->base.b.flags & PIPE_RESOURCE_FLAG_FIXED_ADDRESS) + res->base.b.flags & PIPE_RESOURCE_FLAG_FIXED_ADDRESS || + res->base.b.flags & PIPE_RESOURCE_FLAG_FRONTEND_VM) return false; /* If it's already invalidated, don't bother doing anything. @@ -2737,6 +2749,43 @@ static const struct u_transfer_vtbl transfer_vtbl = { .get_stencil = iris_resource_get_separate_stencil, }; +static struct pipe_vm_allocation * +iris_alloc_vm(struct pipe_screen *pscreen, uint64_t start, uint64_t size) +{ + struct iris_screen *screen = (struct iris_screen *)pscreen; + if (!iris_bufmgr_alloc_heap(screen->bufmgr, start, size)) + return NULL; + + struct pipe_vm_allocation *res = CALLOC_STRUCT(pipe_vm_allocation); + if (!res) { + iris_bufmgr_free_heap(screen->bufmgr, start, size); + return NULL; + } + + res->start = start; + res->size = size; + return res; +} + +static void +iris_free_vm(struct pipe_screen *pscreen, struct pipe_vm_allocation *alloc) +{ + struct iris_screen *screen = (struct iris_screen *)pscreen; + iris_bufmgr_free_heap(screen->bufmgr, alloc->start, alloc->size); + FREE(alloc); +} + +static bool +iris_resource_assign_vma(struct pipe_screen *pscreen, + struct pipe_resource *presource, uint64_t address) +{ + struct iris_screen *screen = (struct iris_screen *)pscreen; + struct iris_resource *res = (struct iris_resource *)presource; + + assert(presource->flags & PIPE_RESOURCE_FLAG_FRONTEND_VM); + return iris_bufmgr_assign_vma(screen->bufmgr, res->bo, address); +} + static uint64_t iris_resource_get_address(struct pipe_screen *pscreen, struct pipe_resource *presrouce) @@ -2763,6 +2812,9 @@ iris_init_screen_resource_functions(struct pipe_screen *pscreen) pscreen->resource_destroy = u_transfer_helper_resource_destroy; pscreen->memobj_create_from_handle = iris_memobj_create_from_handle; pscreen->memobj_destroy = iris_memobj_destroy; + pscreen->alloc_vm = iris_alloc_vm; + pscreen->free_vm = iris_free_vm; + pscreen->resource_assign_vma = iris_resource_assign_vma; pscreen->resource_get_address = iris_resource_get_address; pscreen->transfer_helper = u_transfer_helper_create(&transfer_vtbl, diff --git a/src/gallium/drivers/iris/iris_screen.c b/src/gallium/drivers/iris/iris_screen.c index 0f1605473d7..49f25b95bda 100644 --- a/src/gallium/drivers/iris/iris_screen.c +++ b/src/gallium/drivers/iris/iris_screen.c @@ -491,6 +491,13 @@ iris_init_screen_caps(struct iris_screen *screen) caps->max_texture_anisotropy = 16.0f; caps->max_texture_lod_bias = 15.0f; + + caps->min_vma = IRIS_MEMZONE_OTHER_START; + /* Exclude addresses which would need to be converted to their canonical form. + * The easy way to go about this is to take the highest address and simply + * shift it right by one, so the highest valid address bit gets unset. + */ + caps->max_vma = intel_48b_address(UINT64_MAX) >> 1; } static uint64_t