iris: implement SVM interfaces

Acked-by: Adam Jackson <ajax@redhat.com>
Reviewed-by: José Roberto de Souza <jose.souza@intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/35174>

docs/features.txt

@@ -756,7 +756,7 @@ Rusticl OpenCL 1.2 -- all DONE:
 
 Rusticl OpenCL 2.0 -- all DONE:
 
-  Shared virtual memory                                 DONE (nvc0, llvmpipe)
+  Shared virtual memory                                 DONE (iris, nvc0, llvmpipe)
   Device queues                                         not started
   - cl_khr_create_command_queue                         DONE
   - Additional queries for clGetDeviceInfo              DONE

docs/relnotes/new_features.txt

@@ -28,7 +28,7 @@ VK_EXT_texel_buffer_alignment on panvk
 cl_khr_kernel_clock on freedreno, iris, llvmpipe, nvc0, panfrost, radeonsi and zink with llvm-19 or newer
 GL_KHR_texture_compression_astc_hdr on panfrost and asahi
 cl_ext_buffer_device_address on iris, llvmpipe and zink
-Completed OpenCL 2.0 coarse grain buffer SVM support
+Completed OpenCL 2.0 coarse grain buffer SVM support for iris
 VK_EXT_shader_subgroup_ballot on panvk
 VK_EXT_shader_subgroup_vote on panvk
 Vulkan video support on GFX12 (RDNA4) for RADV

src/gallium/drivers/iris/iris_bufmgr.c

@@ -288,7 +288,7 @@ static struct bo_cache_bucket *
 bucket_for_size(struct iris_bufmgr *bufmgr, uint64_t size,
                 enum iris_heap heap, enum bo_alloc_flags flags)
 {
-   if (flags & BO_ALLOC_PROTECTED)
+   if (flags & (BO_ALLOC_PROTECTED | BO_ALLOC_NO_VMA))
       return NULL;
 
    const struct intel_device_info *devinfo = &bufmgr->devinfo;
@@ -394,6 +394,37 @@ vma_alloc(struct iris_bufmgr *bufmgr,
    return intel_canonical_address(addr);
 }
 
+bool
+iris_bufmgr_alloc_heap(struct iris_bufmgr *bufmgr, uint64_t start, uint64_t size)
+{
+   simple_mtx_lock(&bufmgr->lock);
+   bool res = util_vma_heap_alloc_addr(&bufmgr->vma_allocator[IRIS_MEMZONE_OTHER], start, size);
+   simple_mtx_unlock(&bufmgr->lock);
+   return res;
+}
+
+void
+iris_bufmgr_free_heap(struct iris_bufmgr *bufmgr, uint64_t start, uint64_t size)
+{
+   simple_mtx_lock(&bufmgr->lock);
+   util_vma_heap_free(&bufmgr->vma_allocator[IRIS_MEMZONE_OTHER], start, size);
+   simple_mtx_unlock(&bufmgr->lock);
+}
+
+bool
+iris_bufmgr_assign_vma(struct iris_bufmgr *bufmgr, struct iris_bo *bo, uint64_t address)
+{
+   assert(bo->address == 0 || address == 0);
+   bo->address = intel_canonical_address(address);
+
+   /* we disallow addresses to be assigned which need to be made canonical */
+   assert(bo->address == address);
+   if (address)
+      return bufmgr->kmd_backend->gem_vm_bind(bo, 0);
+   else
+      return bufmgr->kmd_backend->gem_vm_unbind(bo);
+}
+
 static void
 vma_free(struct iris_bufmgr *bufmgr,
          uint64_t address,
@@ -1280,7 +1311,7 @@ iris_bo_alloc(struct iris_bufmgr *bufmgr,
          return NULL;
    }
 
-   if (bo->address == 0ull) {
+   if (bo->address == 0ull && !(flags & BO_ALLOC_NO_VMA)) {
       simple_mtx_lock(&bufmgr->lock);
       bo->address = vma_alloc(bufmgr, memzone, bo->size, alignment);
       simple_mtx_unlock(&bufmgr->lock);
@@ -1339,7 +1370,7 @@ iris_bo_close(int fd, uint32_t gem_handle)
 
 struct iris_bo *
 iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name,
-                       void *ptr, size_t size,
+                       void *ptr, size_t size, unsigned flags,
                        enum iris_memory_zone memzone)
 {
    struct iris_bo *bo;
@@ -1362,13 +1393,6 @@ iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name,
    if (INTEL_DEBUG(DEBUG_CAPTURE_ALL))
       bo->real.capture = true;
 
-   simple_mtx_lock(&bufmgr->lock);
-   bo->address = vma_alloc(bufmgr, memzone, size, 1);
-   simple_mtx_unlock(&bufmgr->lock);
-
-   if (bo->address == 0ull)
-      goto err_close;
-
    p_atomic_set(&bo->refcount, 1);
    bo->index = -1;
    bo->idle = true;
@@ -1376,8 +1400,17 @@ iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name,
    bo->real.mmap_mode = heap_to_mmap_mode(bufmgr, bo->real.heap);
    bo->real.prime_fd = -1;
 
-   if (!bufmgr->kmd_backend->gem_vm_bind(bo, 0))
-      goto err_vma_free;
+   if (!(flags & BO_ALLOC_NO_VMA)) {
+      simple_mtx_lock(&bufmgr->lock);
+      bo->address = vma_alloc(bufmgr, memzone, size, 1);
+      simple_mtx_unlock(&bufmgr->lock);
+
+      if (bo->address == 0ull)
+         goto err_close;
+
+      if (!bufmgr->kmd_backend->gem_vm_bind(bo, 0))
+         goto err_vma_free;
+   }
 
    return bo;
 
@@ -1428,6 +1461,8 @@ iris_bo_gem_create_from_name(struct iris_bufmgr *bufmgr,
 {
    struct iris_bo *bo;
 
+   assert(!(flags & BO_ALLOC_NO_VMA));
+
    /* At the moment most applications only have a few named bo.
     * For instance, in a DRI client only the render buffers passed
     * between X and the client are named. And since X returns the
@@ -1538,10 +1573,12 @@ bo_close(struct iris_bo *bo)
    }
 
    /* Unbind and return the VMA for reuse */
-   if (bufmgr->kmd_backend->gem_vm_unbind(bo))
-      vma_free(bo->bufmgr, bo->address, bo->size);
-   else
-      DBG("Unable to unbind vm of buf %u\n", bo->gem_handle);
+   if (bo->address) {
+      if (bufmgr->kmd_backend->gem_vm_unbind(bo))
+         vma_free(bo->bufmgr, bo->address, bo->size);
+      else
+         DBG("Unable to unbind vm of buf %u\n", bo->gem_handle);
+   }
 
    if (bo->real.prime_fd != -1)
       close(bo->real.prime_fd);
@@ -1951,6 +1988,8 @@ iris_bo_import_dmabuf(struct iris_bufmgr *bufmgr, int prime_fd,
    uint32_t handle;
    struct iris_bo *bo;
 
+   assert(!(flags & BO_ALLOC_NO_VMA));
+
    simple_mtx_lock(&bufmgr->lock);
    int ret = drmPrimeFDToHandle(bufmgr->fd, prime_fd, &handle);
    if (ret) {

src/gallium/drivers/iris/iris_bufmgr.h

@@ -411,6 +411,8 @@ enum bo_alloc_flags {
    BO_ALLOC_CPU_VISIBLE = (1<<9),
    /* BO content is compressed. */
    BO_ALLOC_COMPRESSED = (1<<10),
+   /* Do not allocate or bind a vma */
+   BO_ALLOC_NO_VMA = (1<<11),
 };
 
 /**
@@ -429,7 +431,7 @@ struct iris_bo *iris_bo_alloc(struct iris_bufmgr *bufmgr,
 
 struct iris_bo *
 iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name,
-                       void *ptr, size_t size,
+                       void *ptr, size_t size, unsigned flags,
                        enum iris_memory_zone memzone);
 
 /** Takes a reference on a buffer object */
@@ -696,6 +698,10 @@ bool iris_bufmgr_compute_engine_supported(struct iris_bufmgr *bufmgr);
 uint64_t iris_bufmgr_get_dummy_aux_address(struct iris_bufmgr *bufmgr);
 struct iris_bo *iris_bufmgr_get_mem_fence_bo(struct iris_bufmgr *bufmgr);
 
+bool iris_bufmgr_alloc_heap(struct iris_bufmgr *bufmgr, uint64_t start, uint64_t size);
+void iris_bufmgr_free_heap(struct iris_bufmgr *bufmgr, uint64_t start, uint64_t size);
+bool iris_bufmgr_assign_vma(struct iris_bufmgr *bufmgr, struct iris_bo *bo, uint64_t address);
+
 enum iris_madvice {
    IRIS_MADVICE_WILL_NEED = 0,
    IRIS_MADVICE_DONT_NEED = 1,

src/gallium/drivers/iris/iris_resource.c

@@ -48,6 +48,7 @@
 #include "util/ralloc.h"
 #include "i915/iris_bufmgr.h"
 #include "iris_batch.h"
+#include "iris_bufmgr.h"
 #include "iris_context.h"
 #include "iris_resource.h"
 #include "iris_screen.h"
@@ -489,6 +490,9 @@ iris_resource_alloc_flags(const struct iris_screen *screen,
    if (templ->bind & PIPE_BIND_SCANOUT)
       flags |= BO_ALLOC_SCANOUT;
 
+   if (templ->flags & PIPE_RESOURCE_FLAG_FRONTEND_VM)
+      flags |= BO_ALLOC_NO_SUBALLOC | BO_ALLOC_NO_VMA;
+
    if (templ->flags & (PIPE_RESOURCE_FLAG_MAP_COHERENT |
                        PIPE_RESOURCE_FLAG_MAP_PERSISTENT))
       flags |= BO_ALLOC_SMEM | BO_ALLOC_CACHED_COHERENT;
@@ -533,6 +537,9 @@ iris_resource_destroy(struct pipe_screen *screen,
    if (p_res->target == PIPE_BUFFER)
       util_range_destroy(&res->valid_buffer_range);
 
+   if (p_res->flags & PIPE_RESOURCE_FLAG_FRONTEND_VM)
+      assert(res->bo->address == 0);
+
    iris_resource_disable_aux(res);
 
    threaded_resource_deinit(p_res);
@@ -1241,6 +1248,7 @@ iris_resource_from_user_memory(struct pipe_screen *pscreen,
    struct iris_screen *screen = (struct iris_screen *)pscreen;
    struct iris_bufmgr *bufmgr = screen->bufmgr;
    struct iris_resource *res = iris_alloc_resource(pscreen, templ);
+   unsigned flags = 0;
    if (!res)
       return NULL;
 
@@ -1272,10 +1280,13 @@ iris_resource_from_user_memory(struct pipe_screen *pscreen,
    size_t mem_size = offset + res_size;
    mem_size = ALIGN_NPOT(mem_size, page_size);
 
+   if (templ->flags & PIPE_RESOURCE_FLAG_FRONTEND_VM)
+      flags |= BO_ALLOC_NO_VMA;
+
    res->internal_format = templ->format;
    res->base.is_user_ptr = true;
    res->bo = iris_bo_create_userptr(bufmgr, "user", mem_start, mem_size,
-                                    IRIS_MEMZONE_OTHER);
+                                    flags, IRIS_MEMZONE_OTHER);
    res->offset = offset;
    if (!res->bo) {
       iris_resource_destroy(pscreen, &res->base.b);
@@ -1977,7 +1988,8 @@ iris_invalidate_buffer(struct iris_context *ice, struct iris_resource *res)
    struct iris_screen *screen = (void *) ice->ctx.screen;
 
    if (res->base.b.target != PIPE_BUFFER ||
-       res->base.b.flags & PIPE_RESOURCE_FLAG_FIXED_ADDRESS)
+       res->base.b.flags & PIPE_RESOURCE_FLAG_FIXED_ADDRESS ||
+       res->base.b.flags & PIPE_RESOURCE_FLAG_FRONTEND_VM)
       return false;
 
    /* If it's already invalidated, don't bother doing anything.
@@ -2737,6 +2749,43 @@ static const struct u_transfer_vtbl transfer_vtbl = {
    .get_stencil           = iris_resource_get_separate_stencil,
 };
 
+static struct pipe_vm_allocation *
+iris_alloc_vm(struct pipe_screen *pscreen, uint64_t start, uint64_t size)
+{
+   struct iris_screen *screen = (struct iris_screen *)pscreen;
+   if (!iris_bufmgr_alloc_heap(screen->bufmgr, start, size))
+      return NULL;
+
+   struct pipe_vm_allocation *res = CALLOC_STRUCT(pipe_vm_allocation);
+   if (!res) {
+      iris_bufmgr_free_heap(screen->bufmgr, start, size);
+      return NULL;
+   }
+
+   res->start = start;
+   res->size = size;
+   return res;
+}
+
+static void
+iris_free_vm(struct pipe_screen *pscreen, struct pipe_vm_allocation *alloc)
+{
+   struct iris_screen *screen = (struct iris_screen *)pscreen;
+   iris_bufmgr_free_heap(screen->bufmgr, alloc->start, alloc->size);
+   FREE(alloc);
+}
+
+static bool
+iris_resource_assign_vma(struct pipe_screen *pscreen,
+                         struct pipe_resource *presource, uint64_t address)
+{
+   struct iris_screen *screen = (struct iris_screen *)pscreen;
+   struct iris_resource *res = (struct iris_resource *)presource;
+
+   assert(presource->flags & PIPE_RESOURCE_FLAG_FRONTEND_VM);
+   return iris_bufmgr_assign_vma(screen->bufmgr, res->bo, address);
+}
+
 static uint64_t
 iris_resource_get_address(struct pipe_screen *pscreen,
                           struct pipe_resource *presrouce)
@@ -2763,6 +2812,9 @@ iris_init_screen_resource_functions(struct pipe_screen *pscreen)
    pscreen->resource_destroy = u_transfer_helper_resource_destroy;
    pscreen->memobj_create_from_handle = iris_memobj_create_from_handle;
    pscreen->memobj_destroy = iris_memobj_destroy;
+   pscreen->alloc_vm = iris_alloc_vm;
+   pscreen->free_vm = iris_free_vm;
+   pscreen->resource_assign_vma = iris_resource_assign_vma;
    pscreen->resource_get_address = iris_resource_get_address;
    pscreen->transfer_helper =
       u_transfer_helper_create(&transfer_vtbl,

src/gallium/drivers/iris/iris_screen.c

@@ -491,6 +491,13 @@ iris_init_screen_caps(struct iris_screen *screen)
 
    caps->max_texture_anisotropy = 16.0f;
    caps->max_texture_lod_bias = 15.0f;
+
+   caps->min_vma = IRIS_MEMZONE_OTHER_START;
+   /* Exclude addresses which would need to be converted to their canonical form.
+    * The easy way to go about this is to take the highest address and simply
+    * shift it right by one, so the highest valid address bit gets unset.
+    */
+   caps->max_vma = intel_48b_address(UINT64_MAX) >> 1;
 }
 
 static uint64_t