From f9daddf5d5d1b1060e1136a911d1567d87ea5b73 Mon Sep 17 00:00:00 2001 From: Connor Abbott Date: Fri, 15 Nov 2024 11:26:18 -0500 Subject: [PATCH] tu/knl: Add an API for sparse binding Add a "sparse VMA" abstraction, and functions creating them, destroying them, and submitting commands to map and unmap BOs into them. This mirrors the Vulkan API, but with image offsets resolved to page offsets. Part-of: --- src/freedreno/vulkan/tu_device.h | 4 ++ src/freedreno/vulkan/tu_knl.cc | 39 +++++++++++- src/freedreno/vulkan/tu_knl.h | 75 ++++++++++++++++++++++- src/freedreno/vulkan/tu_knl_drm_msm.cc | 1 + src/freedreno/vulkan/tu_knl_drm_virtio.cc | 1 + src/freedreno/vulkan/tu_knl_kgsl.cc | 1 + src/freedreno/vulkan/tu_queue.cc | 3 +- src/freedreno/vulkan/tu_queue.h | 7 +++ 8 files changed, 128 insertions(+), 3 deletions(-) diff --git a/src/freedreno/vulkan/tu_device.h b/src/freedreno/vulkan/tu_device.h index fb05910e3ab..2e1feb4828c 100644 --- a/src/freedreno/vulkan/tu_device.h +++ b/src/freedreno/vulkan/tu_device.h @@ -122,6 +122,10 @@ struct tu_physical_device bool has_set_iova; bool has_raytracing; bool has_vm_bind; + /* Whether a sparse queue can be created. */ + bool has_sparse; + /* Whether TU_SPARSE_VMA_MAP_ZERO can be used. */ + bool has_sparse_prr; uint64_t va_start; uint64_t va_size; diff --git a/src/freedreno/vulkan/tu_knl.cc b/src/freedreno/vulkan/tu_knl.cc index 1b16dec1144..d29fe9bcab4 100644 --- a/src/freedreno/vulkan/tu_knl.cc +++ b/src/freedreno/vulkan/tu_knl.cc @@ -228,6 +228,29 @@ tu_bo_set_metadata(struct tu_device *dev, struct tu_bo *bo, dev->instance->knl->bo_set_metadata(dev, bo, metadata, metadata_size); } +VkResult +tu_sparse_vma_init(struct tu_device *dev, + struct vk_object_base *base, + struct tu_sparse_vma *out_vma, + uint64_t *out_iova, + enum tu_sparse_vma_flags flags, + uint64_t size, uint64_t client_iova) +{ + size = align64(size, os_page_size); + + out_vma->flags = flags; + return dev->instance->knl->sparse_vma_init(dev, base, out_vma, out_iova, + flags, size, client_iova); + +} + +void +tu_sparse_vma_finish(struct tu_device *dev, + struct tu_sparse_vma *vma) +{ + dev->instance->knl->sparse_vma_finish(dev, vma); +} + int tu_bo_get_metadata(struct tu_device *dev, struct tu_bo *bo, void *metadata, uint32_t metadata_size) @@ -280,10 +303,11 @@ tu_device_check_status(struct vk_device *vk_device) int tu_drm_submitqueue_new(struct tu_device *dev, + enum tu_queue_type type, int priority, uint32_t *queue_id) { - return dev->instance->knl->submitqueue_new(dev, priority, queue_id); + return dev->instance->knl->submitqueue_new(dev, type, priority, queue_id); } void @@ -313,6 +337,19 @@ tu_submit_add_entries(struct tu_device *dev, void *submit, num_entries); } +void +tu_submit_add_bind(struct tu_device *dev, + void *_submit, + struct tu_sparse_vma *vma, uint64_t vma_offset, + struct tu_bo *bo, uint64_t bo_offset, + uint64_t size) +{ + assert(vma_offset % 4096 == 0); + assert(bo_offset % 4096 == 0); + return dev->instance->knl->submit_add_bind(dev, _submit, vma, vma_offset, + bo, bo_offset, size); +} + VkResult tu_queue_submit(struct tu_queue *queue, void *submit, struct vk_sync_wait *waits, uint32_t wait_count, diff --git a/src/freedreno/vulkan/tu_knl.h b/src/freedreno/vulkan/tu_knl.h index f3fe16e5171..edaca35f3e9 100644 --- a/src/freedreno/vulkan/tu_knl.h +++ b/src/freedreno/vulkan/tu_knl.h @@ -11,6 +11,7 @@ #define TU_DRM_H #include "tu_common.h" +#include "tu_queue.h" struct tu_u_trace_syncobj; struct vdrm_bo; @@ -65,6 +66,47 @@ struct tu_bo { struct vk_object_base *base; }; +enum tu_sparse_vma_flags { + TU_SPARSE_VMA_REPLAYABLE = 1 << 0, + + /* Make unmapped pages in the memory region map to the PRR NULL page. This + * applies to all pages initially as well as pages that are subsequently + * unmapped via a queue submission. Writes to the PRR NULL page are + * dropped, and reads return zero. If this flag isn't set then reads and + * writes to unmapped pages will fault instead. + */ + TU_SPARSE_VMA_MAP_ZERO = 1 << 1, +}; + +/* This represents a memory region into which BOs can be mapped. This is + * implemented differently on drm/msm and kgsl: + * + * - msm allows us to control the VA range ourselves, and provides an API to + * map arbitrary parts of BOs to a given VA range. The sparse VMA is + * just a userspace driver abstraction, consisting of an iova range we + * reserve and (optionally) map as NULL initially, and when mapping a BO + * into the sparse VMA we calculate the final iova range to map it to. + * - kgsl doesn't allow userspace control of the iova, and requires that we + * create a "virtual BO" into which we can map BOs. The virtual BO maps + * one-to-one to tu_sparse_vma, and almost one-to-one to a Vulkan VkBuffer + * or VkImage with sparse binding. + * + * tu_sparse_vma is an abstraction to bridge this difference. + */ +struct tu_sparse_vma { + enum tu_sparse_vma_flags flags; + + union { + struct { + uint64_t iova; + uint64_t size; + } msm; + struct { + struct tu_bo *virtual_bo; + } kgsl; + }; +}; + struct tu_knl { const char *name; @@ -73,7 +115,7 @@ struct tu_knl { int (*device_get_gpu_timestamp)(struct tu_device *dev, uint64_t *ts); int (*device_get_suspend_count)(struct tu_device *dev, uint64_t *suspend_count); VkResult (*device_check_status)(struct tu_device *dev); - int (*submitqueue_new)(struct tu_device *dev, int priority, uint32_t *queue_id); + int (*submitqueue_new)(struct tu_device *dev, enum tu_queue_type type, int priority, uint32_t *queue_id); void (*submitqueue_close)(struct tu_device *dev, uint32_t queue_id); VkResult (*bo_init)(struct tu_device *dev, struct vk_object_base *base, struct tu_bo **out_bo, uint64_t size, uint64_t client_iova, @@ -94,12 +136,25 @@ struct tu_knl { void (*submit_add_entries)(struct tu_device *device, void *_submit, struct tu_cs_entry *entries, unsigned num_entries); + void (*submit_add_bind)(struct tu_device *device, + void *_submit, + struct tu_sparse_vma *vma, uint64_t vma_offset, + struct tu_bo *bo, uint64_t bo_offset, + uint64_t size); VkResult (*queue_submit)(struct tu_queue *queue, void *_submit, struct vk_sync_wait *waits, uint32_t wait_count, struct vk_sync_signal *signals, uint32_t signal_count, struct tu_u_trace_submission_data *u_trace_submission_data); VkResult (*queue_wait_fence)(struct tu_queue *queue, uint32_t fence, uint64_t timeout_ns); + VkResult (*sparse_vma_init)(struct tu_device *dev, + struct vk_object_base *base, + struct tu_sparse_vma *out_vma, + uint64_t *out_iova, + enum tu_sparse_vma_flags flags, + uint64_t size, uint64_t client_iova); + void (*sparse_vma_finish)(struct tu_device *device, + struct tu_sparse_vma *vma); const struct vk_device_entrypoint_table *device_entrypoints; }; @@ -178,6 +233,16 @@ tu_bo_get_ref(struct tu_bo *bo) return bo; } +VkResult tu_sparse_vma_init(struct tu_device *dev, + struct vk_object_base *base, + struct tu_sparse_vma *out_vma, + uint64_t *out_iova, + enum tu_sparse_vma_flags flags, + uint64_t size, uint64_t client_iova); + +void tu_sparse_vma_finish(struct tu_device *device, + struct tu_sparse_vma *vma); + VkResult tu_knl_kgsl_load(struct tu_instance *instance, int fd); struct _drmVersion; @@ -217,6 +282,7 @@ tu_device_check_status(struct vk_device *vk_device); int tu_drm_submitqueue_new(struct tu_device *dev, + enum tu_queue_type type, int priority, uint32_t *queue_id); @@ -234,6 +300,13 @@ tu_submit_add_entries(struct tu_device *dev, void *submit, struct tu_cs_entry *entries, unsigned num_entries); +void +tu_submit_add_bind(struct tu_device *device, + void *_submit, + struct tu_sparse_vma *vma, uint64_t vma_offset, + struct tu_bo *bo, uint64_t bo_offset, + uint64_t size); + VkResult tu_queue_submit(struct tu_queue *queue, void *submit, struct vk_sync_wait *waits, uint32_t wait_count, diff --git a/src/freedreno/vulkan/tu_knl_drm_msm.cc b/src/freedreno/vulkan/tu_knl_drm_msm.cc index 8c7079f9fae..cbc5dfcdd53 100644 --- a/src/freedreno/vulkan/tu_knl_drm_msm.cc +++ b/src/freedreno/vulkan/tu_knl_drm_msm.cc @@ -322,6 +322,7 @@ msm_device_check_status(struct tu_device *device) static int msm_submitqueue_new(struct tu_device *dev, + enum tu_queue_type type, int priority, uint32_t *queue_id) { diff --git a/src/freedreno/vulkan/tu_knl_drm_virtio.cc b/src/freedreno/vulkan/tu_knl_drm_virtio.cc index 011d376f7ea..e2aa2e93982 100644 --- a/src/freedreno/vulkan/tu_knl_drm_virtio.cc +++ b/src/freedreno/vulkan/tu_knl_drm_virtio.cc @@ -346,6 +346,7 @@ virtio_device_check_status(struct tu_device *device) static int virtio_submitqueue_new(struct tu_device *dev, + enum tu_queue_type type, int priority, uint32_t *queue_id) { diff --git a/src/freedreno/vulkan/tu_knl_kgsl.cc b/src/freedreno/vulkan/tu_knl_kgsl.cc index 0b26e92b3b6..9b2f4c102d4 100644 --- a/src/freedreno/vulkan/tu_knl_kgsl.cc +++ b/src/freedreno/vulkan/tu_knl_kgsl.cc @@ -51,6 +51,7 @@ safe_ioctl(int fd, unsigned long request, void *arg) static int kgsl_submitqueue_new(struct tu_device *dev, + enum tu_queue_type type, int priority, uint32_t *queue_id) { diff --git a/src/freedreno/vulkan/tu_queue.cc b/src/freedreno/vulkan/tu_queue.cc index 68bafe0cb4b..d27b03296e2 100644 --- a/src/freedreno/vulkan/tu_queue.cc +++ b/src/freedreno/vulkan/tu_queue.cc @@ -282,8 +282,9 @@ tu_queue_init(struct tu_device *device, queue->device = device; queue->priority = priority; queue->vk.driver_submit = queue_submit; + queue->type = TU_QUEUE_GFX; - int ret = tu_drm_submitqueue_new(device, priority, &queue->msm_queue_id); + int ret = tu_drm_submitqueue_new(device, TU_QUEUE_GFX, priority, &queue->msm_queue_id); if (ret) return vk_startup_errorf(device->instance, VK_ERROR_INITIALIZATION_FAILED, "submitqueue create failed"); diff --git a/src/freedreno/vulkan/tu_queue.h b/src/freedreno/vulkan/tu_queue.h index 1b82be54abd..02db20a305d 100644 --- a/src/freedreno/vulkan/tu_queue.h +++ b/src/freedreno/vulkan/tu_queue.h @@ -12,12 +12,19 @@ #include "tu_common.h" +enum tu_queue_type +{ + TU_QUEUE_GFX, + TU_QUEUE_SPARSE, +}; + struct tu_queue { struct vk_queue vk; struct tu_device *device; + enum tu_queue_type type; uint32_t msm_queue_id; uint32_t priority;