diff --git a/src/freedreno/vulkan/tu_device.h b/src/freedreno/vulkan/tu_device.h index fb05910e3ab..2e1feb4828c 100644 --- a/src/freedreno/vulkan/tu_device.h +++ b/src/freedreno/vulkan/tu_device.h @@ -122,6 +122,10 @@ struct tu_physical_device bool has_set_iova; bool has_raytracing; bool has_vm_bind; + /* Whether a sparse queue can be created. */ + bool has_sparse; + /* Whether TU_SPARSE_VMA_MAP_ZERO can be used. */ + bool has_sparse_prr; uint64_t va_start; uint64_t va_size; diff --git a/src/freedreno/vulkan/tu_knl.cc b/src/freedreno/vulkan/tu_knl.cc index 1b16dec1144..d29fe9bcab4 100644 --- a/src/freedreno/vulkan/tu_knl.cc +++ b/src/freedreno/vulkan/tu_knl.cc @@ -228,6 +228,29 @@ tu_bo_set_metadata(struct tu_device *dev, struct tu_bo *bo, dev->instance->knl->bo_set_metadata(dev, bo, metadata, metadata_size); } +VkResult +tu_sparse_vma_init(struct tu_device *dev, + struct vk_object_base *base, + struct tu_sparse_vma *out_vma, + uint64_t *out_iova, + enum tu_sparse_vma_flags flags, + uint64_t size, uint64_t client_iova) +{ + size = align64(size, os_page_size); + + out_vma->flags = flags; + return dev->instance->knl->sparse_vma_init(dev, base, out_vma, out_iova, + flags, size, client_iova); + +} + +void +tu_sparse_vma_finish(struct tu_device *dev, + struct tu_sparse_vma *vma) +{ + dev->instance->knl->sparse_vma_finish(dev, vma); +} + int tu_bo_get_metadata(struct tu_device *dev, struct tu_bo *bo, void *metadata, uint32_t metadata_size) @@ -280,10 +303,11 @@ tu_device_check_status(struct vk_device *vk_device) int tu_drm_submitqueue_new(struct tu_device *dev, + enum tu_queue_type type, int priority, uint32_t *queue_id) { - return dev->instance->knl->submitqueue_new(dev, priority, queue_id); + return dev->instance->knl->submitqueue_new(dev, type, priority, queue_id); } void @@ -313,6 +337,19 @@ tu_submit_add_entries(struct tu_device *dev, void *submit, num_entries); } +void +tu_submit_add_bind(struct tu_device *dev, + void *_submit, + struct tu_sparse_vma *vma, uint64_t vma_offset, + struct tu_bo *bo, uint64_t bo_offset, + uint64_t size) +{ + assert(vma_offset % 4096 == 0); + assert(bo_offset % 4096 == 0); + return dev->instance->knl->submit_add_bind(dev, _submit, vma, vma_offset, + bo, bo_offset, size); +} + VkResult tu_queue_submit(struct tu_queue *queue, void *submit, struct vk_sync_wait *waits, uint32_t wait_count, diff --git a/src/freedreno/vulkan/tu_knl.h b/src/freedreno/vulkan/tu_knl.h index f3fe16e5171..edaca35f3e9 100644 --- a/src/freedreno/vulkan/tu_knl.h +++ b/src/freedreno/vulkan/tu_knl.h @@ -11,6 +11,7 @@ #define TU_DRM_H #include "tu_common.h" +#include "tu_queue.h" struct tu_u_trace_syncobj; struct vdrm_bo; @@ -65,6 +66,47 @@ struct tu_bo { struct vk_object_base *base; }; +enum tu_sparse_vma_flags { + TU_SPARSE_VMA_REPLAYABLE = 1 << 0, + + /* Make unmapped pages in the memory region map to the PRR NULL page. This + * applies to all pages initially as well as pages that are subsequently + * unmapped via a queue submission. Writes to the PRR NULL page are + * dropped, and reads return zero. If this flag isn't set then reads and + * writes to unmapped pages will fault instead. + */ + TU_SPARSE_VMA_MAP_ZERO = 1 << 1, +}; + +/* This represents a memory region into which BOs can be mapped. This is + * implemented differently on drm/msm and kgsl: + * + * - msm allows us to control the VA range ourselves, and provides an API to + * map arbitrary parts of BOs to a given VA range. The sparse VMA is + * just a userspace driver abstraction, consisting of an iova range we + * reserve and (optionally) map as NULL initially, and when mapping a BO + * into the sparse VMA we calculate the final iova range to map it to. + * - kgsl doesn't allow userspace control of the iova, and requires that we + * create a "virtual BO" into which we can map BOs. The virtual BO maps + * one-to-one to tu_sparse_vma, and almost one-to-one to a Vulkan VkBuffer + * or VkImage with sparse binding. + * + * tu_sparse_vma is an abstraction to bridge this difference. + */ +struct tu_sparse_vma { + enum tu_sparse_vma_flags flags; + + union { + struct { + uint64_t iova; + uint64_t size; + } msm; + struct { + struct tu_bo *virtual_bo; + } kgsl; + }; +}; + struct tu_knl { const char *name; @@ -73,7 +115,7 @@ struct tu_knl { int (*device_get_gpu_timestamp)(struct tu_device *dev, uint64_t *ts); int (*device_get_suspend_count)(struct tu_device *dev, uint64_t *suspend_count); VkResult (*device_check_status)(struct tu_device *dev); - int (*submitqueue_new)(struct tu_device *dev, int priority, uint32_t *queue_id); + int (*submitqueue_new)(struct tu_device *dev, enum tu_queue_type type, int priority, uint32_t *queue_id); void (*submitqueue_close)(struct tu_device *dev, uint32_t queue_id); VkResult (*bo_init)(struct tu_device *dev, struct vk_object_base *base, struct tu_bo **out_bo, uint64_t size, uint64_t client_iova, @@ -94,12 +136,25 @@ struct tu_knl { void (*submit_add_entries)(struct tu_device *device, void *_submit, struct tu_cs_entry *entries, unsigned num_entries); + void (*submit_add_bind)(struct tu_device *device, + void *_submit, + struct tu_sparse_vma *vma, uint64_t vma_offset, + struct tu_bo *bo, uint64_t bo_offset, + uint64_t size); VkResult (*queue_submit)(struct tu_queue *queue, void *_submit, struct vk_sync_wait *waits, uint32_t wait_count, struct vk_sync_signal *signals, uint32_t signal_count, struct tu_u_trace_submission_data *u_trace_submission_data); VkResult (*queue_wait_fence)(struct tu_queue *queue, uint32_t fence, uint64_t timeout_ns); + VkResult (*sparse_vma_init)(struct tu_device *dev, + struct vk_object_base *base, + struct tu_sparse_vma *out_vma, + uint64_t *out_iova, + enum tu_sparse_vma_flags flags, + uint64_t size, uint64_t client_iova); + void (*sparse_vma_finish)(struct tu_device *device, + struct tu_sparse_vma *vma); const struct vk_device_entrypoint_table *device_entrypoints; }; @@ -178,6 +233,16 @@ tu_bo_get_ref(struct tu_bo *bo) return bo; } +VkResult tu_sparse_vma_init(struct tu_device *dev, + struct vk_object_base *base, + struct tu_sparse_vma *out_vma, + uint64_t *out_iova, + enum tu_sparse_vma_flags flags, + uint64_t size, uint64_t client_iova); + +void tu_sparse_vma_finish(struct tu_device *device, + struct tu_sparse_vma *vma); + VkResult tu_knl_kgsl_load(struct tu_instance *instance, int fd); struct _drmVersion; @@ -217,6 +282,7 @@ tu_device_check_status(struct vk_device *vk_device); int tu_drm_submitqueue_new(struct tu_device *dev, + enum tu_queue_type type, int priority, uint32_t *queue_id); @@ -234,6 +300,13 @@ tu_submit_add_entries(struct tu_device *dev, void *submit, struct tu_cs_entry *entries, unsigned num_entries); +void +tu_submit_add_bind(struct tu_device *device, + void *_submit, + struct tu_sparse_vma *vma, uint64_t vma_offset, + struct tu_bo *bo, uint64_t bo_offset, + uint64_t size); + VkResult tu_queue_submit(struct tu_queue *queue, void *submit, struct vk_sync_wait *waits, uint32_t wait_count, diff --git a/src/freedreno/vulkan/tu_knl_drm_msm.cc b/src/freedreno/vulkan/tu_knl_drm_msm.cc index 8c7079f9fae..cbc5dfcdd53 100644 --- a/src/freedreno/vulkan/tu_knl_drm_msm.cc +++ b/src/freedreno/vulkan/tu_knl_drm_msm.cc @@ -322,6 +322,7 @@ msm_device_check_status(struct tu_device *device) static int msm_submitqueue_new(struct tu_device *dev, + enum tu_queue_type type, int priority, uint32_t *queue_id) { diff --git a/src/freedreno/vulkan/tu_knl_drm_virtio.cc b/src/freedreno/vulkan/tu_knl_drm_virtio.cc index 011d376f7ea..e2aa2e93982 100644 --- a/src/freedreno/vulkan/tu_knl_drm_virtio.cc +++ b/src/freedreno/vulkan/tu_knl_drm_virtio.cc @@ -346,6 +346,7 @@ virtio_device_check_status(struct tu_device *device) static int virtio_submitqueue_new(struct tu_device *dev, + enum tu_queue_type type, int priority, uint32_t *queue_id) { diff --git a/src/freedreno/vulkan/tu_knl_kgsl.cc b/src/freedreno/vulkan/tu_knl_kgsl.cc index 0b26e92b3b6..9b2f4c102d4 100644 --- a/src/freedreno/vulkan/tu_knl_kgsl.cc +++ b/src/freedreno/vulkan/tu_knl_kgsl.cc @@ -51,6 +51,7 @@ safe_ioctl(int fd, unsigned long request, void *arg) static int kgsl_submitqueue_new(struct tu_device *dev, + enum tu_queue_type type, int priority, uint32_t *queue_id) { diff --git a/src/freedreno/vulkan/tu_queue.cc b/src/freedreno/vulkan/tu_queue.cc index 68bafe0cb4b..d27b03296e2 100644 --- a/src/freedreno/vulkan/tu_queue.cc +++ b/src/freedreno/vulkan/tu_queue.cc @@ -282,8 +282,9 @@ tu_queue_init(struct tu_device *device, queue->device = device; queue->priority = priority; queue->vk.driver_submit = queue_submit; + queue->type = TU_QUEUE_GFX; - int ret = tu_drm_submitqueue_new(device, priority, &queue->msm_queue_id); + int ret = tu_drm_submitqueue_new(device, TU_QUEUE_GFX, priority, &queue->msm_queue_id); if (ret) return vk_startup_errorf(device->instance, VK_ERROR_INITIALIZATION_FAILED, "submitqueue create failed"); diff --git a/src/freedreno/vulkan/tu_queue.h b/src/freedreno/vulkan/tu_queue.h index 1b82be54abd..02db20a305d 100644 --- a/src/freedreno/vulkan/tu_queue.h +++ b/src/freedreno/vulkan/tu_queue.h @@ -12,12 +12,19 @@ #include "tu_common.h" +enum tu_queue_type +{ + TU_QUEUE_GFX, + TU_QUEUE_SPARSE, +}; + struct tu_queue { struct vk_queue vk; struct tu_device *device; + enum tu_queue_type type; uint32_t msm_queue_id; uint32_t priority;