tu/knl: Add an API for sparse binding

Add a "sparse VMA" abstraction, and functions creating them, destroying
them, and submitting commands to map and unmap BOs into them. This
mirrors the Vulkan API, but with image offsets resolved to page offsets.

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/32533>

src/freedreno/vulkan/tu_device.h

@@ -122,6 +122,10 @@ struct tu_physical_device
    bool has_set_iova;
    bool has_raytracing;
    bool has_vm_bind;
+   /* Whether a sparse queue can be created. */
+   bool has_sparse;
+   /* Whether TU_SPARSE_VMA_MAP_ZERO can be used. */
+   bool has_sparse_prr;
    uint64_t va_start;
    uint64_t va_size;
 

src/freedreno/vulkan/tu_knl.cc

@@ -228,6 +228,29 @@ tu_bo_set_metadata(struct tu_device *dev, struct tu_bo *bo,
    dev->instance->knl->bo_set_metadata(dev, bo, metadata, metadata_size);
 }
 
+VkResult
+tu_sparse_vma_init(struct tu_device *dev,
+                   struct vk_object_base *base,
+                   struct tu_sparse_vma *out_vma,
+                   uint64_t *out_iova,
+                   enum tu_sparse_vma_flags flags,
+                   uint64_t size, uint64_t client_iova)
+{
+   size = align64(size, os_page_size);
+
+   out_vma->flags = flags;
+   return dev->instance->knl->sparse_vma_init(dev, base, out_vma, out_iova,
+                                              flags, size, client_iova);
+
+}
+
+void
+tu_sparse_vma_finish(struct tu_device *dev,
+                     struct tu_sparse_vma *vma)
+{
+   dev->instance->knl->sparse_vma_finish(dev, vma);
+}
+
 int
 tu_bo_get_metadata(struct tu_device *dev, struct tu_bo *bo,
                    void *metadata, uint32_t metadata_size)
@@ -280,10 +303,11 @@ tu_device_check_status(struct vk_device *vk_device)
 
 int
 tu_drm_submitqueue_new(struct tu_device *dev,
+                       enum tu_queue_type type,
                        int priority,
                        uint32_t *queue_id)
 {
-   return dev->instance->knl->submitqueue_new(dev, priority, queue_id);
+   return dev->instance->knl->submitqueue_new(dev, type, priority, queue_id);
 }
 
 void
@@ -313,6 +337,19 @@ tu_submit_add_entries(struct tu_device *dev, void *submit,
                                                  num_entries);
 }
 
+void
+tu_submit_add_bind(struct tu_device *dev,
+                   void *_submit,
+                   struct tu_sparse_vma *vma, uint64_t vma_offset,
+                   struct tu_bo *bo, uint64_t bo_offset,
+                   uint64_t size)
+{
+   assert(vma_offset % 4096 == 0);
+   assert(bo_offset % 4096 == 0);
+   return dev->instance->knl->submit_add_bind(dev, _submit, vma, vma_offset,
+                                              bo, bo_offset, size);
+}
+
 VkResult
 tu_queue_submit(struct tu_queue *queue, void *submit,
                 struct vk_sync_wait *waits, uint32_t wait_count,

src/freedreno/vulkan/tu_knl.h

@@ -11,6 +11,7 @@
 #define TU_DRM_H
 
 #include "tu_common.h"
+#include "tu_queue.h"
 
 struct tu_u_trace_syncobj;
 struct vdrm_bo;
@@ -65,6 +66,47 @@ struct tu_bo {
    struct vk_object_base *base;
 };
 
+enum tu_sparse_vma_flags {
+   TU_SPARSE_VMA_REPLAYABLE = 1 << 0,
+
+   /* Make unmapped pages in the memory region map to the PRR NULL page. This
+    * applies to all pages initially as well as pages that are subsequently
+    * unmapped via a queue submission. Writes to the PRR NULL page are
+    * dropped, and reads return zero. If this flag isn't set then reads and
+    * writes to unmapped pages will fault instead.
+    */
+   TU_SPARSE_VMA_MAP_ZERO = 1 << 1,
+};
+
+/* This represents a memory region into which BOs can be mapped. This is
+ * implemented differently on drm/msm and kgsl:
+ *
+ * - msm allows us to control the VA range ourselves, and provides an API to
+ *   map arbitrary parts of BOs to a given VA range. The sparse VMA is
+ *   just a userspace driver abstraction, consisting of an iova range we
+ *   reserve and (optionally) map as NULL initially, and when mapping a BO
+ *   into the sparse VMA we calculate the final iova range to map it to.
+ * - kgsl doesn't allow userspace control of the iova, and requires that we
+ *   create a "virtual BO" into which we can map BOs. The virtual BO maps
+ *   one-to-one to tu_sparse_vma, and almost one-to-one to a Vulkan VkBuffer
+ *   or VkImage with sparse binding.
+ *
+ * tu_sparse_vma is an abstraction to bridge this difference.
+ */
+struct tu_sparse_vma {
+   enum tu_sparse_vma_flags flags;
+
+   union {
+      struct {
+         uint64_t iova;
+         uint64_t size;
+      } msm;
+      struct {
+         struct tu_bo *virtual_bo;
+      } kgsl;
+   };
+};
+
 struct tu_knl {
    const char *name;
 
@@ -73,7 +115,7 @@ struct tu_knl {
    int (*device_get_gpu_timestamp)(struct tu_device *dev, uint64_t *ts);
    int (*device_get_suspend_count)(struct tu_device *dev, uint64_t *suspend_count);
    VkResult (*device_check_status)(struct tu_device *dev);
-   int (*submitqueue_new)(struct tu_device *dev, int priority, uint32_t *queue_id);
+   int (*submitqueue_new)(struct tu_device *dev, enum tu_queue_type type, int priority, uint32_t *queue_id);
    void (*submitqueue_close)(struct tu_device *dev, uint32_t queue_id);
    VkResult (*bo_init)(struct tu_device *dev, struct vk_object_base *base,
                        struct tu_bo **out_bo, uint64_t size, uint64_t client_iova,
@@ -94,12 +136,25 @@ struct tu_knl {
    void (*submit_add_entries)(struct tu_device *device, void *_submit,
                               struct tu_cs_entry *entries,
                               unsigned num_entries);
+   void (*submit_add_bind)(struct tu_device *device,
+                           void *_submit,
+                           struct tu_sparse_vma *vma, uint64_t vma_offset,
+                           struct tu_bo *bo, uint64_t bo_offset,
+                           uint64_t size);
    VkResult (*queue_submit)(struct tu_queue *queue, void *_submit,
                             struct vk_sync_wait *waits, uint32_t wait_count,
                             struct vk_sync_signal *signals, uint32_t signal_count,
                             struct tu_u_trace_submission_data *u_trace_submission_data);
    VkResult (*queue_wait_fence)(struct tu_queue *queue, uint32_t fence,
                                 uint64_t timeout_ns);
+   VkResult (*sparse_vma_init)(struct tu_device *dev,
+                               struct vk_object_base *base,
+                               struct tu_sparse_vma *out_vma,
+                               uint64_t *out_iova,
+                               enum tu_sparse_vma_flags flags,
+                               uint64_t size, uint64_t client_iova);
+   void (*sparse_vma_finish)(struct tu_device *device,
+                             struct tu_sparse_vma *vma);
 
    const struct vk_device_entrypoint_table *device_entrypoints;
 };
@@ -178,6 +233,16 @@ tu_bo_get_ref(struct tu_bo *bo)
    return bo;
 }
 
+VkResult tu_sparse_vma_init(struct tu_device *dev,
+                            struct vk_object_base *base,
+                            struct tu_sparse_vma *out_vma,
+                            uint64_t *out_iova,
+                            enum tu_sparse_vma_flags flags,
+                            uint64_t size, uint64_t client_iova);
+
+void tu_sparse_vma_finish(struct tu_device *device,
+                          struct tu_sparse_vma *vma);
+
 VkResult tu_knl_kgsl_load(struct tu_instance *instance, int fd);
 
 struct _drmVersion;
@@ -217,6 +282,7 @@ tu_device_check_status(struct vk_device *vk_device);
 
 int
 tu_drm_submitqueue_new(struct tu_device *dev,
+                       enum tu_queue_type type,
                        int priority,
                        uint32_t *queue_id);
 
@@ -234,6 +300,13 @@ tu_submit_add_entries(struct tu_device *dev, void *submit,
                       struct tu_cs_entry *entries,
                       unsigned num_entries);
 
+void
+tu_submit_add_bind(struct tu_device *device,
+                   void *_submit,
+                   struct tu_sparse_vma *vma, uint64_t vma_offset,
+                   struct tu_bo *bo, uint64_t bo_offset,
+                   uint64_t size);
+
 VkResult
 tu_queue_submit(struct tu_queue *queue, void *submit,
                 struct vk_sync_wait *waits, uint32_t wait_count,

src/freedreno/vulkan/tu_knl_drm_msm.cc

@@ -322,6 +322,7 @@ msm_device_check_status(struct tu_device *device)
 
 static int
 msm_submitqueue_new(struct tu_device *dev,
+                    enum tu_queue_type type,
                     int priority,
                     uint32_t *queue_id)
 {

src/freedreno/vulkan/tu_knl_drm_virtio.cc

@@ -346,6 +346,7 @@ virtio_device_check_status(struct tu_device *device)
 
 static int
 virtio_submitqueue_new(struct tu_device *dev,
+                       enum tu_queue_type type,
                        int priority,
                        uint32_t *queue_id)
 {

src/freedreno/vulkan/tu_knl_kgsl.cc

@@ -51,6 +51,7 @@ safe_ioctl(int fd, unsigned long request, void *arg)
 
 static int
 kgsl_submitqueue_new(struct tu_device *dev,
+                     enum tu_queue_type type,
                      int priority,
                      uint32_t *queue_id)
 {

src/freedreno/vulkan/tu_queue.cc

@@ -282,8 +282,9 @@ tu_queue_init(struct tu_device *device,
    queue->device = device;
    queue->priority = priority;
    queue->vk.driver_submit = queue_submit;
+   queue->type = TU_QUEUE_GFX;
 
-   int ret = tu_drm_submitqueue_new(device, priority, &queue->msm_queue_id);
+   int ret = tu_drm_submitqueue_new(device, TU_QUEUE_GFX, priority, &queue->msm_queue_id);
    if (ret)
       return vk_startup_errorf(device->instance, VK_ERROR_INITIALIZATION_FAILED,
                                "submitqueue create failed");

src/freedreno/vulkan/tu_queue.h

@@ -12,12 +12,19 @@
 
 #include "tu_common.h"
 
+enum tu_queue_type
+{
+   TU_QUEUE_GFX,
+   TU_QUEUE_SPARSE,
+};
+
 struct tu_queue
 {
    struct vk_queue vk;
 
    struct tu_device *device;
 
+   enum tu_queue_type type;
    uint32_t msm_queue_id;
    uint32_t priority;