venus: add a ring buffer

vn_ring is a fast way to submit commands, skipping vn_renderer entirely.

Signed-off-by: Chia-I Wu <olvaffe@gmail.com>
Reviewed-by: Ryan Neph <ryanneph@google.com>
Reviewed-by: Gert Wollny <gert.wollny@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5800>

src/virtio/vulkan/meson.build

@@ -35,6 +35,7 @@ libvn_files = files(
   'vn_cs.c',
   'vn_device.c',
   'vn_icd.c',
+  'vn_ring.c',
   'vn_renderer_virtgpu.c',
   'vn_renderer_vtest.c',
 )

src/virtio/vulkan/vn_common.c

@@ -102,3 +102,19 @@ vn_default_allocator(void)
    };
    return &allocator;
 }
+
+void
+vn_relax(uint32_t *iter)
+{
+   const uint32_t busy_wait_order = 4;
+   const uint32_t base_sleep_us = 10;
+
+   (*iter)++;
+   if (*iter < (1 << busy_wait_order)) {
+      thrd_yield();
+      return;
+   }
+
+   const uint32_t shift = util_last_bit(*iter) - busy_wait_order - 1;
+   os_time_sleep(base_sleep_us << shift);
+}

src/virtio/vulkan/vn_common.h

@@ -109,6 +109,9 @@ vn_log_result(struct vn_instance *instance,
 const VkAllocationCallbacks *
 vn_default_allocator(void);
 
+void
+vn_relax(uint32_t *iter);
+
 static_assert(sizeof(vn_object_id) >= sizeof(uintptr_t), "");
 
 static inline VkResult

src/virtio/vulkan/vn_ring.c

@@ -0,0 +1,235 @@
+/*
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "vn_ring.h"
+
+#include "vn_renderer.h"
+
+/* must be power-of-two */
+#define VN_RING_BUFFER_SIZE (1u << 11)
+#define VN_RING_BUFFER_MASK (VN_RING_BUFFER_SIZE - 1)
+
+enum vn_ring_status_flag {
+   VN_RING_STATUS_IDLE = 1u << 0,
+};
+
+static uint32_t
+vn_ring_load_head(const struct vn_ring *ring)
+{
+   /* the renderer is expected to store the head with memory_order_release,
+    * forming a release-acquire ordering
+    */
+   return atomic_load_explicit(ring->shared.head, memory_order_acquire);
+}
+
+static void
+vn_ring_store_tail(struct vn_ring *ring)
+{
+   /* the renderer is expected to load the tail with memory_order_acquire,
+    * forming a release-acquire ordering
+    */
+   return atomic_store_explicit(ring->shared.tail, ring->cur,
+                                memory_order_release);
+}
+
+static uint32_t
+vn_ring_load_status(const struct vn_ring *ring)
+{
+   /* this must be called and ordered after vn_ring_store_tail */
+   return atomic_load_explicit(ring->shared.status, memory_order_seq_cst);
+}
+
+static void
+vn_ring_write_buffer(struct vn_ring *ring, const void *data, size_t size)
+{
+   assert(ring->cur + size - vn_ring_load_head(ring) <= VN_RING_BUFFER_SIZE);
+
+   const size_t offset = ring->cur & VN_RING_BUFFER_MASK;
+   if (offset + size <= VN_RING_BUFFER_SIZE) {
+      memcpy(ring->shared.buffer + offset, data, size);
+   } else {
+      const size_t s = VN_RING_BUFFER_SIZE - offset;
+      memcpy(ring->shared.buffer + offset, data, s);
+      memcpy(ring->shared.buffer, data + s, size - s);
+   }
+
+   ring->cur += size;
+}
+
+static bool
+vn_ring_ge_seqno(const struct vn_ring *ring, uint32_t a, uint32_t b)
+{
+   /* this can return false negative when not called fast enough (e.g., when
+    * called once every couple hours), but following calls with larger a's
+    * will correct itself
+    *
+    * TODO use real seqnos?
+    */
+   if (a >= b)
+      return ring->cur >= a || ring->cur < b;
+   else
+      return ring->cur >= a && ring->cur < b;
+}
+
+static void
+vn_ring_retire_submits(struct vn_ring *ring, uint32_t seqno)
+{
+   list_for_each_entry_safe (struct vn_ring_submit, submit, &ring->submits,
+                             head) {
+      if (!vn_ring_ge_seqno(ring, seqno, submit->seqno))
+         break;
+
+      for (uint32_t i = 0; i < submit->bo_count; i++)
+         vn_renderer_bo_unref(submit->bos[i]);
+
+      list_del(&submit->head);
+      list_add(&submit->head, &ring->free_submits);
+   }
+}
+
+static uint32_t
+vn_ring_wait_seqno(const struct vn_ring *ring, uint32_t seqno)
+{
+   /* A renderer wait incurs several hops and the renderer might poll
+    * repeatedly anyway.  Let's just poll here.
+    */
+   uint32_t iter = 0;
+   do {
+      const uint32_t head = vn_ring_load_head(ring);
+      if (vn_ring_ge_seqno(ring, head, seqno))
+         return head;
+      vn_relax(&iter);
+   } while (true);
+}
+
+static uint32_t
+vn_ring_wait_space(const struct vn_ring *ring, uint32_t size)
+{
+   assert(size <= VN_RING_BUFFER_SIZE);
+
+   /* see the reasoning in vn_ring_wait_seqno */
+   uint32_t iter = 0;
+   do {
+      const uint32_t head = vn_ring_load_head(ring);
+      if (ring->cur + size - head <= VN_RING_BUFFER_SIZE)
+         return head;
+      vn_relax(&iter);
+   } while (true);
+}
+
+void
+vn_ring_get_layout(size_t extra_size, struct vn_ring_layout *layout)
+{
+   /* this can be changed/extended quite freely */
+   struct layout {
+      uint32_t head __attribute__((aligned(64)));
+      uint32_t tail __attribute__((aligned(64)));
+      uint32_t status __attribute__((aligned(64)));
+
+      uint8_t buffer[] __attribute__((aligned(64)));
+   };
+   const size_t buf_size = VN_RING_BUFFER_SIZE;
+
+   assert(buf_size && util_is_power_of_two_or_zero(buf_size));
+
+   layout->head_offset = offsetof(struct layout, head);
+   layout->tail_offset = offsetof(struct layout, tail);
+   layout->status_offset = offsetof(struct layout, status);
+
+   layout->buffer_offset = offsetof(struct layout, buffer);
+   layout->buffer_size = buf_size;
+
+   layout->extra_offset = layout->buffer_offset + layout->buffer_size;
+   layout->extra_size = extra_size;
+
+   layout->bo_size = layout->extra_offset + layout->extra_size;
+}
+
+void
+vn_ring_init(struct vn_ring *ring,
+             const struct vn_ring_layout *layout,
+             void *shared)
+{
+   memset(ring, 0, sizeof(*ring));
+   memset(shared, 0, layout->bo_size);
+
+   ring->shared.head = shared + layout->head_offset;
+   ring->shared.tail = shared + layout->tail_offset;
+   ring->shared.status = shared + layout->status_offset;
+   ring->shared.buffer = shared + layout->buffer_offset;
+   ring->shared.extra = shared + layout->extra_offset;
+
+   list_inithead(&ring->submits);
+   list_inithead(&ring->free_submits);
+}
+
+void
+vn_ring_fini(struct vn_ring *ring)
+{
+   vn_ring_retire_submits(ring, ring->cur);
+   assert(list_is_empty(&ring->submits));
+
+   list_for_each_entry_safe (struct vn_ring_submit, submit,
+                             &ring->free_submits, head)
+      free(submit);
+}
+
+struct vn_ring_submit *
+vn_ring_get_submit(struct vn_ring *ring, uint32_t bo_count)
+{
+   const uint32_t min_bo_count = 2;
+   struct vn_ring_submit *submit;
+
+   /* TODO this could be simplified if we could omit bo_count */
+   if (bo_count <= min_bo_count && !list_is_empty(&ring->free_submits)) {
+      submit =
+         list_first_entry(&ring->free_submits, struct vn_ring_submit, head);
+      list_del(&submit->head);
+   } else {
+      bo_count = MAX2(bo_count, min_bo_count);
+      submit = malloc(sizeof(*submit) + sizeof(submit->bos[0]) * bo_count);
+   }
+
+   return submit;
+}
+
+bool
+vn_ring_submit(struct vn_ring *ring,
+               struct vn_ring_submit *submit,
+               const void *cs_data,
+               size_t cs_size,
+               uint32_t *seqno)
+{
+   const uint32_t cur_seqno = vn_ring_wait_space(ring, cs_size);
+   vn_ring_write_buffer(ring, cs_data, cs_size);
+   vn_ring_store_tail(ring);
+   const bool notify = vn_ring_load_status(ring) & VN_RING_STATUS_IDLE;
+
+   vn_ring_retire_submits(ring, cur_seqno);
+
+   submit->seqno = ring->cur;
+   list_addtail(&submit->head, &ring->submits);
+
+   *seqno = submit->seqno;
+   return notify;
+}
+
+/**
+ * This is thread-safe.
+ */
+void
+vn_ring_wait(const struct vn_ring *ring, uint32_t seqno)
+{
+   vn_ring_wait_seqno(ring, seqno);
+}
+
+void
+vn_ring_wait_all(const struct vn_ring *ring)
+{
+   /* load from tail rather than ring->cur for atomicity */
+   const uint32_t pending_seqno =
+      atomic_load_explicit(ring->shared.tail, memory_order_relaxed);
+   vn_ring_wait(ring, pending_seqno);
+}

src/virtio/vulkan/vn_ring.h

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef VN_RING_H
+#define VN_RING_H
+
+#include "vn_common.h"
+
+/**
+ * A ring is a single-producer and single-consumer circular buffer.  The data
+ * in the buffer are produced and consumed in order.  An externally-defined
+ * mechanism is required for ring setup and notifications in both directions.
+ * Notifications for new data from the producer are needed only when the
+ * consumer is not actively polling, which is indicated by the ring status.
+ *
+ * For venus, the data are plain venus commands.  When a venus command is
+ * consumed from the ring's perspective, there can still be ongoing CPU and/or
+ * GPU works.  This is not an issue when the works generated by following
+ * venus commands are correctly queued after the ongoing works.  There are
+ * also venus commands that facilitate polling or waiting for ongoing works.
+ */
+
+/* the layout of a ring in a BO */
+struct vn_ring_layout {
+   size_t head_offset;
+   size_t tail_offset;
+   size_t status_offset;
+
+   size_t buffer_offset;
+   size_t buffer_size;
+
+   size_t extra_offset;
+   size_t extra_size;
+
+   size_t bo_size;
+};
+
+static_assert(ATOMIC_INT_LOCK_FREE == 2 && sizeof(atomic_uint) == 4,
+              "vn_ring_shared requires lock-free 32-bit atomic_uint");
+
+/* pointers to a ring in a BO */
+struct vn_ring_shared {
+   const volatile atomic_uint *head;
+   volatile atomic_uint *tail;
+   const volatile atomic_uint *status;
+   void *buffer;
+   void *extra;
+};
+
+struct vn_ring_submit {
+   uint32_t seqno;
+
+   struct list_head head;
+
+   /* BOs to keep alive (TODO make sure BOs are pinned) */
+   uint32_t bo_count;
+   struct vn_renderer_bo *bos[];
+};
+
+struct vn_ring {
+   struct vn_ring_shared shared;
+   uint32_t cur;
+
+   struct list_head submits;
+   struct list_head free_submits;
+};
+
+void
+vn_ring_get_layout(size_t extra_size, struct vn_ring_layout *layout);
+
+void
+vn_ring_init(struct vn_ring *ring,
+             const struct vn_ring_layout *layout,
+             void *shared);
+
+void
+vn_ring_fini(struct vn_ring *ring);
+
+struct vn_ring_submit *
+vn_ring_get_submit(struct vn_ring *ring, uint32_t bo_count);
+
+bool
+vn_ring_submit(struct vn_ring *ring,
+               struct vn_ring_submit *submit,
+               const void *cs_data,
+               size_t cs_size,
+               uint32_t *seqno);
+
+void
+vn_ring_wait(const struct vn_ring *ring, uint32_t seqno);
+
+void
+vn_ring_wait_all(const struct vn_ring *ring);
+
+#endif /* VN_RING_H */