From 83d7c2bb406a21c710127844dca0cad5eebdbcd7 Mon Sep 17 00:00:00 2001 From: Chia-I Wu Date: Mon, 1 Mar 2021 11:54:21 -0800 Subject: [PATCH] venus: add a ring buffer vn_ring is a fast way to submit commands, skipping vn_renderer entirely. Signed-off-by: Chia-I Wu Reviewed-by: Ryan Neph Reviewed-by: Gert Wollny Part-of: --- src/virtio/vulkan/meson.build | 1 + src/virtio/vulkan/vn_common.c | 16 +++ src/virtio/vulkan/vn_common.h | 3 + src/virtio/vulkan/vn_ring.c | 235 ++++++++++++++++++++++++++++++++++ src/virtio/vulkan/vn_ring.h | 97 ++++++++++++++ 5 files changed, 352 insertions(+) create mode 100644 src/virtio/vulkan/vn_ring.c create mode 100644 src/virtio/vulkan/vn_ring.h diff --git a/src/virtio/vulkan/meson.build b/src/virtio/vulkan/meson.build index 6c147911fa9..165d832494e 100644 --- a/src/virtio/vulkan/meson.build +++ b/src/virtio/vulkan/meson.build @@ -35,6 +35,7 @@ libvn_files = files( 'vn_cs.c', 'vn_device.c', 'vn_icd.c', + 'vn_ring.c', 'vn_renderer_virtgpu.c', 'vn_renderer_vtest.c', ) diff --git a/src/virtio/vulkan/vn_common.c b/src/virtio/vulkan/vn_common.c index f5754a5f222..799a96f9a01 100644 --- a/src/virtio/vulkan/vn_common.c +++ b/src/virtio/vulkan/vn_common.c @@ -102,3 +102,19 @@ vn_default_allocator(void) }; return &allocator; } + +void +vn_relax(uint32_t *iter) +{ + const uint32_t busy_wait_order = 4; + const uint32_t base_sleep_us = 10; + + (*iter)++; + if (*iter < (1 << busy_wait_order)) { + thrd_yield(); + return; + } + + const uint32_t shift = util_last_bit(*iter) - busy_wait_order - 1; + os_time_sleep(base_sleep_us << shift); +} diff --git a/src/virtio/vulkan/vn_common.h b/src/virtio/vulkan/vn_common.h index 612b2d29943..1c8ab8d9ea9 100644 --- a/src/virtio/vulkan/vn_common.h +++ b/src/virtio/vulkan/vn_common.h @@ -109,6 +109,9 @@ vn_log_result(struct vn_instance *instance, const VkAllocationCallbacks * vn_default_allocator(void); +void +vn_relax(uint32_t *iter); + static_assert(sizeof(vn_object_id) >= sizeof(uintptr_t), ""); static inline VkResult diff --git a/src/virtio/vulkan/vn_ring.c b/src/virtio/vulkan/vn_ring.c new file mode 100644 index 00000000000..812be7a4c1d --- /dev/null +++ b/src/virtio/vulkan/vn_ring.c @@ -0,0 +1,235 @@ +/* + * Copyright 2021 Google LLC + * SPDX-License-Identifier: MIT + */ + +#include "vn_ring.h" + +#include "vn_renderer.h" + +/* must be power-of-two */ +#define VN_RING_BUFFER_SIZE (1u << 11) +#define VN_RING_BUFFER_MASK (VN_RING_BUFFER_SIZE - 1) + +enum vn_ring_status_flag { + VN_RING_STATUS_IDLE = 1u << 0, +}; + +static uint32_t +vn_ring_load_head(const struct vn_ring *ring) +{ + /* the renderer is expected to store the head with memory_order_release, + * forming a release-acquire ordering + */ + return atomic_load_explicit(ring->shared.head, memory_order_acquire); +} + +static void +vn_ring_store_tail(struct vn_ring *ring) +{ + /* the renderer is expected to load the tail with memory_order_acquire, + * forming a release-acquire ordering + */ + return atomic_store_explicit(ring->shared.tail, ring->cur, + memory_order_release); +} + +static uint32_t +vn_ring_load_status(const struct vn_ring *ring) +{ + /* this must be called and ordered after vn_ring_store_tail */ + return atomic_load_explicit(ring->shared.status, memory_order_seq_cst); +} + +static void +vn_ring_write_buffer(struct vn_ring *ring, const void *data, size_t size) +{ + assert(ring->cur + size - vn_ring_load_head(ring) <= VN_RING_BUFFER_SIZE); + + const size_t offset = ring->cur & VN_RING_BUFFER_MASK; + if (offset + size <= VN_RING_BUFFER_SIZE) { + memcpy(ring->shared.buffer + offset, data, size); + } else { + const size_t s = VN_RING_BUFFER_SIZE - offset; + memcpy(ring->shared.buffer + offset, data, s); + memcpy(ring->shared.buffer, data + s, size - s); + } + + ring->cur += size; +} + +static bool +vn_ring_ge_seqno(const struct vn_ring *ring, uint32_t a, uint32_t b) +{ + /* this can return false negative when not called fast enough (e.g., when + * called once every couple hours), but following calls with larger a's + * will correct itself + * + * TODO use real seqnos? + */ + if (a >= b) + return ring->cur >= a || ring->cur < b; + else + return ring->cur >= a && ring->cur < b; +} + +static void +vn_ring_retire_submits(struct vn_ring *ring, uint32_t seqno) +{ + list_for_each_entry_safe (struct vn_ring_submit, submit, &ring->submits, + head) { + if (!vn_ring_ge_seqno(ring, seqno, submit->seqno)) + break; + + for (uint32_t i = 0; i < submit->bo_count; i++) + vn_renderer_bo_unref(submit->bos[i]); + + list_del(&submit->head); + list_add(&submit->head, &ring->free_submits); + } +} + +static uint32_t +vn_ring_wait_seqno(const struct vn_ring *ring, uint32_t seqno) +{ + /* A renderer wait incurs several hops and the renderer might poll + * repeatedly anyway. Let's just poll here. + */ + uint32_t iter = 0; + do { + const uint32_t head = vn_ring_load_head(ring); + if (vn_ring_ge_seqno(ring, head, seqno)) + return head; + vn_relax(&iter); + } while (true); +} + +static uint32_t +vn_ring_wait_space(const struct vn_ring *ring, uint32_t size) +{ + assert(size <= VN_RING_BUFFER_SIZE); + + /* see the reasoning in vn_ring_wait_seqno */ + uint32_t iter = 0; + do { + const uint32_t head = vn_ring_load_head(ring); + if (ring->cur + size - head <= VN_RING_BUFFER_SIZE) + return head; + vn_relax(&iter); + } while (true); +} + +void +vn_ring_get_layout(size_t extra_size, struct vn_ring_layout *layout) +{ + /* this can be changed/extended quite freely */ + struct layout { + uint32_t head __attribute__((aligned(64))); + uint32_t tail __attribute__((aligned(64))); + uint32_t status __attribute__((aligned(64))); + + uint8_t buffer[] __attribute__((aligned(64))); + }; + const size_t buf_size = VN_RING_BUFFER_SIZE; + + assert(buf_size && util_is_power_of_two_or_zero(buf_size)); + + layout->head_offset = offsetof(struct layout, head); + layout->tail_offset = offsetof(struct layout, tail); + layout->status_offset = offsetof(struct layout, status); + + layout->buffer_offset = offsetof(struct layout, buffer); + layout->buffer_size = buf_size; + + layout->extra_offset = layout->buffer_offset + layout->buffer_size; + layout->extra_size = extra_size; + + layout->bo_size = layout->extra_offset + layout->extra_size; +} + +void +vn_ring_init(struct vn_ring *ring, + const struct vn_ring_layout *layout, + void *shared) +{ + memset(ring, 0, sizeof(*ring)); + memset(shared, 0, layout->bo_size); + + ring->shared.head = shared + layout->head_offset; + ring->shared.tail = shared + layout->tail_offset; + ring->shared.status = shared + layout->status_offset; + ring->shared.buffer = shared + layout->buffer_offset; + ring->shared.extra = shared + layout->extra_offset; + + list_inithead(&ring->submits); + list_inithead(&ring->free_submits); +} + +void +vn_ring_fini(struct vn_ring *ring) +{ + vn_ring_retire_submits(ring, ring->cur); + assert(list_is_empty(&ring->submits)); + + list_for_each_entry_safe (struct vn_ring_submit, submit, + &ring->free_submits, head) + free(submit); +} + +struct vn_ring_submit * +vn_ring_get_submit(struct vn_ring *ring, uint32_t bo_count) +{ + const uint32_t min_bo_count = 2; + struct vn_ring_submit *submit; + + /* TODO this could be simplified if we could omit bo_count */ + if (bo_count <= min_bo_count && !list_is_empty(&ring->free_submits)) { + submit = + list_first_entry(&ring->free_submits, struct vn_ring_submit, head); + list_del(&submit->head); + } else { + bo_count = MAX2(bo_count, min_bo_count); + submit = malloc(sizeof(*submit) + sizeof(submit->bos[0]) * bo_count); + } + + return submit; +} + +bool +vn_ring_submit(struct vn_ring *ring, + struct vn_ring_submit *submit, + const void *cs_data, + size_t cs_size, + uint32_t *seqno) +{ + const uint32_t cur_seqno = vn_ring_wait_space(ring, cs_size); + vn_ring_write_buffer(ring, cs_data, cs_size); + vn_ring_store_tail(ring); + const bool notify = vn_ring_load_status(ring) & VN_RING_STATUS_IDLE; + + vn_ring_retire_submits(ring, cur_seqno); + + submit->seqno = ring->cur; + list_addtail(&submit->head, &ring->submits); + + *seqno = submit->seqno; + return notify; +} + +/** + * This is thread-safe. + */ +void +vn_ring_wait(const struct vn_ring *ring, uint32_t seqno) +{ + vn_ring_wait_seqno(ring, seqno); +} + +void +vn_ring_wait_all(const struct vn_ring *ring) +{ + /* load from tail rather than ring->cur for atomicity */ + const uint32_t pending_seqno = + atomic_load_explicit(ring->shared.tail, memory_order_relaxed); + vn_ring_wait(ring, pending_seqno); +} diff --git a/src/virtio/vulkan/vn_ring.h b/src/virtio/vulkan/vn_ring.h new file mode 100644 index 00000000000..c12aecc3e45 --- /dev/null +++ b/src/virtio/vulkan/vn_ring.h @@ -0,0 +1,97 @@ +/* + * Copyright 2021 Google LLC + * SPDX-License-Identifier: MIT + */ + +#ifndef VN_RING_H +#define VN_RING_H + +#include "vn_common.h" + +/** + * A ring is a single-producer and single-consumer circular buffer. The data + * in the buffer are produced and consumed in order. An externally-defined + * mechanism is required for ring setup and notifications in both directions. + * Notifications for new data from the producer are needed only when the + * consumer is not actively polling, which is indicated by the ring status. + * + * For venus, the data are plain venus commands. When a venus command is + * consumed from the ring's perspective, there can still be ongoing CPU and/or + * GPU works. This is not an issue when the works generated by following + * venus commands are correctly queued after the ongoing works. There are + * also venus commands that facilitate polling or waiting for ongoing works. + */ + +/* the layout of a ring in a BO */ +struct vn_ring_layout { + size_t head_offset; + size_t tail_offset; + size_t status_offset; + + size_t buffer_offset; + size_t buffer_size; + + size_t extra_offset; + size_t extra_size; + + size_t bo_size; +}; + +static_assert(ATOMIC_INT_LOCK_FREE == 2 && sizeof(atomic_uint) == 4, + "vn_ring_shared requires lock-free 32-bit atomic_uint"); + +/* pointers to a ring in a BO */ +struct vn_ring_shared { + const volatile atomic_uint *head; + volatile atomic_uint *tail; + const volatile atomic_uint *status; + void *buffer; + void *extra; +}; + +struct vn_ring_submit { + uint32_t seqno; + + struct list_head head; + + /* BOs to keep alive (TODO make sure BOs are pinned) */ + uint32_t bo_count; + struct vn_renderer_bo *bos[]; +}; + +struct vn_ring { + struct vn_ring_shared shared; + uint32_t cur; + + struct list_head submits; + struct list_head free_submits; +}; + +void +vn_ring_get_layout(size_t extra_size, struct vn_ring_layout *layout); + +void +vn_ring_init(struct vn_ring *ring, + const struct vn_ring_layout *layout, + void *shared); + +void +vn_ring_fini(struct vn_ring *ring); + +struct vn_ring_submit * +vn_ring_get_submit(struct vn_ring *ring, uint32_t bo_count); + +bool +vn_ring_submit(struct vn_ring *ring, + struct vn_ring_submit *submit, + const void *cs_data, + size_t cs_size, + uint32_t *seqno); + +void +vn_ring_wait(const struct vn_ring *ring, uint32_t seqno); + +void +vn_ring_wait_all(const struct vn_ring *ring); + +#endif /* VN_RING_H */