AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
4151a39b8ab5dbed26c6c0bedc528d2b64df5bf2
mesa/src/virtio/vulkan/vn_queue.c
Antonio Ospite ddf2aa3a4d build: avoid redefining unreachable() which is standard in C23 
In the C23 standard unreachable() is now a predefined function-like
macro in <stddef.h>

See https://android.googlesource.com/platform/bionic/+/HEAD/docs/c23.md#is-now-a-predefined-function_like-macro-in

And this causes build errors when building for C23:

-----------------------------------------------------------------------
In file included from ../src/util/log.h:30,
                 from ../src/util/log.c:30:
../src/util/macros.h:123:9: warning: "unreachable" redefined
  123 | #define unreachable(str)    \
      |         ^~~~~~~~~~~
In file included from ../src/util/macros.h:31:
/usr/lib/gcc/x86_64-linux-gnu/14/include/stddef.h:456:9: note: this is the location of the previous definition
  456 | #define unreachable() (__builtin_unreachable ())
      |         ^~~~~~~~~~~
-----------------------------------------------------------------------

So don't redefine it with the same name, but use the name UNREACHABLE()
to also signify it's a macro.

Using a different name also makes sense because the behavior of the
macro was extending the one of __builtin_unreachable() anyway, and it
also had a different signature, accepting one argument, compared to the
standard unreachable() with no arguments.

This change improves the chances of building mesa with the C23 standard,
which for instance is the default in recent AOSP versions.

All the instances of the macro, including the definition, were updated
with the following command line:

  git grep -l '[^_]unreachable(' -- "src/**" | sort | uniq | \
  while read file; \
  do \
    sed -e 's/\([^_]\)unreachable(/\1UNREACHABLE(/g' -i "$file"; \
  done && \
  sed -e 's/#undef unreachable/#undef UNREACHABLE/g' -i src/intel/isl/isl_aux_info.c

Reviewed-by: Erik Faye-Lund <erik.faye-lund@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/36437>
2025-07-31 17:49:42 +00:00

2477 lines
82 KiB
C
Raw Blame History

/*
* Copyright 2019 Google LLC
* SPDX-License-Identifier: MIT
*
* based in part on anv and radv which are:
* Copyright © 2015 Intel Corporation
* Copyright © 2016 Red Hat.
* Copyright © 2016 Bas Nieuwenhuizen
*/
#include "vn_queue.h"
#include "util/libsync.h"
#include "venus-protocol/vn_protocol_driver_event.h"
#include "venus-protocol/vn_protocol_driver_fence.h"
#include "venus-protocol/vn_protocol_driver_queue.h"
#include "venus-protocol/vn_protocol_driver_semaphore.h"
#include "venus-protocol/vn_protocol_driver_transport.h"
#include "vn_command_buffer.h"
#include "vn_device.h"
#include "vn_device_memory.h"
#include "vn_feedback.h"
#include "vn_instance.h"
#include "vn_physical_device.h"
#include "vn_query_pool.h"
#include "vn_renderer.h"
#include "vn_wsi.h"
/* queue commands */
struct vn_submit_info_pnext_fix {
VkDeviceGroupSubmitInfo group;
VkProtectedSubmitInfo protected;
VkTimelineSemaphoreSubmitInfo timeline;
};
struct vn_queue_submission {
VkStructureType batch_type;
VkQueue queue_handle;
uint32_t batch_count;
union {
const void *batches;
const VkSubmitInfo *submit_batches;
const VkSubmitInfo2 *submit2_batches;
const VkBindSparseInfo *sparse_batches;
};
VkFence fence_handle;
uint32_t cmd_count;
uint32_t feedback_types;
uint32_t pnext_count;
uint32_t dev_mask_count;
bool has_zink_sync_batch;
const struct vn_device_memory *wsi_mem;
struct vn_sync_payload_external external_payload;
/* Temporary storage allocation for submission
*
* A single alloc for storage is performed and the offsets inside storage
* are set as below:
*
* batches
* - non-empty submission: copy of original batches
* - empty submission: a single batch for fence feedback (ffb)
* cmds
* - for each batch:
* - copy of original batch cmds
* - a single cmd for query feedback (qfb)
* - one cmd for each signal semaphore that has feedback (sfb)
* - if last batch, a single cmd for ffb
*/
struct {
void *storage;
union {
void *batches;
VkSubmitInfo *submit_batches;
VkSubmitInfo2 *submit2_batches;
};
union {
void *cmds;
VkCommandBuffer *cmd_handles;
VkCommandBufferSubmitInfo *cmd_infos;
};
struct vn_submit_info_pnext_fix *pnexts;
uint32_t *dev_masks;
} temp;
};
static inline uint32_t
vn_get_wait_semaphore_count(struct vn_queue_submission *submit,
uint32_t batch_index)
{
switch (submit->batch_type) {
case VK_STRUCTURE_TYPE_SUBMIT_INFO:
return submit->submit_batches[batch_index].waitSemaphoreCount;
case VK_STRUCTURE_TYPE_SUBMIT_INFO_2:
return submit->submit2_batches[batch_index].waitSemaphoreInfoCount;
case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
return submit->sparse_batches[batch_index].waitSemaphoreCount;
default:
UNREACHABLE("unexpected batch type");
}
}
static inline uint32_t
vn_get_signal_semaphore_count(struct vn_queue_submission *submit,
uint32_t batch_index)
{
switch (submit->batch_type) {
case VK_STRUCTURE_TYPE_SUBMIT_INFO:
return submit->submit_batches[batch_index].signalSemaphoreCount;
case VK_STRUCTURE_TYPE_SUBMIT_INFO_2:
return submit->submit2_batches[batch_index].signalSemaphoreInfoCount;
case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
return submit->sparse_batches[batch_index].signalSemaphoreCount;
default:
UNREACHABLE("unexpected batch type");
}
}
static inline VkSemaphore
vn_get_wait_semaphore(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t semaphore_index)
{
switch (submit->batch_type) {
case VK_STRUCTURE_TYPE_SUBMIT_INFO:
return submit->submit_batches[batch_index]
.pWaitSemaphores[semaphore_index];
case VK_STRUCTURE_TYPE_SUBMIT_INFO_2:
return submit->submit2_batches[batch_index]
.pWaitSemaphoreInfos[semaphore_index]
.semaphore;
case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
return submit->sparse_batches[batch_index]
.pWaitSemaphores[semaphore_index];
default:
UNREACHABLE("unexpected batch type");
}
}
static inline VkSemaphore
vn_get_signal_semaphore(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t semaphore_index)
{
switch (submit->batch_type) {
case VK_STRUCTURE_TYPE_SUBMIT_INFO:
return submit->submit_batches[batch_index]
.pSignalSemaphores[semaphore_index];
case VK_STRUCTURE_TYPE_SUBMIT_INFO_2:
return submit->submit2_batches[batch_index]
.pSignalSemaphoreInfos[semaphore_index]
.semaphore;
case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
return submit->sparse_batches[batch_index]
.pSignalSemaphores[semaphore_index];
default:
UNREACHABLE("unexpected batch type");
}
}
static inline size_t
vn_get_batch_size(struct vn_queue_submission *submit)
{
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
? sizeof(VkSubmitInfo)
: sizeof(VkSubmitInfo2);
}
static inline size_t
vn_get_cmd_size(struct vn_queue_submission *submit)
{
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
? sizeof(VkCommandBuffer)
: sizeof(VkCommandBufferSubmitInfo);
}
static inline uint32_t
vn_get_cmd_count(struct vn_queue_submission *submit, uint32_t batch_index)
{
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
? submit->submit_batches[batch_index].commandBufferCount
: submit->submit2_batches[batch_index].commandBufferInfoCount;
}
static inline const void *
vn_get_cmds(struct vn_queue_submission *submit, uint32_t batch_index)
{
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
? (const void *)submit->submit_batches[batch_index]
.pCommandBuffers
: (const void *)submit->submit2_batches[batch_index]
.pCommandBufferInfos;
}
static inline struct vn_command_buffer *
vn_get_cmd(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t cmd_index)
{
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
return vn_command_buffer_from_handle(
submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
? submit->submit_batches[batch_index].pCommandBuffers[cmd_index]
: submit->submit2_batches[batch_index]
.pCommandBufferInfos[cmd_index]
.commandBuffer);
}
static inline void
vn_set_temp_cmd(struct vn_queue_submission *submit,
uint32_t cmd_index,
VkCommandBuffer cmd_handle)
{
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
submit->temp.cmd_infos[cmd_index] = (VkCommandBufferSubmitInfo){
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO,
.commandBuffer = cmd_handle,
};
} else {
submit->temp.cmd_handles[cmd_index] = cmd_handle;
}
}
static uint64_t
vn_get_signal_semaphore_counter(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t sem_index)
{
switch (submit->batch_type) {
case VK_STRUCTURE_TYPE_SUBMIT_INFO: {
const struct VkTimelineSemaphoreSubmitInfo *timeline_sem_info =
vk_find_struct_const(submit->submit_batches[batch_index].pNext,
TIMELINE_SEMAPHORE_SUBMIT_INFO);
return timeline_sem_info->pSignalSemaphoreValues[sem_index];
}
case VK_STRUCTURE_TYPE_SUBMIT_INFO_2:
return submit->submit2_batches[batch_index]
.pSignalSemaphoreInfos[sem_index]
.value;
default:
UNREACHABLE("unexpected batch type");
}
}
static bool
vn_has_zink_sync_batch(struct vn_queue_submission *submit)
{
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
struct vn_device *dev = vn_device_from_vk(queue->base.vk.base.device);
struct vn_instance *instance = dev->instance;
const uint32_t last_batch_index = submit->batch_count - 1;
if (!instance->engine_is_zink)
return false;
if (!submit->batch_count || !last_batch_index ||
vn_get_cmd_count(submit, last_batch_index))
return false;
if (vn_get_wait_semaphore_count(submit, last_batch_index))
return false;
const uint32_t signal_count =
vn_get_signal_semaphore_count(submit, last_batch_index);
for (uint32_t i = 0; i < signal_count; i++) {
struct vn_semaphore *sem = vn_semaphore_from_handle(
vn_get_signal_semaphore(submit, last_batch_index, i));
if (sem->feedback.slot) {
return true;
}
}
return false;
}
static bool
vn_fix_batch_cmd_count_for_zink_sync(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t new_cmd_count)
{
/* If the last batch is a zink sync batch which is empty but contains
* feedback, append the feedback to the previous batch instead so that
* the last batch remains empty for perf.
*/
if (batch_index == submit->batch_count - 1 &&
submit->has_zink_sync_batch) {
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
VkSubmitInfo2 *batch =
&submit->temp.submit2_batches[batch_index - 1];
assert(batch->pCommandBufferInfos);
batch->commandBufferInfoCount += new_cmd_count;
} else {
VkSubmitInfo *batch = &submit->temp.submit_batches[batch_index - 1];
assert(batch->pCommandBuffers);
batch->commandBufferCount += new_cmd_count;
}
return true;
}
return false;
}
static void
vn_fix_device_group_cmd_count(struct vn_queue_submission *submit,
uint32_t batch_index)
{
struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle);
struct vn_device *dev = vn_device_from_vk(queue_vk->base.device);
const VkSubmitInfo *src_batch = &submit->submit_batches[batch_index];
struct vn_submit_info_pnext_fix *pnext_fix = submit->temp.pnexts;
VkBaseOutStructure *dst =
(void *)&submit->temp.submit_batches[batch_index];
uint32_t new_cmd_count =
submit->temp.submit_batches[batch_index].commandBufferCount;
vk_foreach_struct_const(src, src_batch->pNext) {
void *pnext = NULL;
switch (src->sType) {
case VK_STRUCTURE_TYPE_DEVICE_GROUP_SUBMIT_INFO: {
uint32_t orig_cmd_count = 0;
memcpy(&pnext_fix->group, src, sizeof(pnext_fix->group));
VkDeviceGroupSubmitInfo *src_device_group =
(VkDeviceGroupSubmitInfo *)src;
if (src_device_group->commandBufferCount) {
orig_cmd_count = src_device_group->commandBufferCount;
memcpy(submit->temp.dev_masks,
src_device_group->pCommandBufferDeviceMasks,
sizeof(uint32_t) * orig_cmd_count);
}
/* Set the group device mask. Unlike sync2, zero means skip. */
for (uint32_t i = orig_cmd_count; i < new_cmd_count; i++) {
submit->temp.dev_masks[i] = dev->device_mask;
}
pnext_fix->group.commandBufferCount = new_cmd_count;
pnext_fix->group.pCommandBufferDeviceMasks = submit->temp.dev_masks;
pnext = &pnext_fix->group;
break;
}
case VK_STRUCTURE_TYPE_PROTECTED_SUBMIT_INFO:
memcpy(&pnext_fix->protected, src, sizeof(pnext_fix->protected));
pnext = &pnext_fix->protected;
break;
case VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO:
memcpy(&pnext_fix->timeline, src, sizeof(pnext_fix->timeline));
pnext = &pnext_fix->timeline;
break;
default:
/* The following structs are not supported by venus so are not
* handled here. VkAmigoProfilingSubmitInfoSEC,
* VkD3D12FenceSubmitInfoKHR, VkFrameBoundaryEXT,
* VkLatencySubmissionPresentIdNV, VkPerformanceQuerySubmitInfoKHR,
* VkWin32KeyedMutexAcquireReleaseInfoKHR,
* VkWin32KeyedMutexAcquireReleaseInfoNV
*/
break;
}
if (pnext) {
dst->pNext = pnext;
dst = pnext;
}
}
submit->temp.pnexts++;
submit->temp.dev_masks += new_cmd_count;
}
static bool
vn_semaphore_wait_external(struct vn_device *dev, struct vn_semaphore *sem);
static VkResult
vn_queue_submission_fix_batch_semaphores(struct vn_queue_submission *submit,
uint32_t batch_index)
{
struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle);
VkDevice dev_handle = vk_device_to_handle(queue_vk->base.device);
struct vn_device *dev = vn_device_from_handle(dev_handle);
const uint32_t wait_count =
vn_get_wait_semaphore_count(submit, batch_index);
for (uint32_t i = 0; i < wait_count; i++) {
VkSemaphore sem_handle = vn_get_wait_semaphore(submit, batch_index, i);
struct vn_semaphore *sem = vn_semaphore_from_handle(sem_handle);
const struct vn_sync_payload *payload = sem->payload;
if (payload->type != VN_SYNC_TYPE_IMPORTED_SYNC_FD)
continue;
if (!vn_semaphore_wait_external(dev, sem))
return VK_ERROR_DEVICE_LOST;
assert(dev->physical_device->renderer_sync_fd.semaphore_importable);
const VkImportSemaphoreResourceInfoMESA res_info = {
.sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_RESOURCE_INFO_MESA,
.semaphore = sem_handle,
.resourceId = 0,
};
vn_async_vkImportSemaphoreResourceMESA(dev->primary_ring, dev_handle,
&res_info);
}
return VK_SUCCESS;
}
static void
vn_queue_submission_count_batch_feedback(struct vn_queue_submission *submit,
uint32_t batch_index)
{
const uint32_t signal_count =
vn_get_signal_semaphore_count(submit, batch_index);
uint32_t extra_cmd_count = 0;
uint32_t feedback_types = 0;
for (uint32_t i = 0; i < signal_count; i++) {
struct vn_semaphore *sem = vn_semaphore_from_handle(
vn_get_signal_semaphore(submit, batch_index, i));
if (sem->feedback.slot) {
feedback_types |= VN_FEEDBACK_TYPE_SEMAPHORE;
extra_cmd_count++;
}
}
if (submit->batch_type != VK_STRUCTURE_TYPE_BIND_SPARSE_INFO) {
const uint32_t cmd_count = vn_get_cmd_count(submit, batch_index);
for (uint32_t i = 0; i < cmd_count; i++) {
struct vn_command_buffer *cmd = vn_get_cmd(submit, batch_index, i);
if (!list_is_empty(&cmd->builder.query_records))
feedback_types |= VN_FEEDBACK_TYPE_QUERY;
/* If a cmd that was submitted previously and already has a feedback
* cmd linked, as long as
* VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT was not set we can
* assume it has completed execution and is no longer in the pending
* state so its safe to recycle the old feedback command.
*/
if (cmd->linked_qfb_cmd) {
assert(!cmd->builder.is_simultaneous);
vn_query_feedback_cmd_free(cmd->linked_qfb_cmd);
cmd->linked_qfb_cmd = NULL;
}
}
if (feedback_types & VN_FEEDBACK_TYPE_QUERY)
extra_cmd_count++;
if (submit->feedback_types & VN_FEEDBACK_TYPE_FENCE &&
batch_index == submit->batch_count - 1) {
feedback_types |= VN_FEEDBACK_TYPE_FENCE;
extra_cmd_count++;
}
/* Space to copy the original cmds to append feedback to it.
* If the last batch is a zink sync batch which is an empty batch with
* sem feedback, feedback will be appended to the second to last batch
* so also need to copy the second to last batch's original cmds even
* if it doesn't have feedback itself.
*/
if (feedback_types || (batch_index == submit->batch_count - 2 &&
submit->has_zink_sync_batch)) {
extra_cmd_count += cmd_count;
}
}
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO &&
extra_cmd_count) {
const VkDeviceGroupSubmitInfo *device_group = vk_find_struct_const(
submit->submit_batches[batch_index].pNext, DEVICE_GROUP_SUBMIT_INFO);
if (device_group) {
submit->pnext_count++;
submit->dev_mask_count += extra_cmd_count;
}
}
submit->feedback_types |= feedback_types;
submit->cmd_count += extra_cmd_count;
}
static VkResult
vn_queue_submission_prepare(struct vn_queue_submission *submit)
{
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
struct vn_fence *fence = vn_fence_from_handle(submit->fence_handle);
assert(!fence || !fence->is_external || !fence->feedback.slot);
if (fence && fence->feedback.slot)
submit->feedback_types |= VN_FEEDBACK_TYPE_FENCE;
if (submit->batch_type != VK_STRUCTURE_TYPE_BIND_SPARSE_INFO)
submit->has_zink_sync_batch = vn_has_zink_sync_batch(submit);
submit->external_payload.ring_idx = queue->ring_idx;
submit->wsi_mem = NULL;
if (submit->batch_count == 1 &&
submit->batch_type != VK_STRUCTURE_TYPE_BIND_SPARSE_INFO) {
const struct wsi_memory_signal_submit_info *info = vk_find_struct_const(
submit->submit_batches[0].pNext, WSI_MEMORY_SIGNAL_SUBMIT_INFO_MESA);
if (info)
submit->wsi_mem = vn_device_memory_from_handle(info->memory);
}
for (uint32_t i = 0; i < submit->batch_count; i++) {
VkResult result = vn_queue_submission_fix_batch_semaphores(submit, i);
if (result != VK_SUCCESS)
return result;
vn_queue_submission_count_batch_feedback(submit, i);
}
return VK_SUCCESS;
}
static VkResult
vn_queue_submission_alloc_storage(struct vn_queue_submission *submit)
{
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
if (!submit->feedback_types)
return VK_SUCCESS;
/* for original batches or a new batch to hold feedback fence cmd */
const size_t total_batch_size =
vn_get_batch_size(submit) * MAX2(submit->batch_count, 1);
/* for fence, timeline semaphore and query feedback cmds */
const size_t total_cmd_size =
vn_get_cmd_size(submit) * MAX2(submit->cmd_count, 1);
/* for fixing command buffer counts in device group info, if it exists */
const size_t total_pnext_size =
submit->pnext_count * sizeof(struct vn_submit_info_pnext_fix);
const size_t total_dev_mask_size =
submit->dev_mask_count * sizeof(uint32_t);
submit->temp.storage = vn_cached_storage_get(
&queue->storage, total_batch_size + total_cmd_size + total_pnext_size +
total_dev_mask_size);
if (!submit->temp.storage)
return VK_ERROR_OUT_OF_HOST_MEMORY;
submit->temp.batches = submit->temp.storage;
submit->temp.cmds = submit->temp.storage + total_batch_size;
submit->temp.pnexts =
submit->temp.storage + total_batch_size + total_cmd_size;
submit->temp.dev_masks = submit->temp.storage + total_batch_size +
total_cmd_size + total_pnext_size;
return VK_SUCCESS;
}
static VkResult
vn_queue_submission_get_resolved_query_records(
struct vn_queue_submission *submit,
uint32_t batch_index,
struct vn_feedback_cmd_pool *fb_cmd_pool,
struct list_head *resolved_records)
{
struct vn_command_pool *cmd_pool =
vn_command_pool_from_handle(fb_cmd_pool->pool_handle);
struct list_head dropped_records;
VkResult result = VK_SUCCESS;
list_inithead(resolved_records);
list_inithead(&dropped_records);
const uint32_t cmd_count = vn_get_cmd_count(submit, batch_index);
for (uint32_t i = 0; i < cmd_count; i++) {
struct vn_command_buffer *cmd = vn_get_cmd(submit, batch_index, i);
list_for_each_entry(struct vn_cmd_query_record, record,
&cmd->builder.query_records, head) {
if (!record->copy) {
list_for_each_entry_safe(struct vn_cmd_query_record, prev,
resolved_records, head) {
/* If we previously added a query feedback that is now getting
* reset, remove it since it is now a no-op and the deferred
* feedback copy will cause a hang waiting for the reset query
* to become available.
*/
if (prev->copy && prev->query_pool == record->query_pool &&
prev->query >= record->query &&
prev->query < record->query + record->query_count)
list_move_to(&prev->head, &dropped_records);
}
}
simple_mtx_lock(&fb_cmd_pool->mutex);
struct vn_cmd_query_record *curr = vn_cmd_pool_alloc_query_record(
cmd_pool, record->query_pool, record->query, record->query_count,
record->copy);
simple_mtx_unlock(&fb_cmd_pool->mutex);
if (!curr) {
list_splicetail(resolved_records, &dropped_records);
result = VK_ERROR_OUT_OF_HOST_MEMORY;
goto out_free_dropped_records;
}
list_addtail(&curr->head, resolved_records);
}
}
/* further resolve to batch sequential queries */
struct vn_cmd_query_record *curr =
list_first_entry(resolved_records, struct vn_cmd_query_record, head);
list_for_each_entry_safe(struct vn_cmd_query_record, next,
resolved_records, head) {
if (curr->query_pool == next->query_pool && curr->copy == next->copy) {
if (curr->query + curr->query_count == next->query) {
curr->query_count += next->query_count;
list_move_to(&next->head, &dropped_records);
} else if (curr->query == next->query + next->query_count) {
curr->query = next->query;
curr->query_count += next->query_count;
list_move_to(&next->head, &dropped_records);
} else {
curr = next;
}
} else {
curr = next;
}
}
out_free_dropped_records:
simple_mtx_lock(&fb_cmd_pool->mutex);
vn_cmd_pool_free_query_records(cmd_pool, &dropped_records);
simple_mtx_unlock(&fb_cmd_pool->mutex);
return result;
}
static VkResult
vn_queue_submission_add_query_feedback(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t *new_cmd_count)
{
struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle);
struct vn_device *dev = vn_device_from_vk(queue_vk->base.device);
VkResult result;
struct vn_feedback_cmd_pool *fb_cmd_pool = NULL;
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
if (dev->queue_families[i] == queue_vk->queue_family_index) {
fb_cmd_pool = &dev->fb_cmd_pools[i];
break;
}
}
assert(fb_cmd_pool);
struct list_head resolved_records;
result = vn_queue_submission_get_resolved_query_records(
submit, batch_index, fb_cmd_pool, &resolved_records);
if (result != VK_SUCCESS)
return result;
/* currently the reset query is always recorded */
assert(!list_is_empty(&resolved_records));
struct vn_query_feedback_cmd *qfb_cmd;
result = vn_query_feedback_cmd_alloc(vn_device_to_handle(dev), fb_cmd_pool,
&resolved_records, &qfb_cmd);
if (result == VK_SUCCESS) {
/* link query feedback cmd lifecycle with a cmd in the original batch so
* that the feedback cmd can be reset and recycled when that cmd gets
* reset/freed.
*
* Avoid cmd buffers with VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT
* since we don't know if all its instances have completed execution.
* Should be rare enough to just log and leak the feedback cmd.
*/
bool found_companion_cmd = false;
const uint32_t cmd_count = vn_get_cmd_count(submit, batch_index);
for (uint32_t i = 0; i < cmd_count; i++) {
struct vn_command_buffer *cmd = vn_get_cmd(submit, batch_index, i);
if (!cmd->builder.is_simultaneous) {
cmd->linked_qfb_cmd = qfb_cmd;
found_companion_cmd = true;
break;
}
}
if (!found_companion_cmd)
vn_log(dev->instance, "WARN: qfb cmd has leaked!");
vn_set_temp_cmd(submit, (*new_cmd_count)++, qfb_cmd->cmd_handle);
}
simple_mtx_lock(&fb_cmd_pool->mutex);
vn_cmd_pool_free_query_records(
vn_command_pool_from_handle(fb_cmd_pool->pool_handle),
&resolved_records);
simple_mtx_unlock(&fb_cmd_pool->mutex);
return result;
}
struct vn_semaphore_feedback_cmd *
vn_semaphore_get_feedback_cmd(struct vn_device *dev,
struct vn_semaphore *sem);
static VkResult
vn_queue_submission_add_semaphore_feedback(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t signal_index,
uint32_t *new_cmd_count)
{
struct vn_semaphore *sem = vn_semaphore_from_handle(
vn_get_signal_semaphore(submit, batch_index, signal_index));
if (!sem->feedback.slot)
return VK_SUCCESS;
VK_FROM_HANDLE(vk_queue, queue_vk, submit->queue_handle);
struct vn_device *dev = vn_device_from_vk(queue_vk->base.device);
struct vn_semaphore_feedback_cmd *sfb_cmd =
vn_semaphore_get_feedback_cmd(dev, sem);
if (!sfb_cmd)
return VK_ERROR_OUT_OF_HOST_MEMORY;
const uint64_t counter =
vn_get_signal_semaphore_counter(submit, batch_index, signal_index);
vn_feedback_set_counter(sfb_cmd->src_slot, counter);
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
if (dev->queue_families[i] == queue_vk->queue_family_index) {
vn_set_temp_cmd(submit, (*new_cmd_count)++, sfb_cmd->cmd_handles[i]);
return VK_SUCCESS;
}
}
UNREACHABLE("bad feedback sem");
}
static void
vn_queue_submission_add_fence_feedback(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t *new_cmd_count)
{
VK_FROM_HANDLE(vk_queue, queue_vk, submit->queue_handle);
struct vn_device *dev = vn_device_from_vk(queue_vk->base.device);
struct vn_fence *fence = vn_fence_from_handle(submit->fence_handle);
VkCommandBuffer ffb_cmd_handle = VK_NULL_HANDLE;
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
if (dev->queue_families[i] == queue_vk->queue_family_index) {
ffb_cmd_handle = fence->feedback.commands[i];
}
}
assert(ffb_cmd_handle != VK_NULL_HANDLE);
vn_set_temp_cmd(submit, (*new_cmd_count)++, ffb_cmd_handle);
}
static VkResult
vn_queue_submission_add_feedback_cmds(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t feedback_types)
{
VkResult result;
uint32_t new_cmd_count = vn_get_cmd_count(submit, batch_index);
if (feedback_types & VN_FEEDBACK_TYPE_QUERY) {
result = vn_queue_submission_add_query_feedback(submit, batch_index,
&new_cmd_count);
if (result != VK_SUCCESS)
return result;
}
if (feedback_types & VN_FEEDBACK_TYPE_SEMAPHORE) {
const uint32_t signal_count =
vn_get_signal_semaphore_count(submit, batch_index);
for (uint32_t i = 0; i < signal_count; i++) {
result = vn_queue_submission_add_semaphore_feedback(
submit, batch_index, i, &new_cmd_count);
if (result != VK_SUCCESS)
return result;
}
if (vn_fix_batch_cmd_count_for_zink_sync(submit, batch_index,
new_cmd_count))
return VK_SUCCESS;
}
if (feedback_types & VN_FEEDBACK_TYPE_FENCE) {
vn_queue_submission_add_fence_feedback(submit, batch_index,
&new_cmd_count);
}
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
VkSubmitInfo2 *batch = &submit->temp.submit2_batches[batch_index];
batch->pCommandBufferInfos = submit->temp.cmd_infos;
batch->commandBufferInfoCount = new_cmd_count;
} else {
VkSubmitInfo *batch = &submit->temp.submit_batches[batch_index];
batch->pCommandBuffers = submit->temp.cmd_handles;
batch->commandBufferCount = new_cmd_count;
const VkDeviceGroupSubmitInfo *device_group = vk_find_struct_const(
submit->submit_batches[batch_index].pNext, DEVICE_GROUP_SUBMIT_INFO);
if (device_group)
vn_fix_device_group_cmd_count(submit, batch_index);
}
return VK_SUCCESS;
}
static VkResult
vn_queue_submission_setup_batch(struct vn_queue_submission *submit,
uint32_t batch_index)
{
uint32_t feedback_types = 0;
uint32_t extra_cmd_count = 0;
const uint32_t signal_count =
vn_get_signal_semaphore_count(submit, batch_index);
for (uint32_t i = 0; i < signal_count; i++) {
struct vn_semaphore *sem = vn_semaphore_from_handle(
vn_get_signal_semaphore(submit, batch_index, i));
if (sem->feedback.slot) {
feedback_types |= VN_FEEDBACK_TYPE_SEMAPHORE;
extra_cmd_count++;
}
}
const uint32_t cmd_count = vn_get_cmd_count(submit, batch_index);
for (uint32_t i = 0; i < cmd_count; i++) {
struct vn_command_buffer *cmd = vn_get_cmd(submit, batch_index, i);
if (!list_is_empty(&cmd->builder.query_records)) {
feedback_types |= VN_FEEDBACK_TYPE_QUERY;
extra_cmd_count++;
break;
}
}
if (submit->feedback_types & VN_FEEDBACK_TYPE_FENCE &&
batch_index == submit->batch_count - 1) {
feedback_types |= VN_FEEDBACK_TYPE_FENCE;
extra_cmd_count++;
}
/* If the batch has qfb, sfb or ffb, copy the original commands and append
* feedback cmds.
* If this is the second to last batch and the last batch a zink sync batch
* which is empty but has feedback, also copy the original commands for
* this batch so that the last batch's feedback can be appended to it.
*/
if (feedback_types || (batch_index == submit->batch_count - 2 &&
submit->has_zink_sync_batch)) {
const size_t cmd_size = vn_get_cmd_size(submit);
const size_t total_cmd_size = cmd_count * cmd_size;
/* copy only needed for non-empty batches */
if (total_cmd_size) {
memcpy(submit->temp.cmds, vn_get_cmds(submit, batch_index),
total_cmd_size);
}
VkResult result = vn_queue_submission_add_feedback_cmds(
submit, batch_index, feedback_types);
if (result != VK_SUCCESS)
return result;
/* advance the temp cmds for working on next batch cmds */
submit->temp.cmds += total_cmd_size + (extra_cmd_count * cmd_size);
}
return VK_SUCCESS;
}
static VkResult
vn_queue_submission_setup_batches(struct vn_queue_submission *submit)
{
assert(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2 ||
submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO);
if (!submit->feedback_types)
return VK_SUCCESS;
/* For a submission that is:
* - non-empty: copy batches for adding feedbacks
* - empty: initialize a batch for fence feedback
*/
if (submit->batch_count) {
memcpy(submit->temp.batches, submit->batches,
vn_get_batch_size(submit) * submit->batch_count);
} else {
assert(submit->feedback_types & VN_FEEDBACK_TYPE_FENCE);
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
submit->temp.submit2_batches[0] = (VkSubmitInfo2){
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
};
} else {
submit->temp.submit_batches[0] = (VkSubmitInfo){
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
};
}
submit->batch_count = 1;
submit->batches = submit->temp.batches;
}
for (uint32_t i = 0; i < submit->batch_count; i++) {
VkResult result = vn_queue_submission_setup_batch(submit, i);
if (result != VK_SUCCESS)
return result;
}
submit->batches = submit->temp.batches;
return VK_SUCCESS;
}
static void
vn_queue_submission_cleanup_semaphore_feedback(
struct vn_queue_submission *submit)
{
struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle);
VkDevice dev_handle = vk_device_to_handle(queue_vk->base.device);
for (uint32_t i = 0; i < submit->batch_count; i++) {
const uint32_t wait_count = vn_get_wait_semaphore_count(submit, i);
for (uint32_t j = 0; j < wait_count; j++) {
VkSemaphore sem_handle = vn_get_wait_semaphore(submit, i, j);
struct vn_semaphore *sem = vn_semaphore_from_handle(sem_handle);
if (!sem->feedback.slot)
continue;
/* sfb pending cmds are recycled when signaled counter is updated */
uint64_t counter = 0;
vn_GetSemaphoreCounterValue(dev_handle, sem_handle, &counter);
}
const uint32_t signal_count = vn_get_signal_semaphore_count(submit, i);
for (uint32_t j = 0; j < signal_count; j++) {
VkSemaphore sem_handle = vn_get_signal_semaphore(submit, i, j);
struct vn_semaphore *sem = vn_semaphore_from_handle(sem_handle);
if (!sem->feedback.slot)
continue;
/* sfb pending cmds are recycled when signaled counter is updated */
uint64_t counter = 0;
vn_GetSemaphoreCounterValue(dev_handle, sem_handle, &counter);
}
}
}
static void
vn_queue_submission_cleanup(struct vn_queue_submission *submit)
{
/* TODO clean up pending src feedbacks on failure? */
if (submit->feedback_types & VN_FEEDBACK_TYPE_SEMAPHORE)
vn_queue_submission_cleanup_semaphore_feedback(submit);
}
static VkResult
vn_queue_submission_prepare_submit(struct vn_queue_submission *submit)
{
VkResult result = vn_queue_submission_prepare(submit);
if (result != VK_SUCCESS)
return result;
result = vn_queue_submission_alloc_storage(submit);
if (result != VK_SUCCESS)
return result;
result = vn_queue_submission_setup_batches(submit);
if (result != VK_SUCCESS) {
vn_queue_submission_cleanup(submit);
return result;
}
return VK_SUCCESS;
}
static void
vn_queue_wsi_present(struct vn_queue_submission *submit)
{
struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle);
struct vn_device *dev = vn_device_from_vk(queue_vk->base.device);
if (!submit->wsi_mem)
return;
if (dev->renderer->info.has_implicit_fencing) {
struct vn_renderer_submit_batch batch = {
.ring_idx = submit->external_payload.ring_idx,
};
uint32_t local_data[8];
struct vn_cs_encoder local_enc =
VN_CS_ENCODER_INITIALIZER_LOCAL(local_data, sizeof(local_data));
if (submit->external_payload.ring_seqno_valid) {
const uint64_t ring_id = vn_ring_get_id(dev->primary_ring);
vn_encode_vkWaitRingSeqnoMESA(&local_enc, 0, ring_id,
submit->external_payload.ring_seqno);
batch.cs_data = local_data;
batch.cs_size = vn_cs_encoder_get_len(&local_enc);
}
const struct vn_renderer_submit renderer_submit = {
.bos = &submit->wsi_mem->base_bo,
.bo_count = 1,
.batches = &batch,
.batch_count = 1,
};
vn_renderer_submit(dev->renderer, &renderer_submit);
} else {
if (VN_DEBUG(WSI)) {
static uint32_t num_rate_limit_warning = 0;
if (num_rate_limit_warning++ < 10)
vn_log(dev->instance,
"forcing vkQueueWaitIdle before presenting");
}
vn_QueueWaitIdle(submit->queue_handle);
}
}
static VkResult
vn_queue_submit(struct vn_queue_submission *submit)
{
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
struct vn_device *dev = vn_device_from_vk(queue->base.vk.base.device);
struct vn_instance *instance = dev->instance;
VkResult result;
/* To ensure external components waiting on the correct fence payload,
* below sync primitives must be installed after the submission:
* - explicit fencing: sync file export
* - implicit fencing: dma-fence attached to the wsi bo
*
* We enforce above via an asynchronous vkQueueSubmit(2) via ring followed
* by an asynchronous renderer submission to wait for the ring submission:
* - struct wsi_memory_signal_submit_info
* - fence is an external fence
* - has an external signal semaphore
*/
result = vn_queue_submission_prepare_submit(submit);
if (result != VK_SUCCESS)
return vn_error(instance, result);
/* skip no-op submit */
if (!submit->batch_count && submit->fence_handle == VK_NULL_HANDLE)
return VK_SUCCESS;
if (VN_PERF(NO_ASYNC_QUEUE_SUBMIT)) {
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
result = vn_call_vkQueueSubmit2(
dev->primary_ring, submit->queue_handle, submit->batch_count,
submit->submit2_batches, submit->fence_handle);
} else {
result = vn_call_vkQueueSubmit(
dev->primary_ring, submit->queue_handle, submit->batch_count,
submit->submit_batches, submit->fence_handle);
}
if (result != VK_SUCCESS) {
vn_queue_submission_cleanup(submit);
return vn_error(instance, result);
}
} else {
struct vn_ring_submit_command ring_submit;
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
vn_submit_vkQueueSubmit2(
dev->primary_ring, 0, submit->queue_handle, submit->batch_count,
submit->submit2_batches, submit->fence_handle, &ring_submit);
} else {
vn_submit_vkQueueSubmit(dev->primary_ring, 0, submit->queue_handle,
submit->batch_count, submit->submit_batches,
submit->fence_handle, &ring_submit);
}
if (!ring_submit.ring_seqno_valid) {
vn_queue_submission_cleanup(submit);
return vn_error(instance, VK_ERROR_DEVICE_LOST);
}
submit->external_payload.ring_seqno_valid = true;
submit->external_payload.ring_seqno = ring_submit.ring_seqno;
}
/* If external fence, track the submission's ring_idx to facilitate
* sync_file export.
*
* Imported syncs don't need a proxy renderer sync on subsequent export,
* because an fd is already available.
*/
struct vn_fence *fence = vn_fence_from_handle(submit->fence_handle);
if (fence && fence->is_external) {
assert(fence->payload->type == VN_SYNC_TYPE_DEVICE_ONLY);
fence->external_payload = submit->external_payload;
}
for (uint32_t i = 0; i < submit->batch_count; i++) {
const uint32_t signal_count = vn_get_signal_semaphore_count(submit, i);
for (uint32_t j = 0; j < signal_count; j++) {
struct vn_semaphore *sem =
vn_semaphore_from_handle(vn_get_signal_semaphore(submit, i, j));
if (sem->is_external) {
assert(sem->payload->type == VN_SYNC_TYPE_DEVICE_ONLY);
sem->external_payload = submit->external_payload;
}
}
}
vn_queue_wsi_present(submit);
vn_queue_submission_cleanup(submit);
return VK_SUCCESS;
}
VkResult
vn_QueueSubmit(VkQueue queue,
uint32_t submitCount,
const VkSubmitInfo *pSubmits,
VkFence fence)
{
VN_TRACE_FUNC();
vn_tls_set_async_pipeline_create();
struct vn_queue_submission submit = {
.batch_type = VK_STRUCTURE_TYPE_SUBMIT_INFO,
.queue_handle = queue,
.batch_count = submitCount,
.submit_batches = pSubmits,
.fence_handle = fence,
};
return vn_queue_submit(&submit);
}
VkResult
vn_QueueSubmit2(VkQueue queue,
uint32_t submitCount,
const VkSubmitInfo2 *pSubmits,
VkFence fence)
{
VN_TRACE_FUNC();
vn_tls_set_async_pipeline_create();
struct vn_queue_submission submit = {
.batch_type = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
.queue_handle = queue,
.batch_count = submitCount,
.submit2_batches = pSubmits,
.fence_handle = fence,
};
return vn_queue_submit(&submit);
}
static VkResult
vn_queue_bind_sparse_submit(struct vn_queue_submission *submit)
{
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
struct vn_device *dev = vn_device_from_vk(queue->base.vk.base.device);
struct vn_instance *instance = dev->instance;
VkResult result;
if (VN_PERF(NO_ASYNC_QUEUE_SUBMIT)) {
result = vn_call_vkQueueBindSparse(
dev->primary_ring, submit->queue_handle, submit->batch_count,
submit->sparse_batches, submit->fence_handle);
if (result != VK_SUCCESS)
return vn_error(instance, result);
} else {
struct vn_ring_submit_command ring_submit;
vn_submit_vkQueueBindSparse(dev->primary_ring, 0, submit->queue_handle,
submit->batch_count, submit->sparse_batches,
submit->fence_handle, &ring_submit);
if (!ring_submit.ring_seqno_valid)
return vn_error(instance, VK_ERROR_DEVICE_LOST);
}
return VK_SUCCESS;
}
static VkResult
vn_queue_bind_sparse_submit_batch(struct vn_queue_submission *submit,
uint32_t batch_index)
{
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
VkDevice dev_handle = vk_device_to_handle(queue->base.vk.base.device);
const VkBindSparseInfo *sparse_info = &submit->sparse_batches[batch_index];
const VkSemaphore *signal_sem = sparse_info->pSignalSemaphores;
uint32_t signal_sem_count = sparse_info->signalSemaphoreCount;
VkResult result;
struct vn_queue_submission sparse_batch = {
.batch_type = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO,
.queue_handle = submit->queue_handle,
.batch_count = 1,
.fence_handle = VK_NULL_HANDLE,
};
/* lazily create sparse semaphore */
if (queue->sparse_semaphore == VK_NULL_HANDLE) {
queue->sparse_semaphore_counter = 1;
const VkSemaphoreTypeCreateInfo sem_type_create_info = {
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
.pNext = NULL,
/* This must be timeline type to adhere to mesa's requirement
* not to mix binary semaphores with wait-before-signal.
*/
.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
.initialValue = 1,
};
const VkSemaphoreCreateInfo create_info = {
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
.pNext = &sem_type_create_info,
.flags = 0,
};
result = vn_CreateSemaphore(dev_handle, &create_info, NULL,
&queue->sparse_semaphore);
if (result != VK_SUCCESS)
return result;
}
/* Setup VkTimelineSemaphoreSubmitInfo's for our queue sparse semaphore
* so that the vkQueueSubmit waits on the vkQueueBindSparse signal.
*/
queue->sparse_semaphore_counter++;
struct VkTimelineSemaphoreSubmitInfo wait_timeline_sem_info = { 0 };
wait_timeline_sem_info.sType =
VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO;
wait_timeline_sem_info.signalSemaphoreValueCount = 1;
wait_timeline_sem_info.pSignalSemaphoreValues =
&queue->sparse_semaphore_counter;
struct VkTimelineSemaphoreSubmitInfo signal_timeline_sem_info = { 0 };
signal_timeline_sem_info.sType =
VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO;
signal_timeline_sem_info.waitSemaphoreValueCount = 1;
signal_timeline_sem_info.pWaitSemaphoreValues =
&queue->sparse_semaphore_counter;
/* Split up the original wait and signal semaphores into its respective
* vkTimelineSemaphoreSubmitInfo
*/
const struct VkTimelineSemaphoreSubmitInfo *timeline_sem_info =
vk_find_struct_const(sparse_info->pNext,
TIMELINE_SEMAPHORE_SUBMIT_INFO);
if (timeline_sem_info) {
if (timeline_sem_info->waitSemaphoreValueCount) {
wait_timeline_sem_info.waitSemaphoreValueCount =
timeline_sem_info->waitSemaphoreValueCount;
wait_timeline_sem_info.pWaitSemaphoreValues =
timeline_sem_info->pWaitSemaphoreValues;
}
if (timeline_sem_info->signalSemaphoreValueCount) {
signal_timeline_sem_info.signalSemaphoreValueCount =
timeline_sem_info->signalSemaphoreValueCount;
signal_timeline_sem_info.pSignalSemaphoreValues =
timeline_sem_info->pSignalSemaphoreValues;
}
}
/* Attach the original VkDeviceGroupBindSparseInfo if it exists */
struct VkDeviceGroupBindSparseInfo batch_device_group_info;
const struct VkDeviceGroupBindSparseInfo *device_group_info =
vk_find_struct_const(sparse_info->pNext, DEVICE_GROUP_BIND_SPARSE_INFO);
if (device_group_info) {
memcpy(&batch_device_group_info, device_group_info,
sizeof(*device_group_info));
batch_device_group_info.pNext = NULL;
wait_timeline_sem_info.pNext = &batch_device_group_info;
}
/* Copy the original batch VkBindSparseInfo modified to signal
* our sparse semaphore.
*/
VkBindSparseInfo batch_sparse_info;
memcpy(&batch_sparse_info, sparse_info, sizeof(*sparse_info));
batch_sparse_info.pNext = &wait_timeline_sem_info;
batch_sparse_info.signalSemaphoreCount = 1;
batch_sparse_info.pSignalSemaphores = &queue->sparse_semaphore;
/* Set up the SubmitInfo to wait on our sparse semaphore before sending
* feedback and signaling the original semaphores/fence
*
* Even if this VkBindSparse batch does not have feedback semaphores,
* we still glue all the batches together to ensure the feedback
* fence occurs after.
*/
VkPipelineStageFlags stage_masks = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT;
VkSubmitInfo batch_submit_info = {
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
.pNext = &signal_timeline_sem_info,
.waitSemaphoreCount = 1,
.pWaitSemaphores = &queue->sparse_semaphore,
.pWaitDstStageMask = &stage_masks,
.signalSemaphoreCount = signal_sem_count,
.pSignalSemaphores = signal_sem,
};
/* Set the possible fence if on the last batch */
VkFence fence_handle = VK_NULL_HANDLE;
if ((submit->feedback_types & VN_FEEDBACK_TYPE_FENCE) &&
batch_index == (submit->batch_count - 1)) {
fence_handle = submit->fence_handle;
}
sparse_batch.sparse_batches = &batch_sparse_info;
result = vn_queue_bind_sparse_submit(&sparse_batch);
if (result != VK_SUCCESS)
return result;
result = vn_QueueSubmit(submit->queue_handle, 1, &batch_submit_info,
fence_handle);
if (result != VK_SUCCESS)
return result;
return VK_SUCCESS;
}
VkResult
vn_QueueBindSparse(VkQueue queue,
uint32_t bindInfoCount,
const VkBindSparseInfo *pBindInfo,
VkFence fence)
{
VN_TRACE_FUNC();
VkResult result;
struct vn_queue_submission submit = {
.batch_type = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO,
.queue_handle = queue,
.batch_count = bindInfoCount,
.sparse_batches = pBindInfo,
.fence_handle = fence,
};
result = vn_queue_submission_prepare(&submit);
if (result != VK_SUCCESS)
return result;
if (!submit.batch_count) {
/* skip no-op submit */
if (submit.fence_handle == VK_NULL_HANDLE)
return VK_SUCCESS;
/* if empty batch, just send a vkQueueSubmit with the fence */
result =
vn_QueueSubmit(submit.queue_handle, 0, NULL, submit.fence_handle);
if (result != VK_SUCCESS)
return result;
}
/* if feedback isn't used in the batch, can directly submit */
if (!submit.feedback_types)
return vn_queue_bind_sparse_submit(&submit);
for (uint32_t i = 0; i < submit.batch_count; i++) {
result = vn_queue_bind_sparse_submit_batch(&submit, i);
if (result != VK_SUCCESS)
return result;
}
return VK_SUCCESS;
}
VkResult
vn_QueueWaitIdle(VkQueue _queue)
{
VN_TRACE_FUNC();
struct vn_queue *queue = vn_queue_from_handle(_queue);
VkDevice dev_handle = vk_device_to_handle(queue->base.vk.base.device);
struct vn_device *dev = vn_device_from_handle(dev_handle);
VkResult result;
/* lazily create queue wait fence for queue idle waiting */
if (queue->wait_fence == VK_NULL_HANDLE) {
const VkFenceCreateInfo create_info = {
.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO,
.flags = 0,
};
result =
vn_CreateFence(dev_handle, &create_info, NULL, &queue->wait_fence);
if (result != VK_SUCCESS)
return result;
}
result = vn_QueueSubmit(_queue, 0, NULL, queue->wait_fence);
if (result != VK_SUCCESS)
return result;
result =
vn_WaitForFences(dev_handle, 1, &queue->wait_fence, true, UINT64_MAX);
vn_ResetFences(dev_handle, 1, &queue->wait_fence);
return vn_result(dev->instance, result);
}
/* fence commands */
static void
vn_sync_payload_release(UNUSED struct vn_device *dev,
struct vn_sync_payload *payload)
{
if (payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD && payload->fd >= 0)
close(payload->fd);
payload->type = VN_SYNC_TYPE_INVALID;
}
static VkResult
vn_fence_init_payloads(struct vn_device *dev,
struct vn_fence *fence,
bool signaled,
const VkAllocationCallbacks *alloc)
{
fence->permanent.type = VN_SYNC_TYPE_DEVICE_ONLY;
fence->temporary.type = VN_SYNC_TYPE_INVALID;
fence->payload = &fence->permanent;
return VK_SUCCESS;
}
static VkResult
vn_fence_feedback_init(struct vn_device *dev,
struct vn_fence *fence,
bool signaled,
const VkAllocationCallbacks *alloc)
{
VkDevice dev_handle = vn_device_to_handle(dev);
struct vn_feedback_slot *slot;
VkCommandBuffer *cmd_handles;
VkResult result;
if (fence->is_external)
return VK_SUCCESS;
if (VN_PERF(NO_FENCE_FEEDBACK))
return VK_SUCCESS;
slot = vn_feedback_pool_alloc(&dev->feedback_pool, VN_FEEDBACK_TYPE_FENCE);
if (!slot)
return VK_ERROR_OUT_OF_HOST_MEMORY;
vn_feedback_set_status(slot, signaled ? VK_SUCCESS : VK_NOT_READY);
cmd_handles =
vk_zalloc(alloc, sizeof(*cmd_handles) * dev->queue_family_count,
VN_DEFAULT_ALIGN, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (!cmd_handles) {
vn_feedback_pool_free(&dev->feedback_pool, slot);
return VK_ERROR_OUT_OF_HOST_MEMORY;
}
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
result = vn_feedback_cmd_alloc(dev_handle, &dev->fb_cmd_pools[i], slot,
NULL, &cmd_handles[i]);
if (result != VK_SUCCESS) {
for (uint32_t j = 0; j < i; j++) {
vn_feedback_cmd_free(dev_handle, &dev->fb_cmd_pools[j],
cmd_handles[j]);
}
break;
}
}
if (result != VK_SUCCESS) {
vk_free(alloc, cmd_handles);
vn_feedback_pool_free(&dev->feedback_pool, slot);
return result;
}
fence->feedback.slot = slot;
fence->feedback.commands = cmd_handles;
return VK_SUCCESS;
}
static void
vn_fence_feedback_fini(struct vn_device *dev,
struct vn_fence *fence,
const VkAllocationCallbacks *alloc)
{
VkDevice dev_handle = vn_device_to_handle(dev);
if (!fence->feedback.slot)
return;
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
vn_feedback_cmd_free(dev_handle, &dev->fb_cmd_pools[i],
fence->feedback.commands[i]);
}
vn_feedback_pool_free(&dev->feedback_pool, fence->feedback.slot);
vk_free(alloc, fence->feedback.commands);
}
VkResult
vn_CreateFence(VkDevice device,
const VkFenceCreateInfo *pCreateInfo,
const VkAllocationCallbacks *pAllocator,
VkFence *pFence)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
const VkAllocationCallbacks *alloc =
pAllocator ? pAllocator : &dev->base.vk.alloc;
const bool signaled = pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT;
VkResult result;
struct vn_fence *fence = vk_zalloc(alloc, sizeof(*fence), VN_DEFAULT_ALIGN,
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (!fence)
return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
vn_object_base_init(&fence->base, VK_OBJECT_TYPE_FENCE, &dev->base);
const struct VkExportFenceCreateInfo *export_info =
vk_find_struct_const(pCreateInfo->pNext, EXPORT_FENCE_CREATE_INFO);
fence->is_external = export_info && export_info->handleTypes;
result = vn_fence_init_payloads(dev, fence, signaled, alloc);
if (result != VK_SUCCESS)
goto out_object_base_fini;
result = vn_fence_feedback_init(dev, fence, signaled, alloc);
if (result != VK_SUCCESS)
goto out_payloads_fini;
*pFence = vn_fence_to_handle(fence);
vn_async_vkCreateFence(dev->primary_ring, device, pCreateInfo, NULL,
pFence);
return VK_SUCCESS;
out_payloads_fini:
vn_sync_payload_release(dev, &fence->permanent);
vn_sync_payload_release(dev, &fence->temporary);
out_object_base_fini:
vn_object_base_fini(&fence->base);
vk_free(alloc, fence);
return vn_error(dev->instance, result);
}
void
vn_DestroyFence(VkDevice device,
VkFence _fence,
const VkAllocationCallbacks *pAllocator)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_fence *fence = vn_fence_from_handle(_fence);
const VkAllocationCallbacks *alloc =
pAllocator ? pAllocator : &dev->base.vk.alloc;
if (!fence)
return;
vn_async_vkDestroyFence(dev->primary_ring, device, _fence, NULL);
vn_fence_feedback_fini(dev, fence, alloc);
vn_sync_payload_release(dev, &fence->permanent);
vn_sync_payload_release(dev, &fence->temporary);
vn_object_base_fini(&fence->base);
vk_free(alloc, fence);
}
VkResult
vn_ResetFences(VkDevice device, uint32_t fenceCount, const VkFence *pFences)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
vn_async_vkResetFences(dev->primary_ring, device, fenceCount, pFences);
for (uint32_t i = 0; i < fenceCount; i++) {
struct vn_fence *fence = vn_fence_from_handle(pFences[i]);
struct vn_sync_payload *perm = &fence->permanent;
vn_sync_payload_release(dev, &fence->temporary);
assert(perm->type == VN_SYNC_TYPE_DEVICE_ONLY);
fence->payload = perm;
if (fence->feedback.slot)
vn_feedback_reset_status(fence->feedback.slot);
}
return VK_SUCCESS;
}
VkResult
vn_GetFenceStatus(VkDevice device, VkFence _fence)
{
struct vn_device *dev = vn_device_from_handle(device);
struct vn_fence *fence = vn_fence_from_handle(_fence);
struct vn_sync_payload *payload = fence->payload;
VkResult result;
switch (payload->type) {
case VN_SYNC_TYPE_DEVICE_ONLY:
if (fence->feedback.slot) {
result = vn_feedback_get_status(fence->feedback.slot);
if (result == VK_SUCCESS) {
/* When fence feedback slot gets signaled, the real fence
* signal operation follows after but the signaling isr can be
* deferred or preempted. To avoid racing, we let the
* renderer wait for the fence. This also helps resolve
* synchronization validation errors, because the layer no
* longer sees any fence status checks and falsely believes the
* caller does not sync.
*/
vn_async_vkWaitForFences(dev->primary_ring, device, 1, &_fence,
VK_TRUE, UINT64_MAX);
}
} else {
result = vn_call_vkGetFenceStatus(dev->primary_ring, device, _fence);
}
break;
case VN_SYNC_TYPE_IMPORTED_SYNC_FD:
if (payload->fd < 0 || sync_wait(payload->fd, 0) == 0)
result = VK_SUCCESS;
else
result = errno == ETIME ? VK_NOT_READY : VK_ERROR_DEVICE_LOST;
break;
default:
UNREACHABLE("unexpected fence payload type");
break;
}
return vn_result(dev->instance, result);
}
static VkResult
vn_find_first_signaled_fence(VkDevice device,
const VkFence *fences,
uint32_t count)
{
for (uint32_t i = 0; i < count; i++) {
VkResult result = vn_GetFenceStatus(device, fences[i]);
if (result == VK_SUCCESS || result < 0)
return result;
}
return VK_NOT_READY;
}
static VkResult
vn_remove_signaled_fences(VkDevice device, VkFence *fences, uint32_t *count)
{
uint32_t cur = 0;
for (uint32_t i = 0; i < *count; i++) {
VkResult result = vn_GetFenceStatus(device, fences[i]);
if (result != VK_SUCCESS) {
if (result < 0)
return result;
fences[cur++] = fences[i];
}
}
*count = cur;
return cur ? VK_NOT_READY : VK_SUCCESS;
}
static VkResult
vn_update_sync_result(struct vn_device *dev,
VkResult result,
int64_t abs_timeout,
struct vn_relax_state *relax_state)
{
switch (result) {
case VK_NOT_READY:
if (abs_timeout != OS_TIMEOUT_INFINITE &&
os_time_get_nano() >= abs_timeout)
result = VK_TIMEOUT;
else
vn_relax(relax_state);
break;
default:
assert(result == VK_SUCCESS || result < 0);
break;
}
return result;
}
VkResult
vn_WaitForFences(VkDevice device,
uint32_t fenceCount,
const VkFence *pFences,
VkBool32 waitAll,
uint64_t timeout)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
const int64_t abs_timeout = os_time_get_absolute_timeout(timeout);
VkResult result = VK_NOT_READY;
if (fenceCount > 1 && waitAll) {
STACK_ARRAY(VkFence, fences, fenceCount);
typed_memcpy(fences, pFences, fenceCount);
struct vn_relax_state relax_state =
vn_relax_init(dev->instance, VN_RELAX_REASON_FENCE);
while (result == VK_NOT_READY) {
result = vn_remove_signaled_fences(device, fences, &fenceCount);
result =
vn_update_sync_result(dev, result, abs_timeout, &relax_state);
}
vn_relax_fini(&relax_state);
STACK_ARRAY_FINISH(fences);
} else {
struct vn_relax_state relax_state =
vn_relax_init(dev->instance, VN_RELAX_REASON_FENCE);
while (result == VK_NOT_READY) {
result = vn_find_first_signaled_fence(device, pFences, fenceCount);
result =
vn_update_sync_result(dev, result, abs_timeout, &relax_state);
}
vn_relax_fini(&relax_state);
}
return vn_result(dev->instance, result);
}
static VkResult
vn_create_sync_file(struct vn_device *dev,
struct vn_sync_payload_external *external_payload,
int *out_fd)
{
struct vn_renderer_sync *sync;
VkResult result = vn_renderer_sync_create(dev->renderer, 0,
VN_RENDERER_SYNC_BINARY, &sync);
if (result != VK_SUCCESS)
return vn_error(dev->instance, result);
struct vn_renderer_submit_batch batch = {
.syncs = &sync,
.sync_values = &(const uint64_t){ 1 },
.sync_count = 1,
.ring_idx = external_payload->ring_idx,
};
uint32_t local_data[8];
struct vn_cs_encoder local_enc =
VN_CS_ENCODER_INITIALIZER_LOCAL(local_data, sizeof(local_data));
if (external_payload->ring_seqno_valid) {
const uint64_t ring_id = vn_ring_get_id(dev->primary_ring);
vn_encode_vkWaitRingSeqnoMESA(&local_enc, 0, ring_id,
external_payload->ring_seqno);
batch.cs_data = local_data;
batch.cs_size = vn_cs_encoder_get_len(&local_enc);
}
const struct vn_renderer_submit submit = {
.batches = &batch,
.batch_count = 1,
};
result = vn_renderer_submit(dev->renderer, &submit);
if (result != VK_SUCCESS) {
vn_renderer_sync_destroy(dev->renderer, sync);
return vn_error(dev->instance, result);
}
*out_fd = vn_renderer_sync_export_syncobj(dev->renderer, sync, true);
vn_renderer_sync_destroy(dev->renderer, sync);
return *out_fd >= 0 ? VK_SUCCESS : VK_ERROR_TOO_MANY_OBJECTS;
}
static inline bool
vn_sync_valid_fd(int fd)
{
/* the special value -1 for fd is treated like a valid sync file descriptor
* referring to an object that has already signaled
*/
return (fd >= 0 && sync_valid_fd(fd)) || fd == -1;
}
VkResult
vn_ImportFenceFdKHR(VkDevice device,
const VkImportFenceFdInfoKHR *pImportFenceFdInfo)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_fence *fence = vn_fence_from_handle(pImportFenceFdInfo->fence);
ASSERTED const bool sync_file = pImportFenceFdInfo->handleType ==
VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
const int fd = pImportFenceFdInfo->fd;
assert(sync_file);
if (!vn_sync_valid_fd(fd))
return vn_error(dev->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
struct vn_sync_payload *temp = &fence->temporary;
vn_sync_payload_release(dev, temp);
temp->type = VN_SYNC_TYPE_IMPORTED_SYNC_FD;
temp->fd = fd;
fence->payload = temp;
return VK_SUCCESS;
}
VkResult
vn_GetFenceFdKHR(VkDevice device,
const VkFenceGetFdInfoKHR *pGetFdInfo,
int *pFd)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_fence *fence = vn_fence_from_handle(pGetFdInfo->fence);
const bool sync_file =
pGetFdInfo->handleType == VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
struct vn_sync_payload *payload = fence->payload;
VkResult result;
assert(sync_file);
assert(dev->physical_device->renderer_sync_fd.fence_exportable);
int fd = -1;
if (payload->type == VN_SYNC_TYPE_DEVICE_ONLY) {
result = vn_create_sync_file(dev, &fence->external_payload, &fd);
if (result != VK_SUCCESS)
return vn_error(dev->instance, result);
vn_async_vkResetFenceResourceMESA(dev->primary_ring, device,
pGetFdInfo->fence);
vn_sync_payload_release(dev, &fence->temporary);
fence->payload = &fence->permanent;
#ifdef VN_USE_WSI_PLATFORM
if (!dev->renderer->info.has_implicit_fencing)
sync_wait(fd, -1);
#endif
} else {
assert(payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD);
/* transfer ownership of imported sync fd to save a dup */
fd = payload->fd;
payload->fd = -1;
/* reset host fence in case in signaled state before import */
result = vn_ResetFences(device, 1, &pGetFdInfo->fence);
if (result != VK_SUCCESS) {
/* transfer sync fd ownership back on error */
payload->fd = fd;
return result;
}
}
*pFd = fd;
return VK_SUCCESS;
}
/* semaphore commands */
static VkResult
vn_semaphore_init_payloads(struct vn_device *dev,
struct vn_semaphore *sem,
uint64_t initial_val,
const VkAllocationCallbacks *alloc)
{
sem->permanent.type = VN_SYNC_TYPE_DEVICE_ONLY;
sem->temporary.type = VN_SYNC_TYPE_INVALID;
sem->payload = &sem->permanent;
return VK_SUCCESS;
}
static bool
vn_semaphore_wait_external(struct vn_device *dev, struct vn_semaphore *sem)
{
struct vn_sync_payload *temp = &sem->temporary;
assert(temp->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD);
if (temp->fd >= 0) {
if (sync_wait(temp->fd, -1))
return false;
}
vn_sync_payload_release(dev, &sem->temporary);
sem->payload = &sem->permanent;
return true;
}
struct vn_semaphore_feedback_cmd *
vn_semaphore_get_feedback_cmd(struct vn_device *dev, struct vn_semaphore *sem)
{
struct vn_semaphore_feedback_cmd *sfb_cmd = NULL;
simple_mtx_lock(&sem->feedback.cmd_mtx);
if (!list_is_empty(&sem->feedback.free_cmds)) {
sfb_cmd = list_first_entry(&sem->feedback.free_cmds,
struct vn_semaphore_feedback_cmd, head);
list_move_to(&sfb_cmd->head, &sem->feedback.pending_cmds);
sem->feedback.free_cmd_count--;
}
simple_mtx_unlock(&sem->feedback.cmd_mtx);
if (!sfb_cmd) {
sfb_cmd = vn_semaphore_feedback_cmd_alloc(dev, sem->feedback.slot);
simple_mtx_lock(&sem->feedback.cmd_mtx);
list_add(&sfb_cmd->head, &sem->feedback.pending_cmds);
simple_mtx_unlock(&sem->feedback.cmd_mtx);
}
return sfb_cmd;
}
static VkResult
vn_semaphore_feedback_init(struct vn_device *dev,
struct vn_semaphore *sem,
uint64_t initial_value,
const VkAllocationCallbacks *alloc)
{
struct vn_feedback_slot *slot;
assert(sem->type == VK_SEMAPHORE_TYPE_TIMELINE);
if (sem->is_external)
return VK_SUCCESS;
if (VN_PERF(NO_SEMAPHORE_FEEDBACK))
return VK_SUCCESS;
slot =
vn_feedback_pool_alloc(&dev->feedback_pool, VN_FEEDBACK_TYPE_SEMAPHORE);
if (!slot)
return VK_ERROR_OUT_OF_HOST_MEMORY;
list_inithead(&sem->feedback.pending_cmds);
list_inithead(&sem->feedback.free_cmds);
vn_feedback_set_counter(slot, initial_value);
simple_mtx_init(&sem->feedback.cmd_mtx, mtx_plain);
simple_mtx_init(&sem->feedback.async_wait_mtx, mtx_plain);
sem->feedback.signaled_counter = initial_value;
sem->feedback.slot = slot;
return VK_SUCCESS;
}
static void
vn_semaphore_feedback_fini(struct vn_device *dev, struct vn_semaphore *sem)
{
if (!sem->feedback.slot)
return;
list_for_each_entry_safe(struct vn_semaphore_feedback_cmd, sfb_cmd,
&sem->feedback.free_cmds, head)
vn_semaphore_feedback_cmd_free(dev, sfb_cmd);
list_for_each_entry_safe(struct vn_semaphore_feedback_cmd, sfb_cmd,
&sem->feedback.pending_cmds, head)
vn_semaphore_feedback_cmd_free(dev, sfb_cmd);
simple_mtx_destroy(&sem->feedback.cmd_mtx);
simple_mtx_destroy(&sem->feedback.async_wait_mtx);
vn_feedback_pool_free(&dev->feedback_pool, sem->feedback.slot);
}
VkResult
vn_CreateSemaphore(VkDevice device,
const VkSemaphoreCreateInfo *pCreateInfo,
const VkAllocationCallbacks *pAllocator,
VkSemaphore *pSemaphore)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
const VkAllocationCallbacks *alloc =
pAllocator ? pAllocator : &dev->base.vk.alloc;
struct vn_semaphore *sem = vk_zalloc(alloc, sizeof(*sem), VN_DEFAULT_ALIGN,
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (!sem)
return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
vn_object_base_init(&sem->base, VK_OBJECT_TYPE_SEMAPHORE, &dev->base);
const VkSemaphoreTypeCreateInfo *type_info =
vk_find_struct_const(pCreateInfo->pNext, SEMAPHORE_TYPE_CREATE_INFO);
uint64_t initial_val = 0;
if (type_info && type_info->semaphoreType == VK_SEMAPHORE_TYPE_TIMELINE) {
sem->type = VK_SEMAPHORE_TYPE_TIMELINE;
initial_val = type_info->initialValue;
} else {
sem->type = VK_SEMAPHORE_TYPE_BINARY;
}
const struct VkExportSemaphoreCreateInfo *export_info =
vk_find_struct_const(pCreateInfo->pNext, EXPORT_SEMAPHORE_CREATE_INFO);
sem->is_external = export_info && export_info->handleTypes;
VkResult result = vn_semaphore_init_payloads(dev, sem, initial_val, alloc);
if (result != VK_SUCCESS)
goto out_object_base_fini;
if (sem->type == VK_SEMAPHORE_TYPE_TIMELINE) {
result = vn_semaphore_feedback_init(dev, sem, initial_val, alloc);
if (result != VK_SUCCESS)
goto out_payloads_fini;
}
VkSemaphore sem_handle = vn_semaphore_to_handle(sem);
vn_async_vkCreateSemaphore(dev->primary_ring, device, pCreateInfo, NULL,
&sem_handle);
*pSemaphore = sem_handle;
return VK_SUCCESS;
out_payloads_fini:
vn_sync_payload_release(dev, &sem->permanent);
vn_sync_payload_release(dev, &sem->temporary);
out_object_base_fini:
vn_object_base_fini(&sem->base);
vk_free(alloc, sem);
return vn_error(dev->instance, result);
}
void
vn_DestroySemaphore(VkDevice device,
VkSemaphore semaphore,
const VkAllocationCallbacks *pAllocator)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_semaphore *sem = vn_semaphore_from_handle(semaphore);
const VkAllocationCallbacks *alloc =
pAllocator ? pAllocator : &dev->base.vk.alloc;
if (!sem)
return;
vn_async_vkDestroySemaphore(dev->primary_ring, device, semaphore, NULL);
if (sem->type == VK_SEMAPHORE_TYPE_TIMELINE)
vn_semaphore_feedback_fini(dev, sem);
vn_sync_payload_release(dev, &sem->permanent);
vn_sync_payload_release(dev, &sem->temporary);
vn_object_base_fini(&sem->base);
vk_free(alloc, sem);
}
VkResult
vn_GetSemaphoreCounterValue(VkDevice device,
VkSemaphore semaphore,
uint64_t *pValue)
{
struct vn_device *dev = vn_device_from_handle(device);
struct vn_semaphore *sem = vn_semaphore_from_handle(semaphore);
ASSERTED struct vn_sync_payload *payload = sem->payload;
assert(payload->type == VN_SYNC_TYPE_DEVICE_ONLY);
if (sem->feedback.slot) {
simple_mtx_lock(&sem->feedback.async_wait_mtx);
const uint64_t counter = vn_feedback_get_counter(sem->feedback.slot);
if (sem->feedback.signaled_counter < counter) {
/* When the timeline semaphore feedback slot gets signaled, the real
* semaphore signal operation follows after but the signaling isr can
* be deferred or preempted. To avoid racing, we let the renderer
* wait for the semaphore by sending an asynchronous wait call for
* the feedback value.
* We also cache the counter value to only send the async call once
* per counter value to prevent spamming redundant async wait calls.
* The cached counter value requires a lock to ensure multiple
* threads querying for the same value are guaranteed to encode after
* the async wait call.
*
* This also helps resolve synchronization validation errors, because
* the layer no longer sees any semaphore status checks and falsely
* believes the caller does not sync.
*/
VkSemaphoreWaitInfo wait_info = {
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
.pNext = NULL,
.flags = 0,
.semaphoreCount = 1,
.pSemaphores = &semaphore,
.pValues = &counter,
};
vn_async_vkWaitSemaphores(dev->primary_ring, device, &wait_info,
UINT64_MAX);
/* search pending cmds for already signaled values */
simple_mtx_lock(&sem->feedback.cmd_mtx);
list_for_each_entry_safe(struct vn_semaphore_feedback_cmd, sfb_cmd,
&sem->feedback.pending_cmds, head) {
if (counter >= vn_feedback_get_counter(sfb_cmd->src_slot)) {
/* avoid over-caching more than normal runtime usage */
if (sem->feedback.free_cmd_count > 5) {
list_del(&sfb_cmd->head);
vn_semaphore_feedback_cmd_free(dev, sfb_cmd);
} else {
list_move_to(&sfb_cmd->head, &sem->feedback.free_cmds);
sem->feedback.free_cmd_count++;
}
}
}
simple_mtx_unlock(&sem->feedback.cmd_mtx);
sem->feedback.signaled_counter = counter;
}
simple_mtx_unlock(&sem->feedback.async_wait_mtx);
*pValue = counter;
return VK_SUCCESS;
} else {
return vn_call_vkGetSemaphoreCounterValue(dev->primary_ring, device,
semaphore, pValue);
}
}
VkResult
vn_SignalSemaphore(VkDevice device, const VkSemaphoreSignalInfo *pSignalInfo)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_semaphore *sem =
vn_semaphore_from_handle(pSignalInfo->semaphore);
vn_async_vkSignalSemaphore(dev->primary_ring, device, pSignalInfo);
if (sem->feedback.slot) {
simple_mtx_lock(&sem->feedback.async_wait_mtx);
vn_feedback_set_counter(sem->feedback.slot, pSignalInfo->value);
/* Update async counters. Since we're signaling, we're aligned with
* the renderer.
*/
sem->feedback.signaled_counter = pSignalInfo->value;
simple_mtx_unlock(&sem->feedback.async_wait_mtx);
}
return VK_SUCCESS;
}
static VkResult
vn_find_first_signaled_semaphore(VkDevice device,
const VkSemaphore *semaphores,
const uint64_t *values,
uint32_t count)
{
for (uint32_t i = 0; i < count; i++) {
uint64_t val = 0;
VkResult result =
vn_GetSemaphoreCounterValue(device, semaphores[i], &val);
if (result != VK_SUCCESS || val >= values[i])
return result;
}
return VK_NOT_READY;
}
static VkResult
vn_remove_signaled_semaphores(VkDevice device,
VkSemaphore *semaphores,
uint64_t *values,
uint32_t *count)
{
uint32_t cur = 0;
for (uint32_t i = 0; i < *count; i++) {
uint64_t val = 0;
VkResult result =
vn_GetSemaphoreCounterValue(device, semaphores[i], &val);
if (result != VK_SUCCESS)
return result;
if (val < values[i])
semaphores[cur++] = semaphores[i];
}
*count = cur;
return cur ? VK_NOT_READY : VK_SUCCESS;
}
VkResult
vn_WaitSemaphores(VkDevice device,
const VkSemaphoreWaitInfo *pWaitInfo,
uint64_t timeout)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
const int64_t abs_timeout = os_time_get_absolute_timeout(timeout);
VkResult result = VK_NOT_READY;
if (pWaitInfo->semaphoreCount > 1 &&
!(pWaitInfo->flags & VK_SEMAPHORE_WAIT_ANY_BIT)) {
uint32_t semaphore_count = pWaitInfo->semaphoreCount;
STACK_ARRAY(VkSemaphore, semaphores, semaphore_count);
STACK_ARRAY(uint64_t, values, semaphore_count);
typed_memcpy(semaphores, pWaitInfo->pSemaphores, semaphore_count);
typed_memcpy(values, pWaitInfo->pValues, semaphore_count);
struct vn_relax_state relax_state =
vn_relax_init(dev->instance, VN_RELAX_REASON_SEMAPHORE);
while (result == VK_NOT_READY) {
result = vn_remove_signaled_semaphores(device, semaphores, values,
&semaphore_count);
result =
vn_update_sync_result(dev, result, abs_timeout, &relax_state);
}
vn_relax_fini(&relax_state);
STACK_ARRAY_FINISH(semaphores);
STACK_ARRAY_FINISH(values);
} else {
struct vn_relax_state relax_state =
vn_relax_init(dev->instance, VN_RELAX_REASON_SEMAPHORE);
while (result == VK_NOT_READY) {
result = vn_find_first_signaled_semaphore(
device, pWaitInfo->pSemaphores, pWaitInfo->pValues,
pWaitInfo->semaphoreCount);
result =
vn_update_sync_result(dev, result, abs_timeout, &relax_state);
}
vn_relax_fini(&relax_state);
}
return vn_result(dev->instance, result);
}
VkResult
vn_ImportSemaphoreFdKHR(
VkDevice device, const VkImportSemaphoreFdInfoKHR *pImportSemaphoreFdInfo)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_semaphore *sem =
vn_semaphore_from_handle(pImportSemaphoreFdInfo->semaphore);
ASSERTED const bool sync_file =
pImportSemaphoreFdInfo->handleType ==
VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
const int fd = pImportSemaphoreFdInfo->fd;
assert(sync_file);
if (!vn_sync_valid_fd(fd))
return vn_error(dev->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
struct vn_sync_payload *temp = &sem->temporary;
vn_sync_payload_release(dev, temp);
temp->type = VN_SYNC_TYPE_IMPORTED_SYNC_FD;
temp->fd = fd;
sem->payload = temp;
return VK_SUCCESS;
}
VkResult
vn_GetSemaphoreFdKHR(VkDevice device,
const VkSemaphoreGetFdInfoKHR *pGetFdInfo,
int *pFd)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_semaphore *sem = vn_semaphore_from_handle(pGetFdInfo->semaphore);
const bool sync_file =
pGetFdInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
struct vn_sync_payload *payload = sem->payload;
assert(sync_file);
assert(dev->physical_device->renderer_sync_fd.semaphore_exportable);
assert(dev->physical_device->renderer_sync_fd.semaphore_importable);
int fd = -1;
if (payload->type == VN_SYNC_TYPE_DEVICE_ONLY) {
VkResult result = vn_create_sync_file(dev, &sem->external_payload, &fd);
if (result != VK_SUCCESS)
return vn_error(dev->instance, result);
#ifdef VN_USE_WSI_PLATFORM
if (!dev->renderer->info.has_implicit_fencing)
sync_wait(fd, -1);
#endif
} else {
assert(payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD);
/* transfer ownership of imported sync fd to save a dup */
fd = payload->fd;
payload->fd = -1;
}
/* When payload->type is VN_SYNC_TYPE_IMPORTED_SYNC_FD, the current
* payload is from a prior temporary sync_fd import. The permanent
* payload of the sempahore might be in signaled state. So we do an
* import here to ensure later wait operation is legit. With resourceId
* 0, renderer does a signaled sync_fd -1 payload import on the host
* semaphore.
*/
if (payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD) {
const VkImportSemaphoreResourceInfoMESA res_info = {
.sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_RESOURCE_INFO_MESA,
.semaphore = pGetFdInfo->semaphore,
.resourceId = 0,
};
vn_async_vkImportSemaphoreResourceMESA(dev->primary_ring, device,
&res_info);
}
/* perform wait operation on the host semaphore */
vn_async_vkWaitSemaphoreResourceMESA(dev->primary_ring, device,
pGetFdInfo->semaphore);
vn_sync_payload_release(dev, &sem->temporary);
sem->payload = &sem->permanent;
*pFd = fd;
return VK_SUCCESS;
}
/* event commands */
static VkResult
vn_event_feedback_init(struct vn_device *dev, struct vn_event *ev)
{
struct vn_feedback_slot *slot;
if (VN_PERF(NO_EVENT_FEEDBACK))
return VK_SUCCESS;
slot = vn_feedback_pool_alloc(&dev->feedback_pool, VN_FEEDBACK_TYPE_EVENT);
if (!slot)
return VK_ERROR_OUT_OF_HOST_MEMORY;
/* newly created event object is in the unsignaled state */
vn_feedback_set_status(slot, VK_EVENT_RESET);
ev->feedback_slot = slot;
return VK_SUCCESS;
}
static inline void
vn_event_feedback_fini(struct vn_device *dev, struct vn_event *ev)
{
if (ev->feedback_slot)
vn_feedback_pool_free(&dev->feedback_pool, ev->feedback_slot);
}
VkResult
vn_CreateEvent(VkDevice device,
const VkEventCreateInfo *pCreateInfo,
const VkAllocationCallbacks *pAllocator,
VkEvent *pEvent)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
const VkAllocationCallbacks *alloc =
pAllocator ? pAllocator : &dev->base.vk.alloc;
struct vn_event *ev = vk_zalloc(alloc, sizeof(*ev), VN_DEFAULT_ALIGN,
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (!ev)
return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
vn_object_base_init(&ev->base, VK_OBJECT_TYPE_EVENT, &dev->base);
/* feedback is only needed to speed up host operations */
if (!(pCreateInfo->flags & VK_EVENT_CREATE_DEVICE_ONLY_BIT)) {
VkResult result = vn_event_feedback_init(dev, ev);
if (result != VK_SUCCESS)
return vn_error(dev->instance, result);
}
VkEvent ev_handle = vn_event_to_handle(ev);
vn_async_vkCreateEvent(dev->primary_ring, device, pCreateInfo, NULL,
&ev_handle);
*pEvent = ev_handle;
return VK_SUCCESS;
}
void
vn_DestroyEvent(VkDevice device,
VkEvent event,
const VkAllocationCallbacks *pAllocator)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_event *ev = vn_event_from_handle(event);
const VkAllocationCallbacks *alloc =
pAllocator ? pAllocator : &dev->base.vk.alloc;
if (!ev)
return;
vn_async_vkDestroyEvent(dev->primary_ring, device, event, NULL);
vn_event_feedback_fini(dev, ev);
vn_object_base_fini(&ev->base);
vk_free(alloc, ev);
}
VkResult
vn_GetEventStatus(VkDevice device, VkEvent event)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_event *ev = vn_event_from_handle(event);
VkResult result;
if (ev->feedback_slot)
result = vn_feedback_get_status(ev->feedback_slot);
else
result = vn_call_vkGetEventStatus(dev->primary_ring, device, event);
return vn_result(dev->instance, result);
}
VkResult
vn_SetEvent(VkDevice device, VkEvent event)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_event *ev = vn_event_from_handle(event);
if (ev->feedback_slot) {
vn_feedback_set_status(ev->feedback_slot, VK_EVENT_SET);
vn_async_vkSetEvent(dev->primary_ring, device, event);
} else {
VkResult result = vn_call_vkSetEvent(dev->primary_ring, device, event);
if (result != VK_SUCCESS)
return vn_error(dev->instance, result);
}
return VK_SUCCESS;
}
VkResult
vn_ResetEvent(VkDevice device, VkEvent event)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_event *ev = vn_event_from_handle(event);
if (ev->feedback_slot) {
vn_feedback_reset_status(ev->feedback_slot);
vn_async_vkResetEvent(dev->primary_ring, device, event);
} else {
VkResult result =
vn_call_vkResetEvent(dev->primary_ring, device, event);
if (result != VK_SUCCESS)
return vn_error(dev->instance, result);
}
return VK_SUCCESS;
}
Reference in New Issue View Git Blame Copy Permalink