8af267eb00
Batches must be ignored if batch count is zero, so all batch inspections
have to be gated behind batch count. For memcpy, it's UB if either src
or dst is NULL even when size is zero.
Side note:
- For original commit, this fixes just the memcpy UB
- For current codes, this fixes to not skip ffb batch prepare
Fixes: 493a3b5cda ("venus: refactor batch submission fixup")
Signed-off-by: Yiwei Zhang <zzyiwei@chromium.org>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/28071>
2423 lines
80 KiB
C
2423 lines
80 KiB
C
/*
|
|
* Copyright 2019 Google LLC
|
|
* SPDX-License-Identifier: MIT
|
|
*
|
|
* based in part on anv and radv which are:
|
|
* Copyright © 2015 Intel Corporation
|
|
* Copyright © 2016 Red Hat.
|
|
* Copyright © 2016 Bas Nieuwenhuizen
|
|
*/
|
|
|
|
#include "vn_queue.h"
|
|
|
|
#include "util/libsync.h"
|
|
#include "venus-protocol/vn_protocol_driver_event.h"
|
|
#include "venus-protocol/vn_protocol_driver_fence.h"
|
|
#include "venus-protocol/vn_protocol_driver_queue.h"
|
|
#include "venus-protocol/vn_protocol_driver_semaphore.h"
|
|
#include "venus-protocol/vn_protocol_driver_transport.h"
|
|
|
|
#include "vn_command_buffer.h"
|
|
#include "vn_device.h"
|
|
#include "vn_device_memory.h"
|
|
#include "vn_feedback.h"
|
|
#include "vn_instance.h"
|
|
#include "vn_physical_device.h"
|
|
#include "vn_query_pool.h"
|
|
#include "vn_renderer.h"
|
|
#include "vn_wsi.h"
|
|
|
|
/* queue commands */
|
|
|
|
struct vn_queue_submission {
|
|
VkStructureType batch_type;
|
|
VkQueue queue_handle;
|
|
uint32_t batch_count;
|
|
union {
|
|
const void *batches;
|
|
const VkSubmitInfo *submit_batches;
|
|
const VkSubmitInfo2 *submit2_batches;
|
|
const VkBindSparseInfo *sparse_batches;
|
|
};
|
|
VkFence fence_handle;
|
|
|
|
uint32_t cmd_count;
|
|
uint32_t feedback_types;
|
|
bool has_zink_sync_batch;
|
|
const struct vn_device_memory *wsi_mem;
|
|
struct vn_sync_payload_external external_payload;
|
|
|
|
/* Temporary storage allocation for submission
|
|
*
|
|
* A single alloc for storage is performed and the offsets inside storage
|
|
* are set as below:
|
|
*
|
|
* batches
|
|
* - non-empty submission: copy of original batches
|
|
* - empty submission: a single batch for fence feedback (ffb)
|
|
* cmds
|
|
* - for each batch:
|
|
* - copy of original batch cmds
|
|
* - a single cmd for query feedback (qfb)
|
|
* - one cmd for each signal semaphore that has feedback (sfb)
|
|
* - if last batch, a single cmd for ffb
|
|
*/
|
|
struct {
|
|
void *storage;
|
|
|
|
union {
|
|
void *batches;
|
|
VkSubmitInfo *submit_batches;
|
|
VkSubmitInfo2 *submit2_batches;
|
|
};
|
|
|
|
union {
|
|
void *cmds;
|
|
VkCommandBuffer *cmd_handles;
|
|
VkCommandBufferSubmitInfo *cmd_infos;
|
|
};
|
|
} temp;
|
|
};
|
|
|
|
static inline uint32_t
|
|
vn_get_wait_semaphore_count(struct vn_queue_submission *submit,
|
|
uint32_t batch_index)
|
|
{
|
|
switch (submit->batch_type) {
|
|
case VK_STRUCTURE_TYPE_SUBMIT_INFO:
|
|
return submit->submit_batches[batch_index].waitSemaphoreCount;
|
|
case VK_STRUCTURE_TYPE_SUBMIT_INFO_2:
|
|
return submit->submit2_batches[batch_index].waitSemaphoreInfoCount;
|
|
case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
|
|
return submit->sparse_batches[batch_index].waitSemaphoreCount;
|
|
default:
|
|
unreachable("unexpected batch type");
|
|
}
|
|
}
|
|
|
|
static inline uint32_t
|
|
vn_get_signal_semaphore_count(struct vn_queue_submission *submit,
|
|
uint32_t batch_index)
|
|
{
|
|
switch (submit->batch_type) {
|
|
case VK_STRUCTURE_TYPE_SUBMIT_INFO:
|
|
return submit->submit_batches[batch_index].signalSemaphoreCount;
|
|
case VK_STRUCTURE_TYPE_SUBMIT_INFO_2:
|
|
return submit->submit2_batches[batch_index].signalSemaphoreInfoCount;
|
|
case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
|
|
return submit->sparse_batches[batch_index].signalSemaphoreCount;
|
|
default:
|
|
unreachable("unexpected batch type");
|
|
}
|
|
}
|
|
|
|
static inline VkSemaphore
|
|
vn_get_wait_semaphore(struct vn_queue_submission *submit,
|
|
uint32_t batch_index,
|
|
uint32_t semaphore_index)
|
|
{
|
|
switch (submit->batch_type) {
|
|
case VK_STRUCTURE_TYPE_SUBMIT_INFO:
|
|
return submit->submit_batches[batch_index]
|
|
.pWaitSemaphores[semaphore_index];
|
|
case VK_STRUCTURE_TYPE_SUBMIT_INFO_2:
|
|
return submit->submit2_batches[batch_index]
|
|
.pWaitSemaphoreInfos[semaphore_index]
|
|
.semaphore;
|
|
case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
|
|
return submit->sparse_batches[batch_index]
|
|
.pWaitSemaphores[semaphore_index];
|
|
default:
|
|
unreachable("unexpected batch type");
|
|
}
|
|
}
|
|
|
|
static inline VkSemaphore
|
|
vn_get_signal_semaphore(struct vn_queue_submission *submit,
|
|
uint32_t batch_index,
|
|
uint32_t semaphore_index)
|
|
{
|
|
switch (submit->batch_type) {
|
|
case VK_STRUCTURE_TYPE_SUBMIT_INFO:
|
|
return submit->submit_batches[batch_index]
|
|
.pSignalSemaphores[semaphore_index];
|
|
case VK_STRUCTURE_TYPE_SUBMIT_INFO_2:
|
|
return submit->submit2_batches[batch_index]
|
|
.pSignalSemaphoreInfos[semaphore_index]
|
|
.semaphore;
|
|
case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
|
|
return submit->sparse_batches[batch_index]
|
|
.pSignalSemaphores[semaphore_index];
|
|
default:
|
|
unreachable("unexpected batch type");
|
|
}
|
|
}
|
|
|
|
static inline size_t
|
|
vn_get_batch_size(struct vn_queue_submission *submit)
|
|
{
|
|
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
|
|
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
|
|
return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
|
|
? sizeof(VkSubmitInfo)
|
|
: sizeof(VkSubmitInfo2);
|
|
}
|
|
|
|
static inline size_t
|
|
vn_get_cmd_size(struct vn_queue_submission *submit)
|
|
{
|
|
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
|
|
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
|
|
return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
|
|
? sizeof(VkCommandBuffer)
|
|
: sizeof(VkCommandBufferSubmitInfo);
|
|
}
|
|
|
|
static inline uint32_t
|
|
vn_get_cmd_count(struct vn_queue_submission *submit, uint32_t batch_index)
|
|
{
|
|
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
|
|
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
|
|
return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
|
|
? submit->submit_batches[batch_index].commandBufferCount
|
|
: submit->submit2_batches[batch_index].commandBufferInfoCount;
|
|
}
|
|
|
|
static inline const void *
|
|
vn_get_cmds(struct vn_queue_submission *submit, uint32_t batch_index)
|
|
{
|
|
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
|
|
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
|
|
return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
|
|
? (const void *)submit->submit_batches[batch_index]
|
|
.pCommandBuffers
|
|
: (const void *)submit->submit2_batches[batch_index]
|
|
.pCommandBufferInfos;
|
|
}
|
|
|
|
static inline struct vn_command_buffer *
|
|
vn_get_cmd(struct vn_queue_submission *submit,
|
|
uint32_t batch_index,
|
|
uint32_t cmd_index)
|
|
{
|
|
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
|
|
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
|
|
return vn_command_buffer_from_handle(
|
|
submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
|
|
? submit->submit_batches[batch_index].pCommandBuffers[cmd_index]
|
|
: submit->submit2_batches[batch_index]
|
|
.pCommandBufferInfos[cmd_index]
|
|
.commandBuffer);
|
|
}
|
|
|
|
static inline VkCommandBuffer *
|
|
vn_get_temp_cmd_ptr(struct vn_queue_submission *submit, uint32_t cmd_index)
|
|
{
|
|
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
|
|
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
|
|
return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
|
|
? &submit->temp.cmd_handles[cmd_index]
|
|
: &submit->temp.cmd_infos[cmd_index].commandBuffer;
|
|
}
|
|
|
|
static inline void
|
|
vn_set_temp_cmd(struct vn_queue_submission *submit,
|
|
uint32_t cmd_index,
|
|
VkCommandBuffer cmd_handle)
|
|
{
|
|
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
|
|
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
|
|
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
|
|
submit->temp.cmd_infos[cmd_index] = (VkCommandBufferSubmitInfo){
|
|
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO,
|
|
.commandBuffer = cmd_handle,
|
|
};
|
|
} else {
|
|
submit->temp.cmd_handles[cmd_index] = cmd_handle;
|
|
}
|
|
}
|
|
|
|
static uint64_t
|
|
vn_get_signal_semaphore_counter(struct vn_queue_submission *submit,
|
|
uint32_t batch_index,
|
|
uint32_t sem_index)
|
|
{
|
|
switch (submit->batch_type) {
|
|
case VK_STRUCTURE_TYPE_SUBMIT_INFO: {
|
|
const struct VkTimelineSemaphoreSubmitInfo *timeline_sem_info =
|
|
vk_find_struct_const(submit->submit_batches[batch_index].pNext,
|
|
TIMELINE_SEMAPHORE_SUBMIT_INFO);
|
|
return timeline_sem_info->pSignalSemaphoreValues[sem_index];
|
|
}
|
|
case VK_STRUCTURE_TYPE_SUBMIT_INFO_2:
|
|
return submit->submit2_batches[batch_index]
|
|
.pSignalSemaphoreInfos[sem_index]
|
|
.value;
|
|
default:
|
|
unreachable("unexpected batch type");
|
|
}
|
|
}
|
|
|
|
static bool
|
|
vn_has_zink_sync_batch(struct vn_queue_submission *submit)
|
|
{
|
|
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
|
|
struct vn_device *dev = (void *)queue->base.base.base.device;
|
|
struct vn_instance *instance = dev->instance;
|
|
const uint32_t last_batch_index = submit->batch_count - 1;
|
|
|
|
if (!instance->engine_is_zink)
|
|
return false;
|
|
|
|
if (!submit->batch_count || !last_batch_index ||
|
|
vn_get_cmd_count(submit, last_batch_index))
|
|
return false;
|
|
|
|
if (vn_get_wait_semaphore_count(submit, last_batch_index))
|
|
return false;
|
|
|
|
const uint32_t signal_count =
|
|
vn_get_signal_semaphore_count(submit, last_batch_index);
|
|
for (uint32_t i = 0; i < signal_count; i++) {
|
|
struct vn_semaphore *sem = vn_semaphore_from_handle(
|
|
vn_get_signal_semaphore(submit, last_batch_index, i));
|
|
if (sem->feedback.slot) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static bool
|
|
vn_fix_batch_cmd_count_for_zink_sync(struct vn_queue_submission *submit,
|
|
uint32_t batch_index,
|
|
uint32_t new_cmd_count)
|
|
{
|
|
/* If the last batch is a zink sync batch which is empty but contains
|
|
* feedback, append the feedback to the previous batch instead so that
|
|
* the last batch remains empty for perf.
|
|
*/
|
|
if (batch_index == submit->batch_count - 1 &&
|
|
submit->has_zink_sync_batch) {
|
|
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
|
|
VkSubmitInfo2 *batch =
|
|
&submit->temp.submit2_batches[batch_index - 1];
|
|
assert(batch->pCommandBufferInfos);
|
|
batch->commandBufferInfoCount += new_cmd_count;
|
|
} else {
|
|
VkSubmitInfo *batch = &submit->temp.submit_batches[batch_index - 1];
|
|
assert(batch->pCommandBuffers);
|
|
batch->commandBufferCount += new_cmd_count;
|
|
}
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static bool
|
|
vn_semaphore_wait_external(struct vn_device *dev, struct vn_semaphore *sem);
|
|
|
|
static VkResult
|
|
vn_queue_submission_fix_batch_semaphores(struct vn_queue_submission *submit,
|
|
uint32_t batch_index)
|
|
{
|
|
struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle);
|
|
VkDevice dev_handle = vk_device_to_handle(queue_vk->base.device);
|
|
struct vn_device *dev = vn_device_from_handle(dev_handle);
|
|
|
|
const uint32_t wait_count =
|
|
vn_get_wait_semaphore_count(submit, batch_index);
|
|
for (uint32_t i = 0; i < wait_count; i++) {
|
|
VkSemaphore sem_handle = vn_get_wait_semaphore(submit, batch_index, i);
|
|
struct vn_semaphore *sem = vn_semaphore_from_handle(sem_handle);
|
|
const struct vn_sync_payload *payload = sem->payload;
|
|
|
|
if (payload->type != VN_SYNC_TYPE_IMPORTED_SYNC_FD)
|
|
continue;
|
|
|
|
if (!vn_semaphore_wait_external(dev, sem))
|
|
return VK_ERROR_DEVICE_LOST;
|
|
|
|
assert(dev->physical_device->renderer_sync_fd.semaphore_importable);
|
|
|
|
const VkImportSemaphoreResourceInfoMESA res_info = {
|
|
.sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_RESOURCE_INFO_MESA,
|
|
.semaphore = sem_handle,
|
|
.resourceId = 0,
|
|
};
|
|
vn_async_vkImportSemaphoreResourceMESA(dev->primary_ring, dev_handle,
|
|
&res_info);
|
|
}
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
static void
|
|
vn_queue_submission_count_batch_feedback(struct vn_queue_submission *submit,
|
|
uint32_t batch_index)
|
|
{
|
|
const uint32_t signal_count =
|
|
vn_get_signal_semaphore_count(submit, batch_index);
|
|
uint32_t feedback_types = 0;
|
|
|
|
for (uint32_t i = 0; i < signal_count; i++) {
|
|
struct vn_semaphore *sem = vn_semaphore_from_handle(
|
|
vn_get_signal_semaphore(submit, batch_index, i));
|
|
if (sem->feedback.slot) {
|
|
feedback_types |= VN_FEEDBACK_TYPE_SEMAPHORE;
|
|
submit->cmd_count++;
|
|
}
|
|
}
|
|
|
|
if (submit->batch_type != VK_STRUCTURE_TYPE_BIND_SPARSE_INFO) {
|
|
const uint32_t cmd_count = vn_get_cmd_count(submit, batch_index);
|
|
for (uint32_t i = 0; i < cmd_count; i++) {
|
|
struct vn_command_buffer *cmd = vn_get_cmd(submit, batch_index, i);
|
|
if (!list_is_empty(&cmd->builder.query_batches))
|
|
feedback_types |= VN_FEEDBACK_TYPE_QUERY;
|
|
|
|
/* If a cmd that was submitted previously and already has a feedback
|
|
* cmd linked, as long as
|
|
* VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT was not set we can
|
|
* assume it has completed execution and is no longer in the pending
|
|
* state so its safe to recycle the old feedback command.
|
|
*/
|
|
if (cmd->linked_qfb_cmd) {
|
|
assert(!cmd->builder.is_simultaneous);
|
|
|
|
vn_feedback_query_cmd_free(cmd->linked_qfb_cmd);
|
|
cmd->linked_qfb_cmd = NULL;
|
|
}
|
|
}
|
|
if (feedback_types & VN_FEEDBACK_TYPE_QUERY)
|
|
submit->cmd_count++;
|
|
|
|
if (submit->feedback_types & VN_FEEDBACK_TYPE_FENCE &&
|
|
batch_index == submit->batch_count - 1) {
|
|
feedback_types |= VN_FEEDBACK_TYPE_FENCE;
|
|
submit->cmd_count++;
|
|
}
|
|
|
|
/* Space to copy the original cmds to append feedback to it.
|
|
* If the last batch is a zink sync batch which is an empty batch with
|
|
* sem feedback, feedback will be appended to the second to last batch
|
|
* so also need to copy the second to last batch's original cmds even
|
|
* if it doesn't have feedback itself.
|
|
*/
|
|
if (feedback_types || (batch_index == submit->batch_count - 2 &&
|
|
submit->has_zink_sync_batch)) {
|
|
submit->cmd_count += cmd_count;
|
|
}
|
|
}
|
|
|
|
submit->feedback_types |= feedback_types;
|
|
}
|
|
|
|
static VkResult
|
|
vn_queue_submission_prepare(struct vn_queue_submission *submit)
|
|
{
|
|
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
|
|
struct vn_fence *fence = vn_fence_from_handle(submit->fence_handle);
|
|
|
|
assert(!fence || !fence->is_external || !fence->feedback.slot);
|
|
if (fence && fence->feedback.slot)
|
|
submit->feedback_types |= VN_FEEDBACK_TYPE_FENCE;
|
|
|
|
if (submit->batch_type != VK_STRUCTURE_TYPE_BIND_SPARSE_INFO)
|
|
submit->has_zink_sync_batch = vn_has_zink_sync_batch(submit);
|
|
|
|
submit->external_payload.ring_idx = queue->ring_idx;
|
|
|
|
submit->wsi_mem = NULL;
|
|
if (submit->batch_count == 1 &&
|
|
submit->batch_type != VK_STRUCTURE_TYPE_BIND_SPARSE_INFO) {
|
|
const struct wsi_memory_signal_submit_info *info = vk_find_struct_const(
|
|
submit->submit_batches[0].pNext, WSI_MEMORY_SIGNAL_SUBMIT_INFO_MESA);
|
|
if (info) {
|
|
submit->wsi_mem = vn_device_memory_from_handle(info->memory);
|
|
assert(!submit->wsi_mem->base_memory && submit->wsi_mem->base_bo);
|
|
}
|
|
}
|
|
|
|
for (uint32_t i = 0; i < submit->batch_count; i++) {
|
|
VkResult result = vn_queue_submission_fix_batch_semaphores(submit, i);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
|
|
vn_queue_submission_count_batch_feedback(submit, i);
|
|
}
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
static VkResult
|
|
vn_queue_submission_alloc_storage(struct vn_queue_submission *submit)
|
|
{
|
|
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
|
|
|
|
if (!submit->feedback_types)
|
|
return VK_SUCCESS;
|
|
|
|
/* for original batches or a new batch to hold feedback fence cmd */
|
|
const size_t total_batch_size =
|
|
vn_get_batch_size(submit) * MAX2(submit->batch_count, 1);
|
|
/* for fence, timeline semaphore and query feedback cmds */
|
|
const size_t total_cmd_size =
|
|
vn_get_cmd_size(submit) * MAX2(submit->cmd_count, 1);
|
|
submit->temp.storage = vn_cached_storage_get(
|
|
&queue->storage, total_batch_size + total_cmd_size);
|
|
if (!submit->temp.storage)
|
|
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
|
|
|
submit->temp.batches = submit->temp.storage;
|
|
submit->temp.cmds = submit->temp.storage + total_batch_size;
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
static VkResult
|
|
vn_combine_query_feedback_batches_and_record(
|
|
VkDevice dev_handle,
|
|
VkCommandBuffer *cmd_handles,
|
|
uint32_t cmd_count,
|
|
uint32_t cmd_stride,
|
|
struct vn_feedback_cmd_pool *fb_cmd_pool,
|
|
struct vn_query_feedback_cmd **out_qfb_cmd)
|
|
{
|
|
struct vn_command_pool *cmd_pool =
|
|
vn_command_pool_from_handle(fb_cmd_pool->pool_handle);
|
|
VkResult result = VK_SUCCESS;
|
|
|
|
struct list_head combined_batches;
|
|
list_inithead(&combined_batches);
|
|
|
|
uintptr_t cmd_handle_ptr = (uintptr_t)cmd_handles;
|
|
for (uint32_t i = 0; i < cmd_count; i++) {
|
|
struct vn_command_buffer *cmd =
|
|
vn_command_buffer_from_handle(*(VkCommandBuffer *)cmd_handle_ptr);
|
|
|
|
list_for_each_entry(struct vn_feedback_query_batch, batch,
|
|
&cmd->builder.query_batches, head) {
|
|
if (!batch->copy) {
|
|
list_for_each_entry_safe(struct vn_feedback_query_batch,
|
|
batch_clone, &combined_batches, head) {
|
|
/* If we previously added a query feedback that is now getting
|
|
* reset, remove it since it is now a no-op and the deferred
|
|
* feedback copy will cause a hang waiting for the reset query
|
|
* to become available.
|
|
*/
|
|
if (batch_clone->copy &&
|
|
(vn_query_pool_to_handle(batch_clone->query_pool) ==
|
|
vn_query_pool_to_handle(batch->query_pool)) &&
|
|
batch_clone->query >= batch->query &&
|
|
batch_clone->query < batch->query + batch->query_count) {
|
|
simple_mtx_lock(&fb_cmd_pool->mutex);
|
|
list_move_to(&batch_clone->head,
|
|
&cmd_pool->free_query_batches);
|
|
simple_mtx_unlock(&fb_cmd_pool->mutex);
|
|
}
|
|
}
|
|
}
|
|
|
|
simple_mtx_lock(&fb_cmd_pool->mutex);
|
|
struct vn_feedback_query_batch *batch_clone =
|
|
vn_cmd_query_batch_alloc(cmd_pool, batch->query_pool,
|
|
batch->query, batch->query_count,
|
|
batch->copy);
|
|
simple_mtx_unlock(&fb_cmd_pool->mutex);
|
|
if (!batch_clone) {
|
|
result = VK_ERROR_OUT_OF_HOST_MEMORY;
|
|
goto recycle_combined_batches;
|
|
}
|
|
|
|
list_addtail(&batch_clone->head, &combined_batches);
|
|
}
|
|
|
|
cmd_handle_ptr += cmd_stride;
|
|
}
|
|
|
|
if (list_is_empty(&combined_batches)) {
|
|
/* On the off chance the combined list resolves to empty due to
|
|
* resets, we can return with a null feedback cmd to indicate
|
|
* the query feedback cmd is noop and can be skipped.
|
|
*/
|
|
*out_qfb_cmd = NULL;
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
struct vn_query_feedback_cmd *qfb_cmd;
|
|
result = vn_feedback_query_cmd_alloc(dev_handle, fb_cmd_pool, &qfb_cmd);
|
|
if (result == VK_SUCCESS) {
|
|
result = vn_feedback_query_batch_record(dev_handle, qfb_cmd,
|
|
&combined_batches);
|
|
if (result != VK_SUCCESS)
|
|
vn_feedback_query_cmd_free(qfb_cmd);
|
|
}
|
|
|
|
recycle_combined_batches:
|
|
simple_mtx_lock(&fb_cmd_pool->mutex);
|
|
list_for_each_entry_safe(struct vn_feedback_query_batch, batch_clone,
|
|
&combined_batches, head)
|
|
list_move_to(&batch_clone->head, &cmd_pool->free_query_batches);
|
|
simple_mtx_unlock(&fb_cmd_pool->mutex);
|
|
|
|
*out_qfb_cmd = qfb_cmd;
|
|
|
|
return result;
|
|
}
|
|
|
|
static VkResult
|
|
vn_queue_submission_add_query_feedback(struct vn_queue_submission *submit,
|
|
uint32_t *cmd_count)
|
|
{
|
|
struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle);
|
|
VkDevice dev_handle = vk_device_to_handle(queue_vk->base.device);
|
|
struct vn_device *dev = vn_device_from_handle(dev_handle);
|
|
VkCommandBuffer *cmd_handles = vn_get_temp_cmd_ptr(submit, 0);
|
|
|
|
struct vn_feedback_cmd_pool *fb_cmd_pool = NULL;
|
|
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
|
|
if (dev->queue_families[i] == queue_vk->queue_family_index) {
|
|
fb_cmd_pool = &dev->fb_cmd_pools[i];
|
|
break;
|
|
}
|
|
}
|
|
|
|
struct vn_query_feedback_cmd *qfb_cmd = NULL;
|
|
VkResult result = vn_combine_query_feedback_batches_and_record(
|
|
dev_handle, cmd_handles, *cmd_count, vn_get_cmd_size(submit),
|
|
fb_cmd_pool, &qfb_cmd);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
|
|
if (!qfb_cmd) {
|
|
/* No query feedback needed, return without incrementing cmd_count */
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
/* link query feedback cmd lifecycle with a cmd in the original batch so
|
|
* that the feedback cmd can be reset and recycled when that cmd gets
|
|
* reset/freed.
|
|
*
|
|
* Avoid cmd buffers with VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT
|
|
* since we don't know if all its instances have completed execution.
|
|
* Should be rare enough to just log and leak the feedback cmd.
|
|
*/
|
|
bool found_companion_cmd = false;
|
|
for (uint32_t i = 0; i < *cmd_count; i++) {
|
|
VkCommandBuffer *cmd_handle = vn_get_temp_cmd_ptr(submit, i);
|
|
struct vn_command_buffer *cmd =
|
|
vn_command_buffer_from_handle(*cmd_handle);
|
|
if (!cmd->builder.is_simultaneous) {
|
|
cmd->linked_qfb_cmd = qfb_cmd;
|
|
found_companion_cmd = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!found_companion_cmd) {
|
|
vn_log(dev->instance,
|
|
"Could not find non simultaneous cmd to link query feedback\n");
|
|
}
|
|
|
|
vn_set_temp_cmd(submit, *cmd_count,
|
|
vn_command_buffer_to_handle(qfb_cmd->cmd));
|
|
(*cmd_count)++;
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
struct vn_semaphore_feedback_cmd *
|
|
vn_semaphore_get_feedback_cmd(struct vn_device *dev,
|
|
struct vn_semaphore *sem);
|
|
|
|
static VkResult
|
|
vn_queue_submission_add_semaphore_feedback(struct vn_queue_submission *submit,
|
|
uint32_t batch_index,
|
|
uint32_t signal_index,
|
|
uint32_t *cmd_count)
|
|
{
|
|
struct vn_semaphore *sem = vn_semaphore_from_handle(
|
|
vn_get_signal_semaphore(submit, batch_index, signal_index));
|
|
if (!sem->feedback.slot)
|
|
return VK_SUCCESS;
|
|
|
|
VK_FROM_HANDLE(vk_queue, queue_vk, submit->queue_handle);
|
|
struct vn_device *dev = (void *)queue_vk->base.device;
|
|
struct vn_semaphore_feedback_cmd *sfb_cmd =
|
|
vn_semaphore_get_feedback_cmd(dev, sem);
|
|
if (!sfb_cmd)
|
|
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
|
|
|
const uint64_t counter =
|
|
vn_get_signal_semaphore_counter(submit, batch_index, signal_index);
|
|
vn_feedback_set_counter(sfb_cmd->src_slot, counter);
|
|
|
|
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
|
|
if (dev->queue_families[i] == queue_vk->queue_family_index) {
|
|
vn_set_temp_cmd(submit, *cmd_count, sfb_cmd->cmd_handles[i]);
|
|
(*cmd_count)++;
|
|
return VK_SUCCESS;
|
|
}
|
|
}
|
|
|
|
unreachable("bad feedback sem");
|
|
}
|
|
|
|
static void
|
|
vn_queue_submission_add_fence_feedback(struct vn_queue_submission *submit,
|
|
uint32_t batch_index,
|
|
uint32_t *cmd_count)
|
|
{
|
|
VK_FROM_HANDLE(vk_queue, queue_vk, submit->queue_handle);
|
|
struct vn_device *dev = (void *)queue_vk->base.device;
|
|
struct vn_fence *fence = vn_fence_from_handle(submit->fence_handle);
|
|
|
|
VkCommandBuffer ffb_cmd_handle = VK_NULL_HANDLE;
|
|
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
|
|
if (dev->queue_families[i] == queue_vk->queue_family_index) {
|
|
ffb_cmd_handle = fence->feedback.commands[i];
|
|
}
|
|
}
|
|
assert(ffb_cmd_handle != VK_NULL_HANDLE);
|
|
|
|
vn_set_temp_cmd(submit, *cmd_count, ffb_cmd_handle);
|
|
(*cmd_count)++;
|
|
}
|
|
|
|
static VkResult
|
|
vn_queue_submission_add_feedback_cmds(struct vn_queue_submission *submit,
|
|
uint32_t batch_index,
|
|
uint32_t cmd_count,
|
|
uint32_t feedback_types)
|
|
{
|
|
VkResult result;
|
|
uint32_t new_cmd_count = cmd_count;
|
|
|
|
if (feedback_types & VN_FEEDBACK_TYPE_QUERY) {
|
|
result = vn_queue_submission_add_query_feedback(submit, &new_cmd_count);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
}
|
|
|
|
if (feedback_types & VN_FEEDBACK_TYPE_SEMAPHORE) {
|
|
const uint32_t signal_count =
|
|
vn_get_signal_semaphore_count(submit, batch_index);
|
|
for (uint32_t i = 0; i < signal_count; i++) {
|
|
result = vn_queue_submission_add_semaphore_feedback(
|
|
submit, batch_index, i, &new_cmd_count);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
}
|
|
if (vn_fix_batch_cmd_count_for_zink_sync(submit, batch_index,
|
|
new_cmd_count))
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
if (feedback_types & VN_FEEDBACK_TYPE_FENCE) {
|
|
vn_queue_submission_add_fence_feedback(submit, batch_index,
|
|
&new_cmd_count);
|
|
}
|
|
|
|
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
|
|
VkSubmitInfo2 *batch = &submit->temp.submit2_batches[batch_index];
|
|
batch->pCommandBufferInfos = submit->temp.cmd_infos;
|
|
batch->commandBufferInfoCount = new_cmd_count;
|
|
} else {
|
|
VkSubmitInfo *batch = &submit->temp.submit_batches[batch_index];
|
|
batch->pCommandBuffers = submit->temp.cmd_handles;
|
|
batch->commandBufferCount = new_cmd_count;
|
|
}
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
static VkResult
|
|
vn_queue_submission_setup_batch(struct vn_queue_submission *submit,
|
|
uint32_t batch_index)
|
|
{
|
|
const uint32_t cmd_count = vn_get_cmd_count(submit, batch_index);
|
|
const uint32_t signal_count =
|
|
vn_get_signal_semaphore_count(submit, batch_index);
|
|
|
|
uint32_t feedback_types = 0;
|
|
uint32_t extra_cmd_count = 0;
|
|
for (uint32_t i = 0; i < signal_count; i++) {
|
|
struct vn_semaphore *sem = vn_semaphore_from_handle(
|
|
vn_get_signal_semaphore(submit, batch_index, i));
|
|
if (sem->feedback.slot) {
|
|
feedback_types |= VN_FEEDBACK_TYPE_SEMAPHORE;
|
|
extra_cmd_count++;
|
|
}
|
|
}
|
|
|
|
for (uint32_t i = 0; i < cmd_count; i++) {
|
|
struct vn_command_buffer *cmd = vn_get_cmd(submit, batch_index, i);
|
|
if (!list_is_empty(&cmd->builder.query_batches)) {
|
|
feedback_types |= VN_FEEDBACK_TYPE_QUERY;
|
|
extra_cmd_count++;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (submit->feedback_types & VN_FEEDBACK_TYPE_FENCE &&
|
|
batch_index == submit->batch_count - 1) {
|
|
feedback_types |= VN_FEEDBACK_TYPE_FENCE;
|
|
extra_cmd_count++;
|
|
}
|
|
|
|
/* If the batch has qfb, sfb or ffb, copy the original commands and append
|
|
* feedback cmds.
|
|
* If this is the second to last batch and the last batch a zink sync batch
|
|
* which is empty but has feedback, also copy the original commands for
|
|
* this batch so that the last batch's feedback can be appended to it.
|
|
*/
|
|
if (extra_cmd_count || (batch_index == submit->batch_count - 2 &&
|
|
submit->has_zink_sync_batch)) {
|
|
const size_t cmd_size = vn_get_cmd_size(submit);
|
|
const size_t total_cmd_size = cmd_count * cmd_size;
|
|
/* copy only needed for non-empty batches */
|
|
if (total_cmd_size) {
|
|
memcpy(submit->temp.cmds, vn_get_cmds(submit, batch_index),
|
|
total_cmd_size);
|
|
}
|
|
|
|
VkResult result = vn_queue_submission_add_feedback_cmds(
|
|
submit, batch_index, cmd_count, feedback_types);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
|
|
/* advance the temp cmds for working on next batch cmds */
|
|
submit->temp.cmds += total_cmd_size + (extra_cmd_count * cmd_size);
|
|
}
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
static VkResult
|
|
vn_queue_submission_setup_batches(struct vn_queue_submission *submit)
|
|
{
|
|
assert(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2 ||
|
|
submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO);
|
|
|
|
if (!submit->feedback_types)
|
|
return VK_SUCCESS;
|
|
|
|
/* For a submission that is:
|
|
* - non-empty: copy batches for adding feedbacks
|
|
* - empty: initialize a batch for fence feedback
|
|
*/
|
|
if (submit->batch_count) {
|
|
memcpy(submit->temp.batches, submit->batches,
|
|
vn_get_batch_size(submit) * submit->batch_count);
|
|
} else {
|
|
assert(submit->feedback_types & VN_FEEDBACK_TYPE_FENCE);
|
|
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
|
|
submit->temp.submit2_batches[0] = (VkSubmitInfo2){
|
|
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
|
|
};
|
|
} else {
|
|
submit->temp.submit_batches[0] = (VkSubmitInfo){
|
|
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
|
|
};
|
|
}
|
|
submit->batch_count = 1;
|
|
submit->batches = submit->temp.batches;
|
|
}
|
|
|
|
for (uint32_t i = 0; i < submit->batch_count; i++) {
|
|
VkResult result = vn_queue_submission_setup_batch(submit, i);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
}
|
|
|
|
submit->batches = submit->temp.batches;
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
static void
|
|
vn_queue_submission_cleanup_semaphore_feedback(
|
|
struct vn_queue_submission *submit)
|
|
{
|
|
struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle);
|
|
VkDevice dev_handle = vk_device_to_handle(queue_vk->base.device);
|
|
|
|
for (uint32_t i = 0; i < submit->batch_count; i++) {
|
|
const uint32_t wait_count = vn_get_wait_semaphore_count(submit, i);
|
|
for (uint32_t j = 0; j < wait_count; j++) {
|
|
VkSemaphore sem_handle = vn_get_wait_semaphore(submit, i, j);
|
|
struct vn_semaphore *sem = vn_semaphore_from_handle(sem_handle);
|
|
if (!sem->feedback.slot)
|
|
continue;
|
|
|
|
/* sfb pending cmds are recycled when signaled counter is updated */
|
|
uint64_t counter = 0;
|
|
vn_GetSemaphoreCounterValue(dev_handle, sem_handle, &counter);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
vn_queue_submission_cleanup(struct vn_queue_submission *submit)
|
|
{
|
|
/* TODO clean up pending src feedbacks on failure? */
|
|
if (submit->feedback_types & VN_FEEDBACK_TYPE_SEMAPHORE)
|
|
vn_queue_submission_cleanup_semaphore_feedback(submit);
|
|
}
|
|
|
|
static VkResult
|
|
vn_queue_submission_prepare_submit(struct vn_queue_submission *submit)
|
|
{
|
|
VkResult result = vn_queue_submission_prepare(submit);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
|
|
result = vn_queue_submission_alloc_storage(submit);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
|
|
result = vn_queue_submission_setup_batches(submit);
|
|
if (result != VK_SUCCESS) {
|
|
vn_queue_submission_cleanup(submit);
|
|
return result;
|
|
}
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
static void
|
|
vn_queue_wsi_present(struct vn_queue_submission *submit)
|
|
{
|
|
struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle);
|
|
struct vn_device *dev = (void *)queue_vk->base.device;
|
|
|
|
if (!submit->wsi_mem)
|
|
return;
|
|
|
|
if (dev->renderer->info.has_implicit_fencing) {
|
|
struct vn_renderer_submit_batch batch = {
|
|
.ring_idx = submit->external_payload.ring_idx,
|
|
};
|
|
|
|
uint32_t local_data[8];
|
|
struct vn_cs_encoder local_enc =
|
|
VN_CS_ENCODER_INITIALIZER_LOCAL(local_data, sizeof(local_data));
|
|
if (submit->external_payload.ring_seqno_valid) {
|
|
const uint64_t ring_id = vn_ring_get_id(dev->primary_ring);
|
|
vn_encode_vkWaitRingSeqnoMESA(&local_enc, 0, ring_id,
|
|
submit->external_payload.ring_seqno);
|
|
batch.cs_data = local_data;
|
|
batch.cs_size = vn_cs_encoder_get_len(&local_enc);
|
|
}
|
|
|
|
const struct vn_renderer_submit renderer_submit = {
|
|
.bos = &submit->wsi_mem->base_bo,
|
|
.bo_count = 1,
|
|
.batches = &batch,
|
|
.batch_count = 1,
|
|
};
|
|
vn_renderer_submit(dev->renderer, &renderer_submit);
|
|
} else {
|
|
if (VN_DEBUG(WSI)) {
|
|
static uint32_t num_rate_limit_warning = 0;
|
|
|
|
if (num_rate_limit_warning++ < 10)
|
|
vn_log(dev->instance,
|
|
"forcing vkQueueWaitIdle before presenting");
|
|
}
|
|
|
|
vn_QueueWaitIdle(submit->queue_handle);
|
|
}
|
|
}
|
|
|
|
static VkResult
|
|
vn_queue_submit(struct vn_queue_submission *submit)
|
|
{
|
|
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
|
|
struct vn_device *dev = (void *)queue->base.base.base.device;
|
|
struct vn_instance *instance = dev->instance;
|
|
VkResult result;
|
|
|
|
/* To ensure external components waiting on the correct fence payload,
|
|
* below sync primitives must be installed after the submission:
|
|
* - explicit fencing: sync file export
|
|
* - implicit fencing: dma-fence attached to the wsi bo
|
|
*
|
|
* We enforce above via an asynchronous vkQueueSubmit(2) via ring followed
|
|
* by an asynchronous renderer submission to wait for the ring submission:
|
|
* - struct wsi_memory_signal_submit_info
|
|
* - fence is an external fence
|
|
* - has an external signal semaphore
|
|
*/
|
|
result = vn_queue_submission_prepare_submit(submit);
|
|
if (result != VK_SUCCESS)
|
|
return vn_error(instance, result);
|
|
|
|
/* skip no-op submit */
|
|
if (!submit->batch_count && submit->fence_handle == VK_NULL_HANDLE)
|
|
return VK_SUCCESS;
|
|
|
|
if (VN_PERF(NO_ASYNC_QUEUE_SUBMIT)) {
|
|
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
|
|
result = vn_call_vkQueueSubmit2(
|
|
dev->primary_ring, submit->queue_handle, submit->batch_count,
|
|
submit->submit2_batches, submit->fence_handle);
|
|
} else {
|
|
result = vn_call_vkQueueSubmit(
|
|
dev->primary_ring, submit->queue_handle, submit->batch_count,
|
|
submit->submit_batches, submit->fence_handle);
|
|
}
|
|
|
|
if (result != VK_SUCCESS) {
|
|
vn_queue_submission_cleanup(submit);
|
|
return vn_error(instance, result);
|
|
}
|
|
} else {
|
|
struct vn_ring_submit_command instance_submit;
|
|
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
|
|
vn_submit_vkQueueSubmit2(
|
|
dev->primary_ring, 0, submit->queue_handle, submit->batch_count,
|
|
submit->submit2_batches, submit->fence_handle, &instance_submit);
|
|
} else {
|
|
vn_submit_vkQueueSubmit(dev->primary_ring, 0, submit->queue_handle,
|
|
submit->batch_count, submit->submit_batches,
|
|
submit->fence_handle, &instance_submit);
|
|
}
|
|
if (!instance_submit.ring_seqno_valid) {
|
|
vn_queue_submission_cleanup(submit);
|
|
return vn_error(instance, VK_ERROR_DEVICE_LOST);
|
|
}
|
|
submit->external_payload.ring_seqno_valid = true;
|
|
submit->external_payload.ring_seqno = instance_submit.ring_seqno;
|
|
}
|
|
|
|
/* If external fence, track the submission's ring_idx to facilitate
|
|
* sync_file export.
|
|
*
|
|
* Imported syncs don't need a proxy renderer sync on subsequent export,
|
|
* because an fd is already available.
|
|
*/
|
|
struct vn_fence *fence = vn_fence_from_handle(submit->fence_handle);
|
|
if (fence && fence->is_external) {
|
|
assert(fence->payload->type == VN_SYNC_TYPE_DEVICE_ONLY);
|
|
fence->external_payload = submit->external_payload;
|
|
}
|
|
|
|
for (uint32_t i = 0; i < submit->batch_count; i++) {
|
|
const uint32_t signal_count = vn_get_signal_semaphore_count(submit, i);
|
|
for (uint32_t j = 0; j < signal_count; j++) {
|
|
struct vn_semaphore *sem =
|
|
vn_semaphore_from_handle(vn_get_signal_semaphore(submit, i, j));
|
|
if (sem->is_external) {
|
|
assert(sem->payload->type == VN_SYNC_TYPE_DEVICE_ONLY);
|
|
sem->external_payload = submit->external_payload;
|
|
}
|
|
}
|
|
}
|
|
|
|
vn_queue_wsi_present(submit);
|
|
|
|
vn_queue_submission_cleanup(submit);
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
VkResult
|
|
vn_QueueSubmit(VkQueue queue,
|
|
uint32_t submitCount,
|
|
const VkSubmitInfo *pSubmits,
|
|
VkFence fence)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
|
|
struct vn_queue_submission submit = {
|
|
.batch_type = VK_STRUCTURE_TYPE_SUBMIT_INFO,
|
|
.queue_handle = queue,
|
|
.batch_count = submitCount,
|
|
.submit_batches = pSubmits,
|
|
.fence_handle = fence,
|
|
};
|
|
|
|
return vn_queue_submit(&submit);
|
|
}
|
|
|
|
VkResult
|
|
vn_QueueSubmit2(VkQueue queue,
|
|
uint32_t submitCount,
|
|
const VkSubmitInfo2 *pSubmits,
|
|
VkFence fence)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
|
|
struct vn_queue_submission submit = {
|
|
.batch_type = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
|
|
.queue_handle = queue,
|
|
.batch_count = submitCount,
|
|
.submit2_batches = pSubmits,
|
|
.fence_handle = fence,
|
|
};
|
|
|
|
return vn_queue_submit(&submit);
|
|
}
|
|
|
|
static VkResult
|
|
vn_queue_bind_sparse_submit(struct vn_queue_submission *submit)
|
|
{
|
|
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
|
|
struct vn_device *dev = (void *)queue->base.base.base.device;
|
|
struct vn_instance *instance = dev->instance;
|
|
VkResult result;
|
|
|
|
if (VN_PERF(NO_ASYNC_QUEUE_SUBMIT)) {
|
|
result = vn_call_vkQueueBindSparse(
|
|
dev->primary_ring, submit->queue_handle, submit->batch_count,
|
|
submit->sparse_batches, submit->fence_handle);
|
|
if (result != VK_SUCCESS)
|
|
return vn_error(instance, result);
|
|
} else {
|
|
struct vn_ring_submit_command instance_submit;
|
|
vn_submit_vkQueueBindSparse(dev->primary_ring, 0, submit->queue_handle,
|
|
submit->batch_count, submit->sparse_batches,
|
|
submit->fence_handle, &instance_submit);
|
|
|
|
if (!instance_submit.ring_seqno_valid)
|
|
return vn_error(instance, VK_ERROR_DEVICE_LOST);
|
|
}
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
static VkResult
|
|
vn_queue_bind_sparse_submit_batch(struct vn_queue_submission *submit,
|
|
uint32_t batch_index)
|
|
{
|
|
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
|
|
VkDevice dev_handle = vk_device_to_handle(queue->base.base.base.device);
|
|
const VkBindSparseInfo *sparse_info = &submit->sparse_batches[batch_index];
|
|
const VkSemaphore *signal_sem = sparse_info->pSignalSemaphores;
|
|
uint32_t signal_sem_count = sparse_info->signalSemaphoreCount;
|
|
VkResult result;
|
|
|
|
struct vn_queue_submission sparse_batch = {
|
|
.batch_type = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO,
|
|
.queue_handle = submit->queue_handle,
|
|
.batch_count = 1,
|
|
.fence_handle = VK_NULL_HANDLE,
|
|
};
|
|
|
|
/* lazily create sparse semaphore */
|
|
if (queue->sparse_semaphore == VK_NULL_HANDLE) {
|
|
queue->sparse_semaphore_counter = 1;
|
|
const VkSemaphoreTypeCreateInfo sem_type_create_info = {
|
|
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
|
|
.pNext = NULL,
|
|
/* This must be timeline type to adhere to mesa's requirement
|
|
* not to mix binary semaphores with wait-before-signal.
|
|
*/
|
|
.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
|
|
.initialValue = 1,
|
|
};
|
|
const VkSemaphoreCreateInfo create_info = {
|
|
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
|
|
.pNext = &sem_type_create_info,
|
|
.flags = 0,
|
|
};
|
|
|
|
result = vn_CreateSemaphore(dev_handle, &create_info, NULL,
|
|
&queue->sparse_semaphore);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
}
|
|
|
|
/* Setup VkTimelineSemaphoreSubmitInfo's for our queue sparse semaphore
|
|
* so that the vkQueueSubmit waits on the vkQueueBindSparse signal.
|
|
*/
|
|
queue->sparse_semaphore_counter++;
|
|
struct VkTimelineSemaphoreSubmitInfo wait_timeline_sem_info = { 0 };
|
|
wait_timeline_sem_info.sType =
|
|
VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO;
|
|
wait_timeline_sem_info.signalSemaphoreValueCount = 1;
|
|
wait_timeline_sem_info.pSignalSemaphoreValues =
|
|
&queue->sparse_semaphore_counter;
|
|
|
|
struct VkTimelineSemaphoreSubmitInfo signal_timeline_sem_info = { 0 };
|
|
signal_timeline_sem_info.sType =
|
|
VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO;
|
|
signal_timeline_sem_info.waitSemaphoreValueCount = 1;
|
|
signal_timeline_sem_info.pWaitSemaphoreValues =
|
|
&queue->sparse_semaphore_counter;
|
|
|
|
/* Split up the original wait and signal semaphores into its respective
|
|
* vkTimelineSemaphoreSubmitInfo
|
|
*/
|
|
const struct VkTimelineSemaphoreSubmitInfo *timeline_sem_info =
|
|
vk_find_struct_const(sparse_info->pNext,
|
|
TIMELINE_SEMAPHORE_SUBMIT_INFO);
|
|
if (timeline_sem_info) {
|
|
if (timeline_sem_info->waitSemaphoreValueCount) {
|
|
wait_timeline_sem_info.waitSemaphoreValueCount =
|
|
timeline_sem_info->waitSemaphoreValueCount;
|
|
wait_timeline_sem_info.pWaitSemaphoreValues =
|
|
timeline_sem_info->pWaitSemaphoreValues;
|
|
}
|
|
|
|
if (timeline_sem_info->signalSemaphoreValueCount) {
|
|
signal_timeline_sem_info.signalSemaphoreValueCount =
|
|
timeline_sem_info->signalSemaphoreValueCount;
|
|
signal_timeline_sem_info.pSignalSemaphoreValues =
|
|
timeline_sem_info->pSignalSemaphoreValues;
|
|
}
|
|
}
|
|
|
|
/* Attach the original VkDeviceGroupBindSparseInfo if it exists */
|
|
struct VkDeviceGroupBindSparseInfo batch_device_group_info;
|
|
const struct VkDeviceGroupBindSparseInfo *device_group_info =
|
|
vk_find_struct_const(sparse_info->pNext, DEVICE_GROUP_BIND_SPARSE_INFO);
|
|
if (device_group_info) {
|
|
memcpy(&batch_device_group_info, device_group_info,
|
|
sizeof(*device_group_info));
|
|
batch_device_group_info.pNext = NULL;
|
|
|
|
wait_timeline_sem_info.pNext = &batch_device_group_info;
|
|
}
|
|
|
|
/* Copy the original batch VkBindSparseInfo modified to signal
|
|
* our sparse semaphore.
|
|
*/
|
|
VkBindSparseInfo batch_sparse_info;
|
|
memcpy(&batch_sparse_info, sparse_info, sizeof(*sparse_info));
|
|
|
|
batch_sparse_info.pNext = &wait_timeline_sem_info;
|
|
batch_sparse_info.signalSemaphoreCount = 1;
|
|
batch_sparse_info.pSignalSemaphores = &queue->sparse_semaphore;
|
|
|
|
/* Set up the SubmitInfo to wait on our sparse semaphore before sending
|
|
* feedback and signaling the original semaphores/fence
|
|
*
|
|
* Even if this VkBindSparse batch does not have feedback semaphores,
|
|
* we still glue all the batches together to ensure the feedback
|
|
* fence occurs after.
|
|
*/
|
|
VkPipelineStageFlags stage_masks = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT;
|
|
VkSubmitInfo batch_submit_info = {
|
|
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
|
|
.pNext = &signal_timeline_sem_info,
|
|
.waitSemaphoreCount = 1,
|
|
.pWaitSemaphores = &queue->sparse_semaphore,
|
|
.pWaitDstStageMask = &stage_masks,
|
|
.signalSemaphoreCount = signal_sem_count,
|
|
.pSignalSemaphores = signal_sem,
|
|
};
|
|
|
|
/* Set the possible fence if on the last batch */
|
|
VkFence fence_handle = VK_NULL_HANDLE;
|
|
if ((submit->feedback_types & VN_FEEDBACK_TYPE_FENCE) &&
|
|
batch_index == (submit->batch_count - 1)) {
|
|
fence_handle = submit->fence_handle;
|
|
}
|
|
|
|
sparse_batch.sparse_batches = &batch_sparse_info;
|
|
result = vn_queue_bind_sparse_submit(&sparse_batch);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
|
|
result = vn_QueueSubmit(submit->queue_handle, 1, &batch_submit_info,
|
|
fence_handle);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
VkResult
|
|
vn_QueueBindSparse(VkQueue queue,
|
|
uint32_t bindInfoCount,
|
|
const VkBindSparseInfo *pBindInfo,
|
|
VkFence fence)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
VkResult result;
|
|
|
|
struct vn_queue_submission submit = {
|
|
.batch_type = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO,
|
|
.queue_handle = queue,
|
|
.batch_count = bindInfoCount,
|
|
.sparse_batches = pBindInfo,
|
|
.fence_handle = fence,
|
|
};
|
|
|
|
result = vn_queue_submission_prepare(&submit);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
|
|
if (!submit.batch_count) {
|
|
/* skip no-op submit */
|
|
if (submit.fence_handle == VK_NULL_HANDLE)
|
|
return VK_SUCCESS;
|
|
|
|
/* if empty batch, just send a vkQueueSubmit with the fence */
|
|
result =
|
|
vn_QueueSubmit(submit.queue_handle, 0, NULL, submit.fence_handle);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
}
|
|
|
|
/* if feedback isn't used in the batch, can directly submit */
|
|
if (!submit.feedback_types)
|
|
return vn_queue_bind_sparse_submit(&submit);
|
|
|
|
for (uint32_t i = 0; i < submit.batch_count; i++) {
|
|
result = vn_queue_bind_sparse_submit_batch(&submit, i);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
}
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
VkResult
|
|
vn_QueueWaitIdle(VkQueue _queue)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_queue *queue = vn_queue_from_handle(_queue);
|
|
VkDevice dev_handle = vk_device_to_handle(queue->base.base.base.device);
|
|
struct vn_device *dev = vn_device_from_handle(dev_handle);
|
|
VkResult result;
|
|
|
|
/* lazily create queue wait fence for queue idle waiting */
|
|
if (queue->wait_fence == VK_NULL_HANDLE) {
|
|
const VkFenceCreateInfo create_info = {
|
|
.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO,
|
|
.flags = 0,
|
|
};
|
|
result =
|
|
vn_CreateFence(dev_handle, &create_info, NULL, &queue->wait_fence);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
}
|
|
|
|
result = vn_QueueSubmit(_queue, 0, NULL, queue->wait_fence);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
|
|
result =
|
|
vn_WaitForFences(dev_handle, 1, &queue->wait_fence, true, UINT64_MAX);
|
|
vn_ResetFences(dev_handle, 1, &queue->wait_fence);
|
|
|
|
return vn_result(dev->instance, result);
|
|
}
|
|
|
|
/* fence commands */
|
|
|
|
static void
|
|
vn_sync_payload_release(UNUSED struct vn_device *dev,
|
|
struct vn_sync_payload *payload)
|
|
{
|
|
if (payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD && payload->fd >= 0)
|
|
close(payload->fd);
|
|
|
|
payload->type = VN_SYNC_TYPE_INVALID;
|
|
}
|
|
|
|
static VkResult
|
|
vn_fence_init_payloads(struct vn_device *dev,
|
|
struct vn_fence *fence,
|
|
bool signaled,
|
|
const VkAllocationCallbacks *alloc)
|
|
{
|
|
fence->permanent.type = VN_SYNC_TYPE_DEVICE_ONLY;
|
|
fence->temporary.type = VN_SYNC_TYPE_INVALID;
|
|
fence->payload = &fence->permanent;
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
void
|
|
vn_fence_signal_wsi(struct vn_device *dev, struct vn_fence *fence)
|
|
{
|
|
struct vn_sync_payload *temp = &fence->temporary;
|
|
|
|
vn_sync_payload_release(dev, temp);
|
|
temp->type = VN_SYNC_TYPE_IMPORTED_SYNC_FD;
|
|
temp->fd = -1;
|
|
fence->payload = temp;
|
|
}
|
|
|
|
static VkResult
|
|
vn_fence_feedback_init(struct vn_device *dev,
|
|
struct vn_fence *fence,
|
|
bool signaled,
|
|
const VkAllocationCallbacks *alloc)
|
|
{
|
|
VkDevice dev_handle = vn_device_to_handle(dev);
|
|
struct vn_feedback_slot *slot;
|
|
VkCommandBuffer *cmd_handles;
|
|
VkResult result;
|
|
|
|
if (fence->is_external)
|
|
return VK_SUCCESS;
|
|
|
|
if (VN_PERF(NO_FENCE_FEEDBACK))
|
|
return VK_SUCCESS;
|
|
|
|
slot = vn_feedback_pool_alloc(&dev->feedback_pool, VN_FEEDBACK_TYPE_FENCE);
|
|
if (!slot)
|
|
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
|
|
|
vn_feedback_set_status(slot, signaled ? VK_SUCCESS : VK_NOT_READY);
|
|
|
|
cmd_handles =
|
|
vk_zalloc(alloc, sizeof(*cmd_handles) * dev->queue_family_count,
|
|
VN_DEFAULT_ALIGN, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
|
|
if (!cmd_handles) {
|
|
vn_feedback_pool_free(&dev->feedback_pool, slot);
|
|
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
|
}
|
|
|
|
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
|
|
result = vn_feedback_cmd_alloc(dev_handle, &dev->fb_cmd_pools[i], slot,
|
|
NULL, &cmd_handles[i]);
|
|
if (result != VK_SUCCESS) {
|
|
for (uint32_t j = 0; j < i; j++) {
|
|
vn_feedback_cmd_free(dev_handle, &dev->fb_cmd_pools[j],
|
|
cmd_handles[j]);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (result != VK_SUCCESS) {
|
|
vk_free(alloc, cmd_handles);
|
|
vn_feedback_pool_free(&dev->feedback_pool, slot);
|
|
return result;
|
|
}
|
|
|
|
fence->feedback.slot = slot;
|
|
fence->feedback.commands = cmd_handles;
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
static void
|
|
vn_fence_feedback_fini(struct vn_device *dev,
|
|
struct vn_fence *fence,
|
|
const VkAllocationCallbacks *alloc)
|
|
{
|
|
VkDevice dev_handle = vn_device_to_handle(dev);
|
|
|
|
if (!fence->feedback.slot)
|
|
return;
|
|
|
|
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
|
|
vn_feedback_cmd_free(dev_handle, &dev->fb_cmd_pools[i],
|
|
fence->feedback.commands[i]);
|
|
}
|
|
|
|
vn_feedback_pool_free(&dev->feedback_pool, fence->feedback.slot);
|
|
|
|
vk_free(alloc, fence->feedback.commands);
|
|
}
|
|
|
|
VkResult
|
|
vn_CreateFence(VkDevice device,
|
|
const VkFenceCreateInfo *pCreateInfo,
|
|
const VkAllocationCallbacks *pAllocator,
|
|
VkFence *pFence)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
const VkAllocationCallbacks *alloc =
|
|
pAllocator ? pAllocator : &dev->base.base.alloc;
|
|
const bool signaled = pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT;
|
|
VkResult result;
|
|
|
|
struct vn_fence *fence = vk_zalloc(alloc, sizeof(*fence), VN_DEFAULT_ALIGN,
|
|
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
|
|
if (!fence)
|
|
return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
|
|
|
|
vn_object_base_init(&fence->base, VK_OBJECT_TYPE_FENCE, &dev->base);
|
|
|
|
const struct VkExportFenceCreateInfo *export_info =
|
|
vk_find_struct_const(pCreateInfo->pNext, EXPORT_FENCE_CREATE_INFO);
|
|
fence->is_external = export_info && export_info->handleTypes;
|
|
|
|
result = vn_fence_init_payloads(dev, fence, signaled, alloc);
|
|
if (result != VK_SUCCESS)
|
|
goto out_object_base_fini;
|
|
|
|
result = vn_fence_feedback_init(dev, fence, signaled, alloc);
|
|
if (result != VK_SUCCESS)
|
|
goto out_payloads_fini;
|
|
|
|
*pFence = vn_fence_to_handle(fence);
|
|
vn_async_vkCreateFence(dev->primary_ring, device, pCreateInfo, NULL,
|
|
pFence);
|
|
|
|
return VK_SUCCESS;
|
|
|
|
out_payloads_fini:
|
|
vn_sync_payload_release(dev, &fence->permanent);
|
|
vn_sync_payload_release(dev, &fence->temporary);
|
|
|
|
out_object_base_fini:
|
|
vn_object_base_fini(&fence->base);
|
|
vk_free(alloc, fence);
|
|
return vn_error(dev->instance, result);
|
|
}
|
|
|
|
void
|
|
vn_DestroyFence(VkDevice device,
|
|
VkFence _fence,
|
|
const VkAllocationCallbacks *pAllocator)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
struct vn_fence *fence = vn_fence_from_handle(_fence);
|
|
const VkAllocationCallbacks *alloc =
|
|
pAllocator ? pAllocator : &dev->base.base.alloc;
|
|
|
|
if (!fence)
|
|
return;
|
|
|
|
vn_async_vkDestroyFence(dev->primary_ring, device, _fence, NULL);
|
|
|
|
vn_fence_feedback_fini(dev, fence, alloc);
|
|
|
|
vn_sync_payload_release(dev, &fence->permanent);
|
|
vn_sync_payload_release(dev, &fence->temporary);
|
|
|
|
vn_object_base_fini(&fence->base);
|
|
vk_free(alloc, fence);
|
|
}
|
|
|
|
VkResult
|
|
vn_ResetFences(VkDevice device, uint32_t fenceCount, const VkFence *pFences)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
|
|
/* TODO if the fence is shared-by-ref, this needs to be synchronous */
|
|
if (false)
|
|
vn_call_vkResetFences(dev->primary_ring, device, fenceCount, pFences);
|
|
else
|
|
vn_async_vkResetFences(dev->primary_ring, device, fenceCount, pFences);
|
|
|
|
for (uint32_t i = 0; i < fenceCount; i++) {
|
|
struct vn_fence *fence = vn_fence_from_handle(pFences[i]);
|
|
struct vn_sync_payload *perm = &fence->permanent;
|
|
|
|
vn_sync_payload_release(dev, &fence->temporary);
|
|
|
|
assert(perm->type == VN_SYNC_TYPE_DEVICE_ONLY);
|
|
fence->payload = perm;
|
|
|
|
if (fence->feedback.slot)
|
|
vn_feedback_reset_status(fence->feedback.slot);
|
|
}
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
VkResult
|
|
vn_GetFenceStatus(VkDevice device, VkFence _fence)
|
|
{
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
struct vn_fence *fence = vn_fence_from_handle(_fence);
|
|
struct vn_sync_payload *payload = fence->payload;
|
|
|
|
VkResult result;
|
|
switch (payload->type) {
|
|
case VN_SYNC_TYPE_DEVICE_ONLY:
|
|
if (fence->feedback.slot) {
|
|
result = vn_feedback_get_status(fence->feedback.slot);
|
|
if (result == VK_SUCCESS) {
|
|
/* When fence feedback slot gets signaled, the real fence
|
|
* signal operation follows after but the signaling isr can be
|
|
* deferred or preempted. To avoid racing, we let the
|
|
* renderer wait for the fence. This also helps resolve
|
|
* synchronization validation errors, because the layer no
|
|
* longer sees any fence status checks and falsely believes the
|
|
* caller does not sync.
|
|
*/
|
|
vn_async_vkWaitForFences(dev->primary_ring, device, 1, &_fence,
|
|
VK_TRUE, UINT64_MAX);
|
|
}
|
|
} else {
|
|
result = vn_call_vkGetFenceStatus(dev->primary_ring, device, _fence);
|
|
}
|
|
break;
|
|
case VN_SYNC_TYPE_IMPORTED_SYNC_FD:
|
|
if (payload->fd < 0 || sync_wait(payload->fd, 0) == 0)
|
|
result = VK_SUCCESS;
|
|
else
|
|
result = errno == ETIME ? VK_NOT_READY : VK_ERROR_DEVICE_LOST;
|
|
break;
|
|
default:
|
|
unreachable("unexpected fence payload type");
|
|
break;
|
|
}
|
|
|
|
return vn_result(dev->instance, result);
|
|
}
|
|
|
|
static VkResult
|
|
vn_find_first_signaled_fence(VkDevice device,
|
|
const VkFence *fences,
|
|
uint32_t count)
|
|
{
|
|
for (uint32_t i = 0; i < count; i++) {
|
|
VkResult result = vn_GetFenceStatus(device, fences[i]);
|
|
if (result == VK_SUCCESS || result < 0)
|
|
return result;
|
|
}
|
|
return VK_NOT_READY;
|
|
}
|
|
|
|
static VkResult
|
|
vn_remove_signaled_fences(VkDevice device, VkFence *fences, uint32_t *count)
|
|
{
|
|
uint32_t cur = 0;
|
|
for (uint32_t i = 0; i < *count; i++) {
|
|
VkResult result = vn_GetFenceStatus(device, fences[i]);
|
|
if (result != VK_SUCCESS) {
|
|
if (result < 0)
|
|
return result;
|
|
fences[cur++] = fences[i];
|
|
}
|
|
}
|
|
|
|
*count = cur;
|
|
return cur ? VK_NOT_READY : VK_SUCCESS;
|
|
}
|
|
|
|
static VkResult
|
|
vn_update_sync_result(struct vn_device *dev,
|
|
VkResult result,
|
|
int64_t abs_timeout,
|
|
struct vn_relax_state *relax_state)
|
|
{
|
|
switch (result) {
|
|
case VK_NOT_READY:
|
|
if (abs_timeout != OS_TIMEOUT_INFINITE &&
|
|
os_time_get_nano() >= abs_timeout)
|
|
result = VK_TIMEOUT;
|
|
else
|
|
vn_relax(relax_state);
|
|
break;
|
|
default:
|
|
assert(result == VK_SUCCESS || result < 0);
|
|
break;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
VkResult
|
|
vn_WaitForFences(VkDevice device,
|
|
uint32_t fenceCount,
|
|
const VkFence *pFences,
|
|
VkBool32 waitAll,
|
|
uint64_t timeout)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
const VkAllocationCallbacks *alloc = &dev->base.base.alloc;
|
|
|
|
const int64_t abs_timeout = os_time_get_absolute_timeout(timeout);
|
|
VkResult result = VK_NOT_READY;
|
|
if (fenceCount > 1 && waitAll) {
|
|
VkFence local_fences[8];
|
|
VkFence *fences = local_fences;
|
|
if (fenceCount > ARRAY_SIZE(local_fences)) {
|
|
fences =
|
|
vk_alloc(alloc, sizeof(*fences) * fenceCount, VN_DEFAULT_ALIGN,
|
|
VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
|
|
if (!fences)
|
|
return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
|
|
}
|
|
memcpy(fences, pFences, sizeof(*fences) * fenceCount);
|
|
|
|
struct vn_relax_state relax_state =
|
|
vn_relax_init(dev->instance, "client");
|
|
while (result == VK_NOT_READY) {
|
|
result = vn_remove_signaled_fences(device, fences, &fenceCount);
|
|
result =
|
|
vn_update_sync_result(dev, result, abs_timeout, &relax_state);
|
|
}
|
|
vn_relax_fini(&relax_state);
|
|
|
|
if (fences != local_fences)
|
|
vk_free(alloc, fences);
|
|
} else {
|
|
struct vn_relax_state relax_state =
|
|
vn_relax_init(dev->instance, "client");
|
|
while (result == VK_NOT_READY) {
|
|
result = vn_find_first_signaled_fence(device, pFences, fenceCount);
|
|
result =
|
|
vn_update_sync_result(dev, result, abs_timeout, &relax_state);
|
|
}
|
|
vn_relax_fini(&relax_state);
|
|
}
|
|
|
|
return vn_result(dev->instance, result);
|
|
}
|
|
|
|
static VkResult
|
|
vn_create_sync_file(struct vn_device *dev,
|
|
struct vn_sync_payload_external *external_payload,
|
|
int *out_fd)
|
|
{
|
|
struct vn_renderer_sync *sync;
|
|
VkResult result = vn_renderer_sync_create(dev->renderer, 0,
|
|
VN_RENDERER_SYNC_BINARY, &sync);
|
|
if (result != VK_SUCCESS)
|
|
return vn_error(dev->instance, result);
|
|
|
|
struct vn_renderer_submit_batch batch = {
|
|
.syncs = &sync,
|
|
.sync_values = &(const uint64_t){ 1 },
|
|
.sync_count = 1,
|
|
.ring_idx = external_payload->ring_idx,
|
|
};
|
|
|
|
uint32_t local_data[8];
|
|
struct vn_cs_encoder local_enc =
|
|
VN_CS_ENCODER_INITIALIZER_LOCAL(local_data, sizeof(local_data));
|
|
if (external_payload->ring_seqno_valid) {
|
|
const uint64_t ring_id = vn_ring_get_id(dev->primary_ring);
|
|
vn_encode_vkWaitRingSeqnoMESA(&local_enc, 0, ring_id,
|
|
external_payload->ring_seqno);
|
|
batch.cs_data = local_data;
|
|
batch.cs_size = vn_cs_encoder_get_len(&local_enc);
|
|
}
|
|
|
|
const struct vn_renderer_submit submit = {
|
|
.batches = &batch,
|
|
.batch_count = 1,
|
|
};
|
|
result = vn_renderer_submit(dev->renderer, &submit);
|
|
if (result != VK_SUCCESS) {
|
|
vn_renderer_sync_destroy(dev->renderer, sync);
|
|
return vn_error(dev->instance, result);
|
|
}
|
|
|
|
*out_fd = vn_renderer_sync_export_syncobj(dev->renderer, sync, true);
|
|
vn_renderer_sync_destroy(dev->renderer, sync);
|
|
|
|
return *out_fd >= 0 ? VK_SUCCESS : VK_ERROR_TOO_MANY_OBJECTS;
|
|
}
|
|
|
|
static inline bool
|
|
vn_sync_valid_fd(int fd)
|
|
{
|
|
/* the special value -1 for fd is treated like a valid sync file descriptor
|
|
* referring to an object that has already signaled
|
|
*/
|
|
return (fd >= 0 && sync_valid_fd(fd)) || fd == -1;
|
|
}
|
|
|
|
VkResult
|
|
vn_ImportFenceFdKHR(VkDevice device,
|
|
const VkImportFenceFdInfoKHR *pImportFenceFdInfo)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
struct vn_fence *fence = vn_fence_from_handle(pImportFenceFdInfo->fence);
|
|
ASSERTED const bool sync_file = pImportFenceFdInfo->handleType ==
|
|
VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
|
|
const int fd = pImportFenceFdInfo->fd;
|
|
|
|
assert(sync_file);
|
|
|
|
if (!vn_sync_valid_fd(fd))
|
|
return vn_error(dev->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
|
|
|
|
struct vn_sync_payload *temp = &fence->temporary;
|
|
vn_sync_payload_release(dev, temp);
|
|
temp->type = VN_SYNC_TYPE_IMPORTED_SYNC_FD;
|
|
temp->fd = fd;
|
|
fence->payload = temp;
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
VkResult
|
|
vn_GetFenceFdKHR(VkDevice device,
|
|
const VkFenceGetFdInfoKHR *pGetFdInfo,
|
|
int *pFd)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
struct vn_fence *fence = vn_fence_from_handle(pGetFdInfo->fence);
|
|
const bool sync_file =
|
|
pGetFdInfo->handleType == VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
|
|
struct vn_sync_payload *payload = fence->payload;
|
|
VkResult result;
|
|
|
|
assert(sync_file);
|
|
assert(dev->physical_device->renderer_sync_fd.fence_exportable);
|
|
|
|
int fd = -1;
|
|
if (payload->type == VN_SYNC_TYPE_DEVICE_ONLY) {
|
|
result = vn_create_sync_file(dev, &fence->external_payload, &fd);
|
|
if (result != VK_SUCCESS)
|
|
return vn_error(dev->instance, result);
|
|
|
|
vn_async_vkResetFenceResourceMESA(dev->primary_ring, device,
|
|
pGetFdInfo->fence);
|
|
|
|
vn_sync_payload_release(dev, &fence->temporary);
|
|
fence->payload = &fence->permanent;
|
|
|
|
#ifdef VN_USE_WSI_PLATFORM
|
|
if (!dev->renderer->info.has_implicit_fencing)
|
|
sync_wait(fd, -1);
|
|
#endif
|
|
} else {
|
|
assert(payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD);
|
|
|
|
/* transfer ownership of imported sync fd to save a dup */
|
|
fd = payload->fd;
|
|
payload->fd = -1;
|
|
|
|
/* reset host fence in case in signaled state before import */
|
|
result = vn_ResetFences(device, 1, &pGetFdInfo->fence);
|
|
if (result != VK_SUCCESS) {
|
|
/* transfer sync fd ownership back on error */
|
|
payload->fd = fd;
|
|
return result;
|
|
}
|
|
}
|
|
|
|
*pFd = fd;
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
/* semaphore commands */
|
|
|
|
static VkResult
|
|
vn_semaphore_init_payloads(struct vn_device *dev,
|
|
struct vn_semaphore *sem,
|
|
uint64_t initial_val,
|
|
const VkAllocationCallbacks *alloc)
|
|
{
|
|
sem->permanent.type = VN_SYNC_TYPE_DEVICE_ONLY;
|
|
sem->temporary.type = VN_SYNC_TYPE_INVALID;
|
|
sem->payload = &sem->permanent;
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
static bool
|
|
vn_semaphore_wait_external(struct vn_device *dev, struct vn_semaphore *sem)
|
|
{
|
|
struct vn_sync_payload *temp = &sem->temporary;
|
|
|
|
assert(temp->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD);
|
|
|
|
if (temp->fd >= 0) {
|
|
if (sync_wait(temp->fd, -1))
|
|
return false;
|
|
}
|
|
|
|
vn_sync_payload_release(dev, &sem->temporary);
|
|
sem->payload = &sem->permanent;
|
|
|
|
return true;
|
|
}
|
|
|
|
void
|
|
vn_semaphore_signal_wsi(struct vn_device *dev, struct vn_semaphore *sem)
|
|
{
|
|
struct vn_sync_payload *temp = &sem->temporary;
|
|
|
|
vn_sync_payload_release(dev, temp);
|
|
temp->type = VN_SYNC_TYPE_IMPORTED_SYNC_FD;
|
|
temp->fd = -1;
|
|
sem->payload = temp;
|
|
}
|
|
|
|
struct vn_semaphore_feedback_cmd *
|
|
vn_semaphore_get_feedback_cmd(struct vn_device *dev, struct vn_semaphore *sem)
|
|
{
|
|
struct vn_semaphore_feedback_cmd *sfb_cmd = NULL;
|
|
|
|
simple_mtx_lock(&sem->feedback.cmd_mtx);
|
|
if (!list_is_empty(&sem->feedback.free_cmds)) {
|
|
sfb_cmd = list_first_entry(&sem->feedback.free_cmds,
|
|
struct vn_semaphore_feedback_cmd, head);
|
|
list_move_to(&sfb_cmd->head, &sem->feedback.pending_cmds);
|
|
}
|
|
simple_mtx_unlock(&sem->feedback.cmd_mtx);
|
|
|
|
if (!sfb_cmd) {
|
|
sfb_cmd = vn_semaphore_feedback_cmd_alloc(dev, sem->feedback.slot);
|
|
|
|
simple_mtx_lock(&sem->feedback.cmd_mtx);
|
|
list_add(&sfb_cmd->head, &sem->feedback.pending_cmds);
|
|
simple_mtx_unlock(&sem->feedback.cmd_mtx);
|
|
}
|
|
|
|
return sfb_cmd;
|
|
}
|
|
|
|
static VkResult
|
|
vn_semaphore_feedback_init(struct vn_device *dev,
|
|
struct vn_semaphore *sem,
|
|
uint64_t initial_value,
|
|
const VkAllocationCallbacks *alloc)
|
|
{
|
|
struct vn_feedback_slot *slot;
|
|
|
|
assert(sem->type == VK_SEMAPHORE_TYPE_TIMELINE);
|
|
|
|
if (sem->is_external)
|
|
return VK_SUCCESS;
|
|
|
|
if (VN_PERF(NO_SEMAPHORE_FEEDBACK))
|
|
return VK_SUCCESS;
|
|
|
|
slot =
|
|
vn_feedback_pool_alloc(&dev->feedback_pool, VN_FEEDBACK_TYPE_SEMAPHORE);
|
|
if (!slot)
|
|
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
|
|
|
list_inithead(&sem->feedback.pending_cmds);
|
|
list_inithead(&sem->feedback.free_cmds);
|
|
|
|
vn_feedback_set_counter(slot, initial_value);
|
|
|
|
simple_mtx_init(&sem->feedback.cmd_mtx, mtx_plain);
|
|
simple_mtx_init(&sem->feedback.async_wait_mtx, mtx_plain);
|
|
|
|
sem->feedback.signaled_counter = initial_value;
|
|
sem->feedback.slot = slot;
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
static void
|
|
vn_semaphore_feedback_fini(struct vn_device *dev, struct vn_semaphore *sem)
|
|
{
|
|
if (!sem->feedback.slot)
|
|
return;
|
|
|
|
list_for_each_entry_safe(struct vn_semaphore_feedback_cmd, sfb_cmd,
|
|
&sem->feedback.free_cmds, head)
|
|
vn_semaphore_feedback_cmd_free(dev, sfb_cmd);
|
|
|
|
list_for_each_entry_safe(struct vn_semaphore_feedback_cmd, sfb_cmd,
|
|
&sem->feedback.pending_cmds, head)
|
|
vn_semaphore_feedback_cmd_free(dev, sfb_cmd);
|
|
|
|
simple_mtx_destroy(&sem->feedback.cmd_mtx);
|
|
simple_mtx_destroy(&sem->feedback.async_wait_mtx);
|
|
|
|
vn_feedback_pool_free(&dev->feedback_pool, sem->feedback.slot);
|
|
}
|
|
|
|
VkResult
|
|
vn_CreateSemaphore(VkDevice device,
|
|
const VkSemaphoreCreateInfo *pCreateInfo,
|
|
const VkAllocationCallbacks *pAllocator,
|
|
VkSemaphore *pSemaphore)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
const VkAllocationCallbacks *alloc =
|
|
pAllocator ? pAllocator : &dev->base.base.alloc;
|
|
|
|
struct vn_semaphore *sem = vk_zalloc(alloc, sizeof(*sem), VN_DEFAULT_ALIGN,
|
|
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
|
|
if (!sem)
|
|
return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
|
|
|
|
vn_object_base_init(&sem->base, VK_OBJECT_TYPE_SEMAPHORE, &dev->base);
|
|
|
|
const VkSemaphoreTypeCreateInfo *type_info =
|
|
vk_find_struct_const(pCreateInfo->pNext, SEMAPHORE_TYPE_CREATE_INFO);
|
|
uint64_t initial_val = 0;
|
|
if (type_info && type_info->semaphoreType == VK_SEMAPHORE_TYPE_TIMELINE) {
|
|
sem->type = VK_SEMAPHORE_TYPE_TIMELINE;
|
|
initial_val = type_info->initialValue;
|
|
} else {
|
|
sem->type = VK_SEMAPHORE_TYPE_BINARY;
|
|
}
|
|
|
|
const struct VkExportSemaphoreCreateInfo *export_info =
|
|
vk_find_struct_const(pCreateInfo->pNext, EXPORT_SEMAPHORE_CREATE_INFO);
|
|
sem->is_external = export_info && export_info->handleTypes;
|
|
|
|
VkResult result = vn_semaphore_init_payloads(dev, sem, initial_val, alloc);
|
|
if (result != VK_SUCCESS)
|
|
goto out_object_base_fini;
|
|
|
|
if (sem->type == VK_SEMAPHORE_TYPE_TIMELINE) {
|
|
result = vn_semaphore_feedback_init(dev, sem, initial_val, alloc);
|
|
if (result != VK_SUCCESS)
|
|
goto out_payloads_fini;
|
|
}
|
|
|
|
VkSemaphore sem_handle = vn_semaphore_to_handle(sem);
|
|
vn_async_vkCreateSemaphore(dev->primary_ring, device, pCreateInfo, NULL,
|
|
&sem_handle);
|
|
|
|
*pSemaphore = sem_handle;
|
|
|
|
return VK_SUCCESS;
|
|
|
|
out_payloads_fini:
|
|
vn_sync_payload_release(dev, &sem->permanent);
|
|
vn_sync_payload_release(dev, &sem->temporary);
|
|
|
|
out_object_base_fini:
|
|
vn_object_base_fini(&sem->base);
|
|
vk_free(alloc, sem);
|
|
return vn_error(dev->instance, result);
|
|
}
|
|
|
|
void
|
|
vn_DestroySemaphore(VkDevice device,
|
|
VkSemaphore semaphore,
|
|
const VkAllocationCallbacks *pAllocator)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
struct vn_semaphore *sem = vn_semaphore_from_handle(semaphore);
|
|
const VkAllocationCallbacks *alloc =
|
|
pAllocator ? pAllocator : &dev->base.base.alloc;
|
|
|
|
if (!sem)
|
|
return;
|
|
|
|
vn_async_vkDestroySemaphore(dev->primary_ring, device, semaphore, NULL);
|
|
|
|
if (sem->type == VK_SEMAPHORE_TYPE_TIMELINE)
|
|
vn_semaphore_feedback_fini(dev, sem);
|
|
|
|
vn_sync_payload_release(dev, &sem->permanent);
|
|
vn_sync_payload_release(dev, &sem->temporary);
|
|
|
|
vn_object_base_fini(&sem->base);
|
|
vk_free(alloc, sem);
|
|
}
|
|
|
|
VkResult
|
|
vn_GetSemaphoreCounterValue(VkDevice device,
|
|
VkSemaphore semaphore,
|
|
uint64_t *pValue)
|
|
{
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
struct vn_semaphore *sem = vn_semaphore_from_handle(semaphore);
|
|
ASSERTED struct vn_sync_payload *payload = sem->payload;
|
|
|
|
assert(payload->type == VN_SYNC_TYPE_DEVICE_ONLY);
|
|
|
|
if (sem->feedback.slot) {
|
|
simple_mtx_lock(&sem->feedback.async_wait_mtx);
|
|
const uint64_t counter = vn_feedback_get_counter(sem->feedback.slot);
|
|
if (sem->feedback.signaled_counter < counter) {
|
|
/* When the timeline semaphore feedback slot gets signaled, the real
|
|
* semaphore signal operation follows after but the signaling isr can
|
|
* be deferred or preempted. To avoid racing, we let the renderer
|
|
* wait for the semaphore by sending an asynchronous wait call for
|
|
* the feedback value.
|
|
* We also cache the counter value to only send the async call once
|
|
* per counter value to prevent spamming redundant async wait calls.
|
|
* The cached counter value requires a lock to ensure multiple
|
|
* threads querying for the same value are guaranteed to encode after
|
|
* the async wait call.
|
|
*
|
|
* This also helps resolve synchronization validation errors, because
|
|
* the layer no longer sees any semaphore status checks and falsely
|
|
* believes the caller does not sync.
|
|
*/
|
|
VkSemaphoreWaitInfo wait_info = {
|
|
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
|
|
.pNext = NULL,
|
|
.flags = 0,
|
|
.semaphoreCount = 1,
|
|
.pSemaphores = &semaphore,
|
|
.pValues = &counter,
|
|
};
|
|
|
|
vn_async_vkWaitSemaphores(dev->primary_ring, device, &wait_info,
|
|
UINT64_MAX);
|
|
|
|
/* search pending cmds for already signaled values */
|
|
simple_mtx_lock(&sem->feedback.cmd_mtx);
|
|
list_for_each_entry_safe(struct vn_semaphore_feedback_cmd, sfb_cmd,
|
|
&sem->feedback.pending_cmds, head) {
|
|
if (counter >= vn_feedback_get_counter(sfb_cmd->src_slot))
|
|
list_move_to(&sfb_cmd->head, &sem->feedback.free_cmds);
|
|
}
|
|
simple_mtx_unlock(&sem->feedback.cmd_mtx);
|
|
|
|
sem->feedback.signaled_counter = counter;
|
|
}
|
|
simple_mtx_unlock(&sem->feedback.async_wait_mtx);
|
|
|
|
*pValue = counter;
|
|
return VK_SUCCESS;
|
|
} else {
|
|
return vn_call_vkGetSemaphoreCounterValue(dev->primary_ring, device,
|
|
semaphore, pValue);
|
|
}
|
|
}
|
|
|
|
VkResult
|
|
vn_SignalSemaphore(VkDevice device, const VkSemaphoreSignalInfo *pSignalInfo)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
struct vn_semaphore *sem =
|
|
vn_semaphore_from_handle(pSignalInfo->semaphore);
|
|
|
|
/* TODO if the semaphore is shared-by-ref, this needs to be synchronous */
|
|
if (false)
|
|
vn_call_vkSignalSemaphore(dev->primary_ring, device, pSignalInfo);
|
|
else
|
|
vn_async_vkSignalSemaphore(dev->primary_ring, device, pSignalInfo);
|
|
|
|
if (sem->feedback.slot) {
|
|
simple_mtx_lock(&sem->feedback.async_wait_mtx);
|
|
|
|
vn_feedback_set_counter(sem->feedback.slot, pSignalInfo->value);
|
|
/* Update async counters. Since we're signaling, we're aligned with
|
|
* the renderer.
|
|
*/
|
|
sem->feedback.signaled_counter = pSignalInfo->value;
|
|
|
|
simple_mtx_unlock(&sem->feedback.async_wait_mtx);
|
|
}
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
static VkResult
|
|
vn_find_first_signaled_semaphore(VkDevice device,
|
|
const VkSemaphore *semaphores,
|
|
const uint64_t *values,
|
|
uint32_t count)
|
|
{
|
|
for (uint32_t i = 0; i < count; i++) {
|
|
uint64_t val = 0;
|
|
VkResult result =
|
|
vn_GetSemaphoreCounterValue(device, semaphores[i], &val);
|
|
if (result != VK_SUCCESS || val >= values[i])
|
|
return result;
|
|
}
|
|
return VK_NOT_READY;
|
|
}
|
|
|
|
static VkResult
|
|
vn_remove_signaled_semaphores(VkDevice device,
|
|
VkSemaphore *semaphores,
|
|
uint64_t *values,
|
|
uint32_t *count)
|
|
{
|
|
uint32_t cur = 0;
|
|
for (uint32_t i = 0; i < *count; i++) {
|
|
uint64_t val = 0;
|
|
VkResult result =
|
|
vn_GetSemaphoreCounterValue(device, semaphores[i], &val);
|
|
if (result != VK_SUCCESS)
|
|
return result;
|
|
if (val < values[i])
|
|
semaphores[cur++] = semaphores[i];
|
|
}
|
|
|
|
*count = cur;
|
|
return cur ? VK_NOT_READY : VK_SUCCESS;
|
|
}
|
|
|
|
VkResult
|
|
vn_WaitSemaphores(VkDevice device,
|
|
const VkSemaphoreWaitInfo *pWaitInfo,
|
|
uint64_t timeout)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
const VkAllocationCallbacks *alloc = &dev->base.base.alloc;
|
|
|
|
const int64_t abs_timeout = os_time_get_absolute_timeout(timeout);
|
|
VkResult result = VK_NOT_READY;
|
|
if (pWaitInfo->semaphoreCount > 1 &&
|
|
!(pWaitInfo->flags & VK_SEMAPHORE_WAIT_ANY_BIT)) {
|
|
uint32_t semaphore_count = pWaitInfo->semaphoreCount;
|
|
VkSemaphore local_semaphores[8];
|
|
uint64_t local_values[8];
|
|
VkSemaphore *semaphores = local_semaphores;
|
|
uint64_t *values = local_values;
|
|
if (semaphore_count > ARRAY_SIZE(local_semaphores)) {
|
|
semaphores = vk_alloc(
|
|
alloc, (sizeof(*semaphores) + sizeof(*values)) * semaphore_count,
|
|
VN_DEFAULT_ALIGN, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
|
|
if (!semaphores)
|
|
return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
|
|
|
|
values = (uint64_t *)&semaphores[semaphore_count];
|
|
}
|
|
memcpy(semaphores, pWaitInfo->pSemaphores,
|
|
sizeof(*semaphores) * semaphore_count);
|
|
memcpy(values, pWaitInfo->pValues, sizeof(*values) * semaphore_count);
|
|
|
|
struct vn_relax_state relax_state =
|
|
vn_relax_init(dev->instance, "client");
|
|
while (result == VK_NOT_READY) {
|
|
result = vn_remove_signaled_semaphores(device, semaphores, values,
|
|
&semaphore_count);
|
|
result =
|
|
vn_update_sync_result(dev, result, abs_timeout, &relax_state);
|
|
}
|
|
vn_relax_fini(&relax_state);
|
|
|
|
if (semaphores != local_semaphores)
|
|
vk_free(alloc, semaphores);
|
|
} else {
|
|
struct vn_relax_state relax_state =
|
|
vn_relax_init(dev->instance, "client");
|
|
while (result == VK_NOT_READY) {
|
|
result = vn_find_first_signaled_semaphore(
|
|
device, pWaitInfo->pSemaphores, pWaitInfo->pValues,
|
|
pWaitInfo->semaphoreCount);
|
|
result =
|
|
vn_update_sync_result(dev, result, abs_timeout, &relax_state);
|
|
}
|
|
vn_relax_fini(&relax_state);
|
|
}
|
|
|
|
return vn_result(dev->instance, result);
|
|
}
|
|
|
|
VkResult
|
|
vn_ImportSemaphoreFdKHR(
|
|
VkDevice device, const VkImportSemaphoreFdInfoKHR *pImportSemaphoreFdInfo)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
struct vn_semaphore *sem =
|
|
vn_semaphore_from_handle(pImportSemaphoreFdInfo->semaphore);
|
|
ASSERTED const bool sync_file =
|
|
pImportSemaphoreFdInfo->handleType ==
|
|
VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
|
|
const int fd = pImportSemaphoreFdInfo->fd;
|
|
|
|
assert(sync_file);
|
|
|
|
if (!vn_sync_valid_fd(fd))
|
|
return vn_error(dev->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
|
|
|
|
struct vn_sync_payload *temp = &sem->temporary;
|
|
vn_sync_payload_release(dev, temp);
|
|
temp->type = VN_SYNC_TYPE_IMPORTED_SYNC_FD;
|
|
temp->fd = fd;
|
|
sem->payload = temp;
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
VkResult
|
|
vn_GetSemaphoreFdKHR(VkDevice device,
|
|
const VkSemaphoreGetFdInfoKHR *pGetFdInfo,
|
|
int *pFd)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
struct vn_semaphore *sem = vn_semaphore_from_handle(pGetFdInfo->semaphore);
|
|
const bool sync_file =
|
|
pGetFdInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
|
|
struct vn_sync_payload *payload = sem->payload;
|
|
|
|
assert(sync_file);
|
|
assert(dev->physical_device->renderer_sync_fd.semaphore_exportable);
|
|
assert(dev->physical_device->renderer_sync_fd.semaphore_importable);
|
|
|
|
int fd = -1;
|
|
if (payload->type == VN_SYNC_TYPE_DEVICE_ONLY) {
|
|
VkResult result = vn_create_sync_file(dev, &sem->external_payload, &fd);
|
|
if (result != VK_SUCCESS)
|
|
return vn_error(dev->instance, result);
|
|
|
|
#ifdef VN_USE_WSI_PLATFORM
|
|
if (!dev->renderer->info.has_implicit_fencing)
|
|
sync_wait(fd, -1);
|
|
#endif
|
|
} else {
|
|
assert(payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD);
|
|
|
|
/* transfer ownership of imported sync fd to save a dup */
|
|
fd = payload->fd;
|
|
payload->fd = -1;
|
|
}
|
|
|
|
/* When payload->type is VN_SYNC_TYPE_IMPORTED_SYNC_FD, the current
|
|
* payload is from a prior temporary sync_fd import. The permanent
|
|
* payload of the sempahore might be in signaled state. So we do an
|
|
* import here to ensure later wait operation is legit. With resourceId
|
|
* 0, renderer does a signaled sync_fd -1 payload import on the host
|
|
* semaphore.
|
|
*/
|
|
if (payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD) {
|
|
const VkImportSemaphoreResourceInfoMESA res_info = {
|
|
.sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_RESOURCE_INFO_MESA,
|
|
.semaphore = pGetFdInfo->semaphore,
|
|
.resourceId = 0,
|
|
};
|
|
vn_async_vkImportSemaphoreResourceMESA(dev->primary_ring, device,
|
|
&res_info);
|
|
}
|
|
|
|
/* perform wait operation on the host semaphore */
|
|
vn_async_vkWaitSemaphoreResourceMESA(dev->primary_ring, device,
|
|
pGetFdInfo->semaphore);
|
|
|
|
vn_sync_payload_release(dev, &sem->temporary);
|
|
sem->payload = &sem->permanent;
|
|
|
|
*pFd = fd;
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
/* event commands */
|
|
|
|
static VkResult
|
|
vn_event_feedback_init(struct vn_device *dev, struct vn_event *ev)
|
|
{
|
|
struct vn_feedback_slot *slot;
|
|
|
|
if (VN_PERF(NO_EVENT_FEEDBACK))
|
|
return VK_SUCCESS;
|
|
|
|
slot = vn_feedback_pool_alloc(&dev->feedback_pool, VN_FEEDBACK_TYPE_EVENT);
|
|
if (!slot)
|
|
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
|
|
|
/* newly created event object is in the unsignaled state */
|
|
vn_feedback_set_status(slot, VK_EVENT_RESET);
|
|
|
|
ev->feedback_slot = slot;
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
static inline void
|
|
vn_event_feedback_fini(struct vn_device *dev, struct vn_event *ev)
|
|
{
|
|
if (ev->feedback_slot)
|
|
vn_feedback_pool_free(&dev->feedback_pool, ev->feedback_slot);
|
|
}
|
|
|
|
VkResult
|
|
vn_CreateEvent(VkDevice device,
|
|
const VkEventCreateInfo *pCreateInfo,
|
|
const VkAllocationCallbacks *pAllocator,
|
|
VkEvent *pEvent)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
const VkAllocationCallbacks *alloc =
|
|
pAllocator ? pAllocator : &dev->base.base.alloc;
|
|
|
|
struct vn_event *ev = vk_zalloc(alloc, sizeof(*ev), VN_DEFAULT_ALIGN,
|
|
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
|
|
if (!ev)
|
|
return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
|
|
|
|
vn_object_base_init(&ev->base, VK_OBJECT_TYPE_EVENT, &dev->base);
|
|
|
|
/* feedback is only needed to speed up host operations */
|
|
if (!(pCreateInfo->flags & VK_EVENT_CREATE_DEVICE_ONLY_BIT)) {
|
|
VkResult result = vn_event_feedback_init(dev, ev);
|
|
if (result != VK_SUCCESS)
|
|
return vn_error(dev->instance, result);
|
|
}
|
|
|
|
VkEvent ev_handle = vn_event_to_handle(ev);
|
|
vn_async_vkCreateEvent(dev->primary_ring, device, pCreateInfo, NULL,
|
|
&ev_handle);
|
|
|
|
*pEvent = ev_handle;
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
void
|
|
vn_DestroyEvent(VkDevice device,
|
|
VkEvent event,
|
|
const VkAllocationCallbacks *pAllocator)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
struct vn_event *ev = vn_event_from_handle(event);
|
|
const VkAllocationCallbacks *alloc =
|
|
pAllocator ? pAllocator : &dev->base.base.alloc;
|
|
|
|
if (!ev)
|
|
return;
|
|
|
|
vn_async_vkDestroyEvent(dev->primary_ring, device, event, NULL);
|
|
|
|
vn_event_feedback_fini(dev, ev);
|
|
|
|
vn_object_base_fini(&ev->base);
|
|
vk_free(alloc, ev);
|
|
}
|
|
|
|
VkResult
|
|
vn_GetEventStatus(VkDevice device, VkEvent event)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
struct vn_event *ev = vn_event_from_handle(event);
|
|
VkResult result;
|
|
|
|
if (ev->feedback_slot)
|
|
result = vn_feedback_get_status(ev->feedback_slot);
|
|
else
|
|
result = vn_call_vkGetEventStatus(dev->primary_ring, device, event);
|
|
|
|
return vn_result(dev->instance, result);
|
|
}
|
|
|
|
VkResult
|
|
vn_SetEvent(VkDevice device, VkEvent event)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
struct vn_event *ev = vn_event_from_handle(event);
|
|
|
|
if (ev->feedback_slot) {
|
|
vn_feedback_set_status(ev->feedback_slot, VK_EVENT_SET);
|
|
vn_async_vkSetEvent(dev->primary_ring, device, event);
|
|
} else {
|
|
VkResult result = vn_call_vkSetEvent(dev->primary_ring, device, event);
|
|
if (result != VK_SUCCESS)
|
|
return vn_error(dev->instance, result);
|
|
}
|
|
|
|
return VK_SUCCESS;
|
|
}
|
|
|
|
VkResult
|
|
vn_ResetEvent(VkDevice device, VkEvent event)
|
|
{
|
|
VN_TRACE_FUNC();
|
|
struct vn_device *dev = vn_device_from_handle(device);
|
|
struct vn_event *ev = vn_event_from_handle(event);
|
|
|
|
if (ev->feedback_slot) {
|
|
vn_feedback_reset_status(ev->feedback_slot);
|
|
vn_async_vkResetEvent(dev->primary_ring, device, event);
|
|
} else {
|
|
VkResult result =
|
|
vn_call_vkResetEvent(dev->primary_ring, device, event);
|
|
if (result != VK_SUCCESS)
|
|
return vn_error(dev->instance, result);
|
|
}
|
|
|
|
return VK_SUCCESS;
|
|
}
|