diff --git a/src/asahi/.clang-format b/src/asahi/.clang-format index ebdf1323eef..7fe240e0fec 100644 --- a/src/asahi/.clang-format +++ b/src/asahi/.clang-format @@ -70,7 +70,8 @@ ForEachMacros: - nir_foreach_variable_with_modes_safe - nir_foreach_variable_with_modes - nir_foreach_shader_out_variable - - foreach_batch + - foreach_active + - foreach_submitted - AGX_BATCH_FOREACH_BO_HANDLE - hash_table_foreach - set_foreach diff --git a/src/gallium/drivers/asahi/agx_batch.c b/src/gallium/drivers/asahi/agx_batch.c index ea22259ce3c..aabf0027a19 100644 --- a/src/gallium/drivers/asahi/agx_batch.c +++ b/src/gallium/drivers/asahi/agx_batch.c @@ -4,11 +4,16 @@ * SPDX-License-Identifier: MIT */ +#include +#include "asahi/lib/decode.h" #include "agx_state.h" -#define foreach_batch(ctx, idx) \ +#define foreach_active(ctx, idx) \ BITSET_FOREACH_SET(idx, ctx->batches.active, AGX_MAX_BATCHES) +#define foreach_submitted(ctx, idx) \ + BITSET_FOREACH_SET(idx, ctx->batches.submitted, AGX_MAX_BATCHES) + static unsigned agx_batch_idx(struct agx_batch *batch) { @@ -21,6 +26,43 @@ agx_batch_is_active(struct agx_batch *batch) return BITSET_TEST(batch->ctx->batches.active, agx_batch_idx(batch)); } +bool +agx_batch_is_submitted(struct agx_batch *batch) +{ + return BITSET_TEST(batch->ctx->batches.submitted, agx_batch_idx(batch)); +} + +static void +agx_batch_mark_active(struct agx_batch *batch) +{ + unsigned batch_idx = agx_batch_idx(batch); + + assert(!BITSET_TEST(batch->ctx->batches.submitted, batch_idx)); + assert(!BITSET_TEST(batch->ctx->batches.active, batch_idx)); + BITSET_SET(batch->ctx->batches.active, batch_idx); +} + +static void +agx_batch_mark_submitted(struct agx_batch *batch) +{ + unsigned batch_idx = agx_batch_idx(batch); + + assert(BITSET_TEST(batch->ctx->batches.active, batch_idx)); + assert(!BITSET_TEST(batch->ctx->batches.submitted, batch_idx)); + BITSET_CLEAR(batch->ctx->batches.active, batch_idx); + BITSET_SET(batch->ctx->batches.submitted, batch_idx); +} + +static void +agx_batch_mark_complete(struct agx_batch *batch) +{ + unsigned batch_idx = agx_batch_idx(batch); + + assert(!BITSET_TEST(batch->ctx->batches.active, batch_idx)); + assert(BITSET_TEST(batch->ctx->batches.submitted, batch_idx)); + BITSET_CLEAR(batch->ctx->batches.submitted, batch_idx); +} + static void agx_batch_init(struct agx_context *ctx, const struct pipe_framebuffer_state *key, @@ -74,11 +116,21 @@ agx_batch_init(struct agx_context *ctx, agx_batch_writes(batch, agx_resource(key->cbufs[i]->texture)); } - unsigned batch_idx = agx_batch_idx(batch); - BITSET_SET(ctx->batches.active, batch_idx); - if (key->width != AGX_COMPUTE_BATCH_WIDTH) agx_batch_init_state(batch); + + if (!batch->syncobj) { + int ret = drmSyncobjCreate(dev->fd, 0, &batch->syncobj); + assert(!ret && batch->syncobj); + } + + agx_batch_mark_active(batch); +} + +static void +agx_batch_print_stats(struct agx_device *dev, struct agx_batch *batch) +{ + unreachable("Linux UAPI not yet upstream"); } void @@ -86,9 +138,9 @@ agx_batch_cleanup(struct agx_context *ctx, struct agx_batch *batch) { struct agx_device *dev = agx_device(ctx->base.screen); assert(batch->ctx == ctx); + assert(agx_batch_is_submitted(batch)); - if (ctx->batch == batch) - ctx->batch = NULL; + assert(ctx->batch != batch); agx_finish_batch_occlusion_queries(batch); batch->occlusion_buffer.cpu = NULL; @@ -96,12 +148,16 @@ agx_batch_cleanup(struct agx_context *ctx, struct agx_batch *batch) int handle; AGX_BATCH_FOREACH_BO_HANDLE(batch, handle) { + struct agx_bo *bo = agx_lookup_bo(dev, handle); + /* There is no more writer on this context for anything we wrote */ struct agx_batch *writer = agx_writer_get(ctx, handle); - assert((writer == NULL || writer == batch) && - "cannot read while another batch writes"); - agx_writer_remove(ctx, handle); + if (writer == batch) { + bo->writer_syncobj = 0; + agx_writer_remove(ctx, handle); + } + agx_bo_unreference(agx_lookup_bo(dev, handle)); } @@ -114,8 +170,40 @@ agx_batch_cleanup(struct agx_context *ctx, struct agx_batch *batch) util_dynarray_fini(&batch->occlusion_queries); util_unreference_framebuffer_state(&batch->key); - unsigned batch_idx = agx_batch_idx(batch); - BITSET_CLEAR(ctx->batches.active, batch_idx); + agx_batch_mark_complete(batch); + + if (!(dev->debug & (AGX_DBG_TRACE | AGX_DBG_SYNC))) { + agx_batch_print_stats(dev, batch); + } +} + +int +agx_cleanup_batches(struct agx_context *ctx) +{ + struct agx_device *dev = agx_device(ctx->base.screen); + + unsigned i; + unsigned count = 0; + struct agx_batch *batches[AGX_MAX_BATCHES]; + uint32_t syncobjs[AGX_MAX_BATCHES]; + uint32_t first = 0; + + foreach_submitted(ctx, i) { + batches[count] = &ctx->batches.slots[i]; + syncobjs[count++] = ctx->batches.slots[i].syncobj; + } + + if (!count) + return -1; + + int ret = drmSyncobjWait(dev->fd, syncobjs, count, 0, 0, &first); + assert(!ret || ret == -ETIME); + if (ret) + return -1; + + assert(first < AGX_MAX_BATCHES); + agx_batch_cleanup(ctx, batches[first]); + return agx_batch_idx(batches[first]); } static struct agx_batch * @@ -124,7 +212,7 @@ agx_get_batch_for_framebuffer(struct agx_context *ctx, { /* Look if we have a matching batch */ unsigned i; - foreach_batch(ctx, i) { + foreach_active(ctx, i) { struct agx_batch *candidate = &ctx->batches.slots[i]; if (util_framebuffer_state_equal(&candidate->key, state)) { @@ -136,26 +224,43 @@ agx_get_batch_for_framebuffer(struct agx_context *ctx, } } - /* Look if we have a free batch */ - struct agx_batch *batch = NULL; - for (unsigned i = 0; i < AGX_MAX_BATCHES; ++i) { - if (!BITSET_TEST(ctx->batches.active, i)) { - batch = &ctx->batches.slots[i]; - break; + /* Look for a free batch */ + for (i = 0; i < AGX_MAX_BATCHES; ++i) { + if (!BITSET_TEST(ctx->batches.active, i) && + !BITSET_TEST(ctx->batches.submitted, i)) { + struct agx_batch *batch = &ctx->batches.slots[i]; + agx_batch_init(ctx, state, batch); + return batch; } } + /* Try to clean up one batch */ + int freed = agx_cleanup_batches(ctx); + if (freed >= 0) { + struct agx_batch *batch = &ctx->batches.slots[freed]; + agx_batch_init(ctx, state, batch); + return batch; + } + /* Else, evict something */ - if (!batch) { - for (unsigned i = 0; i < AGX_MAX_BATCHES; ++i) { - struct agx_batch *candidate = &ctx->batches.slots[i]; + struct agx_batch *batch = NULL; + bool submitted = false; + for (i = 0; i < AGX_MAX_BATCHES; ++i) { + struct agx_batch *candidate = &ctx->batches.slots[i]; + bool cand_submitted = BITSET_TEST(ctx->batches.submitted, i); - if (!batch || batch->seqnum > candidate->seqnum) - batch = candidate; + /* Prefer submitted batches first */ + if (!cand_submitted && submitted) + continue; + + if (!batch || batch->seqnum > candidate->seqnum) { + batch = candidate; + submitted = cand_submitted; } - - agx_flush_batch(ctx, batch); } + assert(batch); + + agx_sync_batch_for_reason(ctx, batch, "Too many batches"); /* Batch is now free */ agx_batch_init(ctx, state, batch); @@ -187,11 +292,11 @@ agx_get_compute_batch(struct agx_context *ctx) void agx_flush_all(struct agx_context *ctx, const char *reason) { - if (reason) - perf_debug_ctx(ctx, "Flushing due to: %s\n", reason); - unsigned idx; - foreach_batch(ctx, idx) { + foreach_active(ctx, idx) { + if (reason) + perf_debug_ctx(ctx, "Flushing due to: %s\n", reason); + agx_flush_batch(ctx, &ctx->batches.slots[idx]); } } @@ -203,16 +308,19 @@ agx_flush_batch_for_reason(struct agx_context *ctx, struct agx_batch *batch, if (reason) perf_debug_ctx(ctx, "Flushing due to: %s\n", reason); - agx_flush_batch(ctx, batch); + if (agx_batch_is_active(batch)) + agx_flush_batch(ctx, batch); } static void agx_flush_readers_except(struct agx_context *ctx, struct agx_resource *rsrc, - struct agx_batch *except, const char *reason) + struct agx_batch *except, const char *reason, + bool sync) { unsigned idx; - foreach_batch(ctx, idx) { + /* Flush everything to the hardware first */ + foreach_active(ctx, idx) { struct agx_batch *batch = &ctx->batches.slots[idx]; if (batch == except) @@ -223,17 +331,41 @@ agx_flush_readers_except(struct agx_context *ctx, struct agx_resource *rsrc, agx_flush_batch(ctx, batch); } } + + /* Then wait on everything if necessary */ + if (sync) { + foreach_submitted(ctx, idx) { + struct agx_batch *batch = &ctx->batches.slots[idx]; + + if (batch == except) + continue; + + if (agx_batch_uses_bo(batch, rsrc->bo)) { + perf_debug_ctx(ctx, "Sync reader due to: %s\n", reason); + agx_sync_batch(ctx, batch); + } + } + } } static void agx_flush_writer_except(struct agx_context *ctx, struct agx_resource *rsrc, - struct agx_batch *except, const char *reason) + struct agx_batch *except, const char *reason, bool sync) { struct agx_batch *writer = agx_writer_get(ctx, rsrc->bo->handle); - if (writer && writer != except) { - perf_debug_ctx(ctx, "Flush writer due to: %s\n", reason); - agx_flush_batch(ctx, writer); + if (writer && writer != except && + (agx_batch_is_active(writer) || agx_batch_is_submitted(writer))) { + if (agx_batch_is_active(writer) || sync) { + perf_debug_ctx(ctx, "%s writer due to: %s\n", sync ? "Sync" : "Flush", + reason); + } + if (agx_batch_is_active(writer)) + agx_flush_batch(ctx, writer); + /* Check for submitted state, because if the batch was a no-op it'll + * already be cleaned up */ + if (sync && agx_batch_is_submitted(writer)) + agx_sync_batch(ctx, writer); } } @@ -241,7 +373,14 @@ bool agx_any_batch_uses_resource(struct agx_context *ctx, struct agx_resource *rsrc) { unsigned idx; - foreach_batch(ctx, idx) { + foreach_active(ctx, idx) { + struct agx_batch *batch = &ctx->batches.slots[idx]; + + if (agx_batch_uses_bo(batch, rsrc->bo)) + return true; + } + + foreach_submitted(ctx, idx) { struct agx_batch *batch = &ctx->batches.slots[idx]; if (agx_batch_uses_bo(batch, rsrc->bo)) @@ -255,21 +394,36 @@ void agx_flush_readers(struct agx_context *ctx, struct agx_resource *rsrc, const char *reason) { - agx_flush_readers_except(ctx, rsrc, NULL, reason); + agx_flush_readers_except(ctx, rsrc, NULL, reason, false); +} + +void +agx_sync_readers(struct agx_context *ctx, struct agx_resource *rsrc, + const char *reason) +{ + agx_flush_readers_except(ctx, rsrc, NULL, reason, true); } void agx_flush_writer(struct agx_context *ctx, struct agx_resource *rsrc, const char *reason) { - agx_flush_writer_except(ctx, rsrc, NULL, reason); + agx_flush_writer_except(ctx, rsrc, NULL, reason, false); +} + +void +agx_sync_writer(struct agx_context *ctx, struct agx_resource *rsrc, + const char *reason) +{ + agx_flush_writer_except(ctx, rsrc, NULL, reason, true); } void agx_batch_reads(struct agx_batch *batch, struct agx_resource *rsrc) { /* Hazard: read-after-write */ - agx_flush_writer_except(batch->ctx, rsrc, batch, "Read from another batch"); + agx_flush_writer_except(batch->ctx, rsrc, batch, "Read from another batch", + false); agx_batch_add_bo(batch, rsrc->bo); @@ -283,7 +437,7 @@ agx_batch_writes(struct agx_batch *batch, struct agx_resource *rsrc) struct agx_context *ctx = batch->ctx; struct agx_batch *writer = agx_writer_get(ctx, rsrc->bo->handle); - agx_flush_readers_except(ctx, rsrc, batch, "Write from other batch"); + agx_flush_readers_except(ctx, rsrc, batch, "Write from other batch", false); /* Nothing to do if we're already writing */ if (writer == batch) @@ -296,7 +450,13 @@ agx_batch_writes(struct agx_batch *batch, struct agx_resource *rsrc) /* Write is strictly stronger than a read */ agx_batch_reads(batch, rsrc); - /* We are now the new writer */ + writer = agx_writer_get(ctx, rsrc->bo->handle); + assert(!writer || agx_batch_is_submitted(writer)); + + /* We are now the new writer. Disregard the previous writer -- anything that + * needs to wait for the writer going forward needs to wait for us. + */ + agx_writer_remove(ctx, rsrc->bo->handle); agx_writer_add(ctx, agx_batch_idx(batch), rsrc->bo->handle); } @@ -314,10 +474,245 @@ void agx_flush_occlusion_queries(struct agx_context *ctx) { unsigned i; - foreach_batch(ctx, i) { + foreach_active(ctx, i) { struct agx_batch *other = &ctx->batches.slots[i]; if (other->occlusion_queries.size != 0) agx_flush_batch_for_reason(ctx, other, "Occlusion query ordering"); } + + foreach_submitted(ctx, i) { + struct agx_batch *other = &ctx->batches.slots[i]; + + if (other->occlusion_queries.size != 0) + agx_sync_batch_for_reason(ctx, other, "Occlusion query ordering"); + } +} + +static int +agx_get_in_sync(struct agx_context *ctx) +{ + struct agx_device *dev = agx_device(ctx->base.screen); + + if (ctx->in_sync_fd >= 0) { + int ret = + drmSyncobjImportSyncFile(dev->fd, ctx->in_sync_obj, ctx->in_sync_fd); + assert(!ret); + + close(ctx->in_sync_fd); + ctx->in_sync_fd = -1; + + return ctx->in_sync_obj; + } else { + return 0; + } +} + +static void +agx_add_sync(struct drm_asahi_sync *syncs, unsigned *count, uint32_t handle) +{ + if (!handle) + return; + + syncs[(*count)++] = (struct drm_asahi_sync){ + .sync_type = DRM_ASAHI_SYNC_SYNCOBJ, + .handle = handle, + }; +} + +void +agx_batch_submit(struct agx_context *ctx, struct agx_batch *batch, + uint32_t barriers, enum drm_asahi_cmd_type cmd_type, + void *cmdbuf) +{ + struct agx_device *dev = agx_device(ctx->base.screen); + + bool feedback = dev->debug & (AGX_DBG_TRACE | AGX_DBG_SYNC | AGX_DBG_STATS); + +#ifndef NDEBUG + /* Debug builds always get feedback (for fault checks) */ + feedback = true; +#endif + + if (!feedback) + batch->result = NULL; + + /* We allocate the worst-case sync array size since this won't be excessive + * for most workloads + */ + unsigned max_syncs = agx_batch_bo_list_bits(batch) + 1; + unsigned in_sync_count = 0; + unsigned shared_bo_count = 0; + struct drm_asahi_sync *in_syncs = + malloc(max_syncs * sizeof(struct drm_asahi_sync)); + struct agx_bo **shared_bos = malloc(max_syncs * sizeof(struct agx_bo *)); + + struct drm_asahi_sync out_sync = { + .sync_type = DRM_ASAHI_SYNC_SYNCOBJ, + .handle = batch->syncobj, + }; + + int handle; + AGX_BATCH_FOREACH_BO_HANDLE(batch, handle) { + struct agx_bo *bo = agx_lookup_bo(dev, handle); + + if (bo->flags & AGX_BO_SHARED) { + /* Get a sync file fd from the buffer */ + int in_sync_fd = agx_export_sync_file(dev, bo); + assert(in_sync_fd >= 0); + + /* Create a new syncobj */ + uint32_t sync_handle; + int ret = drmSyncobjCreate(dev->fd, 0, &sync_handle); + assert(ret >= 0); + + /* Import the sync file into it */ + ret = drmSyncobjImportSyncFile(dev->fd, sync_handle, in_sync_fd); + assert(ret >= 0); + assert(sync_handle); + close(in_sync_fd); + + /* Add it to our wait list */ + agx_add_sync(in_syncs, &in_sync_count, sync_handle); + + /* And keep track of the BO for cloning the out_sync */ + shared_bos[shared_bo_count++] = bo; + } + } + + /* Add an explicit fence from gallium, if any */ + agx_add_sync(in_syncs, &in_sync_count, agx_get_in_sync(ctx)); + + /* Submit! */ + agx_submit_single( + dev, cmd_type, barriers, in_syncs, in_sync_count, &out_sync, 1, cmdbuf, + feedback ? ctx->result_buf->handle : 0, feedback ? batch->result_off : 0, + feedback ? sizeof(union agx_batch_result) : 0); + + /* Now stash our batch fence into any shared BOs. */ + if (shared_bo_count) { + /* Convert our handle to a sync file */ + int out_sync_fd = -1; + int ret = drmSyncobjExportSyncFile(dev->fd, batch->syncobj, &out_sync_fd); + assert(ret >= 0); + assert(out_sync_fd >= 0); + + for (unsigned i = 0; i < shared_bo_count; i++) { + /* Free the in_sync handle we just acquired */ + ret = drmSyncobjDestroy(dev->fd, in_syncs[i].handle); + assert(ret >= 0); + /* And then import the out_sync sync file into it */ + ret = agx_import_sync_file(dev, shared_bos[i], out_sync_fd); + assert(ret >= 0); + } + + close(out_sync_fd); + } + + /* Record the syncobj on each BO we write, so it can be added post-facto as a + * fence if the BO is exported later... + */ + AGX_BATCH_FOREACH_BO_HANDLE(batch, handle) { + struct agx_bo *bo = agx_lookup_bo(dev, handle); + struct agx_batch *writer = agx_writer_get(ctx, handle); + + if (!writer) + continue; + + /* Skip BOs that are written by submitted batches, they're not ours */ + if (agx_batch_is_submitted(writer)) + continue; + + /* But any BOs written by active batches are ours */ + assert(writer == batch && "exclusive writer"); + bo->writer_syncobj = batch->syncobj; + } + + free(in_syncs); + free(shared_bos); + + if (dev->debug & (AGX_DBG_TRACE | AGX_DBG_SYNC)) { + /* Wait so we can get errors reported back */ + int ret = drmSyncobjWait(dev->fd, &batch->syncobj, 1, INT64_MAX, 0, NULL); + assert(!ret); + + if (dev->debug & AGX_DBG_TRACE) { + /* agxdecode DRM commands */ + switch (cmd_type) { + default: + unreachable("Linux UAPI not yet upstream"); + } + agxdecode_next_frame(); + } + + agx_batch_print_stats(dev, batch); + } + + agx_batch_mark_submitted(batch); + + /* Record the last syncobj for fence creation */ + ctx->syncobj = batch->syncobj; + + if (ctx->batch == batch) + ctx->batch = NULL; + + /* Try to clean up up to two batches, to keep memory usage down */ + if (agx_cleanup_batches(ctx) >= 0) + agx_cleanup_batches(ctx); +} + +void +agx_sync_batch(struct agx_context *ctx, struct agx_batch *batch) +{ + struct agx_device *dev = agx_device(ctx->base.screen); + + if (agx_batch_is_active(batch)) + agx_flush_batch(ctx, batch); + + /* Empty batch case, already cleaned up */ + if (!agx_batch_is_submitted(batch)) + return; + + assert(batch->syncobj); + int ret = drmSyncobjWait(dev->fd, &batch->syncobj, 1, INT64_MAX, 0, NULL); + assert(!ret); + agx_batch_cleanup(ctx, batch); +} + +void +agx_sync_batch_for_reason(struct agx_context *ctx, struct agx_batch *batch, + const char *reason) +{ + if (reason) + perf_debug_ctx(ctx, "Syncing due to: %s\n", reason); + + agx_sync_batch(ctx, batch); +} + +void +agx_sync_all(struct agx_context *ctx, const char *reason) +{ + if (reason) + perf_debug_ctx(ctx, "Syncing all due to: %s\n", reason); + + unsigned idx; + foreach_active(ctx, idx) { + agx_flush_batch(ctx, &ctx->batches.slots[idx]); + } + + foreach_submitted(ctx, idx) { + agx_sync_batch(ctx, &ctx->batches.slots[idx]); + } +} + +void +agx_batch_reset(struct agx_context *ctx, struct agx_batch *batch) +{ + /* Reset an empty batch. Like submit, but does nothing. */ + agx_batch_mark_submitted(batch); + + if (ctx->batch == batch) + ctx->batch = NULL; + + agx_batch_cleanup(ctx, batch); } diff --git a/src/gallium/drivers/asahi/agx_fence.c b/src/gallium/drivers/asahi/agx_fence.c new file mode 100644 index 00000000000..4ca2cc89075 --- /dev/null +++ b/src/gallium/drivers/asahi/agx_fence.c @@ -0,0 +1,156 @@ +/* + * Based on panfrost/pan_fence.c: + * + * Copyright (c) 2022 Amazon.com, Inc. or its affiliates. + * Copyright (C) 2008 VMware, Inc. + * Copyright (C) 2014 Broadcom + * Copyright (C) 2018 Alyssa Rosenzweig + * Copyright (C) 2019 Collabora, Ltd. + * Copyright (C) 2012 Rob Clark + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include + +#include "agx_fence.h" +#include "agx_state.h" + +#include "util/os_time.h" +#include "util/u_inlines.h" + +void +agx_fence_reference(struct pipe_screen *pscreen, struct pipe_fence_handle **ptr, + struct pipe_fence_handle *fence) +{ + struct agx_device *dev = agx_device(pscreen); + struct pipe_fence_handle *old = *ptr; + + if (pipe_reference(old ? &old->reference : NULL, + fence ? &fence->reference : NULL)) { + drmSyncobjDestroy(dev->fd, old->syncobj); + free(old); + } + + *ptr = fence; +} + +bool +agx_fence_finish(struct pipe_screen *pscreen, struct pipe_context *ctx, + struct pipe_fence_handle *fence, uint64_t timeout) +{ + struct agx_device *dev = agx_device(pscreen); + int ret; + + if (fence->signaled) + return true; + + uint64_t abs_timeout = os_time_get_absolute_timeout(timeout); + if (abs_timeout == OS_TIMEOUT_INFINITE) + abs_timeout = INT64_MAX; + + ret = drmSyncobjWait(dev->fd, &fence->syncobj, 1, abs_timeout, + DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL, NULL); + + assert(ret >= 0 || ret == -ETIME); + fence->signaled = (ret >= 0); + return fence->signaled; +} + +int +agx_fence_get_fd(struct pipe_screen *screen, struct pipe_fence_handle *f) +{ + struct agx_device *dev = agx_device(screen); + int fd = -1; + + int ret = drmSyncobjExportSyncFile(dev->fd, f->syncobj, &fd); + assert(ret >= 0); + assert(fd >= 0); + + return fd; +} + +struct pipe_fence_handle * +agx_fence_from_fd(struct agx_context *ctx, int fd, enum pipe_fd_type type) +{ + struct agx_device *dev = agx_device(ctx->base.screen); + int ret; + + struct pipe_fence_handle *f = calloc(1, sizeof(*f)); + if (!f) + return NULL; + + if (type == PIPE_FD_TYPE_NATIVE_SYNC) { + ret = drmSyncobjCreate(dev->fd, 0, &f->syncobj); + if (ret) { + fprintf(stderr, "create syncobj failed\n"); + goto err_free_fence; + } + + ret = drmSyncobjImportSyncFile(dev->fd, f->syncobj, fd); + if (ret) { + fprintf(stderr, "import syncfile failed\n"); + goto err_destroy_syncobj; + } + } else { + assert(type == PIPE_FD_TYPE_SYNCOBJ); + ret = drmSyncobjFDToHandle(dev->fd, fd, &f->syncobj); + if (ret) { + fprintf(stderr, "import syncobj FD failed\n"); + goto err_free_fence; + } + } + + pipe_reference_init(&f->reference, 1); + + return f; + +err_destroy_syncobj: + drmSyncobjDestroy(dev->fd, f->syncobj); +err_free_fence: + free(f); + return NULL; +} + +struct pipe_fence_handle * +agx_fence_create(struct agx_context *ctx) +{ + struct agx_device *dev = agx_device(ctx->base.screen); + int fd = -1, ret; + + /* Snapshot the last rendering out fence. We'd rather have another + * syncobj instead of a sync file, but this is all we get. + * (HandleToFD/FDToHandle just gives you another syncobj ID for the + * same syncobj). + */ + ret = drmSyncobjExportSyncFile(dev->fd, ctx->syncobj, &fd); + assert(ret >= 0 && fd != -1 && "export failed"); + if (ret || fd == -1) { + fprintf(stderr, "export failed\n"); + return NULL; + } + + struct pipe_fence_handle *f = + agx_fence_from_fd(ctx, fd, PIPE_FD_TYPE_NATIVE_SYNC); + + close(fd); + + return f; +} diff --git a/src/gallium/drivers/asahi/agx_fence.h b/src/gallium/drivers/asahi/agx_fence.h new file mode 100644 index 00000000000..cf92eca26a9 --- /dev/null +++ b/src/gallium/drivers/asahi/agx_fence.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2022 Amazon.com, Inc. or its affiliates. + * Copyright 2018-2019 Alyssa Rosenzweig + * Copyright 2018-2019 Collabora, Ltd. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. + * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR + * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef AGX_FENCE_H +#define AGX_FENCE_H + +#include "pipe/p_state.h" + +struct agx_context; + +struct pipe_fence_handle { + struct pipe_reference reference; + uint32_t syncobj; + bool signaled; +}; + +void agx_fence_reference(struct pipe_screen *pscreen, + struct pipe_fence_handle **ptr, + struct pipe_fence_handle *fence); + +bool agx_fence_finish(struct pipe_screen *pscreen, struct pipe_context *ctx, + struct pipe_fence_handle *fence, uint64_t timeout); + +int agx_fence_get_fd(struct pipe_screen *screen, struct pipe_fence_handle *f); + +struct pipe_fence_handle *agx_fence_from_fd(struct agx_context *ctx, int fd, + enum pipe_fd_type type); + +struct pipe_fence_handle *agx_fence_create(struct agx_context *ctx); + +#endif diff --git a/src/gallium/drivers/asahi/agx_pipe.c b/src/gallium/drivers/asahi/agx_pipe.c index 5556c69efde..7fb5a8a940b 100644 --- a/src/gallium/drivers/asahi/agx_pipe.c +++ b/src/gallium/drivers/asahi/agx_pipe.c @@ -25,6 +25,7 @@ */ #include #include +#include #include "asahi/compiler/agx_compile.h" #include "asahi/layout/layout.h" #include "asahi/lib/agx_formats.h" @@ -51,6 +52,7 @@ #include "util/u_upload_mgr.h" #include "agx_device.h" #include "agx_disk_cache.h" +#include "agx_fence.h" #include "agx_public.h" #include "agx_state.h" @@ -620,7 +622,7 @@ agx_shadow(struct agx_context *ctx, struct agx_resource *rsrc) /* * Perform the required synchronization before a transfer_map operation can - * complete. This may require flushing batches. + * complete. This may require syncing batches. */ static void agx_prepare_for_map(struct agx_context *ctx, struct agx_resource *rsrc, @@ -648,10 +650,10 @@ agx_prepare_for_map(struct agx_context *ctx, struct agx_resource *rsrc, if (usage & PIPE_MAP_UNSYNCHRONIZED) return; - /* Both writing and reading need writers flushed */ - agx_flush_writer(ctx, rsrc, "Unsynchronized transfer"); + /* Both writing and reading need writers synced */ + agx_sync_writer(ctx, rsrc, "Unsynchronized transfer"); - /* Additionally, writing needs readers flushed */ + /* Additionally, writing needs readers synced */ if (!(usage & PIPE_MAP_WRITE)) return; @@ -661,12 +663,12 @@ agx_prepare_for_map(struct agx_context *ctx, struct agx_resource *rsrc, if (!agx_any_batch_uses_resource(ctx, rsrc)) return; - /* There are readers. Try to shadow the resource to avoid a flush */ + /* There are readers. Try to shadow the resource to avoid a sync */ if ((usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) && agx_shadow(ctx, rsrc)) return; - /* Otherwise, we need to flush */ - agx_flush_readers(ctx, rsrc, "Unsynchronized write"); + /* Otherwise, we need to sync */ + agx_sync_readers(ctx, rsrc, "Unsynchronized write"); } /* Most of the time we can do CPU-side transfers, but sometimes we need to use @@ -798,7 +800,7 @@ agx_transfer_map(struct pipe_context *pctx, struct pipe_resource *resource, if ((usage & PIPE_MAP_READ) && agx_resource_valid(rsrc, level)) { agx_blit_to_staging(pctx, transfer); - agx_flush_writer(ctx, staging, "GPU read staging blit"); + agx_sync_writer(ctx, staging, "GPU read staging blit"); } agx_bo_mmap(staging->bo); @@ -997,7 +999,6 @@ agx_flush_resource(struct pipe_context *pctx, struct pipe_resource *pres) pres->bind |= PIPE_BIND_SHARED; agx_flush_writer(agx_context(pctx), rsrc, "flush_resource"); } - } /* @@ -1009,10 +1010,24 @@ agx_flush(struct pipe_context *pctx, struct pipe_fence_handle **fence, { struct agx_context *ctx = agx_context(pctx); - if (fence) - *fence = NULL; - agx_flush_all(ctx, "Gallium flush"); + + /* At this point all pending work has been submitted. Since jobs are + * started and completed sequentially from a UAPI perspective, and since + * we submit all jobs with compute+render barriers on the prior job, + * waiting on the last submitted job is sufficient to guarantee completion + * of all GPU work thus far, so we can create a fence out of the latest + * syncobj. + * + * See this page for more info on how the GPU/UAPI queueing works: + * https://github.com/AsahiLinux/docs/wiki/SW:AGX-driver-notes#queues + */ + + if (fence) { + struct pipe_fence_handle *f = agx_fence_create(ctx); + pctx->screen->fence_reference(pctx->screen, fence, NULL); + *fence = f; + } } void @@ -1021,10 +1036,11 @@ agx_flush_batch(struct agx_context *ctx, struct agx_batch *batch) struct agx_device *dev = agx_device(ctx->base.screen); assert(agx_batch_is_active(batch)); + assert(!agx_batch_is_submitted(batch)); /* Make sure there's something to submit. */ if (!batch->clear && !batch->any_draws) { - agx_batch_cleanup(ctx, batch); + agx_batch_reset(ctx, batch); return; } @@ -1120,14 +1136,23 @@ agx_flush_batch(struct agx_context *ctx, struct agx_batch *batch) (void)pipeline_background; (void)pipeline_background_partial; - agx_batch_cleanup(ctx, batch); + unreachable("Linux UAPI not yet upstream"); + agx_batch_submit(ctx, batch, 0, 0, NULL); } static void agx_destroy_context(struct pipe_context *pctx) { + struct agx_device *dev = agx_device(pctx->screen); struct agx_context *ctx = agx_context(pctx); + /* Batch state needs to be freed on completion, and we don't want to yank + * buffers out from in-progress GPU jobs to avoid faults, so just wait until + * everything in progress is actually done on context destroy. This will + * ensure everything is cleaned up properly. + */ + agx_sync_all(ctx, "destroy context"); + if (pctx->stream_uploader) u_upload_destroy(pctx->stream_uploader); @@ -1140,6 +1165,16 @@ agx_destroy_context(struct pipe_context *pctx) agx_bo_unreference(ctx->result_buf); + drmSyncobjDestroy(dev->fd, ctx->in_sync_obj); + drmSyncobjDestroy(dev->fd, ctx->dummy_syncobj); + if (ctx->in_sync_fd != -1) + close(ctx->in_sync_fd); + + for (unsigned i = 0; i < AGX_MAX_BATCHES; ++i) { + if (ctx->batches.slots[i].syncobj) + drmSyncobjDestroy(dev->fd, ctx->batches.slots[i].syncobj); + } + ralloc_free(ctx); } @@ -1167,6 +1202,7 @@ agx_create_context(struct pipe_screen *screen, void *priv, unsigned flags) { struct agx_context *ctx = rzalloc(NULL, struct agx_context); struct pipe_context *pctx = &ctx->base; + int ret; if (!ctx) return NULL; @@ -1209,10 +1245,21 @@ agx_create_context(struct pipe_screen *screen, void *priv, unsigned flags) ctx->blitter = util_blitter_create(pctx); ctx->result_buf = agx_bo_create( - agx_device(screen), sizeof(union agx_batch_result) * AGX_MAX_BATCHES, 0, - "Batch result buffer"); + agx_device(screen), sizeof(union agx_batch_result) * AGX_MAX_BATCHES, + AGX_BO_WRITEBACK, "Batch result buffer"); assert(ctx->result_buf); + /* Sync object/FD used for NATIVE_FENCE_FD. */ + ctx->in_sync_fd = -1; + ret = drmSyncobjCreate(agx_device(screen)->fd, 0, &ctx->in_sync_obj); + assert(!ret); + + /* Dummy sync object used before any work has been submitted. */ + ret = drmSyncobjCreate(agx_device(screen)->fd, DRM_SYNCOBJ_CREATE_SIGNALED, + &ctx->dummy_syncobj); + assert(!ret); + ctx->syncobj = ctx->dummy_syncobj; + return pctx; } @@ -1309,6 +1356,7 @@ agx_get_param(struct pipe_screen *pscreen, enum pipe_cap param) case PIPE_CAP_PRIMITIVE_RESTART: case PIPE_CAP_PRIMITIVE_RESTART_FIXED_INDEX: case PIPE_CAP_ANISOTROPIC_FILTER: + case PIPE_CAP_NATIVE_FENCE_FD: return true; case PIPE_CAP_SAMPLER_VIEW_TARGET: @@ -1783,19 +1831,6 @@ agx_destroy_screen(struct pipe_screen *pscreen) ralloc_free(screen); } -static void -agx_fence_reference(struct pipe_screen *screen, struct pipe_fence_handle **ptr, - struct pipe_fence_handle *fence) -{ -} - -static bool -agx_fence_finish(struct pipe_screen *screen, struct pipe_context *ctx, - struct pipe_fence_handle *fence, uint64_t timeout) -{ - return true; -} - static const void * agx_get_compiler_options(struct pipe_screen *pscreen, enum pipe_shader_ir ir, enum pipe_shader_type shader) @@ -1905,6 +1940,7 @@ agx_screen_create(int fd, struct renderonly *ro, struct sw_winsys *winsys) screen->get_timestamp = u_default_get_timestamp; screen->fence_reference = agx_fence_reference; screen->fence_finish = agx_fence_finish; + screen->fence_get_fd = agx_fence_get_fd; screen->get_compiler_options = agx_get_compiler_options; screen->get_disk_shader_cache = agx_get_disk_shader_cache; diff --git a/src/gallium/drivers/asahi/agx_state.h b/src/gallium/drivers/asahi/agx_state.h index e4df175e41a..5b4ec9b2412 100644 --- a/src/gallium/drivers/asahi/agx_state.h +++ b/src/gallium/drivers/asahi/agx_state.h @@ -179,6 +179,7 @@ struct agx_batch { struct agx_context *ctx; struct pipe_framebuffer_state key; uint64_t seqnum; + uint32_t syncobj; struct agx_tilebuffer_layout tilebuffer_layout; @@ -283,7 +284,11 @@ enum agx_dirty { AGX_DIRTY_QUERY = BITFIELD_BIT(13), }; -#define AGX_MAX_BATCHES (2) +/* Maximum number of in-progress + under-construction GPU batches. + * Must be large enough for silly workloads that do things like + * glGenerateMipmap on every frame, otherwise we end up losing performance. + */ +#define AGX_MAX_BATCHES (128) struct agx_context { struct pipe_context base; @@ -299,6 +304,9 @@ struct agx_context { /** Set of active batches for faster traversal */ BITSET_DECLARE(active, AGX_MAX_BATCHES); + + /** Set of submitted batches for faster traversal */ + BITSET_DECLARE(submitted, AGX_MAX_BATCHES); } batches; struct agx_batch *batch; @@ -346,6 +354,11 @@ struct agx_context { struct util_dynarray writer; struct agx_meta_cache meta; + + uint32_t syncobj; + uint32_t dummy_syncobj; + int in_sync_fd; + uint32_t in_sync_obj; }; static void @@ -573,6 +586,7 @@ bool agx_nir_lower_sysvals(nir_shader *shader, unsigned *push_size); bool agx_batch_is_active(struct agx_batch *batch); +bool agx_batch_is_submitted(struct agx_batch *batch); uint64_t agx_batch_upload_pbe(struct agx_batch *batch, unsigned rt); @@ -624,6 +638,10 @@ agx_batch_num_bo(struct agx_batch *batch) BITSET_FOREACH_SET(handle, (batch)->bo_list.set, \ agx_batch_bo_list_bits(batch)) +void agx_batch_submit(struct agx_context *ctx, struct agx_batch *batch, + uint32_t barriers, enum drm_asahi_cmd_type cmd_type, + void *cmdbuf); + void agx_flush_batch(struct agx_context *ctx, struct agx_batch *batch); void agx_flush_batch_for_reason(struct agx_context *ctx, struct agx_batch *batch, const char *reason); @@ -635,6 +653,15 @@ void agx_flush_writer(struct agx_context *ctx, struct agx_resource *rsrc, void agx_flush_batches_writing_occlusion_queries(struct agx_context *ctx); void agx_flush_occlusion_queries(struct agx_context *ctx); +void agx_sync_writer(struct agx_context *ctx, struct agx_resource *rsrc, + const char *reason); +void agx_sync_readers(struct agx_context *ctx, struct agx_resource *rsrc, + const char *reason); +void agx_sync_batch(struct agx_context *ctx, struct agx_batch *batch); +void agx_sync_all(struct agx_context *ctx, const char *reason); +void agx_sync_batch_for_reason(struct agx_context *ctx, struct agx_batch *batch, + const char *reason); + /* Use these instead of batch_add_bo for proper resource tracking */ void agx_batch_reads(struct agx_batch *batch, struct agx_resource *rsrc); void agx_batch_writes(struct agx_batch *batch, struct agx_resource *rsrc); @@ -651,7 +678,9 @@ bool agx_any_batch_uses_resource(struct agx_context *ctx, struct agx_batch *agx_get_batch(struct agx_context *ctx); struct agx_batch *agx_get_compute_batch(struct agx_context *ctx); +void agx_batch_reset(struct agx_context *ctx, struct agx_batch *batch); void agx_batch_cleanup(struct agx_context *ctx, struct agx_batch *batch); +int agx_cleanup_batches(struct agx_context *ctx); /* Blit shaders */ void agx_blitter_save(struct agx_context *ctx, struct blitter_context *blitter, diff --git a/src/gallium/drivers/asahi/meson.build b/src/gallium/drivers/asahi/meson.build index 65a81614464..696f1828db4 100644 --- a/src/gallium/drivers/asahi/meson.build +++ b/src/gallium/drivers/asahi/meson.build @@ -22,6 +22,7 @@ files_asahi = files( 'agx_batch.c', 'agx_blit.c', 'agx_disk_cache.c', + 'agx_fence.c', 'agx_pipe.c', 'agx_nir_lower_sysvals.c', 'agx_query.c', @@ -35,7 +36,7 @@ libasahi = static_library( include_directories : [inc_gallium_aux, inc_gallium, inc_include, inc_src], c_args : [c_msvc_compat_args], gnu_symbol_visibility : 'hidden', - dependencies : [idep_nir, idep_mesautil, idep_agx_pack], + dependencies : [idep_nir, idep_mesautil, idep_agx_pack, dep_libdrm], ) driver_asahi = declare_dependency(