AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

asahi: Extend batch tracking for explicit sync

Browse Source 
Now that we have stub sync support in the submission API, we can
implement the batch tracking changes required to support an explicit
sync world. This excludes the UAPI-specific bits (command decoding and
status parsing).

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Signed-off-by: Asahi Lina <lina@asahilina.net>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/21620>
This commit is contained in:
Asahi Lina
2023-03-01 18:23:10 +09:00
committed by Marge Bot
parent c41f10eb9e
commit 04387269dd
7 changed files with 748 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/asahi/.clang-format
+2 -1
View File
@@ -70,7 +70,8 @@ ForEachMacros:
- nir_foreach_variable_with_modes_safe
- nir_foreach_variable_with_modes
- nir_foreach_shader_out_variable
- foreach_batch
- foreach_active
- foreach_submitted
- AGX_BATCH_FOREACH_BO_HANDLE
- hash_table_foreach
- set_foreach
src/gallium/drivers/asahi/agx_batch.c
+438 -43
View File
@@ -4,11 +4,16 @@
* SPDX-License-Identifier: MIT
*/
#include <xf86drm.h>
#include "asahi/lib/decode.h"
#include "agx_state.h"
#define foreach_batch(ctx, idx) \
#define foreach_active(ctx, idx) \
BITSET_FOREACH_SET(idx, ctx->batches.active, AGX_MAX_BATCHES)
#define foreach_submitted(ctx, idx) \
BITSET_FOREACH_SET(idx, ctx->batches.submitted, AGX_MAX_BATCHES)
static unsigned
agx_batch_idx(struct agx_batch *batch)
{
@@ -21,6 +26,43 @@ agx_batch_is_active(struct agx_batch *batch)
return BITSET_TEST(batch->ctx->batches.active, agx_batch_idx(batch));
}
bool
agx_batch_is_submitted(struct agx_batch *batch)
{
return BITSET_TEST(batch->ctx->batches.submitted, agx_batch_idx(batch));
}
static void
agx_batch_mark_active(struct agx_batch *batch)
{
unsigned batch_idx = agx_batch_idx(batch);
assert(!BITSET_TEST(batch->ctx->batches.submitted, batch_idx));
assert(!BITSET_TEST(batch->ctx->batches.active, batch_idx));
BITSET_SET(batch->ctx->batches.active, batch_idx);
}
static void
agx_batch_mark_submitted(struct agx_batch *batch)
{
unsigned batch_idx = agx_batch_idx(batch);
assert(BITSET_TEST(batch->ctx->batches.active, batch_idx));
assert(!BITSET_TEST(batch->ctx->batches.submitted, batch_idx));
BITSET_CLEAR(batch->ctx->batches.active, batch_idx);
BITSET_SET(batch->ctx->batches.submitted, batch_idx);
}
static void
agx_batch_mark_complete(struct agx_batch *batch)
{
unsigned batch_idx = agx_batch_idx(batch);
assert(!BITSET_TEST(batch->ctx->batches.active, batch_idx));
assert(BITSET_TEST(batch->ctx->batches.submitted, batch_idx));
BITSET_CLEAR(batch->ctx->batches.submitted, batch_idx);
}
static void
agx_batch_init(struct agx_context *ctx,
const struct pipe_framebuffer_state *key,
@@ -74,11 +116,21 @@ agx_batch_init(struct agx_context *ctx,
agx_batch_writes(batch, agx_resource(key->cbufs[i]->texture));
}
unsigned batch_idx = agx_batch_idx(batch);
BITSET_SET(ctx->batches.active, batch_idx);
if (key->width != AGX_COMPUTE_BATCH_WIDTH)
agx_batch_init_state(batch);
if (!batch->syncobj) {
int ret = drmSyncobjCreate(dev->fd, 0, &batch->syncobj);
assert(!ret && batch->syncobj);
}
agx_batch_mark_active(batch);
}
static void
agx_batch_print_stats(struct agx_device *dev, struct agx_batch *batch)
{
unreachable("Linux UAPI not yet upstream");
}
void
@@ -86,9 +138,9 @@ agx_batch_cleanup(struct agx_context *ctx, struct agx_batch *batch)
{
struct agx_device *dev = agx_device(ctx->base.screen);
assert(batch->ctx == ctx);
assert(agx_batch_is_submitted(batch));
if (ctx->batch == batch)
ctx->batch = NULL;
assert(ctx->batch != batch);
agx_finish_batch_occlusion_queries(batch);
batch->occlusion_buffer.cpu = NULL;
@@ -96,12 +148,16 @@ agx_batch_cleanup(struct agx_context *ctx, struct agx_batch *batch)
int handle;
AGX_BATCH_FOREACH_BO_HANDLE(batch, handle) {
struct agx_bo *bo = agx_lookup_bo(dev, handle);
/* There is no more writer on this context for anything we wrote */
struct agx_batch *writer = agx_writer_get(ctx, handle);
assert((writer == NULL || writer == batch) &&
"cannot read while another batch writes");
agx_writer_remove(ctx, handle);
if (writer == batch) {
bo->writer_syncobj = 0;
agx_writer_remove(ctx, handle);
}
agx_bo_unreference(agx_lookup_bo(dev, handle));
}
@@ -114,8 +170,40 @@ agx_batch_cleanup(struct agx_context *ctx, struct agx_batch *batch)
util_dynarray_fini(&batch->occlusion_queries);
util_unreference_framebuffer_state(&batch->key);
unsigned batch_idx = agx_batch_idx(batch);
BITSET_CLEAR(ctx->batches.active, batch_idx);
agx_batch_mark_complete(batch);
if (!(dev->debug & (AGX_DBG_TRACE | AGX_DBG_SYNC))) {
agx_batch_print_stats(dev, batch);
}
}
int
agx_cleanup_batches(struct agx_context *ctx)
{
struct agx_device *dev = agx_device(ctx->base.screen);
unsigned i;
unsigned count = 0;
struct agx_batch *batches[AGX_MAX_BATCHES];
uint32_t syncobjs[AGX_MAX_BATCHES];
uint32_t first = 0;
foreach_submitted(ctx, i) {
batches[count] = &ctx->batches.slots[i];
syncobjs[count++] = ctx->batches.slots[i].syncobj;
}
if (!count)
return -1;
int ret = drmSyncobjWait(dev->fd, syncobjs, count, 0, 0, &first);
assert(!ret || ret == -ETIME);
if (ret)
return -1;
assert(first < AGX_MAX_BATCHES);
agx_batch_cleanup(ctx, batches[first]);
return agx_batch_idx(batches[first]);
}
static struct agx_batch *
@@ -124,7 +212,7 @@ agx_get_batch_for_framebuffer(struct agx_context *ctx,
{
/* Look if we have a matching batch */
unsigned i;
foreach_batch(ctx, i) {
foreach_active(ctx, i) {
struct agx_batch *candidate = &ctx->batches.slots[i];
if (util_framebuffer_state_equal(&candidate->key, state)) {
@@ -136,26 +224,43 @@ agx_get_batch_for_framebuffer(struct agx_context *ctx,
}
}
/* Look if we have a free batch */
struct agx_batch *batch = NULL;
for (unsigned i = 0; i < AGX_MAX_BATCHES; ++i) {
if (!BITSET_TEST(ctx->batches.active, i)) {
batch = &ctx->batches.slots[i];
break;
/* Look for a free batch */
for (i = 0; i < AGX_MAX_BATCHES; ++i) {
if (!BITSET_TEST(ctx->batches.active, i) &&
!BITSET_TEST(ctx->batches.submitted, i)) {
struct agx_batch *batch = &ctx->batches.slots[i];
agx_batch_init(ctx, state, batch);
return batch;
}
}
/* Try to clean up one batch */
int freed = agx_cleanup_batches(ctx);
if (freed >= 0) {
struct agx_batch *batch = &ctx->batches.slots[freed];
agx_batch_init(ctx, state, batch);
return batch;
}
/* Else, evict something */
if (!batch) {
for (unsigned i = 0; i < AGX_MAX_BATCHES; ++i) {
struct agx_batch *candidate = &ctx->batches.slots[i];
struct agx_batch *batch = NULL;
bool submitted = false;
for (i = 0; i < AGX_MAX_BATCHES; ++i) {
struct agx_batch *candidate = &ctx->batches.slots[i];
bool cand_submitted = BITSET_TEST(ctx->batches.submitted, i);
if (!batch || batch->seqnum > candidate->seqnum)
batch = candidate;
/* Prefer submitted batches first */
if (!cand_submitted && submitted)
continue;
if (!batch || batch->seqnum > candidate->seqnum) {
batch = candidate;
submitted = cand_submitted;
}
agx_flush_batch(ctx, batch);
}
assert(batch);
agx_sync_batch_for_reason(ctx, batch, "Too many batches");
/* Batch is now free */
agx_batch_init(ctx, state, batch);
@@ -187,11 +292,11 @@ agx_get_compute_batch(struct agx_context *ctx)
void
agx_flush_all(struct agx_context *ctx, const char *reason)
{
if (reason)
perf_debug_ctx(ctx, "Flushing due to: %s\n", reason);
unsigned idx;
foreach_batch(ctx, idx) {
foreach_active(ctx, idx) {
if (reason)
perf_debug_ctx(ctx, "Flushing due to: %s\n", reason);
agx_flush_batch(ctx, &ctx->batches.slots[idx]);
}
}
@@ -203,16 +308,19 @@ agx_flush_batch_for_reason(struct agx_context *ctx, struct agx_batch *batch,
if (reason)
perf_debug_ctx(ctx, "Flushing due to: %s\n", reason);
agx_flush_batch(ctx, batch);
if (agx_batch_is_active(batch))
agx_flush_batch(ctx, batch);
}
static void
agx_flush_readers_except(struct agx_context *ctx, struct agx_resource *rsrc,
struct agx_batch *except, const char *reason)
struct agx_batch *except, const char *reason,
bool sync)
{
unsigned idx;
foreach_batch(ctx, idx) {
/* Flush everything to the hardware first */
foreach_active(ctx, idx) {
struct agx_batch *batch = &ctx->batches.slots[idx];
if (batch == except)
@@ -223,17 +331,41 @@ agx_flush_readers_except(struct agx_context *ctx, struct agx_resource *rsrc,
agx_flush_batch(ctx, batch);
}
}
/* Then wait on everything if necessary */
if (sync) {
foreach_submitted(ctx, idx) {
struct agx_batch *batch = &ctx->batches.slots[idx];
if (batch == except)
continue;
if (agx_batch_uses_bo(batch, rsrc->bo)) {
perf_debug_ctx(ctx, "Sync reader due to: %s\n", reason);
agx_sync_batch(ctx, batch);
}
}
}
}
static void
agx_flush_writer_except(struct agx_context *ctx, struct agx_resource *rsrc,
struct agx_batch *except, const char *reason)
struct agx_batch *except, const char *reason, bool sync)
{
struct agx_batch *writer = agx_writer_get(ctx, rsrc->bo->handle);
if (writer && writer != except) {
perf_debug_ctx(ctx, "Flush writer due to: %s\n", reason);
agx_flush_batch(ctx, writer);
if (writer && writer != except &&
(agx_batch_is_active(writer) || agx_batch_is_submitted(writer))) {
if (agx_batch_is_active(writer) || sync) {
perf_debug_ctx(ctx, "%s writer due to: %s\n", sync ? "Sync" : "Flush",
reason);
}
if (agx_batch_is_active(writer))
agx_flush_batch(ctx, writer);
/* Check for submitted state, because if the batch was a no-op it'll
* already be cleaned up */
if (sync && agx_batch_is_submitted(writer))
agx_sync_batch(ctx, writer);
}
}
@@ -241,7 +373,14 @@ bool
agx_any_batch_uses_resource(struct agx_context *ctx, struct agx_resource *rsrc)
{
unsigned idx;
foreach_batch(ctx, idx) {
foreach_active(ctx, idx) {
struct agx_batch *batch = &ctx->batches.slots[idx];
if (agx_batch_uses_bo(batch, rsrc->bo))
return true;
}
foreach_submitted(ctx, idx) {
struct agx_batch *batch = &ctx->batches.slots[idx];
if (agx_batch_uses_bo(batch, rsrc->bo))
@@ -255,21 +394,36 @@ void
agx_flush_readers(struct agx_context *ctx, struct agx_resource *rsrc,
const char *reason)
{
agx_flush_readers_except(ctx, rsrc, NULL, reason);
agx_flush_readers_except(ctx, rsrc, NULL, reason, false);
}
void
agx_sync_readers(struct agx_context *ctx, struct agx_resource *rsrc,
const char *reason)
{
agx_flush_readers_except(ctx, rsrc, NULL, reason, true);
}
void
agx_flush_writer(struct agx_context *ctx, struct agx_resource *rsrc,
const char *reason)
{
agx_flush_writer_except(ctx, rsrc, NULL, reason);
agx_flush_writer_except(ctx, rsrc, NULL, reason, false);
}
void
agx_sync_writer(struct agx_context *ctx, struct agx_resource *rsrc,
const char *reason)
{
agx_flush_writer_except(ctx, rsrc, NULL, reason, true);
}
void
agx_batch_reads(struct agx_batch *batch, struct agx_resource *rsrc)
{
/* Hazard: read-after-write */
agx_flush_writer_except(batch->ctx, rsrc, batch, "Read from another batch");
agx_flush_writer_except(batch->ctx, rsrc, batch, "Read from another batch",
false);
agx_batch_add_bo(batch, rsrc->bo);
@@ -283,7 +437,7 @@ agx_batch_writes(struct agx_batch *batch, struct agx_resource *rsrc)
struct agx_context *ctx = batch->ctx;
struct agx_batch *writer = agx_writer_get(ctx, rsrc->bo->handle);
agx_flush_readers_except(ctx, rsrc, batch, "Write from other batch");
agx_flush_readers_except(ctx, rsrc, batch, "Write from other batch", false);
/* Nothing to do if we're already writing */
if (writer == batch)
@@ -296,7 +450,13 @@ agx_batch_writes(struct agx_batch *batch, struct agx_resource *rsrc)
/* Write is strictly stronger than a read */
agx_batch_reads(batch, rsrc);
/* We are now the new writer */
writer = agx_writer_get(ctx, rsrc->bo->handle);
assert(!writer || agx_batch_is_submitted(writer));
/* We are now the new writer. Disregard the previous writer -- anything that
* needs to wait for the writer going forward needs to wait for us.
*/
agx_writer_remove(ctx, rsrc->bo->handle);
agx_writer_add(ctx, agx_batch_idx(batch), rsrc->bo->handle);
}
@@ -314,10 +474,245 @@ void
agx_flush_occlusion_queries(struct agx_context *ctx)
{
unsigned i;
foreach_batch(ctx, i) {
foreach_active(ctx, i) {
struct agx_batch *other = &ctx->batches.slots[i];
if (other->occlusion_queries.size != 0)
agx_flush_batch_for_reason(ctx, other, "Occlusion query ordering");
}
foreach_submitted(ctx, i) {
struct agx_batch *other = &ctx->batches.slots[i];
if (other->occlusion_queries.size != 0)
agx_sync_batch_for_reason(ctx, other, "Occlusion query ordering");
}
}
static int
agx_get_in_sync(struct agx_context *ctx)
{
struct agx_device *dev = agx_device(ctx->base.screen);
if (ctx->in_sync_fd >= 0) {
int ret =
drmSyncobjImportSyncFile(dev->fd, ctx->in_sync_obj, ctx->in_sync_fd);
assert(!ret);
close(ctx->in_sync_fd);
ctx->in_sync_fd = -1;
return ctx->in_sync_obj;
} else {
return 0;
}
}
static void
agx_add_sync(struct drm_asahi_sync *syncs, unsigned *count, uint32_t handle)
{
if (!handle)
return;
syncs[(*count)++] = (struct drm_asahi_sync){
.sync_type = DRM_ASAHI_SYNC_SYNCOBJ,
.handle = handle,
};
}
void
agx_batch_submit(struct agx_context *ctx, struct agx_batch *batch,
uint32_t barriers, enum drm_asahi_cmd_type cmd_type,
void *cmdbuf)
{
struct agx_device *dev = agx_device(ctx->base.screen);
bool feedback = dev->debug & (AGX_DBG_TRACE | AGX_DBG_SYNC | AGX_DBG_STATS);
#ifndef NDEBUG
/* Debug builds always get feedback (for fault checks) */
feedback = true;
#endif
if (!feedback)
batch->result = NULL;
/* We allocate the worst-case sync array size since this won't be excessive
* for most workloads
*/
unsigned max_syncs = agx_batch_bo_list_bits(batch) + 1;
unsigned in_sync_count = 0;
unsigned shared_bo_count = 0;
struct drm_asahi_sync *in_syncs =
malloc(max_syncs * sizeof(struct drm_asahi_sync));
struct agx_bo **shared_bos = malloc(max_syncs * sizeof(struct agx_bo *));
struct drm_asahi_sync out_sync = {
.sync_type = DRM_ASAHI_SYNC_SYNCOBJ,
.handle = batch->syncobj,
};
int handle;
AGX_BATCH_FOREACH_BO_HANDLE(batch, handle) {
struct agx_bo *bo = agx_lookup_bo(dev, handle);
if (bo->flags & AGX_BO_SHARED) {
/* Get a sync file fd from the buffer */
int in_sync_fd = agx_export_sync_file(dev, bo);
assert(in_sync_fd >= 0);
/* Create a new syncobj */
uint32_t sync_handle;
int ret = drmSyncobjCreate(dev->fd, 0, &sync_handle);
assert(ret >= 0);
/* Import the sync file into it */
ret = drmSyncobjImportSyncFile(dev->fd, sync_handle, in_sync_fd);
assert(ret >= 0);
assert(sync_handle);
close(in_sync_fd);
/* Add it to our wait list */
agx_add_sync(in_syncs, &in_sync_count, sync_handle);
/* And keep track of the BO for cloning the out_sync */
shared_bos[shared_bo_count++] = bo;
}
}
/* Add an explicit fence from gallium, if any */
agx_add_sync(in_syncs, &in_sync_count, agx_get_in_sync(ctx));
/* Submit! */
agx_submit_single(
dev, cmd_type, barriers, in_syncs, in_sync_count, &out_sync, 1, cmdbuf,
feedback ? ctx->result_buf->handle : 0, feedback ? batch->result_off : 0,
feedback ? sizeof(union agx_batch_result) : 0);
/* Now stash our batch fence into any shared BOs. */
if (shared_bo_count) {
/* Convert our handle to a sync file */
int out_sync_fd = -1;
int ret = drmSyncobjExportSyncFile(dev->fd, batch->syncobj, &out_sync_fd);
assert(ret >= 0);
assert(out_sync_fd >= 0);
for (unsigned i = 0; i < shared_bo_count; i++) {
/* Free the in_sync handle we just acquired */
ret = drmSyncobjDestroy(dev->fd, in_syncs[i].handle);
assert(ret >= 0);
/* And then import the out_sync sync file into it */
ret = agx_import_sync_file(dev, shared_bos[i], out_sync_fd);
assert(ret >= 0);
}
close(out_sync_fd);
}
/* Record the syncobj on each BO we write, so it can be added post-facto as a
* fence if the BO is exported later...
*/
AGX_BATCH_FOREACH_BO_HANDLE(batch, handle) {
struct agx_bo *bo = agx_lookup_bo(dev, handle);
struct agx_batch *writer = agx_writer_get(ctx, handle);
if (!writer)
continue;
/* Skip BOs that are written by submitted batches, they're not ours */
if (agx_batch_is_submitted(writer))
continue;
/* But any BOs written by active batches are ours */
assert(writer == batch && "exclusive writer");
bo->writer_syncobj = batch->syncobj;
}
free(in_syncs);
free(shared_bos);
if (dev->debug & (AGX_DBG_TRACE | AGX_DBG_SYNC)) {
/* Wait so we can get errors reported back */
int ret = drmSyncobjWait(dev->fd, &batch->syncobj, 1, INT64_MAX, 0, NULL);
assert(!ret);
if (dev->debug & AGX_DBG_TRACE) {
/* agxdecode DRM commands */
switch (cmd_type) {
default:
unreachable("Linux UAPI not yet upstream");
}
agxdecode_next_frame();
}
agx_batch_print_stats(dev, batch);
}
agx_batch_mark_submitted(batch);
/* Record the last syncobj for fence creation */
ctx->syncobj = batch->syncobj;
if (ctx->batch == batch)
ctx->batch = NULL;
/* Try to clean up up to two batches, to keep memory usage down */
if (agx_cleanup_batches(ctx) >= 0)
agx_cleanup_batches(ctx);
}
void
agx_sync_batch(struct agx_context *ctx, struct agx_batch *batch)
{
struct agx_device *dev = agx_device(ctx->base.screen);
if (agx_batch_is_active(batch))
agx_flush_batch(ctx, batch);
/* Empty batch case, already cleaned up */
if (!agx_batch_is_submitted(batch))
return;
assert(batch->syncobj);
int ret = drmSyncobjWait(dev->fd, &batch->syncobj, 1, INT64_MAX, 0, NULL);
assert(!ret);
agx_batch_cleanup(ctx, batch);
}
void
agx_sync_batch_for_reason(struct agx_context *ctx, struct agx_batch *batch,
const char *reason)
{
if (reason)
perf_debug_ctx(ctx, "Syncing due to: %s\n", reason);
agx_sync_batch(ctx, batch);
}
void
agx_sync_all(struct agx_context *ctx, const char *reason)
{
if (reason)
perf_debug_ctx(ctx, "Syncing all due to: %s\n", reason);
unsigned idx;
foreach_active(ctx, idx) {
agx_flush_batch(ctx, &ctx->batches.slots[idx]);
}
foreach_submitted(ctx, idx) {
agx_sync_batch(ctx, &ctx->batches.slots[idx]);
}
}
void
agx_batch_reset(struct agx_context *ctx, struct agx_batch *batch)
{
/* Reset an empty batch. Like submit, but does nothing. */
agx_batch_mark_submitted(batch);
if (ctx->batch == batch)
ctx->batch = NULL;
agx_batch_cleanup(ctx, batch);
}
src/gallium/drivers/asahi/agx_fence.c
+156
View File
@@ -0,0 +1,156 @@
/*
* Based on panfrost/pan_fence.c:
*
* Copyright (c) 2022 Amazon.com, Inc. or its affiliates.
* Copyright (C) 2008 VMware, Inc.
* Copyright (C) 2014 Broadcom
* Copyright (C) 2018 Alyssa Rosenzweig
* Copyright (C) 2019 Collabora, Ltd.
* Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <xf86drm.h>
#include "agx_fence.h"
#include "agx_state.h"
#include "util/os_time.h"
#include "util/u_inlines.h"
void
agx_fence_reference(struct pipe_screen *pscreen, struct pipe_fence_handle **ptr,
struct pipe_fence_handle *fence)
{
struct agx_device *dev = agx_device(pscreen);
struct pipe_fence_handle *old = *ptr;
if (pipe_reference(old ? &old->reference : NULL,
fence ? &fence->reference : NULL)) {
drmSyncobjDestroy(dev->fd, old->syncobj);
free(old);
}
*ptr = fence;
}
bool
agx_fence_finish(struct pipe_screen *pscreen, struct pipe_context *ctx,
struct pipe_fence_handle *fence, uint64_t timeout)
{
struct agx_device *dev = agx_device(pscreen);
int ret;
if (fence->signaled)
return true;
uint64_t abs_timeout = os_time_get_absolute_timeout(timeout);
if (abs_timeout == OS_TIMEOUT_INFINITE)
abs_timeout = INT64_MAX;
ret = drmSyncobjWait(dev->fd, &fence->syncobj, 1, abs_timeout,
DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL, NULL);
assert(ret >= 0 || ret == -ETIME);
fence->signaled = (ret >= 0);
return fence->signaled;
}
int
agx_fence_get_fd(struct pipe_screen *screen, struct pipe_fence_handle *f)
{
struct agx_device *dev = agx_device(screen);
int fd = -1;
int ret = drmSyncobjExportSyncFile(dev->fd, f->syncobj, &fd);
assert(ret >= 0);
assert(fd >= 0);
return fd;
}
struct pipe_fence_handle *
agx_fence_from_fd(struct agx_context *ctx, int fd, enum pipe_fd_type type)
{
struct agx_device *dev = agx_device(ctx->base.screen);
int ret;
struct pipe_fence_handle *f = calloc(1, sizeof(*f));
if (!f)
return NULL;
if (type == PIPE_FD_TYPE_NATIVE_SYNC) {
ret = drmSyncobjCreate(dev->fd, 0, &f->syncobj);
if (ret) {
fprintf(stderr, "create syncobj failed\n");
goto err_free_fence;
}
ret = drmSyncobjImportSyncFile(dev->fd, f->syncobj, fd);
if (ret) {
fprintf(stderr, "import syncfile failed\n");
goto err_destroy_syncobj;
}
} else {
assert(type == PIPE_FD_TYPE_SYNCOBJ);
ret = drmSyncobjFDToHandle(dev->fd, fd, &f->syncobj);
if (ret) {
fprintf(stderr, "import syncobj FD failed\n");
goto err_free_fence;
}
}
pipe_reference_init(&f->reference, 1);
return f;
err_destroy_syncobj:
drmSyncobjDestroy(dev->fd, f->syncobj);
err_free_fence:
free(f);
return NULL;
}
struct pipe_fence_handle *
agx_fence_create(struct agx_context *ctx)
{
struct agx_device *dev = agx_device(ctx->base.screen);
int fd = -1, ret;
/* Snapshot the last rendering out fence. We'd rather have another
* syncobj instead of a sync file, but this is all we get.
* (HandleToFD/FDToHandle just gives you another syncobj ID for the
* same syncobj).
*/
ret = drmSyncobjExportSyncFile(dev->fd, ctx->syncobj, &fd);
assert(ret >= 0 && fd != -1 && "export failed");
if (ret || fd == -1) {
fprintf(stderr, "export failed\n");
return NULL;
}
struct pipe_fence_handle *f =
agx_fence_from_fd(ctx, fd, PIPE_FD_TYPE_NATIVE_SYNC);
close(fd);
return f;
}
src/gallium/drivers/asahi/agx_fence.h
+55
View File
@@ -0,0 +1,55 @@
/*
* Copyright (c) 2022 Amazon.com, Inc. or its affiliates.
* Copyright 2018-2019 Alyssa Rosenzweig
* Copyright 2018-2019 Collabora, Ltd.
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sub license, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial portions
* of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
* IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef AGX_FENCE_H
#define AGX_FENCE_H
#include "pipe/p_state.h"
struct agx_context;
struct pipe_fence_handle {
struct pipe_reference reference;
uint32_t syncobj;
bool signaled;
};
void agx_fence_reference(struct pipe_screen *pscreen,
struct pipe_fence_handle **ptr,
struct pipe_fence_handle *fence);
bool agx_fence_finish(struct pipe_screen *pscreen, struct pipe_context *ctx,
struct pipe_fence_handle *fence, uint64_t timeout);
int agx_fence_get_fd(struct pipe_screen *screen, struct pipe_fence_handle *f);
struct pipe_fence_handle *agx_fence_from_fd(struct agx_context *ctx, int fd,
enum pipe_fd_type type);
struct pipe_fence_handle *agx_fence_create(struct agx_context *ctx);
#endif
src/gallium/drivers/asahi/agx_pipe.c
+65 -29
View File
@@ -25,6 +25,7 @@
*/
#include <errno.h>
#include <stdio.h>
#include <xf86drm.h>
#include "asahi/compiler/agx_compile.h"
#include "asahi/layout/layout.h"
#include "asahi/lib/agx_formats.h"
@@ -51,6 +52,7 @@
#include "util/u_upload_mgr.h"
#include "agx_device.h"
#include "agx_disk_cache.h"
#include "agx_fence.h"
#include "agx_public.h"
#include "agx_state.h"
@@ -620,7 +622,7 @@ agx_shadow(struct agx_context *ctx, struct agx_resource *rsrc)
/*
* Perform the required synchronization before a transfer_map operation can
* complete. This may require flushing batches.
* complete. This may require syncing batches.
*/
static void
agx_prepare_for_map(struct agx_context *ctx, struct agx_resource *rsrc,
@@ -648,10 +650,10 @@ agx_prepare_for_map(struct agx_context *ctx, struct agx_resource *rsrc,
if (usage & PIPE_MAP_UNSYNCHRONIZED)
return;
/* Both writing and reading need writers flushed */
agx_flush_writer(ctx, rsrc, "Unsynchronized transfer");
/* Both writing and reading need writers synced */
agx_sync_writer(ctx, rsrc, "Unsynchronized transfer");
/* Additionally, writing needs readers flushed */
/* Additionally, writing needs readers synced */
if (!(usage & PIPE_MAP_WRITE))
return;
@@ -661,12 +663,12 @@ agx_prepare_for_map(struct agx_context *ctx, struct agx_resource *rsrc,
if (!agx_any_batch_uses_resource(ctx, rsrc))
return;
/* There are readers. Try to shadow the resource to avoid a flush */
/* There are readers. Try to shadow the resource to avoid a sync */
if ((usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) && agx_shadow(ctx, rsrc))
return;
/* Otherwise, we need to flush */
agx_flush_readers(ctx, rsrc, "Unsynchronized write");
/* Otherwise, we need to sync */
agx_sync_readers(ctx, rsrc, "Unsynchronized write");
}
/* Most of the time we can do CPU-side transfers, but sometimes we need to use
@@ -798,7 +800,7 @@ agx_transfer_map(struct pipe_context *pctx, struct pipe_resource *resource,
if ((usage & PIPE_MAP_READ) && agx_resource_valid(rsrc, level)) {
agx_blit_to_staging(pctx, transfer);
agx_flush_writer(ctx, staging, "GPU read staging blit");
agx_sync_writer(ctx, staging, "GPU read staging blit");
}
agx_bo_mmap(staging->bo);
@@ -997,7 +999,6 @@ agx_flush_resource(struct pipe_context *pctx, struct pipe_resource *pres)
pres->bind |= PIPE_BIND_SHARED;
agx_flush_writer(agx_context(pctx), rsrc, "flush_resource");
}
}
/*
@@ -1009,10 +1010,24 @@ agx_flush(struct pipe_context *pctx, struct pipe_fence_handle **fence,
{
struct agx_context *ctx = agx_context(pctx);
if (fence)
*fence = NULL;
agx_flush_all(ctx, "Gallium flush");
/* At this point all pending work has been submitted. Since jobs are
* started and completed sequentially from a UAPI perspective, and since
* we submit all jobs with compute+render barriers on the prior job,
* waiting on the last submitted job is sufficient to guarantee completion
* of all GPU work thus far, so we can create a fence out of the latest
* syncobj.
*
* See this page for more info on how the GPU/UAPI queueing works:
* https://github.com/AsahiLinux/docs/wiki/SW:AGX-driver-notes#queues
*/
if (fence) {
struct pipe_fence_handle *f = agx_fence_create(ctx);
pctx->screen->fence_reference(pctx->screen, fence, NULL);
*fence = f;
}
}
void
@@ -1021,10 +1036,11 @@ agx_flush_batch(struct agx_context *ctx, struct agx_batch *batch)
struct agx_device *dev = agx_device(ctx->base.screen);
assert(agx_batch_is_active(batch));
assert(!agx_batch_is_submitted(batch));
/* Make sure there's something to submit. */
if (!batch->clear && !batch->any_draws) {
agx_batch_cleanup(ctx, batch);
agx_batch_reset(ctx, batch);
return;
}
@@ -1120,14 +1136,23 @@ agx_flush_batch(struct agx_context *ctx, struct agx_batch *batch)
(void)pipeline_background;
(void)pipeline_background_partial;
agx_batch_cleanup(ctx, batch);
unreachable("Linux UAPI not yet upstream");
agx_batch_submit(ctx, batch, 0, 0, NULL);
}
static void
agx_destroy_context(struct pipe_context *pctx)
{
struct agx_device *dev = agx_device(pctx->screen);
struct agx_context *ctx = agx_context(pctx);
/* Batch state needs to be freed on completion, and we don't want to yank
* buffers out from in-progress GPU jobs to avoid faults, so just wait until
* everything in progress is actually done on context destroy. This will
* ensure everything is cleaned up properly.
*/
agx_sync_all(ctx, "destroy context");
if (pctx->stream_uploader)
u_upload_destroy(pctx->stream_uploader);
@@ -1140,6 +1165,16 @@ agx_destroy_context(struct pipe_context *pctx)
agx_bo_unreference(ctx->result_buf);
drmSyncobjDestroy(dev->fd, ctx->in_sync_obj);
drmSyncobjDestroy(dev->fd, ctx->dummy_syncobj);
if (ctx->in_sync_fd != -1)
close(ctx->in_sync_fd);
for (unsigned i = 0; i < AGX_MAX_BATCHES; ++i) {
if (ctx->batches.slots[i].syncobj)
drmSyncobjDestroy(dev->fd, ctx->batches.slots[i].syncobj);
}
ralloc_free(ctx);
}
@@ -1167,6 +1202,7 @@ agx_create_context(struct pipe_screen *screen, void *priv, unsigned flags)
{
struct agx_context *ctx = rzalloc(NULL, struct agx_context);
struct pipe_context *pctx = &ctx->base;
int ret;
if (!ctx)
return NULL;
@@ -1209,10 +1245,21 @@ agx_create_context(struct pipe_screen *screen, void *priv, unsigned flags)
ctx->blitter = util_blitter_create(pctx);
ctx->result_buf = agx_bo_create(
agx_device(screen), sizeof(union agx_batch_result) * AGX_MAX_BATCHES, 0,
"Batch result buffer");
agx_device(screen), sizeof(union agx_batch_result) * AGX_MAX_BATCHES,
AGX_BO_WRITEBACK, "Batch result buffer");
assert(ctx->result_buf);
/* Sync object/FD used for NATIVE_FENCE_FD. */
ctx->in_sync_fd = -1;
ret = drmSyncobjCreate(agx_device(screen)->fd, 0, &ctx->in_sync_obj);
assert(!ret);
/* Dummy sync object used before any work has been submitted. */
ret = drmSyncobjCreate(agx_device(screen)->fd, DRM_SYNCOBJ_CREATE_SIGNALED,
&ctx->dummy_syncobj);
assert(!ret);
ctx->syncobj = ctx->dummy_syncobj;
return pctx;
}
@@ -1309,6 +1356,7 @@ agx_get_param(struct pipe_screen *pscreen, enum pipe_cap param)
case PIPE_CAP_PRIMITIVE_RESTART:
case PIPE_CAP_PRIMITIVE_RESTART_FIXED_INDEX:
case PIPE_CAP_ANISOTROPIC_FILTER:
case PIPE_CAP_NATIVE_FENCE_FD:
return true;
case PIPE_CAP_SAMPLER_VIEW_TARGET:
@@ -1783,19 +1831,6 @@ agx_destroy_screen(struct pipe_screen *pscreen)
ralloc_free(screen);
}
static void
agx_fence_reference(struct pipe_screen *screen, struct pipe_fence_handle **ptr,
struct pipe_fence_handle *fence)
{
}
static bool
agx_fence_finish(struct pipe_screen *screen, struct pipe_context *ctx,
struct pipe_fence_handle *fence, uint64_t timeout)
{
return true;
}
static const void *
agx_get_compiler_options(struct pipe_screen *pscreen, enum pipe_shader_ir ir,
enum pipe_shader_type shader)
@@ -1905,6 +1940,7 @@ agx_screen_create(int fd, struct renderonly *ro, struct sw_winsys *winsys)
screen->get_timestamp = u_default_get_timestamp;
screen->fence_reference = agx_fence_reference;
screen->fence_finish = agx_fence_finish;
screen->fence_get_fd = agx_fence_get_fd;
screen->get_compiler_options = agx_get_compiler_options;
screen->get_disk_shader_cache = agx_get_disk_shader_cache;
src/gallium/drivers/asahi/agx_state.h
+30 -1
View File
@@ -179,6 +179,7 @@ struct agx_batch {
struct agx_context *ctx;
struct pipe_framebuffer_state key;
uint64_t seqnum;
uint32_t syncobj;
struct agx_tilebuffer_layout tilebuffer_layout;
@@ -283,7 +284,11 @@ enum agx_dirty {
AGX_DIRTY_QUERY = BITFIELD_BIT(13),
};
#define AGX_MAX_BATCHES (2)
/* Maximum number of in-progress + under-construction GPU batches.
* Must be large enough for silly workloads that do things like
* glGenerateMipmap on every frame, otherwise we end up losing performance.
*/
#define AGX_MAX_BATCHES (128)
struct agx_context {
struct pipe_context base;
@@ -299,6 +304,9 @@ struct agx_context {
/** Set of active batches for faster traversal */
BITSET_DECLARE(active, AGX_MAX_BATCHES);
/** Set of submitted batches for faster traversal */
BITSET_DECLARE(submitted, AGX_MAX_BATCHES);
} batches;
struct agx_batch *batch;
@@ -346,6 +354,11 @@ struct agx_context {
struct util_dynarray writer;
struct agx_meta_cache meta;
uint32_t syncobj;
uint32_t dummy_syncobj;
int in_sync_fd;
uint32_t in_sync_obj;
};
static void
@@ -573,6 +586,7 @@ bool agx_nir_lower_sysvals(nir_shader *shader,
unsigned *push_size);
bool agx_batch_is_active(struct agx_batch *batch);
bool agx_batch_is_submitted(struct agx_batch *batch);
uint64_t agx_batch_upload_pbe(struct agx_batch *batch, unsigned rt);
@@ -624,6 +638,10 @@ agx_batch_num_bo(struct agx_batch *batch)
BITSET_FOREACH_SET(handle, (batch)->bo_list.set, \
agx_batch_bo_list_bits(batch))
void agx_batch_submit(struct agx_context *ctx, struct agx_batch *batch,
uint32_t barriers, enum drm_asahi_cmd_type cmd_type,
void *cmdbuf);
void agx_flush_batch(struct agx_context *ctx, struct agx_batch *batch);
void agx_flush_batch_for_reason(struct agx_context *ctx,
struct agx_batch *batch, const char *reason);
@@ -635,6 +653,15 @@ void agx_flush_writer(struct agx_context *ctx, struct agx_resource *rsrc,
void agx_flush_batches_writing_occlusion_queries(struct agx_context *ctx);
void agx_flush_occlusion_queries(struct agx_context *ctx);
void agx_sync_writer(struct agx_context *ctx, struct agx_resource *rsrc,
const char *reason);
void agx_sync_readers(struct agx_context *ctx, struct agx_resource *rsrc,
const char *reason);
void agx_sync_batch(struct agx_context *ctx, struct agx_batch *batch);
void agx_sync_all(struct agx_context *ctx, const char *reason);
void agx_sync_batch_for_reason(struct agx_context *ctx, struct agx_batch *batch,
const char *reason);
/* Use these instead of batch_add_bo for proper resource tracking */
void agx_batch_reads(struct agx_batch *batch, struct agx_resource *rsrc);
void agx_batch_writes(struct agx_batch *batch, struct agx_resource *rsrc);
@@ -651,7 +678,9 @@ bool agx_any_batch_uses_resource(struct agx_context *ctx,
struct agx_batch *agx_get_batch(struct agx_context *ctx);
struct agx_batch *agx_get_compute_batch(struct agx_context *ctx);
void agx_batch_reset(struct agx_context *ctx, struct agx_batch *batch);
void agx_batch_cleanup(struct agx_context *ctx, struct agx_batch *batch);
int agx_cleanup_batches(struct agx_context *ctx);
/* Blit shaders */
void agx_blitter_save(struct agx_context *ctx, struct blitter_context *blitter,
src/gallium/drivers/asahi/meson.build
+2 -1
View File
@@ -22,6 +22,7 @@ files_asahi = files(
'agx_batch.c',
'agx_blit.c',
'agx_disk_cache.c',
'agx_fence.c',
'agx_pipe.c',
'agx_nir_lower_sysvals.c',
'agx_query.c',
@@ -35,7 +36,7 @@ libasahi = static_library(
include_directories : [inc_gallium_aux, inc_gallium, inc_include, inc_src],
c_args : [c_msvc_compat_args],
gnu_symbol_visibility : 'hidden',
dependencies : [idep_nir, idep_mesautil, idep_agx_pack],
dependencies : [idep_nir, idep_mesautil, idep_agx_pack, dep_libdrm],
)
driver_asahi = declare_dependency(