From c7f52551a7f9a63c22f2553897adea0af1ecb0cb Mon Sep 17 00:00:00 2001 From: Jonathan Date: Sun, 24 Jul 2022 17:48:46 +0200 Subject: [PATCH] gallium/u_threaded: buffer subdata merging (v2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In a scenario where a sequence of calls happens like: * subdata(buffer_a, offset=0, size=64) * subdata(buffer_a, offset=64, size=64) * subdata(buffer_a, offset=128, size=64) * subdata(buffer_a, offset=192, size=64) and the buffer can't be directly mapped (e.g., because it has bindings), the subdata calls will now be merged together into one larger subdata call. This achieves a 3x perf gain in KHR-GL46.CommonBugs.CommonBug_SparseBuffersWithCopyOps on radeonsi Before: real 0m1,923s user 0m1,017s sys 0m0,051s After: real 0m0,686s user 0m0,502s sys 0m0,071s Reviewed-By: Mike Blumenkrantz Reviewed-by: Marek Olšák Part-of: --- .../auxiliary/util/u_threaded_context.c | 94 +++++++++++++++++++ .../auxiliary/util/u_threaded_context.h | 6 ++ 2 files changed, 100 insertions(+) diff --git a/src/gallium/auxiliary/util/u_threaded_context.c b/src/gallium/auxiliary/util/u_threaded_context.c index 74d39f8da9b..4a778c8c903 100644 --- a/src/gallium/auxiliary/util/u_threaded_context.c +++ b/src/gallium/auxiliary/util/u_threaded_context.c @@ -237,6 +237,7 @@ tc_batch_execute(void *job, UNUSED void *gdata, int thread_index) tc_clear_driver_thread(batch->tc); tc_batch_check(batch); batch->num_total_slots = 0; + batch->last_mergeable_call = NULL; } static void @@ -295,6 +296,7 @@ tc_add_sized_call(struct threaded_context *tc, enum tc_call_id id, tc_batch_flush(tc); next = &tc->batch_slots[tc->next]; tc_assert(next->num_total_slots == 0); + tc_assert(next->last_mergeable_call == NULL); } tc_assert(util_queue_fence_is_signalled(&next->fence)); @@ -323,6 +325,56 @@ tc_add_sized_call(struct threaded_context *tc, enum tc_call_id id, ((struct type*)tc_add_sized_call(tc, execute, \ call_size_with_slots(type, num_slots))) +/* Returns the last mergeable call that was added to the unflushed + * batch, or NULL if the address of that call is not currently known + * or no such call exists in the unflushed batch. + */ +static struct tc_call_base * +tc_get_last_mergeable_call(struct threaded_context *tc) +{ + struct tc_batch *batch = &tc->batch_slots[tc->next]; + struct tc_call_base *call = batch->last_mergeable_call; + + tc_assert(call == NULL || call->num_slots <= batch->num_total_slots); + + if (call && (uint64_t *)call == &batch->slots[batch->num_total_slots - call->num_slots]) + return call; + else + return NULL; +} + +/* Increases the size of the last call in the unflushed batch to the + * given number of slots, if possible, without changing the call's data. + */ +static bool +tc_enlarge_last_mergeable_call(struct threaded_context *tc, unsigned desired_num_slots) +{ + struct tc_batch *batch = &tc->batch_slots[tc->next]; + struct tc_call_base *call = tc_get_last_mergeable_call(tc); + + tc_assert(call); + tc_assert(desired_num_slots >= call->num_slots); + + unsigned added_slots = desired_num_slots - call->num_slots; + + if (unlikely(batch->num_total_slots + added_slots > TC_SLOTS_PER_BATCH)) + return false; + + batch->num_total_slots += added_slots; + call->num_slots += added_slots; + + return true; +} + +static void +tc_mark_call_mergeable(struct threaded_context *tc, struct tc_call_base *call) +{ + struct tc_batch *batch = &tc->batch_slots[tc->next]; + tc_assert(call->num_slots <= batch->num_total_slots); + tc_assert((uint64_t *)call == &batch->slots[batch->num_total_slots - call->num_slots]); + batch->last_mergeable_call = call; +} + static bool tc_is_sync(struct threaded_context *tc) { @@ -2558,6 +2610,20 @@ tc_call_buffer_subdata(struct pipe_context *pipe, void *call, uint64_t *last) return p->base.num_slots; } +static bool +is_mergeable_buffer_subdata(const struct tc_call_base *previous_call, + unsigned usage, unsigned offset, + struct pipe_resource *resource) +{ + if (!previous_call || previous_call->call_id != TC_CALL_buffer_subdata) + return false; + + struct tc_buffer_subdata *subdata = (struct tc_buffer_subdata *)previous_call; + + return subdata->usage == usage && subdata->resource == resource + && (subdata->offset + subdata->size) == offset; +} + static void tc_buffer_subdata(struct pipe_context *_pipe, struct pipe_resource *resource, @@ -2601,6 +2667,32 @@ tc_buffer_subdata(struct pipe_context *_pipe, util_range_add(&tres->b, &tres->valid_buffer_range, offset, offset + size); + /* We can potentially merge this subdata call with the previous one (if any), + * if the application does a whole-buffer upload piecewise. */ + { + struct tc_call_base *last_call = tc_get_last_mergeable_call(tc); + struct tc_buffer_subdata *merge_dest = (struct tc_buffer_subdata *)last_call; + + if (is_mergeable_buffer_subdata(last_call, usage, offset, resource) && + tc_enlarge_last_mergeable_call(tc, call_size_with_slots(tc_buffer_subdata, merge_dest->size + size))) { + memcpy(merge_dest->slot + merge_dest->size, data, size); + merge_dest->size += size; + + /* TODO: We *could* do an invalidate + upload here if we detect that + * the merged subdata call overwrites the entire buffer. However, that's + * a little complicated since we can't add further calls to our batch + * until we have removed the merged subdata call, which means that + * calling tc_invalidate_buffer before we have removed the call will + * blow things up. + * + * Just leave a large, merged subdata call in the batch for now, which is + * at least better than tons of tiny subdata calls. + */ + + return; + } + } + /* The upload is small. Enqueue it. */ struct tc_buffer_subdata *p = tc_add_slot_based_call(tc, TC_CALL_buffer_subdata, tc_buffer_subdata, size); @@ -2614,6 +2706,8 @@ tc_buffer_subdata(struct pipe_context *_pipe, p->offset = offset; p->size = size; memcpy(p->slot, data, size); + + tc_mark_call_mergeable(tc, &p->base); } struct tc_texture_subdata { diff --git a/src/gallium/auxiliary/util/u_threaded_context.h b/src/gallium/auxiliary/util/u_threaded_context.h index 848c56abb89..9b330a849d2 100644 --- a/src/gallium/auxiliary/util/u_threaded_context.h +++ b/src/gallium/auxiliary/util/u_threaded_context.h @@ -414,6 +414,12 @@ struct tc_batch { #endif uint16_t num_total_slots; uint16_t buffer_list_index; + + /* The last mergeable call that was added to this batch (i.e. + * buffer subdata). This might be out-of-date or NULL. + */ + struct tc_call_base *last_mergeable_call; + struct util_queue_fence fence; struct tc_unflushed_batch_token *token; uint64_t slots[TC_SLOTS_PER_BATCH];