From 0c49d3cf97745aae8d1b3dedf0ef937845bd2853 Mon Sep 17 00:00:00 2001 From: Sagar Ghuge Date: Mon, 15 May 2023 22:45:53 -0700 Subject: [PATCH] anv: Setup companion RCS command buffer submission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add all the wait fences from the main (CCS/BCS) command buffer to the companion RCS command buffer so that the companion RCS batch starts at the same time as the main (CCS/BCS) batch. v2: - Drop unncessary flush (Jose) Signed-off-by: Sagar Ghuge Reviewed-by: Lionel Landwerlin Reviewed-by: José Roberto de Souza Part-of: --- src/intel/vulkan/anv_batch_chain.c | 26 ++++-- src/intel/vulkan/anv_private.h | 15 ++-- src/intel/vulkan/i915/anv_batch_chain.c | 103 ++++++++++++++++++++++-- src/intel/vulkan/xe/anv_batch_chain.c | 66 +++++++++++++-- 4 files changed, 186 insertions(+), 24 deletions(-) diff --git a/src/intel/vulkan/anv_batch_chain.c b/src/intel/vulkan/anv_batch_chain.c index bd35ef35a9e..0f86685cbb0 100644 --- a/src/intel/vulkan/anv_batch_chain.c +++ b/src/intel/vulkan/anv_batch_chain.c @@ -1176,8 +1176,10 @@ anv_cmd_buffer_chain_command_buffers(struct anv_cmd_buffer **cmd_buffers, } /* Chain the N-1 first batch buffers */ - for (uint32_t i = 0; i < (num_cmd_buffers - 1); i++) + for (uint32_t i = 0; i < (num_cmd_buffers - 1); i++) { + assert(cmd_buffers[i]->companion_rcs_cmd_buffer == NULL); anv_cmd_buffer_record_chain_submit(cmd_buffers[i], cmd_buffers[i + 1]); + } /* Put an end to the last one */ anv_cmd_buffer_record_end_submit(cmd_buffers[num_cmd_buffers - 1]); @@ -1188,7 +1190,8 @@ anv_cmd_buffer_exec_batch_debug(struct anv_queue *queue, uint32_t cmd_buffer_count, struct anv_cmd_buffer **cmd_buffers, struct anv_query_pool *perf_query_pool, - uint32_t perf_query_pass) + uint32_t perf_query_pass, + bool is_companion_rcs_cmd_buffer) { if (!INTEL_DEBUG(DEBUG_BATCH | DEBUG_BATCH_STATS)) return; @@ -1217,15 +1220,26 @@ anv_cmd_buffer_exec_batch_debug(struct anv_queue *queue, } for (uint32_t i = 0; i < cmd_buffer_count; i++) { + struct anv_cmd_buffer *cmd_buffer = + is_companion_rcs_cmd_buffer ? + cmd_buffers[i]->companion_rcs_cmd_buffer : + cmd_buffers[i]; struct anv_batch_bo *bbo = - list_first_entry(&cmd_buffers[i]->batch_bos, struct anv_batch_bo, link); - device->cmd_buffer_being_decoded = cmd_buffers[i]; + list_first_entry(&cmd_buffer->batch_bos, struct anv_batch_bo, link); + device->cmd_buffer_being_decoded = cmd_buffer; + struct intel_batch_decode_ctx *ctx = queue->decoder; + if (is_companion_rcs_cmd_buffer) { + int render_queue_idx = + anv_get_first_render_queue_index(device->physical); + ctx = &device->decoder[render_queue_idx]; + } + if (INTEL_DEBUG(DEBUG_BATCH)) { - intel_print_batch(queue->decoder, bbo->bo->map, + intel_print_batch(ctx, bbo->bo->map, bbo->bo->size, bbo->bo->offset, false); } if (INTEL_DEBUG(DEBUG_BATCH_STATS)) { - intel_batch_stats(queue->decoder, bbo->bo->map, + intel_batch_stats(ctx, bbo->bo->map, bbo->bo->size, bbo->bo->offset, false); } device->cmd_buffer_being_decoded = NULL; diff --git a/src/intel/vulkan/anv_private.h b/src/intel/vulkan/anv_private.h index 8405a18dd4d..21718b0f2ee 100644 --- a/src/intel/vulkan/anv_private.h +++ b/src/intel/vulkan/anv_private.h @@ -3502,15 +3502,19 @@ struct anv_cmd_buffer { extern const struct vk_command_buffer_ops anv_cmd_buffer_ops; /* Determine whether we can chain a given cmd_buffer to another one. We need - * softpin and we also need to make sure that we can edit the end of the batch - * to point to next one, which requires the command buffer to not be used - * simultaneously. + * to make sure that we can edit the end of the batch to point to next one, + * which requires the command buffer to not be used simultaneously. + * + * We could in theory also implement chaining with companion command buffers, + * but let's sparse ourselves some pain and misery. This optimization has no + * benefit on the brand new Xe kernel driver. */ static inline bool anv_cmd_buffer_is_chainable(struct anv_cmd_buffer *cmd_buffer) { return !(cmd_buffer->usage_flags & - VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT); + VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT) && + !(cmd_buffer->is_companion_rcs_cmd_buffer); } static inline bool @@ -3587,7 +3591,8 @@ anv_cmd_buffer_exec_batch_debug(struct anv_queue *queue, uint32_t cmd_buffer_count, struct anv_cmd_buffer **cmd_buffers, struct anv_query_pool *perf_query_pool, - uint32_t perf_query_pass); + uint32_t perf_query_pass, + bool is_companion_rcs_cmd_buffer); void anv_cmd_buffer_clflush(struct anv_cmd_buffer **cmd_buffers, uint32_t num_cmd_buffers); diff --git a/src/intel/vulkan/i915/anv_batch_chain.c b/src/intel/vulkan/i915/anv_batch_chain.c index a52ba522df4..3ded767a631 100644 --- a/src/intel/vulkan/i915/anv_batch_chain.c +++ b/src/intel/vulkan/i915/anv_batch_chain.c @@ -342,7 +342,8 @@ static VkResult setup_execbuf_for_cmd_buffers(struct anv_execbuf *execbuf, struct anv_queue *queue, struct anv_cmd_buffer **cmd_buffers, - uint32_t num_cmd_buffers) + uint32_t num_cmd_buffers, + bool is_companion_rcs_cmd_buffer) { struct anv_device *device = queue->device; VkResult result; @@ -353,8 +354,11 @@ setup_execbuf_for_cmd_buffers(struct anv_execbuf *execbuf, anv_cmd_buffer_chain_command_buffers(cmd_buffers, num_cmd_buffers); for (uint32_t i = 0; i < num_cmd_buffers; i++) { - anv_measure_submit(cmd_buffers[i]); - result = setup_execbuf_for_cmd_buffer(execbuf, cmd_buffers[i]); + struct anv_cmd_buffer *cmd_buf = + is_companion_rcs_cmd_buffer ? + cmd_buffers[i]->companion_rcs_cmd_buffer : cmd_buffers[i]; + anv_measure_submit(cmd_buf); + result = setup_execbuf_for_cmd_buffer(execbuf, cmd_buf); if (result != VK_SUCCESS) return result; } @@ -418,8 +422,12 @@ setup_execbuf_for_cmd_buffers(struct anv_execbuf *execbuf, return result; } + struct list_head *batch_bo = + is_companion_rcs_cmd_buffer && cmd_buffers[0]->companion_rcs_cmd_buffer ? + &cmd_buffers[0]->companion_rcs_cmd_buffer->batch_bos : + &cmd_buffers[0]->batch_bos; struct anv_batch_bo *first_batch_bo = - list_first_entry(&cmd_buffers[0]->batch_bos, struct anv_batch_bo, link); + list_first_entry(batch_bo, struct anv_batch_bo, link); /* The kernel requires that the last entry in the validation list be the * batch buffer to execute. We can simply swap the element @@ -447,9 +455,11 @@ setup_execbuf_for_cmd_buffers(struct anv_execbuf *execbuf, anv_cmd_buffer_clflush(cmd_buffers, num_cmd_buffers); #endif + assert(!is_companion_rcs_cmd_buffer || device->physical->has_vm_control); uint64_t exec_flags = 0; uint32_t context_id; - get_context_and_exec_flags(queue, false, &exec_flags, &context_id); + get_context_and_exec_flags(queue, is_companion_rcs_cmd_buffer, &exec_flags, + &context_id); execbuf->execbuf = (struct drm_i915_gem_execbuffer2) { .buffers_ptr = (uintptr_t) execbuf->objects, @@ -637,6 +647,73 @@ anv_i915_debug_submit(const struct anv_execbuf *execbuf) } } +static VkResult +i915_companion_rcs_queue_exec_locked(struct anv_queue *queue, + uint32_t cmd_buffer_count, + struct anv_cmd_buffer **cmd_buffers, + uint32_t wait_count, + const struct vk_sync_wait *waits) +{ + struct anv_device *device = queue->device; + struct anv_execbuf execbuf = { + .alloc = &queue->device->vk.alloc, + .alloc_scope = VK_SYSTEM_ALLOCATION_SCOPE_DEVICE, + }; + + /* Always add the workaround BO as it includes a driver identifier for the + * error_state. + */ + VkResult result = + anv_execbuf_add_bo(device, &execbuf, device->workaround_bo, NULL, 0); + if (result != VK_SUCCESS) + goto error; + + for (uint32_t i = 0; i < wait_count; i++) { + result = anv_execbuf_add_sync(device, &execbuf, + waits[i].sync, + false /* is_signal */, + waits[i].wait_value); + if (result != VK_SUCCESS) + goto error; + } + + result = setup_execbuf_for_cmd_buffers(&execbuf, queue, cmd_buffers, + cmd_buffer_count, + true /* is_companion_rcs_cmd_buffer */); + if (result != VK_SUCCESS) + goto error; + + if (INTEL_DEBUG(DEBUG_SUBMIT)) + anv_i915_debug_submit(&execbuf); + + anv_cmd_buffer_exec_batch_debug(queue, cmd_buffer_count, cmd_buffers, + NULL, 0, true /*is_companion_rcs_cmd_buffer */); + + if (execbuf.syncobj_values) { + execbuf.timeline_fences.fence_count = execbuf.syncobj_count; + execbuf.timeline_fences.handles_ptr = (uintptr_t)execbuf.syncobjs; + execbuf.timeline_fences.values_ptr = (uintptr_t)execbuf.syncobj_values; + anv_execbuf_add_ext(&execbuf, + DRM_I915_GEM_EXECBUFFER_EXT_TIMELINE_FENCES, + &execbuf.timeline_fences.base); + } else if (execbuf.syncobjs) { + execbuf.execbuf.flags |= I915_EXEC_FENCE_ARRAY; + execbuf.execbuf.num_cliprects = execbuf.syncobj_count; + execbuf.execbuf.cliprects_ptr = (uintptr_t)execbuf.syncobjs; + } + + int ret = queue->device->info->no_hw ? 0 : + anv_gem_execbuffer(queue->device, &execbuf.execbuf); + if (ret) { + anv_i915_debug_submit(&execbuf); + result = vk_queue_set_lost(&queue->vk, "execbuf2 failed: %m"); + } + + error: + anv_execbuf_finish(&execbuf); + return result; +} + VkResult i915_queue_exec_locked(struct anv_queue *queue, uint32_t wait_count, @@ -715,7 +792,8 @@ i915_queue_exec_locked(struct anv_queue *queue, if (cmd_buffer_count) { result = setup_execbuf_for_cmd_buffers(&execbuf, queue, cmd_buffers, - cmd_buffer_count); + cmd_buffer_count, + false /* is_companion_rcs_cmd_buffer */); } else { result = setup_empty_execbuf(&execbuf, queue); } @@ -730,7 +808,8 @@ i915_queue_exec_locked(struct anv_queue *queue, anv_i915_debug_submit(&execbuf); anv_cmd_buffer_exec_batch_debug(queue, cmd_buffer_count, cmd_buffers, - perf_query_pool, perf_query_pass); + perf_query_pool, perf_query_pass, + false /* is_companion_rcs_cmd_buffer */); if (execbuf.syncobj_values) { execbuf.timeline_fences.fence_count = execbuf.syncobj_count; @@ -799,6 +878,15 @@ i915_queue_exec_locked(struct anv_queue *queue, result = vk_queue_set_lost(&queue->vk, "execbuf2 failed: %m"); } + if (cmd_buffer_count != 0 && cmd_buffers[0]->companion_rcs_cmd_buffer) { + struct anv_cmd_buffer *companion_rcs_cmd_buffer = + cmd_buffers[0]->companion_rcs_cmd_buffer; + assert(companion_rcs_cmd_buffer->is_companion_rcs_cmd_buffer); + result = i915_companion_rcs_queue_exec_locked(queue, cmd_buffer_count, + cmd_buffers, wait_count, + waits); + } + if (result == VK_SUCCESS && queue->sync) { result = vk_sync_wait(&device->vk, queue->sync, 0, VK_SYNC_WAIT_COMPLETE, UINT64_MAX); @@ -829,6 +917,7 @@ i915_execute_simple_batch(struct anv_queue *queue, struct anv_bo *batch_bo, if (result != VK_SUCCESS) goto fail; + assert(!is_companion_rcs_batch || device->physical->has_vm_control); uint64_t exec_flags = 0; uint32_t context_id; get_context_and_exec_flags(queue, is_companion_rcs_batch, &exec_flags, diff --git a/src/intel/vulkan/xe/anv_batch_chain.c b/src/intel/vulkan/xe/anv_batch_chain.c index 0f713e7ac8a..5fadf847e19 100644 --- a/src/intel/vulkan/xe/anv_batch_chain.c +++ b/src/intel/vulkan/xe/anv_batch_chain.c @@ -106,12 +106,12 @@ xe_exec_process_syncs(struct anv_queue *queue, uint32_t wait_count, const struct vk_sync_wait *waits, uint32_t signal_count, const struct vk_sync_signal *signals, struct anv_utrace_submit *utrace_submit, + bool is_companion_rcs_queue, struct drm_xe_sync **ret, uint32_t *ret_count) { struct anv_device *device = queue->device; uint32_t num_syncs = wait_count + signal_count + (utrace_submit ? 1 : 0) + - (queue->sync ? 1 : 0); - + ((queue->sync && !is_companion_rcs_queue) ? 1 : 0); if (!num_syncs) return VK_SUCCESS; @@ -148,7 +148,7 @@ xe_exec_process_syncs(struct anv_queue *queue, TYPE_SIGNAL); } - if (queue->sync) { + if (queue->sync && !is_companion_rcs_queue) { struct drm_xe_sync *xe_sync = &xe_syncs[count++]; xe_exec_fill_sync(xe_sync, queue->sync, 0, @@ -164,14 +164,16 @@ xe_exec_process_syncs(struct anv_queue *queue, static void xe_exec_print_debug(struct anv_queue *queue, uint32_t cmd_buffer_count, struct anv_cmd_buffer **cmd_buffers, struct anv_query_pool *perf_query_pool, - uint32_t perf_query_pass, struct drm_xe_exec *exec) + uint32_t perf_query_pass, struct drm_xe_exec *exec, + bool is_companion_rcs_cmd_buffer) { if (INTEL_DEBUG(DEBUG_SUBMIT)) fprintf(stderr, "Batch offset=0x%016"PRIx64" on queue %u\n", (uint64_t)exec->address, queue->vk.index_in_family); anv_cmd_buffer_exec_batch_debug(queue, cmd_buffer_count, cmd_buffers, - perf_query_pool, perf_query_pass); + perf_query_pool, perf_query_pass, + is_companion_rcs_cmd_buffer); } VkResult @@ -204,6 +206,51 @@ xe_queue_exec_utrace_locked(struct anv_queue *queue, return VK_SUCCESS; } +static VkResult +xe_companion_rcs_queue_exec_locked(struct anv_queue *queue, + uint32_t cmd_buffer_count, + struct anv_cmd_buffer **cmd_buffers, + uint32_t wait_count, + const struct vk_sync_wait *waits) +{ + struct anv_device *device = queue->device; + VkResult result; + + struct drm_xe_sync *xe_syncs = NULL; + uint32_t xe_syncs_count = 0; + result = xe_exec_process_syncs(queue, wait_count, waits, 0, NULL, NULL, + true, /* is_companion_rcs_queue */ + &xe_syncs, + &xe_syncs_count); + if (result != VK_SUCCESS) + return result; + + struct drm_xe_exec exec = { + .exec_queue_id = queue->companion_rcs_id, + .num_batch_buffer = 1, + .syncs = (uintptr_t)xe_syncs, + .num_syncs = xe_syncs_count, + }; + + struct anv_cmd_buffer *first_cmd_buffer = + cmd_buffers[0]->companion_rcs_cmd_buffer; + struct anv_batch_bo *first_batch_bo = + list_first_entry(&first_cmd_buffer->batch_bos, struct anv_batch_bo, + link); + exec.address = first_batch_bo->bo->offset; + + xe_exec_print_debug(queue, cmd_buffer_count, cmd_buffers, NULL, 0, &exec, + true /* is_companion_rcs_cmd_buffer */); + + if (!device->info->no_hw) { + if (intel_ioctl(device->fd, DRM_IOCTL_XE_EXEC, &exec)) + result = vk_device_set_lost(&device->vk, "anv_xe_queue_exec_locked failed: %m"); + } + vk_free(&device->vk.alloc, xe_syncs); + + return result; +} + VkResult xe_queue_exec_locked(struct anv_queue *queue, uint32_t wait_count, @@ -229,6 +276,7 @@ xe_queue_exec_locked(struct anv_queue *queue, result = xe_exec_process_syncs(queue, wait_count, waits, signal_count, signals, utrace_submit, + false, /* is_companion_rcs_queue */ &xe_syncs, &xe_syncs_count); if (result != VK_SUCCESS) return result; @@ -261,7 +309,8 @@ xe_queue_exec_locked(struct anv_queue *queue, } xe_exec_print_debug(queue, cmd_buffer_count, cmd_buffers, perf_query_pool, - perf_query_pass, &exec); + perf_query_pass, &exec, + false /* is_companion_rcs_cmd_buffer */); /* TODO: add perfetto stuff when Xe supports it */ @@ -271,6 +320,11 @@ xe_queue_exec_locked(struct anv_queue *queue, } vk_free(&device->vk.alloc, xe_syncs); + if (cmd_buffer_count != 0 && cmd_buffers[0]->companion_rcs_cmd_buffer) + result = xe_companion_rcs_queue_exec_locked(queue, cmd_buffer_count, + cmd_buffers, wait_count, + waits); + if (result == VK_SUCCESS && queue->sync) { result = vk_sync_wait(&device->vk, queue->sync, 0, VK_SYNC_WAIT_COMPLETE, UINT64_MAX);