diff --git a/src/broadcom/vulkan/meson.build b/src/broadcom/vulkan/meson.build index 5d4de9ec4b5..e1c90dbc59a 100644 --- a/src/broadcom/vulkan/meson.build +++ b/src/broadcom/vulkan/meson.build @@ -67,6 +67,7 @@ libv3dv_files = files( 'v3dv_pipeline.c', 'v3dv_pipeline_cache.c', 'v3dv_private.h', + 'v3dv_query.c', 'v3dv_queue.c', 'v3dv_uniforms.c', 'v3dv_util.c', diff --git a/src/broadcom/vulkan/v3dv_cmd_buffer.c b/src/broadcom/vulkan/v3dv_cmd_buffer.c index 9398e2c4813..9f869dcfbe3 100644 --- a/src/broadcom/vulkan/v3dv_cmd_buffer.c +++ b/src/broadcom/vulkan/v3dv_cmd_buffer.c @@ -161,29 +161,31 @@ v3dv_job_destroy(struct v3dv_job *job) list_del(&job->list_link); - v3dv_cl_destroy(&job->bcl); - v3dv_cl_destroy(&job->rcl); - v3dv_cl_destroy(&job->indirect); + if (job->type == V3DV_JOB_TYPE_GPU) { + v3dv_cl_destroy(&job->bcl); + v3dv_cl_destroy(&job->rcl); + v3dv_cl_destroy(&job->indirect); - /* Since we don't ref BOs, when we add them to the command buffer, don't - * unref them here either. - */ + /* Since we don't ref BOs, when we add them to the command buffer, don't + * unref them here either. + */ #if 0 - set_foreach(job->bos, entry) { - struct v3dv_bo *bo = (struct v3dv_bo *)entry->key; - v3dv_bo_free(cmd_buffer->device, bo); - } + set_foreach(job->bos, entry) { + struct v3dv_bo *bo = (struct v3dv_bo *)entry->key; + v3dv_bo_free(cmd_buffer->device, bo); + } #endif - _mesa_set_destroy(job->bos, NULL); + _mesa_set_destroy(job->bos, NULL); - set_foreach(job->extra_bos, entry) { - struct v3dv_bo *bo = (struct v3dv_bo *)entry->key; - v3dv_bo_free(job->device, bo); + set_foreach(job->extra_bos, entry) { + struct v3dv_bo *bo = (struct v3dv_bo *)entry->key; + v3dv_bo_free(job->device, bo); + } + _mesa_set_destroy(job->extra_bos, NULL); + + v3dv_bo_free(job->device, job->tile_alloc); + v3dv_bo_free(job->device, job->tile_state); } - _mesa_set_destroy(job->extra_bos, NULL); - - v3dv_bo_free(job->device, job->tile_alloc); - v3dv_bo_free(job->device, job->tile_state); vk_free(&job->device->alloc, job); } @@ -206,6 +208,9 @@ cmd_buffer_free_resources(struct v3dv_cmd_buffer *cmd_buffer) vk_free(&cmd_buffer->pool->alloc, cmd_buffer->state.attachments); } + if (cmd_buffer->state.query.end.alloc_count > 0) + vk_free(&cmd_buffer->device->alloc, cmd_buffer->state.query.end.states); + if (cmd_buffer->push_constants_resource.bo) v3dv_bo_free(cmd_buffer->device, cmd_buffer->push_constants_resource.bo); } @@ -447,9 +452,6 @@ v3dv_job_start_frame(struct v3dv_job *job, /* There's definitely nothing in the VCD cache we want. */ cl_emit(&job->bcl, FLUSH_VCD_CACHE, bin); - /* Disable any leftover OQ state from another job. */ - cl_emit(&job->bcl, OCCLUSION_QUERY_COUNTER, counter); - /* "Binning mode lists must have a Start Tile Binning item (6) after * any prefix state data before the binning list proper starts." */ @@ -478,6 +480,38 @@ cmd_buffer_end_render_pass_frame(struct v3dv_cmd_buffer *cmd_buffer) v3dv_job_emit_binning_flush(cmd_buffer->state.job); } +static struct v3dv_job * +cmd_buffer_create_cpu_job(struct v3dv_device *device, + enum v3dv_job_type type, + struct v3dv_cmd_buffer *cmd_buffer, + uint32_t subpass_idx) +{ + struct v3dv_job *job = vk_zalloc(&device->alloc, + sizeof(struct v3dv_job), 8, + VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + v3dv_job_init(job, type, device, cmd_buffer, subpass_idx); + return job; +} + +static void +cmd_buffer_add_cpu_jobs_for_pending_state(struct v3dv_cmd_buffer *cmd_buffer) +{ + struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; + + if (state->query.end.used_count > 0) { + const uint32_t query_count = state->query.end.used_count; + for (uint32_t i = 0; i < query_count; i++) { + assert(i < state->query.end.used_count); + struct v3dv_job *job = + cmd_buffer_create_cpu_job(cmd_buffer->device, + V3DV_JOB_TYPE_CPU_END_QUERY, + cmd_buffer, -1); + job->cpu.query_end = state->query.end.states[i]; + list_addtail(&job->list_link, &cmd_buffer->submit_jobs); + } + } +} + void v3dv_cmd_buffer_finish_job(struct v3dv_cmd_buffer *cmd_buffer) { @@ -498,37 +532,47 @@ v3dv_cmd_buffer_finish_job(struct v3dv_cmd_buffer *cmd_buffer) list_addtail(&job->list_link, &cmd_buffer->submit_jobs); cmd_buffer->state.job = NULL; + + /* If we have recorded any state with this last GPU job that requires to + * emit CPU jobs after the job is completed, add them now. + */ + cmd_buffer_add_cpu_jobs_for_pending_state(cmd_buffer); } void v3dv_job_init(struct v3dv_job *job, + enum v3dv_job_type type, struct v3dv_device *device, struct v3dv_cmd_buffer *cmd_buffer, int32_t subpass_idx) { assert(job); + job->type = type; + job->device = device; job->cmd_buffer = cmd_buffer; - job->bos = - _mesa_set_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal); - job->bo_count = 0; + if (type == V3DV_JOB_TYPE_GPU) { + job->bos = + _mesa_set_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal); + job->bo_count = 0; - job->extra_bos = - _mesa_set_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal); + job->extra_bos = + _mesa_set_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal); - v3dv_cl_init(job, &job->bcl); - v3dv_cl_begin(&job->bcl); + v3dv_cl_init(job, &job->bcl); + v3dv_cl_begin(&job->bcl); - v3dv_cl_init(job, &job->rcl); - v3dv_cl_begin(&job->rcl); + v3dv_cl_init(job, &job->rcl); + v3dv_cl_begin(&job->rcl); - v3dv_cl_init(job, &job->indirect); - v3dv_cl_begin(&job->indirect); + v3dv_cl_init(job, &job->indirect); + v3dv_cl_begin(&job->indirect); - if (V3D_DEBUG & V3D_DEBUG_ALWAYS_FLUSH) - job->always_flush = true; + if (V3D_DEBUG & V3D_DEBUG_ALWAYS_FLUSH) + job->always_flush = true; + } if (cmd_buffer) { /* Flag all state as dirty. Generally, we need to re-emit state for each @@ -579,7 +623,8 @@ v3dv_cmd_buffer_start_job(struct v3dv_cmd_buffer *cmd_buffer, return NULL; } - v3dv_job_init(job, cmd_buffer->device, cmd_buffer, subpass_idx); + v3dv_job_init(job, V3DV_JOB_TYPE_GPU, cmd_buffer->device, + cmd_buffer, subpass_idx); return job; } @@ -1813,13 +1858,15 @@ v3dv_EndCommandBuffer(VkCommandBuffer commandBuffer) cmd_buffer->status = V3DV_CMD_BUFFER_STATUS_EXECUTABLE; struct v3dv_job *job = cmd_buffer->state.job; - if (!job) - return VK_SUCCESS; - - /* We get here if we recorded commands after the last render pass in the - * command buffer. Make sure we finish this last job. */ - assert(v3dv_cl_offset(&job->bcl) != 0); - v3dv_cmd_buffer_finish_job(cmd_buffer); + if (job) { + /* We get here if we recorded commands after the last render pass in the + * command buffer. Make sure we finish this last job. + * + * FIXME: is this even possible? + */ + assert(v3dv_cl_offset(&job->bcl) != 0); + v3dv_cmd_buffer_finish_job(cmd_buffer); + } return VK_SUCCESS; } @@ -2734,6 +2781,22 @@ emit_gl_shader_state(struct v3dv_cmd_buffer *cmd_buffer) V3DV_CMD_DIRTY_PUSH_CONSTANTS); } +static void +emit_occlusion_query(struct v3dv_cmd_buffer *cmd_buffer) +{ + struct v3dv_job *job = cmd_buffer->state.job; + assert(job); + + cl_emit(&job->bcl, OCCLUSION_QUERY_COUNTER, counter) { + if (cmd_buffer->state.query.active_query) { + counter.address = + v3dv_cl_address(cmd_buffer->state.query.active_query, 0); + } + } + + cmd_buffer->state.dirty &= ~V3DV_CMD_DIRTY_OCCLUSION_QUERY; +} + /* This stores command buffer state for the current subpass that we are * about to interrupt to emit a meta pass. */ @@ -2964,6 +3027,9 @@ cmd_buffer_emit_pre_draw(struct v3dv_cmd_buffer *cmd_buffer) if (*dirty & (V3DV_CMD_DIRTY_PIPELINE | V3DV_CMD_DIRTY_BLEND_CONSTANTS)) emit_blend(cmd_buffer); + if (*dirty & V3DV_CMD_DIRTY_OCCLUSION_QUERY) + emit_occlusion_query(cmd_buffer); + cmd_buffer->state.dirty &= ~V3DV_CMD_DIRTY_PIPELINE; } @@ -3296,3 +3362,144 @@ v3dv_CmdSetBlendConstants(VkCommandBuffer commandBuffer, cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_BLEND_CONSTANTS; } + +void +v3dv_cmd_buffer_reset_queries(struct v3dv_cmd_buffer *cmd_buffer, + struct v3dv_query_pool *pool, + uint32_t first, + uint32_t count) +{ + /* Resets can only happen outside a render pass instance so we should not + * be in the middle of job recording. + */ + assert(cmd_buffer->state.pass == NULL); + assert(cmd_buffer->state.job == NULL); + + assert(first < pool->query_count); + assert(first + count <= pool->query_count); + + struct v3dv_job *job = + cmd_buffer_create_cpu_job(cmd_buffer->device, + V3DV_JOB_TYPE_CPU_RESET_QUERIES, + cmd_buffer, -1); + job->cpu.query_reset.pool = pool; + job->cpu.query_reset.first = first; + job->cpu.query_reset.count = count; + + list_addtail(&job->list_link, &cmd_buffer->submit_jobs); +} + +static bool +ensure_query_state(const struct v3dv_device *device, + uint32_t slot_size, + uint32_t used_count, + uint32_t *alloc_count, + void **ptr) +{ + if (used_count >= *alloc_count) { + const uint32_t prev_slot_count = *alloc_count; + void *old_buffer = *ptr; + + const uint32_t new_slot_count = MAX2(*alloc_count * 2, 4); + const uint32_t bytes = new_slot_count * slot_size; + *ptr = vk_alloc(&device->alloc, bytes, 8, + VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + if (*ptr == NULL) { + fprintf(stderr, "Error: failed to allocate CPU buffer for query.\n"); + return false; + } + + memcpy(*ptr, old_buffer, prev_slot_count * slot_size); + *alloc_count = new_slot_count; + } + assert(used_count < *alloc_count); + return true; +} + +void +v3dv_cmd_buffer_begin_query(struct v3dv_cmd_buffer *cmd_buffer, + struct v3dv_query_pool *pool, + uint32_t query, + VkQueryControlFlags flags) +{ + /* FIXME: we only support one active query for now */ + assert(cmd_buffer->state.query.active_query == NULL); + assert(query < pool->query_count); + + cmd_buffer->state.query.active_query = pool->queries[query].bo; + cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_OCCLUSION_QUERY; +} + +void +v3dv_cmd_buffer_end_query(struct v3dv_cmd_buffer *cmd_buffer, + struct v3dv_query_pool *pool, + uint32_t query) +{ + assert(query < pool->query_count); + assert(cmd_buffer->state.query.active_query != NULL); + + if (cmd_buffer->state.pass) { + /* Queue the EndQuery in the command buffer state, we will create a CPU + * job to flag all of these queries as possibly available right after the + * render pass job in which they have been recorded. + */ + struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; + ensure_query_state(cmd_buffer->device, + sizeof(struct v3dv_end_query_cpu_job_info), + state->query.end.used_count, + &state->query.end.alloc_count, + (void **) &state->query.end.states); + + struct v3dv_end_query_cpu_job_info *info = + &state->query.end.states[state->query.end.used_count++]; + + info->pool = pool; + info->query = query; + } else { + /* Otherwise, schedule the CPU job immediately */ + struct v3dv_job *job = + cmd_buffer_create_cpu_job(cmd_buffer->device, + V3DV_JOB_TYPE_CPU_END_QUERY, + cmd_buffer, -1); + job->cpu.query_end.pool = pool; + job->cpu.query_end.query = query; + list_addtail(&job->list_link, &cmd_buffer->submit_jobs); + } + + cmd_buffer->state.query.active_query = NULL; + cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_OCCLUSION_QUERY; +} + +void +v3dv_cmd_buffer_copy_query_results(struct v3dv_cmd_buffer *cmd_buffer, + struct v3dv_query_pool *pool, + uint32_t first, + uint32_t count, + struct v3dv_buffer *dst, + uint32_t offset, + uint32_t stride, + VkQueryResultFlags flags) +{ + /* Copies can only happen outside a render pass instance so we should not + * be in the middle of job recording. + */ + assert(cmd_buffer->state.pass == NULL); + assert(cmd_buffer->state.job == NULL); + + assert(first < pool->query_count); + assert(first + count <= pool->query_count); + + struct v3dv_job *job = + cmd_buffer_create_cpu_job(cmd_buffer->device, + V3DV_JOB_TYPE_CPU_COPY_QUERY_RESULTS, + cmd_buffer, -1); + job->cpu.query_copy_results.pool = pool; + job->cpu.query_copy_results.first = first; + job->cpu.query_copy_results.count = count; + job->cpu.query_copy_results.dst = dst; + job->cpu.query_copy_results.offset = offset; + job->cpu.query_copy_results.stride = stride; + job->cpu.query_copy_results.flags = flags; + + list_addtail(&job->list_link, &cmd_buffer->submit_jobs); +} diff --git a/src/broadcom/vulkan/v3dv_device.c b/src/broadcom/vulkan/v3dv_device.c index afb332f45e1..db6b6ab614a 100644 --- a/src/broadcom/vulkan/v3dv_device.c +++ b/src/broadcom/vulkan/v3dv_device.c @@ -559,7 +559,7 @@ v3dv_GetPhysicalDeviceFeatures(VkPhysicalDevice physicalDevice, .textureCompressionETC2 = true, .textureCompressionASTC_LDR = false, .textureCompressionBC = false, - .occlusionQueryPrecise = false, + .occlusionQueryPrecise = true, .pipelineStatisticsQuery = false, .vertexPipelineStoresAndAtomics = true, .fragmentStoresAndAtomics = true, diff --git a/src/broadcom/vulkan/v3dv_private.h b/src/broadcom/vulkan/v3dv_private.h index 21990481a18..99a9ee2a403 100644 --- a/src/broadcom/vulkan/v3dv_private.h +++ b/src/broadcom/vulkan/v3dv_private.h @@ -553,6 +553,7 @@ enum v3dv_cmd_dirty_bits { V3DV_CMD_DIRTY_PUSH_CONSTANTS = 1 << 8, V3DV_CMD_DIRTY_BLEND_CONSTANTS = 1 << 9, V3DV_CMD_DIRTY_SHADER_VARIANTS = 1 << 10, + V3DV_CMD_DIRTY_OCCLUSION_QUERY = 1 << 11, }; @@ -598,9 +599,39 @@ enum v3dv_ez_state { VC5_EZ_DISABLED, }; +enum v3dv_job_type { + V3DV_JOB_TYPE_GPU = 0, + V3DV_JOB_TYPE_CPU_RESET_QUERIES, + V3DV_JOB_TYPE_CPU_END_QUERY, + V3DV_JOB_TYPE_CPU_COPY_QUERY_RESULTS, +}; + +struct v3dv_reset_query_cpu_job_info { + struct v3dv_query_pool *pool; + uint32_t first; + uint32_t count; +}; + +struct v3dv_end_query_cpu_job_info { + struct v3dv_query_pool *pool; + uint32_t query; +}; + +struct v3dv_copy_query_results_cpu_job_info { + struct v3dv_query_pool *pool; + uint32_t first; + uint32_t count; + struct v3dv_buffer *dst; + uint32_t offset; + uint32_t stride; + VkQueryResultFlags flags; +}; + struct v3dv_job { struct list_head list_link; + enum v3dv_job_type type; + struct v3dv_device *device; struct v3dv_cmd_buffer *cmd_buffer; @@ -657,9 +688,17 @@ struct v3dv_job { * require this behavior. */ bool always_flush; + + /* Job specs for CPU jobs */ + union { + struct v3dv_reset_query_cpu_job_info query_reset; + struct v3dv_end_query_cpu_job_info query_end; + struct v3dv_copy_query_results_cpu_job_info query_copy_results; + } cpu; }; void v3dv_job_init(struct v3dv_job *job, + enum v3dv_job_type type, struct v3dv_device *device, struct v3dv_cmd_buffer *cmd_buffer, int32_t subpass_idx); @@ -742,6 +781,25 @@ struct v3dv_cmd_buffer_state { struct v3dv_dynamic_state dynamic; } meta; + + /* Command buffer state for queries */ + struct { + /* A list of vkCmdQueryEnd commands recorded in the command buffer during + * a render pass. We queue these here and then schedule the corresponding + * CPU jobs for them at the time we finish the GPU job in which they have + * been recorded. + */ + struct { + uint32_t used_count; + uint32_t alloc_count; + struct v3dv_end_query_cpu_job_info *states; + } end; + + /* This is not NULL if we have an active query, that is, we have called + * vkCmdBeginQuery but not vkCmdEndQuery. + */ + struct v3dv_bo *active_query; + } query; }; struct v3dv_descriptor { @@ -770,6 +828,24 @@ struct v3dv_resource { uint32_t offset; }; +struct v3dv_query { + bool maybe_available; + struct v3dv_bo *bo; +}; + +struct v3dv_query_pool { + uint32_t query_count; + struct v3dv_query *queries; +}; + +VkResult v3dv_get_query_pool_results_cpu(struct v3dv_device *device, + struct v3dv_query_pool *pool, + uint32_t first, + uint32_t count, + void *data, + VkDeviceSize stride, + VkQueryResultFlags flags); + struct v3dv_cmd_buffer { VK_LOADER_DATA _loader_data; @@ -813,6 +889,29 @@ void v3dv_render_pass_setup_render_target(struct v3dv_cmd_buffer *cmd_buffer, uint32_t *rt_type, uint32_t *rt_clamp); +void v3dv_cmd_buffer_reset_queries(struct v3dv_cmd_buffer *cmd_buffer, + struct v3dv_query_pool *pool, + uint32_t first, + uint32_t count); + +void v3dv_cmd_buffer_begin_query(struct v3dv_cmd_buffer *cmd_buffer, + struct v3dv_query_pool *pool, + uint32_t query, + VkQueryControlFlags flags); + +void v3dv_cmd_buffer_end_query(struct v3dv_cmd_buffer *cmd_buffer, + struct v3dv_query_pool *pool, + uint32_t query); + +void v3dv_cmd_buffer_copy_query_results(struct v3dv_cmd_buffer *cmd_buffer, + struct v3dv_query_pool *pool, + uint32_t first, + uint32_t count, + struct v3dv_buffer *dst, + uint32_t offset, + uint32_t stride, + VkQueryResultFlags flags); + struct v3dv_semaphore { /* A syncobject handle associated with this semaphore */ uint32_t sync; @@ -1356,6 +1455,7 @@ V3DV_DEFINE_NONDISP_HANDLE_CASTS(v3dv_image, VkImage) V3DV_DEFINE_NONDISP_HANDLE_CASTS(v3dv_image_view, VkImageView) V3DV_DEFINE_NONDISP_HANDLE_CASTS(v3dv_pipeline, VkPipeline) V3DV_DEFINE_NONDISP_HANDLE_CASTS(v3dv_pipeline_layout, VkPipelineLayout) +V3DV_DEFINE_NONDISP_HANDLE_CASTS(v3dv_query_pool, VkQueryPool) V3DV_DEFINE_NONDISP_HANDLE_CASTS(v3dv_render_pass, VkRenderPass) V3DV_DEFINE_NONDISP_HANDLE_CASTS(v3dv_sampler, VkSampler) V3DV_DEFINE_NONDISP_HANDLE_CASTS(v3dv_semaphore, VkSemaphore) diff --git a/src/broadcom/vulkan/v3dv_query.c b/src/broadcom/vulkan/v3dv_query.c new file mode 100644 index 00000000000..2d7f40c2d50 --- /dev/null +++ b/src/broadcom/vulkan/v3dv_query.c @@ -0,0 +1,261 @@ +/* + * Copyright © 2020 Raspberry Pi + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "v3dv_private.h" + +VkResult +v3dv_CreateQueryPool(VkDevice _device, + const VkQueryPoolCreateInfo *pCreateInfo, + const VkAllocationCallbacks *pAllocator, + VkQueryPool *pQueryPool) +{ + V3DV_FROM_HANDLE(v3dv_device, device, _device); + + assert(pCreateInfo->queryType == VK_QUERY_TYPE_OCCLUSION); + assert(pCreateInfo->queryCount > 0); + + + /* FIXME: the hw allows us to allocate up to 16 queries in a single block + * so we should try to use that. + */ + struct v3dv_query_pool *pool = + vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8, + VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + if (pool == NULL) + return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); + + pool->query_count = pCreateInfo->queryCount; + + VkResult result; + + const uint32_t pool_bytes = sizeof(struct v3dv_query) * pool->query_count; + pool->queries = vk_alloc2(&device->alloc, pAllocator, pool_bytes, 8, + VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + if (pool->queries == NULL) { + result = vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); + goto fail_alloc_bo_list; + } + + uint32_t i; + for (i = 0; i < pool->query_count; i++) { + pool->queries[i].maybe_available = false; + pool->queries[i].bo = v3dv_bo_alloc(device, 4096, "query"); + if (!pool->queries[i].bo) { + result = vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY); + goto fail_alloc_bo; + } + + /* For occlusion queries we only need a 4-byte counter */ + if (!v3dv_bo_map(device, pool->queries[i].bo, 4)) { + result = vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY); + goto fail_alloc_bo; + } + } + + *pQueryPool = v3dv_query_pool_to_handle(pool); + + return VK_SUCCESS; + +fail_alloc_bo: + for (uint32_t j = 0; j < i; j++) + v3dv_bo_free(device, pool->queries[j].bo); + vk_free2(&device->alloc, pAllocator, pool->queries); + +fail_alloc_bo_list: + vk_free2(&device->alloc, pAllocator, pool); + + return result; +} + +void +v3dv_DestroyQueryPool(VkDevice _device, + VkQueryPool queryPool, + const VkAllocationCallbacks *pAllocator) +{ + V3DV_FROM_HANDLE(v3dv_device, device, _device); + V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool); + + if (!pool) + return; + + for (uint32_t i = 0; i < pool->query_count; i++) + v3dv_bo_free(device, pool->queries[i].bo); + vk_free2(&device->alloc, pAllocator, pool->queries); + vk_free2(&device->alloc, pAllocator, pool); +} + +static void +write_query_result(void *dst, uint32_t idx, bool do_64bit, uint32_t value) +{ + if (do_64bit) { + uint64_t *dst64 = (uint64_t *) dst; + dst64[idx] = value; + } else { + uint32_t *dst32 = (uint32_t *) dst; + dst32[idx] = value; + } +} + +VkResult +v3dv_get_query_pool_results_cpu(struct v3dv_device *device, + struct v3dv_query_pool *pool, + uint32_t first, + uint32_t count, + void *data, + VkDeviceSize stride, + VkQueryResultFlags flags) +{ + assert(first < pool->query_count); + assert(first + count <= pool->query_count); + assert(data); + + const bool do_64bit = flags & VK_QUERY_RESULT_64_BIT; + const bool do_wait = flags & VK_QUERY_RESULT_WAIT_BIT; + const bool do_partial = flags & VK_QUERY_RESULT_PARTIAL_BIT; + + VkResult result = VK_SUCCESS; + for (uint32_t i = first; i < first + count; i++) { + assert(pool->queries[i].bo && pool->queries[i].bo->map); + struct v3dv_bo *bo = pool->queries[i].bo; + const uint32_t *counter = (const uint32_t *) bo->map; + + bool available; + if (do_wait) { + /* From the Vulkan 1.0 spec: + * + * "If VK_QUERY_RESULT_WAIT_BIT is set, (...) If the query does not + * become available in a finite amount of time (e.g. due to not + * issuing a query since the last reset), a VK_ERROR_DEVICE_LOST + * error may occur." + */ + if (!pool->queries[i].maybe_available) + return vk_error(device->instance, VK_ERROR_DEVICE_LOST); + + if (!v3dv_bo_wait(device, bo, 0xffffffffffffffffull)) + return vk_error(device->instance, VK_ERROR_DEVICE_LOST); + + available = true; + } else { + available = pool->queries[i].maybe_available && + v3dv_bo_wait(device, bo, 0); + } + + /** + * From the Vulkan 1.0 spec: + * + * "If VK_QUERY_RESULT_WAIT_BIT and VK_QUERY_RESULT_PARTIAL_BIT are + * both not set then no result values are written to pData for queries + * that are in the unavailable state at the time of the call, and + * vkGetQueryPoolResults returns VK_NOT_READY. However, availability + * state is still written to pData for those queries if + * VK_QUERY_RESULT_WITH_AVAILABILITY_BIT is set." + */ + uint32_t slot = 0; + + const bool write_result = available || do_partial; + if (write_result) + write_query_result(data, slot, do_64bit, *counter); + slot++; + + if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) + write_query_result(data, slot++, do_64bit, available ? 1u : 0u); + + if (!write_result) + result = VK_NOT_READY; + + data += stride; + } + + return result; +} + +VkResult +v3dv_GetQueryPoolResults(VkDevice _device, + VkQueryPool queryPool, + uint32_t firstQuery, + uint32_t queryCount, + size_t dataSize, + void *pData, + VkDeviceSize stride, + VkQueryResultFlags flags) +{ + V3DV_FROM_HANDLE(v3dv_device, device, _device); + V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool); + + return v3dv_get_query_pool_results_cpu(device, pool, firstQuery, queryCount, + pData, stride, flags); +} + +void +v3dv_CmdResetQueryPool(VkCommandBuffer commandBuffer, + VkQueryPool queryPool, + uint32_t firstQuery, + uint32_t queryCount) +{ + V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); + V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool); + + v3dv_cmd_buffer_reset_queries(cmd_buffer, pool, firstQuery, queryCount); +} + +void +v3dv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, + VkQueryPool queryPool, + uint32_t firstQuery, + uint32_t queryCount, + VkBuffer dstBuffer, + VkDeviceSize dstOffset, + VkDeviceSize stride, + VkQueryResultFlags flags) +{ + V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); + V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool); + V3DV_FROM_HANDLE(v3dv_buffer, dst, dstBuffer); + + v3dv_cmd_buffer_copy_query_results(cmd_buffer, pool, + firstQuery, queryCount, + dst, dstOffset, stride, flags); +} + +void +v3dv_CmdBeginQuery(VkCommandBuffer commandBuffer, + VkQueryPool queryPool, + uint32_t query, + VkQueryControlFlags flags) +{ + V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); + V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool); + + v3dv_cmd_buffer_begin_query(cmd_buffer, pool, query, flags); +} + +void +v3dv_CmdEndQuery(VkCommandBuffer commandBuffer, + VkQueryPool queryPool, + uint32_t query) +{ + V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); + V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool); + + v3dv_cmd_buffer_end_query(cmd_buffer, pool, query); +} diff --git a/src/broadcom/vulkan/v3dv_queue.c b/src/broadcom/vulkan/v3dv_queue.c index cc45de1f64f..b98380c4215 100644 --- a/src/broadcom/vulkan/v3dv_queue.c +++ b/src/broadcom/vulkan/v3dv_queue.c @@ -76,6 +76,101 @@ get_absolute_timeout(uint64_t timeout) return (current_time + timeout); } +static VkResult +handle_reset_query_cpu_job(struct v3dv_job *job) +{ + /* We are about to reset query counters so we need to make sure that + * The GPU is not using them. + * + * FIXME: we could avoid blocking the main thread for this if we use + * submission thread. + */ + v3dv_DeviceWaitIdle(v3dv_device_to_handle(job->device)); + + struct v3dv_reset_query_cpu_job_info *info = &job->cpu.query_reset; + for (uint32_t i = info->first; i < info->first + info->count; i++) { + assert(i < info->pool->query_count); + struct v3dv_query *query = &info->pool->queries[i]; + query->maybe_available = false; + uint32_t *counter = (uint32_t *) query->bo->map; + *counter = 0; + } + + return VK_SUCCESS; +} + +static VkResult +handle_end_query_cpu_job(struct v3dv_job *job) +{ + struct v3dv_end_query_cpu_job_info *info = &job->cpu.query_end; + assert(info->query < info->pool->query_count); + struct v3dv_query *query = &info->pool->queries[info->query]; + query->maybe_available = true; + + return VK_SUCCESS; +} + +static VkResult +handle_copy_query_results_cpu_job(struct v3dv_job *job) +{ + struct v3dv_copy_query_results_cpu_job_info *info = + &job->cpu.query_copy_results; + + assert(info->dst && info->dst->mem && info->dst->mem->bo); + struct v3dv_bo *bo = info->dst->mem->bo; + const uint32_t bo_size = info->dst->size; + + /* Map the entire dst buffer for the CPU copy */ + bool dst_was_mapped = bo->map != NULL; + uint32_t map_size = bo->map_size; + bool needs_map = false; + if (!dst_was_mapped) { + needs_map = true; + } else if (map_size < bo_size) { + v3dv_bo_unmap(job->device, bo); + needs_map = true; + } + if (needs_map && !v3dv_bo_map(job->device, bo, bo_size)) + return vk_error(job->device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); + + /* FIXME: if flags includes VK_QUERY_RESULT_WAIT_BIT this could trigger a + * sync wait on the CPU for the corresponding GPU jobs to finish. We might + * want to use a submission thread to avoid blocking on the main thread. + */ + v3dv_get_query_pool_results_cpu(job->device, + info->pool, + info->first, + info->count, + bo->map + info->dst->mem_offset, + info->stride, + info->flags); + + if (needs_map) { + v3dv_bo_unmap(job->device, bo); + if (dst_was_mapped) { + if (!v3dv_bo_map(job->device, bo, map_size)) + return vk_error(job->device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); + } + } + + return VK_SUCCESS; +} + +static VkResult +handle_cpu_job(struct v3dv_job *job) +{ + switch (job->type) { + case V3DV_JOB_TYPE_CPU_RESET_QUERIES: + return handle_reset_query_cpu_job(job); + case V3DV_JOB_TYPE_CPU_END_QUERY: + return handle_end_query_cpu_job(job); + case V3DV_JOB_TYPE_CPU_COPY_QUERY_RESULTS: + return handle_copy_query_results_cpu_job(job); + default: + unreachable("Unhandled job type"); + } +} + static VkResult process_semaphores_to_signal(struct v3dv_device *device, uint32_t count, const VkSemaphore *sems) @@ -132,10 +227,15 @@ process_fence_to_signal(struct v3dv_device *device, VkFence _fence) } static VkResult -queue_submit_job(struct v3dv_queue *queue, struct v3dv_job *job, bool do_wait) +queue_submit_job(struct v3dv_queue *queue, + struct v3dv_job *job, + bool do_wait) { assert(job); + if (job->type != V3DV_JOB_TYPE_GPU) + return handle_cpu_job(job); + struct v3dv_device *device = queue->device; struct drm_v3d_submit_cl submit; @@ -311,7 +411,7 @@ queue_create_noop_job(struct v3dv_queue *queue, struct v3dv_job **job) VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!*job) return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); - v3dv_job_init(*job, device, NULL, -1); + v3dv_job_init(*job, V3DV_JOB_TYPE_GPU, device, NULL, -1); emit_noop_bin(*job); emit_noop_render(*job);