From 675d7f4bd0d09a80e8f0a5a60ca9eab20a6e3b74 Mon Sep 17 00:00:00 2001 From: Samuel Pitoiset Date: Mon, 1 Apr 2024 13:25:49 +0200 Subject: [PATCH] radv: remove radv_cmd_buffer::device Get the logical device object using the base object. Signed-off-by: Samuel Pitoiset Part-of: --- .../vulkan/layers/radv_annotate_layer_gen.py | 6 +- src/amd/vulkan/layers/radv_rage2.c | 2 +- src/amd/vulkan/layers/radv_rra_layer.c | 34 +- src/amd/vulkan/layers/radv_sqtt_layer.c | 65 +- src/amd/vulkan/meta/radv_meta_astc_decode.c | 13 +- src/amd/vulkan/meta/radv_meta_blit.c | 8 +- src/amd/vulkan/meta/radv_meta_blit2d.c | 19 +- src/amd/vulkan/meta/radv_meta_buffer.c | 30 +- src/amd/vulkan/meta/radv_meta_bufimage.c | 70 +- src/amd/vulkan/meta/radv_meta_clear.c | 74 +- src/amd/vulkan/meta/radv_meta_copy.c | 38 +- .../vulkan/meta/radv_meta_copy_vrs_htile.c | 8 +- src/amd/vulkan/meta/radv_meta_dcc_retile.c | 10 +- src/amd/vulkan/meta/radv_meta_decompress.c | 17 +- src/amd/vulkan/meta/radv_meta_etc_decode.c | 9 +- src/amd/vulkan/meta/radv_meta_fast_clear.c | 14 +- src/amd/vulkan/meta/radv_meta_fmask_copy.c | 13 +- src/amd/vulkan/meta/radv_meta_fmask_expand.c | 8 +- src/amd/vulkan/meta/radv_meta_resolve.c | 31 +- src/amd/vulkan/meta/radv_meta_resolve_cs.c | 30 +- src/amd/vulkan/meta/radv_meta_resolve_fs.c | 24 +- src/amd/vulkan/radix_sort/radv_radix_sort.c | 28 +- src/amd/vulkan/radv_acceleration_structure.c | 114 +-- src/amd/vulkan/radv_cmd_buffer.c | 695 ++++++++++-------- .../vulkan/radv_device_generated_commands.c | 21 +- src/amd/vulkan/radv_perfcounter.c | 52 +- src/amd/vulkan/radv_private.h | 8 +- src/amd/vulkan/radv_query.c | 123 ++-- src/amd/vulkan/radv_sqtt.c | 4 +- src/amd/vulkan/radv_video.c | 48 +- src/amd/vulkan/si_cmd_buffer.c | 34 +- 31 files changed, 937 insertions(+), 713 deletions(-) diff --git a/src/amd/vulkan/layers/radv_annotate_layer_gen.py b/src/amd/vulkan/layers/radv_annotate_layer_gen.py index 66ddc8e6cd9..3fd75a0e1a6 100644 --- a/src/amd/vulkan/layers/radv_annotate_layer_gen.py +++ b/src/amd/vulkan/layers/radv_annotate_layer_gen.py @@ -47,8 +47,10 @@ TEMPLATE = Template(COPYRIGHT + """ #include "radv_private.h" #define ANNOTATE(command, ...) \ - radv_cmd_buffer_annotate(radv_cmd_buffer_from_handle(commandBuffer), #command); \ - radv_cmd_buffer_from_handle(commandBuffer)->device->layer_dispatch.annotate.command(__VA_ARGS__) + struct radv_cmd_buffer *cmd_buffer = radv_cmd_buffer_from_handle(commandBuffer); \ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); \ + radv_cmd_buffer_annotate(cmd_buffer, #command); \ + device->layer_dispatch.annotate.command(__VA_ARGS__) % for c in commands: % if c.guard is not None: diff --git a/src/amd/vulkan/layers/radv_rage2.c b/src/amd/vulkan/layers/radv_rage2.c index 471b23e3b99..1f7d114ba14 100644 --- a/src/amd/vulkan/layers/radv_rage2.c +++ b/src/amd/vulkan/layers/radv_rage2.c @@ -31,7 +31,7 @@ rage2_CmdBeginRenderPass(VkCommandBuffer commandBuffer, const VkRenderPassBeginI { VK_FROM_HANDLE(vk_framebuffer, framebuffer, pRenderPassBegin->framebuffer); RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); VkRenderPassBeginInfo render_pass_begin = { .sType = pRenderPassBegin->sType, diff --git a/src/amd/vulkan/layers/radv_rra_layer.c b/src/amd/vulkan/layers/radv_rra_layer.c index 99a4ca4669e..829f625c722 100644 --- a/src/amd/vulkan/layers/radv_rra_layer.c +++ b/src/amd/vulkan/layers/radv_rra_layer.c @@ -194,6 +194,7 @@ handle_accel_struct_write(VkCommandBuffer commandBuffer, struct vk_acceleration_ struct radv_rra_accel_struct_data *data) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); VkMemoryBarrier2 barrier = { .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, @@ -215,7 +216,7 @@ handle_accel_struct_write(VkCommandBuffer commandBuffer, struct vk_acceleration_ if (!data->va) { data->va = vk_acceleration_structure_get_va(accel_struct); - _mesa_hash_table_u64_insert(cmd_buffer->device->rra_trace.accel_struct_vas, data->va, accel_struct); + _mesa_hash_table_u64_insert(device->rra_trace.accel_struct_vas, data->va, accel_struct); } if (!data->buffer) @@ -244,39 +245,42 @@ rra_CmdBuildAccelerationStructuresKHR(VkCommandBuffer commandBuffer, uint32_t in const VkAccelerationStructureBuildRangeInfoKHR *const *ppBuildRangeInfos) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - cmd_buffer->device->layer_dispatch.rra.CmdBuildAccelerationStructuresKHR(commandBuffer, infoCount, pInfos, - ppBuildRangeInfos); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); - simple_mtx_lock(&cmd_buffer->device->rra_trace.data_mtx); + device->layer_dispatch.rra.CmdBuildAccelerationStructuresKHR(commandBuffer, infoCount, pInfos, ppBuildRangeInfos); + + simple_mtx_lock(&device->rra_trace.data_mtx); for (uint32_t i = 0; i < infoCount; ++i) { RADV_FROM_HANDLE(vk_acceleration_structure, structure, pInfos[i].dstAccelerationStructure); - struct hash_entry *entry = _mesa_hash_table_search(cmd_buffer->device->rra_trace.accel_structs, structure); + struct hash_entry *entry = _mesa_hash_table_search(device->rra_trace.accel_structs, structure); assert(entry); struct radv_rra_accel_struct_data *data = entry->data; handle_accel_struct_write(commandBuffer, structure, data); } - simple_mtx_unlock(&cmd_buffer->device->rra_trace.data_mtx); + simple_mtx_unlock(&device->rra_trace.data_mtx); } VKAPI_ATTR void VKAPI_CALL rra_CmdCopyAccelerationStructureKHR(VkCommandBuffer commandBuffer, const VkCopyAccelerationStructureInfoKHR *pInfo) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - cmd_buffer->device->layer_dispatch.rra.CmdCopyAccelerationStructureKHR(commandBuffer, pInfo); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); - simple_mtx_lock(&cmd_buffer->device->rra_trace.data_mtx); + device->layer_dispatch.rra.CmdCopyAccelerationStructureKHR(commandBuffer, pInfo); + + simple_mtx_lock(&device->rra_trace.data_mtx); RADV_FROM_HANDLE(vk_acceleration_structure, structure, pInfo->dst); - struct hash_entry *entry = _mesa_hash_table_search(cmd_buffer->device->rra_trace.accel_structs, structure); + struct hash_entry *entry = _mesa_hash_table_search(device->rra_trace.accel_structs, structure); assert(entry); struct radv_rra_accel_struct_data *data = entry->data; handle_accel_struct_write(commandBuffer, structure, data); - simple_mtx_unlock(&cmd_buffer->device->rra_trace.data_mtx); + simple_mtx_unlock(&device->rra_trace.data_mtx); } VKAPI_ATTR void VKAPI_CALL @@ -284,19 +288,21 @@ rra_CmdCopyMemoryToAccelerationStructureKHR(VkCommandBuffer commandBuffer, const VkCopyMemoryToAccelerationStructureInfoKHR *pInfo) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - cmd_buffer->device->layer_dispatch.rra.CmdCopyMemoryToAccelerationStructureKHR(commandBuffer, pInfo); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); - simple_mtx_lock(&cmd_buffer->device->rra_trace.data_mtx); + device->layer_dispatch.rra.CmdCopyMemoryToAccelerationStructureKHR(commandBuffer, pInfo); + + simple_mtx_lock(&device->rra_trace.data_mtx); RADV_FROM_HANDLE(vk_acceleration_structure, structure, pInfo->dst); - struct hash_entry *entry = _mesa_hash_table_search(cmd_buffer->device->rra_trace.accel_structs, structure); + struct hash_entry *entry = _mesa_hash_table_search(device->rra_trace.accel_structs, structure); assert(entry); struct radv_rra_accel_struct_data *data = entry->data; handle_accel_struct_write(commandBuffer, structure, data); - simple_mtx_unlock(&cmd_buffer->device->rra_trace.data_mtx); + simple_mtx_unlock(&device->rra_trace.data_mtx); } VKAPI_ATTR void VKAPI_CALL diff --git a/src/amd/vulkan/layers/radv_sqtt_layer.c b/src/amd/vulkan/layers/radv_sqtt_layer.c index e33b6099244..d525eea15f7 100644 --- a/src/amd/vulkan/layers/radv_sqtt_layer.c +++ b/src/amd/vulkan/layers/radv_sqtt_layer.c @@ -35,13 +35,14 @@ void radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv_graphics_pipeline *pipeline) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const enum amd_gfx_level gfx_level = pdev->info.gfx_level; struct radv_sqtt_shaders_reloc *reloc = pipeline->sqtt_shaders_reloc; struct radeon_cmdbuf *cs = cmd_buffer->cs; uint64_t va; - radv_cs_add_buffer(cmd_buffer->device->ws, cs, reloc->bo); + radv_cs_add_buffer(device->ws, cs, reloc->bo); /* VS */ if (pipeline->base.shaders[MESA_SHADER_VERTEX]) { @@ -304,7 +305,9 @@ void radv_write_user_event_marker(struct radv_cmd_buffer *cmd_buffer, enum rgp_sqtt_marker_user_event_type type, const char *str) { - if (likely(!cmd_buffer->device->sqtt.bo)) + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + + if (likely(!device->sqtt.bo)) return; if (type == UserEventPop) { @@ -334,16 +337,17 @@ radv_write_user_event_marker(struct radv_cmd_buffer *cmd_buffer, enum rgp_sqtt_m void radv_describe_begin_cmd_buffer(struct radv_cmd_buffer *cmd_buffer) { - uint64_t device_id = (uintptr_t)cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + uint64_t device_id = (uintptr_t)device; struct rgp_sqtt_marker_cb_start marker = {0}; - if (likely(!cmd_buffer->device->sqtt.bo)) + if (likely(!device->sqtt.bo)) return; /* Reserve a command buffer ID for SQTT. */ - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + const struct radv_physical_device *pdev = radv_device_physical(device); enum amd_ip_type ip_type = radv_queue_family_to_ring(pdev, cmd_buffer->qf); - union rgp_sqtt_marker_cb_id cb_id = ac_sqtt_get_next_cmdbuf_id(&cmd_buffer->device->sqtt, ip_type); + union rgp_sqtt_marker_cb_id cb_id = ac_sqtt_get_next_cmdbuf_id(&device->sqtt, ip_type); cmd_buffer->sqtt_cb_id = cb_id.all; marker.identifier = RGP_SQTT_MARKER_IDENTIFIER_CB_START; @@ -365,10 +369,11 @@ radv_describe_begin_cmd_buffer(struct radv_cmd_buffer *cmd_buffer) void radv_describe_end_cmd_buffer(struct radv_cmd_buffer *cmd_buffer) { - uint64_t device_id = (uintptr_t)cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + uint64_t device_id = (uintptr_t)device; struct rgp_sqtt_marker_cb_end marker = {0}; - if (likely(!cmd_buffer->device->sqtt.bo)) + if (likely(!device->sqtt.bo)) return; marker.identifier = RGP_SQTT_MARKER_IDENTIFIER_CB_END; @@ -382,7 +387,9 @@ radv_describe_end_cmd_buffer(struct radv_cmd_buffer *cmd_buffer) void radv_describe_draw(struct radv_cmd_buffer *cmd_buffer) { - if (likely(!cmd_buffer->device->sqtt.bo)) + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + + if (likely(!device->sqtt.bo)) return; radv_write_event_marker(cmd_buffer, cmd_buffer->state.current_event_type, UINT_MAX, UINT_MAX, UINT_MAX); @@ -391,7 +398,9 @@ radv_describe_draw(struct radv_cmd_buffer *cmd_buffer) void radv_describe_dispatch(struct radv_cmd_buffer *cmd_buffer, const struct radv_dispatch_info *info) { - if (likely(!cmd_buffer->device->sqtt.bo)) + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + + if (likely(!device->sqtt.bo)) return; if (info->indirect) { @@ -430,9 +439,10 @@ radv_describe_end_render_pass_resolve(struct radv_cmd_buffer *cmd_buffer) void radv_describe_barrier_end_delayed(struct radv_cmd_buffer *cmd_buffer) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct rgp_sqtt_marker_barrier_end marker = {0}; - if (likely(!cmd_buffer->device->sqtt.bo) || !cmd_buffer->state.pending_sqtt_barrier_end) + if (likely(!device->sqtt.bo) || !cmd_buffer->state.pending_sqtt_barrier_end) return; cmd_buffer->state.pending_sqtt_barrier_end = false; @@ -483,9 +493,10 @@ radv_describe_barrier_end_delayed(struct radv_cmd_buffer *cmd_buffer) void radv_describe_barrier_start(struct radv_cmd_buffer *cmd_buffer, enum rgp_barrier_reason reason) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct rgp_sqtt_marker_barrier_start marker = {0}; - if (likely(!cmd_buffer->device->sqtt.bo)) + if (likely(!device->sqtt.bo)) return; if (cmd_buffer->state.in_barrier) { @@ -514,9 +525,10 @@ radv_describe_barrier_end(struct radv_cmd_buffer *cmd_buffer) void radv_describe_layout_transition(struct radv_cmd_buffer *cmd_buffer, const struct radv_barrier_data *barrier) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct rgp_sqtt_marker_layout_transition marker = {0}; - if (likely(!cmd_buffer->device->sqtt.bo)) + if (likely(!device->sqtt.bo)) return; if (!cmd_buffer->state.in_barrier) { @@ -542,7 +554,9 @@ radv_describe_layout_transition(struct radv_cmd_buffer *cmd_buffer, const struct void radv_describe_begin_accel_struct_build(struct radv_cmd_buffer *cmd_buffer, uint32_t count) { - if (likely(!cmd_buffer->device->sqtt.bo)) + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + + if (likely(!device->sqtt.bo)) return; char marker[64]; @@ -560,9 +574,10 @@ static void radv_describe_pipeline_bind(struct radv_cmd_buffer *cmd_buffer, VkPipelineBindPoint pipelineBindPoint, struct radv_pipeline *pipeline) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct rgp_sqtt_marker_pipeline_bind marker = {0}; - if (likely(!cmd_buffer->device->sqtt.bo)) + if (likely(!device->sqtt.bo)) return; marker.identifier = RGP_SQTT_MARKER_IDENTIFIER_BIND_PIPELINE; @@ -907,9 +922,10 @@ fail: #define EVENT_MARKER_BASE(cmd_name, api_name, event_name, ...) \ RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); \ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); \ radv_write_begin_general_api_marker(cmd_buffer, ApiCmd##api_name); \ cmd_buffer->state.current_event_type = EventCmd##event_name; \ - cmd_buffer->device->layer_dispatch.rgp.Cmd##cmd_name(__VA_ARGS__); \ + device->layer_dispatch.rgp.Cmd##cmd_name(__VA_ARGS__); \ cmd_buffer->state.current_event_type = EventInternalUnknown; \ radv_write_end_general_api_marker(cmd_buffer, ApiCmd##api_name); @@ -1159,8 +1175,9 @@ sqtt_CmdDrawMeshTasksIndirectCountEXT(VkCommandBuffer commandBuffer, VkBuffer bu #define API_MARKER_ALIAS(cmd_name, api_name, ...) \ RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); \ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); \ radv_write_begin_general_api_marker(cmd_buffer, ApiCmd##api_name); \ - cmd_buffer->device->layer_dispatch.rgp.Cmd##cmd_name(__VA_ARGS__); \ + device->layer_dispatch.rgp.Cmd##cmd_name(__VA_ARGS__); \ radv_write_end_general_api_marker(cmd_buffer, ApiCmd##api_name); #define API_MARKER(cmd_name, ...) API_MARKER_ALIAS(cmd_name, cmd_name, __VA_ARGS__); @@ -1349,27 +1366,33 @@ VKAPI_ATTR void VKAPI_CALL sqtt_CmdBeginDebugUtilsLabelEXT(VkCommandBuffer commandBuffer, const VkDebugUtilsLabelEXT *pLabelInfo) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + radv_write_user_event_marker(cmd_buffer, UserEventPush, pLabelInfo->pLabelName); - cmd_buffer->device->layer_dispatch.rgp.CmdBeginDebugUtilsLabelEXT(commandBuffer, pLabelInfo); + device->layer_dispatch.rgp.CmdBeginDebugUtilsLabelEXT(commandBuffer, pLabelInfo); } VKAPI_ATTR void VKAPI_CALL sqtt_CmdEndDebugUtilsLabelEXT(VkCommandBuffer commandBuffer) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + radv_write_user_event_marker(cmd_buffer, UserEventPop, NULL); - cmd_buffer->device->layer_dispatch.rgp.CmdEndDebugUtilsLabelEXT(commandBuffer); + device->layer_dispatch.rgp.CmdEndDebugUtilsLabelEXT(commandBuffer); } VKAPI_ATTR void VKAPI_CALL sqtt_CmdInsertDebugUtilsLabelEXT(VkCommandBuffer commandBuffer, const VkDebugUtilsLabelEXT *pLabelInfo) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + radv_write_user_event_marker(cmd_buffer, UserEventTrigger, pLabelInfo->pLabelName); - cmd_buffer->device->layer_dispatch.rgp.CmdInsertDebugUtilsLabelEXT(commandBuffer, pLabelInfo); + device->layer_dispatch.rgp.CmdInsertDebugUtilsLabelEXT(commandBuffer, pLabelInfo); } /* Pipelines */ diff --git a/src/amd/vulkan/meta/radv_meta_astc_decode.c b/src/amd/vulkan/meta/radv_meta_astc_decode.c index c03d77501ca..055f4a50687 100644 --- a/src/amd/vulkan/meta/radv_meta_astc_decode.c +++ b/src/amd/vulkan/meta/radv_meta_astc_decode.c @@ -58,7 +58,7 @@ static void decode_astc(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview, struct radv_image_view *dst_iview, VkImageLayout layout, const VkOffset3D *offset, const VkExtent3D *extent) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_state *state = &device->meta_state; struct vk_texcompress_astc_write_descriptor_set write_desc_set; VkFormat format = src_iview->image->vk.format; @@ -136,6 +136,7 @@ void radv_meta_decode_astc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout layout, const VkImageSubresourceLayers *subresource, VkOffset3D offset, VkExtent3D extent) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_COMPUTE_PIPELINE | RADV_META_SAVE_CONSTANTS | RADV_META_SAVE_DESCRIPTORS | @@ -150,12 +151,10 @@ radv_meta_decode_astc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *ima offset = vk_image_sanitize_offset(&image->vk, offset); struct radv_image_view src_iview, dst_iview; - image_view_init(cmd_buffer->device, image, VK_FORMAT_R32G32B32A32_UINT, VK_IMAGE_ASPECT_COLOR_BIT, - subresource->mipLevel, subresource->baseArrayLayer, - vk_image_subresource_layer_count(&image->vk, subresource), &src_iview); - image_view_init(cmd_buffer->device, image, VK_FORMAT_R8G8B8A8_UINT, VK_IMAGE_ASPECT_PLANE_1_BIT, - subresource->mipLevel, subresource->baseArrayLayer, - vk_image_subresource_layer_count(&image->vk, subresource), &dst_iview); + image_view_init(device, image, VK_FORMAT_R32G32B32A32_UINT, VK_IMAGE_ASPECT_COLOR_BIT, subresource->mipLevel, + subresource->baseArrayLayer, vk_image_subresource_layer_count(&image->vk, subresource), &src_iview); + image_view_init(device, image, VK_FORMAT_R8G8B8A8_UINT, VK_IMAGE_ASPECT_PLANE_1_BIT, subresource->mipLevel, + subresource->baseArrayLayer, vk_image_subresource_layer_count(&image->vk, subresource), &dst_iview); VkExtent3D extent_copy = { .width = extent.width, diff --git a/src/amd/vulkan/meta/radv_meta_blit.c b/src/amd/vulkan/meta/radv_meta_blit.c index 0f733fe8630..36b6b60c2e6 100644 --- a/src/amd/vulkan/meta/radv_meta_blit.c +++ b/src/amd/vulkan/meta/radv_meta_blit.c @@ -194,7 +194,7 @@ meta_emit_blit(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, struct radv_image *dst_image, struct radv_image_view *dst_iview, VkImageLayout dst_image_layout, VkRect2D dst_box, VkSampler sampler) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); uint32_t src_width = radv_minify(src_iview->image->vk.extent.width, src_iview->vk.base_mip_level); uint32_t src_height = radv_minify(src_iview->image->vk.extent.height, src_iview->vk.base_mip_level); uint32_t src_depth = radv_minify(src_iview->image->vk.extent.depth, src_iview->vk.base_mip_level); @@ -389,9 +389,9 @@ static void blit_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkImageLayout src_image_layout, struct radv_image *dst_image, VkImageLayout dst_image_layout, const VkImageBlit2 *region, VkFilter filter) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const VkImageSubresourceLayers *src_res = ®ion->srcSubresource; const VkImageSubresourceLayers *dst_res = ®ion->dstSubresource; - struct radv_device *device = cmd_buffer->device; struct radv_meta_saved_state saved_state; VkSampler sampler; @@ -515,7 +515,7 @@ blit_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkI /* 3D images have just 1 layer */ const uint32_t src_array_slice = src_image->vk.image_type == VK_IMAGE_TYPE_3D ? 0 : src_start + i; - radv_image_view_init(&dst_iview, cmd_buffer->device, + radv_image_view_init(&dst_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(dst_image), @@ -528,7 +528,7 @@ blit_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkI .layerCount = 1}, }, 0, NULL); - radv_image_view_init(&src_iview, cmd_buffer->device, + radv_image_view_init(&src_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(src_image), diff --git a/src/amd/vulkan/meta/radv_meta_blit2d.c b/src/amd/vulkan/meta/radv_meta_blit2d.c index ea973e47c9e..5dcdb9d74d8 100644 --- a/src/amd/vulkan/meta/radv_meta_blit2d.c +++ b/src/amd/vulkan/meta/radv_meta_blit2d.c @@ -49,6 +49,7 @@ static void create_iview(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *surf, struct radv_image_view *iview, VkFormat depth_format, VkImageAspectFlagBits aspects) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); VkFormat format; if (depth_format) @@ -56,7 +57,7 @@ create_iview(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *s else format = surf->format; - radv_image_view_init(iview, cmd_buffer->device, + radv_image_view_init(iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(surf->image), @@ -75,13 +76,14 @@ static void create_bview(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_buffer *src, struct radv_buffer_view *bview, VkFormat depth_format) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); VkFormat format; if (depth_format) format = depth_format; else format = src->format; - radv_buffer_view_init(bview, cmd_buffer->device, + radv_buffer_view_init(bview, device, &(VkBufferViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO, .flags = 0, @@ -102,7 +104,7 @@ blit2d_bind_src(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf struct radv_meta_blit2d_buffer *src_buf, struct blit2d_src_temps *tmp, enum blit2d_src_type src_type, VkFormat depth_format, VkImageAspectFlagBits aspects, uint32_t log2_samples) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); if (src_type == BLIT2D_SRC_TYPE_BUFFER) { create_bview(cmd_buffer, src_buf, &tmp->bview, depth_format); @@ -156,7 +158,8 @@ struct blit2d_dst_temps { static void bind_pipeline(struct radv_cmd_buffer *cmd_buffer, enum blit2d_src_type src_type, unsigned fs_key, uint32_t log2_samples) { - VkPipeline pipeline = cmd_buffer->device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + VkPipeline pipeline = device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]; radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); } @@ -164,7 +167,8 @@ bind_pipeline(struct radv_cmd_buffer *cmd_buffer, enum blit2d_src_type src_type, static void bind_depth_pipeline(struct radv_cmd_buffer *cmd_buffer, enum blit2d_src_type src_type, uint32_t log2_samples) { - VkPipeline pipeline = cmd_buffer->device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + VkPipeline pipeline = device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]; radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); } @@ -172,7 +176,8 @@ bind_depth_pipeline(struct radv_cmd_buffer *cmd_buffer, enum blit2d_src_type src static void bind_stencil_pipeline(struct radv_cmd_buffer *cmd_buffer, enum blit2d_src_type src_type, uint32_t log2_samples) { - VkPipeline pipeline = cmd_buffer->device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + VkPipeline pipeline = device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]; radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); } @@ -183,7 +188,7 @@ radv_meta_blit2d_normal_dst(struct radv_cmd_buffer *cmd_buffer, struct radv_meta unsigned num_rects, struct radv_meta_blit2d_rect *rects, enum blit2d_src_type src_type, uint32_t log2_samples) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); for (unsigned r = 0; r < num_rects; ++r) { radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, diff --git a/src/amd/vulkan/meta/radv_meta_buffer.c b/src/amd/vulkan/meta/radv_meta_buffer.c index 104d1ef6721..e82e9781ff1 100644 --- a/src/amd/vulkan/meta/radv_meta_buffer.c +++ b/src/amd/vulkan/meta/radv_meta_buffer.c @@ -158,7 +158,7 @@ radv_device_finish_meta_buffer_state(struct radv_device *device) static void fill_buffer_shader(struct radv_cmd_buffer *cmd_buffer, uint64_t va, uint64_t size, uint32_t data) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_COMPUTE_PIPELINE | RADV_META_SAVE_CONSTANTS); @@ -185,7 +185,7 @@ fill_buffer_shader(struct radv_cmd_buffer *cmd_buffer, uint64_t va, uint64_t siz static void copy_buffer_shader(struct radv_cmd_buffer *cmd_buffer, uint64_t src_va, uint64_t dst_va, uint64_t size) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_COMPUTE_PIPELINE | RADV_META_SAVE_CONSTANTS); @@ -231,17 +231,18 @@ uint32_t radv_fill_buffer(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *image, struct radeon_winsys_bo *bo, uint64_t va, uint64_t size, uint32_t value) { - bool use_compute = radv_prefer_compute_dma(cmd_buffer->device, size, NULL, bo); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + bool use_compute = radv_prefer_compute_dma(device, size, NULL, bo); uint32_t flush_bits = 0; assert(!(va & 3)); assert(!(size & 3)); if (bo) - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, bo); if (cmd_buffer->qf == RADV_QUEUE_TRANSFER) { - radv_sdma_fill_buffer(cmd_buffer->device, cmd_buffer->cs, va, size, value); + radv_sdma_fill_buffer(device, cmd_buffer->cs, va, size, value); } else if (use_compute) { cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_WRITE_BIT, image); @@ -259,17 +260,18 @@ void radv_copy_buffer(struct radv_cmd_buffer *cmd_buffer, struct radeon_winsys_bo *src_bo, struct radeon_winsys_bo *dst_bo, uint64_t src_offset, uint64_t dst_offset, uint64_t size) { - bool use_compute = !(size & 3) && !(src_offset & 3) && !(dst_offset & 3) && - radv_prefer_compute_dma(cmd_buffer->device, size, src_bo, dst_bo); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + bool use_compute = + !(size & 3) && !(src_offset & 3) && !(dst_offset & 3) && radv_prefer_compute_dma(device, size, src_bo, dst_bo); uint64_t src_va = radv_buffer_get_va(src_bo) + src_offset; uint64_t dst_va = radv_buffer_get_va(dst_bo) + dst_offset; - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, src_bo); - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, dst_bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, src_bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, dst_bo); if (cmd_buffer->qf == RADV_QUEUE_TRANSFER) - radv_sdma_copy_buffer(cmd_buffer->device, cmd_buffer->cs, src_va, dst_va, size); + radv_sdma_copy_buffer(device, cmd_buffer->cs, src_va, dst_va, size); else if (use_compute) copy_buffer_shader(cmd_buffer, src_va, dst_va, size); else if (size) @@ -323,13 +325,14 @@ radv_CmdCopyBuffer2(VkCommandBuffer commandBuffer, const VkCopyBufferInfo2 *pCop void radv_update_buffer_cp(struct radv_cmd_buffer *cmd_buffer, uint64_t va, const void *data, uint64_t size) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); uint64_t words = size / 4; bool mec = radv_cmd_buffer_uses_mec(cmd_buffer); assert(size < RADV_BUFFER_UPDATE_THRESHOLD); radv_emit_cache_flush(cmd_buffer); - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, words + 4); + radeon_check_space(device->ws, cmd_buffer->cs, words + 4); radeon_emit(cmd_buffer->cs, PKT3(PKT3_WRITE_DATA, 2 + words, 0)); radeon_emit(cmd_buffer->cs, @@ -338,7 +341,7 @@ radv_update_buffer_cp(struct radv_cmd_buffer *cmd_buffer, uint64_t va, const voi radeon_emit(cmd_buffer->cs, va >> 32); radeon_emit_array(cmd_buffer->cs, data, words); - if (radv_device_fault_detection_enabled(cmd_buffer->device)) + if (radv_device_fault_detection_enabled(device)) radv_cmd_buffer_trace_emit(cmd_buffer); } @@ -348,6 +351,7 @@ radv_CmdUpdateBuffer(VkCommandBuffer commandBuffer, VkBuffer dstBuffer, VkDevice { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_buffer, dst_buffer, dstBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); uint64_t va = radv_buffer_get_va(dst_buffer->bo); va += dstOffset + dst_buffer->offset; @@ -358,7 +362,7 @@ radv_CmdUpdateBuffer(VkCommandBuffer commandBuffer, VkBuffer dstBuffer, VkDevice return; if (dataSize < RADV_BUFFER_UPDATE_THRESHOLD && cmd_buffer->qf != RADV_QUEUE_TRANSFER) { - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, dst_buffer->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, dst_buffer->bo); radv_update_buffer_cp(cmd_buffer, va, pData, dataSize); } else { uint32_t buf_offset; diff --git a/src/amd/vulkan/meta/radv_meta_bufimage.c b/src/amd/vulkan/meta/radv_meta_bufimage.c index 67df8b003f5..1972745ce08 100644 --- a/src/amd/vulkan/meta/radv_meta_bufimage.c +++ b/src/amd/vulkan/meta/radv_meta_bufimage.c @@ -1059,10 +1059,12 @@ static void create_iview(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *surf, struct radv_image_view *iview, VkFormat format, VkImageAspectFlagBits aspects) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + if (format == VK_FORMAT_UNDEFINED) format = surf->format; - radv_image_view_init(iview, cmd_buffer->device, + radv_image_view_init(iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(surf->image), @@ -1084,7 +1086,9 @@ static void create_bview(struct radv_cmd_buffer *cmd_buffer, struct radv_buffer *buffer, unsigned offset, VkFormat format, struct radv_buffer_view *bview) { - radv_buffer_view_init(bview, cmd_buffer->device, + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + + radv_buffer_view_init(bview, device, &(VkBufferViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO, .flags = 0, @@ -1099,7 +1103,7 @@ static void create_buffer_from_image(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *surf, VkBufferUsageFlagBits2KHR usage, VkBuffer *buffer) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_device_memory mem; radv_device_memory_init(&mem, device, surf->image->bindings[0].bo); @@ -1133,6 +1137,7 @@ static void create_bview_for_r32g32b32(struct radv_cmd_buffer *cmd_buffer, struct radv_buffer *buffer, unsigned offset, VkFormat src_format, struct radv_buffer_view *bview) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); VkFormat format; switch (src_format) { @@ -1149,7 +1154,7 @@ create_bview_for_r32g32b32(struct radv_cmd_buffer *cmd_buffer, struct radv_buffe unreachable("invalid R32G32B32 format"); } - radv_buffer_view_init(bview, cmd_buffer->device, + radv_buffer_view_init(bview, device, &(VkBufferViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO, .flags = 0, @@ -1174,11 +1179,11 @@ fixup_gfx9_cs_copy(struct radv_cmd_buffer *cmd_buffer, const struct radv_meta_bl const struct radv_meta_blit2d_surf *img_bsurf, const struct radv_meta_blit2d_rect *rect, bool to_image) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const unsigned mip_level = img_bsurf->level; const struct radv_image *image = img_bsurf->image; const struct radeon_surf *surf = &image->planes[0].surface; - struct radv_device *device = cmd_buffer->device; const struct radeon_info *gpu_info = &pdev->info; struct ac_addrlib *addrlib = device->ws->get_addrlib(device->ws); struct ac_surf_info surf_info = radv_get_ac_surf_info(device, image); @@ -1244,7 +1249,8 @@ fixup_gfx9_cs_copy(struct radv_cmd_buffer *cmd_buffer, const struct radv_meta_bl static unsigned get_image_stride_for_r32g32b32(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *surf) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); unsigned stride; if (pdev->info.gfx_level >= GFX9) { @@ -1259,7 +1265,7 @@ get_image_stride_for_r32g32b32(struct radv_cmd_buffer *cmd_buffer, struct radv_m static void itob_bind_descriptors(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src, struct radv_buffer_view *dst) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); radv_meta_push_descriptor_set( cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, device->meta_state.itob.img_p_layout, 0, /* set */ @@ -1291,8 +1297,8 @@ void radv_meta_image_to_buffer(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src, struct radv_meta_blit2d_buffer *dst, unsigned num_rects, struct radv_meta_blit2d_rect *rects) { - VkPipeline pipeline = cmd_buffer->device->meta_state.itob.pipeline; - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + VkPipeline pipeline = device->meta_state.itob.pipeline; struct radv_image_view src_view; struct radv_buffer_view dst_view; @@ -1301,7 +1307,7 @@ radv_meta_image_to_buffer(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_b itob_bind_descriptors(cmd_buffer, &src_view, &dst_view); if (src->image->vk.image_type == VK_IMAGE_TYPE_3D) - pipeline = cmd_buffer->device->meta_state.itob.pipeline_3d; + pipeline = device->meta_state.itob.pipeline_3d; radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); @@ -1322,7 +1328,7 @@ static void btoi_r32g32b32_bind_descriptors(struct radv_cmd_buffer *cmd_buffer, struct radv_buffer_view *src, struct radv_buffer_view *dst) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); radv_meta_push_descriptor_set( cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, device->meta_state.btoi_r32g32b32.img_p_layout, 0, /* set */ @@ -1350,8 +1356,8 @@ radv_meta_buffer_to_image_cs_r32g32b32(struct radv_cmd_buffer *cmd_buffer, struc struct radv_meta_blit2d_surf *dst, unsigned num_rects, struct radv_meta_blit2d_rect *rects) { - VkPipeline pipeline = cmd_buffer->device->meta_state.btoi_r32g32b32.pipeline; - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + VkPipeline pipeline = device->meta_state.btoi_r32g32b32.pipeline; struct radv_buffer_view src_view, dst_view; unsigned dst_offset = 0; unsigned stride; @@ -1393,7 +1399,7 @@ radv_meta_buffer_to_image_cs_r32g32b32(struct radv_cmd_buffer *cmd_buffer, struc static void btoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer, struct radv_buffer_view *src, struct radv_image_view *dst) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); radv_meta_push_descriptor_set( cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, device->meta_state.btoi.img_p_layout, 0, /* set */ @@ -1424,8 +1430,8 @@ void radv_meta_buffer_to_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_buffer *src, struct radv_meta_blit2d_surf *dst, unsigned num_rects, struct radv_meta_blit2d_rect *rects) { - VkPipeline pipeline = cmd_buffer->device->meta_state.btoi.pipeline; - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + VkPipeline pipeline = device->meta_state.btoi.pipeline; struct radv_buffer_view src_view; struct radv_image_view dst_view; @@ -1440,7 +1446,7 @@ radv_meta_buffer_to_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_met btoi_bind_descriptors(cmd_buffer, &src_view, &dst_view); if (dst->image->vk.image_type == VK_IMAGE_TYPE_3D) - pipeline = cmd_buffer->device->meta_state.btoi.pipeline_3d; + pipeline = device->meta_state.btoi.pipeline_3d; radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); for (unsigned r = 0; r < num_rects; ++r) { @@ -1465,7 +1471,7 @@ static void itoi_r32g32b32_bind_descriptors(struct radv_cmd_buffer *cmd_buffer, struct radv_buffer_view *src, struct radv_buffer_view *dst) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); radv_meta_push_descriptor_set( cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, device->meta_state.itoi_r32g32b32.img_p_layout, 0, /* set */ @@ -1493,8 +1499,8 @@ radv_meta_image_to_image_cs_r32g32b32(struct radv_cmd_buffer *cmd_buffer, struct struct radv_meta_blit2d_surf *dst, unsigned num_rects, struct radv_meta_blit2d_rect *rects) { - VkPipeline pipeline = cmd_buffer->device->meta_state.itoi_r32g32b32.pipeline; - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + VkPipeline pipeline = device->meta_state.itoi_r32g32b32.pipeline; struct radv_buffer_view src_view, dst_view; unsigned src_offset = 0, dst_offset = 0; unsigned src_stride, dst_stride; @@ -1539,7 +1545,7 @@ radv_meta_image_to_image_cs_r32g32b32(struct radv_cmd_buffer *cmd_buffer, struct static void itoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src, struct radv_image_view *dst) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, device->meta_state.itoi.img_p_layout, 0, /* set */ @@ -1575,7 +1581,7 @@ void radv_meta_image_to_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src, struct radv_meta_blit2d_surf *dst, unsigned num_rects, struct radv_meta_blit2d_rect *rects) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_image_view src_view, dst_view; uint32_t samples = src->image->vk.samples; uint32_t samples_log2 = ffs(samples) - 1; @@ -1599,9 +1605,9 @@ radv_meta_image_to_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta itoi_bind_descriptors(cmd_buffer, &src_view, &dst_view); - VkPipeline pipeline = cmd_buffer->device->meta_state.itoi.pipeline[samples_log2]; + VkPipeline pipeline = device->meta_state.itoi.pipeline[samples_log2]; if (src->image->vk.image_type == VK_IMAGE_TYPE_3D || dst->image->vk.image_type == VK_IMAGE_TYPE_3D) - pipeline = cmd_buffer->device->meta_state.itoi.pipeline_3d; + pipeline = device->meta_state.itoi.pipeline_3d; radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); for (unsigned r = 0; r < num_rects; ++r) { @@ -1622,7 +1628,7 @@ radv_meta_image_to_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta static void cleari_r32g32b32_bind_descriptors(struct radv_cmd_buffer *cmd_buffer, struct radv_buffer_view *view) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, device->meta_state.cleari_r32g32b32.img_p_layout, 0, /* set */ @@ -1641,8 +1647,8 @@ static void radv_meta_clear_image_cs_r32g32b32(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *dst, const VkClearColorValue *clear_color) { - VkPipeline pipeline = cmd_buffer->device->meta_state.cleari_r32g32b32.pipeline; - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + VkPipeline pipeline = device->meta_state.cleari_r32g32b32.pipeline; struct radv_buffer_view dst_view; unsigned stride; VkBuffer buffer; @@ -1679,7 +1685,7 @@ radv_meta_clear_image_cs_r32g32b32(struct radv_cmd_buffer *cmd_buffer, struct ra static void cleari_bind_descriptors(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *dst_iview) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, device->meta_state.cleari.img_p_layout, 0, /* set */ @@ -1705,7 +1711,7 @@ void radv_meta_clear_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *dst, const VkClearColorValue *clear_color) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_image_view dst_iview; uint32_t samples = dst->image->vk.samples; uint32_t samples_log2 = ffs(samples) - 1; @@ -1719,9 +1725,9 @@ radv_meta_clear_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_bl create_iview(cmd_buffer, dst, &dst_iview, VK_FORMAT_UNDEFINED, dst->aspect_mask); cleari_bind_descriptors(cmd_buffer, &dst_iview); - VkPipeline pipeline = cmd_buffer->device->meta_state.cleari.pipeline[samples_log2]; + VkPipeline pipeline = device->meta_state.cleari.pipeline[samples_log2]; if (dst->image->vk.image_type == VK_IMAGE_TYPE_3D) - pipeline = cmd_buffer->device->meta_state.cleari.pipeline_3d; + pipeline = device->meta_state.cleari.pipeline_3d; radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); diff --git a/src/amd/vulkan/meta/radv_meta_clear.c b/src/amd/vulkan/meta/radv_meta_clear.c index 1163687e314..c9bb51d2e46 100644 --- a/src/amd/vulkan/meta/radv_meta_clear.c +++ b/src/amd/vulkan/meta/radv_meta_clear.c @@ -291,7 +291,7 @@ static void emit_color_clear(struct radv_cmd_buffer *cmd_buffer, const VkClearAttachment *clear_att, const VkClearRect *clear_rect, uint32_t view_mask) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_rendering_state *render = &cmd_buffer->state.render; uint32_t samples, samples_log2; VkFormat format; @@ -486,7 +486,8 @@ static VkPipeline pick_depthstencil_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_state *meta_state, int samples_log2, VkImageAspectFlags aspects, bool fast) { - bool unrestricted = cmd_buffer->device->vk.enabled_extensions.EXT_depth_range_unrestricted; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + bool unrestricted = device->vk.enabled_extensions.EXT_depth_range_unrestricted; int index = fast ? DEPTH_CLEAR_FAST : DEPTH_CLEAR_SLOW; VkPipeline *pipeline; @@ -508,8 +509,7 @@ pick_depthstencil_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_ } if (*pipeline == VK_NULL_HANDLE) { - VkResult ret = - create_depthstencil_pipeline(cmd_buffer->device, aspects, 1u << samples_log2, index, unrestricted, pipeline); + VkResult ret = create_depthstencil_pipeline(device, aspects, 1u << samples_log2, index, unrestricted, pipeline); if (ret != VK_SUCCESS) { vk_command_buffer_set_error(&cmd_buffer->vk, ret); return VK_NULL_HANDLE; @@ -523,7 +523,7 @@ emit_depthstencil_clear(struct radv_cmd_buffer *cmd_buffer, VkClearDepthStencilV VkImageAspectFlags aspects, const VkClearRect *clear_rect, uint32_t view_mask, bool can_fast_clear) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_state *meta_state = &device->meta_state; const struct radv_rendering_state *render = &cmd_buffer->state.render; uint32_t samples, samples_log2; @@ -547,7 +547,7 @@ emit_depthstencil_clear(struct radv_cmd_buffer *cmd_buffer, VkClearDepthStencilV if (!(aspects & VK_IMAGE_ASPECT_DEPTH_BIT)) clear_value.depth = 1.0f; - if (cmd_buffer->device->vk.enabled_extensions.EXT_depth_range_unrestricted) { + if (device->vk.enabled_extensions.EXT_depth_range_unrestricted) { vk_common_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), device->meta_state.clear_depth_unrestricted_p_layout, VK_SHADER_STAGE_FRAGMENT_BIT, 0, 4, &clear_value.depth); @@ -596,7 +596,7 @@ static uint32_t clear_htile_mask(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *image, struct radeon_winsys_bo *bo, uint64_t offset, uint64_t size, uint32_t htile_value, uint32_t htile_mask) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_state *state = &device->meta_state; uint64_t block_count = DIV_ROUND_UP(size, 1024); struct radv_meta_saved_state saved_state; @@ -721,10 +721,12 @@ radv_can_fast_clear_depth(struct radv_cmd_buffer *cmd_buffer, const struct radv_ VkImageLayout image_layout, VkImageAspectFlags aspects, const VkClearRect *clear_rect, const VkClearDepthStencilValue clear_value, uint32_t view_mask) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + if (!iview || !iview->support_fast_clear) return false; - if (!radv_layout_is_htile_compressed(cmd_buffer->device, iview->image, image_layout, + if (!radv_layout_is_htile_compressed(device, iview->image, image_layout, radv_image_queue_family_mask(iview->image, cmd_buffer->qf, cmd_buffer->qf))) return false; @@ -740,8 +742,8 @@ radv_can_fast_clear_depth(struct radv_cmd_buffer *cmd_buffer, const struct radv_ if (!view_mask && clear_rect->layerCount != iview->image->vk.array_layers) return false; - if (cmd_buffer->device->vk.enabled_extensions.EXT_depth_range_unrestricted && - (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) && (clear_value.depth < 0.0 || clear_value.depth > 1.0)) + if (device->vk.enabled_extensions.EXT_depth_range_unrestricted && (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) && + (clear_value.depth < 0.0 || clear_value.depth > 1.0)) return false; if (radv_image_is_tc_compat_htile(iview->image) && @@ -765,9 +767,10 @@ radv_fast_clear_depth(struct radv_cmd_buffer *cmd_buffer, const struct radv_imag VkClearDepthStencilValue clear_value, VkImageAspectFlags aspects, enum radv_cmd_flush_bits *pre_flush, enum radv_cmd_flush_bits *post_flush) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); uint32_t clear_word, flush_bits; - clear_word = radv_get_htile_fast_clear_value(cmd_buffer->device, iview->image, clear_value); + clear_word = radv_get_htile_fast_clear_value(device, iview->image, clear_value); if (pre_flush) { enum radv_cmd_flush_bits bits = @@ -1139,7 +1142,8 @@ uint32_t radv_clear_cmask(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *range, uint32_t value) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); uint64_t offset = image->bindings[0].offset + image->planes[0].surface.cmask_offset; uint64_t size; @@ -1179,7 +1183,8 @@ uint32_t radv_clear_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *range, uint32_t value) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); uint32_t level_count = vk_image_subresource_level_count(&image->vk, range); uint32_t layer_count = vk_image_subresource_layer_count(&image->vk, range); uint32_t flush_bits = 0; @@ -1229,7 +1234,7 @@ static uint32_t radv_clear_dcc_comp_to_single(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *range, uint32_t color_values[4]) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); unsigned bytes_per_pixel = vk_format_get_blocksize(image->vk.format); unsigned layer_count = vk_image_subresource_layer_count(&image->vk, range); struct radv_meta_saved_state saved_state; @@ -1274,7 +1279,7 @@ radv_clear_dcc_comp_to_single(struct radv_cmd_buffer *cmd_buffer, struct radv_im width = radv_minify(image->vk.extent.width, range->baseMipLevel + l); height = radv_minify(image->vk.extent.height, range->baseMipLevel + l); - radv_image_view_init(&iview, cmd_buffer->device, + radv_image_view_init(&iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(image), @@ -1333,12 +1338,13 @@ uint32_t radv_clear_htile(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *image, const VkImageSubresourceRange *range, uint32_t value) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); uint32_t level_count = vk_image_subresource_level_count(&image->vk, range); uint32_t flush_bits = 0; uint32_t htile_mask; - htile_mask = radv_get_htile_mask(cmd_buffer->device, image, range->aspectMask); + htile_mask = radv_get_htile_mask(device, image, range->aspectMask); if (level_count != image->vk.mip_levels) { assert(pdev->info.gfx_level >= GFX10); @@ -1609,13 +1615,14 @@ radv_can_fast_clear_color(struct radv_cmd_buffer *cmd_buffer, const struct radv_ VkImageLayout image_layout, const VkClearRect *clear_rect, VkClearColorValue clear_value, uint32_t view_mask) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); uint32_t clear_color[2]; if (!iview || !iview->support_fast_clear) return false; - if (!radv_layout_can_fast_clear(cmd_buffer->device, iview->image, iview->vk.base_mip_level, image_layout, + if (!radv_layout_can_fast_clear(device, iview->image, iview->vk.base_mip_level, image_layout, radv_image_queue_family_mask(iview->image, cmd_buffer->qf, cmd_buffer->qf))) return false; @@ -1647,11 +1654,10 @@ radv_can_fast_clear_color(struct radv_cmd_buffer *cmd_buffer, const struct radv_ uint32_t reset_value; if (pdev->info.gfx_level >= GFX11) { - if (!gfx11_get_fast_clear_parameters(cmd_buffer->device, iview, &clear_value, &reset_value)) + if (!gfx11_get_fast_clear_parameters(device, iview, &clear_value, &reset_value)) return false; } else { - gfx8_get_fast_clear_parameters(cmd_buffer->device, iview, &clear_value, &reset_value, - &can_avoid_fast_clear_elim); + gfx8_get_fast_clear_parameters(device, iview, &clear_value, &reset_value, &can_avoid_fast_clear_elim); } if (iview->image->vk.mip_levels > 1) { @@ -1685,7 +1691,8 @@ radv_fast_clear_color(struct radv_cmd_buffer *cmd_buffer, const struct radv_imag const VkClearAttachment *clear_att, enum radv_cmd_flush_bits *pre_flush, enum radv_cmd_flush_bits *post_flush) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); VkClearColorValue clear_value = clear_att->clearValue.color; uint32_t clear_color[4], flush_bits = 0; uint32_t cmask_clear_value; @@ -1717,11 +1724,10 @@ radv_fast_clear_color(struct radv_cmd_buffer *cmd_buffer, const struct radv_imag bool can_avoid_fast_clear_elim = true; if (pdev->info.gfx_level >= GFX11) { - ASSERTED bool result = gfx11_get_fast_clear_parameters(cmd_buffer->device, iview, &clear_value, &reset_value); + ASSERTED bool result = gfx11_get_fast_clear_parameters(device, iview, &clear_value, &reset_value); assert(result); } else { - gfx8_get_fast_clear_parameters(cmd_buffer->device, iview, &clear_value, &reset_value, - &can_avoid_fast_clear_elim); + gfx8_get_fast_clear_parameters(device, iview, &clear_value, &reset_value, &can_avoid_fast_clear_elim); } if (radv_image_has_cmask(iview->image)) { @@ -1733,7 +1739,7 @@ radv_fast_clear_color(struct radv_cmd_buffer *cmd_buffer, const struct radv_imag flush_bits |= radv_clear_dcc(cmd_buffer, iview->image, &range, reset_value); - if (reset_value == radv_dcc_single_clear_value(cmd_buffer->device)) { + if (reset_value == radv_dcc_single_clear_value(device)) { /* Write the clear color to the first byte of each 256B block when the image supports DCC * fast clears with comp-to-single. */ @@ -1941,11 +1947,12 @@ radv_clear_image_layer(struct radv_cmd_buffer *cmd_buffer, struct radv_image *im const VkImageSubresourceRange *range, VkFormat format, int level, unsigned layer_count, const VkClearValue *clear_val) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_image_view iview; uint32_t width = radv_minify(image->vk.extent.width, range->baseMipLevel + level); uint32_t height = radv_minify(image->vk.extent.height, range->baseMipLevel + level); - radv_image_view_init(&iview, cmd_buffer->device, + radv_image_view_init(&iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(image), @@ -2018,10 +2025,11 @@ static bool radv_fast_clear_range(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkFormat format, VkImageLayout image_layout, const VkImageSubresourceRange *range, const VkClearValue *clear_val) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_image_view iview; bool fast_cleared = false; - radv_image_view_init(&iview, cmd_buffer->device, + radv_image_view_init(&iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(image), @@ -2080,7 +2088,8 @@ radv_cmd_clear_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *imag const VkClearValue *clear_value, uint32_t range_count, const VkImageSubresourceRange *ranges, bool cs) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); VkFormat format = image->vk.format; VkClearValue internal_clear_value; @@ -2104,7 +2113,7 @@ radv_cmd_clear_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *imag const VkImageSubresourceRange *range = &ranges[r]; /* Don't use compressed image stores because they will use an incompatible format. */ - if (radv_layout_dcc_compressed(cmd_buffer->device, image, range->baseMipLevel, image_layout, queue_mask)) { + if (radv_layout_dcc_compressed(device, image, range->baseMipLevel, image_layout, queue_mask)) { disable_compression = cs; break; } @@ -2169,10 +2178,11 @@ radv_CmdClearColorImage(VkCommandBuffer commandBuffer, VkImage image_h, VkImageL { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_image, image, image_h); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; bool cs; - cs = cmd_buffer->qf == RADV_QUEUE_COMPUTE || !radv_image_is_renderable(cmd_buffer->device, image); + cs = cmd_buffer->qf == RADV_QUEUE_COMPUTE || !radv_image_is_renderable(device, image); /* Clear commands (except vkCmdClearAttachments) should not be affected by conditional rendering. */ diff --git a/src/amd/vulkan/meta/radv_meta_copy.c b/src/amd/vulkan/meta/radv_meta_copy.c index 95c6f1152e5..2183c74d2fa 100644 --- a/src/amd/vulkan/meta/radv_meta_copy.c +++ b/src/amd/vulkan/meta/radv_meta_copy.c @@ -76,7 +76,7 @@ alloc_transfer_temp_bo(struct radv_cmd_buffer *cmd_buffer) if (cmd_buffer->transfer.copy_temp) return true; - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const VkResult r = radv_bo_create(device, RADV_SDMA_TRANSFER_TEMP_BYTES, 4096, RADEON_DOMAIN_VRAM, RADEON_FLAG_NO_CPU_ACCESS | RADEON_FLAG_NO_INTERPROCESS_SHARING, RADV_BO_PRIORITY_SCRATCH, 0, true, &cmd_buffer->transfer.copy_temp); @@ -94,7 +94,7 @@ static void transfer_copy_buffer_image(struct radv_cmd_buffer *cmd_buffer, struct radv_buffer *buffer, struct radv_image *image, const VkBufferImageCopy2 *region, bool to_image) { - const struct radv_device *device = cmd_buffer->device; + const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radeon_cmdbuf *cs = cmd_buffer->cs; const VkImageAspectFlags aspect_mask = region->imageSubresource.aspectMask; const unsigned binding_idx = image->disjoint ? radv_plane_from_aspect(aspect_mask) : 0; @@ -127,6 +127,7 @@ copy_buffer_to_image(struct radv_cmd_buffer *cmd_buffer, struct radv_buffer *buf return; } + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; bool cs; @@ -135,7 +136,7 @@ copy_buffer_to_image(struct radv_cmd_buffer *cmd_buffer, struct radv_buffer *buf */ assert(image->vk.samples == 1); - cs = cmd_buffer->qf == RADV_QUEUE_COMPUTE || !radv_image_is_renderable(cmd_buffer->device, image); + cs = cmd_buffer->qf == RADV_QUEUE_COMPUTE || !radv_image_is_renderable(device, image); /* VK_EXT_conditional_rendering says that copy commands should not be * affected by conditional rendering. @@ -170,7 +171,7 @@ copy_buffer_to_image(struct radv_cmd_buffer *cmd_buffer, struct radv_buffer *buf if (!radv_is_buffer_format_supported(img_bsurf.format, NULL)) { uint32_t queue_mask = radv_image_queue_family_mask(image, cmd_buffer->qf, cmd_buffer->qf); bool compressed = - radv_layout_dcc_compressed(cmd_buffer->device, image, region->imageSubresource.mipLevel, layout, queue_mask); + radv_layout_dcc_compressed(device, image, region->imageSubresource.mipLevel, layout, queue_mask); if (compressed) { radv_describe_barrier_start(cmd_buffer, RGP_BARRIER_UNKNOWN_REASON); @@ -239,7 +240,8 @@ radv_CmdCopyBufferToImage2(VkCommandBuffer commandBuffer, const VkCopyBufferToIm RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_buffer, src_buffer, pCopyBufferToImageInfo->srcBuffer); RADV_FROM_HANDLE(radv_image, dst_image, pCopyBufferToImageInfo->dstImage); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); for (unsigned r = 0; r < pCopyBufferToImageInfo->regionCount; r++) { copy_buffer_to_image(cmd_buffer, src_buffer, dst_image, pCopyBufferToImageInfo->dstImageLayout, @@ -273,7 +275,7 @@ static void copy_image_to_buffer(struct radv_cmd_buffer *cmd_buffer, struct radv_buffer *buffer, struct radv_image *image, VkImageLayout layout, const VkBufferImageCopy2 *region) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); if (cmd_buffer->qf == RADV_QUEUE_TRANSFER) { transfer_copy_buffer_image(cmd_buffer, buffer, image, region, false); return; @@ -389,7 +391,7 @@ static void transfer_copy_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkImageLayout src_image_layout, struct radv_image *dst_image, VkImageLayout dst_image_layout, const VkImageCopy2 *region) { - const struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radeon_cmdbuf *cs = cmd_buffer->cs; unsigned int dst_aspect_mask_remaining = region->dstSubresource.aspectMask; @@ -423,7 +425,8 @@ static void copy_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkImageLayout src_image_layout, struct radv_image *dst_image, VkImageLayout dst_image_layout, const VkImageCopy2 *region) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); if (cmd_buffer->qf == RADV_QUEUE_TRANSFER) { transfer_copy_image(cmd_buffer, src_image, src_image_layout, dst_image, dst_image_layout, region); @@ -448,7 +451,7 @@ copy_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkI assert(src_image->plane_count == 1 || util_is_power_of_two_nonzero(region->srcSubresource.aspectMask)); assert(dst_image->plane_count == 1 || util_is_power_of_two_nonzero(region->dstSubresource.aspectMask)); - cs = cmd_buffer->qf == RADV_QUEUE_COMPUTE || !radv_image_is_renderable(cmd_buffer->device, dst_image); + cs = cmd_buffer->qf == RADV_QUEUE_COMPUTE || !radv_image_is_renderable(device, dst_image); /* VK_EXT_conditional_rendering says that copy commands should not be * affected by conditional rendering. @@ -463,7 +466,7 @@ copy_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkI */ uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, cmd_buffer->qf); - if (radv_layout_is_htile_compressed(cmd_buffer->device, dst_image, dst_image_layout, queue_mask) && + if (radv_layout_is_htile_compressed(device, dst_image, dst_image_layout, queue_mask) && (region->dstOffset.x || region->dstOffset.y || region->dstOffset.z || region->extent.width != dst_image->vk.extent.width || region->extent.height != dst_image->vk.extent.height || region->extent.depth != dst_image->vk.extent.depth)) { @@ -495,11 +498,11 @@ copy_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkI dst_image, dst_image_layout, ®ion->dstSubresource, region->dstSubresource.aspectMask); uint32_t dst_queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, cmd_buffer->qf); - bool dst_compressed = radv_layout_dcc_compressed(cmd_buffer->device, dst_image, region->dstSubresource.mipLevel, - dst_image_layout, dst_queue_mask); + bool dst_compressed = + radv_layout_dcc_compressed(device, dst_image, region->dstSubresource.mipLevel, dst_image_layout, dst_queue_mask); uint32_t src_queue_mask = radv_image_queue_family_mask(src_image, cmd_buffer->qf, cmd_buffer->qf); - bool src_compressed = radv_layout_dcc_compressed(cmd_buffer->device, src_image, region->srcSubresource.mipLevel, - src_image_layout, src_queue_mask); + bool src_compressed = + radv_layout_dcc_compressed(device, src_image, region->srcSubresource.mipLevel, src_image_layout, src_queue_mask); bool need_dcc_sign_reinterpret = false; if (!src_compressed || @@ -590,7 +593,7 @@ copy_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkI /* Fixup HTILE after a copy on compute. */ uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, cmd_buffer->qf); - if (radv_layout_is_htile_compressed(cmd_buffer->device, dst_image, dst_image_layout, queue_mask)) { + if (radv_layout_is_htile_compressed(device, dst_image, dst_image_layout, queue_mask)) { cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE; VkImageSubresourceRange range = { @@ -601,7 +604,7 @@ copy_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkI .layerCount = vk_image_subresource_layer_count(&dst_image->vk, ®ion->dstSubresource), }; - uint32_t htile_value = radv_get_htile_initial_value(cmd_buffer->device, dst_image); + uint32_t htile_value = radv_get_htile_initial_value(device, dst_image); cmd_buffer->state.flush_bits |= radv_clear_htile(cmd_buffer, dst_image, &range, htile_value); } @@ -616,7 +619,8 @@ radv_CmdCopyImage2(VkCommandBuffer commandBuffer, const VkCopyImageInfo2 *pCopyI RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_image, src_image, pCopyImageInfo->srcImage); RADV_FROM_HANDLE(radv_image, dst_image, pCopyImageInfo->dstImage); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); for (unsigned r = 0; r < pCopyImageInfo->regionCount; r++) { copy_image(cmd_buffer, src_image, pCopyImageInfo->srcImageLayout, dst_image, pCopyImageInfo->dstImageLayout, diff --git a/src/amd/vulkan/meta/radv_meta_copy_vrs_htile.c b/src/amd/vulkan/meta/radv_meta_copy_vrs_htile.c index 6de669d33c1..b6f80d71357 100644 --- a/src/amd/vulkan/meta/radv_meta_copy_vrs_htile.c +++ b/src/amd/vulkan/meta/radv_meta_copy_vrs_htile.c @@ -193,15 +193,15 @@ void radv_copy_vrs_htile(struct radv_cmd_buffer *cmd_buffer, struct radv_image *vrs_image, const VkRect2D *rect, struct radv_image *dst_image, struct radv_buffer *htile_buffer, bool read_htile_value) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_state *state = &device->meta_state; struct radv_meta_saved_state saved_state; struct radv_image_view vrs_iview; assert(radv_image_has_htile(dst_image)); - if (!cmd_buffer->device->meta_state.copy_vrs_htile_pipeline) { - VkResult ret = radv_device_init_meta_copy_vrs_htile_state(cmd_buffer->device, &dst_image->planes[0].surface); + if (!device->meta_state.copy_vrs_htile_pipeline) { + VkResult ret = radv_device_init_meta_copy_vrs_htile_state(device, &dst_image->planes[0].surface); if (ret != VK_SUCCESS) { vk_command_buffer_set_error(&cmd_buffer->vk, ret); return; @@ -218,7 +218,7 @@ radv_copy_vrs_htile(struct radv_cmd_buffer *cmd_buffer, struct radv_image *vrs_i radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE, state->copy_vrs_htile_pipeline); - radv_image_view_init(&vrs_iview, cmd_buffer->device, + radv_image_view_init(&vrs_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(vrs_image), diff --git a/src/amd/vulkan/meta/radv_meta_dcc_retile.c b/src/amd/vulkan/meta/radv_meta_dcc_retile.c index 7348df7abe8..938a2663768 100644 --- a/src/amd/vulkan/meta/radv_meta_dcc_retile.c +++ b/src/amd/vulkan/meta/radv_meta_dcc_retile.c @@ -172,7 +172,7 @@ void radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image) { struct radv_meta_saved_state saved_state; - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_buffer buffer; assert(image->vk.image_type == VK_IMAGE_TYPE_2D); @@ -186,8 +186,8 @@ radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image) unsigned swizzle_mode = image->planes[0].surface.u.gfx9.swizzle_mode; /* Compile pipelines if not already done so. */ - if (!cmd_buffer->device->meta_state.dcc_retile.pipeline[swizzle_mode]) { - VkResult ret = radv_device_init_meta_dcc_retile_state(cmd_buffer->device, &image->planes[0].surface); + if (!device->meta_state.dcc_retile.pipeline[swizzle_mode]) { + VkResult ret = radv_device_init_meta_dcc_retile_state(device, &image->planes[0].surface); if (ret != VK_SUCCESS) { vk_command_buffer_set_error(&cmd_buffer->vk, ret); return; @@ -204,7 +204,7 @@ radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image) struct radv_buffer_view views[2]; VkBufferView view_handles[2]; - radv_buffer_view_init(views, cmd_buffer->device, + radv_buffer_view_init(views, device, &(VkBufferViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO, .buffer = radv_buffer_to_handle(&buffer), @@ -212,7 +212,7 @@ radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image) .range = image->planes[0].surface.meta_size, .format = VK_FORMAT_R8_UINT, }); - radv_buffer_view_init(views + 1, cmd_buffer->device, + radv_buffer_view_init(views + 1, device, &(VkBufferViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO, .buffer = radv_buffer_to_handle(&buffer), diff --git a/src/amd/vulkan/meta/radv_meta_decompress.c b/src/amd/vulkan/meta/radv_meta_decompress.c index 4fd295a22bb..2d2a84f9586 100644 --- a/src/amd/vulkan/meta/radv_meta_decompress.c +++ b/src/amd/vulkan/meta/radv_meta_decompress.c @@ -344,7 +344,8 @@ static VkPipeline * radv_get_depth_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *subresourceRange, enum radv_depth_op op) { - struct radv_meta_state *state = &cmd_buffer->device->meta_state; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + struct radv_meta_state *state = &device->meta_state; uint32_t samples = image->vk.samples; uint32_t samples_log2 = ffs(samples) - 1; VkPipeline *pipeline; @@ -352,14 +353,14 @@ radv_get_depth_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_image *i if (!state->depth_decomp[samples_log2].decompress_pipeline) { VkResult ret; - ret = create_pipeline(cmd_buffer->device, samples, state->depth_decomp[samples_log2].p_layout, DEPTH_DECOMPRESS, + ret = create_pipeline(device, samples, state->depth_decomp[samples_log2].p_layout, DEPTH_DECOMPRESS, &state->depth_decomp[samples_log2].decompress_pipeline); if (ret != VK_SUCCESS) { vk_command_buffer_set_error(&cmd_buffer->vk, ret); return NULL; } - ret = create_pipeline(cmd_buffer->device, samples, state->depth_decomp[samples_log2].p_layout, DEPTH_RESUMMARIZE, + ret = create_pipeline(device, samples, state->depth_decomp[samples_log2].p_layout, DEPTH_RESUMMARIZE, &state->depth_decomp[samples_log2].resummarize_pipeline); if (ret != VK_SUCCESS) { vk_command_buffer_set_error(&cmd_buffer->vk, ret); @@ -385,7 +386,7 @@ static void radv_process_depth_image_layer(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *range, int level, int layer) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_image_view iview; uint32_t width, height; @@ -503,10 +504,10 @@ static void radv_expand_depth_stencil_compute(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *subresourceRange) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; struct radv_image_view load_iview = {0}; struct radv_image_view store_iview = {0}; - struct radv_device *device = cmd_buffer->device; assert(radv_image_is_tc_compat_htile(image)); @@ -528,7 +529,7 @@ radv_expand_depth_stencil_compute(struct radv_cmd_buffer *cmd_buffer, struct rad height = radv_minify(image->vk.extent.height, subresourceRange->baseMipLevel + l); for (uint32_t s = 0; s < vk_image_subresource_layer_count(&image->vk, subresourceRange); s++) { - radv_image_view_init(&load_iview, cmd_buffer->device, + radv_image_view_init(&load_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(image), @@ -541,7 +542,7 @@ radv_expand_depth_stencil_compute(struct radv_cmd_buffer *cmd_buffer, struct rad .layerCount = 1}, }, 0, &(struct radv_image_view_extra_create_info){.enable_compression = true}); - radv_image_view_init(&store_iview, cmd_buffer->device, + radv_image_view_init(&store_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(image), @@ -598,7 +599,7 @@ radv_expand_depth_stencil_compute(struct radv_cmd_buffer *cmd_buffer, struct rad radv_src_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_WRITE_BIT, image); /* Initialize the HTILE metadata as "fully expanded". */ - uint32_t htile_value = radv_get_htile_initial_value(cmd_buffer->device, image); + uint32_t htile_value = radv_get_htile_initial_value(device, image); cmd_buffer->state.flush_bits |= radv_clear_htile(cmd_buffer, image, subresourceRange, htile_value); } diff --git a/src/amd/vulkan/meta/radv_meta_etc_decode.c b/src/amd/vulkan/meta/radv_meta_etc_decode.c index aa18016ddd3..2b2eaf0a079 100644 --- a/src/amd/vulkan/meta/radv_meta_etc_decode.c +++ b/src/amd/vulkan/meta/radv_meta_etc_decode.c @@ -61,7 +61,7 @@ radv_device_finish_meta_etc_decode_state(struct radv_device *device) static VkPipeline radv_get_etc_decode_pipeline(struct radv_cmd_buffer *cmd_buffer) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_state *state = &device->meta_state; VkResult ret; @@ -78,7 +78,7 @@ static void decode_etc(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview, struct radv_image_view *dst_iview, const VkOffset3D *offset, const VkExtent3D *extent) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); VkPipeline pipeline = radv_get_etc_decode_pipeline(cmd_buffer); radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, @@ -123,6 +123,7 @@ void radv_meta_decode_etc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout layout, const VkImageSubresourceLayers *subresource, VkOffset3D offset, VkExtent3D extent) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_COMPUTE_PIPELINE | RADV_META_SAVE_CONSTANTS | RADV_META_SAVE_DESCRIPTORS | @@ -139,7 +140,7 @@ radv_meta_decode_etc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *imag VkFormat load_format = vk_texcompress_etc2_load_format(image->vk.format); struct radv_image_view src_iview; radv_image_view_init( - &src_iview, cmd_buffer->device, + &src_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(image), @@ -159,7 +160,7 @@ radv_meta_decode_etc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *imag VkFormat store_format = vk_texcompress_etc2_store_format(image->vk.format); struct radv_image_view dst_iview; radv_image_view_init( - &dst_iview, cmd_buffer->device, + &dst_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(image), diff --git a/src/amd/vulkan/meta/radv_meta_fast_clear.c b/src/amd/vulkan/meta/radv_meta_fast_clear.c index 2e4e0501854..7910101f213 100644 --- a/src/amd/vulkan/meta/radv_meta_fast_clear.c +++ b/src/amd/vulkan/meta/radv_meta_fast_clear.c @@ -460,7 +460,7 @@ static void radv_process_color_image_layer(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *range, int level, int layer, bool flush_cb) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_image_view iview; uint32_t width, height; @@ -519,7 +519,7 @@ static void radv_process_color_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *subresourceRange, enum radv_color_op op) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; bool old_predicating = false; bool flush_cb = false; @@ -685,15 +685,15 @@ static void radv_decompress_dcc_compute(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *subresourceRange) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; struct radv_image_view load_iview = {0}; struct radv_image_view store_iview = {0}; - struct radv_device *device = cmd_buffer->device; cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_WRITE_BIT, image); - if (!cmd_buffer->device->meta_state.fast_clear_flush.cmask_eliminate_pipeline) { - VkResult ret = radv_device_init_meta_fast_clear_flush_state_internal(cmd_buffer->device); + if (!device->meta_state.fast_clear_flush.cmask_eliminate_pipeline) { + VkResult ret = radv_device_init_meta_fast_clear_flush_state_internal(device); if (ret != VK_SUCCESS) { vk_command_buffer_set_error(&cmd_buffer->vk, ret); return; @@ -716,7 +716,7 @@ radv_decompress_dcc_compute(struct radv_cmd_buffer *cmd_buffer, struct radv_imag height = radv_minify(image->vk.extent.height, subresourceRange->baseMipLevel + l); for (uint32_t s = 0; s < vk_image_subresource_layer_count(&image->vk, subresourceRange); s++) { - radv_image_view_init(&load_iview, cmd_buffer->device, + radv_image_view_init(&load_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(image), @@ -729,7 +729,7 @@ radv_decompress_dcc_compute(struct radv_cmd_buffer *cmd_buffer, struct radv_imag .layerCount = 1}, }, 0, &(struct radv_image_view_extra_create_info){.enable_compression = true}); - radv_image_view_init(&store_iview, cmd_buffer->device, + radv_image_view_init(&store_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(image), diff --git a/src/amd/vulkan/meta/radv_meta_fmask_copy.c b/src/amd/vulkan/meta/radv_meta_fmask_copy.c index 7d8308ee45d..fdffc20a456 100644 --- a/src/amd/vulkan/meta/radv_meta_fmask_copy.c +++ b/src/amd/vulkan/meta/radv_meta_fmask_copy.c @@ -232,7 +232,8 @@ radv_can_use_fmask_copy(struct radv_cmd_buffer *cmd_buffer, const struct radv_im const struct radv_image *dst_image, unsigned num_rects, const struct radv_meta_blit2d_rect *rects) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); /* TODO: Test on pre GFX10 chips. */ if (pdev->info.gfx_level < GFX10) @@ -274,7 +275,7 @@ void radv_fmask_copy(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src, struct radv_meta_blit2d_surf *dst) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_image_view src_iview, dst_iview; uint32_t samples = src->image->vk.samples; uint32_t samples_log2 = ffs(samples) - 1; @@ -286,7 +287,7 @@ radv_fmask_copy(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf } radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE, - cmd_buffer->device->meta_state.fmask_copy.pipeline[samples_log2]); + device->meta_state.fmask_copy.pipeline[samples_log2]); radv_image_view_init(&src_iview, device, &(VkImageViewCreateInfo){ @@ -322,9 +323,9 @@ radv_fmask_copy(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf }, 0, NULL); - radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, - cmd_buffer->device->meta_state.fmask_copy.p_layout, 0, /* set */ - 2, /* descriptorWriteCount */ + radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, device->meta_state.fmask_copy.p_layout, + 0, /* set */ + 2, /* descriptorWriteCount */ (VkWriteDescriptorSet[]){{.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = 0, .dstArrayElement = 0, diff --git a/src/amd/vulkan/meta/radv_meta_fmask_expand.c b/src/amd/vulkan/meta/radv_meta_fmask_expand.c index 37e3d51bb59..3c289f82623 100644 --- a/src/amd/vulkan/meta/radv_meta_fmask_expand.c +++ b/src/amd/vulkan/meta/radv_meta_fmask_expand.c @@ -73,7 +73,7 @@ void radv_expand_fmask_image_inplace(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *subresourceRange) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; const uint32_t samples = image->vk.samples; const uint32_t samples_log2 = ffs(samples) - 1; @@ -112,9 +112,9 @@ radv_expand_fmask_image_inplace(struct radv_cmd_buffer *cmd_buffer, struct radv_ }, 0, NULL); - radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, - cmd_buffer->device->meta_state.fmask_expand.p_layout, 0, /* set */ - 2, /* descriptorWriteCount */ + radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, device->meta_state.fmask_expand.p_layout, + 0, /* set */ + 2, /* descriptorWriteCount */ (VkWriteDescriptorSet[]){{.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = 0, .dstArrayElement = 0, diff --git a/src/amd/vulkan/meta/radv_meta_resolve.c b/src/amd/vulkan/meta/radv_meta_resolve.c index 2a20d84cb2f..19ceefe2c4a 100644 --- a/src/amd/vulkan/meta/radv_meta_resolve.c +++ b/src/amd/vulkan/meta/radv_meta_resolve.c @@ -229,7 +229,7 @@ static void emit_resolve(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *src_image, const struct radv_image *dst_image, VkFormat vk_format) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); VkCommandBuffer cmd_buffer_h = radv_cmd_buffer_to_handle(cmd_buffer); unsigned fs_key = radv_format_meta_fs_key(device, vk_format); @@ -331,7 +331,7 @@ radv_meta_resolve_hardware_image(struct radv_cmd_buffer *cmd_buffer, struct radv VkImageLayout src_image_layout, struct radv_image *dst_image, VkImageLayout dst_image_layout, const VkImageResolve2 *region) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_GRAPHICS_PIPELINE); @@ -369,8 +369,7 @@ radv_meta_resolve_hardware_image(struct radv_cmd_buffer *cmd_buffer, struct radv uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, cmd_buffer->qf); - if (radv_layout_dcc_compressed(cmd_buffer->device, dst_image, region->dstSubresource.mipLevel, dst_image_layout, - queue_mask)) { + if (radv_layout_dcc_compressed(device, dst_image, region->dstSubresource.mipLevel, dst_image_layout, queue_mask)) { VkImageSubresourceRange range = { .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, .baseMipLevel = region->dstSubresource.mipLevel, @@ -404,7 +403,7 @@ radv_meta_resolve_hardware_image(struct radv_cmd_buffer *cmd_buffer, struct radv } struct radv_image_view src_iview; - radv_image_view_init(&src_iview, cmd_buffer->device, + radv_image_view_init(&src_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(src_image), @@ -422,7 +421,7 @@ radv_meta_resolve_hardware_image(struct radv_cmd_buffer *cmd_buffer, struct radv 0, NULL); struct radv_image_view dst_iview; - radv_image_view_init(&dst_iview, cmd_buffer->device, + radv_image_view_init(&dst_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(dst_image), @@ -507,7 +506,8 @@ radv_CmdResolveImage2(VkCommandBuffer commandBuffer, const VkResolveImageInfo2 * RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_image, src_image, pResolveImageInfo->srcImage); RADV_FROM_HANDLE(radv_image, dst_image, pResolveImageInfo->dstImage); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); VkImageLayout src_image_layout = pResolveImageInfo->srcImageLayout; VkImageLayout dst_image_layout = pResolveImageInfo->dstImageLayout; enum radv_resolve_method resolve_method = pdev->info.gfx_level >= GFX11 ? RESOLVE_FRAGMENT : RESOLVE_HW; @@ -531,7 +531,7 @@ radv_CmdResolveImage2(VkCommandBuffer commandBuffer, const VkResolveImageInfo2 * for (uint32_t r = 0; r < pResolveImageInfo->regionCount; r++) { const VkImageResolve2 *region = &pResolveImageInfo->pRegions[r]; - radv_pick_resolve_method_images(cmd_buffer->device, src_image, src_image->vk.format, dst_image, + radv_pick_resolve_method_images(device, src_image, src_image->vk.format, dst_image, region->dstSubresource.mipLevel, dst_image_layout, cmd_buffer, &resolve_method); resolve_image(cmd_buffer, src_image, src_image_layout, dst_image, dst_image_layout, region, resolve_method); @@ -543,6 +543,7 @@ radv_cmd_buffer_resolve_rendering_hw(struct radv_cmd_buffer *cmd_buffer, struct VkImageLayout src_layout, struct radv_image_view *dst_iview, VkImageLayout dst_layout) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_GRAPHICS_PIPELINE | RADV_META_SAVE_RENDER); @@ -563,7 +564,7 @@ radv_cmd_buffer_resolve_rendering_hw(struct radv_cmd_buffer *cmd_buffer, struct struct radv_image *dst_img = dst_iview->image; uint32_t queue_mask = radv_image_queue_family_mask(dst_img, cmd_buffer->qf, cmd_buffer->qf); - if (radv_layout_dcc_compressed(cmd_buffer->device, dst_img, dst_iview->vk.base_mip_level, dst_layout, queue_mask)) { + if (radv_layout_dcc_compressed(device, dst_img, dst_iview->vk.base_mip_level, dst_layout, queue_mask)) { VkImageSubresourceRange range = { .aspectMask = dst_iview->vk.aspects, .baseMipLevel = dst_iview->vk.base_mip_level, @@ -603,8 +604,7 @@ radv_cmd_buffer_resolve_rendering_hw(struct radv_cmd_buffer *cmd_buffer, struct radv_CmdBeginRendering(radv_cmd_buffer_to_handle(cmd_buffer), &rendering_info); - VkResult ret = - build_resolve_pipeline(cmd_buffer->device, radv_format_meta_fs_key(cmd_buffer->device, dst_iview->vk.format)); + VkResult ret = build_resolve_pipeline(device, radv_format_meta_fs_key(device, dst_iview->vk.format)); if (ret != VK_SUCCESS) { vk_command_buffer_set_error(&cmd_buffer->vk, ret); return; @@ -623,7 +623,8 @@ radv_cmd_buffer_resolve_rendering_hw(struct radv_cmd_buffer *cmd_buffer, struct void radv_cmd_buffer_resolve_rendering(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_rendering_state *render = &cmd_buffer->state.render; enum radv_resolve_method resolve_method = pdev->info.gfx_level >= GFX11 ? RESOLVE_FRAGMENT : RESOLVE_HW; @@ -643,7 +644,7 @@ radv_cmd_buffer_resolve_rendering(struct radv_cmd_buffer *cmd_buffer) struct radv_image_view *src_iview = render->ds_att.iview; struct radv_image_view *dst_iview = render->ds_att.resolve_iview; - radv_pick_resolve_method_images(cmd_buffer->device, src_iview->image, src_iview->vk.format, dst_iview->image, + radv_pick_resolve_method_images(device, src_iview->image, src_iview->vk.format, dst_iview->image, dst_iview->vk.base_mip_level, VK_IMAGE_LAYOUT_UNDEFINED, cmd_buffer, &resolve_method); @@ -717,8 +718,8 @@ radv_cmd_buffer_resolve_rendering(struct radv_cmd_buffer *cmd_buffer) VkImageLayout dst_layout = render->color_att[i].resolve_layout; struct radv_image *dst_img = dst_iview->image; - radv_pick_resolve_method_images(cmd_buffer->device, src_img, src_iview->vk.format, dst_img, - dst_iview->vk.base_mip_level, dst_layout, cmd_buffer, &resolve_method); + radv_pick_resolve_method_images(device, src_img, src_iview->vk.format, dst_img, dst_iview->vk.base_mip_level, + dst_layout, cmd_buffer, &resolve_method); VkImageResolve2 region = { .sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2, .extent = diff --git a/src/amd/vulkan/meta/radv_meta_resolve_cs.c b/src/amd/vulkan/meta/radv_meta_resolve_cs.c index aa990ee7ab2..855d1e66f35 100644 --- a/src/amd/vulkan/meta/radv_meta_resolve_cs.c +++ b/src/amd/vulkan/meta/radv_meta_resolve_cs.c @@ -415,7 +415,7 @@ radv_device_finish_meta_resolve_compute_state(struct radv_device *device) static VkPipeline * radv_get_resolve_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_state *state = &device->meta_state; uint32_t samples = src_iview->image->vk.samples; uint32_t samples_log2 = ffs(samples) - 1; @@ -446,7 +446,7 @@ static void emit_resolve(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview, struct radv_image_view *dst_iview, const VkOffset2D *src_offset, const VkOffset2D *dst_offset, const VkExtent2D *resolve_extent) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); VkPipeline *pipeline; radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, @@ -497,7 +497,7 @@ emit_depth_stencil_resolve(struct radv_cmd_buffer *cmd_buffer, struct radv_image const VkExtent3D *resolve_extent, VkImageAspectFlags aspects, VkResolveModeFlagBits resolve_mode) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const uint32_t samples = src_iview->image->vk.samples; const uint32_t samples_log2 = ffs(samples) - 1; VkPipeline *pipeline; @@ -582,6 +582,7 @@ radv_meta_resolve_compute_image(struct radv_cmd_buffer *cmd_buffer, struct radv_ VkImageLayout src_image_layout, struct radv_image *dst_image, VkFormat dst_format, VkImageLayout dst_image_layout, const VkImageResolve2 *region) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; /* For partial resolves, DCC should be decompressed before resolving @@ -589,9 +590,8 @@ radv_meta_resolve_compute_image(struct radv_cmd_buffer *cmd_buffer, struct radv_ */ uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, cmd_buffer->qf); - if (!radv_image_use_dcc_image_stores(cmd_buffer->device, dst_image) && - radv_layout_dcc_compressed(cmd_buffer->device, dst_image, region->dstSubresource.mipLevel, dst_image_layout, - queue_mask) && + if (!radv_image_use_dcc_image_stores(device, dst_image) && + radv_layout_dcc_compressed(device, dst_image, region->dstSubresource.mipLevel, dst_image_layout, queue_mask) && (region->dstOffset.x || region->dstOffset.y || region->dstOffset.z || region->extent.width != dst_image->vk.extent.width || region->extent.height != dst_image->vk.extent.height || region->extent.depth != dst_image->vk.extent.depth)) { @@ -623,7 +623,7 @@ radv_meta_resolve_compute_image(struct radv_cmd_buffer *cmd_buffer, struct radv_ for (uint32_t layer = 0; layer < src_layer_count; ++layer) { struct radv_image_view src_iview; - radv_image_view_init(&src_iview, cmd_buffer->device, + radv_image_view_init(&src_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(src_image), @@ -641,7 +641,7 @@ radv_meta_resolve_compute_image(struct radv_cmd_buffer *cmd_buffer, struct radv_ 0, NULL); struct radv_image_view dst_iview; - radv_image_view_init(&dst_iview, cmd_buffer->device, + radv_image_view_init(&dst_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(dst_image), @@ -667,9 +667,8 @@ radv_meta_resolve_compute_image(struct radv_cmd_buffer *cmd_buffer, struct radv_ radv_meta_restore(&saved_state, cmd_buffer); - if (!radv_image_use_dcc_image_stores(cmd_buffer->device, dst_image) && - radv_layout_dcc_compressed(cmd_buffer->device, dst_image, region->dstSubresource.mipLevel, dst_image_layout, - queue_mask)) { + if (!radv_image_use_dcc_image_stores(device, dst_image) && + radv_layout_dcc_compressed(device, dst_image, region->dstSubresource.mipLevel, dst_image_layout, queue_mask)) { cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE; @@ -701,6 +700,7 @@ void radv_depth_stencil_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, VkImageAspectFlags aspects, VkResolveModeFlagBits resolve_mode) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_rendering_state *render = &cmd_buffer->state.render; VkRect2D resolve_area = render->area; struct radv_meta_saved_state saved_state; @@ -739,7 +739,7 @@ radv_depth_stencil_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, VkIm struct radv_image *dst_image = dst_iview->image; struct radv_image_view tsrc_iview; - radv_image_view_init(&tsrc_iview, cmd_buffer->device, + radv_image_view_init(&tsrc_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(src_image), @@ -757,7 +757,7 @@ radv_depth_stencil_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, VkIm 0, NULL); struct radv_image_view tdst_iview; - radv_image_view_init(&tdst_iview, cmd_buffer->device, + radv_image_view_init(&tdst_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(dst_image), @@ -783,7 +783,7 @@ radv_depth_stencil_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, VkIm uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, cmd_buffer->qf); - if (radv_layout_is_htile_compressed(cmd_buffer->device, dst_image, dst_layout, queue_mask)) { + if (radv_layout_is_htile_compressed(device, dst_image, dst_layout, queue_mask)) { VkImageSubresourceRange range = {0}; range.aspectMask = aspects; range.baseMipLevel = dst_iview->vk.base_mip_level; @@ -791,7 +791,7 @@ radv_depth_stencil_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, VkIm range.baseArrayLayer = dst_iview->vk.base_array_layer; range.layerCount = layer_count; - uint32_t htile_value = radv_get_htile_initial_value(cmd_buffer->device, dst_image); + uint32_t htile_value = radv_get_htile_initial_value(device, dst_image); cmd_buffer->state.flush_bits |= radv_clear_htile(cmd_buffer, dst_image, &range, htile_value); } diff --git a/src/amd/vulkan/meta/radv_meta_resolve_fs.c b/src/amd/vulkan/meta/radv_meta_resolve_fs.c index 572828fb641..dcc7a1def7c 100644 --- a/src/amd/vulkan/meta/radv_meta_resolve_fs.c +++ b/src/amd/vulkan/meta/radv_meta_resolve_fs.c @@ -559,8 +559,8 @@ static VkPipeline * radv_get_resolve_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview, struct radv_image_view *dst_iview) { - struct radv_device *device = cmd_buffer->device; - unsigned fs_key = radv_format_meta_fs_key(cmd_buffer->device, dst_iview->vk.format); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + unsigned fs_key = radv_format_meta_fs_key(device, dst_iview->vk.format); const uint32_t samples = src_iview->image->vk.samples; const uint32_t samples_log2 = ffs(samples) - 1; VkPipeline *pipeline; @@ -583,13 +583,13 @@ static void emit_resolve(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview, struct radv_image_view *dst_iview, const VkOffset2D *src_offset, const VkOffset2D *dst_offset) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); VkCommandBuffer cmd_buffer_h = radv_cmd_buffer_to_handle(cmd_buffer); VkPipeline *pipeline; radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, - cmd_buffer->device->meta_state.resolve_fragment.p_layout, 0, /* set */ - 1, /* descriptorWriteCount */ + device->meta_state.resolve_fragment.p_layout, 0, /* set */ + 1, /* descriptorWriteCount */ (VkWriteDescriptorSet[]){ {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = 0, @@ -632,14 +632,14 @@ emit_depth_stencil_resolve(struct radv_cmd_buffer *cmd_buffer, struct radv_image const VkExtent2D *resolve_extent, VkImageAspectFlags aspects, VkResolveModeFlagBits resolve_mode) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const uint32_t samples = src_iview->image->vk.samples; const uint32_t samples_log2 = ffs(samples) - 1; VkPipeline *pipeline; radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, - cmd_buffer->device->meta_state.resolve_fragment.p_layout, 0, /* set */ - 1, /* descriptorWriteCount */ + device->meta_state.resolve_fragment.p_layout, 0, /* set */ + 1, /* descriptorWriteCount */ (VkWriteDescriptorSet[]){ {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = 0, @@ -718,6 +718,7 @@ radv_meta_resolve_fragment_image(struct radv_cmd_buffer *cmd_buffer, struct radv VkImageLayout src_image_layout, struct radv_image *dst_image, VkImageLayout dst_image_layout, const VkImageResolve2 *region) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; unsigned dst_layout = radv_meta_dst_layout_from_layout(dst_image_layout); VkImageLayout layout = radv_meta_dst_layout_to_layout(dst_layout); @@ -751,7 +752,7 @@ radv_meta_resolve_fragment_image(struct radv_cmd_buffer *cmd_buffer, struct radv radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &resolve_area); struct radv_image_view src_iview; - radv_image_view_init(&src_iview, cmd_buffer->device, + radv_image_view_init(&src_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(src_image), @@ -769,7 +770,7 @@ radv_meta_resolve_fragment_image(struct radv_cmd_buffer *cmd_buffer, struct radv 0, NULL); struct radv_image_view dst_iview; - radv_image_view_init(&dst_iview, cmd_buffer->device, + radv_image_view_init(&dst_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(dst_image), @@ -871,6 +872,7 @@ void radv_depth_stencil_resolve_rendering_fs(struct radv_cmd_buffer *cmd_buffer, VkImageAspectFlags aspects, VkResolveModeFlagBits resolve_mode) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_rendering_state *render = &cmd_buffer->state.render; VkRect2D resolve_area = render->area; struct radv_meta_saved_state saved_state; @@ -930,7 +932,7 @@ radv_depth_stencil_resolve_rendering_fs(struct radv_cmd_buffer *cmd_buffer, VkIm radv_CmdBeginRendering(radv_cmd_buffer_to_handle(cmd_buffer), &rendering_info); struct radv_image_view tsrc_iview; - radv_image_view_init(&tsrc_iview, cmd_buffer->device, + radv_image_view_init(&tsrc_iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(src_image), diff --git a/src/amd/vulkan/radix_sort/radv_radix_sort.c b/src/amd/vulkan/radix_sort/radv_radix_sort.c index 004198912c8..d1f0bf07714 100644 --- a/src/amd/vulkan/radix_sort/radv_radix_sort.c +++ b/src/amd/vulkan/radix_sort/radv_radix_sort.c @@ -151,9 +151,11 @@ vkCmdPipelineBarrier(VkCommandBuffer commandBuffer, VkPipelineStageFlags srcStag const VkImageMemoryBarrier *pImageMemoryBarriers) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - cmd_buffer->device->vk.dispatch_table.CmdPipelineBarrier( - commandBuffer, srcStageMask, dstStageMask, dependencyFlags, memoryBarrierCount, pMemoryBarriers, - bufferMemoryBarrierCount, pBufferMemoryBarriers, imageMemoryBarrierCount, pImageMemoryBarriers); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + + device->vk.dispatch_table.CmdPipelineBarrier(commandBuffer, srcStageMask, dstStageMask, dependencyFlags, + memoryBarrierCount, pMemoryBarriers, bufferMemoryBarrierCount, + pBufferMemoryBarriers, imageMemoryBarrierCount, pImageMemoryBarriers); } VKAPI_ATTR void VKAPI_CALL @@ -161,21 +163,27 @@ vkCmdPushConstants(VkCommandBuffer commandBuffer, VkPipelineLayout layout, VkSha uint32_t offset, uint32_t size, const void *pValues) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - cmd_buffer->device->vk.dispatch_table.CmdPushConstants(commandBuffer, layout, stageFlags, offset, size, pValues); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + + device->vk.dispatch_table.CmdPushConstants(commandBuffer, layout, stageFlags, offset, size, pValues); } VKAPI_ATTR void VKAPI_CALL vkCmdBindPipeline(VkCommandBuffer commandBuffer, VkPipelineBindPoint pipelineBindPoint, VkPipeline pipeline) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - cmd_buffer->device->vk.dispatch_table.CmdBindPipeline(commandBuffer, pipelineBindPoint, pipeline); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + + device->vk.dispatch_table.CmdBindPipeline(commandBuffer, pipelineBindPoint, pipeline); } VKAPI_ATTR void VKAPI_CALL vkCmdDispatch(VkCommandBuffer commandBuffer, uint32_t groupCountX, uint32_t groupCountY, uint32_t groupCountZ) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - cmd_buffer->device->vk.dispatch_table.CmdDispatch(commandBuffer, groupCountX, groupCountY, groupCountZ); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + + device->vk.dispatch_table.CmdDispatch(commandBuffer, groupCountX, groupCountY, groupCountZ); } VKAPI_ATTR VkDeviceAddress VKAPI_CALL @@ -190,12 +198,16 @@ vkCmdFillBuffer(VkCommandBuffer commandBuffer, VkBuffer dstBuffer, VkDeviceSize uint32_t data) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - cmd_buffer->device->vk.dispatch_table.CmdFillBuffer(commandBuffer, dstBuffer, dstOffset, size, data); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + + device->vk.dispatch_table.CmdFillBuffer(commandBuffer, dstBuffer, dstOffset, size, data); } VKAPI_ATTR void VKAPI_CALL vkCmdDispatchIndirect(VkCommandBuffer commandBuffer, VkBuffer buffer, VkDeviceSize offset) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - cmd_buffer->device->vk.dispatch_table.CmdDispatchIndirect(commandBuffer, buffer, offset); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + + device->vk.dispatch_table.CmdDispatchIndirect(commandBuffer, buffer, offset); } diff --git a/src/amd/vulkan/radv_acceleration_structure.c b/src/amd/vulkan/radv_acceleration_structure.c index 0925ea5d9b5..8a886e6be55 100644 --- a/src/amd/vulkan/radv_acceleration_structure.c +++ b/src/amd/vulkan/radv_acceleration_structure.c @@ -733,13 +733,13 @@ build_leaves(VkCommandBuffer commandBuffer, uint32_t infoCount, bool updateable) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); radv_write_user_event_marker(cmd_buffer, UserEventPush, "leaves"); - cmd_buffer->device->vk.dispatch_table.CmdBindPipeline( - commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, - updateable ? cmd_buffer->device->meta_state.accel_struct_build.leaf_updateable_pipeline - : cmd_buffer->device->meta_state.accel_struct_build.leaf_pipeline); + device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, + updateable ? device->meta_state.accel_struct_build.leaf_updateable_pipeline + : device->meta_state.accel_struct_build.leaf_pipeline); for (uint32_t i = 0; i < infoCount; ++i) { if (bvh_states[i].config.internal_type == INTERNAL_BUILD_TYPE_UPDATE) @@ -764,7 +764,7 @@ build_leaves(VkCommandBuffer commandBuffer, uint32_t infoCount, leaf_consts.geom_data = fill_geometry_data(pInfos[i].type, &bvh_states[i], j, geom, build_range_info); - vk_common_CmdPushConstants(commandBuffer, cmd_buffer->device->meta_state.accel_struct_build.leaf_p_layout, + vk_common_CmdPushConstants(commandBuffer, device->meta_state.accel_struct_build.leaf_p_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(leaf_consts), &leaf_consts); radv_unaligned_dispatch(cmd_buffer, build_range_info->primitiveCount, 1, 1); @@ -782,11 +782,12 @@ morton_generate(VkCommandBuffer commandBuffer, uint32_t infoCount, enum radv_cmd_flush_bits flush_bits) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); radv_write_user_event_marker(cmd_buffer, UserEventPush, "morton"); - cmd_buffer->device->vk.dispatch_table.CmdBindPipeline( - commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, cmd_buffer->device->meta_state.accel_struct_build.morton_pipeline); + device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, + device->meta_state.accel_struct_build.morton_pipeline); for (uint32_t i = 0; i < infoCount; ++i) { if (bvh_states[i].config.internal_type == INTERNAL_BUILD_TYPE_UPDATE) @@ -797,7 +798,7 @@ morton_generate(VkCommandBuffer commandBuffer, uint32_t infoCount, .ids = pInfos[i].scratchData.deviceAddress + bvh_states[i].scratch.sort_buffer_offset[0], }; - vk_common_CmdPushConstants(commandBuffer, cmd_buffer->device->meta_state.accel_struct_build.morton_p_layout, + vk_common_CmdPushConstants(commandBuffer, device->meta_state.accel_struct_build.morton_p_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(consts), &consts); radv_unaligned_dispatch(cmd_buffer, bvh_states[i].node_count, 1, 1); } @@ -814,10 +815,11 @@ morton_sort(VkCommandBuffer commandBuffer, uint32_t infoCount, { /* Copyright 2019 The Fuchsia Authors. */ RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); radv_write_user_event_marker(cmd_buffer, UserEventPush, "sort"); - radix_sort_vk_t *rs = cmd_buffer->device->meta_state.accel_struct_build.radix_sort; + radix_sort_vk_t *rs = device->meta_state.accel_struct_build.radix_sort; /* * OVERVIEW @@ -923,8 +925,8 @@ morton_sort(VkCommandBuffer commandBuffer, uint32_t infoCount, */ vk_barrier_transfer_w_to_compute_r(commandBuffer); - cmd_buffer->device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, - rs->pipelines.named.histogram); + device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, + rs->pipelines.named.histogram); for (uint32_t i = 0; i < infoCount; ++i) { if (!bvh_states[i].node_count) @@ -955,8 +957,7 @@ morton_sort(VkCommandBuffer commandBuffer, uint32_t infoCount, */ vk_barrier_compute_w_to_compute_r(commandBuffer); - cmd_buffer->device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, - rs->pipelines.named.prefix); + device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, rs->pipelines.named.prefix); for (uint32_t i = 0; i < infoCount; ++i) { if (!bvh_states[i].node_count) @@ -1002,7 +1003,7 @@ morton_sort(VkCommandBuffer commandBuffer, uint32_t infoCount, /* Bind new pipeline */ VkPipeline p = is_even ? rs->pipelines.named.scatter[pass_dword].even : rs->pipelines.named.scatter[pass_dword].odd; - cmd_buffer->device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, p); + device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, p); /* Update push constants that changed */ VkPipelineLayout pl = is_even ? rs->pipeline_layouts.named.scatter[pass_dword].even @@ -1044,12 +1045,12 @@ lbvh_build_internal(VkCommandBuffer commandBuffer, uint32_t infoCount, enum radv_cmd_flush_bits flush_bits) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); radv_write_user_event_marker(cmd_buffer, UserEventPush, "lbvh"); - cmd_buffer->device->vk.dispatch_table.CmdBindPipeline( - commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, - cmd_buffer->device->meta_state.accel_struct_build.lbvh_main_pipeline); + device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, + device->meta_state.accel_struct_build.lbvh_main_pipeline); for (uint32_t i = 0; i < infoCount; ++i) { if (bvh_states[i].config.internal_type != INTERNAL_BUILD_TYPE_LBVH) @@ -1066,7 +1067,7 @@ lbvh_build_internal(VkCommandBuffer commandBuffer, uint32_t infoCount, .internal_node_base = bvh_states[i].scratch.internal_node_offset - bvh_states[i].scratch.ir_offset, }; - vk_common_CmdPushConstants(commandBuffer, cmd_buffer->device->meta_state.accel_struct_build.lbvh_main_p_layout, + vk_common_CmdPushConstants(commandBuffer, device->meta_state.accel_struct_build.lbvh_main_p_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(consts), &consts); radv_unaligned_dispatch(cmd_buffer, internal_node_count, 1, 1); bvh_states[i].node_count = internal_node_count; @@ -1075,9 +1076,8 @@ lbvh_build_internal(VkCommandBuffer commandBuffer, uint32_t infoCount, cmd_buffer->state.flush_bits |= flush_bits; - cmd_buffer->device->vk.dispatch_table.CmdBindPipeline( - commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, - cmd_buffer->device->meta_state.accel_struct_build.lbvh_generate_ir_pipeline); + device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, + device->meta_state.accel_struct_build.lbvh_generate_ir_pipeline); for (uint32_t i = 0; i < infoCount; ++i) { if (bvh_states[i].config.internal_type != INTERNAL_BUILD_TYPE_LBVH) @@ -1090,8 +1090,7 @@ lbvh_build_internal(VkCommandBuffer commandBuffer, uint32_t infoCount, .internal_node_base = bvh_states[i].scratch.internal_node_offset - bvh_states[i].scratch.ir_offset, }; - vk_common_CmdPushConstants(commandBuffer, - cmd_buffer->device->meta_state.accel_struct_build.lbvh_generate_ir_p_layout, + vk_common_CmdPushConstants(commandBuffer, device->meta_state.accel_struct_build.lbvh_generate_ir_p_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(consts), &consts); radv_unaligned_dispatch(cmd_buffer, bvh_states[i].internal_node_count, 1, 1); } @@ -1104,11 +1103,12 @@ ploc_build_internal(VkCommandBuffer commandBuffer, uint32_t infoCount, const VkAccelerationStructureBuildGeometryInfoKHR *pInfos, struct bvh_state *bvh_states) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); radv_write_user_event_marker(cmd_buffer, UserEventPush, "ploc"); - cmd_buffer->device->vk.dispatch_table.CmdBindPipeline( - commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, cmd_buffer->device->meta_state.accel_struct_build.ploc_pipeline); + device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, + device->meta_state.accel_struct_build.ploc_pipeline); for (uint32_t i = 0; i < infoCount; ++i) { if (bvh_states[i].config.internal_type != INTERNAL_BUILD_TYPE_PLOC) @@ -1129,7 +1129,7 @@ ploc_build_internal(VkCommandBuffer commandBuffer, uint32_t infoCount, .internal_node_offset = bvh_states[i].scratch.internal_node_offset - bvh_states[i].scratch.ir_offset, }; - vk_common_CmdPushConstants(commandBuffer, cmd_buffer->device->meta_state.accel_struct_build.ploc_p_layout, + vk_common_CmdPushConstants(commandBuffer, device->meta_state.accel_struct_build.ploc_p_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(consts), &consts); vk_common_CmdDispatch(commandBuffer, MAX2(DIV_ROUND_UP(bvh_states[i].node_count, PLOC_WORKGROUP_SIZE), 1), 1, 1); } @@ -1142,13 +1142,13 @@ encode_nodes(VkCommandBuffer commandBuffer, uint32_t infoCount, const VkAccelerationStructureBuildGeometryInfoKHR *pInfos, struct bvh_state *bvh_states, bool compact) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); radv_write_user_event_marker(cmd_buffer, UserEventPush, "encode"); - cmd_buffer->device->vk.dispatch_table.CmdBindPipeline( - commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, - compact ? cmd_buffer->device->meta_state.accel_struct_build.encode_compact_pipeline - : cmd_buffer->device->meta_state.accel_struct_build.encode_pipeline); + device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, + compact ? device->meta_state.accel_struct_build.encode_compact_pipeline + : device->meta_state.accel_struct_build.encode_pipeline); for (uint32_t i = 0; i < infoCount; ++i) { if (compact != bvh_states[i].config.compact) @@ -1183,7 +1183,7 @@ encode_nodes(VkCommandBuffer commandBuffer, uint32_t infoCount, .leaf_node_count = bvh_states[i].leaf_node_count, .geometry_type = geometry_type, }; - vk_common_CmdPushConstants(commandBuffer, cmd_buffer->device->meta_state.accel_struct_build.encode_p_layout, + vk_common_CmdPushConstants(commandBuffer, device->meta_state.accel_struct_build.encode_p_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(args), &args); struct radv_dispatch_info dispatch = { @@ -1206,13 +1206,13 @@ init_header(VkCommandBuffer commandBuffer, uint32_t infoCount, struct radv_bvh_batch_state *batch_state) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); if (batch_state->any_compact) { radv_write_user_event_marker(cmd_buffer, UserEventPush, "header"); - cmd_buffer->device->vk.dispatch_table.CmdBindPipeline( - commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, - cmd_buffer->device->meta_state.accel_struct_build.header_pipeline); + device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, + device->meta_state.accel_struct_build.header_pipeline); } for (uint32_t i = 0; i < infoCount; ++i) { @@ -1234,7 +1234,7 @@ init_header(VkCommandBuffer commandBuffer, uint32_t infoCount, .instance_count = instance_count, }; - vk_common_CmdPushConstants(commandBuffer, cmd_buffer->device->meta_state.accel_struct_build.header_p_layout, + vk_common_CmdPushConstants(commandBuffer, device->meta_state.accel_struct_build.header_p_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(args), &args); radv_unaligned_dispatch(cmd_buffer, 1, 1, 1); @@ -1305,11 +1305,12 @@ update(VkCommandBuffer commandBuffer, uint32_t infoCount, const VkAccelerationSt const VkAccelerationStructureBuildRangeInfoKHR *const *ppBuildRangeInfos, struct bvh_state *bvh_states) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); radv_write_user_event_marker(cmd_buffer, UserEventPush, "update"); - cmd_buffer->device->vk.dispatch_table.CmdBindPipeline( - commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, cmd_buffer->device->meta_state.accel_struct_build.update_pipeline); + device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, + device->meta_state.accel_struct_build.update_pipeline); for (uint32_t i = 0; i < infoCount; ++i) { if (bvh_states[i].config.internal_type != INTERNAL_BUILD_TYPE_UPDATE) @@ -1339,7 +1340,7 @@ update(VkCommandBuffer commandBuffer, uint32_t infoCount, const VkAccelerationSt update_consts.geom_data = fill_geometry_data(pInfos[i].type, &bvh_states[i], j, geom, build_range_info); - vk_common_CmdPushConstants(commandBuffer, cmd_buffer->device->meta_state.accel_struct_build.update_p_layout, + vk_common_CmdPushConstants(commandBuffer, device->meta_state.accel_struct_build.update_p_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(update_consts), &update_consts); radv_unaligned_dispatch(cmd_buffer, build_range_info->primitiveCount, 1, 1); @@ -1357,9 +1358,10 @@ radv_CmdBuildAccelerationStructuresKHR(VkCommandBuffer commandBuffer, uint32_t i const VkAccelerationStructureBuildRangeInfoKHR *const *ppBuildRangeInfos) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; - VkResult result = radv_device_init_accel_struct_build_state(cmd_buffer->device); + VkResult result = radv_device_init_accel_struct_build_state(device); if (result != VK_SUCCESS) { vk_command_buffer_set_error(&cmd_buffer->vk, result); return; @@ -1384,8 +1386,7 @@ radv_CmdBuildAccelerationStructuresKHR(VkCommandBuffer commandBuffer, uint32_t i leaf_node_count += ppBuildRangeInfos[i][j].primitiveCount; } - get_build_layout(cmd_buffer->device, leaf_node_count, pInfos + i, &bvh_states[i].accel_struct, - &bvh_states[i].scratch); + get_build_layout(device, leaf_node_count, pInfos + i, &bvh_states[i].accel_struct, &bvh_states[i].scratch); struct build_config config = build_config(leaf_node_count, pInfos + i); bvh_states[i].config = config; @@ -1482,7 +1483,7 @@ radv_CmdBuildAccelerationStructuresKHR(VkCommandBuffer commandBuffer, uint32_t i init_header(commandBuffer, infoCount, pInfos, bvh_states, &batch_state); - if (cmd_buffer->device->rra_trace.accel_structs) + if (device->rra_trace.accel_structs) init_geometry_infos(commandBuffer, infoCount, pInfos, bvh_states, ppBuildRangeInfos); if (batch_state.any_update) @@ -1501,9 +1502,10 @@ radv_CmdCopyAccelerationStructureKHR(VkCommandBuffer commandBuffer, const VkCopy RADV_FROM_HANDLE(vk_acceleration_structure, src, pInfo->src); RADV_FROM_HANDLE(vk_acceleration_structure, dst, pInfo->dst); RADV_FROM_HANDLE(radv_buffer, src_buffer, src->buffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; - VkResult result = radv_device_init_accel_struct_copy_state(cmd_buffer->device); + VkResult result = radv_device_init_accel_struct_copy_state(device); if (result != VK_SUCCESS) { vk_command_buffer_set_error(&cmd_buffer->vk, result); return; @@ -1513,7 +1515,7 @@ radv_CmdCopyAccelerationStructureKHR(VkCommandBuffer commandBuffer, const VkCopy RADV_META_SAVE_COMPUTE_PIPELINE | RADV_META_SAVE_DESCRIPTORS | RADV_META_SAVE_CONSTANTS); radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE, - cmd_buffer->device->meta_state.accel_struct_build.copy_pipeline); + device->meta_state.accel_struct_build.copy_pipeline); struct copy_args consts = { .src_addr = vk_acceleration_structure_get_va(src), @@ -1522,8 +1524,8 @@ radv_CmdCopyAccelerationStructureKHR(VkCommandBuffer commandBuffer, const VkCopy }; vk_common_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), - cmd_buffer->device->meta_state.accel_struct_build.copy_p_layout, - VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(consts), &consts); + device->meta_state.accel_struct_build.copy_p_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, + sizeof(consts), &consts); cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT, NULL); @@ -1568,9 +1570,10 @@ radv_CmdCopyMemoryToAccelerationStructureKHR(VkCommandBuffer commandBuffer, { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(vk_acceleration_structure, dst, pInfo->dst); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; - VkResult result = radv_device_init_accel_struct_copy_state(cmd_buffer->device); + VkResult result = radv_device_init_accel_struct_copy_state(device); if (result != VK_SUCCESS) { vk_command_buffer_set_error(&cmd_buffer->vk, result); return; @@ -1580,7 +1583,7 @@ radv_CmdCopyMemoryToAccelerationStructureKHR(VkCommandBuffer commandBuffer, RADV_META_SAVE_COMPUTE_PIPELINE | RADV_META_SAVE_DESCRIPTORS | RADV_META_SAVE_CONSTANTS); radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE, - cmd_buffer->device->meta_state.accel_struct_build.copy_pipeline); + device->meta_state.accel_struct_build.copy_pipeline); const struct copy_args consts = { .src_addr = pInfo->src.deviceAddress, @@ -1589,8 +1592,8 @@ radv_CmdCopyMemoryToAccelerationStructureKHR(VkCommandBuffer commandBuffer, }; vk_common_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), - cmd_buffer->device->meta_state.accel_struct_build.copy_p_layout, - VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(consts), &consts); + device->meta_state.accel_struct_build.copy_p_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, + sizeof(consts), &consts); vk_common_CmdDispatch(commandBuffer, 512, 1, 1); radv_meta_restore(&saved_state, cmd_buffer); @@ -1603,10 +1606,11 @@ radv_CmdCopyAccelerationStructureToMemoryKHR(VkCommandBuffer commandBuffer, RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(vk_acceleration_structure, src, pInfo->src); RADV_FROM_HANDLE(radv_buffer, src_buffer, src->buffer); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_meta_saved_state saved_state; - VkResult result = radv_device_init_accel_struct_copy_state(cmd_buffer->device); + VkResult result = radv_device_init_accel_struct_copy_state(device); if (result != VK_SUCCESS) { vk_command_buffer_set_error(&cmd_buffer->vk, result); return; @@ -1616,7 +1620,7 @@ radv_CmdCopyAccelerationStructureToMemoryKHR(VkCommandBuffer commandBuffer, RADV_META_SAVE_COMPUTE_PIPELINE | RADV_META_SAVE_DESCRIPTORS | RADV_META_SAVE_CONSTANTS); radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE, - cmd_buffer->device->meta_state.accel_struct_build.copy_pipeline); + device->meta_state.accel_struct_build.copy_pipeline); const struct copy_args consts = { .src_addr = vk_acceleration_structure_get_va(src), @@ -1625,8 +1629,8 @@ radv_CmdCopyAccelerationStructureToMemoryKHR(VkCommandBuffer commandBuffer, }; vk_common_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), - cmd_buffer->device->meta_state.accel_struct_build.copy_p_layout, - VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(consts), &consts); + device->meta_state.accel_struct_build.copy_p_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, + sizeof(consts), &consts); cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT, NULL); diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c index 9a1e9b392e1..e9e5a4a8a7f 100644 --- a/src/amd/vulkan/radv_cmd_buffer.c +++ b/src/amd/vulkan/radv_cmd_buffer.c @@ -66,7 +66,8 @@ static void radv_handle_image_transition(struct radv_cmd_buffer *cmd_buffer, str static void radv_bind_dynamic_state(struct radv_cmd_buffer *cmd_buffer, const struct radv_dynamic_state *src) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_dynamic_state *dest = &cmd_buffer->state.dynamic; uint64_t copy_mask = src->mask; uint64_t dest_mask = 0; @@ -258,7 +259,8 @@ radv_bind_dynamic_state(struct radv_cmd_buffer *cmd_buffer, const struct radv_dy bool radv_cmd_buffer_uses_mec(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); return cmd_buffer->qf == RADV_QUEUE_COMPUTE && pdev->info.gfx_level >= GFX7; } @@ -285,7 +287,9 @@ static void radv_write_data(struct radv_cmd_buffer *cmd_buffer, const unsigned engine_sel, const uint64_t va, const unsigned count, const uint32_t *data, const bool predicating) { - radv_cs_write_data(cmd_buffer->device, cmd_buffer->cs, cmd_buffer->qf, engine_sel, va, count, data, predicating); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + + radv_cs_write_data(device, cmd_buffer->cs, cmd_buffer->qf, engine_sel, va, count, data, predicating); } static void @@ -321,28 +325,29 @@ static void radv_destroy_cmd_buffer(struct vk_command_buffer *vk_cmd_buffer) { struct radv_cmd_buffer *cmd_buffer = container_of(vk_cmd_buffer, struct radv_cmd_buffer, vk); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); if (cmd_buffer->qf != RADV_QUEUE_SPARSE) { util_dynarray_fini(&cmd_buffer->ray_history); list_for_each_entry_safe (struct radv_cmd_buffer_upload, up, &cmd_buffer->upload.list, list) { - radv_rmv_log_command_buffer_bo_destroy(cmd_buffer->device, up->upload_bo); - radv_bo_destroy(cmd_buffer->device, up->upload_bo); + radv_rmv_log_command_buffer_bo_destroy(device, up->upload_bo); + radv_bo_destroy(device, up->upload_bo); list_del(&up->list); free(up); } if (cmd_buffer->upload.upload_bo) { - radv_rmv_log_command_buffer_bo_destroy(cmd_buffer->device, cmd_buffer->upload.upload_bo); - radv_bo_destroy(cmd_buffer->device, cmd_buffer->upload.upload_bo); + radv_rmv_log_command_buffer_bo_destroy(device, cmd_buffer->upload.upload_bo); + radv_bo_destroy(device, cmd_buffer->upload.upload_bo); } if (cmd_buffer->cs) - cmd_buffer->device->ws->cs_destroy(cmd_buffer->cs); + device->ws->cs_destroy(cmd_buffer->cs); if (cmd_buffer->gang.cs) - cmd_buffer->device->ws->cs_destroy(cmd_buffer->gang.cs); + device->ws->cs_destroy(cmd_buffer->gang.cs); if (cmd_buffer->transfer.copy_temp) - radv_bo_destroy(cmd_buffer->device, cmd_buffer->transfer.copy_temp); + radv_bo_destroy(device, cmd_buffer->transfer.copy_temp); radv_cmd_buffer_finish_shader_part_cache(cmd_buffer); @@ -350,7 +355,7 @@ radv_destroy_cmd_buffer(struct vk_command_buffer *vk_cmd_buffer) struct radv_descriptor_set_header *set = &cmd_buffer->descriptors[i].push_set.set; free(set->mapped_ptr); if (set->layout) - vk_descriptor_set_layout_unref(&cmd_buffer->device->vk, &set->layout->vk); + vk_descriptor_set_layout_unref(&device->vk, &set->layout->vk); vk_object_base_finish(&set->base); } @@ -378,8 +383,6 @@ radv_create_cmd_buffer(struct vk_command_pool *pool, struct vk_command_buffer ** return result; } - cmd_buffer->device = device; - cmd_buffer->qf = vk_queue_to_radv(pdev, pool->queue_family_index); if (cmd_buffer->qf != RADV_QUEUE_SPARSE) { @@ -422,19 +425,20 @@ static void radv_reset_cmd_buffer(struct vk_command_buffer *vk_cmd_buffer, UNUSED VkCommandBufferResetFlags flags) { struct radv_cmd_buffer *cmd_buffer = container_of(vk_cmd_buffer, struct radv_cmd_buffer, vk); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); vk_command_buffer_reset(&cmd_buffer->vk); if (cmd_buffer->qf == RADV_QUEUE_SPARSE) return; - cmd_buffer->device->ws->cs_reset(cmd_buffer->cs); + device->ws->cs_reset(cmd_buffer->cs); if (cmd_buffer->gang.cs) - cmd_buffer->device->ws->cs_reset(cmd_buffer->gang.cs); + device->ws->cs_reset(cmd_buffer->gang.cs); list_for_each_entry_safe (struct radv_cmd_buffer_upload, up, &cmd_buffer->upload.list, list) { - radv_rmv_log_command_buffer_bo_destroy(cmd_buffer->device, up->upload_bo); - radv_bo_destroy(cmd_buffer->device, up->upload_bo); + radv_rmv_log_command_buffer_bo_destroy(device, up->upload_bo); + radv_bo_destroy(device, up->upload_bo); list_del(&up->list); free(up); } @@ -461,7 +465,7 @@ radv_reset_cmd_buffer(struct vk_command_buffer *vk_cmd_buffer, UNUSED VkCommandB cmd_buffer->has_indirect_pipeline_binds = false; if (cmd_buffer->upload.upload_bo) - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, cmd_buffer->upload.upload_bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, cmd_buffer->upload.upload_bo); cmd_buffer->upload.offset = 0; memset(cmd_buffer->vertex_binding_buffers, 0, sizeof(struct radv_buffer *) * cmd_buffer->used_vertex_bindings); @@ -484,10 +488,10 @@ const struct vk_command_buffer_ops radv_cmd_buffer_ops = { static bool radv_cmd_buffer_resize_upload_buf(struct radv_cmd_buffer *cmd_buffer, uint64_t min_needed) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); uint64_t new_size; struct radeon_winsys_bo *bo = NULL; struct radv_cmd_buffer_upload *upload; - struct radv_device *device = cmd_buffer->device; new_size = MAX2(min_needed, 16 * 1024); new_size = MAX2(new_size, 2 * cmd_buffer->upload.size); @@ -536,7 +540,8 @@ radv_cmd_buffer_upload_alloc_aligned(struct radv_cmd_buffer *cmd_buffer, unsigne { assert(size % 4 == 0); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radeon_info *gpu_info = &pdev->info; /* Align to the scalar cache line size if it results in this allocation @@ -585,7 +590,7 @@ radv_cmd_buffer_upload_data(struct radv_cmd_buffer *cmd_buffer, unsigned size, c void radv_cmd_buffer_trace_emit(struct radv_cmd_buffer *cmd_buffer) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radeon_cmdbuf *cs = cmd_buffer->cs; uint64_t va; @@ -599,7 +604,7 @@ radv_cmd_buffer_trace_emit(struct radv_cmd_buffer *cmd_buffer) ++cmd_buffer->state.trace_id; radv_write_data(cmd_buffer, V_370_ME, va, 1, &cmd_buffer->state.trace_id, false); - radeon_check_space(cmd_buffer->device->ws, cs, 2); + radeon_check_space(device->ws, cs, 2); radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); radeon_emit(cs, AC_ENCODE_TRACE_POINT(cmd_buffer->state.trace_id)); @@ -608,7 +613,9 @@ radv_cmd_buffer_trace_emit(struct radv_cmd_buffer *cmd_buffer) void radv_cmd_buffer_annotate(struct radv_cmd_buffer *cmd_buffer, const char *annotation) { - cmd_buffer->device->ws->cs_annotate(cmd_buffer->cs, annotation); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + + device->ws->cs_annotate(cmd_buffer->cs, annotation); } static void @@ -639,13 +646,14 @@ radv_gang_barrier(struct radv_cmd_buffer *cmd_buffer, VkPipelineStageFlags2 src_ void radv_gang_cache_flush(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *ace_cs = cmd_buffer->gang.cs; const uint32_t flush_bits = cmd_buffer->gang.flush_bits; enum rgp_flush_bits sqtt_flush_bits = 0; - radv_cs_emit_cache_flush(cmd_buffer->device->ws, ace_cs, pdev->info.gfx_level, NULL, 0, RADV_QUEUE_COMPUTE, - flush_bits, &sqtt_flush_bits, 0); + radv_cs_emit_cache_flush(device->ws, ace_cs, pdev->info.gfx_level, NULL, 0, RADV_QUEUE_COMPUTE, flush_bits, + &sqtt_flush_bits, 0); cmd_buffer->gang.flush_bits = 0; } @@ -686,12 +694,13 @@ ALWAYS_INLINE static bool radv_flush_gang_semaphore(struct radv_cmd_buffer *cmd_buffer, struct radeon_cmdbuf *cs, const enum radv_queue_family qf, const uint32_t va_off, const uint32_t value) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); if (!radv_gang_sem_init(cmd_buffer)) return false; - ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 12); + ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cs, 12); radv_cs_emit_write_event_eop(cs, pdev->info.gfx_level, qf, V_028A90_BOTTOM_OF_PIPE_TS, 0, EOP_DST_SEL_MEM, EOP_DATA_SEL_VALUE_32BIT, cmd_buffer->gang.sem.va + va_off, value, @@ -728,8 +737,10 @@ ALWAYS_INLINE static void radv_wait_gang_semaphore(struct radv_cmd_buffer *cmd_buffer, struct radeon_cmdbuf *cs, const enum radv_queue_family qf, const uint32_t va_off, const uint32_t value) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + assert(cmd_buffer->gang.sem.va); - radeon_check_space(cmd_buffer->device->ws, cs, 7); + radeon_check_space(device->ws, cs, 7); radv_cp_wait_mem(cs, qf, WAIT_REG_MEM_GREATER_OR_EQUAL, cmd_buffer->gang.sem.va + va_off, value, 0xffffffff); } @@ -750,10 +761,11 @@ radv_wait_gang_follower(struct radv_cmd_buffer *cmd_buffer) bool radv_gang_init(struct radv_cmd_buffer *cmd_buffer) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + if (cmd_buffer->gang.cs) return true; - struct radv_device *device = cmd_buffer->device; struct radeon_cmdbuf *ace_cs = device->ws->cs_create(device->ws, AMD_IP_COMPUTE, cmd_buffer->vk.level == VK_COMMAND_BUFFER_LEVEL_SECONDARY); @@ -769,8 +781,9 @@ radv_gang_init(struct radv_cmd_buffer *cmd_buffer) static VkResult radv_gang_finalize(struct radv_cmd_buffer *cmd_buffer) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + assert(cmd_buffer->gang.cs); - struct radv_device *device = cmd_buffer->device; struct radeon_cmdbuf *ace_cs = cmd_buffer->gang.cs; /* Emit pending cache flush. */ @@ -797,7 +810,7 @@ radv_gang_finalize(struct radv_cmd_buffer *cmd_buffer) static void radv_cmd_buffer_after_draw(struct radv_cmd_buffer *cmd_buffer, enum radv_cmd_flush_bits flags, bool dgc) { - const struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_physical_device *pdev = radv_device_physical(device); if (unlikely(device->sqtt.bo) && !dgc) { @@ -830,7 +843,7 @@ radv_cmd_buffer_after_draw(struct radv_cmd_buffer *cmd_buffer, enum radv_cmd_flu static void radv_save_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_physical_device *pdev = radv_device_physical(device); enum amd_ip_type ring; uint32_t data[2]; @@ -861,7 +874,7 @@ radv_save_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pip static void radv_save_vertex_descriptors(struct radv_cmd_buffer *cmd_buffer, uint64_t vb_ptr) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); uint32_t data[2]; uint64_t va; @@ -877,7 +890,7 @@ radv_save_vertex_descriptors(struct radv_cmd_buffer *cmd_buffer, uint64_t vb_ptr static void radv_save_vs_prolog(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader_part *prolog) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); uint32_t data[2]; uint64_t va; @@ -907,7 +920,7 @@ static void radv_save_descriptors(struct radv_cmd_buffer *cmd_buffer, VkPipelineBindPoint bind_point) { struct radv_descriptor_state *descriptors_state = radv_get_descriptors_state(cmd_buffer, bind_point); - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); uint32_t data[MAX_SETS * 2] = {0}; uint64_t va; va = radv_buffer_get_va(device->trace_bo) + 40; @@ -1212,7 +1225,8 @@ struct radv_bin_size_entry { static VkExtent2D radv_gfx10_compute_bin_size(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_rendering_state *render = &cmd_buffer->state.render; const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; VkExtent2D extent = {512, 512}; @@ -1296,7 +1310,8 @@ radv_gfx10_compute_bin_size(struct radv_cmd_buffer *cmd_buffer) static VkExtent2D radv_gfx9_compute_bin_size(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_rendering_state *render = &cmd_buffer->state.render; const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; static const struct radv_bin_size_entry color_size_table[][3][9] = { @@ -1562,7 +1577,8 @@ radv_gfx9_compute_bin_size(struct radv_cmd_buffer *cmd_buffer) static unsigned radv_get_disabled_binning_state(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_rendering_state *render = &cmd_buffer->state.render; const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; uint32_t pa_sc_binner_cntl_0; @@ -1603,7 +1619,7 @@ radv_get_disabled_binning_state(struct radv_cmd_buffer *cmd_buffer) static unsigned radv_get_binning_state(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_physical_device *pdev = radv_device_physical(device); unsigned pa_sc_binner_cntl_0; VkExtent2D bin_size; @@ -1639,7 +1655,8 @@ radv_get_binning_state(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_binning_state(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); unsigned pa_sc_binner_cntl_0; if (pdev->info.gfx_level < GFX9) @@ -1709,7 +1726,8 @@ radv_emit_prefetch_L2(struct radv_cmd_buffer *cmd_buffer, bool first_stage_only) static void radv_emit_rbplus_state(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); assert(pdev->info.rbplus_allowed); @@ -1865,7 +1883,7 @@ static void radv_emit_epilog(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *shader, const struct radv_shader_part *epilog) { - const struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; @@ -1885,7 +1903,7 @@ static void radv_emit_ps_epilog_state(struct radv_cmd_buffer *cmd_buffer, struct radv_shader_part *ps_epilog) { struct radv_shader *ps_shader = cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT]; - const struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); if (cmd_buffer->state.emitted_ps_epilog == ps_epilog) return; @@ -1918,7 +1936,7 @@ static void radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer) { struct radv_graphics_pipeline *pipeline = cmd_buffer->state.graphics_pipeline; - const struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_physical_device *pdev = radv_device_physical(device); if (cmd_buffer->state.emitted_graphics_pipeline == pipeline) @@ -1982,11 +2000,11 @@ radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer) if (!shader) continue; - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, shader->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, shader->bo); } if (cmd_buffer->state.gs_copy_shader) { - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, cmd_buffer->state.gs_copy_shader->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, cmd_buffer->state.gs_copy_shader->bo); } struct radv_shader *task_shader = cmd_buffer->state.shaders[MESA_SHADER_TASK]; @@ -2002,7 +2020,7 @@ radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer) } } - if (radv_device_fault_detection_enabled(cmd_buffer->device)) + if (radv_device_fault_detection_enabled(device)) radv_save_pipeline(cmd_buffer, &pipeline->base); cmd_buffer->state.emitted_graphics_pipeline = pipeline; @@ -2030,7 +2048,7 @@ radv_get_depth_clamp_mode(struct radv_cmd_buffer *cmd_buffer) { const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; bool depth_clip_enable = radv_get_depth_clip_enable(cmd_buffer); - const struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); enum radv_depth_clamp_mode mode; mode = RADV_DEPTH_CLAMP_MODE_VIEWPORT; @@ -2221,7 +2239,8 @@ radv_emit_depth_bias(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_line_stipple(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; enum amd_gfx_level gfx_level = pdev->info.gfx_level; /* GFX9 chips fail linestrip CTS tests unless this is set to 0 = no reset */ @@ -2239,7 +2258,8 @@ radv_emit_line_stipple(struct radv_cmd_buffer *cmd_buffer) static uint32_t radv_get_pa_su_sc_mode_cntl(const struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); enum amd_gfx_level gfx_level = pdev->info.gfx_level; const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; unsigned pa_su_sc_mode_cntl; @@ -2303,7 +2323,8 @@ radv_emit_provoking_vertex_mode(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_primitive_topology(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_shader *last_vgt_shader = cmd_buffer->state.last_vgt_shader; const struct radv_userdata_info *loc = radv_get_user_sgpr(last_vgt_shader, AC_UD_NUM_VERTS_PER_PRIM); const uint32_t vgt_gs_out_prim_type = radv_get_rasterization_prim(cmd_buffer); @@ -2318,7 +2339,7 @@ radv_emit_primitive_topology(struct radv_cmd_buffer *cmd_buffer) radeon_set_config_reg(cmd_buffer->cs, R_008958_VGT_PRIMITIVE_TYPE, d->vk.ia.primitive_topology); } - radv_emit_vgt_gs_out(cmd_buffer->device, cmd_buffer->cs, vgt_gs_out_prim_type); + radv_emit_vgt_gs_out(device, cmd_buffer->cs, vgt_gs_out_prim_type); if (loc->sgpr_idx == -1) return; @@ -2363,7 +2384,8 @@ radv_emit_stencil_control(struct radv_cmd_buffer *cmd_buffer) static bool radv_should_force_vrs1x1(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_shader *ps = cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT]; return pdev->info.gfx_level >= GFX10_3 && @@ -2373,13 +2395,14 @@ radv_should_force_vrs1x1(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_fragment_shading_rate(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; /* When per-vertex VRS is forced and the dynamic fragment shading rate is a no-op, ignore * it. This is needed for vkd3d-proton because it always declares per-draw VRS as dynamic. */ - if (cmd_buffer->device->force_vrs != RADV_FORCE_VRS_1x1 && d->vk.fsr.fragment_size.width == 1 && + if (device->force_vrs != RADV_FORCE_VRS_1x1 && d->vk.fsr.fragment_size.width == 1 && d->vk.fsr.fragment_size.height == 1 && d->vk.fsr.combiner_ops[0] == VK_FRAGMENT_SHADING_RATE_COMBINER_OP_KEEP_KHR && d->vk.fsr.combiner_ops[1] == VK_FRAGMENT_SHADING_RATE_COMBINER_OP_KEEP_KHR) @@ -2471,7 +2494,8 @@ radv_get_primitive_reset_index(const struct radv_cmd_buffer *cmd_buffer) static void radv_emit_primitive_restart_enable(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const enum amd_gfx_level gfx_level = pdev->info.gfx_level; const struct radv_dynamic_state *const d = &cmd_buffer->state.dynamic; struct radeon_cmdbuf *cs = cmd_buffer->cs; @@ -2531,7 +2555,8 @@ radv_is_mrt0_dual_src(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_logic_op(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; unsigned cb_color_control = 0; @@ -2574,7 +2599,7 @@ radv_emit_logic_op(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_color_write(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_binning_settings *settings = &pdev->binning_settings; const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; @@ -2600,7 +2625,8 @@ radv_emit_color_write(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_patch_control_points(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_shader *vs = radv_get_shader(cmd_buffer->state.shaders, MESA_SHADER_VERTEX); const struct radv_shader *tcs = cmd_buffer->state.shaders[MESA_SHADER_TESS_CTRL]; const struct radv_shader *tes = radv_get_shader(cmd_buffer->state.shaders, MESA_SHADER_TESS_EVAL); @@ -2687,7 +2713,8 @@ radv_emit_patch_control_points(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_conservative_rast_mode(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; if (pdev->info.gfx_level >= GFX9) { @@ -2730,7 +2757,8 @@ radv_emit_depth_clamp_enable(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_rasterization_samples(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); unsigned rasterization_samples = radv_get_rasterization_samples(cmd_buffer); unsigned ps_iter_samples = radv_get_ps_iter_samples(cmd_buffer); const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; @@ -2776,13 +2804,14 @@ static void radv_emit_fb_color_state(struct radv_cmd_buffer *cmd_buffer, int index, struct radv_color_buffer_info *cb, struct radv_image_view *iview, VkImageLayout layout) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); bool is_vi = pdev->info.gfx_level >= GFX8; uint32_t cb_fdcc_control = cb->cb_dcc_control; uint32_t cb_color_info = cb->cb_color_info; struct radv_image *image = iview->image; - if (!radv_layout_dcc_compressed(cmd_buffer->device, image, iview->vk.base_mip_level, layout, + if (!radv_layout_dcc_compressed(device, image, iview->vk.base_mip_level, layout, radv_image_queue_family_mask(image, cmd_buffer->qf, cmd_buffer->qf))) { if (pdev->info.gfx_level >= GFX11) { cb_fdcc_control &= C_028C78_FDCC_ENABLE; @@ -2792,7 +2821,7 @@ radv_emit_fb_color_state(struct radv_cmd_buffer *cmd_buffer, int index, struct r } const enum radv_fmask_compression fmask_comp = radv_layout_fmask_compression( - cmd_buffer->device, image, layout, radv_image_queue_family_mask(image, cmd_buffer->qf, cmd_buffer->qf)); + device, image, layout, radv_image_queue_family_mask(image, cmd_buffer->qf, cmd_buffer->qf)); if (fmask_comp == RADV_FMASK_COMPRESSION_NONE) { cb_color_info &= C_028C70_COMPRESSION; } @@ -2888,7 +2917,8 @@ static void radv_update_zrange_precision(struct radv_cmd_buffer *cmd_buffer, struct radv_ds_buffer_info *ds, const struct radv_image_view *iview, bool requires_cond_exec) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_image *image = iview->image; uint32_t db_z_info = ds->db_z_info; uint32_t db_z_info_reg; @@ -2911,7 +2941,7 @@ radv_update_zrange_precision(struct radv_cmd_buffer *cmd_buffer, struct radv_ds_ if (requires_cond_exec) { uint64_t va = radv_get_tc_compat_zrange_va(image, iview->vk.base_mip_level); - radv_emit_cond_exec(cmd_buffer->device, cmd_buffer->cs, va, 3 /* SET_CONTEXT_REG size */); + radv_emit_cond_exec(device, cmd_buffer->cs, va, 3 /* SET_CONTEXT_REG size */); } radeon_set_context_reg(cmd_buffer->cs, db_z_info_reg, db_z_info); @@ -2920,7 +2950,7 @@ radv_update_zrange_precision(struct radv_cmd_buffer *cmd_buffer, struct radv_ds_ static struct radv_image * radv_cmd_buffer_get_vrs_image(struct radv_cmd_buffer *cmd_buffer) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); if (!device->vrs.image) { VkResult result; @@ -2940,7 +2970,8 @@ static void radv_emit_fb_ds_state(struct radv_cmd_buffer *cmd_buffer, struct radv_ds_buffer_info *ds, struct radv_image_view *iview, bool depth_compressed, bool stencil_compressed) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); uint64_t db_htile_data_base = ds->db_htile_data_base; uint32_t db_htile_surface = ds->db_htile_surface; uint32_t db_render_control = ds->db_render_control | cmd_buffer->state.db_render_control; @@ -2959,7 +2990,7 @@ radv_emit_fb_ds_state(struct radv_cmd_buffer *cmd_buffer, struct radv_ds_buffer_ * our internal HTILE buffer. */ if (!radv_htile_enabled(iview->image, iview->vk.base_mip_level) && radv_cmd_buffer_get_vrs_image(cmd_buffer)) { - struct radv_buffer *htile_buffer = cmd_buffer->device->vrs.buffer; + struct radv_buffer *htile_buffer = device->vrs.buffer; assert(!G_028038_TILE_SURFACE_ENABLE(db_z_info) && !db_htile_data_base && !db_htile_surface); db_z_info |= S_028038_TILE_SURFACE_ENABLE(1); @@ -3041,7 +3072,8 @@ radv_emit_fb_ds_state(struct radv_cmd_buffer *cmd_buffer, struct radv_ds_buffer_ static void radv_emit_null_ds_state(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const enum amd_gfx_level gfx_level = pdev->info.gfx_level; unsigned db_render_control = 0; unsigned num_samples = 0; @@ -3051,7 +3083,7 @@ radv_emit_null_ds_state(struct radv_cmd_buffer *cmd_buffer) */ if (gfx_level == GFX11) { num_samples = util_logbase2(radv_get_rasterization_samples(cmd_buffer)); - radv_gfx11_set_db_render_control(cmd_buffer->device, 1, &db_render_control); + radv_gfx11_set_db_render_control(device, 1, &db_render_control); } if (gfx_level == GFX9) { @@ -3110,6 +3142,7 @@ radv_set_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image const VkImageSubresourceRange *range, VkClearDepthStencilValue ds_clear_value, VkImageAspectFlags aspects) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radeon_cmdbuf *cs = cmd_buffer->cs; uint32_t level_count = vk_image_subresource_level_count(&image->vk, range); @@ -3117,8 +3150,8 @@ radv_set_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image uint64_t va = radv_get_ds_clear_value_va(image, range->baseMipLevel); /* Use the fastest way when both aspects are used. */ - ASSERTED unsigned cdw_end = radv_cs_write_data_head(cmd_buffer->device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, - va, 2 * level_count, cmd_buffer->state.predicating); + ASSERTED unsigned cdw_end = radv_cs_write_data_head(device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, va, + 2 * level_count, cmd_buffer->state.predicating); for (uint32_t l = 0; l < level_count; l++) { radeon_emit(cs, ds_clear_value.stencil); @@ -3152,7 +3185,8 @@ static void radv_set_tc_compat_zrange_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *range, uint32_t value) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; if (!pdev->info.has_tc_compat_zrange_bug) @@ -3161,8 +3195,8 @@ radv_set_tc_compat_zrange_metadata(struct radv_cmd_buffer *cmd_buffer, struct ra uint64_t va = radv_get_tc_compat_zrange_va(image, range->baseMipLevel); uint32_t level_count = vk_image_subresource_level_count(&image->vk, range); - ASSERTED unsigned cdw_end = radv_cs_write_data_head(cmd_buffer->device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, - va, level_count, cmd_buffer->state.predicating); + ASSERTED unsigned cdw_end = radv_cs_write_data_head(device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, va, + level_count, cmd_buffer->state.predicating); for (uint32_t l = 0; l < level_count; l++) radeon_emit(cs, value); @@ -3224,7 +3258,8 @@ radv_update_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, const struct r static void radv_load_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, const struct radv_image_view *iview) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; const struct radv_image *image = iview->image; VkImageAspectFlags aspects = vk_format_aspects(image->vk.format); @@ -3273,6 +3308,8 @@ void radv_update_fce_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *range, bool value) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + if (!image->fce_pred_offset) return; @@ -3280,8 +3317,8 @@ radv_update_fce_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image * uint64_t va = radv_image_get_fce_pred_va(image, range->baseMipLevel); uint32_t level_count = vk_image_subresource_level_count(&image->vk, range); - ASSERTED unsigned cdw_end = radv_cs_write_data_head(cmd_buffer->device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, - va, 2 * level_count, false); + ASSERTED unsigned cdw_end = + radv_cs_write_data_head(device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, va, 2 * level_count, false); for (uint32_t l = 0; l < level_count; l++) { radeon_emit(cmd_buffer->cs, pred_val); @@ -3298,6 +3335,8 @@ void radv_update_dcc_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *range, bool value) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + if (image->dcc_pred_offset == 0) return; @@ -3307,8 +3346,8 @@ radv_update_dcc_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image * assert(radv_dcc_enabled(image, range->baseMipLevel)); - ASSERTED unsigned cdw_end = radv_cs_write_data_head(cmd_buffer->device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, - va, 2 * level_count, false); + ASSERTED unsigned cdw_end = + radv_cs_write_data_head(device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, va, 2 * level_count, false); for (uint32_t l = 0; l < level_count; l++) { radeon_emit(cmd_buffer->cs, pred_val); @@ -3325,13 +3364,14 @@ static void radv_update_bound_fast_clear_color(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, int cb_idx, uint32_t color_values[2]) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radeon_cmdbuf *cs = cmd_buffer->cs; if (cb_idx >= cmd_buffer->state.render.color_att_count || cmd_buffer->state.render.color_att[cb_idx].iview == NULL || cmd_buffer->state.render.color_att[cb_idx].iview->image != image) return; - ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 4); + ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, 4); radeon_set_context_reg_seq(cs, R_028C8C_CB_COLOR0_CLEAR_WORD0 + cb_idx * 0x3c, 2); radeon_emit(cs, color_values[0]); @@ -3349,6 +3389,7 @@ static void radv_set_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *range, uint32_t color_values[2]) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radeon_cmdbuf *cs = cmd_buffer->cs; uint32_t level_count = vk_image_subresource_level_count(&image->vk, range); @@ -3357,8 +3398,8 @@ radv_set_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_im if (radv_image_has_clear_value(image)) { uint64_t va = radv_image_get_fast_clear_va(image, range->baseMipLevel); - ASSERTED unsigned cdw_end = radv_cs_write_data_head(cmd_buffer->device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, - va, 2 * level_count, cmd_buffer->state.predicating); + ASSERTED unsigned cdw_end = radv_cs_write_data_head(device, cmd_buffer->cs, cmd_buffer->qf, V_370_PFP, va, + 2 * level_count, cmd_buffer->state.predicating); for (uint32_t l = 0; l < level_count; l++) { radeon_emit(cs, color_values[0]); @@ -3407,7 +3448,8 @@ radv_update_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, const struc static void radv_load_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *iview, int cb_idx) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; struct radv_image *image = iview->image; @@ -3454,7 +3496,8 @@ radv_load_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_i static void radv_emit_fb_mip_change_flush(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_rendering_state *render = &cmd_buffer->state.render; bool color_mip_changed = false; @@ -3498,7 +3541,8 @@ radv_emit_fb_mip_change_flush(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_mip_change_flush_default(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); /* Entire workaround is not applicable before GFX9 */ if (pdev->info.gfx_level < GFX9) @@ -3527,7 +3571,8 @@ radv_emit_mip_change_flush_default(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_rendering_state *render = &cmd_buffer->state.render; int i; bool disable_constant_encode_ac01 = false; @@ -3535,7 +3580,7 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) : S_028C70_FORMAT_GFX6(V_028C70_COLOR_INVALID); VkExtent2D extent = {MAX_FRAMEBUFFER_WIDTH, MAX_FRAMEBUFFER_HEIGHT}; - ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 51 + MAX_RTS * 70); + ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, 51 + MAX_RTS * 70); for (i = 0; i < render->color_att_count; ++i) { struct radv_image_view *iview = render->color_att[i].iview; @@ -3546,18 +3591,18 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) VkImageLayout layout = render->color_att[i].layout; - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, iview->image->bindings[0].bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, iview->image->bindings[0].bo); assert(iview->vk.aspects & (VK_IMAGE_ASPECT_COLOR_BIT | VK_IMAGE_ASPECT_PLANE_0_BIT | VK_IMAGE_ASPECT_PLANE_1_BIT | VK_IMAGE_ASPECT_PLANE_2_BIT)); if (iview->image->disjoint && iview->vk.aspects == VK_IMAGE_ASPECT_COLOR_BIT) { for (uint32_t plane_id = 0; plane_id < iview->image->plane_count; plane_id++) { - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, iview->image->bindings[plane_id].bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, iview->image->bindings[plane_id].bo); } } else { uint32_t plane_id = iview->image->disjoint ? iview->plane_id : 0; - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, iview->image->bindings[plane_id].bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, iview->image->bindings[plane_id].bo); } radv_emit_fb_color_state(cmd_buffer, i, &render->color_att[i].cb, iview, layout); @@ -3582,13 +3627,11 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) if (render->ds_att.iview) { struct radv_image_view *iview = render->ds_att.iview; const struct radv_image *image = iview->image; - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, image->bindings[0].bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, image->bindings[0].bo); uint32_t qf_mask = radv_image_queue_family_mask(image, cmd_buffer->qf, cmd_buffer->qf); - bool depth_compressed = - radv_layout_is_htile_compressed(cmd_buffer->device, image, render->ds_att.layout, qf_mask); - bool stencil_compressed = - radv_layout_is_htile_compressed(cmd_buffer->device, image, render->ds_att.stencil_layout, qf_mask); + bool depth_compressed = radv_layout_is_htile_compressed(device, image, render->ds_att.layout, qf_mask); + bool stencil_compressed = radv_layout_is_htile_compressed(device, image, render->ds_att.stencil_layout, qf_mask); radv_emit_fb_ds_state(cmd_buffer, &render->ds_att.ds, iview, depth_compressed, stencil_compressed); @@ -3606,12 +3649,12 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) * bind our internal depth buffer that contains the VRS data as part of HTILE. */ VkImageLayout layout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; - struct radv_buffer *htile_buffer = cmd_buffer->device->vrs.buffer; - struct radv_image *image = cmd_buffer->device->vrs.image; + struct radv_buffer *htile_buffer = device->vrs.buffer; + struct radv_image *image = device->vrs.image; struct radv_ds_buffer_info ds; struct radv_image_view iview; - radv_image_view_init(&iview, cmd_buffer->device, + radv_image_view_init(&iview, device, &(VkImageViewCreateInfo){ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = radv_image_to_handle(image), @@ -3630,10 +3673,10 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) radv_initialise_vrs_surface(image, htile_buffer, &ds); - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, htile_buffer->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, htile_buffer->bo); bool depth_compressed = radv_layout_is_htile_compressed( - cmd_buffer->device, image, layout, radv_image_queue_family_mask(image, cmd_buffer->qf, cmd_buffer->qf)); + device, image, layout, radv_image_queue_family_mask(image, cmd_buffer->qf, cmd_buffer->qf)); radv_emit_fb_ds_state(cmd_buffer, &ds, &iview, depth_compressed, false); radv_image_view_finish(&iview); @@ -3726,7 +3769,8 @@ radv_handle_zero_index_buffer_bug(struct radv_cmd_buffer *cmd_buffer, uint64_t * static void radv_emit_index_buffer(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; struct radv_cmd_state *state = &cmd_buffer->state; uint32_t max_index_count = state->max_index_count; @@ -3755,7 +3799,8 @@ radv_emit_index_buffer(struct radv_cmd_buffer *cmd_buffer) static void radv_flush_occlusion_query_state(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const enum amd_gfx_level gfx_level = pdev->info.gfx_level; const bool enable_occlusion_queries = cmd_buffer->state.active_occlusion_queries || cmd_buffer->state.inherited_occlusion_queries; @@ -3826,8 +3871,8 @@ lookup_vs_prolog(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *v { assert(vs_shader->info.vs.dynamic_inputs); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_vs_input_state *state = &cmd_buffer->state.dynamic_vs_input; - struct radv_device *device = cmd_buffer->device; const struct radv_physical_device *pdev = radv_device_physical(device); unsigned num_attributes = util_last_bit(vs_shader->info.vs.vb_desc_usage_mask); @@ -3922,7 +3967,8 @@ static void emit_prolog_regs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *vs_shader, const struct radv_shader_part *prolog) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); uint32_t rsrc1, rsrc2; /* no need to re-emit anything in this case */ @@ -3991,7 +4037,7 @@ emit_prolog_regs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *v } } - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, prolog->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, prolog->bo); } static void @@ -4003,6 +4049,7 @@ emit_prolog_inputs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader !cmd_buffer->state.emitted_vs_prolog->nontrivial_divisors) return; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_vs_input_state *state = &cmd_buffer->state.dynamic_vs_input; uint64_t input_va = radv_shader_get_va(vs_shader); @@ -4038,13 +4085,14 @@ emit_prolog_inputs(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader uint32_t base_reg = vs_shader->info.user_data_0; assert(loc->sgpr_idx != -1); assert(loc->num_sgprs == 2); - radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, input_va, true); + radv_emit_shader_pointer(device, cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, input_va, true); } static void radv_emit_vertex_input(struct radv_cmd_buffer *cmd_buffer) { const struct radv_shader *vs_shader = radv_get_shader(cmd_buffer->state.shaders, MESA_SHADER_VERTEX); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); assert(!cmd_buffer->state.mesh_shading); @@ -4064,14 +4112,15 @@ radv_emit_vertex_input(struct radv_cmd_buffer *cmd_buffer) cmd_buffer->state.emitted_vs_prolog = prolog; - if (radv_device_fault_detection_enabled(cmd_buffer->device)) + if (radv_device_fault_detection_enabled(device)) radv_save_vs_prolog(cmd_buffer, prolog); } static void radv_emit_tess_domain_origin(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_shader *tes = radv_get_shader(cmd_buffer->state.shaders, MESA_SHADER_TESS_EVAL); const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; unsigned type = 0, partitioning = 0, distribution_mode = 0; @@ -4136,10 +4185,11 @@ radv_emit_tess_domain_origin(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_alpha_to_coverage_enable(struct radv_cmd_buffer *cmd_buffer) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; unsigned db_alpha_to_mask = 0; - if (cmd_buffer->device->instance->debug_flags & RADV_DEBUG_NO_ATOC_DITHERING) { + if (device->instance->debug_flags & RADV_DEBUG_NO_ATOC_DITHERING) { db_alpha_to_mask = S_028B70_ALPHA_TO_MASK_OFFSET0(2) | S_028B70_ALPHA_TO_MASK_OFFSET1(2) | S_028B70_ALPHA_TO_MASK_OFFSET2(2) | S_028B70_ALPHA_TO_MASK_OFFSET3(2) | S_028B70_OFFSET_ROUND(0); @@ -4167,7 +4217,8 @@ radv_emit_sample_mask(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_color_blend(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const enum amd_gfx_level gfx_level = pdev->info.gfx_level; const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; unsigned cb_blend_control[MAX_RTS], sx_mrt_blend_opt[MAX_RTS]; @@ -4275,10 +4326,10 @@ radv_emit_color_blend(struct radv_cmd_buffer *cmd_buffer) static struct radv_shader_part * lookup_ps_epilog(struct radv_cmd_buffer *cmd_buffer) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_shader *ps = cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT]; const struct radv_rendering_state *render = &cmd_buffer->state.render; const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; - struct radv_device *device = cmd_buffer->device; const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_ps_epilog_state state = {0}; @@ -4335,7 +4386,8 @@ lookup_ps_epilog(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_msaa_state(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_shader *ps = cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT]; unsigned rasterization_samples = radv_get_rasterization_samples(cmd_buffer); const struct radv_rendering_state *render = &cmd_buffer->state.render; @@ -4432,7 +4484,8 @@ radv_emit_line_rasterization_mode(struct radv_cmd_buffer *cmd_buffer) static void radv_cmd_buffer_flush_dynamic_state(struct radv_cmd_buffer *cmd_buffer, const uint64_t states) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); if (states & (RADV_CMD_DIRTY_DYNAMIC_VIEWPORT | RADV_CMD_DIRTY_DYNAMIC_DEPTH_CLIP_ENABLE | RADV_CMD_DIRTY_DYNAMIC_DEPTH_CLIP_NEGATIVE_ONE_TO_ONE | RADV_CMD_DIRTY_DYNAMIC_DEPTH_CLAMP_ENABLE)) @@ -4579,7 +4632,7 @@ radv_flush_indirect_descriptor_sets(struct radv_cmd_buffer *cmd_buffer, VkPipeli } struct radeon_cmdbuf *cs = cmd_buffer->cs; - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); uint64_t va = radv_buffer_get_va(cmd_buffer->upload.upload_bo); va += offset; @@ -4619,7 +4672,7 @@ ALWAYS_INLINE static void radv_flush_descriptors(struct radv_cmd_buffer *cmd_buffer, VkShaderStageFlags stages, VkPipelineBindPoint bind_point) { struct radv_descriptor_state *descriptors_state = radv_get_descriptors_state(cmd_buffer, bind_point); - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radeon_cmdbuf *cs = cmd_buffer->cs; bool flush_indirect_descriptors; @@ -4660,7 +4713,7 @@ radv_flush_descriptors(struct radv_cmd_buffer *cmd_buffer, VkShaderStageFlags st assert(cmd_buffer->cs->cdw <= cdw_max); - if (radv_device_fault_detection_enabled(cmd_buffer->device)) + if (radv_device_fault_detection_enabled(device)) radv_save_descriptors(cmd_buffer, bind_point); } @@ -4704,7 +4757,7 @@ radv_must_flush_constants(const struct radv_cmd_buffer *cmd_buffer, VkShaderStag static void radv_flush_constants(struct radv_cmd_buffer *cmd_buffer, VkShaderStageFlags stages, VkPipelineBindPoint bind_point) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radeon_cmdbuf *cs = cmd_buffer->cs; struct radv_descriptor_state *descriptors_state = radv_get_descriptors_state(cmd_buffer, bind_point); const struct radv_push_constant_state *push_constants = radv_get_push_constants_state(cmd_buffer, bind_point); @@ -4768,8 +4821,7 @@ radv_flush_constants(struct radv_cmd_buffer *cmd_buffer, VkShaderStageFlags stag va = radv_buffer_get_va(cmd_buffer->upload.upload_bo); va += offset; - ASSERTED unsigned cdw_max = - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, MESA_VULKAN_SHADER_STAGES * 4); + ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, MESA_VULKAN_SHADER_STAGES * 4); if (internal_stages & VK_SHADER_STAGE_COMPUTE_BIT) { struct radv_shader *compute_shader = bind_point == VK_PIPELINE_BIND_POINT_COMPUTE @@ -4810,7 +4862,8 @@ void radv_write_vertex_descriptors(const struct radv_cmd_buffer *cmd_buffer, const struct radv_graphics_pipeline *pipeline, bool full_null_descriptors, void *vb_ptr) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_shader *vs_shader = radv_get_shader(cmd_buffer->state.shaders, MESA_SHADER_VERTEX); enum amd_gfx_level chip = pdev->info.gfx_level; enum radeon_family family = pdev->info.family; @@ -4974,6 +5027,7 @@ static void radv_flush_vertex_descriptors(struct radv_cmd_buffer *cmd_buffer) { struct radv_shader *vs = radv_get_shader(cmd_buffer->state.shaders, MESA_SHADER_VERTEX); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); if (!vs->info.vs.vb_desc_usage_mask) return; @@ -4996,14 +5050,13 @@ radv_flush_vertex_descriptors(struct radv_cmd_buffer *cmd_buffer) va = radv_buffer_get_va(cmd_buffer->upload.upload_bo); va += vb_offset; - radv_emit_userdata_address(cmd_buffer->device, cmd_buffer->cs, vs, vs->info.user_data_0, AC_UD_VS_VERTEX_BUFFERS, - va); + radv_emit_userdata_address(device, cmd_buffer->cs, vs, vs->info.user_data_0, AC_UD_VS_VERTEX_BUFFERS, va); cmd_buffer->state.vb_va = va; cmd_buffer->state.vb_size = vb_desc_alloc_size; cmd_buffer->state.prefetch_L2_mask |= RADV_PREFETCH_VBO_DESCRIPTORS; - if (radv_device_fault_detection_enabled(cmd_buffer->device)) + if (radv_device_fault_detection_enabled(device)) radv_save_vertex_descriptors(cmd_buffer, (uintptr_t)vb_ptr); cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_VERTEX_BUFFER; @@ -5014,6 +5067,7 @@ radv_emit_streamout_buffers(struct radv_cmd_buffer *cmd_buffer, uint64_t va) { const struct radv_shader *last_vgt_shader = cmd_buffer->state.last_vgt_shader; const struct radv_userdata_info *loc = radv_get_user_sgpr(last_vgt_shader, AC_UD_STREAMOUT_BUFFERS); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); uint32_t base_reg; if (loc->sgpr_idx == -1) @@ -5021,14 +5075,14 @@ radv_emit_streamout_buffers(struct radv_cmd_buffer *cmd_buffer, uint64_t va) base_reg = last_vgt_shader->info.user_data_0; - radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, va, false); + radv_emit_shader_pointer(device, cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, va, false); if (cmd_buffer->state.gs_copy_shader) { loc = &cmd_buffer->state.gs_copy_shader->info.user_sgprs_locs.shader_data[AC_UD_STREAMOUT_BUFFERS]; if (loc->sgpr_idx != -1) { base_reg = R_00B130_SPI_SHADER_USER_DATA_VS_0; - radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, va, false); + radv_emit_shader_pointer(device, cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, va, false); } } } @@ -5036,7 +5090,8 @@ radv_emit_streamout_buffers(struct radv_cmd_buffer *cmd_buffer, uint64_t va) static void radv_flush_streamout_descriptors(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_STREAMOUT_BUFFER) { struct radv_streamout_binding *sb = cmd_buffer->streamout_bindings; @@ -5107,7 +5162,8 @@ radv_flush_streamout_descriptors(struct radv_cmd_buffer *cmd_buffer) static void radv_flush_shader_query_state_gfx(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_shader *last_vgt_shader = cmd_buffer->state.last_vgt_shader; const struct radv_userdata_info *loc = radv_get_user_sgpr(last_vgt_shader, AC_UD_SHADER_QUERY_STATE); enum radv_shader_query_state shader_query_state = radv_shader_query_none; @@ -5170,7 +5226,8 @@ radv_flush_shader_query_state_ace(struct radv_cmd_buffer *cmd_buffer, struct rad static void radv_flush_shader_query_state(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); radv_flush_shader_query_state_gfx(cmd_buffer); @@ -5183,7 +5240,8 @@ radv_flush_shader_query_state(struct radv_cmd_buffer *cmd_buffer) static void radv_flush_force_vrs_state(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_shader *last_vgt_shader = cmd_buffer->state.last_vgt_shader; if (!last_vgt_shader->info.force_vrs_per_vertex) { @@ -5208,7 +5266,7 @@ radv_flush_force_vrs_state(struct radv_cmd_buffer *cmd_buffer) enum amd_gfx_level gfx_level = pdev->info.gfx_level; uint32_t vrs_rates = 0; - switch (cmd_buffer->device->force_vrs) { + switch (device->force_vrs) { case RADV_FORCE_VRS_2x2: vrs_rates = gfx_level >= GFX11 ? V_0283D0_VRS_SHADING_RATE_2X2 : (1u << 2) | (1u << 4); break; @@ -5293,7 +5351,8 @@ static void radv_emit_ia_multi_vgt_param(struct radv_cmd_buffer *cmd_buffer, bool instanced_draw, bool indirect_draw, bool count_from_stream_output, uint32_t draw_vertex_count) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radeon_info *gpu_info = &pdev->info; struct radv_cmd_state *state = &cmd_buffer->state; const unsigned patch_control_points = state->dynamic.vk.ts.patch_control_points; @@ -5360,7 +5419,8 @@ gfx10_emit_ge_cntl(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_draw_registers(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *draw_info) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radeon_info *gpu_info = &pdev->info; struct radv_cmd_state *state = &cmd_buffer->state; struct radeon_cmdbuf *cs = cmd_buffer->cs; @@ -5558,7 +5618,8 @@ radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer, VkAccessFlags2 src_fla enum radv_cmd_flush_bits radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer, VkAccessFlags2 dst_flags, const struct radv_image *image) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); bool has_CB_meta = true, has_DB_meta = true; enum radv_cmd_flush_bits flush_bits = 0; bool flush_CB = true, flush_DB = true; @@ -5578,17 +5639,17 @@ radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer, VkAccessFlags2 dst_fla /* All the L2 invalidations below are not the CB/DB. So if there are no incoherent images * in the L2 cache in CB/DB mode then they are already usable from all the other L2 clients. */ - image_is_coherent |= can_skip_buffer_l2_flushes(cmd_buffer->device) && !cmd_buffer->state.rb_noncoherent_dirty; + image_is_coherent |= can_skip_buffer_l2_flushes(device) && !cmd_buffer->state.rb_noncoherent_dirty; u_foreach_bit64 (b, dst_flags) { switch ((VkAccessFlags2)BITFIELD64_BIT(b)) { case VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT: /* SMEM loads are used to read compute dispatch size in shaders */ - if (!cmd_buffer->device->load_grid_size_from_user_sgpr) + if (!device->load_grid_size_from_user_sgpr) flush_bits |= RADV_CMD_FLAG_INV_SCACHE; /* Ensure the DGC meta shader can read the commands. */ - if (radv_uses_device_generated_commands(cmd_buffer->device)) { + if (radv_uses_device_generated_commands(device)) { flush_bits |= RADV_CMD_FLAG_INV_SCACHE | RADV_CMD_FLAG_INV_VCACHE; if (pdev->info.gfx_level < GFX9) @@ -5779,7 +5840,8 @@ VKAPI_ATTR VkResult VKAPI_CALL radv_BeginCommandBuffer(VkCommandBuffer commandBuffer, const VkCommandBufferBeginInfo *pBeginInfo) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); VkResult result = VK_SUCCESS; vk_command_buffer_begin(&cmd_buffer->vk, pBeginInfo); @@ -5812,7 +5874,7 @@ radv_BeginCommandBuffer(VkCommandBuffer commandBuffer, const VkCommandBufferBegi if (cmd_buffer->qf == RADV_QUEUE_GENERAL) vk_dynamic_graphics_state_init(&cmd_buffer->state.dynamic.vk); - if (cmd_buffer->qf == RADV_QUEUE_COMPUTE || cmd_buffer->device->vk.enabled_features.taskShader) { + if (cmd_buffer->qf == RADV_QUEUE_COMPUTE || device->vk.enabled_features.taskShader) { uint32_t pred_value = 0; uint32_t pred_offset; if (!radv_cmd_buffer_upload_data(cmd_buffer, 4, &pred_value, &pred_offset)) @@ -5897,7 +5959,7 @@ radv_BeginCommandBuffer(VkCommandBuffer commandBuffer, const VkCommandBufferBegi cmd_buffer->state.dirty |= RADV_CMD_DIRTY_OCCLUSION_QUERY; } - if (radv_device_fault_detection_enabled(cmd_buffer->device)) + if (radv_device_fault_detection_enabled(device)) radv_cmd_buffer_trace_emit(cmd_buffer); radv_describe_begin_cmd_buffer(cmd_buffer); @@ -5911,7 +5973,8 @@ radv_CmdBindVertexBuffers2(VkCommandBuffer commandBuffer, uint32_t firstBinding, const VkDeviceSize *pStrides) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_vertex_binding *vb = cmd_buffer->vertex_bindings; const struct radv_vs_input_state *state = &cmd_buffer->state.dynamic_vs_input; @@ -5945,7 +6008,7 @@ radv_CmdBindVertexBuffers2(VkCommandBuffer commandBuffer, uint32_t firstBinding, uint32_t bit = BITFIELD_BIT(idx); if (buffer) { - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, cmd_buffer->vertex_binding_buffers[idx]->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, cmd_buffer->vertex_binding_buffers[idx]->bo); cmd_buffer->state.vbo_bound_mask |= bit; } else { cmd_buffer->state.vbo_bound_mask &= ~bit; @@ -5997,7 +6060,8 @@ radv_CmdBindIndexBuffer2KHR(VkCommandBuffer commandBuffer, VkBuffer buffer, VkDe { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_buffer, index_buffer, buffer); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); cmd_buffer->state.index_type = vk_to_index_type(indexType); @@ -6007,7 +6071,7 @@ radv_CmdBindIndexBuffer2KHR(VkCommandBuffer commandBuffer, VkBuffer buffer, VkDe int index_size = radv_get_vgt_index_size(vk_to_index_type(indexType)); cmd_buffer->state.max_index_count = (vk_buffer_range(&index_buffer->vk, offset, size)) / index_size; - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, index_buffer->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, index_buffer->bo); } else { cmd_buffer->state.index_va = 0; cmd_buffer->state.max_index_count = 0; @@ -6027,14 +6091,15 @@ static void radv_bind_descriptor_set(struct radv_cmd_buffer *cmd_buffer, VkPipelineBindPoint bind_point, struct radv_descriptor_set *set, unsigned idx) { - struct radeon_winsys *ws = cmd_buffer->device->ws; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + struct radeon_winsys *ws = device->ws; radv_set_descriptor_set(cmd_buffer, bind_point, set, idx); assert(set); assert(!(set->header.layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR)); - if (!cmd_buffer->device->use_global_bo_list) { + if (!device->use_global_bo_list) { for (unsigned j = 0; j < set->header.buffer_count; ++j) if (set->descriptors[j]) radv_cs_add_buffer(ws, cmd_buffer->cs, set->descriptors[j]); @@ -6049,8 +6114,9 @@ radv_bind_descriptor_sets(struct radv_cmd_buffer *cmd_buffer, const VkBindDescriptorSetsInfoKHR *pBindDescriptorSetsInfo, VkPipelineBindPoint bind_point) { RADV_FROM_HANDLE(radv_pipeline_layout, layout, pBindDescriptorSetsInfo->layout); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); - const bool no_dynamic_bounds = cmd_buffer->device->instance->debug_flags & RADV_DEBUG_NO_DYNAMIC_BOUNDS; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + const bool no_dynamic_bounds = device->instance->debug_flags & RADV_DEBUG_NO_DYNAMIC_BOUNDS; struct radv_descriptor_state *descriptors_state = radv_get_descriptors_state(cmd_buffer, bind_point); unsigned dyn_idx = 0; @@ -6124,11 +6190,12 @@ radv_init_push_descriptor_set(struct radv_cmd_buffer *cmd_buffer, struct radv_de struct radv_descriptor_set_layout *layout, VkPipelineBindPoint bind_point) { struct radv_descriptor_state *descriptors_state = radv_get_descriptors_state(cmd_buffer, bind_point); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); set->header.size = layout->size; if (set->header.layout != layout) { if (set->header.layout) - vk_descriptor_set_layout_unref(&cmd_buffer->device->vk, &set->header.layout->vk); + vk_descriptor_set_layout_unref(&device->vk, &set->header.layout->vk); vk_descriptor_set_layout_ref(&layout->vk); set->header.layout = layout; } @@ -6160,6 +6227,7 @@ radv_meta_push_descriptor_set(struct radv_cmd_buffer *cmd_buffer, VkPipelineBind { RADV_FROM_HANDLE(radv_pipeline_layout, layout, _layout); struct radv_descriptor_set *push_set = (struct radv_descriptor_set *)&cmd_buffer->meta_push_descriptors; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); unsigned bo_offset; assert(set == 0); @@ -6175,8 +6243,8 @@ radv_meta_push_descriptor_set(struct radv_cmd_buffer *cmd_buffer, VkPipelineBind push_set->header.va = radv_buffer_get_va(cmd_buffer->upload.upload_bo); push_set->header.va += bo_offset; - radv_cmd_update_descriptor_sets(cmd_buffer->device, cmd_buffer, radv_descriptor_set_to_handle(push_set), - descriptorWriteCount, pDescriptorWrites, 0, NULL); + radv_cmd_update_descriptor_sets(device, cmd_buffer, radv_descriptor_set_to_handle(push_set), descriptorWriteCount, + pDescriptorWrites, 0, NULL); radv_set_descriptor_set(cmd_buffer, pipelineBindPoint, push_set, set); } @@ -6188,6 +6256,7 @@ radv_push_descriptor_set(struct radv_cmd_buffer *cmd_buffer, const VkPushDescrip RADV_FROM_HANDLE(radv_pipeline_layout, layout, pPushDescriptorSetInfo->layout); struct radv_descriptor_state *descriptors_state = radv_get_descriptors_state(cmd_buffer, bind_point); struct radv_descriptor_set *push_set = (struct radv_descriptor_set *)&descriptors_state->push_set.set; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); assert(layout->set[pPushDescriptorSetInfo->set].layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR); @@ -6204,7 +6273,7 @@ radv_push_descriptor_set(struct radv_cmd_buffer *cmd_buffer, const VkPushDescrip assert(writeset->descriptorType != VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK); } - radv_cmd_update_descriptor_sets(cmd_buffer->device, cmd_buffer, radv_descriptor_set_to_handle(push_set), + radv_cmd_update_descriptor_sets(device, cmd_buffer, radv_descriptor_set_to_handle(push_set), pPushDescriptorSetInfo->descriptorWriteCount, pPushDescriptorSetInfo->pDescriptorWrites, 0, NULL); @@ -6241,6 +6310,7 @@ radv_CmdPushDescriptorSetWithTemplate2KHR( pPushDescriptorSetWithTemplateInfo->descriptorUpdateTemplate); struct radv_descriptor_state *descriptors_state = radv_get_descriptors_state(cmd_buffer, templ->bind_point); struct radv_descriptor_set *push_set = (struct radv_descriptor_set *)&descriptors_state->push_set.set; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); assert(layout->set[pPushDescriptorSetWithTemplateInfo->set].layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR); @@ -6249,7 +6319,7 @@ radv_CmdPushDescriptorSetWithTemplate2KHR( templ->bind_point)) return; - radv_cmd_update_descriptor_set_with_template(cmd_buffer->device, cmd_buffer, push_set, + radv_cmd_update_descriptor_set_with_template(device, cmd_buffer, push_set, pPushDescriptorSetWithTemplateInfo->descriptorUpdateTemplate, pPushDescriptorSetWithTemplateInfo->pData); @@ -6271,7 +6341,8 @@ VKAPI_ATTR VkResult VKAPI_CALL radv_EndCommandBuffer(VkCommandBuffer commandBuffer) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); if (cmd_buffer->qf == RADV_QUEUE_SPARSE) return vk_command_buffer_end(&cmd_buffer->vk); @@ -6293,7 +6364,7 @@ radv_EndCommandBuffer(VkCommandBuffer commandBuffer) /* Flush noncoherent images on GFX9+ so we can assume they're clean on the start of a * command buffer. */ - if (cmd_buffer->state.rb_noncoherent_dirty && !can_skip_buffer_l2_flushes(cmd_buffer->device)) + if (cmd_buffer->state.rb_noncoherent_dirty && !can_skip_buffer_l2_flushes(device)) cmd_buffer->state.flush_bits |= radv_src_access_flush( cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, NULL); @@ -6323,7 +6394,7 @@ radv_EndCommandBuffer(VkCommandBuffer commandBuffer) radv_describe_end_cmd_buffer(cmd_buffer); - VkResult result = cmd_buffer->device->ws->cs_finalize(cmd_buffer->cs); + VkResult result = device->ws->cs_finalize(cmd_buffer->cs); if (result != VK_SUCCESS) return vk_error(cmd_buffer, result); @@ -6333,6 +6404,8 @@ radv_EndCommandBuffer(VkCommandBuffer commandBuffer) static void radv_emit_compute_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_compute_pipeline *pipeline) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + if (pipeline == cmd_buffer->state.emitted_compute_pipeline) return; @@ -6340,27 +6413,26 @@ radv_emit_compute_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_compu cmd_buffer->state.emitted_compute_pipeline = pipeline; - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, pipeline->base.cs.cdw); + radeon_check_space(device->ws, cmd_buffer->cs, pipeline->base.cs.cdw); radeon_emit_array(cmd_buffer->cs, pipeline->base.cs.buf, pipeline->base.cs.cdw); if (pipeline->base.type == RADV_PIPELINE_COMPUTE) { - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, cmd_buffer->state.shaders[MESA_SHADER_COMPUTE]->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, cmd_buffer->state.shaders[MESA_SHADER_COMPUTE]->bo); } else { - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, cmd_buffer->state.rt_prolog->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, cmd_buffer->state.rt_prolog->bo); if (cmd_buffer->state.shaders[MESA_SHADER_INTERSECTION]) - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, - cmd_buffer->state.shaders[MESA_SHADER_INTERSECTION]->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, cmd_buffer->state.shaders[MESA_SHADER_INTERSECTION]->bo); struct radv_ray_tracing_pipeline *rt_pipeline = radv_pipeline_to_ray_tracing(&pipeline->base); for (unsigned i = 0; i < rt_pipeline->stage_count; ++i) { struct radv_shader *shader = rt_pipeline->stages[i].shader; if (shader) - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, shader->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, shader->bo); } } - if (radv_device_fault_detection_enabled(cmd_buffer->device)) + if (radv_device_fault_detection_enabled(device)) radv_save_pipeline(cmd_buffer, &pipeline->base); } @@ -6375,7 +6447,8 @@ radv_mark_descriptor_sets_dirty(struct radv_cmd_buffer *cmd_buffer, VkPipelineBi static void radv_bind_vs_input_state(struct radv_cmd_buffer *cmd_buffer, const struct radv_graphics_pipeline *pipeline) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_shader *vs_shader = radv_get_shader(cmd_buffer->state.shaders, MESA_SHADER_VERTEX); const struct radv_vs_input_state *src = &pipeline->vs_input_state; @@ -6419,7 +6492,8 @@ radv_bind_custom_blend_mode(struct radv_cmd_buffer *cmd_buffer, unsigned custom_ static void radv_bind_pre_rast_shader(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *shader) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); bool mesh_shading = shader->info.stage == MESA_SHADER_MESH; const struct radv_userdata_info *loc; @@ -6568,7 +6642,8 @@ radv_bind_mesh_shader(struct radv_cmd_buffer *cmd_buffer, const struct radv_shad static void radv_bind_fragment_shader(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *ps) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const enum amd_gfx_level gfx_level = pdev->info.gfx_level; const struct radv_shader *previous_ps = cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT]; const float min_sample_shading = 1.0f; @@ -6628,7 +6703,8 @@ radv_bind_rt_prolog(struct radv_cmd_buffer *cmd_buffer, struct radv_shader *rt_p { cmd_buffer->state.rt_prolog = rt_prolog; - const unsigned max_scratch_waves = radv_get_max_scratch_waves(cmd_buffer->device, rt_prolog); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const unsigned max_scratch_waves = radv_get_max_scratch_waves(device, rt_prolog); cmd_buffer->compute_scratch_waves_wanted = MAX2(cmd_buffer->compute_scratch_waves_wanted, max_scratch_waves); cmd_buffer->shader_upload_seq = MAX2(cmd_buffer->shader_upload_seq, rt_prolog->upload_seq); @@ -6638,7 +6714,7 @@ radv_bind_rt_prolog(struct radv_cmd_buffer *cmd_buffer, struct radv_shader *rt_p static void radv_bind_shader(struct radv_cmd_buffer *cmd_buffer, struct radv_shader *shader, gl_shader_stage stage) { - const struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); if (!shader) { cmd_buffer->state.shaders[stage] = NULL; @@ -6739,7 +6815,8 @@ radv_CmdBindPipeline(VkCommandBuffer commandBuffer, VkPipelineBindPoint pipeline { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_pipeline, pipeline, _pipeline); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); radv_reset_shader_object_state(cmd_buffer, pipelineBindPoint); @@ -7277,7 +7354,8 @@ radv_CmdSetVertexInputEXT(VkCommandBuffer commandBuffer, uint32_t vertexBindingD const VkVertexInputAttributeDescription2EXT *pVertexAttributeDescriptions) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_cmd_state *state = &cmd_buffer->state; struct radv_vs_input_state *vs_state = &state->dynamic_vs_input; @@ -7494,7 +7572,8 @@ radv_CmdSetColorWriteMaskEXT(VkCommandBuffer commandBuffer, uint32_t firstAttach const VkColorComponentFlags *pColorWriteMasks) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_cmd_state *state = &cmd_buffer->state; assert(firstAttachment + attachmentCount <= MAX_RTS); @@ -7640,7 +7719,8 @@ VKAPI_ATTR void VKAPI_CALL radv_CmdExecuteCommands(VkCommandBuffer commandBuffer, uint32_t commandBufferCount, const VkCommandBuffer *pCmdBuffers) { RADV_FROM_HANDLE(radv_cmd_buffer, primary, commandBuffer); - const struct radv_physical_device *pdev = radv_device_physical(primary->device); + struct radv_device *device = radv_cmd_buffer_device(primary); + const struct radv_physical_device *pdev = radv_device_physical(device); assert(commandBufferCount > 0); @@ -7715,7 +7795,7 @@ radv_CmdExecuteCommands(VkCommandBuffer commandBuffer, uint32_t commandBufferCou /* Execute the secondary compute cmdbuf. * Don't use IB2 packets because they are not supported on compute queues. */ - primary->device->ws->cs_execute_secondary(ace_primary, ace_secondary, false); + device->ws->cs_execute_secondary(ace_primary, ace_secondary, false); } /* Update pending ACE internal flush bits from the secondary cmdbuf */ @@ -7730,7 +7810,7 @@ radv_CmdExecuteCommands(VkCommandBuffer commandBuffer, uint32_t commandBufferCou if (radv_gang_follower_sem_dirty(secondary)) primary->gang.sem.follower_value++; - primary->device->ws->cs_execute_secondary(primary->cs, secondary->cs, allow_ib2); + device->ws->cs_execute_secondary(primary->cs, secondary->cs, allow_ib2); /* When the secondary command buffer is compute only we don't * need to re-emit the current graphics pipeline. @@ -7829,7 +7909,8 @@ VKAPI_ATTR void VKAPI_CALL radv_CmdBeginRendering(VkCommandBuffer commandBuffer, const VkRenderingInfo *pRenderingInfo) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct VkSampleLocationsInfoEXT *sample_locs_info = vk_find_struct_const(pRenderingInfo->pNext, SAMPLE_LOCATIONS_INFO_EXT); @@ -7868,7 +7949,7 @@ radv_CmdBeginRendering(VkCommandBuffer commandBuffer, const VkRenderingInfo *pRe color_att[i].format = iview->vk.format; color_att[i].iview = iview; color_att[i].layout = att_info->imageLayout; - radv_initialise_color_surface(cmd_buffer->device, &color_att[i].cb, iview); + radv_initialise_color_surface(device, &color_att[i].cb, iview); if (att_info->resolveMode != VK_RESOLVE_MODE_NONE && att_info->resolveImageView != VK_NULL_HANDLE) { color_att[i].resolve_mode = att_info->resolveMode; @@ -7933,7 +8014,7 @@ radv_CmdBeginRendering(VkCommandBuffer commandBuffer, const VkRenderingInfo *pRe ds_att_aspects = VK_IMAGE_ASPECT_STENCIL_BIT; } - radv_initialise_ds_surface(cmd_buffer->device, &ds_att.ds, ds_att.iview, ds_att_aspects); + radv_initialise_ds_surface(device, &ds_att.ds, ds_att.iview, ds_att_aspects); assert(d_res_iview == NULL || s_res_iview == NULL || d_res_iview == s_res_iview); ds_att.resolve_iview = d_res_iview ? d_res_iview : s_res_iview; @@ -8008,7 +8089,7 @@ radv_CmdBeginRendering(VkCommandBuffer commandBuffer, const VkRenderingInfo *pRe uint64_t htile_size = ds_image->planes[0].surface.u.gfx9.meta_levels[level].size; struct radv_buffer htile_buffer; - radv_buffer_init(&htile_buffer, cmd_buffer->device, ds_image->bindings[0].bo, htile_size, htile_offset); + radv_buffer_init(&htile_buffer, device, ds_image->bindings[0].bo, htile_size, htile_offset); assert(render->area.offset.x + render->area.extent.width <= ds_image->vk.extent.width && render->area.offset.x + render->area.extent.height <= ds_image->vk.extent.height); @@ -8026,7 +8107,7 @@ radv_CmdBeginRendering(VkCommandBuffer commandBuffer, const VkRenderingInfo *pRe if (ds_image && render->area.offset.x < ds_image->vk.extent.width && render->area.offset.y < ds_image->vk.extent.height) { /* HTILE buffer */ - struct radv_buffer *htile_buffer = cmd_buffer->device->vrs.buffer; + struct radv_buffer *htile_buffer = device->vrs.buffer; VkRect2D area = render->area; area.extent.width = MIN2(area.extent.width, ds_image->vk.extent.width - area.offset.x); @@ -8038,7 +8119,7 @@ radv_CmdBeginRendering(VkCommandBuffer commandBuffer, const VkRenderingInfo *pRe } } - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 6); + radeon_check_space(device->ws, cmd_buffer->cs, 6); radeon_set_context_reg(cmd_buffer->cs, R_028204_PA_SC_WINDOW_SCISSOR_TL, S_028204_TL_X(render->area.offset.x) | S_028204_TL_Y(render->area.offset.y)); radeon_set_context_reg(cmd_buffer->cs, R_028208_PA_SC_WINDOW_SCISSOR_BR, @@ -8225,7 +8306,8 @@ ALWAYS_INLINE static void radv_cs_emit_indirect_mesh_draw_packet(struct radv_cmd_buffer *cmd_buffer, uint32_t draw_count, uint64_t count_va, uint32_t stride) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_shader *mesh_shader = cmd_buffer->state.shaders[MESA_SHADER_MESH]; struct radeon_cmdbuf *cs = cmd_buffer->cs; uint32_t base_reg = cmd_buffer->state.vtx_base_sgpr; @@ -8467,7 +8549,8 @@ radv_emit_draw_packets_indexed(struct radv_cmd_buffer *cmd_buffer, const struct const int32_t *vertexOffset) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_cmd_state *state = &cmd_buffer->state; struct radeon_cmdbuf *cs = cmd_buffer->cs; const int index_size = radv_get_vgt_index_size(state->index_type); @@ -8645,7 +8728,8 @@ radv_cs_emit_mesh_dispatch_packet(struct radv_cmd_buffer *cmd_buffer, uint32_t x ALWAYS_INLINE static void radv_emit_direct_mesh_draw_packet(struct radv_cmd_buffer *cmd_buffer, uint32_t x, uint32_t y, uint32_t z) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const uint32_t view_mask = cmd_buffer->state.render.view_mask; radv_emit_userdata_mesh(cmd_buffer, x, y, z); @@ -8676,7 +8760,8 @@ ALWAYS_INLINE static void radv_emit_indirect_mesh_draw_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *info) { const struct radv_cmd_state *state = &cmd_buffer->state; - struct radeon_winsys *ws = cmd_buffer->device->ws; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + struct radeon_winsys *ws = device->ws; struct radeon_cmdbuf *cs = cmd_buffer->cs; const uint64_t va = radv_buffer_get_va(info->indirect->bo) + info->indirect->offset + info->indirect_offset; const uint64_t count_va = !info->count_buffer ? 0 @@ -8825,7 +8910,8 @@ static void radv_emit_indirect_draw_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *info) { const struct radv_cmd_state *state = &cmd_buffer->state; - struct radeon_winsys *ws = cmd_buffer->device->ws; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + struct radeon_winsys *ws = device->ws; struct radeon_cmdbuf *cs = cmd_buffer->cs; const uint64_t va = radv_buffer_get_va(info->indirect->bo) + info->indirect->offset + info->indirect_offset; const uint64_t count_va = info->count_buffer ? radv_buffer_get_va(info->count_buffer->bo) + @@ -8857,7 +8943,8 @@ radv_emit_indirect_draw_packets(struct radv_cmd_buffer *cmd_buffer, const struct static uint64_t radv_get_needed_dynamic_states(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); uint64_t dynamic_states = RADV_DYNAMIC_ALL; if (cmd_buffer->state.graphics_pipeline) @@ -9043,7 +9130,8 @@ radv_emit_fs_state(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_db_shader_control(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radeon_info *gpu_info = &pdev->info; const struct radv_shader *ps = cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT]; const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; @@ -9110,7 +9198,8 @@ radv_emit_db_shader_control(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_streamout_enable_state(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radv_streamout_state *so = &cmd_buffer->state.streamout; const bool streamout_enabled = radv_is_streamout_enabled(cmd_buffer); uint32_t enabled_stream_buffers_mask = 0; @@ -9149,10 +9238,10 @@ radv_cmdbuf_get_last_vgt_api_stage(const struct radv_cmd_buffer *cmd_buffer) static void radv_emit_graphics_shaders(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const gl_shader_stage last_vgt_api_stage = radv_cmdbuf_get_last_vgt_api_stage(cmd_buffer); const struct radv_shader *last_vgt_shader = cmd_buffer->state.shaders[last_vgt_api_stage]; - struct radv_device *device = cmd_buffer->device; struct radeon_cmdbuf *cs = cmd_buffer->cs; radv_foreach_stage(s, cmd_buffer->state.active_stages & RADV_GRAPHICS_STAGE_BITS) @@ -9213,7 +9302,7 @@ radv_emit_graphics_shaders(struct radv_cmd_buffer *cmd_buffer) /* Emit graphics states related to shaders. */ const struct radv_vgt_shader_key vgt_shader_cfg_key = - radv_get_vgt_shader_key(cmd_buffer->device, cmd_buffer->state.shaders, cmd_buffer->state.gs_copy_shader); + radv_get_vgt_shader_key(device, cmd_buffer->state.shaders, cmd_buffer->state.gs_copy_shader); radv_emit_vgt_gs_mode(device, cs, last_vgt_shader); radv_emit_vgt_reuse(device, cs, radv_get_shader(cmd_buffer->state.shaders, MESA_SHADER_TESS_EVAL), @@ -9236,7 +9325,7 @@ radv_emit_graphics_shaders(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_all_graphics_states(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *info) { - const struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_shader_part *ps_epilog = NULL; @@ -9347,7 +9436,7 @@ radv_emit_all_graphics_states(struct radv_cmd_buffer *cmd_buffer, const struct r static void radv_bind_graphics_shaders(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_physical_device *pdev = radv_device_physical(device); uint32_t push_constant_size = 0, dynamic_offset_count = 0; bool need_indirect_descriptor_sets = false; @@ -9382,7 +9471,7 @@ radv_bind_graphics_shaders(struct radv_cmd_buffer *cmd_buffer) if (!shader) continue; - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, shader->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, shader->bo); /* Compute push constants/indirect descriptors state. */ need_indirect_descriptor_sets |= radv_get_user_sgpr(shader, AC_UD_INDIRECT_DESCRIPTOR_SETS)->sgpr_idx != -1; @@ -9412,7 +9501,7 @@ radv_bind_graphics_shaders(struct radv_cmd_buffer *cmd_buffer) radv_bind_gs_copy_shader(cmd_buffer, gs_copy_shader); if (cmd_buffer->state.gs_copy_shader) { - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, cmd_buffer->state.gs_copy_shader->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, cmd_buffer->state.gs_copy_shader->bo); } /* Determine NGG GS info. */ @@ -9485,11 +9574,11 @@ radv_bind_graphics_shaders(struct radv_cmd_buffer *cmd_buffer) ALWAYS_INLINE static bool radv_before_draw(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *info, uint32_t drawCount, bool dgc) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const bool has_prefetch = pdev->info.gfx_level >= GFX7; - ASSERTED const unsigned cdw_max = - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 4096 + 128 * (drawCount - 1)); + ASSERTED const unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, 4096 + 128 * (drawCount - 1)); if (likely(!info->indirect)) { /* GFX6-GFX7 treat instance_count==0 as instance_count==1. There is @@ -9584,6 +9673,7 @@ radv_before_taskmesh_draw(struct radv_cmd_buffer *cmd_buffer, const struct radv_ radv_bind_graphics_shaders(cmd_buffer); } + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radeon_cmdbuf *ace_cs = cmd_buffer->gang.cs; struct radv_shader *task_shader = cmd_buffer->state.shaders[MESA_SHADER_TASK]; @@ -9593,10 +9683,9 @@ radv_before_taskmesh_draw(struct radv_cmd_buffer *cmd_buffer, const struct radv_ VK_SHADER_STAGE_MESH_BIT_EXT | VK_SHADER_STAGE_FRAGMENT_BIT | (task_shader ? VK_SHADER_STAGE_TASK_BIT_EXT : 0); const bool need_task_semaphore = task_shader && radv_flush_gang_leader_semaphore(cmd_buffer); - ASSERTED const unsigned cdw_max = - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 4096 + 128 * (drawCount - 1)); + ASSERTED const unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, 4096 + 128 * (drawCount - 1)); ASSERTED const unsigned ace_cdw_max = - !ace_cs ? 0 : radeon_check_space(cmd_buffer->device->ws, ace_cs, 4096 + 128 * (drawCount - 1)); + !ace_cs ? 0 : radeon_check_space(device->ws, ace_cs, 4096 + 128 * (drawCount - 1)); radv_emit_all_graphics_states(cmd_buffer, info); @@ -9639,7 +9728,8 @@ radv_before_taskmesh_draw(struct radv_cmd_buffer *cmd_buffer, const struct radv_ ALWAYS_INLINE static void radv_after_draw(struct radv_cmd_buffer *cmd_buffer, bool dgc) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radeon_info *gpu_info = &pdev->info; bool has_prefetch = pdev->info.gfx_level >= GFX7; /* Start prefetches after the draw has been started. Both will @@ -9852,6 +9942,7 @@ VKAPI_ATTR void VKAPI_CALL radv_CmdDrawMeshTasksEXT(VkCommandBuffer commandBuffer, uint32_t x, uint32_t y, uint32_t z) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_draw_info info; info.count = x * y * z; @@ -9867,8 +9958,7 @@ radv_CmdDrawMeshTasksEXT(VkCommandBuffer commandBuffer, uint32_t x, uint32_t y, return; if (radv_cmdbuf_has_stage(cmd_buffer, MESA_SHADER_TASK)) { - radv_emit_direct_taskmesh_draw_packets(cmd_buffer->device, &cmd_buffer->state, cmd_buffer->cs, - cmd_buffer->gang.cs, x, y, z); + radv_emit_direct_taskmesh_draw_packets(device, &cmd_buffer->state, cmd_buffer->cs, cmd_buffer->gang.cs, x, y, z); } else { radv_emit_direct_mesh_draw_packet(cmd_buffer, x, y, z); } @@ -9885,7 +9975,7 @@ radv_CmdDrawMeshTasksIndirectEXT(VkCommandBuffer commandBuffer, VkBuffer _buffer RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_buffer, buffer, _buffer); - + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_draw_info info; info.indirect = buffer; @@ -9901,8 +9991,8 @@ radv_CmdDrawMeshTasksIndirectEXT(VkCommandBuffer commandBuffer, VkBuffer _buffer return; if (radv_cmdbuf_has_stage(cmd_buffer, MESA_SHADER_TASK)) { - radv_emit_indirect_taskmesh_draw_packets(cmd_buffer->device, &cmd_buffer->state, cmd_buffer->cs, - cmd_buffer->gang.cs, &info, 0); + radv_emit_indirect_taskmesh_draw_packets(device, &cmd_buffer->state, cmd_buffer->cs, cmd_buffer->gang.cs, &info, + 0); } else { radv_emit_indirect_mesh_draw_packets(cmd_buffer, &info); } @@ -9919,7 +10009,8 @@ radv_CmdDrawMeshTasksIndirectCountEXT(VkCommandBuffer commandBuffer, VkBuffer _b RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_buffer, buffer, _buffer); RADV_FROM_HANDLE(radv_buffer, count_buffer, _countBuffer); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_draw_info info; info.indirect = buffer; @@ -9949,8 +10040,8 @@ radv_CmdDrawMeshTasksIndirectCountEXT(VkCommandBuffer commandBuffer, VkBuffer _b workaround_cond_va = radv_buffer_get_va(cmd_buffer->upload.upload_bo) + workaround_cond_off; } - radv_emit_indirect_taskmesh_draw_packets(cmd_buffer->device, &cmd_buffer->state, cmd_buffer->cs, - cmd_buffer->gang.cs, &info, workaround_cond_va); + radv_emit_indirect_taskmesh_draw_packets(device, &cmd_buffer->state, cmd_buffer->cs, cmd_buffer->gang.cs, &info, + workaround_cond_va); } else { radv_emit_indirect_mesh_draw_packets(cmd_buffer, &info); } @@ -9970,9 +10061,9 @@ radv_CmdExecuteGeneratedCommandsNV(VkCommandBuffer commandBuffer, VkBool32 isPre VK_FROM_HANDLE(radv_indirect_command_layout, layout, pGeneratedCommandsInfo->indirectCommandsLayout); VK_FROM_HANDLE(radv_pipeline, pipeline, pGeneratedCommandsInfo->pipeline); VK_FROM_HANDLE(radv_buffer, prep_buffer, pGeneratedCommandsInfo->preprocessBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const bool compute = layout->pipeline_bind_point == VK_PIPELINE_BIND_POINT_COMPUTE; const bool use_predication = radv_use_dgc_predication(cmd_buffer, pGeneratedCommandsInfo); - const struct radv_device *device = cmd_buffer->device; const struct radv_physical_device *pdev = radv_device_physical(device); /* Secondary command buffers are needed for the full extension but can't use @@ -10097,9 +10188,10 @@ static void radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader *compute_shader, const struct radv_dispatch_info *info) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); - unsigned dispatch_initiator = cmd_buffer->device->dispatch_initiator; - struct radeon_winsys *ws = cmd_buffer->device->ws; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + unsigned dispatch_initiator = device->dispatch_initiator; + struct radeon_winsys *ws = device->ws; bool predicating = cmd_buffer->state.predicating; struct radeon_cmdbuf *cs = cmd_buffer->cs; const struct radv_userdata_info *loc = radv_get_user_sgpr(compute_shader, AC_UD_CS_GRID_SIZE); @@ -10132,7 +10224,7 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv if (loc->sgpr_idx != -1) { unsigned reg = R_00B900_COMPUTE_USER_DATA_0 + loc->sgpr_idx * 4; - if (cmd_buffer->device->load_grid_size_from_user_sgpr) { + if (device->load_grid_size_from_user_sgpr) { assert(pdev->info.gfx_level >= GFX10_3); radeon_emit(cs, PKT3(PKT3_LOAD_SH_REG_INDEX, 3, 0)); radeon_emit(cs, info->va); @@ -10140,7 +10232,7 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv radeon_emit(cs, (reg - SI_SH_REG_OFFSET) >> 2); radeon_emit(cs, 3); } else { - radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, reg, info->va, true); + radv_emit_shader_pointer(device, cmd_buffer->cs, reg, info->va, true); } } @@ -10152,7 +10244,7 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv const unsigned ace_predication_size = 4 /* DISPATCH_INDIRECT */ + (needs_align32_workaround ? 6 * 3 /* 3x COPY_DATA */ : 0); - radv_cs_emit_compute_predication(cmd_buffer->device, &cmd_buffer->state, cs, cmd_buffer->state.mec_inv_pred_va, + radv_cs_emit_compute_predication(device, &cmd_buffer->state, cs, cmd_buffer->state.mec_inv_pred_va, &cmd_buffer->state.mec_inv_pred_emitted, ace_predication_size); if (needs_align32_workaround) { @@ -10190,9 +10282,8 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv radeon_emit(cs, info->va >> 32); if (cmd_buffer->qf == RADV_QUEUE_COMPUTE) { - radv_cs_emit_compute_predication(cmd_buffer->device, &cmd_buffer->state, cs, - cmd_buffer->state.mec_inv_pred_va, &cmd_buffer->state.mec_inv_pred_emitted, - 3 /* PKT3_DISPATCH_INDIRECT */); + radv_cs_emit_compute_predication(device, &cmd_buffer->state, cs, cmd_buffer->state.mec_inv_pred_va, + &cmd_buffer->state.mec_inv_pred_emitted, 3 /* PKT3_DISPATCH_INDIRECT */); predicating = false; } @@ -10233,7 +10324,7 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv } if (loc->sgpr_idx != -1) { - if (cmd_buffer->device->load_grid_size_from_user_sgpr) { + if (device->load_grid_size_from_user_sgpr) { assert(loc->num_sgprs == 3); radeon_set_sh_reg_seq(cs, R_00B900_COMPUTE_USER_DATA_0 + loc->sgpr_idx * 4, 3); @@ -10246,8 +10337,8 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv return; uint64_t va = radv_buffer_get_va(cmd_buffer->upload.upload_bo) + offset; - radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, - R_00B900_COMPUTE_USER_DATA_0 + loc->sgpr_idx * 4, va, true); + radv_emit_shader_pointer(device, cmd_buffer->cs, R_00B900_COMPUTE_USER_DATA_0 + loc->sgpr_idx * 4, va, + true); } } @@ -10265,7 +10356,7 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv } if (cmd_buffer->qf == RADV_QUEUE_COMPUTE) { - radv_cs_emit_compute_predication(cmd_buffer->device, &cmd_buffer->state, cs, cmd_buffer->state.mec_inv_pred_va, + radv_cs_emit_compute_predication(device, &cmd_buffer->state, cs, cmd_buffer->state.mec_inv_pred_va, &cmd_buffer->state.mec_inv_pred_emitted, 5 /* DISPATCH_DIRECT size */); predicating = false; } @@ -10308,11 +10399,12 @@ radv_upload_compute_shader_descriptors(struct radv_cmd_buffer *cmd_buffer, VkPip static void radv_emit_rt_stack_size(struct radv_cmd_buffer *cmd_buffer) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); unsigned rsrc2 = cmd_buffer->state.rt_prolog->config.rsrc2; if (cmd_buffer->state.rt_stack_size) rsrc2 |= S_00B12C_SCRATCH_EN(1); - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 3); + radeon_check_space(device->ws, cmd_buffer->cs, 3); radeon_set_sh_reg(cmd_buffer->cs, R_00B84C_COMPUTE_PGM_RSRC2, rsrc2); } @@ -10321,7 +10413,8 @@ radv_dispatch(struct radv_cmd_buffer *cmd_buffer, const struct radv_dispatch_inf struct radv_compute_pipeline *pipeline, struct radv_shader *compute_shader, VkPipelineBindPoint bind_point) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); bool has_prefetch = pdev->info.gfx_level >= GFX7; bool pipeline_is_dirty = pipeline != cmd_buffer->state.emitted_compute_pipeline; @@ -10395,7 +10488,8 @@ radv_dispatch(struct radv_cmd_buffer *cmd_buffer, const struct radv_dispatch_inf static void radv_dgc_before_dispatch(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_compute_pipeline *pipeline = cmd_buffer->state.compute_pipeline; struct radv_shader *compute_shader = cmd_buffer->state.shaders[MESA_SHADER_COMPUTE]; bool pipeline_is_dirty = pipeline != cmd_buffer->state.emitted_compute_pipeline; @@ -10513,9 +10607,10 @@ radv_trace_trace_rays(struct radv_cmd_buffer *cmd_buffer, const VkTraceRaysIndir if (!data) return; - uint32_t width = DIV_ROUND_UP(cmd->width, cmd_buffer->device->rra_trace.ray_history_resolution_scale); - uint32_t height = DIV_ROUND_UP(cmd->height, cmd_buffer->device->rra_trace.ray_history_resolution_scale); - uint32_t depth = DIV_ROUND_UP(cmd->depth, cmd_buffer->device->rra_trace.ray_history_resolution_scale); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + uint32_t width = DIV_ROUND_UP(cmd->width, device->rra_trace.ray_history_resolution_scale); + uint32_t height = DIV_ROUND_UP(cmd->height, device->rra_trace.ray_history_resolution_scale); + uint32_t depth = DIV_ROUND_UP(cmd->depth, device->rra_trace.ray_history_resolution_scale); struct radv_rra_ray_history_counter counter = { .dispatch_size = {width, height, depth}, @@ -10561,10 +10656,9 @@ radv_trace_trace_rays(struct radv_cmd_buffer *cmd_buffer, const VkTraceRaysIndir radv_src_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT, NULL) | radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT, NULL); - radv_update_buffer_cp( - cmd_buffer, - cmd_buffer->device->rra_trace.ray_history_addr + offsetof(struct radv_ray_history_header, dispatch_index), - &dispatch_index, sizeof(dispatch_index)); + radv_update_buffer_cp(cmd_buffer, + device->rra_trace.ray_history_addr + offsetof(struct radv_ray_history_header, dispatch_index), + &dispatch_index, sizeof(dispatch_index)); } enum radv_rt_mode { @@ -10596,12 +10690,13 @@ static void radv_trace_rays(struct radv_cmd_buffer *cmd_buffer, VkTraceRaysIndirectCommand2KHR *tables, uint64_t indirect_va, enum radv_rt_mode mode) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); - if (cmd_buffer->device->instance->debug_flags & RADV_DEBUG_NO_RT) + if (device->instance->debug_flags & RADV_DEBUG_NO_RT) return; - if (unlikely(cmd_buffer->device->rra_trace.ray_history_buffer)) + if (unlikely(device->rra_trace.ray_history_buffer)) radv_trace_trace_rays(cmd_buffer, tables, indirect_va); struct radv_compute_pipeline *pipeline = &cmd_buffer->state.rt_pipeline->base; @@ -10659,17 +10754,16 @@ radv_trace_rays(struct radv_cmd_buffer *cmd_buffer, VkTraceRaysIndirectCommand2K } else info.va = launch_size_va; - ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 15); + ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, 15); const struct radv_userdata_info *desc_loc = radv_get_user_sgpr(rt_prolog, AC_UD_CS_SBT_DESCRIPTORS); if (desc_loc->sgpr_idx != -1) { - radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, base_reg + desc_loc->sgpr_idx * 4, sbt_va, true); + radv_emit_shader_pointer(device, cmd_buffer->cs, base_reg + desc_loc->sgpr_idx * 4, sbt_va, true); } const struct radv_userdata_info *size_loc = radv_get_user_sgpr(rt_prolog, AC_UD_CS_RAY_LAUNCH_SIZE_ADDR); if (size_loc->sgpr_idx != -1) { - radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, base_reg + size_loc->sgpr_idx * 4, launch_size_va, - true); + radv_emit_shader_pointer(device, cmd_buffer->cs, base_reg + size_loc->sgpr_idx * 4, launch_size_va, true); } const struct radv_userdata_info *base_loc = radv_get_user_sgpr(rt_prolog, AC_UD_CS_RAY_DYNAMIC_CALLABLE_STACK_BASE); @@ -10683,8 +10777,7 @@ radv_trace_rays(struct radv_cmd_buffer *cmd_buffer, VkTraceRaysIndirectCommand2K struct radv_shader *traversal_shader = cmd_buffer->state.shaders[MESA_SHADER_INTERSECTION]; if (shader_loc->sgpr_idx != -1 && traversal_shader) { uint64_t traversal_va = traversal_shader->va | radv_rt_priority_traversal; - radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, base_reg + shader_loc->sgpr_idx * 4, traversal_va, - true); + radv_emit_shader_pointer(device, cmd_buffer->cs, base_reg + shader_loc->sgpr_idx * 4, traversal_va, true); } assert(cmd_buffer->cs->cdw <= cdw_max); @@ -10700,8 +10793,7 @@ radv_trace_rays(struct radv_cmd_buffer *cmd_buffer, VkTraceRaysIndirectCommand2K tables->height = 1; radv_upload_trace_rays_params(cmd_buffer, tables, mode, &launch_size_va, NULL); if (size_loc->sgpr_idx != -1) { - radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, base_reg + size_loc->sgpr_idx * 4, launch_size_va, - true); + radv_emit_shader_pointer(device, cmd_buffer->cs, base_reg + size_loc->sgpr_idx * 4, launch_size_va, true); } radv_dispatch(cmd_buffer, &info, pipeline, rt_prolog, VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR); @@ -10746,8 +10838,9 @@ radv_CmdTraceRaysIndirectKHR(VkCommandBuffer commandBuffer, VkDeviceAddress indirectDeviceAddress) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); - assert(cmd_buffer->device->use_global_bo_list); + assert(device->use_global_bo_list); VkTraceRaysIndirectCommand2KHR tables = { .raygenShaderRecordAddress = pRaygenShaderBindingTable->deviceAddress, @@ -10770,8 +10863,9 @@ VKAPI_ATTR void VKAPI_CALL radv_CmdTraceRaysIndirect2KHR(VkCommandBuffer commandBuffer, VkDeviceAddress indirectDeviceAddress) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); - assert(cmd_buffer->device->use_global_bo_list); + assert(device->use_global_bo_list); radv_trace_rays(cmd_buffer, NULL, indirectDeviceAddress, radv_rt_mode_indirect2); } @@ -10794,8 +10888,9 @@ static void radv_initialize_htile(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *range) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_cmd_state *state = &cmd_buffer->state; - uint32_t htile_value = radv_get_htile_initial_value(cmd_buffer->device, image); + uint32_t htile_value = radv_get_htile_initial_value(device, image); VkClearDepthStencilValue value = {0}; struct radv_barrier_data barrier = {0}; @@ -10834,7 +10929,7 @@ radv_handle_depth_image_transition(struct radv_cmd_buffer *cmd_buffer, struct ra unsigned dst_queue_mask, const VkImageSubresourceRange *range, struct radv_sample_locations_state *sample_locs) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); if (!radv_htile_enabled(image, range->baseMipLevel)) return; @@ -10884,7 +10979,8 @@ uint32_t radv_init_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *range, uint32_t value) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_barrier_data barrier = {0}; uint32_t flush_bits = 0; unsigned size = 0; @@ -10930,6 +11026,7 @@ radv_init_color_image_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_i VkImageLayout dst_layout, unsigned src_queue_mask, unsigned dst_queue_mask, const VkImageSubresourceRange *range) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); uint32_t flush_bits = 0; /* Transitioning from LAYOUT_UNDEFINED layout not everyone is @@ -10951,7 +11048,7 @@ radv_init_color_image_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_i if (radv_dcc_enabled(image, range->baseMipLevel)) { uint32_t value = 0xffffffffu; /* Fully expanded mode. */ - if (radv_layout_dcc_compressed(cmd_buffer->device, image, range->baseMipLevel, dst_layout, dst_queue_mask)) { + if (radv_layout_dcc_compressed(device, image, range->baseMipLevel, dst_layout, dst_queue_mask)) { value = 0u; } @@ -10996,6 +11093,7 @@ radv_handle_color_image_transition(struct radv_cmd_buffer *cmd_buffer, struct ra VkImageLayout src_layout, VkImageLayout dst_layout, unsigned src_queue_mask, unsigned dst_queue_mask, const VkImageSubresourceRange *range) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); bool dcc_decompressed = false, fast_clear_flushed = false; if (!radv_image_has_cmask(image) && !radv_image_has_fmask(image) && !radv_dcc_enabled(image, range->baseMipLevel)) @@ -11012,16 +11110,12 @@ radv_handle_color_image_transition(struct radv_cmd_buffer *cmd_buffer, struct ra if (radv_dcc_enabled(image, range->baseMipLevel)) { if (src_layout == VK_IMAGE_LAYOUT_PREINITIALIZED) { cmd_buffer->state.flush_bits |= radv_init_dcc(cmd_buffer, image, range, 0xffffffffu); - } else if (radv_layout_dcc_compressed(cmd_buffer->device, image, range->baseMipLevel, src_layout, - src_queue_mask) && - !radv_layout_dcc_compressed(cmd_buffer->device, image, range->baseMipLevel, dst_layout, - dst_queue_mask)) { + } else if (radv_layout_dcc_compressed(device, image, range->baseMipLevel, src_layout, src_queue_mask) && + !radv_layout_dcc_compressed(device, image, range->baseMipLevel, dst_layout, dst_queue_mask)) { radv_decompress_dcc(cmd_buffer, image, range); dcc_decompressed = true; - } else if (radv_layout_can_fast_clear(cmd_buffer->device, image, range->baseMipLevel, src_layout, - src_queue_mask) && - !radv_layout_can_fast_clear(cmd_buffer->device, image, range->baseMipLevel, dst_layout, - dst_queue_mask)) { + } else if (radv_layout_can_fast_clear(device, image, range->baseMipLevel, src_layout, src_queue_mask) && + !radv_layout_can_fast_clear(device, image, range->baseMipLevel, dst_layout, dst_queue_mask)) { radv_fast_clear_flush_image_inplace(cmd_buffer, image, range); fast_clear_flushed = true; } @@ -11029,8 +11123,8 @@ radv_handle_color_image_transition(struct radv_cmd_buffer *cmd_buffer, struct ra if (radv_image_need_retile(cmd_buffer, image)) radv_retile_transition(cmd_buffer, image, src_layout, dst_layout, dst_queue_mask); } else if (radv_image_has_cmask(image) || radv_image_has_fmask(image)) { - if (radv_layout_can_fast_clear(cmd_buffer->device, image, range->baseMipLevel, src_layout, src_queue_mask) && - !radv_layout_can_fast_clear(cmd_buffer->device, image, range->baseMipLevel, dst_layout, dst_queue_mask)) { + if (radv_layout_can_fast_clear(device, image, range->baseMipLevel, src_layout, src_queue_mask) && + !radv_layout_can_fast_clear(device, image, range->baseMipLevel, dst_layout, dst_queue_mask)) { radv_fast_clear_flush_image_inplace(cmd_buffer, image, range); fast_clear_flushed = true; } @@ -11038,14 +11132,14 @@ radv_handle_color_image_transition(struct radv_cmd_buffer *cmd_buffer, struct ra /* MSAA color decompress. */ const enum radv_fmask_compression src_fmask_comp = - radv_layout_fmask_compression(cmd_buffer->device, image, src_layout, src_queue_mask); + radv_layout_fmask_compression(device, image, src_layout, src_queue_mask); const enum radv_fmask_compression dst_fmask_comp = - radv_layout_fmask_compression(cmd_buffer->device, image, dst_layout, dst_queue_mask); + radv_layout_fmask_compression(device, image, dst_layout, dst_queue_mask); if (src_fmask_comp <= dst_fmask_comp) return; if (src_fmask_comp == RADV_FMASK_COMPRESSION_FULL) { - if (radv_dcc_enabled(image, range->baseMipLevel) && !radv_image_use_dcc_image_stores(cmd_buffer->device, image) && + if (radv_dcc_enabled(image, range->baseMipLevel) && !radv_image_use_dcc_image_stores(device, image) && !dcc_decompressed) { /* A DCC decompress is required before expanding FMASK * when DCC stores aren't supported to avoid being in @@ -11075,7 +11169,8 @@ radv_handle_image_transition(struct radv_cmd_buffer *cmd_buffer, struct radv_ima VkImageLayout dst_layout, uint32_t src_family_index, uint32_t dst_family_index, const VkImageSubresourceRange *range, struct radv_sample_locations_state *sample_locs) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); enum radv_queue_family src_qf = vk_queue_to_radv(pdev, src_family_index); enum radv_queue_family dst_qf = vk_queue_to_radv(pdev, dst_family_index); if (image->exclusive && src_family_index != dst_family_index) { @@ -11127,6 +11222,7 @@ radv_cp_dma_wait_for_stages(struct radv_cmd_buffer *cmd_buffer, VkPipelineStageF static void radv_barrier(struct radv_cmd_buffer *cmd_buffer, const VkDependencyInfo *dep_info, enum rgp_barrier_reason reason) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); enum radv_cmd_flush_bits src_flush_bits = 0; enum radv_cmd_flush_bits dst_flush_bits = 0; VkPipelineStageFlags2 src_stage_mask = 0; @@ -11206,7 +11302,7 @@ radv_barrier(struct radv_cmd_buffer *cmd_buffer, const VkDependencyInfo *dep_inf * Note that GFX9+ is supposed to have RAW dependency tracking, but it's buggy * so we can't rely on it fow now. */ - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 1); + radeon_check_space(device->ws, cmd_buffer->cs, 1); radeon_emit(cmd_buffer->cs, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); } else { const bool is_gfx_or_ace = cmd_buffer->qf == RADV_QUEUE_GENERAL || cmd_buffer->qf == RADV_QUEUE_COMPUTE; @@ -11238,7 +11334,8 @@ static void write_event(struct radv_cmd_buffer *cmd_buffer, struct radv_event *event, VkPipelineStageFlags2 stageMask, unsigned value) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; uint64_t va = radv_buffer_get_va(event->bo); @@ -11247,9 +11344,9 @@ write_event(struct radv_cmd_buffer *cmd_buffer, struct radv_event *event, VkPipe radv_emit_cache_flush(cmd_buffer); - radv_cs_add_buffer(cmd_buffer->device->ws, cs, event->bo); + radv_cs_add_buffer(device->ws, cs, event->bo); - ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 28); + ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cs, 28); if (stageMask & (VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_RESOLVE_BIT | VK_PIPELINE_STAGE_2_BLIT_BIT | VK_PIPELINE_STAGE_2_CLEAR_BIT)) { @@ -11336,6 +11433,7 @@ radv_CmdWaitEvents2(VkCommandBuffer commandBuffer, uint32_t eventCount, const Vk const VkDependencyInfo *pDependencyInfos) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radeon_cmdbuf *cs = cmd_buffer->cs; if (cmd_buffer->qf == RADV_QUEUE_VIDEO_DEC) @@ -11345,9 +11443,9 @@ radv_CmdWaitEvents2(VkCommandBuffer commandBuffer, uint32_t eventCount, const Vk RADV_FROM_HANDLE(radv_event, event, pEvents[i]); uint64_t va = radv_buffer_get_va(event->bo); - radv_cs_add_buffer(cmd_buffer->device->ws, cs, event->bo); + radv_cs_add_buffer(device->ws, cs, event->bo); - ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 7); + ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cs, 7); radv_cp_wait_mem(cs, cmd_buffer->qf, WAIT_REG_MEM_EQUAL, va, 1, 0xffffffff); assert(cmd_buffer->cs->cdw <= cdw_max); @@ -11359,7 +11457,8 @@ radv_CmdWaitEvents2(VkCommandBuffer commandBuffer, uint32_t eventCount, const Vk void radv_begin_conditional_rendering(struct radv_cmd_buffer *cmd_buffer, uint64_t va, bool draw_visible) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; unsigned pred_op = PREDICATION_OP_BOOL32; @@ -11400,7 +11499,7 @@ radv_begin_conditional_rendering(struct radv_cmd_buffer *cmd_buffer, uint64_t va pred_va = radv_buffer_get_va(cmd_buffer->upload.upload_bo) + pred_offset; - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 8); + radeon_check_space(device->ws, cmd_buffer->cs, 8); radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); radeon_emit( @@ -11486,6 +11585,7 @@ radv_CmdBindTransformFeedbackBuffersEXT(VkCommandBuffer commandBuffer, uint32_t const VkDeviceSize *pSizes) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_streamout_binding *sb = cmd_buffer->streamout_bindings; uint8_t enabled_mask = 0; @@ -11502,7 +11602,7 @@ radv_CmdBindTransformFeedbackBuffersEXT(VkCommandBuffer commandBuffer, uint32_t sb[idx].size = pSizes[i]; } - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, sb[idx].buffer->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, sb[idx].buffer->bo); enabled_mask |= 1 << idx; } @@ -11515,7 +11615,8 @@ radv_CmdBindTransformFeedbackBuffersEXT(VkCommandBuffer commandBuffer, uint32_t static void radv_set_streamout_enable(struct radv_cmd_buffer *cmd_buffer, bool enable) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_streamout_state *so = &cmd_buffer->state.streamout; bool old_streamout_enabled = radv_is_streamout_enabled(cmd_buffer); uint32_t old_hw_enabled_mask = so->hw_enabled_mask; @@ -11540,11 +11641,12 @@ radv_set_streamout_enable(struct radv_cmd_buffer *cmd_buffer, bool enable) static void radv_flush_vgt_streamout(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; unsigned reg_strmout_cntl; - ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 14); + ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, 14); /* The register is at different places on different ASICs. */ if (pdev->info.gfx_level >= GFX9) { @@ -11582,7 +11684,8 @@ radv_CmdBeginTransformFeedbackEXT(VkCommandBuffer commandBuffer, uint32_t firstC const VkDeviceSize *pCounterBufferOffsets) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_streamout_binding *sb = cmd_buffer->streamout_bindings; struct radv_streamout_state *so = &cmd_buffer->state.streamout; struct radeon_cmdbuf *cs = cmd_buffer->cs; @@ -11591,7 +11694,7 @@ radv_CmdBeginTransformFeedbackEXT(VkCommandBuffer commandBuffer, uint32_t firstC if (!pdev->use_ngg_streamout) radv_flush_vgt_streamout(cmd_buffer); - ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, MAX_SO_BUFFERS * 10); + ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, MAX_SO_BUFFERS * 10); u_foreach_bit (i, so->enabled_mask) { int32_t counter_buffer_idx = i - firstCounterBuffer; @@ -11611,7 +11714,7 @@ radv_CmdBeginTransformFeedbackEXT(VkCommandBuffer commandBuffer, uint32_t firstC va += radv_buffer_get_va(buffer->bo); va += buffer->offset + counter_buffer_offset; - radv_cs_add_buffer(cmd_buffer->device->ws, cs, buffer->bo); + radv_cs_add_buffer(device->ws, cs, buffer->bo); } if (pdev->use_ngg_streamout) { @@ -11670,7 +11773,8 @@ radv_CmdEndTransformFeedbackEXT(VkCommandBuffer commandBuffer, uint32_t firstCou const VkBuffer *pCounterBuffers, const VkDeviceSize *pCounterBufferOffsets) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_streamout_state *so = &cmd_buffer->state.streamout; struct radeon_cmdbuf *cs = cmd_buffer->cs; @@ -11684,7 +11788,7 @@ radv_CmdEndTransformFeedbackEXT(VkCommandBuffer commandBuffer, uint32_t firstCou radv_flush_vgt_streamout(cmd_buffer); } - ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, MAX_SO_BUFFERS * 12); + ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, MAX_SO_BUFFERS * 12); u_foreach_bit (i, so->enabled_mask) { int32_t counter_buffer_idx = i - firstCounterBuffer; @@ -11704,7 +11808,7 @@ radv_CmdEndTransformFeedbackEXT(VkCommandBuffer commandBuffer, uint32_t firstCou va += radv_buffer_get_va(buffer->bo); va += buffer->offset + counter_buffer_offset; - radv_cs_add_buffer(cmd_buffer->device->ws, cs, buffer->bo); + radv_cs_add_buffer(device->ws, cs, buffer->bo); } if (pdev->use_ngg_streamout) { @@ -11748,7 +11852,8 @@ radv_CmdEndTransformFeedbackEXT(VkCommandBuffer commandBuffer, uint32_t firstCou static void radv_emit_strmout_buffer(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *draw_info) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const enum amd_gfx_level gfx_level = pdev->info.gfx_level; uint64_t va = radv_buffer_get_va(draw_info->strmout_buffer->bo); struct radeon_cmdbuf *cs = cmd_buffer->cs; @@ -11778,7 +11883,7 @@ radv_emit_strmout_buffer(struct radv_cmd_buffer *cmd_buffer, const struct radv_d radeon_emit(cs, 0); /* unused */ } - radv_cs_add_buffer(cmd_buffer->device->ws, cs, draw_info->strmout_buffer->bo); + radv_cs_add_buffer(device->ws, cs, draw_info->strmout_buffer->bo); } VKAPI_ATTR void VKAPI_CALL @@ -11814,12 +11919,13 @@ radv_CmdWriteBufferMarker2AMD(VkCommandBuffer commandBuffer, VkPipelineStageFlag { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_buffer, buffer, dstBuffer); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; const uint64_t va = radv_buffer_get_va(buffer->bo) + buffer->offset + dstOffset; if (cmd_buffer->qf == RADV_QUEUE_TRANSFER) { - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 4); + radeon_check_space(device->ws, cmd_buffer->cs, 4); radeon_emit(cmd_buffer->cs, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, SDMA_FENCE_MTYPE_UC)); radeon_emit(cs, va); radeon_emit(cs, va >> 32); @@ -11829,7 +11935,7 @@ radv_CmdWriteBufferMarker2AMD(VkCommandBuffer commandBuffer, VkPipelineStageFlag radv_emit_cache_flush(cmd_buffer); - ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 12); + ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, 12); if (!(stage & ~VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT)) { radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); @@ -11861,11 +11967,12 @@ radv_CmdUpdatePipelineIndirectBufferNV(VkCommandBuffer commandBuffer, VkPipeline { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_pipeline, pipeline, _pipeline); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_compute_pipeline *compute_pipeline = radv_pipeline_to_compute(pipeline); const uint64_t va = compute_pipeline->indirect.va; struct radv_compute_pipeline_metadata metadata; - radv_get_compute_pipeline_metadata(cmd_buffer->device, compute_pipeline, &metadata); + radv_get_compute_pipeline_metadata(device, compute_pipeline, &metadata); assert(sizeof(metadata) <= compute_pipeline->indirect.size); radv_write_data(cmd_buffer, V_370_ME, va, sizeof(metadata) / 4, (const uint32_t *)&metadata, false); @@ -11985,7 +12092,7 @@ static void radv_bind_compute_shader(struct radv_cmd_buffer *cmd_buffer, struct radv_shader_object *shader_obj) { struct radv_shader *shader = shader_obj ? shader_obj->shader : NULL; - const struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; @@ -11994,9 +12101,9 @@ radv_bind_compute_shader(struct radv_cmd_buffer *cmd_buffer, struct radv_shader_ if (!shader_obj) return; - ASSERTED const unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 128); + ASSERTED const unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, 128); - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, shader->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, shader->bo); radv_emit_compute_shader(pdev, cs, shader); diff --git a/src/amd/vulkan/radv_device_generated_commands.c b/src/amd/vulkan/radv_device_generated_commands.c index a0131106f15..454b125eab6 100644 --- a/src/amd/vulkan/radv_device_generated_commands.c +++ b/src/amd/vulkan/radv_device_generated_commands.c @@ -2068,7 +2068,8 @@ radv_prepare_dgc_compute(struct radv_cmd_buffer *cmd_buffer, const VkGeneratedCo { VK_FROM_HANDLE(radv_indirect_command_layout, layout, pGeneratedCommandsInfo->indirectCommandsLayout); VK_FROM_HANDLE(radv_pipeline, pipeline, pGeneratedCommandsInfo->pipeline); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); *upload_size = MAX2(*upload_size, 16); @@ -2078,7 +2079,7 @@ radv_prepare_dgc_compute(struct radv_cmd_buffer *cmd_buffer, const VkGeneratedCo } params->dispatch_params_offset = layout->dispatch_params_offset; - params->dispatch_initiator = cmd_buffer->device->dispatch_initiator | S_00B800_FORCE_START_AT_000(1); + params->dispatch_initiator = device->dispatch_initiator | S_00B800_FORCE_START_AT_000(1); params->is_dispatch = 1; if (cond_render_enabled) { @@ -2114,6 +2115,7 @@ radv_prepare_dgc(struct radv_cmd_buffer *cmd_buffer, const VkGeneratedCommandsIn VK_FROM_HANDLE(radv_pipeline, pipeline, pGeneratedCommandsInfo->pipeline); VK_FROM_HANDLE(radv_buffer, prep_buffer, pGeneratedCommandsInfo->preprocessBuffer); VK_FROM_HANDLE(radv_buffer, stream_buffer, pGeneratedCommandsInfo->pStreams[0].buffer); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; unsigned upload_offset, upload_size; struct radv_buffer token_buffer; @@ -2123,7 +2125,7 @@ radv_prepare_dgc(struct radv_cmd_buffer *cmd_buffer, const VkGeneratedCommandsIn radv_get_sequence_size(layout, pipeline, &cmd_stride, &upload_stride); unsigned cmd_buf_size = - radv_align_cmdbuf_size(cmd_buffer->device, cmd_stride * pGeneratedCommandsInfo->sequencesCount, AMD_IP_GFX); + radv_align_cmdbuf_size(device, cmd_stride * pGeneratedCommandsInfo->sequencesCount, AMD_IP_GFX); uint64_t upload_addr = radv_buffer_get_va(prep_buffer->bo) + prep_buffer->offset + pGeneratedCommandsInfo->preprocessOffset; @@ -2213,7 +2215,7 @@ radv_prepare_dgc(struct radv_cmd_buffer *cmd_buffer, const VkGeneratedCommandsIn upload_data = (char *)upload_data + 16 * layout->dynamic_offset_count; } - radv_buffer_init(&token_buffer, cmd_buffer->device, cmd_buffer->upload.upload_bo, upload_size, upload_offset); + radv_buffer_init(&token_buffer, device, cmd_buffer->upload.upload_bo, upload_size, upload_offset); VkWriteDescriptorSet ds_writes[5]; VkDescriptorBufferInfo buf_info[ARRAY_SIZE(ds_writes)]; @@ -2270,14 +2272,13 @@ radv_prepare_dgc(struct radv_cmd_buffer *cmd_buffer, const VkGeneratedCommandsIn RADV_META_SAVE_COMPUTE_PIPELINE | RADV_META_SAVE_DESCRIPTORS | RADV_META_SAVE_CONSTANTS); radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE, - cmd_buffer->device->meta_state.dgc_prepare.pipeline); + device->meta_state.dgc_prepare.pipeline); - vk_common_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), - cmd_buffer->device->meta_state.dgc_prepare.p_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, - sizeof(params), ¶ms); + vk_common_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), device->meta_state.dgc_prepare.p_layout, + VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(params), ¶ms); - radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, - cmd_buffer->device->meta_state.dgc_prepare.p_layout, 0, ds_cnt, ds_writes); + radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, device->meta_state.dgc_prepare.p_layout, 0, + ds_cnt, ds_writes); unsigned block_count = MAX2(1, DIV_ROUND_UP(pGeneratedCommandsInfo->sequencesCount, 64)); vk_common_CmdDispatch(radv_cmd_buffer_to_handle(cmd_buffer), block_count, 1, 1); diff --git a/src/amd/vulkan/radv_perfcounter.c b/src/amd/vulkan/radv_perfcounter.c index 709d2e515c5..2ee813f244c 100644 --- a/src/amd/vulkan/radv_perfcounter.c +++ b/src/amd/vulkan/radv_perfcounter.c @@ -470,7 +470,8 @@ radv_emit_instance(struct radv_cmd_buffer *cmd_buffer, int se, int instance) static void radv_emit_select(struct radv_cmd_buffer *cmd_buffer, struct ac_pc_block *block, unsigned count, unsigned *selectors) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const enum amd_gfx_level gfx_level = pdev->info.gfx_level; const enum radv_queue_family qf = cmd_buffer->qf; struct ac_pc_block_base *regs = block->b->b; @@ -497,7 +498,8 @@ static void radv_pc_emit_block_instance_read(struct radv_cmd_buffer *cmd_buffer, struct ac_pc_block *block, unsigned count, uint64_t va) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct ac_pc_block_base *regs = block->b->b; struct radeon_cmdbuf *cs = cmd_buffer->cs; unsigned reg = regs->counter0_lo; @@ -524,7 +526,8 @@ radv_pc_emit_block_instance_read(struct radv_cmd_buffer *cmd_buffer, struct ac_p static void radv_pc_sample_block(struct radv_cmd_buffer *cmd_buffer, struct ac_pc_block *block, unsigned count, uint64_t va) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); unsigned se_end = 1; if (block->b->b->flags & AC_PC_BLOCK_SE) se_end = pdev->info.max_se; @@ -562,7 +565,8 @@ radv_pc_wait_idle(struct radv_cmd_buffer *cmd_buffer) static void radv_pc_stop_and_sample(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query_pool *pool, uint64_t va, bool end) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); @@ -571,14 +575,14 @@ radv_pc_stop_and_sample(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query radv_pc_wait_idle(cmd_buffer); radv_emit_instance(cmd_buffer, -1, -1); - radv_emit_windowed_counters(cmd_buffer->device, cs, cmd_buffer->qf, false); + radv_emit_windowed_counters(device, cs, cmd_buffer->qf, false); radeon_set_uconfig_reg( cs, R_036020_CP_PERFMON_CNTL, S_036020_PERFMON_STATE(V_036020_CP_PERFMON_STATE_STOP_COUNTING) | S_036020_PERFMON_SAMPLE_ENABLE(1)); for (unsigned pass = 0; pass < pool->num_passes; ++pass) { - uint64_t pred_va = radv_buffer_get_va(cmd_buffer->device->perf_counter_bo) + PERF_CTR_BO_PASS_OFFSET + 8 * pass; + uint64_t pred_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_PASS_OFFSET + 8 * pass; uint64_t reg_va = va + (end ? 8 : 0); radeon_emit(cs, PKT3(PKT3_COND_EXEC, 3, 0)); @@ -627,21 +631,22 @@ radv_pc_stop_and_sample(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query void radv_pc_begin_query(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query_pool *pool, uint64_t va) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radeon_cmdbuf *cs = cmd_buffer->cs; - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + const struct radv_physical_device *pdev = radv_device_physical(device); ASSERTED unsigned cdw_max; cmd_buffer->state.uses_perf_counters = true; - cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, + cdw_max = radeon_check_space(device->ws, cs, 256 + /* Random one time stuff */ 10 * pool->num_passes + /* COND_EXECs */ pool->b.stride / 8 * (5 + 8)); - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, pool->b.bo); - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, cmd_buffer->device->perf_counter_bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, pool->b.bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, device->perf_counter_bo); - uint64_t perf_ctr_va = radv_buffer_get_va(cmd_buffer->device->perf_counter_bo) + PERF_CTR_BO_FENCE_OFFSET; + uint64_t perf_ctr_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_FENCE_OFFSET; radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0)); radeon_emit(cs, S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME)); radeon_emit(cs, perf_ctr_va); @@ -653,12 +658,12 @@ radv_pc_begin_query(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query_poo radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, S_036020_PERFMON_STATE(V_036020_CP_PERFMON_STATE_DISABLE_AND_RESET)); - radv_emit_inhibit_clockgating(cmd_buffer->device, cs, true); - radv_emit_spi_config_cntl(cmd_buffer->device, cs, true); - radv_perfcounter_emit_shaders(cmd_buffer->device, cs, 0x7f); + radv_emit_inhibit_clockgating(device, cs, true); + radv_emit_spi_config_cntl(device, cs, true); + radv_perfcounter_emit_shaders(device, cs, 0x7f); for (unsigned pass = 0; pass < pool->num_passes; ++pass) { - uint64_t pred_va = radv_buffer_get_va(cmd_buffer->device->perf_counter_bo) + PERF_CTR_BO_PASS_OFFSET + 8 * pass; + uint64_t pred_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_PASS_OFFSET + 8 * pass; radeon_emit(cs, PKT3(PKT3_COND_EXEC, 3, 0)); radeon_emit(cs, pred_va); @@ -697,7 +702,7 @@ radv_pc_begin_query(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query_poo radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, S_036020_PERFMON_STATE(V_036020_CP_PERFMON_STATE_START_COUNTING)); - radv_emit_windowed_counters(cmd_buffer->device, cs, cmd_buffer->qf, true); + radv_emit_windowed_counters(device, cs, cmd_buffer->qf, true); assert(cmd_buffer->cs->cdw <= cdw_max); } @@ -705,19 +710,20 @@ radv_pc_begin_query(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query_poo void radv_pc_end_query(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query_pool *pool, uint64_t va) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; ASSERTED unsigned cdw_max; - cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, + cdw_max = radeon_check_space(device->ws, cs, 256 + /* Reserved for things that don't scale with passes/counters */ 5 * pool->num_passes + /* COND_EXECs */ pool->b.stride / 8 * 8); - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, pool->b.bo); - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, cmd_buffer->device->perf_counter_bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, pool->b.bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, device->perf_counter_bo); - uint64_t perf_ctr_va = radv_buffer_get_va(cmd_buffer->device->perf_counter_bo) + PERF_CTR_BO_FENCE_OFFSET; + uint64_t perf_ctr_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_FENCE_OFFSET; radv_cs_emit_write_event_eop(cs, pdev->info.gfx_level, cmd_buffer->qf, V_028A90_BOTTOM_OF_PIPE_TS, 0, EOP_DST_SEL_MEM, EOP_DATA_SEL_VALUE_32BIT, perf_ctr_va, 1, cmd_buffer->gfx9_fence_va); radv_cp_wait_mem(cs, cmd_buffer->qf, WAIT_REG_MEM_EQUAL, perf_ctr_va, 1, 0xffffffff); @@ -727,8 +733,8 @@ radv_pc_end_query(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query_pool radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, S_036020_PERFMON_STATE(V_036020_CP_PERFMON_STATE_DISABLE_AND_RESET)); - radv_emit_spi_config_cntl(cmd_buffer->device, cs, false); - radv_emit_inhibit_clockgating(cmd_buffer->device, cs, false); + radv_emit_spi_config_cntl(device, cs, false); + radv_emit_inhibit_clockgating(device, cs, false); assert(cmd_buffer->cs->cdw <= cdw_max); } diff --git a/src/amd/vulkan/radv_private.h b/src/amd/vulkan/radv_private.h index a0de7d91482..1d177adcec5 100644 --- a/src/amd/vulkan/radv_private.h +++ b/src/amd/vulkan/radv_private.h @@ -1844,8 +1844,6 @@ struct radv_cmd_buffer_upload { struct radv_cmd_buffer { struct vk_command_buffer vk; - struct radv_device *device; - VkCommandBufferUsageFlags usage_flags; struct radeon_cmdbuf *cs; struct radv_cmd_state state; @@ -1949,6 +1947,12 @@ struct radv_cmd_buffer { struct util_dynarray ray_history; }; +static inline struct radv_device * +radv_cmd_buffer_device(const struct radv_cmd_buffer *cmd_buffer) +{ + return (struct radv_device *)cmd_buffer->vk.base.device; +} + static inline bool radv_cmdbuf_has_stage(const struct radv_cmd_buffer *cmd_buffer, gl_shader_stage stage) { diff --git a/src/amd/vulkan/radv_query.c b/src/amd/vulkan/radv_query.c index 3c0e4398949..addcc67571c 100644 --- a/src/amd/vulkan/radv_query.c +++ b/src/amd/vulkan/radv_query.c @@ -1112,7 +1112,7 @@ radv_query_shader(struct radv_cmd_buffer *cmd_buffer, VkPipeline *pipeline, stru uint32_t dst_stride, size_t dst_size, uint32_t count, uint32_t flags, uint32_t pipeline_stats_mask, uint32_t avail_offset, bool uses_gds) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_meta_saved_state saved_state; struct radv_buffer src_buffer, dst_buffer; @@ -1734,7 +1734,8 @@ radv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQueryPool queryPoo RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_query_pool, pool, queryPool); RADV_FROM_HANDLE(radv_buffer, dst_buffer, dstBuffer); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; uint64_t va = radv_buffer_get_va(pool->bo); uint64_t dest_va = radv_buffer_get_va(dst_buffer->bo); @@ -1744,13 +1745,13 @@ radv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQueryPool queryPoo if (!queryCount) return; - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, pool->bo); - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, dst_buffer->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, pool->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, dst_buffer->bo); /* Workaround engines that forget to properly specify WAIT_BIT because some driver implicitly * synchronizes before query copy. */ - if (cmd_buffer->device->instance->drirc.flush_before_query_copy) + if (device->instance->drirc.flush_before_query_copy) cmd_buffer->state.flush_bits |= cmd_buffer->active_query_flush_bits; /* From the Vulkan spec 1.1.108: @@ -1765,20 +1766,20 @@ radv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQueryPool queryPoo switch (pool->vk.query_type) { case VK_QUERY_TYPE_OCCLUSION: - radv_query_shader(cmd_buffer, &cmd_buffer->device->meta_state.query.occlusion_query_pipeline, pool->bo, - dst_buffer->bo, firstQuery * pool->stride, dst_buffer->offset + dstOffset, pool->stride, stride, - dst_size, queryCount, flags, 0, 0, false); + radv_query_shader(cmd_buffer, &device->meta_state.query.occlusion_query_pipeline, pool->bo, dst_buffer->bo, + firstQuery * pool->stride, dst_buffer->offset + dstOffset, pool->stride, stride, dst_size, + queryCount, flags, 0, 0, false); break; case VK_QUERY_TYPE_PIPELINE_STATISTICS: if (flags & VK_QUERY_RESULT_WAIT_BIT) { const uint32_t task_invoc_offset = radv_get_pipelinestat_query_offset(VK_QUERY_PIPELINE_STATISTIC_TASK_SHADER_INVOCATIONS_BIT_EXT); - const unsigned pipelinestat_block_size = radv_get_pipelinestat_query_size(cmd_buffer->device); + const unsigned pipelinestat_block_size = radv_get_pipelinestat_query_size(device); for (unsigned i = 0; i < queryCount; ++i, dest_va += stride) { unsigned query = firstQuery + i; - radeon_check_space(cmd_buffer->device->ws, cs, 7); + radeon_check_space(device->ws, cs, 7); uint64_t avail_va = va + pool->availability_offset + 4 * query; @@ -1790,14 +1791,14 @@ radv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQueryPool queryPoo const uint64_t start_va = src_va + task_invoc_offset + 4; const uint64_t stop_va = start_va + pipelinestat_block_size; - radeon_check_space(cmd_buffer->device->ws, cs, 7 * 2); + radeon_check_space(device->ws, cs, 7 * 2); radv_cp_wait_mem(cs, cmd_buffer->qf, WAIT_REG_MEM_GREATER_OR_EQUAL, start_va, 0x80000000, 0xffffffff); radv_cp_wait_mem(cs, cmd_buffer->qf, WAIT_REG_MEM_GREATER_OR_EQUAL, stop_va, 0x80000000, 0xffffffff); } } } - radv_query_shader(cmd_buffer, &cmd_buffer->device->meta_state.query.pipeline_statistics_query_pipeline, pool->bo, + radv_query_shader(cmd_buffer, &device->meta_state.query.pipeline_statistics_query_pipeline, pool->bo, dst_buffer->bo, firstQuery * pool->stride, dst_buffer->offset + dstOffset, pool->stride, stride, dst_size, queryCount, flags, pool->vk.pipeline_statistics, pool->availability_offset + 4 * firstQuery, false); @@ -1812,7 +1813,7 @@ radv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQueryPool queryPoo unsigned query = firstQuery + i; uint64_t local_src_va = va + query * pool->stride; - radeon_check_space(cmd_buffer->device->ws, cs, 7); + radeon_check_space(device->ws, cs, 7); /* Wait on the high 32 bits of the timestamp in * case the low part is 0xffffffff. @@ -1822,9 +1823,9 @@ radv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQueryPool queryPoo } } - radv_query_shader(cmd_buffer, &cmd_buffer->device->meta_state.query.timestamp_query_pipeline, pool->bo, - dst_buffer->bo, firstQuery * pool->stride, dst_buffer->offset + dstOffset, pool->stride, stride, - dst_size, queryCount, flags, 0, 0, false); + radv_query_shader(cmd_buffer, &device->meta_state.query.timestamp_query_pipeline, pool->bo, dst_buffer->bo, + firstQuery * pool->stride, dst_buffer->offset + dstOffset, pool->stride, stride, dst_size, + queryCount, flags, 0, 0, false); break; case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT: if (flags & VK_QUERY_RESULT_WAIT_BIT) { @@ -1832,7 +1833,7 @@ radv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQueryPool queryPoo unsigned query = firstQuery + i; uint64_t src_va = va + query * pool->stride; - radeon_check_space(cmd_buffer->device->ws, cs, 7 * 4); + radeon_check_space(device->ws, cs, 7 * 4); /* Wait on the upper word of all results. */ for (unsigned j = 0; j < 4; j++, src_va += 8) { @@ -1841,7 +1842,7 @@ radv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQueryPool queryPoo } } - radv_query_shader(cmd_buffer, &cmd_buffer->device->meta_state.query.tfb_query_pipeline, pool->bo, dst_buffer->bo, + radv_query_shader(cmd_buffer, &device->meta_state.query.tfb_query_pipeline, pool->bo, dst_buffer->bo, firstQuery * pool->stride, dst_buffer->offset + dstOffset, pool->stride, stride, dst_size, queryCount, flags, 0, 0, false); break; @@ -1853,7 +1854,7 @@ radv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQueryPool queryPoo unsigned query = firstQuery + i; uint64_t src_va = va + query * pool->stride; - radeon_check_space(cmd_buffer->device->ws, cs, 7 * 4); + radeon_check_space(device->ws, cs, 7 * 4); /* Wait on the upper word of the PrimitiveStorageNeeded result. */ radv_cp_wait_mem(cs, cmd_buffer->qf, WAIT_REG_MEM_GREATER_OR_EQUAL, src_va + 4, 0x80000000, 0xffffffff); @@ -1866,7 +1867,7 @@ radv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQueryPool queryPoo } } - radv_query_shader(cmd_buffer, &cmd_buffer->device->meta_state.query.pg_query_pipeline, pool->bo, dst_buffer->bo, + radv_query_shader(cmd_buffer, &device->meta_state.query.pg_query_pipeline, pool->bo, dst_buffer->bo, firstQuery * pool->stride, dst_buffer->offset + dstOffset, pool->stride, stride, dst_size, queryCount, flags, 0, 0, pool->uses_gds && pdev->info.gfx_level < GFX11); break; @@ -1876,7 +1877,7 @@ radv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQueryPool queryPoo for (unsigned i = 0; i < queryCount; ++i, dest_va += stride) { unsigned query = firstQuery + i; - radeon_check_space(cmd_buffer->device->ws, cs, 7); + radeon_check_space(device->ws, cs, 7); uint64_t avail_va = va + pool->availability_offset + 4 * query; @@ -1884,17 +1885,17 @@ radv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQueryPool queryPoo radv_cp_wait_mem(cs, cmd_buffer->qf, WAIT_REG_MEM_EQUAL, avail_va, 1, 0xffffffff); } } - radv_query_shader(cmd_buffer, &cmd_buffer->device->meta_state.query.pipeline_statistics_query_pipeline, - pool->bo, dst_buffer->bo, firstQuery * pool->stride, dst_buffer->offset + dstOffset, - pool->stride, stride, dst_size, queryCount, flags, 1 << 13, - pool->availability_offset + 4 * firstQuery, false); + radv_query_shader(cmd_buffer, &device->meta_state.query.pipeline_statistics_query_pipeline, pool->bo, + dst_buffer->bo, firstQuery * pool->stride, dst_buffer->offset + dstOffset, pool->stride, + stride, dst_size, queryCount, flags, 1 << 13, pool->availability_offset + 4 * firstQuery, + false); } else { if (flags & VK_QUERY_RESULT_WAIT_BIT) { for (unsigned i = 0; i < queryCount; i++) { unsigned query = firstQuery + i; uint64_t src_va = va + query * pool->stride; - radeon_check_space(cmd_buffer->device->ws, cs, 7 * 2); + radeon_check_space(device->ws, cs, 7 * 2); /* Wait on the upper word. */ radv_cp_wait_mem(cs, cmd_buffer->qf, WAIT_REG_MEM_GREATER_OR_EQUAL, src_va + 4, 0x80000000, 0xffffffff); @@ -1902,9 +1903,9 @@ radv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQueryPool queryPoo } } - radv_query_shader(cmd_buffer, &cmd_buffer->device->meta_state.query.ms_prim_gen_query_pipeline, pool->bo, - dst_buffer->bo, firstQuery * pool->stride, dst_buffer->offset + dstOffset, pool->stride, - stride, dst_size, queryCount, flags, 0, 0, false); + radv_query_shader(cmd_buffer, &device->meta_state.query.ms_prim_gen_query_pipeline, pool->bo, dst_buffer->bo, + firstQuery * pool->stride, dst_buffer->offset + dstOffset, pool->stride, stride, dst_size, + queryCount, flags, 0, 0, false); } break; default: @@ -1932,7 +1933,8 @@ radv_CmdResetQueryPool(VkCommandBuffer commandBuffer, VkQueryPool queryPool, uin { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_query_pool, pool, queryPool); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); uint32_t value = query_clear_value(pool->vk.query_type); uint32_t flush_bits = 0; @@ -1998,9 +2000,10 @@ event_type_for_stream(unsigned stream) static void emit_sample_streamout(struct radv_cmd_buffer *cmd_buffer, uint64_t va, uint32_t index) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radeon_cmdbuf *cs = cmd_buffer->cs; - radeon_check_space(cmd_buffer->device->ws, cs, 4); + radeon_check_space(device->ws, cs, 4); assert(index < MAX_SO_STREAMS); @@ -2059,11 +2062,12 @@ static void emit_begin_query(struct radv_cmd_buffer *cmd_buffer, struct radv_query_pool *pool, uint64_t va, VkQueryType query_type, VkQueryControlFlags flags, uint32_t index) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; switch (query_type) { case VK_QUERY_TYPE_OCCLUSION: - radeon_check_space(cmd_buffer->device->ws, cs, 11); + radeon_check_space(device->ws, cs, 11); ++cmd_buffer->state.active_occlusion_queries; if (cmd_buffer->state.active_occlusion_queries == 1) { @@ -2101,7 +2105,7 @@ emit_begin_query(struct radv_cmd_buffer *cmd_buffer, struct radv_query_pool *poo radeon_emit(cs, va >> 32); break; case VK_QUERY_TYPE_PIPELINE_STATISTICS: { - radeon_check_space(cmd_buffer->device->ws, cs, 4); + radeon_check_space(device->ws, cs, 4); ++cmd_buffer->state.active_pipeline_queries; @@ -2156,14 +2160,14 @@ emit_begin_query(struct radv_cmd_buffer *cmd_buffer, struct radv_query_pool *poo if (pdev->info.gfx_level >= GFX11) { va += task_invoc_offset; - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->gang.cs, 4); + radeon_check_space(device->ws, cmd_buffer->gang.cs, 4); radeon_emit(cmd_buffer->gang.cs, PKT3(PKT3_EVENT_WRITE, 2, 0)); radeon_emit(cmd_buffer->gang.cs, EVENT_TYPE(V_028A90_SAMPLE_PIPELINESTAT) | EVENT_INDEX(2)); radeon_emit(cmd_buffer->gang.cs, va); radeon_emit(cmd_buffer->gang.cs, va >> 32); } else { - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->gang.cs, 11); + radeon_check_space(device->ws, cmd_buffer->gang.cs, 11); gfx10_copy_gds_query_ace(cmd_buffer, RADV_SHADER_QUERY_TS_INVOCATION_OFFSET, va + task_invoc_offset); radv_cs_write_data_imm(cmd_buffer->gang.cs, V_370_ME, va + task_invoc_offset + 4, 0x80000000); @@ -2256,7 +2260,7 @@ emit_begin_query(struct radv_cmd_buffer *cmd_buffer, struct radv_query_pool *poo } case VK_QUERY_TYPE_MESH_PRIMITIVES_GENERATED_EXT: { if (pdev->info.gfx_level >= GFX11) { - radeon_check_space(cmd_buffer->device->ws, cs, 4); + radeon_check_space(device->ws, cs, 4); ++cmd_buffer->state.active_pipeline_queries; @@ -2289,11 +2293,12 @@ static void emit_end_query(struct radv_cmd_buffer *cmd_buffer, struct radv_query_pool *pool, uint64_t va, uint64_t avail_va, VkQueryType query_type, uint32_t index) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; switch (query_type) { case VK_QUERY_TYPE_OCCLUSION: - radeon_check_space(cmd_buffer->device->ws, cs, 14); + radeon_check_space(device->ws, cs, 14); cmd_buffer->state.active_occlusion_queries--; if (cmd_buffer->state.active_occlusion_queries == 0) { @@ -2320,9 +2325,9 @@ emit_end_query(struct radv_cmd_buffer *cmd_buffer, struct radv_query_pool *pool, break; case VK_QUERY_TYPE_PIPELINE_STATISTICS: { - unsigned pipelinestat_block_size = radv_get_pipelinestat_query_size(cmd_buffer->device); + unsigned pipelinestat_block_size = radv_get_pipelinestat_query_size(device); - radeon_check_space(cmd_buffer->device->ws, cs, 16); + radeon_check_space(device->ws, cs, 16); cmd_buffer->state.active_pipeline_queries--; @@ -2376,14 +2381,14 @@ emit_end_query(struct radv_cmd_buffer *cmd_buffer, struct radv_query_pool *pool, if (pdev->info.gfx_level >= GFX11) { va += task_invoc_offset; - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->gang.cs, 4); + radeon_check_space(device->ws, cmd_buffer->gang.cs, 4); radeon_emit(cmd_buffer->gang.cs, PKT3(PKT3_EVENT_WRITE, 2, 0)); radeon_emit(cmd_buffer->gang.cs, EVENT_TYPE(V_028A90_SAMPLE_PIPELINESTAT) | EVENT_INDEX(2)); radeon_emit(cmd_buffer->gang.cs, va); radeon_emit(cmd_buffer->gang.cs, va >> 32); } else { - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->gang.cs, 11); + radeon_check_space(device->ws, cmd_buffer->gang.cs, 11); gfx10_copy_gds_query_ace(cmd_buffer, RADV_SHADER_QUERY_TS_INVOCATION_OFFSET, va + task_invoc_offset); radv_cs_write_data_imm(cmd_buffer->gang.cs, V_370_ME, va + task_invoc_offset + 4, 0x80000000); @@ -2467,9 +2472,9 @@ emit_end_query(struct radv_cmd_buffer *cmd_buffer, struct radv_query_pool *pool, } case VK_QUERY_TYPE_MESH_PRIMITIVES_GENERATED_EXT: { if (pdev->info.gfx_level >= GFX11) { - unsigned pipelinestat_block_size = radv_get_pipelinestat_query_size(cmd_buffer->device); + unsigned pipelinestat_block_size = radv_get_pipelinestat_query_size(device); - radeon_check_space(cmd_buffer->device->ws, cs, 16); + radeon_check_space(device->ws, cs, 16); cmd_buffer->state.active_pipeline_queries--; @@ -2513,10 +2518,11 @@ radv_CmdBeginQueryIndexedEXT(VkCommandBuffer commandBuffer, VkQueryPool queryPoo { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_query_pool, pool, queryPool); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radeon_cmdbuf *cs = cmd_buffer->cs; uint64_t va = radv_buffer_get_va(pool->bo); - radv_cs_add_buffer(cmd_buffer->device->ws, cs, pool->bo); + radv_cs_add_buffer(device->ws, cs, pool->bo); emit_query_flush(cmd_buffer, pool); @@ -2526,7 +2532,7 @@ radv_CmdBeginQueryIndexedEXT(VkCommandBuffer commandBuffer, VkQueryPool queryPoo if (!radv_gang_init(cmd_buffer)) return; - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->gang.cs, pool->bo); + radv_cs_add_buffer(device->ws, cmd_buffer->gang.cs, pool->bo); } emit_begin_query(cmd_buffer, pool, va, pool->vk.query_type, flags, index); @@ -2567,7 +2573,8 @@ radv_CmdEndQueryIndexedEXT(VkCommandBuffer commandBuffer, VkQueryPool queryPool, void radv_write_timestamp(struct radv_cmd_buffer *cmd_buffer, uint64_t va, VkPipelineStageFlags2 stage) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radeon_cmdbuf *cs = cmd_buffer->cs; if (stage == VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT) { @@ -2590,22 +2597,23 @@ radv_CmdWriteTimestamp2(VkCommandBuffer commandBuffer, VkPipelineStageFlags2 sta { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_query_pool, pool, queryPool); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const unsigned num_queries = MAX2(util_bitcount(cmd_buffer->state.render.view_mask), 1); struct radeon_cmdbuf *cs = cmd_buffer->cs; const uint64_t va = radv_buffer_get_va(pool->bo); uint64_t query_va = va + pool->stride * query; - radv_cs_add_buffer(cmd_buffer->device->ws, cs, pool->bo); + radv_cs_add_buffer(device->ws, cs, pool->bo); if (cmd_buffer->qf == RADV_QUEUE_TRANSFER) { - if (cmd_buffer->device->instance->drirc.flush_before_timestamp_write) { - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 1); + if (device->instance->drirc.flush_before_timestamp_write) { + radeon_check_space(device->ws, cmd_buffer->cs, 1); radeon_emit(cmd_buffer->cs, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); } for (unsigned i = 0; i < num_queries; ++i, query_va += pool->stride) { - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 3); + radeon_check_space(device->ws, cmd_buffer->cs, 3); radeon_emit(cmd_buffer->cs, SDMA_PACKET(SDMA_OPCODE_TIMESTAMP, SDMA_TS_SUB_OPCODE_GET_GLOBAL_TIMESTAMP, 0)); radeon_emit(cs, query_va); radeon_emit(cs, query_va >> 32); @@ -2613,14 +2621,14 @@ radv_CmdWriteTimestamp2(VkCommandBuffer commandBuffer, VkPipelineStageFlags2 sta return; } - if (cmd_buffer->device->instance->drirc.flush_before_timestamp_write) { + if (device->instance->drirc.flush_before_timestamp_write) { /* Make sure previously launched waves have finished */ cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_PS_PARTIAL_FLUSH | RADV_CMD_FLAG_CS_PARTIAL_FLUSH; } radv_emit_cache_flush(cmd_buffer); - ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 28 * num_queries); + ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cs, 28 * num_queries); for (unsigned i = 0; i < num_queries; i++) { radv_write_timestamp(cmd_buffer, query_va, stage); @@ -2643,15 +2651,16 @@ radv_CmdWriteAccelerationStructuresPropertiesKHR(VkCommandBuffer commandBuffer, { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_query_pool, pool, queryPool); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radeon_cmdbuf *cs = cmd_buffer->cs; uint64_t pool_va = radv_buffer_get_va(pool->bo); uint64_t query_va = pool_va + pool->stride * firstQuery; - radv_cs_add_buffer(cmd_buffer->device->ws, cs, pool->bo); + radv_cs_add_buffer(device->ws, cs, pool->bo); radv_emit_cache_flush(cmd_buffer); - ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 6 * accelerationStructureCount); + ASSERTED unsigned cdw_max = radeon_check_space(device->ws, cs, 6 * accelerationStructureCount); for (uint32_t i = 0; i < accelerationStructureCount; ++i) { RADV_FROM_HANDLE(vk_acceleration_structure, accel_struct, pAccelerationStructures[i]); diff --git a/src/amd/vulkan/radv_sqtt.c b/src/amd/vulkan/radv_sqtt.c index 867f90a04e0..b1e6c31d709 100644 --- a/src/amd/vulkan/radv_sqtt.c +++ b/src/amd/vulkan/radv_sqtt.c @@ -433,10 +433,10 @@ radv_emit_sqtt_stop(const struct radv_device *device, struct radeon_cmdbuf *cs, void radv_emit_sqtt_userdata(const struct radv_cmd_buffer *cmd_buffer, const void *data, uint32_t num_dwords) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const enum amd_gfx_level gfx_level = pdev->info.gfx_level; const enum radv_queue_family qf = cmd_buffer->qf; - struct radv_device *device = cmd_buffer->device; struct radeon_cmdbuf *cs = cmd_buffer->cs; const uint32_t *dwords = (uint32_t *)data; diff --git a/src/amd/vulkan/radv_video.c b/src/amd/vulkan/radv_video.c index 66447013592..d28f72c07a2 100644 --- a/src/amd/vulkan/radv_video.c +++ b/src/amd/vulkan/radv_video.c @@ -115,7 +115,9 @@ radv_vcn_sq_tail(struct radeon_cmdbuf *cs, struct rvcn_sq_var *sq) static void radv_vcn_sq_start(struct radv_cmd_buffer *cmd_buffer) { - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 256); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + + radeon_check_space(device->ws, cmd_buffer->cs, 256); radv_vcn_sq_header(cmd_buffer->cs, &cmd_buffer->video.sq, false); rvcn_decode_ib_package_t *ib_header = (rvcn_decode_ib_package_t *)&(cmd_buffer->cs->buf[cmd_buffer->cs->cdw]); ib_header->package_size = sizeof(struct rvcn_decode_buffer_s) + sizeof(struct rvcn_decode_ib_package_s); @@ -763,16 +765,16 @@ set_reg(struct radv_cmd_buffer *cmd_buffer, unsigned reg, uint32_t val) static void send_cmd(struct radv_cmd_buffer *cmd_buffer, unsigned cmd, struct radeon_winsys_bo *bo, uint32_t offset) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_physical_device *pdev = radv_device_physical(device); uint64_t addr; - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, bo); addr = radv_buffer_get_va(bo); addr += offset; if (pdev->vid_decode_ip != AMD_IP_VCN_UNIFIED) { - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 6); + radeon_check_space(device->ws, cmd_buffer->cs, 6); set_reg(cmd_buffer, pdev->vid_dec_reg.data0, addr); set_reg(cmd_buffer, pdev->vid_dec_reg.data1, addr >> 32); set_reg(cmd_buffer, pdev->vid_dec_reg.cmd, cmd << 1); @@ -2100,7 +2102,7 @@ rvcn_dec_message_decode(struct radv_cmd_buffer *cmd_buffer, struct radv_video_se struct radv_video_session_params *params, void *ptr, void *it_probs_ptr, uint32_t *slice_offset, const struct VkVideoDecodeInfoKHR *frame_info) { - struct radv_device *device = cmd_buffer->device; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_physical_device *pdev = radv_device_physical(device); rvcn_dec_message_header_t *header; rvcn_dec_message_index_t *index_codec; @@ -2267,7 +2269,7 @@ rvcn_dec_message_decode(struct radv_cmd_buffer *cmd_buffer, struct radv_video_se return true; uint64_t addr; - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, dpb->bindings[0].bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, dpb->bindings[0].bo); addr = radv_buffer_get_va(dpb->bindings[0].bo) + dpb->bindings[0].offset; dynamic_dpb_t2->dpbCurrLo = addr; dynamic_dpb_t2->dpbCurrHi = addr >> 32; @@ -2292,7 +2294,7 @@ rvcn_dec_message_decode(struct radv_cmd_buffer *cmd_buffer, struct radv_video_se assert(f_dpb_iv != NULL); struct radv_image *dpb_img = f_dpb_iv->image; - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, dpb_img->bindings[0].bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, dpb_img->bindings[0].bo); addr = radv_buffer_get_va(dpb_img->bindings[0].bo) + dpb_img->bindings[0].offset; dynamic_dpb_t2->dpbAddrLo[i] = addr; @@ -2300,7 +2302,7 @@ rvcn_dec_message_decode(struct radv_cmd_buffer *cmd_buffer, struct radv_video_se ++dynamic_dpb_t2->dpbArraySize; } - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, dpb->bindings[0].bo); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, dpb->bindings[0].bo); addr = radv_buffer_get_va(dpb->bindings[0].bo) + dpb->bindings[0].offset; dynamic_dpb_t2->dpbCurrLo = addr; @@ -2747,7 +2749,8 @@ radv_CmdBeginVideoCodingKHR(VkCommandBuffer commandBuffer, const VkVideoBeginCod static void radv_vcn_cmd_reset(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_video_session *vid = cmd_buffer->video.vid; uint32_t size = sizeof(rvcn_dec_message_header_t) + sizeof(rvcn_dec_message_create_t); @@ -2759,7 +2762,7 @@ radv_vcn_cmd_reset(struct radv_cmd_buffer *cmd_buffer) ? align(sizeof(rvcn_av1_frame_context_t), 2048) : align(sizeof(rvcn_av1_vcn4_frame_context_t), 2048); - uint8_t *ctxptr = radv_buffer_map(cmd_buffer->device->ws, vid->ctx.mem->bo); + uint8_t *ctxptr = radv_buffer_map(device->ws, vid->ctx.mem->bo); ctxptr += vid->ctx.offset; if (pdev->av1_version == RDECODE_AV1_VER_0) { for (unsigned i = 0; i < 4; ++i) { @@ -2774,7 +2777,7 @@ radv_vcn_cmd_reset(struct radv_cmd_buffer *cmd_buffer) rvcn_vcn4_av1_default_coef_probs((void *)(ctxptr + i * frame_ctxt_size), i); } } - cmd_buffer->device->ws->buffer_unmap(cmd_buffer->device->ws, vid->ctx.mem->bo, false); + device->ws->buffer_unmap(device->ws, vid->ctx.mem->bo, false); } radv_vid_buffer_upload_alloc(cmd_buffer, size, &out_offset, &ptr); @@ -2787,7 +2790,7 @@ radv_vcn_cmd_reset(struct radv_cmd_buffer *cmd_buffer) /* pad out the IB to the 16 dword boundary - otherwise the fw seems to be unhappy */ if (pdev->vid_decode_ip != AMD_IP_VCN_UNIFIED) { - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 8); + radeon_check_space(device->ws, cmd_buffer->cs, 8); for (unsigned i = 0; i < 8; i++) radeon_emit(cmd_buffer->cs, 0x81ff); } else @@ -2797,6 +2800,7 @@ radv_vcn_cmd_reset(struct radv_cmd_buffer *cmd_buffer) static void radv_uvd_cmd_reset(struct radv_cmd_buffer *cmd_buffer) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_video_session *vid = cmd_buffer->video.vid; uint32_t size = sizeof(struct ruvd_msg); void *ptr; @@ -2810,7 +2814,7 @@ radv_uvd_cmd_reset(struct radv_cmd_buffer *cmd_buffer) /* pad out the IB to the 16 dword boundary - otherwise the fw seems to be unhappy */ int padsize = vid->sessionctx.mem ? 4 : 6; - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, padsize); + radeon_check_space(device->ws, cmd_buffer->cs, padsize); for (unsigned i = 0; i < padsize; i++) radeon_emit(cmd_buffer->cs, PKT2_NOP_PAD); } @@ -2819,7 +2823,8 @@ VKAPI_ATTR void VKAPI_CALL radv_CmdControlVideoCodingKHR(VkCommandBuffer commandBuffer, const VkVideoCodingControlInfoKHR *pCodingControlInfo) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + struct radv_physical_device *pdev = radv_device_physical(device); if (pCodingControlInfo->flags & VK_VIDEO_CODING_CONTROL_RESET_BIT_KHR) { if (radv_has_uvd(pdev)) @@ -2838,7 +2843,8 @@ static void radv_uvd_decode_video(struct radv_cmd_buffer *cmd_buffer, const VkVideoDecodeInfoKHR *frame_info) { RADV_FROM_HANDLE(radv_buffer, src_buffer, frame_info->srcBuffer); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_video_session *vid = cmd_buffer->video.vid; struct radv_video_session_params *params = cmd_buffer->video.params; unsigned size = sizeof(struct ruvd_msg); @@ -2858,7 +2864,7 @@ radv_uvd_decode_video(struct radv_cmd_buffer *cmd_buffer, const VkVideoDecodeInf msg_bo = cmd_buffer->upload.upload_bo; uint32_t slice_offset; - ruvd_dec_message_decode(cmd_buffer->device, vid, params, ptr, it_probs_ptr, &slice_offset, frame_info); + ruvd_dec_message_decode(device, vid, params, ptr, it_probs_ptr, &slice_offset, frame_info); rvcn_dec_message_feedback(fb_ptr); if (vid->sessionctx.mem) send_cmd(cmd_buffer, RDECODE_CMD_SESSION_CONTEXT_BUFFER, vid->sessionctx.mem->bo, vid->sessionctx.offset); @@ -2884,7 +2890,7 @@ radv_uvd_decode_video(struct radv_cmd_buffer *cmd_buffer, const VkVideoDecodeInf if (have_it(vid)) send_cmd(cmd_buffer, RDECODE_CMD_IT_SCALING_TABLE_BUFFER, it_probs_bo, it_probs_offset); - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 2); + radeon_check_space(device->ws, cmd_buffer->cs, 2); set_reg(cmd_buffer, pdev->vid_dec_reg.cntl, 1); } @@ -2892,7 +2898,8 @@ static void radv_vcn_decode_video(struct radv_cmd_buffer *cmd_buffer, const VkVideoDecodeInfoKHR *frame_info) { RADV_FROM_HANDLE(radv_buffer, src_buffer, frame_info->srcBuffer); - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); struct radv_video_session *vid = cmd_buffer->video.vid; struct radv_video_session_params *params = cmd_buffer->video.params; unsigned size = 0; @@ -2966,7 +2973,7 @@ radv_vcn_decode_video(struct radv_cmd_buffer *cmd_buffer, const VkVideoDecodeInf send_cmd(cmd_buffer, RDECODE_CMD_PROB_TBL_BUFFER, it_probs_bo, it_probs_offset); if (pdev->vid_decode_ip != AMD_IP_VCN_UNIFIED) { - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 2); + radeon_check_space(device->ws, cmd_buffer->cs, 2); set_reg(cmd_buffer, pdev->vid_dec_reg.cntl, 1); } else radv_vcn_sq_tail(cmd_buffer->cs, &cmd_buffer->video.sq); @@ -2976,7 +2983,8 @@ VKAPI_ATTR void VKAPI_CALL radv_CmdDecodeVideoKHR(VkCommandBuffer commandBuffer, const VkVideoDecodeInfoKHR *frame_info) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + struct radv_physical_device *pdev = radv_device_physical(device); if (radv_has_uvd(pdev)) radv_uvd_decode_video(cmd_buffer, frame_info); diff --git a/src/amd/vulkan/si_cmd_buffer.c b/src/amd/vulkan/si_cmd_buffer.c index 84f083ef4fa..fe99e68e313 100644 --- a/src/amd/vulkan/si_cmd_buffer.c +++ b/src/amd/vulkan/si_cmd_buffer.c @@ -810,7 +810,8 @@ radv_get_ia_multi_vgt_param(struct radv_cmd_buffer *cmd_buffer, bool instanced_d bool count_from_stream_output, uint32_t draw_vertex_count, unsigned topology, bool prim_restart_enable, unsigned patch_control_points, unsigned num_tess_patches) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); const struct radeon_info *gpu_info = &pdev->info; const unsigned max_primgroup_in_wave = 2; /* SWITCH_ON_EOP(0) is always preferable. */ @@ -1489,7 +1490,8 @@ radv_cs_emit_cache_flush(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, enu void radv_emit_cache_flush(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); bool is_compute = cmd_buffer->qf == RADV_QUEUE_COMPUTE; if (is_compute) @@ -1504,12 +1506,12 @@ radv_emit_cache_flush(struct radv_cmd_buffer *cmd_buffer) return; } - radv_cs_emit_cache_flush(cmd_buffer->device->ws, cmd_buffer->cs, pdev->info.gfx_level, &cmd_buffer->gfx9_fence_idx, + radv_cs_emit_cache_flush(device->ws, cmd_buffer->cs, pdev->info.gfx_level, &cmd_buffer->gfx9_fence_idx, cmd_buffer->gfx9_fence_va, radv_cmd_buffer_uses_mec(cmd_buffer), cmd_buffer->state.flush_bits, &cmd_buffer->state.sqtt_flush_bits, cmd_buffer->gfx9_eop_bug_va); - if (radv_device_fault_detection_enabled(cmd_buffer->device)) + if (radv_device_fault_detection_enabled(device)) radv_cmd_buffer_trace_emit(cmd_buffer); if (cmd_buffer->state.flush_bits & RADV_CMD_FLAG_INV_L2) @@ -1534,10 +1536,11 @@ radv_emit_cache_flush(struct radv_cmd_buffer *cmd_buffer) void radv_emit_set_predication_state(struct radv_cmd_buffer *cmd_buffer, bool draw_visible, unsigned pred_op, uint64_t va) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); uint32_t op = 0; - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 4); + radeon_check_space(device->ws, cmd_buffer->cs, 4); if (va) { assert(pred_op == PREDICATION_OP_BOOL32 || pred_op == PREDICATION_OP_BOOL64); @@ -1668,8 +1671,8 @@ radv_cs_emit_cp_dma(struct radv_device *device, struct radeon_cmdbuf *cs, bool p static void radv_emit_cp_dma(struct radv_cmd_buffer *cmd_buffer, uint64_t dst_va, uint64_t src_va, unsigned size, unsigned flags) { + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radeon_cmdbuf *cs = cmd_buffer->cs; - struct radv_device *device = cmd_buffer->device; bool predicating = cmd_buffer->state.predicating; radv_cs_emit_cp_dma(device, cs, predicating, dst_va, src_va, size, flags); @@ -1689,7 +1692,7 @@ radv_emit_cp_dma(struct radv_cmd_buffer *cmd_buffer, uint64_t dst_va, uint64_t s cmd_buffer->state.dma_is_busy = false; } - if (radv_device_fault_detection_enabled(cmd_buffer->device)) + if (radv_device_fault_detection_enabled(device)) radv_cmd_buffer_trace_emit(cmd_buffer); } @@ -1734,9 +1737,11 @@ radv_cs_cp_dma_prefetch(const struct radv_device *device, struct radeon_cmdbuf * void radv_cp_dma_prefetch(struct radv_cmd_buffer *cmd_buffer, uint64_t va, unsigned size) { - radv_cs_cp_dma_prefetch(cmd_buffer->device, cmd_buffer->cs, va, size, cmd_buffer->state.predicating); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); - if (radv_device_fault_detection_enabled(cmd_buffer->device)) + radv_cs_cp_dma_prefetch(device, cmd_buffer->cs, va, size, cmd_buffer->state.predicating); + + if (radv_device_fault_detection_enabled(device)) radv_cmd_buffer_trace_emit(cmd_buffer); } @@ -1783,7 +1788,8 @@ radv_cp_dma_realign_engine(struct radv_cmd_buffer *cmd_buffer, unsigned size) void radv_cp_dma_buffer_copy(struct radv_cmd_buffer *cmd_buffer, uint64_t src_va, uint64_t dest_va, uint64_t size) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); enum amd_gfx_level gfx_level = pdev->info.gfx_level; uint64_t main_src_va, main_dest_va; uint64_t skipped_size = 0, realign_size = 0; @@ -1857,7 +1863,8 @@ radv_cp_dma_buffer_copy(struct radv_cmd_buffer *cmd_buffer, uint64_t src_va, uin void radv_cp_dma_clear_buffer(struct radv_cmd_buffer *cmd_buffer, uint64_t va, uint64_t size, unsigned value) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); if (!size) return; @@ -1896,7 +1903,8 @@ radv_cp_dma_clear_buffer(struct radv_cmd_buffer *cmd_buffer, uint64_t va, uint64 void radv_cp_dma_wait_for_idle(struct radv_cmd_buffer *cmd_buffer) { - const struct radv_physical_device *pdev = radv_device_physical(cmd_buffer->device); + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); if (pdev->info.gfx_level < GFX7) return;