From 4a2a261a79e086c111a29bffb29368ba1948e638 Mon Sep 17 00:00:00 2001 From: Samuel Pitoiset Date: Tue, 27 Feb 2024 10:30:52 +0100 Subject: [PATCH] radv: stop passing radv_cmd_buffer to draw functions with task shaders In order to remove the ambiguity because for task shaders the driver needs to emit to both the GFX CS and the ACE CS but all states come from the main cmdbuf (ie. GFX) from the application point of view. Signed-off-by: Samuel Pitoiset Part-of: --- src/amd/vulkan/radv_cmd_buffer.c | 119 ++++++++++++++++--------------- 1 file changed, 62 insertions(+), 57 deletions(-) diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c index 87c9c89dd20..1b776377689 100644 --- a/src/amd/vulkan/radv_cmd_buffer.c +++ b/src/amd/vulkan/radv_cmd_buffer.c @@ -8227,14 +8227,14 @@ radv_cs_emit_indirect_mesh_draw_packet(struct radv_cmd_buffer *cmd_buffer, uint3 } ALWAYS_INLINE static void -radv_cs_emit_dispatch_taskmesh_direct_ace_packet(struct radv_cmd_buffer *cmd_buffer, const uint32_t x, const uint32_t y, - const uint32_t z) +radv_cs_emit_dispatch_taskmesh_direct_ace_packet(const struct radv_device *device, + const struct radv_cmd_state *cmd_state, struct radeon_cmdbuf *ace_cs, + const uint32_t x, const uint32_t y, const uint32_t z) { - struct radv_shader *task_shader = cmd_buffer->state.shaders[MESA_SHADER_TASK]; - struct radeon_cmdbuf *ace_cs = cmd_buffer->gang.cs; - const bool predicating = cmd_buffer->state.predicating; + const struct radv_shader *task_shader = cmd_state->shaders[MESA_SHADER_TASK]; + const bool predicating = cmd_state->predicating; const uint32_t dispatch_initiator = - cmd_buffer->device->dispatch_initiator_task | S_00B800_CS_W32_EN(task_shader->info.wave_size == 32); + device->dispatch_initiator_task | S_00B800_CS_W32_EN(task_shader->info.wave_size == 32); const struct radv_userdata_info *ring_entry_loc = radv_get_user_sgpr(task_shader, AC_UD_TASK_RING_ENTRY); assert(ring_entry_loc && ring_entry_loc->sgpr_idx != -1 && ring_entry_loc->num_sgprs == 1); @@ -8250,19 +8250,20 @@ radv_cs_emit_dispatch_taskmesh_direct_ace_packet(struct radv_cmd_buffer *cmd_buf } ALWAYS_INLINE static void -radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(struct radv_cmd_buffer *cmd_buffer, uint64_t data_va, +radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(const struct radv_device *device, + const struct radv_cmd_state *cmd_state, + struct radeon_cmdbuf *ace_cs, uint64_t data_va, uint32_t draw_count, uint64_t count_va, uint32_t stride) { assert((data_va & 0x03) == 0); assert((count_va & 0x03) == 0); - struct radv_shader *task_shader = cmd_buffer->state.shaders[MESA_SHADER_TASK]; - struct radeon_cmdbuf *ace_cs = cmd_buffer->gang.cs; + const struct radv_shader *task_shader = cmd_state->shaders[MESA_SHADER_TASK]; const uint32_t xyz_dim_enable = task_shader->info.cs.uses_grid_size; const uint32_t draw_id_enable = task_shader->info.vs.needs_draw_id; const uint32_t dispatch_initiator = - cmd_buffer->device->dispatch_initiator_task | S_00B800_CS_W32_EN(task_shader->info.wave_size == 32); + device->dispatch_initiator_task | S_00B800_CS_W32_EN(task_shader->info.wave_size == 32); const struct radv_userdata_info *ring_entry_loc = radv_get_user_sgpr(task_shader, AC_UD_TASK_RING_ENTRY); const struct radv_userdata_info *xyz_dim_loc = radv_get_user_sgpr(task_shader, AC_UD_CS_GRID_SIZE); @@ -8294,27 +8295,27 @@ radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(struct radv_cmd_buffer } ALWAYS_INLINE static void -radv_cs_emit_dispatch_taskmesh_gfx_packet(struct radv_cmd_buffer *cmd_buffer) +radv_cs_emit_dispatch_taskmesh_gfx_packet(const struct radv_device *device, const struct radv_cmd_state *cmd_state, + struct radeon_cmdbuf *cs) { - const struct radv_shader *mesh_shader = cmd_buffer->state.shaders[MESA_SHADER_MESH]; - struct radeon_cmdbuf *cs = cmd_buffer->cs; - bool predicating = cmd_buffer->state.predicating; + const struct radv_shader *mesh_shader = cmd_state->shaders[MESA_SHADER_MESH]; + const bool predicating = cmd_state->predicating; const struct radv_userdata_info *ring_entry_loc = - radv_get_user_sgpr(cmd_buffer->state.last_vgt_shader, AC_UD_TASK_RING_ENTRY); + radv_get_user_sgpr(cmd_state->last_vgt_shader, AC_UD_TASK_RING_ENTRY); assert(ring_entry_loc->sgpr_idx != -1); uint32_t xyz_dim_en = mesh_shader->info.cs.uses_grid_size; - uint32_t xyz_dim_reg = !xyz_dim_en ? 0 : (cmd_buffer->state.vtx_base_sgpr - SI_SH_REG_OFFSET) >> 2; + uint32_t xyz_dim_reg = !xyz_dim_en ? 0 : (cmd_state->vtx_base_sgpr - SI_SH_REG_OFFSET) >> 2; uint32_t ring_entry_reg = ((mesh_shader->info.user_data_0 - SI_SH_REG_OFFSET) >> 2) + ring_entry_loc->sgpr_idx; - uint32_t mode1_en = !cmd_buffer->device->physical_device->mesh_fast_launch_2; - uint32_t linear_dispatch_en = cmd_buffer->state.shaders[MESA_SHADER_TASK]->info.cs.linear_taskmesh_dispatch; - const bool sqtt_en = !!cmd_buffer->device->sqtt.bo; + uint32_t mode1_en = !device->physical_device->mesh_fast_launch_2; + uint32_t linear_dispatch_en = cmd_state->shaders[MESA_SHADER_TASK]->info.cs.linear_taskmesh_dispatch; + const bool sqtt_en = !!device->sqtt.bo; radeon_emit(cs, PKT3(PKT3_DISPATCH_TASKMESH_GFX, 2, predicating) | PKT3_RESET_FILTER_CAM_S(1)); radeon_emit(cs, S_4D0_RING_ENTRY_REG(ring_entry_reg) | S_4D0_XYZ_DIM_REG(xyz_dim_reg)); - if (cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX11) + if (device->physical_device->rad_info.gfx_level >= GFX11) radeon_emit(cs, S_4D1_XYZ_DIM_ENABLE(xyz_dim_en) | S_4D1_MODE1_ENABLE(mode1_en) | S_4D1_LINEAR_DISPATCH_ENABLE(linear_dispatch_en) | S_4D1_THREAD_TRACE_MARKER_ENABLE(sqtt_en)); else @@ -8398,10 +8399,10 @@ radv_emit_userdata_mesh(struct radv_cmd_buffer *cmd_buffer, const uint32_t x, co } ALWAYS_INLINE static void -radv_emit_userdata_task(struct radv_cmd_buffer *cmd_buffer, uint32_t x, uint32_t y, uint32_t z, uint32_t draw_id) +radv_emit_userdata_task(const struct radv_cmd_state *cmd_state, struct radeon_cmdbuf *ace_cs, uint32_t x, uint32_t y, + uint32_t z, uint32_t draw_id) { - struct radv_shader *task_shader = cmd_buffer->state.shaders[MESA_SHADER_TASK]; - struct radeon_cmdbuf *ace_cs = cmd_buffer->gang.cs; + const struct radv_shader *task_shader = cmd_state->shaders[MESA_SHADER_TASK]; const struct radv_userdata_info *xyz_loc = radv_get_user_sgpr(task_shader, AC_UD_CS_GRID_SIZE); const struct radv_userdata_info *draw_id_loc = radv_get_user_sgpr(task_shader, AC_UD_CS_TASK_DRAW_ID); @@ -8674,42 +8675,43 @@ radv_emit_indirect_mesh_draw_packets(struct radv_cmd_buffer *cmd_buffer, const s } ALWAYS_INLINE static void -radv_emit_direct_taskmesh_draw_packets(struct radv_cmd_buffer *cmd_buffer, uint32_t x, uint32_t y, uint32_t z) +radv_emit_direct_taskmesh_draw_packets(const struct radv_device *device, struct radv_cmd_state *cmd_state, + struct radeon_cmdbuf *cs, struct radeon_cmdbuf *ace_cs, uint32_t x, uint32_t y, + uint32_t z) { - const uint32_t view_mask = cmd_buffer->state.render.view_mask; + const uint32_t view_mask = cmd_state->render.view_mask; const unsigned num_views = MAX2(1, util_bitcount(view_mask)); unsigned ace_predication_size = num_views * 6; /* DISPATCH_TASKMESH_DIRECT_ACE size */ if (num_views > 1) ace_predication_size += num_views * 3; /* SET_SH_REG size (view index SGPR) */ - radv_emit_userdata_task(cmd_buffer, x, y, z, 0); - radv_cs_emit_compute_predication(cmd_buffer->device, &cmd_buffer->state, cmd_buffer->gang.cs, - cmd_buffer->state.mec_inv_pred_va, &cmd_buffer->state.mec_inv_pred_emitted, - ace_predication_size); + radv_emit_userdata_task(cmd_state, ace_cs, x, y, z, 0); + radv_cs_emit_compute_predication(device, cmd_state, ace_cs, cmd_state->mec_inv_pred_va, + &cmd_state->mec_inv_pred_emitted, ace_predication_size); if (!view_mask) { - radv_cs_emit_dispatch_taskmesh_direct_ace_packet(cmd_buffer, x, y, z); - radv_cs_emit_dispatch_taskmesh_gfx_packet(cmd_buffer); + radv_cs_emit_dispatch_taskmesh_direct_ace_packet(device, cmd_state, ace_cs, x, y, z); + radv_cs_emit_dispatch_taskmesh_gfx_packet(device, cmd_state, cs); } else { u_foreach_bit (view, view_mask) { - radv_emit_view_index_with_task(&cmd_buffer->state, cmd_buffer->cs, cmd_buffer->gang.cs, view); + radv_emit_view_index_with_task(cmd_state, cs, ace_cs, view); - radv_cs_emit_dispatch_taskmesh_direct_ace_packet(cmd_buffer, x, y, z); - radv_cs_emit_dispatch_taskmesh_gfx_packet(cmd_buffer); + radv_cs_emit_dispatch_taskmesh_direct_ace_packet(device, cmd_state, ace_cs, x, y, z); + radv_cs_emit_dispatch_taskmesh_gfx_packet(device, cmd_state, cs); } } } static void -radv_emit_indirect_taskmesh_draw_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *info, - uint64_t workaround_cond_va) +radv_emit_indirect_taskmesh_draw_packets(const struct radv_device *device, struct radv_cmd_state *cmd_state, + struct radeon_cmdbuf *cs, struct radeon_cmdbuf *ace_cs, + const struct radv_draw_info *info, uint64_t workaround_cond_va) { - const uint32_t view_mask = cmd_buffer->state.render.view_mask; - struct radeon_winsys *ws = cmd_buffer->device->ws; + const uint32_t view_mask = cmd_state->render.view_mask; + struct radeon_winsys *ws = device->ws; const unsigned num_views = MAX2(1, util_bitcount(view_mask)); unsigned ace_predication_size = num_views * 11; /* DISPATCH_TASKMESH_INDIRECT_MULTI_ACE size */ - struct radeon_cmdbuf *ace_cs = cmd_buffer->gang.cs; const uint64_t va = radv_buffer_get_va(info->indirect->bo) + info->indirect->offset + info->indirect_offset; const uint64_t count_va = !info->count_buffer ? 0 @@ -8720,9 +8722,9 @@ radv_emit_indirect_taskmesh_draw_packets(struct radv_cmd_buffer *cmd_buffer, con ace_predication_size += num_views * 3; /* SET_SH_REG size (view index SGPR) */ if (count_va) - radv_cs_add_buffer(ws, cmd_buffer->gang.cs, info->count_buffer->bo); + radv_cs_add_buffer(ws, ace_cs, info->count_buffer->bo); - if (cmd_buffer->device->physical_device->rad_info.has_taskmesh_indirect0_bug && count_va) { + if (device->physical_device->rad_info.has_taskmesh_indirect0_bug && count_va) { /* MEC firmware bug workaround. * When the count buffer contains zero, DISPATCH_TASKMESH_INDIRECT_MULTI_ACE hangs. * - We must ensure that DISPATCH_TASKMESH_INDIRECT_MULTI_ACE @@ -8747,13 +8749,12 @@ radv_emit_indirect_taskmesh_draw_packets(struct radv_cmd_buffer *cmd_buffer, con ace_predication_size += 2 * 5 + 6 + 6 * num_views; } - radv_cs_add_buffer(ws, cmd_buffer->gang.cs, info->indirect->bo); - radv_cs_emit_compute_predication(cmd_buffer->device, &cmd_buffer->state, cmd_buffer->gang.cs, - cmd_buffer->state.mec_inv_pred_va, &cmd_buffer->state.mec_inv_pred_emitted, - ace_predication_size); + radv_cs_add_buffer(ws, ace_cs, info->indirect->bo); + radv_cs_emit_compute_predication(device, cmd_state, ace_cs, cmd_state->mec_inv_pred_va, + &cmd_state->mec_inv_pred_emitted, ace_predication_size); if (workaround_cond_va) { - radv_emit_cond_exec(cmd_buffer->device, ace_cs, count_va, + radv_emit_cond_exec(device, ace_cs, count_va, 6 + 11 * num_views /* 1x COPY_DATA + Nx DISPATCH_TASKMESH_INDIRECT_MULTI_ACE */); radeon_emit(ace_cs, PKT3(PKT3_COPY_DATA, 4, 0)); @@ -8766,23 +8767,24 @@ radv_emit_indirect_taskmesh_draw_packets(struct radv_cmd_buffer *cmd_buffer, con } if (!view_mask) { - radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(cmd_buffer, va, info->count, count_va, info->stride); - radv_cs_emit_dispatch_taskmesh_gfx_packet(cmd_buffer); + radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(device, cmd_state, ace_cs, va, info->count, count_va, + info->stride); + radv_cs_emit_dispatch_taskmesh_gfx_packet(device, cmd_state, cs); } else { u_foreach_bit (view, view_mask) { - radv_emit_view_index_with_task(&cmd_buffer->state, cmd_buffer->cs, cmd_buffer->gang.cs, view); + radv_emit_view_index_with_task(cmd_state, cs, ace_cs, view); - radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(cmd_buffer, va, info->count, count_va, info->stride); - radv_cs_emit_dispatch_taskmesh_gfx_packet(cmd_buffer); + radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(device, cmd_state, ace_cs, va, info->count, count_va, + info->stride); + radv_cs_emit_dispatch_taskmesh_gfx_packet(device, cmd_state, cs); } } if (workaround_cond_va) { - radv_emit_cond_exec(cmd_buffer->device, ace_cs, workaround_cond_va, - 6 * num_views /* Nx DISPATCH_TASKMESH_DIRECT_ACE */); + radv_emit_cond_exec(device, ace_cs, workaround_cond_va, 6 * num_views /* Nx DISPATCH_TASKMESH_DIRECT_ACE */); for (unsigned v = 0; v < num_views; ++v) { - radv_cs_emit_dispatch_taskmesh_direct_ace_packet(cmd_buffer, 0, 0, 0); + radv_cs_emit_dispatch_taskmesh_direct_ace_packet(device, cmd_state, ace_cs, 0, 0, 0); } } } @@ -9815,7 +9817,8 @@ radv_CmdDrawMeshTasksEXT(VkCommandBuffer commandBuffer, uint32_t x, uint32_t y, return; if (radv_cmdbuf_has_stage(cmd_buffer, MESA_SHADER_TASK)) { - radv_emit_direct_taskmesh_draw_packets(cmd_buffer, x, y, z); + radv_emit_direct_taskmesh_draw_packets(cmd_buffer->device, &cmd_buffer->state, cmd_buffer->cs, + cmd_buffer->gang.cs, x, y, z); } else { radv_emit_direct_mesh_draw_packet(cmd_buffer, x, y, z); } @@ -9848,7 +9851,8 @@ radv_CmdDrawMeshTasksIndirectEXT(VkCommandBuffer commandBuffer, VkBuffer _buffer return; if (radv_cmdbuf_has_stage(cmd_buffer, MESA_SHADER_TASK)) { - radv_emit_indirect_taskmesh_draw_packets(cmd_buffer, &info, 0); + radv_emit_indirect_taskmesh_draw_packets(cmd_buffer->device, &cmd_buffer->state, cmd_buffer->cs, + cmd_buffer->gang.cs, &info, 0); } else { radv_emit_indirect_mesh_draw_packets(cmd_buffer, &info); } @@ -9895,7 +9899,8 @@ radv_CmdDrawMeshTasksIndirectCountEXT(VkCommandBuffer commandBuffer, VkBuffer _b workaround_cond_va = radv_buffer_get_va(cmd_buffer->upload.upload_bo) + workaround_cond_off; } - radv_emit_indirect_taskmesh_draw_packets(cmd_buffer, &info, workaround_cond_va); + radv_emit_indirect_taskmesh_draw_packets(cmd_buffer->device, &cmd_buffer->state, cmd_buffer->cs, + cmd_buffer->gang.cs, &info, workaround_cond_va); } else { radv_emit_indirect_mesh_draw_packets(cmd_buffer, &info); }