diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c index 87c9c89dd20..1b776377689 100644 --- a/src/amd/vulkan/radv_cmd_buffer.c +++ b/src/amd/vulkan/radv_cmd_buffer.c @@ -8227,14 +8227,14 @@ radv_cs_emit_indirect_mesh_draw_packet(struct radv_cmd_buffer *cmd_buffer, uint3 } ALWAYS_INLINE static void -radv_cs_emit_dispatch_taskmesh_direct_ace_packet(struct radv_cmd_buffer *cmd_buffer, const uint32_t x, const uint32_t y, - const uint32_t z) +radv_cs_emit_dispatch_taskmesh_direct_ace_packet(const struct radv_device *device, + const struct radv_cmd_state *cmd_state, struct radeon_cmdbuf *ace_cs, + const uint32_t x, const uint32_t y, const uint32_t z) { - struct radv_shader *task_shader = cmd_buffer->state.shaders[MESA_SHADER_TASK]; - struct radeon_cmdbuf *ace_cs = cmd_buffer->gang.cs; - const bool predicating = cmd_buffer->state.predicating; + const struct radv_shader *task_shader = cmd_state->shaders[MESA_SHADER_TASK]; + const bool predicating = cmd_state->predicating; const uint32_t dispatch_initiator = - cmd_buffer->device->dispatch_initiator_task | S_00B800_CS_W32_EN(task_shader->info.wave_size == 32); + device->dispatch_initiator_task | S_00B800_CS_W32_EN(task_shader->info.wave_size == 32); const struct radv_userdata_info *ring_entry_loc = radv_get_user_sgpr(task_shader, AC_UD_TASK_RING_ENTRY); assert(ring_entry_loc && ring_entry_loc->sgpr_idx != -1 && ring_entry_loc->num_sgprs == 1); @@ -8250,19 +8250,20 @@ radv_cs_emit_dispatch_taskmesh_direct_ace_packet(struct radv_cmd_buffer *cmd_buf } ALWAYS_INLINE static void -radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(struct radv_cmd_buffer *cmd_buffer, uint64_t data_va, +radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(const struct radv_device *device, + const struct radv_cmd_state *cmd_state, + struct radeon_cmdbuf *ace_cs, uint64_t data_va, uint32_t draw_count, uint64_t count_va, uint32_t stride) { assert((data_va & 0x03) == 0); assert((count_va & 0x03) == 0); - struct radv_shader *task_shader = cmd_buffer->state.shaders[MESA_SHADER_TASK]; - struct radeon_cmdbuf *ace_cs = cmd_buffer->gang.cs; + const struct radv_shader *task_shader = cmd_state->shaders[MESA_SHADER_TASK]; const uint32_t xyz_dim_enable = task_shader->info.cs.uses_grid_size; const uint32_t draw_id_enable = task_shader->info.vs.needs_draw_id; const uint32_t dispatch_initiator = - cmd_buffer->device->dispatch_initiator_task | S_00B800_CS_W32_EN(task_shader->info.wave_size == 32); + device->dispatch_initiator_task | S_00B800_CS_W32_EN(task_shader->info.wave_size == 32); const struct radv_userdata_info *ring_entry_loc = radv_get_user_sgpr(task_shader, AC_UD_TASK_RING_ENTRY); const struct radv_userdata_info *xyz_dim_loc = radv_get_user_sgpr(task_shader, AC_UD_CS_GRID_SIZE); @@ -8294,27 +8295,27 @@ radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(struct radv_cmd_buffer } ALWAYS_INLINE static void -radv_cs_emit_dispatch_taskmesh_gfx_packet(struct radv_cmd_buffer *cmd_buffer) +radv_cs_emit_dispatch_taskmesh_gfx_packet(const struct radv_device *device, const struct radv_cmd_state *cmd_state, + struct radeon_cmdbuf *cs) { - const struct radv_shader *mesh_shader = cmd_buffer->state.shaders[MESA_SHADER_MESH]; - struct radeon_cmdbuf *cs = cmd_buffer->cs; - bool predicating = cmd_buffer->state.predicating; + const struct radv_shader *mesh_shader = cmd_state->shaders[MESA_SHADER_MESH]; + const bool predicating = cmd_state->predicating; const struct radv_userdata_info *ring_entry_loc = - radv_get_user_sgpr(cmd_buffer->state.last_vgt_shader, AC_UD_TASK_RING_ENTRY); + radv_get_user_sgpr(cmd_state->last_vgt_shader, AC_UD_TASK_RING_ENTRY); assert(ring_entry_loc->sgpr_idx != -1); uint32_t xyz_dim_en = mesh_shader->info.cs.uses_grid_size; - uint32_t xyz_dim_reg = !xyz_dim_en ? 0 : (cmd_buffer->state.vtx_base_sgpr - SI_SH_REG_OFFSET) >> 2; + uint32_t xyz_dim_reg = !xyz_dim_en ? 0 : (cmd_state->vtx_base_sgpr - SI_SH_REG_OFFSET) >> 2; uint32_t ring_entry_reg = ((mesh_shader->info.user_data_0 - SI_SH_REG_OFFSET) >> 2) + ring_entry_loc->sgpr_idx; - uint32_t mode1_en = !cmd_buffer->device->physical_device->mesh_fast_launch_2; - uint32_t linear_dispatch_en = cmd_buffer->state.shaders[MESA_SHADER_TASK]->info.cs.linear_taskmesh_dispatch; - const bool sqtt_en = !!cmd_buffer->device->sqtt.bo; + uint32_t mode1_en = !device->physical_device->mesh_fast_launch_2; + uint32_t linear_dispatch_en = cmd_state->shaders[MESA_SHADER_TASK]->info.cs.linear_taskmesh_dispatch; + const bool sqtt_en = !!device->sqtt.bo; radeon_emit(cs, PKT3(PKT3_DISPATCH_TASKMESH_GFX, 2, predicating) | PKT3_RESET_FILTER_CAM_S(1)); radeon_emit(cs, S_4D0_RING_ENTRY_REG(ring_entry_reg) | S_4D0_XYZ_DIM_REG(xyz_dim_reg)); - if (cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX11) + if (device->physical_device->rad_info.gfx_level >= GFX11) radeon_emit(cs, S_4D1_XYZ_DIM_ENABLE(xyz_dim_en) | S_4D1_MODE1_ENABLE(mode1_en) | S_4D1_LINEAR_DISPATCH_ENABLE(linear_dispatch_en) | S_4D1_THREAD_TRACE_MARKER_ENABLE(sqtt_en)); else @@ -8398,10 +8399,10 @@ radv_emit_userdata_mesh(struct radv_cmd_buffer *cmd_buffer, const uint32_t x, co } ALWAYS_INLINE static void -radv_emit_userdata_task(struct radv_cmd_buffer *cmd_buffer, uint32_t x, uint32_t y, uint32_t z, uint32_t draw_id) +radv_emit_userdata_task(const struct radv_cmd_state *cmd_state, struct radeon_cmdbuf *ace_cs, uint32_t x, uint32_t y, + uint32_t z, uint32_t draw_id) { - struct radv_shader *task_shader = cmd_buffer->state.shaders[MESA_SHADER_TASK]; - struct radeon_cmdbuf *ace_cs = cmd_buffer->gang.cs; + const struct radv_shader *task_shader = cmd_state->shaders[MESA_SHADER_TASK]; const struct radv_userdata_info *xyz_loc = radv_get_user_sgpr(task_shader, AC_UD_CS_GRID_SIZE); const struct radv_userdata_info *draw_id_loc = radv_get_user_sgpr(task_shader, AC_UD_CS_TASK_DRAW_ID); @@ -8674,42 +8675,43 @@ radv_emit_indirect_mesh_draw_packets(struct radv_cmd_buffer *cmd_buffer, const s } ALWAYS_INLINE static void -radv_emit_direct_taskmesh_draw_packets(struct radv_cmd_buffer *cmd_buffer, uint32_t x, uint32_t y, uint32_t z) +radv_emit_direct_taskmesh_draw_packets(const struct radv_device *device, struct radv_cmd_state *cmd_state, + struct radeon_cmdbuf *cs, struct radeon_cmdbuf *ace_cs, uint32_t x, uint32_t y, + uint32_t z) { - const uint32_t view_mask = cmd_buffer->state.render.view_mask; + const uint32_t view_mask = cmd_state->render.view_mask; const unsigned num_views = MAX2(1, util_bitcount(view_mask)); unsigned ace_predication_size = num_views * 6; /* DISPATCH_TASKMESH_DIRECT_ACE size */ if (num_views > 1) ace_predication_size += num_views * 3; /* SET_SH_REG size (view index SGPR) */ - radv_emit_userdata_task(cmd_buffer, x, y, z, 0); - radv_cs_emit_compute_predication(cmd_buffer->device, &cmd_buffer->state, cmd_buffer->gang.cs, - cmd_buffer->state.mec_inv_pred_va, &cmd_buffer->state.mec_inv_pred_emitted, - ace_predication_size); + radv_emit_userdata_task(cmd_state, ace_cs, x, y, z, 0); + radv_cs_emit_compute_predication(device, cmd_state, ace_cs, cmd_state->mec_inv_pred_va, + &cmd_state->mec_inv_pred_emitted, ace_predication_size); if (!view_mask) { - radv_cs_emit_dispatch_taskmesh_direct_ace_packet(cmd_buffer, x, y, z); - radv_cs_emit_dispatch_taskmesh_gfx_packet(cmd_buffer); + radv_cs_emit_dispatch_taskmesh_direct_ace_packet(device, cmd_state, ace_cs, x, y, z); + radv_cs_emit_dispatch_taskmesh_gfx_packet(device, cmd_state, cs); } else { u_foreach_bit (view, view_mask) { - radv_emit_view_index_with_task(&cmd_buffer->state, cmd_buffer->cs, cmd_buffer->gang.cs, view); + radv_emit_view_index_with_task(cmd_state, cs, ace_cs, view); - radv_cs_emit_dispatch_taskmesh_direct_ace_packet(cmd_buffer, x, y, z); - radv_cs_emit_dispatch_taskmesh_gfx_packet(cmd_buffer); + radv_cs_emit_dispatch_taskmesh_direct_ace_packet(device, cmd_state, ace_cs, x, y, z); + radv_cs_emit_dispatch_taskmesh_gfx_packet(device, cmd_state, cs); } } } static void -radv_emit_indirect_taskmesh_draw_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *info, - uint64_t workaround_cond_va) +radv_emit_indirect_taskmesh_draw_packets(const struct radv_device *device, struct radv_cmd_state *cmd_state, + struct radeon_cmdbuf *cs, struct radeon_cmdbuf *ace_cs, + const struct radv_draw_info *info, uint64_t workaround_cond_va) { - const uint32_t view_mask = cmd_buffer->state.render.view_mask; - struct radeon_winsys *ws = cmd_buffer->device->ws; + const uint32_t view_mask = cmd_state->render.view_mask; + struct radeon_winsys *ws = device->ws; const unsigned num_views = MAX2(1, util_bitcount(view_mask)); unsigned ace_predication_size = num_views * 11; /* DISPATCH_TASKMESH_INDIRECT_MULTI_ACE size */ - struct radeon_cmdbuf *ace_cs = cmd_buffer->gang.cs; const uint64_t va = radv_buffer_get_va(info->indirect->bo) + info->indirect->offset + info->indirect_offset; const uint64_t count_va = !info->count_buffer ? 0 @@ -8720,9 +8722,9 @@ radv_emit_indirect_taskmesh_draw_packets(struct radv_cmd_buffer *cmd_buffer, con ace_predication_size += num_views * 3; /* SET_SH_REG size (view index SGPR) */ if (count_va) - radv_cs_add_buffer(ws, cmd_buffer->gang.cs, info->count_buffer->bo); + radv_cs_add_buffer(ws, ace_cs, info->count_buffer->bo); - if (cmd_buffer->device->physical_device->rad_info.has_taskmesh_indirect0_bug && count_va) { + if (device->physical_device->rad_info.has_taskmesh_indirect0_bug && count_va) { /* MEC firmware bug workaround. * When the count buffer contains zero, DISPATCH_TASKMESH_INDIRECT_MULTI_ACE hangs. * - We must ensure that DISPATCH_TASKMESH_INDIRECT_MULTI_ACE @@ -8747,13 +8749,12 @@ radv_emit_indirect_taskmesh_draw_packets(struct radv_cmd_buffer *cmd_buffer, con ace_predication_size += 2 * 5 + 6 + 6 * num_views; } - radv_cs_add_buffer(ws, cmd_buffer->gang.cs, info->indirect->bo); - radv_cs_emit_compute_predication(cmd_buffer->device, &cmd_buffer->state, cmd_buffer->gang.cs, - cmd_buffer->state.mec_inv_pred_va, &cmd_buffer->state.mec_inv_pred_emitted, - ace_predication_size); + radv_cs_add_buffer(ws, ace_cs, info->indirect->bo); + radv_cs_emit_compute_predication(device, cmd_state, ace_cs, cmd_state->mec_inv_pred_va, + &cmd_state->mec_inv_pred_emitted, ace_predication_size); if (workaround_cond_va) { - radv_emit_cond_exec(cmd_buffer->device, ace_cs, count_va, + radv_emit_cond_exec(device, ace_cs, count_va, 6 + 11 * num_views /* 1x COPY_DATA + Nx DISPATCH_TASKMESH_INDIRECT_MULTI_ACE */); radeon_emit(ace_cs, PKT3(PKT3_COPY_DATA, 4, 0)); @@ -8766,23 +8767,24 @@ radv_emit_indirect_taskmesh_draw_packets(struct radv_cmd_buffer *cmd_buffer, con } if (!view_mask) { - radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(cmd_buffer, va, info->count, count_va, info->stride); - radv_cs_emit_dispatch_taskmesh_gfx_packet(cmd_buffer); + radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(device, cmd_state, ace_cs, va, info->count, count_va, + info->stride); + radv_cs_emit_dispatch_taskmesh_gfx_packet(device, cmd_state, cs); } else { u_foreach_bit (view, view_mask) { - radv_emit_view_index_with_task(&cmd_buffer->state, cmd_buffer->cs, cmd_buffer->gang.cs, view); + radv_emit_view_index_with_task(cmd_state, cs, ace_cs, view); - radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(cmd_buffer, va, info->count, count_va, info->stride); - radv_cs_emit_dispatch_taskmesh_gfx_packet(cmd_buffer); + radv_cs_emit_dispatch_taskmesh_indirect_multi_ace_packet(device, cmd_state, ace_cs, va, info->count, count_va, + info->stride); + radv_cs_emit_dispatch_taskmesh_gfx_packet(device, cmd_state, cs); } } if (workaround_cond_va) { - radv_emit_cond_exec(cmd_buffer->device, ace_cs, workaround_cond_va, - 6 * num_views /* Nx DISPATCH_TASKMESH_DIRECT_ACE */); + radv_emit_cond_exec(device, ace_cs, workaround_cond_va, 6 * num_views /* Nx DISPATCH_TASKMESH_DIRECT_ACE */); for (unsigned v = 0; v < num_views; ++v) { - radv_cs_emit_dispatch_taskmesh_direct_ace_packet(cmd_buffer, 0, 0, 0); + radv_cs_emit_dispatch_taskmesh_direct_ace_packet(device, cmd_state, ace_cs, 0, 0, 0); } } } @@ -9815,7 +9817,8 @@ radv_CmdDrawMeshTasksEXT(VkCommandBuffer commandBuffer, uint32_t x, uint32_t y, return; if (radv_cmdbuf_has_stage(cmd_buffer, MESA_SHADER_TASK)) { - radv_emit_direct_taskmesh_draw_packets(cmd_buffer, x, y, z); + radv_emit_direct_taskmesh_draw_packets(cmd_buffer->device, &cmd_buffer->state, cmd_buffer->cs, + cmd_buffer->gang.cs, x, y, z); } else { radv_emit_direct_mesh_draw_packet(cmd_buffer, x, y, z); } @@ -9848,7 +9851,8 @@ radv_CmdDrawMeshTasksIndirectEXT(VkCommandBuffer commandBuffer, VkBuffer _buffer return; if (radv_cmdbuf_has_stage(cmd_buffer, MESA_SHADER_TASK)) { - radv_emit_indirect_taskmesh_draw_packets(cmd_buffer, &info, 0); + radv_emit_indirect_taskmesh_draw_packets(cmd_buffer->device, &cmd_buffer->state, cmd_buffer->cs, + cmd_buffer->gang.cs, &info, 0); } else { radv_emit_indirect_mesh_draw_packets(cmd_buffer, &info); } @@ -9895,7 +9899,8 @@ radv_CmdDrawMeshTasksIndirectCountEXT(VkCommandBuffer commandBuffer, VkBuffer _b workaround_cond_va = radv_buffer_get_va(cmd_buffer->upload.upload_bo) + workaround_cond_off; } - radv_emit_indirect_taskmesh_draw_packets(cmd_buffer, &info, workaround_cond_va); + radv_emit_indirect_taskmesh_draw_packets(cmd_buffer->device, &cmd_buffer->state, cmd_buffer->cs, + cmd_buffer->gang.cs, &info, workaround_cond_va); } else { radv_emit_indirect_mesh_draw_packets(cmd_buffer, &info); }