diff --git a/src/amd/vulkan/radv_queue.c b/src/amd/vulkan/radv_queue.c index 2f58018ff16..1fc9f972642 100644 --- a/src/amd/vulkan/radv_queue.c +++ b/src/amd/vulkan/radv_queue.c @@ -231,6 +231,44 @@ radv_queue_submit_empty(struct radv_queue *queue, struct vk_queue_submit *submis submission->signals); } +static void +radv_set_ring_buffer(const struct radv_physical_device *pdev, struct radeon_winsys_bo *bo, uint32_t offset, + uint32_t ring_size, bool add_tid, bool swizzle_enable, bool oob_select_raw, uint32_t element_size, + uint32_t index_stride, uint32_t desc[4]) +{ + const uint8_t oob_select = oob_select_raw ? V_008F0C_OOB_SELECT_RAW : V_008F0C_OOB_SELECT_DISABLED; + const uint64_t va = radv_buffer_get_va(bo) + offset; + + uint32_t rsrc_word1 = S_008F04_BASE_ADDRESS_HI(va >> 32); + if (pdev->info.gfx_level >= GFX11) { + rsrc_word1 |= S_008F04_SWIZZLE_ENABLE_GFX11(swizzle_enable); + } else { + rsrc_word1 |= S_008F04_SWIZZLE_ENABLE_GFX6(swizzle_enable); + } + + uint32_t rsrc_word3 = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) | + S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) | + S_008F0C_INDEX_STRIDE(index_stride) | S_008F0C_ADD_TID_ENABLE(add_tid); + + if (pdev->info.gfx_level >= GFX11) { + rsrc_word3 |= S_008F0C_FORMAT_GFX10(V_008F0C_GFX11_FORMAT_32_FLOAT) | S_008F0C_OOB_SELECT(oob_select); + } else if (pdev->info.gfx_level >= GFX10) { + rsrc_word3 |= S_008F0C_FORMAT_GFX10(V_008F0C_GFX10_FORMAT_32_FLOAT) | S_008F0C_OOB_SELECT(oob_select) | + S_008F0C_RESOURCE_LEVEL(1); + } else { + /* DATA_FORMAT is STRIDE[14:17] for MUBUF with ADD_TID_ENABLE=1 */ + const uint32_t data_format = pdev->info.gfx_level >= GFX8 && add_tid ? 0 : V_008F0C_BUF_DATA_FORMAT_32; + + rsrc_word3 |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) | S_008F0C_DATA_FORMAT(data_format) | + S_008F0C_ELEMENT_SIZE(element_size); + } + + desc[0] = va; + desc[1] = rsrc_word1; + desc[2] = ring_size; + desc[3] = rsrc_word3; +} + static void radv_fill_shader_rings(struct radv_device *device, uint32_t *desc, struct radeon_winsys_bo *scratch_bo, uint32_t esgs_ring_size, struct radeon_winsys_bo *esgs_ring_bo, uint32_t gsvs_ring_size, @@ -256,212 +294,55 @@ radv_fill_shader_rings(struct radv_device *device, uint32_t *desc, struct radeon desc += 4; if (esgs_ring_bo) { - uint64_t esgs_va = radv_buffer_get_va(esgs_ring_bo); - /* stride 0, num records - size, add tid, swizzle, elsize4, index stride 64 */ - desc[0] = esgs_va; - desc[1] = S_008F04_BASE_ADDRESS_HI(esgs_va >> 32); - desc[2] = esgs_ring_size; - desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) | - S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) | - S_008F0C_INDEX_STRIDE(3) | S_008F0C_ADD_TID_ENABLE(1); - - if (pdev->info.gfx_level >= GFX11) - desc[1] |= S_008F04_SWIZZLE_ENABLE_GFX11(1); - else - desc[1] |= S_008F04_SWIZZLE_ENABLE_GFX6(1); - - if (pdev->info.gfx_level >= GFX11) { - desc[3] |= - S_008F0C_FORMAT_GFX10(V_008F0C_GFX11_FORMAT_32_FLOAT) | S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED); - } else if (pdev->info.gfx_level >= GFX10) { - desc[3] |= S_008F0C_FORMAT_GFX10(V_008F0C_GFX10_FORMAT_32_FLOAT) | - S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) | S_008F0C_RESOURCE_LEVEL(1); - } else if (pdev->info.gfx_level >= GFX8) { - /* DATA_FORMAT is STRIDE[14:17] for MUBUF with ADD_TID_ENABLE=1 */ - desc[3] |= - S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) | S_008F0C_DATA_FORMAT(0) | S_008F0C_ELEMENT_SIZE(1); - } else { - desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) | - S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) | S_008F0C_ELEMENT_SIZE(1); - } + radv_set_ring_buffer(pdev, esgs_ring_bo, 0, esgs_ring_size, true, true, false, 1, 3, &desc[0]); /* GS entry for ES->GS ring */ /* stride 0, num records - size, elsize0, index stride 0 */ - desc[4] = esgs_va; - desc[5] = S_008F04_BASE_ADDRESS_HI(esgs_va >> 32); - desc[6] = esgs_ring_size; - desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) | - S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W); - - if (pdev->info.gfx_level >= GFX11) { - desc[7] |= - S_008F0C_FORMAT_GFX10(V_008F0C_GFX11_FORMAT_32_FLOAT) | S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED); - } else if (pdev->info.gfx_level >= GFX10) { - desc[7] |= S_008F0C_FORMAT_GFX10(V_008F0C_GFX10_FORMAT_32_FLOAT) | - S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) | S_008F0C_RESOURCE_LEVEL(1); - } else { - desc[7] |= - S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) | S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32); - } + radv_set_ring_buffer(pdev, esgs_ring_bo, 0, esgs_ring_size, false, false, false, 0, 0, &desc[4]); } desc += 8; if (gsvs_ring_bo) { - uint64_t gsvs_va = radv_buffer_get_va(gsvs_ring_bo); - /* VS entry for GS->VS ring */ /* stride 0, num records - size, elsize0, index stride 0 */ - desc[0] = gsvs_va; - desc[1] = S_008F04_BASE_ADDRESS_HI(gsvs_va >> 32); - desc[2] = gsvs_ring_size; - desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) | - S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W); - - if (pdev->info.gfx_level >= GFX11) { - desc[3] |= - S_008F0C_FORMAT_GFX10(V_008F0C_GFX11_FORMAT_32_FLOAT) | S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED); - } else if (pdev->info.gfx_level >= GFX10) { - desc[3] |= S_008F0C_FORMAT_GFX10(V_008F0C_GFX10_FORMAT_32_FLOAT) | - S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) | S_008F0C_RESOURCE_LEVEL(1); - } else { - desc[3] |= - S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) | S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32); - } + radv_set_ring_buffer(pdev, gsvs_ring_bo, 0, gsvs_ring_size, false, false, false, 0, 0, &desc[0]); /* stride gsvs_itemsize, num records 64 elsize 4, index stride 16 */ /* shader will patch stride and desc[2] */ - desc[4] = gsvs_va; - desc[5] = S_008F04_BASE_ADDRESS_HI(gsvs_va >> 32); - desc[6] = 0; - desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) | - S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) | - S_008F0C_INDEX_STRIDE(1) | S_008F0C_ADD_TID_ENABLE(true); - - if (pdev->info.gfx_level >= GFX11) - desc[5] |= S_008F04_SWIZZLE_ENABLE_GFX11(1); - else - desc[5] |= S_008F04_SWIZZLE_ENABLE_GFX6(1); - - if (pdev->info.gfx_level >= GFX11) { - desc[7] |= - S_008F0C_FORMAT_GFX10(V_008F0C_GFX11_FORMAT_32_FLOAT) | S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED); - } else if (pdev->info.gfx_level >= GFX10) { - desc[7] |= S_008F0C_FORMAT_GFX10(V_008F0C_GFX10_FORMAT_32_FLOAT) | - S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) | S_008F0C_RESOURCE_LEVEL(1); - } else if (pdev->info.gfx_level >= GFX8) { - /* DATA_FORMAT is STRIDE[14:17] for MUBUF with ADD_TID_ENABLE=1 */ - desc[7] |= - S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) | S_008F0C_DATA_FORMAT(0) | S_008F0C_ELEMENT_SIZE(1); - } else { - desc[7] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) | - S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) | S_008F0C_ELEMENT_SIZE(1); - } + radv_set_ring_buffer(pdev, gsvs_ring_bo, 0, 0, true, true, false, 1, 1, &desc[4]); } desc += 8; if (tess_rings_bo) { - uint64_t tess_va = radv_buffer_get_va(tess_rings_bo); - uint64_t tess_offchip_va = tess_va + pdev->hs.tess_offchip_ring_offset; + radv_set_ring_buffer(pdev, tess_rings_bo, 0, pdev->hs.tess_factor_ring_size, false, false, true, 0, 0, &desc[0]); - desc[0] = tess_va; - desc[1] = S_008F04_BASE_ADDRESS_HI(tess_va >> 32); - desc[2] = pdev->hs.tess_factor_ring_size; - desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) | - S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W); - - if (pdev->info.gfx_level >= GFX11) { - desc[3] |= - S_008F0C_FORMAT_GFX10(V_008F0C_GFX11_FORMAT_32_FLOAT) | S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW); - } else if (pdev->info.gfx_level >= GFX10) { - desc[3] |= S_008F0C_FORMAT_GFX10(V_008F0C_GFX10_FORMAT_32_FLOAT) | - S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) | S_008F0C_RESOURCE_LEVEL(1); - } else { - desc[3] |= - S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) | S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32); - } - - desc[4] = tess_offchip_va; - desc[5] = S_008F04_BASE_ADDRESS_HI(tess_offchip_va >> 32); - desc[6] = pdev->hs.tess_offchip_ring_size; - desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) | - S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W); - - if (pdev->info.gfx_level >= GFX11) { - desc[7] |= - S_008F0C_FORMAT_GFX10(V_008F0C_GFX11_FORMAT_32_FLOAT) | S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW); - } else if (pdev->info.gfx_level >= GFX10) { - desc[7] |= S_008F0C_FORMAT_GFX10(V_008F0C_GFX10_FORMAT_32_FLOAT) | - S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) | S_008F0C_RESOURCE_LEVEL(1); - } else { - desc[7] |= - S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) | S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32); - } + radv_set_ring_buffer(pdev, tess_rings_bo, pdev->hs.tess_offchip_ring_offset, pdev->hs.tess_offchip_ring_size, + false, false, true, 0, 0, &desc[4]); } desc += 8; if (task_rings_bo) { - uint64_t task_va = radv_buffer_get_va(task_rings_bo); - uint64_t task_draw_ring_va = task_va + pdev->task_info.draw_ring_offset; - uint64_t task_payload_ring_va = task_va + pdev->task_info.payload_ring_offset; + radv_set_ring_buffer(pdev, task_rings_bo, pdev->task_info.draw_ring_offset, + pdev->task_info.num_entries * AC_TASK_DRAW_ENTRY_BYTES, false, false, false, 0, 0, &desc[0]); - desc[0] = task_draw_ring_va; - desc[1] = S_008F04_BASE_ADDRESS_HI(task_draw_ring_va >> 32); - desc[2] = pdev->task_info.num_entries * AC_TASK_DRAW_ENTRY_BYTES; - desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) | - S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W); - - if (pdev->info.gfx_level >= GFX11) { - desc[3] |= - S_008F0C_FORMAT_GFX10(V_008F0C_GFX11_FORMAT_32_UINT) | S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED); - } else { - assert(pdev->info.gfx_level >= GFX10_3); - desc[3] |= S_008F0C_FORMAT_GFX10(V_008F0C_GFX10_FORMAT_32_UINT) | - S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) | S_008F0C_RESOURCE_LEVEL(1); - } - - desc[4] = task_payload_ring_va; - desc[5] = S_008F04_BASE_ADDRESS_HI(task_payload_ring_va >> 32); - desc[6] = pdev->task_info.num_entries * AC_TASK_PAYLOAD_ENTRY_BYTES; - desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) | - S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W); - - if (pdev->info.gfx_level >= GFX11) { - desc[7] |= - S_008F0C_FORMAT_GFX10(V_008F0C_GFX11_FORMAT_32_UINT) | S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED); - } else { - assert(pdev->info.gfx_level >= GFX10_3); - desc[7] |= S_008F0C_FORMAT_GFX10(V_008F0C_GFX10_FORMAT_32_UINT) | - S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) | S_008F0C_RESOURCE_LEVEL(1); - } + radv_set_ring_buffer(pdev, task_rings_bo, pdev->task_info.payload_ring_offset, + pdev->task_info.num_entries * AC_TASK_PAYLOAD_ENTRY_BYTES, false, false, false, 0, 0, + &desc[4]); } desc += 8; if (mesh_scratch_ring_bo) { - uint64_t va = radv_buffer_get_va(mesh_scratch_ring_bo); - - desc[0] = va; - desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32); - desc[2] = RADV_MESH_SCRATCH_NUM_ENTRIES * RADV_MESH_SCRATCH_ENTRY_BYTES; - desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) | - S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W); - - if (pdev->info.gfx_level >= GFX11) { - desc[3] |= - S_008F0C_FORMAT_GFX10(V_008F0C_GFX11_FORMAT_32_UINT) | S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED); - } else { - assert(pdev->info.gfx_level >= GFX10_3); - desc[3] |= S_008F0C_FORMAT_GFX10(V_008F0C_GFX10_FORMAT_32_UINT) | - S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) | S_008F0C_RESOURCE_LEVEL(1); - } + radv_set_ring_buffer(pdev, mesh_scratch_ring_bo, 0, RADV_MESH_SCRATCH_NUM_ENTRIES * RADV_MESH_SCRATCH_ENTRY_BYTES, + false, false, false, 0, 0, &desc[0]); } desc += 4;