From 683904f4c0aeeebec7b4d36fab13e895a1f16e6a Mon Sep 17 00:00:00 2001 From: Simon Perretta Date: Wed, 1 Jan 2025 15:10:25 +0000 Subject: [PATCH] pvr, pco: initial ssbo and atomics support Signed-off-by: Simon Perretta Acked-by: Erik Faye-Lund Part-of: --- src/imagination/pco/pco_internal.h | 13 ++ src/imagination/pco/pco_map.py | 81 ++++++++++ src/imagination/pco/pco_nir.c | 58 ++++++- src/imagination/pco/pco_ops.py | 4 + src/imagination/pco/pco_trans_nir.c | 163 +++++++++++++++++++- src/imagination/vulkan/pvr_descriptor_set.c | 2 + 6 files changed, 314 insertions(+), 7 deletions(-) diff --git a/src/imagination/pco/pco_internal.h b/src/imagination/pco/pco_internal.h index 0533f4c0f5d..1573c73c4c5 100644 --- a/src/imagination/pco/pco_internal.h +++ b/src/imagination/pco/pco_internal.h @@ -2025,6 +2025,19 @@ static inline void pco_ref_new_ssa_addr_comps(pco_func *func, addr_comps[1] = pco_ref_new_ssa32(func); } +/** + * \brief Builds and returns a new 32x(2+n) SSA address and data reference. + * + * \param[in,out] func The function. + * \param[in] data_size The size of the data. + * \return SSA address and data reference. + */ +static inline pco_ref pco_ref_new_ssa_addr_data(pco_func *func, + unsigned data_size) +{ + return pco_ref_new_ssa(func, 32, 2 + data_size); +} + /** * \brief Builds and returns a virtual register reference. * diff --git a/src/imagination/pco/pco_map.py b/src/imagination/pco/pco_map.py index e919810ad18..850eed971ae 100644 --- a/src/imagination/pco/pco_map.py +++ b/src/imagination/pco/pco_map.py @@ -210,6 +210,12 @@ enum_map(OM_MCU_CACHE_MODE_LD.t, F_CACHEMODE_LD, [ ('force_line_fill', 'force_line_fill'), ]) +enum_map(OM_MCU_CACHE_MODE_ST.t, F_CACHEMODE_ST, [ + ('write_through', 'write_through'), + ('write_back', 'write_back'), + ('lazy_write_back', 'write_back_lazy'), +]) + enum_map(OM_TST_OP_MAIN.t, F_TST_OP, [ ('zero', 'z'), ('gzero', 'gz'), @@ -277,6 +283,19 @@ enum_map(OM_BRANCH_CND.t, F_BPRED, [ ('anyinst', 'anyp'), ]) +enum_map(OM_ATOM_OP.t, F_ATOMIC_OP, [ + ('add', 'add'), + ('sub', 'sub'), + ('xchg', 'xchg'), + ('umin', 'umin'), + ('imin', 'imin'), + ('umax', 'umax'), + ('imax', 'imax'), + ('and', 'and'), + ('or', 'or'), + ('xor', 'xor'), +]) + class OpRef(object): def __init__(self, ref_type, index, mods): self.type = ref_type @@ -1177,6 +1196,32 @@ encode_map(O_LD, op_ref_maps=[('backend', ['s3'], ['drc', 'imm', ['s0', 's1', 's2', 's3', 's4', 's5']])] ) +encode_map(O_ST, + encodings=[ + (I_ST_IMMBL, [ + ('drc', ('pco_ref_get_drc', SRC(2))), + ('srcseladd', ('pco_ref_srcsel', SRC(4))), + ('burstlen', ('pco_ref_get_imm', SRC(3))), + ('cachemode_st', OM_MCU_CACHE_MODE_ST), + ('srcseldata', ('pco_ref_srcsel', SRC(0))), + ('dsize', ('pco_ref_get_imm', SRC(1))) + ]) + ], + op_ref_maps=[('backend', [], [['s0', 's1', 's2', 's3', 's4', 's5'], 'imm', 'drc', 'imm', ['s0', 's1', 's2', 's3', 's4', 's5'], ['s0', 's1', 's2', 's3', 's4', 's5', '_']])] +) + +encode_map(O_ATOMIC, + encodings=[ + (I_ATOMIC, [ + ('drc', ('pco_ref_get_drc', SRC(0))), + ('atomic_op', OM_ATOM_OP), + ('srcsel', ('pco_ref_srcsel', SRC(1))), + ('dstsel', ('pco_ref_srcsel', DEST(0))) + ]) + ], + op_ref_maps=[('backend', [['s0', 's1', 's2', 's3', 's4', 's5']], ['drc', ['s0', 's1', 's2', 's3', 's4', 's5']])] +) + encode_map(O_BBYP0BM, encodings=[ (I_PHASE0_SRC, [ @@ -1831,6 +1876,24 @@ group_map(O_IMUL32, ] ) +group_map(O_ST32, + hdr=(I_IGRP_HDR_MAIN, [ + ('oporg', 'be'), + ('olchk', OM_OLCHK), + ('w1p', False), + ('w0p', False), + ('cc', OM_EXEC_CND), + ('end', OM_END), + ('atom', OM_ATOM), + ('rpt', 1) + ]), + enc_ops=[('backend', O_ST, [], [SRC(0), 'pco_ref_imm8(PCO_DSIZE_32BIT)', SRC(1), SRC(2), SRC(3), SRC(4)], [(OM_MCU_CACHE_MODE_ST, OM_MCU_CACHE_MODE_ST)])], + srcs=[ + ('s[0]', ('backend', SRC(0)), 's0'), + ('s[3]', ('backend', SRC(4)), 's3') + ] +) + group_map(O_UVSW_WRITE, hdr=(I_IGRP_HDR_MAIN, [ ('oporg', 'be'), @@ -1966,6 +2029,24 @@ group_map(O_LD, ] ) +group_map(O_ATOMIC, + hdr=(I_IGRP_HDR_MAIN, [ + ('oporg', 'be'), + ('olchk', OM_OLCHK), + ('w1p', False), + ('w0p', False), + ('cc', OM_EXEC_CND), + ('end', OM_END), + ('atom', False), + ('rpt', 1) + ]), + enc_ops=[('backend', O_ATOMIC)], + srcs=[ + ('s[0]', ('backend', SRC(1)), 's0'), + ('s[3]', ('backend', DEST(0)), 's3') + ] +) + group_map(O_MOVI32, hdr=(I_IGRP_HDR_BITWISE, [ ('opcnt', 'p0'), diff --git a/src/imagination/pco/pco_nir.c b/src/imagination/pco/pco_nir.c index 35d6a8f1586..f817f51d899 100644 --- a/src/imagination/pco/pco_nir.c +++ b/src/imagination/pco/pco_nir.c @@ -22,6 +22,7 @@ static const struct spirv_to_nir_options spirv_options = { .environment = NIR_SPIRV_VULKAN, .ubo_addr_format = nir_address_format_vec2_index_32bit_offset, + .ssbo_addr_format = nir_address_format_vec2_index_32bit_offset, .min_ubo_alignment = PVR_UNIFORM_BUFFER_OFFSET_ALIGNMENT, .min_ssbo_alignment = PVR_STORAGE_BUFFER_OFFSET_ALIGNMENT, @@ -220,8 +221,8 @@ void pco_lower_nir(pco_ctx *ctx, nir_shader *nir, pco_data *data) NIR_PASS(_, nir, nir_lower_explicit_io, - nir_var_mem_ubo, - spirv_options.ubo_addr_format); + nir_var_mem_ubo | nir_var_mem_ssbo, + nir_address_format_vec2_index_32bit_offset); NIR_PASS(_, nir, pco_nir_lower_vk, &data->common); @@ -382,6 +383,57 @@ static void gather_fs_data(nir_shader *nir, pco_data *data) } } +/** + * \brief Checks whether a NIR intrinsic op is atomic. + * + * \param[in] op The NIR intrinsic op. + * \return True if the intrinsic op is atomic, else false. + */ +static inline bool intr_op_is_atomic(nir_intrinsic_op op) +{ + switch (op) { + case nir_intrinsic_ssbo_atomic: + case nir_intrinsic_shared_atomic: + case nir_intrinsic_shared_atomic_swap: + return true; + + default: + break; + } + return false; +} + +/** + * \brief Gather common data pass. + * + * \param[in] b NIR builder. + * \param[in] intr NIR intrinsic instruction. + * \param[in,out] cb_data Callback data. + * \return True if the shader was modified (always return false). + */ +static bool gather_common_data_pass(UNUSED struct nir_builder *b, + nir_intrinsic_instr *intr, + void *cb_data) +{ + pco_data *data = cb_data; + data->common.uses.atomics |= intr_op_is_atomic(intr->intrinsic); + return false; +} + +/** + * \brief Gathers data common to all shader stages. + * + * \param[in] nir NIR shader. + * \param[in,out] data Shader data. + */ +static void gather_common_data(nir_shader *nir, pco_data *data) +{ + nir_shader_intrinsics_pass(nir, + gather_common_data_pass, + nir_metadata_all, + data); +} + /** * \brief Gathers shader data. * @@ -390,6 +442,8 @@ static void gather_fs_data(nir_shader *nir, pco_data *data) */ static void gather_data(nir_shader *nir, pco_data *data) { + gather_common_data(nir, data); + switch (nir->info.stage) { case MESA_SHADER_FRAGMENT: return gather_fs_data(nir, data); diff --git a/src/imagination/pco/pco_ops.py b/src/imagination/pco/pco_ops.py index 114a36ecfcf..32996e0405c 100644 --- a/src/imagination/pco/pco_ops.py +++ b/src/imagination/pco/pco_ops.py @@ -341,6 +341,8 @@ O_FITR = hw_op('fitr', OM_ALU + [OM_ITR_MODE, OM_SAT], 1, 3) O_FITRP = hw_op('fitrp', OM_ALU + [OM_ITR_MODE, OM_SAT], 1, 4) O_LD = hw_op('ld', OM_ALU_RPT1 + [OM_MCU_CACHE_MODE_LD], 1, 3) +O_ST = hw_direct_op('st', [OM_MCU_CACHE_MODE_ST], 0, 6) +O_ATOMIC = hw_op('atomic', [OM_OLCHK, OM_EXEC_CND, OM_END, OM_ATOM_OP], 1, 2) ## Bitwise. O_MOVI32 = hw_op('movi32', OM_ALU, 1, 1) @@ -377,6 +379,8 @@ O_MAX = hw_op('max', OM_ALU + [OM_TST_TYPE_MAIN], 1, 2, [], [[RM_ABS, RM_NEG], [ O_IADD32 = hw_op('iadd32', OM_ALU + [OM_S], 1, 3, [], [[RM_ABS, RM_NEG], [RM_ABS, RM_NEG]]) O_IMUL32 = hw_op('imul32', OM_ALU + [OM_S], 1, 3, [], [[RM_ABS, RM_NEG], [RM_ABS, RM_NEG]]) +O_ST32 = hw_op('st32', OM_ALU_RPT1 + [OM_MCU_CACHE_MODE_ST], 0, 5) + # Pseudo-ops (unmapped). O_FNEG = pseudo_op('fneg', OM_ALU, 1, 1) O_FABS = pseudo_op('fabs', OM_ALU, 1, 1) diff --git a/src/imagination/pco/pco_trans_nir.c b/src/imagination/pco/pco_trans_nir.c index c4060d1caa4..9e3cbe07546 100644 --- a/src/imagination/pco/pco_trans_nir.c +++ b/src/imagination/pco/pco_trans_nir.c @@ -445,10 +445,10 @@ static unsigned fetch_resource_base_reg(const pco_common_data *common, return reg_index; } -static pco_instr *trans_load_ubo(trans_ctx *tctx, - nir_intrinsic_instr *intr, - pco_ref dest, - pco_ref offset_src) +static pco_instr *trans_load_buffer(trans_ctx *tctx, + nir_intrinsic_instr *intr, + pco_ref dest, + pco_ref offset_src) { const pco_common_data *common = &tctx->shader->data.common; @@ -485,6 +485,150 @@ static pco_instr *trans_load_ubo(trans_ctx *tctx, addr); } +static pco_instr *trans_store_buffer(trans_ctx *tctx, + nir_intrinsic_instr *intr, + pco_ref data_src, + pco_ref offset_src) +{ + const pco_common_data *common = &tctx->shader->data.common; + + unsigned chans = pco_ref_get_chans(data_src); + unsigned bits = pco_ref_get_bits(data_src); + + uint32_t packed_desc = nir_src_comp_as_uint(intr->src[1], 0); + unsigned elem = nir_src_comp_as_uint(intr->src[1], 1); + unsigned sh_index = fetch_resource_base_reg(common, packed_desc, elem); + + pco_ref base_addr[2]; + pco_ref_hwreg_addr_comps(sh_index, PCO_REG_CLASS_SHARED, base_addr); + + pco_ref addr_data_comps[3] = { + [2] = data_src, + }; + pco_ref_new_ssa_addr_comps(tctx->func, addr_data_comps); + + pco_add64_32(&tctx->b, + addr_data_comps[0], + addr_data_comps[1], + base_addr[0], + base_addr[1], + offset_src, + pco_ref_null(), + .s = true); + + pco_ref addr_data = pco_ref_new_ssa_addr_data(tctx->func, chans); + pco_vec(&tctx->b, addr_data, ARRAY_SIZE(addr_data_comps), addr_data_comps); + + pco_ref data_comp = pco_ref_new_ssa(tctx->func, + pco_ref_get_bits(data_src), + pco_ref_get_chans(data_src)); + pco_comp(&tctx->b, data_comp, addr_data, pco_ref_val16(2)); + + switch (bits) { + case 32: + return pco_st32(&tctx->b, + data_comp, + pco_ref_drc(PCO_DRC_0), + pco_ref_imm8(chans), + addr_data, + pco_ref_null()); + + default: + break; + } + + UNREACHABLE(""); +} + +static inline enum pco_atom_op to_atom_op(nir_atomic_op op) +{ + switch (op) { + case nir_atomic_op_iadd: + return PCO_ATOM_OP_ADD; + + case nir_atomic_op_xchg: + return PCO_ATOM_OP_XCHG; + + case nir_atomic_op_umin: + return PCO_ATOM_OP_UMIN; + + case nir_atomic_op_imin: + return PCO_ATOM_OP_IMIN; + + case nir_atomic_op_umax: + return PCO_ATOM_OP_UMAX; + + case nir_atomic_op_imax: + return PCO_ATOM_OP_IMAX; + + case nir_atomic_op_iand: + return PCO_ATOM_OP_AND; + + case nir_atomic_op_ior: + return PCO_ATOM_OP_OR; + + case nir_atomic_op_ixor: + return PCO_ATOM_OP_XOR; + + default: + break; + } + + UNREACHABLE(""); +} + +static pco_instr *trans_atomic_buffer(trans_ctx *tctx, + nir_intrinsic_instr *intr, + pco_ref dest, + pco_ref offset_src, + pco_ref data_src) +{ + const pco_common_data *common = &tctx->shader->data.common; + + enum pco_atom_op atom_op = to_atom_op(nir_intrinsic_atomic_op(intr)); + + unsigned chans = pco_ref_get_chans(dest); + unsigned bits = pco_ref_get_bits(dest); + + uint32_t packed_desc = nir_src_comp_as_uint(intr->src[0], 0); + unsigned elem = nir_src_comp_as_uint(intr->src[0], 1); + unsigned sh_index = fetch_resource_base_reg(common, packed_desc, elem); + + pco_ref base_addr[2]; + pco_ref_hwreg_addr_comps(sh_index, PCO_REG_CLASS_SHARED, base_addr); + + pco_ref addr_data_comps[3] = { + [2] = data_src, + }; + pco_ref_new_ssa_addr_comps(tctx->func, addr_data_comps); + + pco_add64_32(&tctx->b, + addr_data_comps[0], + addr_data_comps[1], + base_addr[0], + base_addr[1], + offset_src, + pco_ref_null(), + .s = true); + + pco_ref addr_data = pco_ref_new_ssa_addr_data(tctx->func, chans); + pco_vec(&tctx->b, addr_data, ARRAY_SIZE(addr_data_comps), addr_data_comps); + + switch (bits) { + case 32: + return pco_atomic(&tctx->b, + dest, + pco_ref_drc(PCO_DRC_0), + addr_data, + .atom_op = atom_op); + + default: + break; + } + + UNREACHABLE(""); +} + /** * \brief Translates a NIR vs load system value intrinsic into PCO. * @@ -545,7 +689,16 @@ static pco_instr *trans_intr(trans_ctx *tctx, nir_intrinsic_instr *intr) break; case nir_intrinsic_load_ubo: - instr = trans_load_ubo(tctx, intr, dest, src[1]); + case nir_intrinsic_load_ssbo: + instr = trans_load_buffer(tctx, intr, dest, src[1]); + break; + + case nir_intrinsic_store_ssbo: + instr = trans_store_buffer(tctx, intr, src[0], src[2]); + break; + + case nir_intrinsic_ssbo_atomic: + instr = trans_atomic_buffer(tctx, intr, dest, src[1], src[2]); break; case nir_intrinsic_load_vertex_id: diff --git a/src/imagination/vulkan/pvr_descriptor_set.c b/src/imagination/vulkan/pvr_descriptor_set.c index 90d2bc50011..2204def9da6 100644 --- a/src/imagination/vulkan/pvr_descriptor_set.c +++ b/src/imagination/vulkan/pvr_descriptor_set.c @@ -76,6 +76,7 @@ static unsigned pvr_descriptor_size(VkDescriptorType type) { switch (type) { case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: return sizeof(struct pvr_buffer_descriptor); default: @@ -501,6 +502,7 @@ void pvr_UpdateDescriptorSets(VkDevice _device, switch (write->descriptorType) { case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: for (uint32_t j = 0; j < write->descriptorCount; j++) { write_buffer(set, &write->pBufferInfo[j],