diff --git a/src/imagination/.clang-format b/src/imagination/.clang-format index 2883b36f0ef..c56e0f277a3 100644 --- a/src/imagination/.clang-format +++ b/src/imagination/.clang-format @@ -197,10 +197,12 @@ MacroBlockEnd: '' #AttributeMacros: [] # Requires clang-12 ForEachMacros: [ + 'BITSET_FOREACH_SET', 'foreach_instr', 'foreach_instr_safe', 'foreach_list_typed', 'hash_table_foreach', + 'hash_table_u64_foreach', 'LIST_FOR_EACH_ENTRY', 'LIST_FOR_EACH_ENTRY_FROM', 'LIST_FOR_EACH_ENTRY_FROM_REV', @@ -230,14 +232,28 @@ ForEachMacros: [ 'nir_foreach_use_safe', 'nir_foreach_variable_with_modes', 'pco_foreach_block_in_func', + 'pco_foreach_block_in_func_from', + 'pco_foreach_block_in_func_from_rev', + 'pco_foreach_block_in_func_rev', 'pco_foreach_cf_node_in_func', 'pco_foreach_cf_node_in_if_else', 'pco_foreach_cf_node_in_if_then', 'pco_foreach_cf_node_in_loop', 'pco_foreach_func_in_shader', + 'pco_foreach_func_in_shader_rev', 'pco_foreach_igrp_in_block', + 'pco_foreach_instr_dest', + 'pco_foreach_instr_dest_ssa', + 'pco_foreach_instr_src', + 'pco_foreach_instr_src_ssa', 'pco_foreach_instr_in_block', 'pco_foreach_instr_in_block_safe', + 'pco_foreach_instr_in_func', + 'pco_foreach_instr_in_func_from', + 'pco_foreach_instr_in_func_from_rev', + 'pco_foreach_instr_in_func_safe', + 'pco_foreach_instr_in_func_rev', + 'pco_foreach_instr_in_func_safe_rev', 'pco_foreach_phi_src_in_instr', 'rogue_foreach_block', 'rogue_foreach_block_safe', @@ -284,6 +300,7 @@ ForEachMacros: [ 'u_foreach_bit64', 'u_vector_foreach', 'util_dynarray_foreach', + 'util_dynarray_foreach_reverse', 'vk_foreach_struct', 'vk_foreach_struct_const', # FIXME: vk_outarray_append doesn't fit here, remove diff --git a/src/imagination/pco/meson.build b/src/imagination/pco/meson.build index e5b9bd01ee8..a912cca2666 100644 --- a/src/imagination/pco/meson.build +++ b/src/imagination/pco/meson.build @@ -6,12 +6,18 @@ inc_powervr_compiler = include_directories(['.']) libpowervr_compiler_files = files( 'pco.c', 'pco_binary.c', + 'pco_const_imms.c', 'pco_debug.c', 'pco_end.c', 'pco_group_instrs.c', + 'pco_index.c', 'pco_ir.c', 'pco_nir.c', + 'pco_nir_pvfio.c', + 'pco_opt.c', 'pco_print.c', + 'pco_ra.c', + 'pco_schedule.c', 'pco_trans_nir.c', 'pco_validate.c', ) diff --git a/src/imagination/pco/pco.c b/src/imagination/pco/pco.c index a7cd1c8a58f..80517af3d24 100644 --- a/src/imagination/pco/pco.c +++ b/src/imagination/pco/pco.c @@ -270,3 +270,24 @@ pco_igrp *pco_igrp_create(pco_func *func) return igrp; } + +/** + * \brief Deletes a PCO instruction. + * + * \param[in,out] instr PCO instruction. + */ +void pco_instr_delete(pco_instr *instr) +{ + list_del(&instr->link); + ralloc_free(instr); +} + +/** + * \brief Returns the number of temps allocated to the entrypoint function. + * + * \param[in] shader PCO shader. + */ +unsigned pco_shader_temps(pco_shader *shader) +{ + return pco_entrypoint(shader)->temps; +} diff --git a/src/imagination/pco/pco.h b/src/imagination/pco/pco.h index 32d33df515b..0ddce469be6 100644 --- a/src/imagination/pco/pco.h +++ b/src/imagination/pco/pco.h @@ -40,6 +40,8 @@ void pco_process_ir(pco_ctx *ctx, pco_shader *shader); void pco_encode_ir(pco_ctx *ctx, pco_shader *shader); void pco_shader_finalize(pco_ctx *ctx, pco_shader *shader); +unsigned pco_shader_temps(pco_shader *shader); + unsigned pco_shader_binary_size(pco_shader *shader); const void *pco_shader_binary_data(pco_shader *shader); diff --git a/src/imagination/pco/pco_builder.h b/src/imagination/pco/pco_builder.h index 4a3fbd33c12..84764b07464 100644 --- a/src/imagination/pco/pco_builder.h +++ b/src/imagination/pco/pco_builder.h @@ -431,4 +431,17 @@ static inline void pco_builder_insert_igrp(pco_builder *b, pco_igrp *igrp) /* Generated op building functions. */ #include "pco_builder_ops.h" +/** + * \brief Returns whether the instruction has the default execution condition. + * + * \param[in] instr The instruction. + * \return True if the instruction has the default execution condition. + */ +static inline bool pco_instr_default_exec(pco_instr *instr) +{ + if (!pco_instr_has_exec_cnd(instr)) + return true; + + return pco_instr_get_exec_cnd(instr) == PCO_EXEC_CND_E1_ZX; +} #endif /* PCO_BUILDER_H */ diff --git a/src/imagination/pco/pco_const_imms.c b/src/imagination/pco/pco_const_imms.c new file mode 100644 index 00000000000..523e03196fc --- /dev/null +++ b/src/imagination/pco/pco_const_imms.c @@ -0,0 +1,215 @@ +/* + * Copyright © 2024 Imagination Technologies Ltd. + * + * SPDX-License-Identifier: MIT + */ + +/** + * \file pco_const_imms.c + * + * \brief PCO constant immediates lowering pass. + */ + +#include "pco.h" +#include "pco_builder.h" +#include "pco_internal.h" +#include "util/macros.h" + +#include +#include +#include + +/** Mapping of constant register values and their indices. */ +struct const_reg_def { + uint32_t val; + uint8_t idx; + bool flr : 1; + bool neg : 1; +}; + +/** Constant register values (sorted for bsearch). */ +static const struct const_reg_def const_reg_defs[] = { + { 0x00000000, 0, false, false }, { 0x00000001, 1, false, false }, + { 0x00000002, 2, false, false }, { 0x00000003, 3, false, false }, + { 0x00000004, 4, false, false }, { 0x00000005, 5, false, false }, + { 0x00000006, 6, false, false }, { 0x00000007, 7, false, false }, + { 0x00000008, 8, false, false }, { 0x00000009, 9, false, false }, + { 0x0000000a, 10, false, false }, { 0x0000000b, 11, false, false }, + { 0x0000000c, 12, false, false }, { 0x0000000d, 13, false, false }, + { 0x0000000e, 14, false, false }, { 0x0000000f, 15, false, false }, + { 0x00000010, 16, false, false }, { 0x00000011, 17, false, false }, + { 0x00000012, 18, false, false }, { 0x00000013, 19, false, false }, + { 0x00000014, 20, false, false }, { 0x00000015, 21, false, false }, + { 0x00000016, 22, false, false }, { 0x00000017, 23, false, false }, + { 0x00000018, 24, false, false }, { 0x00000019, 25, false, false }, + { 0x0000001a, 26, false, false }, { 0x0000001b, 27, false, false }, + { 0x0000001c, 28, false, false }, { 0x0000001d, 29, false, false }, + { 0x0000001e, 30, false, false }, { 0x0000001f, 31, false, false }, + { 0x0000007f, 147, false, false }, { 0x37800000, 134, false, false }, + { 0x38000000, 135, false, false }, { 0x38800000, 88, false, false }, + { 0x39000000, 87, false, false }, { 0x39800000, 86, false, false }, + { 0x3a000000, 85, false, false }, { 0x3a800000, 84, false, false }, + { 0x3b000000, 83, false, false }, { 0x3b4d2e1c, 136, false, false }, + { 0x3b800000, 82, false, false }, { 0x3c000000, 81, false, false }, + { 0x3c800000, 80, false, false }, { 0x3d000000, 79, false, false }, + { 0x3d25aee6, 156, false, false }, { 0x3d6147ae, 140, false, false }, + { 0x3d800000, 78, false, false }, { 0x3d9e8391, 157, false, false }, + { 0x3e000000, 77, false, false }, { 0x3e2aaaab, 153, false, false }, + { 0x3e800000, 76, false, false }, { 0x3e9a209b, 145, false, false }, + { 0x3ea2f983, 128, false, false }, { 0x3eaaaaab, 152, false, false }, + { 0x3ebc5ab2, 90, false, false }, { 0x3ed55555, 138, false, false }, + { 0x3f000000, 75, false, false }, { 0x3f22f983, 129, false, false }, + { 0x3f317218, 146, false, false }, { 0x3f3504f3, 92, false, false }, + { 0x3f490fdb, 93, false, false }, { 0x3f72a76f, 158, false, false }, + { 0x3f800000, 64, false, false }, { 0x3f860a92, 151, false, false }, + { 0x3f870a3d, 139, false, false }, { 0x3fa2f983, 130, false, false }, + { 0x3fb504f3, 91, false, false }, { 0x3fb8aa3b, 155, false, false }, + { 0x3fc90fdb, 94, false, false }, { 0x40000000, 65, false, false }, + { 0x4019999a, 159, false, false }, { 0x402df854, 89, false, false }, + { 0x40400000, 95, true, false }, { 0x40490fdb, 95, false, false }, + { 0x40549a78, 154, false, false }, { 0x40800000, 66, false, false }, + { 0x40c00000, 131, true, false }, { 0x40c90fdb, 131, false, false }, + { 0x41000000, 67, false, false }, { 0x41400000, 132, true, false }, + { 0x41490fdb, 132, false, false }, { 0x414eb852, 137, false, false }, + { 0x41800000, 68, false, false }, { 0x41c80000, 133, true, false }, + { 0x41c90fdb, 133, false, false }, { 0x42000000, 69, false, false }, + { 0x42800000, 70, false, false }, { 0x43000000, 71, false, false }, + { 0x43800000, 72, false, false }, { 0x44000000, 73, false, false }, + { 0x44800000, 74, false, false }, { 0x4b000000, 149, false, false }, + { 0x4b800000, 150, false, false }, { 0x7f7fffff, 148, false, false }, + { 0x7f800000, 142, false, false }, { 0x7fff7fff, 144, false, false }, + { 0x7fffffff, 143, false, true }, { 0x80000000, 141, false, false }, + { 0x80000001, 1, false, true }, { 0x80000002, 2, false, true }, + { 0x80000003, 3, false, true }, { 0x80000004, 4, false, true }, + { 0x80000005, 5, false, true }, { 0x80000006, 6, false, true }, + { 0x80000007, 7, false, true }, { 0x80000008, 8, false, true }, + { 0x80000009, 9, false, true }, { 0x8000000a, 10, false, true }, + { 0x8000000b, 11, false, true }, { 0x8000000c, 12, false, true }, + { 0x8000000d, 13, false, true }, { 0x8000000e, 14, false, true }, + { 0x8000000f, 15, false, true }, { 0x80000010, 16, false, true }, + { 0x80000011, 17, false, true }, { 0x80000012, 18, false, true }, + { 0x80000013, 19, false, true }, { 0x80000014, 20, false, true }, + { 0x80000015, 21, false, true }, { 0x80000016, 22, false, true }, + { 0x80000017, 23, false, true }, { 0x80000018, 24, false, true }, + { 0x80000019, 25, false, true }, { 0x8000001a, 26, false, true }, + { 0x8000001b, 27, false, true }, { 0x8000001c, 28, false, true }, + { 0x8000001d, 29, false, true }, { 0x8000001e, 30, false, true }, + { 0x8000001f, 31, false, true }, { 0x8000007f, 147, false, true }, + { 0xb7800000, 134, false, true }, { 0xb8000000, 135, false, true }, + { 0xb8800000, 88, false, true }, { 0xb9000000, 87, false, true }, + { 0xb9800000, 86, false, true }, { 0xba000000, 85, false, true }, + { 0xba800000, 84, false, true }, { 0xbb000000, 83, false, true }, + { 0xbb4d2e1c, 136, false, true }, { 0xbb800000, 82, false, true }, + { 0xbc000000, 81, false, true }, { 0xbc800000, 80, false, true }, + { 0xbd000000, 79, false, true }, { 0xbd25aee6, 156, false, true }, + { 0xbd6147ae, 140, false, true }, { 0xbd800000, 78, false, true }, + { 0xbd9e8391, 157, false, true }, { 0xbe000000, 77, false, true }, + { 0xbe2aaaab, 153, false, true }, { 0xbe800000, 76, false, true }, + { 0xbe9a209b, 145, false, true }, { 0xbea2f983, 128, false, true }, + { 0xbeaaaaab, 152, false, true }, { 0xbebc5ab2, 90, false, true }, + { 0xbed55555, 138, false, true }, { 0xbf000000, 75, false, true }, + { 0xbf22f983, 129, false, true }, { 0xbf317218, 146, false, true }, + { 0xbf3504f3, 92, false, true }, { 0xbf490fdb, 93, false, true }, + { 0xbf72a76f, 158, false, true }, { 0xbf800000, 64, false, true }, + { 0xbf860a92, 151, false, true }, { 0xbf870a3d, 139, false, true }, + { 0xbfa2f983, 130, false, true }, { 0xbfb504f3, 91, false, true }, + { 0xbfb8aa3b, 155, false, true }, { 0xbfc90fdb, 94, false, true }, + { 0xc0000000, 65, false, true }, { 0xc019999a, 159, false, true }, + { 0xc02df854, 89, false, true }, { 0xc0400000, 95, true, true }, + { 0xc0490fdb, 95, false, true }, { 0xc0549a78, 154, false, true }, + { 0xc0800000, 66, false, true }, { 0xc0c00000, 131, true, true }, + { 0xc0c90fdb, 131, false, true }, { 0xc1000000, 67, false, true }, + { 0xc1400000, 132, true, true }, { 0xc1490fdb, 132, false, true }, + { 0xc14eb852, 137, false, true }, { 0xc1800000, 68, false, true }, + { 0xc1c80000, 133, true, true }, { 0xc1c90fdb, 133, false, true }, + { 0xc2000000, 69, false, true }, { 0xc2800000, 70, false, true }, + { 0xc3000000, 71, false, true }, { 0xc3800000, 72, false, true }, + { 0xc4000000, 73, false, true }, { 0xc4800000, 74, false, true }, + { 0xcb000000, 149, false, true }, { 0xcb800000, 150, false, true }, + { 0xff7fffff, 148, false, true }, { 0xff800000, 142, false, true }, + { 0xffff7fff, 144, false, true }, { 0xffffffff, 143, false, false }, +}; + +/** + * \brief Comparison function for bsearch() to support rogue_const_reg_def. + * + * \param[in] lhs The left hand side of the comparison. + * \param[in] rhs The right hand side of the comparison. + * \return 0 if (lhs == rhs), -1 if (lhs < rhs), 1 if (lhs > rhs). + */ +static int constreg_cmp(const void *lhs, const void *rhs) +{ + const struct const_reg_def *l = lhs; + const struct const_reg_def *r = rhs; + + if (l->val < r->val) + return -1; + else if (l->val > r->val) + return 1; + + return 0; +} + +/** + * \brief Looks up an immediate in constant registers. + * + * \param[in] imm The immediate to lookup. + * \return A pointer to the constant register definition, + * or NULL if no constant register was found. + */ +inline static const struct const_reg_def *constreg_lookup(uint32_t imm) +{ + return bsearch(&(struct const_reg_def){ .val = imm }, + const_reg_defs, + ARRAY_SIZE(const_reg_defs), + sizeof(*const_reg_defs), + constreg_cmp); +} + +/** + * \brief Converts immediates into constant register lookups where possible. + * + * \param[in,out] shader PCO shader. + * \return True if the pass made progress. + */ +bool pco_const_imms(pco_shader *shader) +{ + bool progress = false; + + pco_foreach_func_in_shader (func, shader) { + pco_foreach_instr_in_func_safe (instr, func) { + if (instr->op != PCO_OP_MOVI32) + continue; + + const struct const_reg_def *const_reg_def = + constreg_lookup(pco_ref_get_imm(instr->src[0])); + + if (!const_reg_def) + continue; + + pco_builder b = + pco_builder_create(func, pco_cursor_after_instr(instr)); + + pco_ref dest = instr->dest[0]; + pco_ref const_reg = + pco_ref_hwreg(const_reg_def->idx, PCO_REG_CLASS_CONST); + + if (!const_reg_def->flr && !const_reg_def->neg) { + pco_mov(&b, dest, const_reg); + } else { + if (const_reg_def->flr) + const_reg = pco_ref_flr(const_reg); + + if (const_reg_def->neg) + const_reg = pco_ref_neg(const_reg); + + pco_fadd(&b, dest, const_reg, pco_zero); + } + + pco_instr_delete(instr); + progress = true; + } + } + + return progress; +} diff --git a/src/imagination/pco/pco_end.c b/src/imagination/pco/pco_end.c index 8158344d033..4461b9c6992 100644 --- a/src/imagination/pco/pco_end.c +++ b/src/imagination/pco/pco_end.c @@ -21,6 +21,7 @@ * \brief Processes end of shader instruction(s). * * \param[in,out] shader PCO shader. + * \return True if the pass made progress. */ bool pco_end(pco_shader *shader) { @@ -32,6 +33,21 @@ bool pco_end(pco_shader *shader) pco_builder b = pco_builder_create(entry, pco_cursor_after_block(last_block)); + if (shader->stage == MESA_SHADER_VERTEX) { + if (last_instr->op == PCO_OP_UVSW_WRITE && + pco_instr_default_exec(last_instr) && + pco_instr_get_rpt(last_instr) == 1) { + pco_instr *new_last_instr = + pco_uvsw_write_emit_endtask(&b, + last_instr->src[0], + last_instr->src[1]); + pco_instr_delete(last_instr); + last_instr = new_last_instr; + } else { + last_instr = pco_uvsw_emit_endtask(&b); + } + } + if (last_instr && pco_instr_has_end(last_instr)) { pco_instr_set_end(last_instr, true); return true; diff --git a/src/imagination/pco/pco_index.c b/src/imagination/pco/pco_index.c new file mode 100644 index 00000000000..55394949751 --- /dev/null +++ b/src/imagination/pco/pco_index.c @@ -0,0 +1,66 @@ +/* + * Copyright © 2024 Imagination Technologies Ltd. + * + * SPDX-License-Identifier: MIT + */ + +/** + * \file pco_index.c + * + * \brief PCO indexing pass. + */ + +#include "pco.h" +#include "pco_internal.h" +#include "util/macros.h" +#include "util/ralloc.h" + +#include +#include + +/** + * \brief Indexes all shader child structures. + * + * \param[in,out] shader PCO shader. + * \return True if the pass made progress. + */ +bool pco_index(pco_shader *shader) +{ + assert(!shader->is_grouped); + + shader->next_func = 0; + pco_foreach_func_in_shader (func, shader) { + unsigned *ssa_idx_map = + rzalloc_array_size(NULL, sizeof(*ssa_idx_map), func->next_ssa); + + func->index = shader->next_func++; + func->next_ssa = 0; + func->next_instr = 0; + func->next_block = 0; + + pco_foreach_block_in_func (block, func) { + block->index = func->next_block++; + pco_foreach_instr_in_block (instr, block) { + instr->index = func->next_instr++; + pco_foreach_instr_dest_ssa (pdest, instr) { + ssa_idx_map[pdest->val] = func->next_ssa++; + pdest->val = ssa_idx_map[pdest->val]; + } + } + } + + pco_foreach_instr_in_func (instr, func) { + pco_foreach_instr_src_ssa (psrc, instr) { + psrc->val = ssa_idx_map[psrc->val]; + } + } + + /* TODO: */ + /* pco_foreach_if_in_func */ + /* pco_foreach_loop_in_func */ + + ralloc_free(ssa_idx_map); + } + + return true; +} diff --git a/src/imagination/pco/pco_internal.h b/src/imagination/pco/pco_internal.h index 48fa82e8436..d871c0116b9 100644 --- a/src/imagination/pco/pco_internal.h +++ b/src/imagination/pco/pco_internal.h @@ -379,6 +379,8 @@ pco_instr *pco_instr_create(pco_func *func, unsigned num_srcs); pco_igrp *pco_igrp_create(pco_func *func); +void pco_instr_delete(pco_instr *instr); + /* Cast helpers. */ /* CF nodes. */ @@ -437,6 +439,12 @@ PCO_DEFINE_CAST(pco_cf_node_as_func, for (pco_block *block = pco_func_first_block(func); block != NULL; \ block = pco_next_block(block)) +#define pco_foreach_block_in_func_from(block, from) \ + for (pco_block *block = from; block != NULL; block = pco_next_block(block)) + +#define pco_foreach_block_in_func_from_rev(block, from) \ + for (pco_block *block = from; block != NULL; block = pco_prev_block(block)) + #define pco_foreach_block_in_func_rev(block, func) \ for (pco_block *block = pco_func_last_block(func); block != NULL; \ block = pco_prev_block(block)) @@ -469,6 +477,20 @@ PCO_DEFINE_CAST(pco_cf_node_as_func, pco_foreach_block_in_func (block, func) \ list_for_each_entry (pco_instr, instr, &(block)->instrs, link) +#define pco_foreach_instr_in_func_from(instr, from) \ + assert(!from->parent_func->parent_shader->is_grouped); \ + pco_foreach_block_in_func_from (block, from->parent_block) \ + list_for_each_entry_from (pco_instr, instr, from, &(block)->instrs, link) + +#define pco_foreach_instr_in_func_from_rev(instr, from) \ + assert(!from->parent_func->parent_shader->is_grouped); \ + pco_foreach_block_in_func_from_rev (block, from->parent_block) \ + list_for_each_entry_from_rev (pco_instr, \ + instr, \ + from, \ + &(block)->instrs, \ + link) + #define pco_foreach_instr_in_func_safe(instr, func) \ assert(!func->parent_shader->is_grouped); \ pco_foreach_block_in_func (block, func) \ @@ -1066,8 +1088,16 @@ static inline bool pco_should_print_binary(pco_shader *shader) } /* PCO IR passes. */ +bool pco_const_imms(pco_shader *shader); +bool pco_dce(pco_shader *shader); bool pco_end(pco_shader *shader); bool pco_group_instrs(pco_shader *shader); +bool pco_index(pco_shader *shader); +bool pco_nir_pfo(nir_shader *nir); +bool pco_nir_pvi(nir_shader *nir); +bool pco_opt(pco_shader *shader); +bool pco_ra(pco_shader *shader); +bool pco_schedule(pco_shader *shader); /** * \brief Returns the PCO bits for a bit size. @@ -1393,6 +1423,36 @@ static inline pco_ref pco_ref_ssa(unsigned index, unsigned bits, unsigned chans) }; } +/** + * \brief Builds and returns a new SSA reference. + * + * \param[in,out] func The function. + * \param[in] bits Number of bits. + * \param[in] chans Number of channels. + * \return SSA reference. + */ +static inline pco_ref +pco_ref_new_ssa(pco_func *func, unsigned bits, unsigned chans) +{ + return (pco_ref){ + .val = func->next_ssa++, + .chans = chans - 1, + .bits = pco_bits(bits), + .type = PCO_REF_TYPE_SSA, + }; +} + +/** + * \brief Builds and returns a new 32x1 SSA reference. + * + * \param[in,out] func The function. + * \return SSA reference. + */ +static inline pco_ref pco_ref_new_ssa32(pco_func *func) +{ + return pco_ref_new_ssa(func, 32, 1); +} + /** * \brief Builds and returns a virtual register reference. * @@ -1410,7 +1470,7 @@ static inline pco_ref pco_ref_vreg(unsigned index) } /** - * \brief Builds and returns a hardware register reference. + * \brief Builds and returns a scalar hardware register reference. * * \param[in] index Register index. * \param[in] reg_class Register class. @@ -1430,6 +1490,29 @@ static inline pco_ref pco_ref_hwreg(unsigned index, }; } +/** + * \brief Builds and returns a vector hardware register reference. + * + * \param[in] index Register index. + * \param[in] reg_class Register class. + * \param[in] chans Number of channels. + * \return Hardware register reference. + */ +static inline pco_ref +pco_ref_hwreg_vec(unsigned index, enum pco_reg_class reg_class, unsigned chans) +{ + assert(index < 256); + assert(reg_class != PCO_REG_CLASS_VIRT); + + return (pco_ref){ + .val = index, + .chans = chans - 1, + .bits = PCO_BITS_32, + .type = PCO_REF_TYPE_REG, + .reg_class = reg_class, + }; +} + /** * \brief Builds and returns an immediate reference. * @@ -1549,7 +1632,7 @@ static inline pco_ref pco_ref_pred(enum pco_pred pred) * \param[in] drc Dependent read counter. * \return Dependent read counter reference. */ -static inline pco_ref pco_ref_drc(enum pco_io drc) +static inline pco_ref pco_ref_drc(enum pco_drc drc) { return (pco_ref){ .val = drc, @@ -1585,6 +1668,137 @@ static inline void pco_ref_xfer_mods(pco_ref *dest, pco_ref *source, bool reset) } } +/** + * \brief Updates a reference to set the oneminus modifier. + * + * \param[in] ref Base reference. + * \return Updated reference. + */ +static inline pco_ref pco_ref_oneminus(pco_ref ref) +{ + ref.oneminus = true; + return ref; +} + +/** + * \brief Updates a reference to set the clamp modifier. + * + * \param[in] ref Base reference. + * \return Updated reference. + */ +static inline pco_ref pco_ref_clamp(pco_ref ref) +{ + ref.clamp = true; + return ref; +} + +/** + * \brief Updates a reference to set the floor modifier. + * + * \param[in] ref Base reference. + * \return Updated reference. + */ +static inline pco_ref pco_ref_flr(pco_ref ref) +{ + ref.flr = true; + ref.abs = false; + ref.neg = false; + return ref; +} + +/** + * \brief Updates a reference to set the abs modifier. + * + * \param[in] ref Base reference. + * \return Updated reference. + */ +static inline pco_ref pco_ref_abs(pco_ref ref) +{ + ref.abs = true; + ref.neg = false; + return ref; +} + +/** + * \brief Updates a reference to set the negate modifier. + * + * \param[in] ref Base reference. + * \return Updated reference. + */ +static inline pco_ref pco_ref_neg(pco_ref ref) +{ + ref.neg = !ref.neg; + return ref; +} + +/** + * \brief Updates a reference to set the element modifier. + * + * \param[in] ref Base reference. + * \param[in] elem New element modifier. + * \return Updated reference. + */ +static inline pco_ref pco_ref_elem(pco_ref ref, enum pco_elem elem) +{ + ref.elem = elem; + return ref; +} + +/** + * \brief Checks whether two reference modifiers are the same. + * + * \param[in] ref0 First reference. + * \param[in] ref1 Second reference. + * \return True if both reference modifiers are the same. + */ +static inline bool pco_ref_mods_are_equal(pco_ref ref0, pco_ref ref1) +{ + return (ref0.oneminus == ref1.oneminus) && (ref0.clamp == ref1.clamp) && + (ref0.flr == ref1.flr) && (ref0.abs == ref1.abs) && + (ref0.neg == ref1.neg) && (ref0.elem == ref1.elem); +} + +/** + * \brief Checks whether two references are the same. + * + * \param[in] ref0 First reference. + * \param[in] ref1 Second reference. + * \return True if both references are the same. + */ +/* TODO: can this be simplified? */ +static inline bool pco_refs_are_equal(pco_ref ref0, pco_ref ref1) +{ + if (ref0.type != ref1.type) + return false; + + if (pco_ref_is_idx_reg(ref0)) { + if ((ref0.idx_reg.num != ref1.idx_reg.num) || + (ref0.idx_reg.offset != ref1.idx_reg.offset)) { + return false; + } + } else if (ref0.val != ref1.val) { + return false; + } + + if (pco_ref_is_idx_reg(ref0) || pco_ref_is_reg(ref0)) + if (ref0.reg_class != ref1.reg_class) + return false; + + if (!pco_ref_mods_are_equal(ref0, ref1)) + return false; + + if (ref0.chans != ref1.chans) + return false; + + if (pco_ref_get_dtype(ref0) != pco_ref_get_dtype(ref1)) + return false; + + if (pco_ref_get_bits(ref0) != pco_ref_get_bits(ref1)) + return false; + + return true; +} + /** * \brief Returns whether none of the lower/upper sources in an instruction * group are set. @@ -1635,4 +1849,19 @@ static inline bool pco_igrp_dests_unset(pco_igrp *igrp) return true; } + +/* Common hw constants. */ + +/** Integer/float zero. */ +#define pco_zero pco_ref_hwreg(0, PCO_REG_CLASS_CONST) + +/** Integer one. */ +#define pco_one pco_ref_hwreg(1, PCO_REG_CLASS_CONST) + +/** Integer -1/true/0xffffffff. */ +#define pco_true pco_ref_hwreg(143, PCO_REG_CLASS_CONST) + +/** Float 1. */ +#define pco_fone pco_ref_hwreg(64, PCO_REG_CLASS_CONST) + #endif /* PCO_INTERNAL_H */ diff --git a/src/imagination/pco/pco_ir.c b/src/imagination/pco/pco_ir.c index 303a2579dc0..346626af162 100644 --- a/src/imagination/pco/pco_ir.c +++ b/src/imagination/pco/pco_ir.c @@ -50,6 +50,11 @@ void pco_process_ir(pco_ctx *ctx, pco_shader *shader) { pco_validate_shader(shader, "before passes"); + PCO_PASS(_, shader, pco_const_imms); + PCO_PASS(_, shader, pco_opt); + PCO_PASS(_, shader, pco_dce); + PCO_PASS(_, shader, pco_schedule); + PCO_PASS(_, shader, pco_ra); PCO_PASS(_, shader, pco_end); PCO_PASS(_, shader, pco_group_instrs); @@ -57,6 +62,4 @@ void pco_process_ir(pco_ctx *ctx, pco_shader *shader) if (pco_should_print_shader(shader)) pco_print_shader(shader, stdout, "after passes"); - - puts("finishme: pco_process_ir"); } diff --git a/src/imagination/pco/pco_nir.c b/src/imagination/pco/pco_nir.c index e7350f22b1d..59e4fe34f9b 100644 --- a/src/imagination/pco/pco_nir.c +++ b/src/imagination/pco/pco_nir.c @@ -21,7 +21,13 @@ static const struct spirv_to_nir_options pco_base_spirv_options = { }; /** Base/common NIR options. */ -static const nir_shader_compiler_options pco_base_nir_options = {}; +static const nir_shader_compiler_options pco_base_nir_options = { + .fuse_ffma32 = true, + + .lower_fquantize2f16 = true, + .lower_layer_fs_input_to_sysval = true, + .compact_arrays = true, +}; /** * \brief Sets up device/core-specific SPIR-V to NIR options. @@ -64,19 +70,76 @@ void pco_preprocess_nir(pco_ctx *ctx, nir_shader *nir) if (nir->info.internal) NIR_PASS(_, nir, nir_lower_returns); + NIR_PASS(_, nir, nir_lower_global_vars_to_local); + NIR_PASS(_, nir, nir_lower_vars_to_ssa); + NIR_PASS(_, nir, nir_split_var_copies); + NIR_PASS(_, nir, nir_lower_var_copies); + NIR_PASS(_, nir, nir_split_per_member_structs); + NIR_PASS(_, + nir, + nir_split_struct_vars, + nir_var_function_temp | nir_var_shader_temp); + NIR_PASS(_, + nir, + nir_split_array_vars, + nir_var_function_temp | nir_var_shader_temp); + NIR_PASS(_, + nir, + nir_lower_indirect_derefs, + nir_var_shader_in | nir_var_shader_out, + UINT32_MAX); + + NIR_PASS(_, + nir, + nir_remove_dead_variables, + nir_var_function_temp | nir_var_shader_temp, + NULL); + NIR_PASS(_, nir, nir_opt_dce); + if (pco_should_print_nir(nir)) { puts("after pco_preprocess_nir:"); nir_print_shader(nir, stdout); } - - puts("finishme: pco_preprocess_nir"); } +/** + * \brief Returns the GLSL type size. + * + * \param[in] type Type. + * \param[in] bindless Whether the access is bindless. + * \return The size. + */ static int glsl_type_size(const struct glsl_type *type, UNUSED bool bindless) { return glsl_count_attribute_slots(type, false); } +/** + * \brief Returns the vectorization with for a given instruction. + * + * \param[in] instr Instruction. + * \param[in] data User data. + * \return The vectorization width. + */ +static uint8_t vectorize_filter(const nir_instr *instr, UNUSED const void *data) +{ + if (instr->type == nir_instr_type_load_const) + return 1; + + if (instr->type != nir_instr_type_alu) + return 0; + + /* TODO */ + nir_alu_instr *alu = nir_instr_as_alu(instr); + switch (alu->op) { + default: + break; + } + + /* Basic for now. */ + return 2; +} + /** * \brief Lowers a NIR shader. * @@ -99,12 +162,61 @@ void pco_lower_nir(pco_ctx *ctx, nir_shader *nir) nir_io_add_const_offset_to_base, nir_var_shader_in | nir_var_shader_out); + if (nir->info.stage == MESA_SHADER_FRAGMENT) { + NIR_PASS(_, nir, pco_nir_pfo); + } else if (nir->info.stage == MESA_SHADER_VERTEX) { + NIR_PASS(_, nir, pco_nir_pvi); + } + + /* TODO: this should happen in the linking stage to cull unused I/O. */ + NIR_PASS(_, + nir, + nir_lower_io_to_scalar, + nir_var_shader_in | nir_var_shader_out, + NULL, + NULL); + + NIR_PASS(_, nir, nir_lower_vars_to_ssa); + NIR_PASS(_, nir, nir_opt_copy_prop_vars); + NIR_PASS(_, nir, nir_opt_dead_write_vars); + NIR_PASS(_, nir, nir_opt_combine_stores, nir_var_all); + + bool progress; + NIR_PASS(_, nir, nir_lower_alu); + NIR_PASS(_, nir, nir_lower_pack); + NIR_PASS(_, nir, nir_opt_algebraic); + do { + progress = false; + + NIR_PASS(progress, nir, nir_opt_algebraic_late); + NIR_PASS(_, nir, nir_opt_constant_folding); + NIR_PASS(_, nir, nir_lower_load_const_to_scalar); + NIR_PASS(_, nir, nir_copy_prop); + NIR_PASS(_, nir, nir_opt_dce); + NIR_PASS(_, nir, nir_opt_cse); + } while (progress); + + NIR_PASS(_, + nir, + nir_opt_vectorize_io, + nir_var_shader_in | nir_var_shader_out); + + NIR_PASS(_, nir, nir_lower_alu_to_scalar, NULL, NULL); + + do { + progress = false; + + NIR_PASS(progress, nir, nir_copy_prop); + NIR_PASS(progress, nir, nir_opt_dce); + NIR_PASS(progress, nir, nir_opt_cse); + NIR_PASS(progress, nir, nir_opt_constant_folding); + NIR_PASS(progress, nir, nir_opt_undef); + } while (progress); + if (pco_should_print_nir(nir)) { puts("after pco_lower_nir:"); nir_print_shader(nir, stdout); } - - puts("finishme: pco_lower_nir"); } /** @@ -115,11 +227,7 @@ void pco_lower_nir(pco_ctx *ctx, nir_shader *nir) */ void pco_postprocess_nir(pco_ctx *ctx, nir_shader *nir) { - NIR_PASS(_, nir, nir_opt_constant_folding); - NIR_PASS(_, nir, nir_opt_copy_prop_vars); - NIR_PASS(_, nir, nir_copy_prop); - NIR_PASS(_, nir, nir_opt_dce); - NIR_PASS(_, nir, nir_opt_cse); + NIR_PASS(_, nir, nir_move_vec_src_uses_to_dest, false); /* Re-index everything. */ nir_foreach_function_with_impl (_, impl, nir) { @@ -134,8 +242,6 @@ void pco_postprocess_nir(pco_ctx *ctx, nir_shader *nir) puts("after pco_postprocess_nir:"); nir_print_shader(nir, stdout); } - - puts("finishme: pco_postprocess_nir"); } /** diff --git a/src/imagination/pco/pco_nir_pvfio.c b/src/imagination/pco/pco_nir_pvfio.c new file mode 100644 index 00000000000..7daca7d10c3 --- /dev/null +++ b/src/imagination/pco/pco_nir_pvfio.c @@ -0,0 +1,163 @@ +/* + * Copyright © 2024 Imagination Technologies Ltd. + * + * SPDX-License-Identifier: MIT + */ + +/** + * \file pco_nir_pvfio.c + * + * \brief PCO NIR per-vertex/fragment input/output passes. + */ + +#include "nir.h" +#include "nir_builder.h" +#include "pco.h" +#include "pco_builder.h" +#include "pco_internal.h" +#include "util/macros.h" +#include "util/u_dynarray.h" + +#include +#include +#include + +/** Per-fragment output pass state. */ +struct pfo_state { + struct util_dynarray stores; /** List of fragment stores. */ +}; + +/** + * \brief Returns a NIR intrinsic instruction if a NIR instruction matches the + * provided intrinsic op. + * + * \param[in] instr NIR instruction. + * \param[in] op Desired intrinsic op. + * \return The intrinsic instruction, else NULL. + */ +static inline nir_intrinsic_instr *is_intr(nir_instr *instr, + nir_intrinsic_op op) +{ + nir_intrinsic_instr *intr = NULL; + + if (instr->type != nir_instr_type_intrinsic) + return NULL; + + intr = nir_instr_as_intrinsic(instr); + + if (intr->intrinsic != op) + return NULL; + + return intr; +} + +/** + * \brief Lowers a PFO-related instruction. + * + * \param[in] b NIR builder. + * \param[in] instr NIR instruction. + * \param[in] cb_data User callback data. + * \return True if the instruction was lowered. + */ +static bool lower_pfo(nir_builder *b, nir_instr *instr, void *cb_data) +{ + struct pfo_state *state = cb_data; + + /* TODO NEXT: move into separate function (pack_to_pbe), + * and use data from driver to actually figure out format stuff! + */ + nir_intrinsic_instr *intr; + if ((intr = is_intr(instr, nir_intrinsic_store_output))) { + /* Skip stores we've already processed. */ + util_dynarray_foreach (&state->stores, nir_intrinsic_instr *, store) { + if (intr == *store) + return false; + } + + nir_src *value = &intr->src[0]; + nir_src *offset = &intr->src[1]; + + /* TODO: more accurate way of detecting this */ + /* Already in expected format. */ + if (b->shader->info.internal && nir_src_num_components(*value) == 1) { + util_dynarray_append(&state->stores, nir_intrinsic_instr *, intr); + return false; + } + + assert(nir_src_as_uint(*offset) == 0); + + assert(nir_src_num_components(*value) == 4); + assert(nir_src_bit_size(*value) == 32); + + /* Update the type of the stored variable. */ + nir_variable *var = nir_find_variable_with_location( + b->shader, + nir_var_shader_out, + nir_intrinsic_io_semantics(intr).location); + + var->type = glsl_uint_type(); + + b->cursor = nir_after_block( + nir_impl_last_block(nir_shader_get_entrypoint(b->shader))); + + /* Emit and track the new store. */ + /* TODO NEXT: base is calculated to be the register offset. */ + nir_intrinsic_instr *store = + nir_store_output(b, + nir_pack_unorm_4x8(b, value->ssa), + offset->ssa, + .base = nir_intrinsic_base(intr), + .write_mask = 1, + .component = 0, + .src_type = nir_type_uint32, + .io_semantics = nir_intrinsic_io_semantics(intr), + .io_xfb = nir_intrinsic_io_xfb(intr), + .io_xfb2 = nir_intrinsic_io_xfb2(intr)); + util_dynarray_append(&state->stores, nir_intrinsic_instr *, store); + + /* Remove the old store. */ + b->cursor = nir_instr_remove(instr); + + return true; + } + + return false; +} + +/** + * \brief Per-fragment output pass. + * + * \param[in,out] nir NIR shader. + * \return True if the pass made progress. + */ +bool pco_nir_pfo(nir_shader *nir) +{ + assert(nir->info.stage == MESA_SHADER_FRAGMENT); + + struct pfo_state state = {}; + util_dynarray_init(&state.stores, NULL); + + bool progress = + nir_shader_instructions_pass(nir, lower_pfo, nir_metadata_none, &state); + + util_dynarray_fini(&state.stores); + + return progress; +} + +/** + * \brief Per-vertex input pass. + * + * \param[in,out] nir NIR shader. + * \return True if the pass made progress. + */ +bool pco_nir_pvi(nir_shader *nir) +{ + assert(nir->info.stage == MESA_SHADER_VERTEX); + + puts("finishme: pco_nir_pvi"); + + /* TODO: format conversion and inserting unspecified/missing components. */ + + return false; +} diff --git a/src/imagination/pco/pco_ops.py b/src/imagination/pco/pco_ops.py index b1fe2dc56ec..368e8d57e21 100644 --- a/src/imagination/pco/pco_ops.py +++ b/src/imagination/pco/pco_ops.py @@ -275,6 +275,8 @@ O_FMAD = hw_op('fmad', OM_ALU + [OM_SAT, OM_LP], 1, 3, [], [[RM_ABS, RM_NEG], [R O_MBYP0 = hw_op('mbyp0', OM_ALU, 1, 1, [], [[RM_ABS, RM_NEG]]) O_PCK = hw_op('pck', OM_ALU + [OM_PCK_FMT, OM_ROUNDZERO, OM_SCALE], 1, 1) +# TODO +# O_PCK_ELEM = pseudo_op('pck.elem', OM_ALU_RPT1 + [OM_PCK_FMT, OM_ROUNDZERO, OM_SCALE], 1, 2) ## Backend. O_UVSW_WRITE = hw_op('uvsw.write', [OM_EXEC_CND, OM_RPT], 0, 2) diff --git a/src/imagination/pco/pco_opt.c b/src/imagination/pco/pco_opt.c new file mode 100644 index 00000000000..4756792a4ea --- /dev/null +++ b/src/imagination/pco/pco_opt.c @@ -0,0 +1,355 @@ +/* + * Copyright © 2024 Imagination Technologies Ltd. + * + * SPDX-License-Identifier: MIT + */ + +/** + * \file pco_opt.c + * + * \brief PCO optimization passes. + */ + +#include "pco.h" +#include "pco_builder.h" +#include "util/bitscan.h" +#include "util/bitset.h" +#include "util/macros.h" +#include "util/ralloc.h" + +#include +#include + +/** Source use. */ +struct pco_use { + pco_instr *instr; + pco_ref *psrc; +}; + +/** + * \brief Checks if an instruction can be back-propagated. + * + * \param[in] to Instruction being back-propagated to. + * \param[in] from Instruction being back-propagated from. + * \return True if from can be back-propagated. + */ +static inline bool can_back_prop_instr(const pco_instr *to, + const pco_instr *from) +{ + const struct pco_op_info *info = &pco_op_info[from->op]; + + /* Ensure any op mods set in from can also be set in to. */ + u_foreach_bit64 (mod, info->mods) { + if (pco_instr_has_mod(from, mod) && pco_instr_mod_is_set(from, mod) && + !pco_instr_has_mod(to, mod)) + return false; + } + + return true; +} + +/** + * \brief Transfers any op mods that have been set. + * + * \param[in] to Instruction receiving the op mods. + * \param[in] from Instruction providing the op mods. + */ +static inline void xfer_set_op_mods(pco_instr *to, const pco_instr *from) +{ + const struct pco_op_info *info = &pco_op_info[from->op]; + + /* Transfer set op mods. */ + u_foreach_bit64 (mod, info->mods) { + if (pco_instr_has_mod(from, mod) && pco_instr_mod_is_set(from, mod)) { + assert(pco_instr_has_mod(to, mod)); + pco_instr_set_mod(to, mod, pco_instr_get_mod(from, mod)); + } + } +} + +/** + * \brief Tries to back-propagate an instruction. + * + * \param[in] uses Global list of uses. + * \param[in] instr Instruction to try and back-propagate. + * \return True if back-propagation was successful. + */ +static inline bool try_back_prop_instr(struct pco_use *uses, pco_instr *instr) +{ + pco_ref *pdest_to = &instr->dest[0]; + if (instr->num_dests != 1 || !pco_ref_is_ssa(*pdest_to)) + return false; + + struct pco_use *use = &uses[pdest_to->val]; + if (!use->instr) + return false; + + /* TODO: allow propagating instructions which can have their dest/op + * modifiers set to perform the same operations as use source modifiers. + * + * Make sure to check in can_back_prop_instr when implementing this. + * We're fine for now since mov has no settable dest mods. + */ + if (use->instr->op != PCO_OP_MOV || pco_ref_has_mods_set(*use->psrc)) + return false; + + if (!can_back_prop_instr(instr, use->instr)) + return false; + + pco_ref *pdest_from = &use->instr->dest[0]; + + assert(pco_ref_get_bits(*pdest_from) == pco_ref_get_bits(*pdest_to)); + assert(pco_ref_get_chans(*pdest_from) == pco_ref_get_chans(*pdest_to)); + assert(!pco_ref_has_mods_set(*pdest_from) && + !pco_ref_has_mods_set(*pdest_to)); + + /* Propagate the destination and the set op mods. */ + /* TODO: types? */ + *pdest_to = *pdest_from; + xfer_set_op_mods(instr, use->instr); + pco_instr_delete(use->instr); + + return true; +} + +/** + * \brief Instruction back-propagation pass. + * + * \param[in,out] shader PCO shader. + * \return True if any back-propagations were performed. + */ +static inline bool back_prop(pco_shader *shader) +{ + bool progress = false; + struct pco_use *uses; + BITSET_WORD *multi_uses; + + pco_foreach_func_in_shader_rev (func, shader) { + uses = rzalloc_array_size(NULL, sizeof(*uses), func->next_ssa); + multi_uses = rzalloc_array_size(uses, + sizeof(*multi_uses), + BITSET_WORDS(func->next_ssa)); + + pco_foreach_instr_in_func_safe_rev (instr, func) { + pco_foreach_instr_src_ssa (psrc, instr) { + if (BITSET_TEST(multi_uses, psrc->val) || uses[psrc->val].instr) { + BITSET_SET(multi_uses, psrc->val); + uses[psrc->val].instr = NULL; + continue; + } + + uses[psrc->val] = (struct pco_use){ + .instr = instr, + .psrc = psrc, + }; + } + + progress |= try_back_prop_instr(uses, instr); + } + + ralloc_free(uses); + } + + return progress; +} + +/** + * \brief Checks if a source can be forward-propagated. + * + * \param[in] to_instr Instruction being forward-propagated to. + * \param[in] to Source being forward-propagated to. + * \param[in] from Source being forward-propagated from. + * \return True if from can be forward-propagated. + */ +static inline bool can_fwd_prop_src(const pco_instr *to_instr, + const pco_ref *to, + const pco_ref *from) +{ + /* Check sizes. */ + if (pco_ref_get_bits(*from) != pco_ref_get_bits(*to)) + return false; + + if (pco_ref_get_chans(*from) != pco_ref_get_chans(*to)) + return false; + + /* See if the modifiers can be propagated. */ + unsigned to_src_index = to - to_instr->src; + if (pco_ref_has_mods_set(*from)) { + if (from->oneminus && !pco_instr_src_has_oneminus(to_instr, to_src_index)) + return false; + if (from->clamp && !pco_instr_src_has_clamp(to_instr, to_src_index)) + return false; + if (from->flr && !pco_instr_src_has_flr(to_instr, to_src_index)) + return false; + if (from->abs && !pco_instr_src_has_abs(to_instr, to_src_index)) + return false; + if (from->neg && !pco_instr_src_has_neg(to_instr, to_src_index)) + return false; + if (from->elem && !pco_instr_src_has_elem(to_instr, to_src_index)) + return false; + } + + /* TODO: Also need to consider whether the source can be represented in the + * propagated instruction. + * Or, a legalize pass to insert movs; probably better since + * feature/arch-agnostic. + */ + + return true; +} + +/** + * \brief Tries to forward-propagate an instruction. + * + * \param[in] writes Global list of writes. + * \param[in] instr Instruction to try and forward-propagate. + * \return True if forward-propagation was successful. + */ +static inline bool try_fwd_prop_instr(pco_instr **writes, pco_instr *instr) +{ + bool progress = false; + + pco_foreach_instr_src_ssa (psrc, instr) { + pco_instr *parent_instr = writes[psrc->val]; + + if (!parent_instr || parent_instr->op != PCO_OP_MOV) + continue; + + if (!can_fwd_prop_src(instr, psrc, &parent_instr->src[0])) + continue; + + /* Propagate the source. */ + pco_ref repl = parent_instr->src[0]; + if (psrc->flr) + repl = pco_ref_flr(repl); + else if (psrc->abs) + repl = pco_ref_abs(repl); + + repl.neg ^= psrc->neg; + + /* TODO: types? */ + *psrc = repl; + + progress = true; + } + + return progress; +} + +/** + * \brief Instruction forward-propagation pass. + * + * \param[in,out] shader PCO shader. + * \return True if any forward-propagations were performed. + */ +static inline bool fwd_prop(pco_shader *shader) +{ + bool progress = false; + pco_instr **writes; + + pco_foreach_func_in_shader (func, shader) { + writes = rzalloc_array_size(NULL, sizeof(*writes), func->next_ssa); + + pco_foreach_instr_in_func (instr, func) { + pco_foreach_instr_dest_ssa (pdest, instr) { + writes[pdest->val] = instr; + } + + progress |= try_fwd_prop_instr(writes, instr); + } + + ralloc_free(writes); + } + + return progress; +} + +/** + * \brief Performs shader optimizations. + * + * \param[in,out] shader PCO shader. + * \return True if the pass made progress. + */ +bool pco_opt(pco_shader *shader) +{ + bool progress = false; + + pco_index(shader); + + progress |= back_prop(shader); + progress |= fwd_prop(shader); + + return progress; +} + +/** + * \brief Checks whether an instruction has side-effects. + * + * \param[in] instr Instruction to check. + * \return True if the instruction has side-effects. + */ +static inline bool instr_has_side_effects(pco_instr *instr) +{ + /* Atomic instructions. */ + if (pco_instr_has_atom(instr) && pco_instr_get_atom(instr)) + return true; + + /* TODO: + * - gradient + * - conditional + * - sample writes (+ set the destination pointer to point to the write data) + * - others + */ + return false; +} + +/** + * \brief Performs DCE. + * + * \param[in,out] shader PCO shader. + * \return True if the pass made progress. + */ +bool pco_dce(pco_shader *shader) +{ + bool progress = false; + BITSET_WORD *ssa_used; + + pco_index(shader); + pco_foreach_func_in_shader (func, shader) { + ssa_used = rzalloc_array_size(NULL, + sizeof(*ssa_used), + BITSET_WORDS(func->next_ssa)); + + /* Collect used SSA sources. */ + pco_foreach_instr_in_func (instr, func) { + pco_foreach_instr_src_ssa (psrc, instr) { + BITSET_SET(ssa_used, psrc->val); + } + } + + /* Remove instructions with unused SSA destinations (if they also have no + * side-effects). + */ + pco_foreach_instr_in_func_safe (instr, func) { + bool has_ssa_dests = false; + bool dests_used = false; + + pco_foreach_instr_dest_ssa (pdest, instr) { + has_ssa_dests = true; + dests_used |= BITSET_TEST(ssa_used, pdest->val); + } + + if (has_ssa_dests && !dests_used && !instr_has_side_effects(instr)) { + pco_instr_delete(instr); + progress = true; + } + } + + ralloc_free(ssa_used); + } + + if (progress) + pco_index(shader); + + return progress; +} diff --git a/src/imagination/pco/pco_ra.c b/src/imagination/pco/pco_ra.c new file mode 100644 index 00000000000..7a510136642 --- /dev/null +++ b/src/imagination/pco/pco_ra.c @@ -0,0 +1,309 @@ +/* + * Copyright © 2024 Imagination Technologies Ltd. + * + * SPDX-License-Identifier: MIT + */ + +/** + * \file pco_ra.c + * + * \brief PCO register allocator. + */ + +#include "hwdef/rogue_hw_utils.h" +#include "pco.h" +#include "pco_builder.h" +#include "pco_internal.h" +#include "util/bitset.h" +#include "util/hash_table.h" +#include "util/macros.h" +#include "util/register_allocate.h" +#include "util/sparse_array.h" +#include "util/u_dynarray.h" + +#include +#include +#include + +/** Live range of an SSA variable. */ +struct live_range { + unsigned start; + unsigned end; +}; + +/** Vector user. */ +struct vec_user { + pco_instr *instr; + unsigned src; + pco_instr *vec; +}; + +/** + * \brief Performs register allocation on a function. + * + * \param[in,out] func PCO shader. + * \param[in] allocable_temps Number of allocatable temp registers. + * \param[in] allocable_vtxins Number of allocatable vertex input registers. + * \param[in] allocable_interns Number of allocatable internal registers. + * \return True if registers were allocated. + */ +static bool pco_ra_func(pco_func *func, + unsigned allocable_temps, + unsigned allocable_vtxins, + unsigned allocable_interns) +{ + /* TODO: support multiple functions and calls. */ + assert(func->type == PCO_FUNC_TYPE_ENTRYPOINT); + + /* No registers to allocate. */ + if (!func->next_ssa) + return false; + + /* TODO: loop lifetime extension. + * TODO: track successors/predecessors. + */ + + /* Collect used bit sizes. */ + uint8_t ssa_bits = 0; + pco_foreach_instr_in_func (instr, func) { + pco_foreach_instr_dest_ssa (pdest, instr) { + ssa_bits |= (1 << pdest->bits); + } + } + + /* 64-bit SSA should've been lowered by now. */ + assert(!(ssa_bits & (1 << PCO_BITS_64))); + + /* TODO: support multiple bit sizes. */ + bool only_32bit = ssa_bits == (1 << PCO_BITS_32); + assert(only_32bit); + + struct ra_regs *ra_regs = + ra_alloc_reg_set(func, allocable_temps, !only_32bit); + + /* Allocate classes. */ + struct hash_table_u64 *ra_classes = _mesa_hash_table_u64_create(ra_regs); + pco_foreach_instr_in_func (instr, func) { + pco_foreach_instr_dest_ssa (pdest, instr) { + unsigned chans = pco_ref_get_chans(*pdest); + if (_mesa_hash_table_u64_search(ra_classes, chans)) + continue; + + struct ra_class *ra_class = ra_alloc_contig_reg_class(ra_regs, chans); + _mesa_hash_table_u64_insert(ra_classes, chans, ra_class); + } + } + + /* Assign registers to classes. */ + hash_table_u64_foreach (ra_classes, entry) { + const unsigned stride = entry.key; + struct ra_class *ra_class = entry.data; + + for (unsigned t = 0; t < allocable_temps - (stride - 1); ++t) + ra_class_add_reg(ra_class, t); + } + + ra_set_finalize(ra_regs, NULL); + + struct ra_graph *ra_graph = + ra_alloc_interference_graph(ra_regs, func->next_ssa); + ralloc_steal(ra_regs, ra_graph); + + /* Allocate and calculate live ranges. */ + struct live_range *live_ranges = + rzalloc_array_size(ra_regs, sizeof(*live_ranges), func->next_ssa); + + for (unsigned u = 0; u < func->next_ssa; ++u) { + live_ranges[u].start = ~0U; + } + + pco_foreach_instr_in_func (instr, func) { + pco_foreach_instr_dest_ssa (pdest, instr) { + assert(live_ranges[pdest->val].start == ~0U); + live_ranges[pdest->val].start = instr->index; + + unsigned chans = pco_ref_get_chans(*pdest); + struct ra_class *ra_class = + _mesa_hash_table_u64_search(ra_classes, chans); + assert(ra_class); + + ra_set_node_class(ra_graph, pdest->val, ra_class); + } + + pco_foreach_instr_src_ssa (psrc, instr) { + live_ranges[psrc->val].end = + MAX2(live_ranges[psrc->val].end, instr->index); + } + } + + /* Build interference graph from overlapping live ranges. */ + for (unsigned ssa0 = 0; ssa0 < func->next_ssa; ++ssa0) { + for (unsigned ssa1 = ssa0 + 1; ssa1 < func->next_ssa; ++ssa1) { + /* If the live ranges overlap, the register nodes interfere. */ + if (!(live_ranges[ssa0].start >= live_ranges[ssa1].end || + live_ranges[ssa1].start >= live_ranges[ssa0].end)) { + ra_add_node_interference(ra_graph, ssa0, ssa1); + } + } + } + + bool allocated = ra_allocate(ra_graph); + assert(allocated); + /* TODO: spilling. */ + + /* Collect info on users of vec ops. */ + struct util_dynarray vec_users; + struct util_dynarray vecs; + BITSET_WORD *instrs_using_vecs = + rzalloc_array_size(ra_regs, + sizeof(*instrs_using_vecs), + BITSET_WORDS(func->next_instr)); + BITSET_WORD *instrs_using_multi_vecs = + rzalloc_array_size(ra_regs, + sizeof(*instrs_using_multi_vecs), + BITSET_WORDS(func->next_instr)); + + util_dynarray_init(&vec_users, ra_regs); + util_dynarray_init(&vecs, ra_regs); + + pco_foreach_instr_in_func (vec, func) { + if (vec->op != PCO_OP_VEC) + continue; + + util_dynarray_append(&vecs, pco_instr *, vec); + + const pco_ref vec_dest = vec->dest[0]; + assert(pco_ref_is_ssa(vec_dest)); + + pco_foreach_instr_in_func_from (instr, vec) { + pco_foreach_instr_src_ssa (psrc, instr) { + if (psrc->val != vec_dest.val) + continue; + + /* TODO: for now we're just supporting instructions producing + * scalars (or with no outputs). + * */ + assert(!instr->num_dests || + (instr->num_dests == 1 && + pco_ref_get_chans(instr->dest[0]) == 1)); + + if (BITSET_TEST(instrs_using_vecs, instr->index)) + BITSET_SET(instrs_using_multi_vecs, instr->index); + + BITSET_SET(instrs_using_vecs, instr->index); + + struct vec_user vec_user = { + .instr = instr, + .src = psrc - instr->src, + .vec = vec, + }; + util_dynarray_append(&vec_users, struct vec_user, vec_user); + } + } + } + + /* TODO: support this. */ + assert(__bitset_is_empty(instrs_using_multi_vecs, + BITSET_WORDS(func->next_instr))); + + /* Replace SSA regs with allocated temps. */ + unsigned temps = 0; + pco_foreach_instr_in_func_safe (instr, func) { + pco_foreach_instr_dest_ssa (pdest, instr) { + pdest->type = PCO_REF_TYPE_REG; + pdest->reg_class = PCO_REG_CLASS_TEMP; + pdest->val = ra_get_node_reg(ra_graph, pdest->val); + temps = MAX2(temps, pdest->val + pco_ref_get_chans(*pdest)); + } + + pco_foreach_instr_src_ssa (psrc, instr) { + psrc->type = PCO_REF_TYPE_REG; + psrc->reg_class = PCO_REG_CLASS_TEMP; + psrc->val = ra_get_node_reg(ra_graph, psrc->val); + } + } + + /* Scalarize the users of any vec ops that haven't been consumed in + * other passes; no point wasting regs with copies unless it's unavoidable. + */ + /* TODO: distinguish between and support instructions that need vecs/can't be + * scalarized, e.g. sample data words. + */ + /* TODO: try and do this in a separate earlier pass, taking into account the + * cost/benefit analysis of scalarizing. + */ + util_dynarray_foreach (&vec_users, struct vec_user, vec_user) { + pco_instr *instr = vec_user->instr; + pco_instr *vec = vec_user->vec; + + switch (instr->op) { + case PCO_OP_UVSW_WRITE: { + assert(vec_user->src == 0); + assert(pco_instr_get_rpt(instr) == vec->num_srcs); + + pco_builder b = + pco_builder_create(func, pco_cursor_after_instr(instr)); + uint8_t vtxout_base_addr = pco_ref_get_imm(instr->src[1]); + + for (unsigned s = 0; s < vec->num_srcs; ++s) + pco_uvsw_write(&b, vec->src[s], pco_ref_val8(vtxout_base_addr + s)); + + break; + } + + default: + unreachable(); + } + + pco_instr_delete(instr); + } + + /* TODO: process/fold comp ops as well? */ + + /* Drop vec ops. */ + util_dynarray_foreach (&vecs, pco_instr *, vec) { + pco_instr_delete(*vec); + } + + ralloc_free(ra_regs); + + func->temps = temps; + + return true; +} + +/** + * \brief Register allocation pass. + * + * \param[in,out] shader PCO shader. + * \return True if the pass made progress. + */ +bool pco_ra(pco_shader *shader) +{ + assert(!shader->is_grouped); + + pco_index(shader); + + unsigned hw_temps = rogue_get_temps(shader->ctx->dev_info); + /* TODO: + * unsigned opt_temps = rogue_get_optimal_temps(shader->ctx->dev_info); + */ + + /* TODO: different number of temps available if preamble/phase change. */ + /* TODO: different number of temps available if barriers are in use. */ + /* TODO: support for internal and vtxin registers. */ + unsigned allocable_temps = hw_temps; + unsigned allocable_vtxins = 0; + unsigned allocable_interns = 0; + + /* Perform register allocation for each function. */ + bool progress = false; + pco_foreach_func_in_shader (func, shader) { + progress |= pco_ra_func(func, + allocable_temps, + allocable_vtxins, + allocable_interns); + } + + return progress; +} diff --git a/src/imagination/pco/pco_schedule.c b/src/imagination/pco/pco_schedule.c new file mode 100644 index 00000000000..6f592503077 --- /dev/null +++ b/src/imagination/pco/pco_schedule.c @@ -0,0 +1,50 @@ +/* + * Copyright © 2024 Imagination Technologies Ltd. + * + * SPDX-License-Identifier: MIT + */ + +/** + * \file pco_schedule.c + * + * \brief PCO instruction scheduling pass. + */ + +#include "pco.h" +#include "pco_builder.h" +#include "pco_internal.h" +#include "util/macros.h" + +#include + +/** + * \brief Schedules instructions and inserts waits. + * + * \param[in,out] shader PCO shader. + * \return True if the pass made progress. + */ +bool pco_schedule(pco_shader *shader) +{ + bool progress = false; + pco_builder b; + + pco_foreach_func_in_shader (func, shader) { + pco_foreach_instr_in_func_safe (instr, func) { + pco_foreach_instr_src (psrc, instr) { + if (!pco_ref_is_drc(*psrc)) + continue; + + b = pco_builder_create(func, pco_cursor_after_instr(instr)); + pco_wdf(&b, *psrc); + + progress = true; + break; + } + } + } + + if (progress) + pco_index(shader); + + return progress; +} diff --git a/src/imagination/pco/pco_trans_nir.c b/src/imagination/pco/pco_trans_nir.c index 4eabfeabb4e..9a3bc7d90d1 100644 --- a/src/imagination/pco/pco_trans_nir.c +++ b/src/imagination/pco/pco_trans_nir.c @@ -11,6 +11,7 @@ */ #include "compiler/glsl/list.h" +#include "compiler/shader_enums.h" #include "pco.h" #include "pco_builder.h" #include "pco_internal.h" @@ -28,6 +29,7 @@ typedef struct _trans_ctx { pco_shader *shader; /** Current shader. */ pco_func *func; /** Current function. */ pco_builder b; /** Builder. */ + gl_shader_stage stage; /** Shader stage. */ BITSET_WORD *float_types; /** NIR SSA float vars. */ BITSET_WORD *int_types; /** NIR SSA int vars. */ @@ -39,6 +41,373 @@ static pco_block *trans_cf_nodes(trans_ctx *tctx, struct list_head *cf_node_list, struct exec_list *nir_cf_node_list); +/** + * \brief Translates a NIR def into a PCO reference. + * + * \param[in] def The nir def. + * \return The PCO reference. + */ +static inline pco_ref pco_ref_nir_def(const nir_def *def) +{ + return pco_ref_ssa(def->index, def->bit_size, def->num_components); +} + +/** + * \brief Translates a NIR src into a PCO reference. + * + * \param[in] src The nir src. + * \return The PCO reference. + */ +static inline pco_ref pco_ref_nir_src(const nir_src *src) +{ + return pco_ref_nir_def(src->ssa); +} + +/** + * \brief Translates a NIR def into a PCO reference with type information. + * + * \param[in] def The nir def. + * \param[in] tctx Translation context. + * \return The PCO reference. + */ +static inline pco_ref pco_ref_nir_def_t(const nir_def *def, trans_ctx *tctx) +{ + pco_ref ref = pco_ref_nir_def(def); + + bool is_float = BITSET_TEST(tctx->float_types, def->index); + bool is_int = BITSET_TEST(tctx->int_types, def->index); + + if (is_float) + ref.dtype = PCO_DTYPE_FLOAT; + else if (is_int) + ref.dtype = PCO_DTYPE_UNSIGNED; + + return ref; +} + +/** + * \brief Translates a NIR src into a PCO reference with type information. + * + * \param[in] src The nir src. + * \param[in] tctx Translation context. + * \return The PCO reference. + */ +static inline pco_ref pco_ref_nir_src_t(const nir_src *src, trans_ctx *tctx) +{ + return pco_ref_nir_def_t(src->ssa, tctx); +} + +/** + * \brief Translates a NIR alu src into a PCO reference with type information, + * extracting and building vectors as needed. + * + * \param[in] src The nir src. + * \param[in,out] tctx Translation context. + * \return The PCO reference. + */ +static inline pco_ref +pco_ref_nir_alu_src_t(const nir_alu_instr *alu, unsigned src, trans_ctx *tctx) +{ + const nir_alu_src *alu_src = &alu->src[src]; + /* unsigned chans = nir_src_num_components(alu_src->src); */ + unsigned chans = nir_ssa_alu_instr_src_components(alu, src); + + bool seq_comps = + nir_is_sequential_comp_swizzle((uint8_t *)alu_src->swizzle, chans); + pco_ref ref = pco_ref_nir_src_t(&alu_src->src, tctx); + unsigned swizzle0 = alu_src->swizzle[0]; + + /* Multiple channels, but referencing the entire vector; return as-is. */ + if (!swizzle0 && seq_comps && chans == nir_src_num_components(alu_src->src)) + return ref; + + /* One channel; just extract it. */ + pco_ref var = pco_ref_new_ssa(tctx->func, pco_ref_get_bits(ref), chans); + if (chans == 1) { + pco_ref comp = pco_ref_val16(swizzle0); + pco_comp(&tctx->b, var, ref, comp); + return var; + } + + /* Multiple channels; extract each into a vec. */ + pco_ref chan_comps[NIR_MAX_VEC_COMPONENTS] = { 0 }; + for (unsigned u = 0; u < chans; ++u) { + pco_ref comp = pco_ref_val16(alu_src->swizzle[u]); + chan_comps[u] = pco_ref_new_ssa(tctx->func, pco_ref_get_bits(ref), 1); + pco_comp(&tctx->b, chan_comps[u], ref, comp); + } + + pco_vec(&tctx->b, var, chans, chan_comps); + return var; +} + +/** + * \brief Translates a NIR vs load_input intrinsic into PCO. + * + * \param[in,out] tctx Translation context. + * \param[in] intr load_input intrinsic. + * \param[in] dest Instruction destination. + * \return The translated PCO instruction. + */ +static pco_instr * +trans_load_input_vs(trans_ctx *tctx, nir_intrinsic_instr *intr, pco_ref dest) +{ + puts("finishme: trans_load_input_vs"); + + unsigned base = nir_intrinsic_base(intr); + unsigned component = nir_intrinsic_component(intr); + unsigned chans = pco_ref_get_chans(dest); + + const nir_src offset = intr->src[0]; + assert(nir_src_as_uint(offset) == 0); + + /* TODO NEXT: Wrong! Do properly! */ + unsigned vtxin_offset = (4 * base) + component; + pco_ref src = pco_ref_hwreg_vec(vtxin_offset, PCO_REG_CLASS_VTXIN, chans); + + return pco_mov(&tctx->b, dest, src, .rpt = chans); +} + +/** + * \brief Translates a NIR vs store_output intrinsic into PCO. + * + * \param[in,out] tctx Translation context. + * \param[in] intr store_output intrinsic. + * \param[in] src Instruction source. + * \return The translated PCO instruction. + */ +static pco_instr * +trans_store_output_vs(trans_ctx *tctx, nir_intrinsic_instr *intr, pco_ref src) +{ + puts("finishme: trans_store_output_vs"); + + unsigned base = nir_intrinsic_base(intr); + unsigned component = nir_intrinsic_component(intr); + unsigned chans = pco_ref_get_chans(src); + + const nir_src offset = intr->src[1]; + assert(nir_src_as_uint(offset) == 0); + + /* TODO NEXT: Wrong! Do properly! */ + pco_ref vtxout_addr = pco_ref_val8((4 * base) + component); + + return pco_uvsw_write(&tctx->b, src, vtxout_addr, .rpt = chans); +} + +/** + * \brief Translates a NIR fs load_input intrinsic into PCO. + * + * \param[in,out] tctx Translation context. + * \param[in] intr load_input intrinsic. + * \param[in] dest Instruction destination. + * \return The translated PCO instruction. + */ +static pco_instr * +trans_load_input_fs(trans_ctx *tctx, nir_intrinsic_instr *intr, pco_ref dest) +{ + puts("finishme: trans_load_input_fs"); + + UNUSED unsigned base = nir_intrinsic_base(intr); + UNUSED unsigned component = nir_intrinsic_component(intr); + unsigned chans = pco_ref_get_chans(dest); + + const nir_src offset = intr->src[0]; + assert(nir_src_as_uint(offset) == 0); + + /* TODO NEXT: Wrong! Do properly! */ + unsigned coeffs_index = + 4 * ((nir_intrinsic_io_semantics(intr).location - VARYING_SLOT_VAR0) + 1); + pco_ref coeffs = pco_ref_hwreg_vec(coeffs_index, PCO_REG_CLASS_COEFF, 4); + pco_ref wcoeffs = pco_ref_hwreg_vec(0, PCO_REG_CLASS_COEFF, 4); + pco_ref itr_count = pco_ref_val16(chans); + + return pco_fitrp(&tctx->b, + dest, + pco_ref_drc(PCO_DRC_0), + coeffs, + wcoeffs, + itr_count, + .itr_mode = PCO_ITR_MODE_PIXEL); +} + +/** + * \brief Translates a NIR fs store_output intrinsic into PCO. + * + * \param[in,out] tctx Translation context. + * \param[in] intr store_output intrinsic. + * \param[in] src Instruction source. + * \return The translated PCO instruction. + */ +static pco_instr * +trans_store_output_fs(trans_ctx *tctx, nir_intrinsic_instr *intr, pco_ref src) +{ + assert(pco_ref_is_scalar(src)); + puts("finishme: trans_store_output_fs"); + + bool is_reg_store = nir_src_is_const(intr->src[1]); + unsigned base = nir_intrinsic_base(intr); + + if (is_reg_store) { + /* TODO NEXT: Wrong! Do properly! */ + pco_ref dest = pco_ref_hwreg(base, PCO_REG_CLASS_PIXOUT); + /* TODO NEXT: optimize this to be propagated (backwards?) */ + /* return pco_mbyp0(&tctx->b, dest, src, .olchk = true); */ + return pco_mov(&tctx->b, dest, src, .olchk = true); + } + + unreachable(); +} + +/** + * \brief Translates a NIR intrinsic instruction into PCO. + * + * \param[in,out] tctx Translation context. + * \param[in] intr The nir intrinsic instruction. + * \return The PCO instruction. + */ +static pco_instr *trans_intr(trans_ctx *tctx, nir_intrinsic_instr *intr) +{ + const nir_intrinsic_info *info = &nir_intrinsic_infos[intr->intrinsic]; + + pco_ref dest = info->has_dest ? pco_ref_nir_def_t(&intr->def, tctx) + : pco_ref_null(); + + pco_ref src[NIR_MAX_VEC_COMPONENTS] = { 0 }; + for (unsigned s = 0; s < info->num_srcs; ++s) + src[s] = pco_ref_nir_src_t(&intr->src[s], tctx); + + switch (intr->intrinsic) { + case nir_intrinsic_load_input: + if (tctx->stage == MESA_SHADER_VERTEX) + return trans_load_input_vs(tctx, intr, dest); + else if (tctx->stage == MESA_SHADER_FRAGMENT) + return trans_load_input_fs(tctx, intr, dest); + break; + + case nir_intrinsic_store_output: + if (tctx->stage == MESA_SHADER_VERTEX) + return trans_store_output_vs(tctx, intr, src[0]); + else if (tctx->stage == MESA_SHADER_FRAGMENT) + return trans_store_output_fs(tctx, intr, src[0]); + break; + + default: + break; + } + + printf("Unsupported intrinsic: \""); + nir_print_instr(&intr->instr, stdout); + printf("\"\n"); + unreachable(); +} + +/** + * \brief Translates a NIR alu instruction into PCO. + * + * \param[in] tctx Translation context. + * \param[in] alu The nir alu instruction. + * \return The PCO instruction. + */ +static pco_instr *trans_alu(trans_ctx *tctx, nir_alu_instr *alu) +{ + const nir_op_info *info = &nir_op_infos[alu->op]; + unsigned num_srcs = info->num_inputs; + + pco_ref dest = pco_ref_nir_def_t(&alu->def, tctx); + UNUSED unsigned chans = pco_ref_get_chans(dest); + + pco_ref src[NIR_MAX_VEC_COMPONENTS] = { 0 }; + for (unsigned s = 0; s < num_srcs; ++s) + src[s] = pco_ref_nir_alu_src_t(alu, s, tctx); + + switch (alu->op) { + case nir_op_fneg: + return pco_mov(&tctx->b, dest, pco_ref_neg(src[0])); + + case nir_op_fadd: + return pco_fadd(&tctx->b, dest, src[0], src[1]); + + case nir_op_fmul: + return pco_fmul(&tctx->b, dest, src[0], src[1]); + + case nir_op_ffma: + return pco_fmad(&tctx->b, dest, src[0], src[1], src[2]); + + case nir_op_pack_unorm_4x8: + return pco_pck(&tctx->b, + dest, + src[0], + .rpt = 4, + .pck_fmt = PCO_PCK_FMT_U8888, + .scale = true); + + case nir_op_vec2: + case nir_op_vec3: + case nir_op_vec4: + case nir_op_vec5: + case nir_op_vec8: + case nir_op_vec16: + return pco_vec(&tctx->b, dest, num_srcs, src); + + default: + break; + } + + printf("Unsupported alu instruction: \""); + nir_print_instr(&alu->instr, stdout); + printf("\"\n"); + unreachable(); +} + +/** + * \brief Translates a NIR load constant instruction into PCO. + * + * \param[in] tctx Translation context. + * \param[in] nconst The nir load constant instruction. + * \return The PCO instruction. + */ +static pco_instr *trans_const(trans_ctx *tctx, nir_load_const_instr *nconst) +{ + unsigned num_bits = nconst->def.bit_size; + unsigned comps = nconst->def.num_components; + + /* TODO: support more bit sizes/components. */ + assert(num_bits == 32); + assert(comps == 1); + uint64_t val = nir_const_value_as_uint(nconst->value[0], num_bits); + + pco_ref dest = pco_ref_nir_def_t(&nconst->def, tctx); + pco_ref imm = pco_ref_imm(val, pco_bits(num_bits), pco_ref_get_dtype(dest)); + + return pco_movi32(&tctx->b, dest, imm); +} + +/** + * \brief Translates a NIR instruction into PCO. + * + * \param[in] tctx Translation context. + * \param[in] ninstr The nir instruction. + * \return The PCO instruction. + */ +static pco_instr *trans_instr(trans_ctx *tctx, nir_instr *ninstr) +{ + switch (ninstr->type) { + case nir_instr_type_intrinsic: + return trans_intr(tctx, nir_instr_as_intrinsic(ninstr)); + + case nir_instr_type_load_const: + return trans_const(tctx, nir_instr_as_load_const(ninstr)); + + case nir_instr_type_alu: + return trans_alu(tctx, nir_instr_as_alu(ninstr)); + + default: + break; + } + + unreachable(); +} + /** * \brief Translates a NIR block into PCO. * @@ -51,6 +420,10 @@ static pco_block *trans_block(trans_ctx *tctx, nir_block *nblock) pco_block *block = pco_block_create(tctx->func); tctx->b = pco_builder_create(tctx->func, pco_cursor_after_block(block)); + nir_foreach_instr (ninstr, nblock) { + trans_instr(tctx, ninstr); + } + return block; } @@ -192,6 +565,7 @@ pco_shader *pco_trans_nir(pco_ctx *ctx, nir_shader *nir, void *mem_ctx) trans_ctx tctx = { .pco_ctx = ctx, .shader = shader, + .stage = shader->stage, }; nir_foreach_function_with_impl (func, impl, nir) {