diff --git a/src/freedreno/ir3/ir3_merge_regs.c b/src/freedreno/ir3/ir3_merge_regs.c index 3259050a315..0d9020bd702 100644 --- a/src/freedreno/ir3/ir3_merge_regs.c +++ b/src/freedreno/ir3/ir3_merge_regs.c @@ -401,6 +401,39 @@ aggressive_coalesce_collect(struct ir3_liveness *live, } } +static void +aggressive_coalesce_rpt(struct ir3_liveness *live, + struct ir3_instruction *instr) +{ + if (!ir3_instr_is_first_rpt(instr)) + return; + + struct ir3_register *def = instr->dsts[0]; + unsigned def_offset = 0; + unsigned src_offsets[instr->srcs_count]; + memset(src_offsets, 0, sizeof(unsigned) * instr->srcs_count); + + foreach_instr_rpt_excl (rpt, instr) { + if (!(rpt->dsts[0]->flags & IR3_REG_SSA)) + continue; + + def_offset += reg_elem_size(def); + try_merge_defs(live, def, rpt->dsts[0], def_offset); + + foreach_src_n (src, src_n, instr) { + struct ir3_register *rpt_src = rpt->srcs[src_n]; + + if (!(src->flags & IR3_REG_SSA) || !(rpt_src->flags & IR3_REG_SSA)) + continue; + if (src->def == rpt_src->def) + continue; + + src_offsets[src_n] += reg_elem_size(src->def); + try_merge_defs(live, src->def, rpt_src->def, src_offsets[src_n]); + } + } +} + static void create_parallel_copy(struct ir3_block *block) { @@ -594,6 +627,12 @@ ir3_merge_regs(struct ir3_liveness *live, struct ir3 *ir) } } + foreach_block (block, &ir->block_list) { + foreach_instr (instr, &block->instr_list) { + aggressive_coalesce_rpt(live, instr); + } + } + index_merge_sets(live, ir); if (ir3_shader_debug & IR3_DBG_RAMSGS) diff --git a/src/freedreno/ir3/ir3_ra.c b/src/freedreno/ir3/ir3_ra.c index 72fd598e95f..204879ef318 100644 --- a/src/freedreno/ir3/ir3_ra.c +++ b/src/freedreno/ir3/ir3_ra.c @@ -1370,6 +1370,55 @@ find_best_gap(struct ra_ctx *ctx, struct ra_file *file, return (physreg_t)~0; } +static physreg_t +try_allocate_src(struct ra_ctx *ctx, struct ra_file *file, + struct ir3_register *reg) +{ + unsigned file_size = reg_file_size(file, reg); + unsigned size = reg_size(reg); + for (unsigned i = 0; i < reg->instr->srcs_count; i++) { + struct ir3_register *src = reg->instr->srcs[i]; + if (!ra_reg_is_src(src)) + continue; + if (ra_get_file(ctx, src) == file && reg_size(src) >= size) { + struct ra_interval *src_interval = &ctx->intervals[src->def->name]; + physreg_t src_physreg = ra_interval_get_physreg(src_interval); + if (src_physreg % reg_elem_size(reg) == 0 && + src_physreg + size <= file_size && + get_reg_specified(ctx, file, reg, src_physreg, false)) + return src_physreg; + } + } + + return ~0; +} + +static bool +rpt_has_unique_merge_set(struct ir3_instruction *instr) +{ + assert(ir3_instr_is_rpt(instr)); + + if (!instr->dsts[0]->merge_set) + return false; + + struct ir3_instruction *first = ir3_instr_first_rpt(instr); + struct ir3_register *def = first->dsts[0]; + + if (def->merge_set != instr->dsts[0]->merge_set || + def->merge_set->regs_count != ir3_instr_rpt_length(first)) { + return false; + } + + unsigned i = 0; + + foreach_instr_rpt (rpt, first) { + if (rpt->dsts[0] != def->merge_set->regs[i++]) + return false; + } + + return true; +} + /* This is the main entrypoint for picking a register. Pick a free register * for "reg", shuffling around sources if necessary. In the normal case where * "is_source" is false, this register can overlap with killed sources @@ -1392,6 +1441,18 @@ get_reg(struct ra_ctx *ctx, struct ra_file *file, struct ir3_register *reg) return preferred_reg; } + /* For repeated instructions whose merge set is unique (i.e., only used for + * these repeated instructions), try to first allocate one of their sources + * (for the same reason as for ALU/SFU instructions explained below). This + * also prevents us from allocating a new register range for this merge set + * when the one from a source could be reused. + */ + if (ir3_instr_is_rpt(reg->instr) && rpt_has_unique_merge_set(reg->instr)) { + physreg_t src_reg = try_allocate_src(ctx, file, reg); + if (src_reg != (physreg_t)~0) + return src_reg; + } + /* If this register is a subset of a merge set which we have not picked a * register for, first try to allocate enough space for the entire merge * set. @@ -1414,19 +1475,9 @@ get_reg(struct ra_ctx *ctx, struct ra_file *file, struct ir3_register *reg) * SFU instructions: */ if (is_sfu(reg->instr) || is_alu(reg->instr)) { - for (unsigned i = 0; i < reg->instr->srcs_count; i++) { - struct ir3_register *src = reg->instr->srcs[i]; - if (!ra_reg_is_src(src)) - continue; - if (ra_get_file(ctx, src) == file && reg_size(src) >= size) { - struct ra_interval *src_interval = &ctx->intervals[src->def->name]; - physreg_t src_physreg = ra_interval_get_physreg(src_interval); - if (src_physreg % reg_elem_size(reg) == 0 && - src_physreg + size <= file_size && - get_reg_specified(ctx, file, reg, src_physreg, false)) - return src_physreg; - } - } + physreg_t src_reg = try_allocate_src(ctx, file, reg); + if (src_reg != (physreg_t)~0) + return src_reg; } physreg_t best_reg =