diff --git a/src/freedreno/ir3/ir3_ra.h b/src/freedreno/ir3/ir3_ra.h index 6dea545c3ce..b67b9dd7839 100644 --- a/src/freedreno/ir3/ir3_ra.h +++ b/src/freedreno/ir3/ir3_ra.h @@ -59,6 +59,8 @@ ra_physreg_to_num(physreg_t physreg, unsigned flags) physreg /= 2; if (flags & IR3_REG_SHARED) physreg += 48 * 4; + else if (flags & IR3_REG_PREDICATE) + physreg += REG_P0 * 4; return physreg; } @@ -67,6 +69,8 @@ ra_num_to_physreg(unsigned num, unsigned flags) { if (flags & IR3_REG_SHARED) num -= 48 * 4; + else if (flags & IR3_REG_PREDICATE) + num -= REG_P0 * 4; if (!(flags & IR3_REG_HALF)) num *= 2; return num; diff --git a/src/freedreno/ir3/ir3_ra_predicates.c b/src/freedreno/ir3/ir3_ra_predicates.c index 9c3e553275b..43600a24f47 100644 --- a/src/freedreno/ir3/ir3_ra_predicates.c +++ b/src/freedreno/ir3/ir3_ra_predicates.c @@ -259,6 +259,10 @@ reload_into(struct ra_predicates_ctx *ctx, struct block_liveness *live, } reloaded_instr->block = use->block; + + /* Keep track of the original def for validation. */ + reloaded_instr->data = def; + ir3_instr_move_before(reloaded_instr, use); struct ir3_register *reloaded_def = reloaded_instr->dsts[0]; return assign_reg(ctx, live, def, reloaded_def, comp); @@ -331,6 +335,12 @@ ra_block(struct ra_predicates_ctx *ctx, struct ir3_block *block) */ dst->flags |= IR3_REG_UNUSED; + /* For validation, we keep track of which def an instruction produces. + * Normally, this will be the instruction's dst but in case of + * reloading, it will point to the original instruction's dst. + */ + dst->instr->data = dst; + /* If we don't have any free registers, ignore the def for now. If we * start spilling right away, we might end-up with a cascade of spills * when there are a lot of defs before their first uses. diff --git a/src/freedreno/ir3/ir3_ra_validate.c b/src/freedreno/ir3/ir3_ra_validate.c index 75edda3331a..1b94acf4284 100644 --- a/src/freedreno/ir3/ir3_ra_validate.c +++ b/src/freedreno/ir3/ir3_ra_validate.c @@ -86,7 +86,7 @@ struct file_state { }; struct reaching_state { - struct file_state half, full, shared; + struct file_state half, full, shared, predicate; }; struct ra_val_ctx { @@ -107,7 +107,7 @@ struct ra_val_ctx { */ struct hash_table *spill_reaching; - unsigned full_size, half_size; + unsigned full_size, half_size, predicate_size; bool merged_regs; bool shared_ra; @@ -136,6 +136,8 @@ get_file_size(struct ra_val_ctx *ctx, struct ir3_register *reg) { if (reg->flags & IR3_REG_SHARED) return RA_SHARED_SIZE; + else if (reg->flags & IR3_REG_PREDICATE) + return ctx->predicate_size; else if (ctx->merged_regs || !(reg->flags & IR3_REG_HALF)) return ctx->full_size; else @@ -164,6 +166,18 @@ get_or_create_spill_state(struct ra_val_ctx *ctx, struct ir3_register *dst) return state; } +static bool +validate_reg_is_src(const struct ir3_register *reg) +{ + return ra_reg_is_src(reg) || ra_reg_is_predicate(reg); +} + +static bool +validate_reg_is_dst(const struct ir3_register *reg) +{ + return ra_reg_is_dst(reg) || ra_reg_is_predicate(reg); +} + /* Validate simple things, like the registers being in-bounds. This way we * don't have to worry about out-of-bounds accesses later. */ @@ -172,7 +186,7 @@ static void validate_simple(struct ra_val_ctx *ctx, struct ir3_instruction *instr) { ctx->current_instr = instr; - ra_foreach_dst (dst, instr) { + foreach_dst_if (dst, instr, validate_reg_is_dst) { if (ctx->shared_ra && !(dst->flags & IR3_REG_SHARED)) continue; validate_assert(ctx, dst->num != INVALID_REG); @@ -182,7 +196,7 @@ validate_simple(struct ra_val_ctx *ctx, struct ir3_instruction *instr) validate_assert(ctx, ra_reg_get_num(dst) == ra_reg_get_num(dst->tied)); } - ra_foreach_src (src, instr) { + foreach_src_if (src, instr, validate_reg_is_src) { if (ctx->shared_ra && !(src->flags & IR3_REG_SHARED)) continue; validate_assert(ctx, src->num != INVALID_REG); @@ -234,6 +248,8 @@ merge_state(struct ra_val_ctx *ctx, struct reaching_state *dst, bool progress = false; progress |= merge_file(&dst->full, &src->full, ctx->full_size); progress |= merge_file(&dst->half, &src->half, ctx->half_size); + progress |= + merge_file(&dst->predicate, &src->predicate, ctx->predicate_size); return progress; } @@ -249,16 +265,33 @@ ra_val_get_file(struct ra_val_ctx *ctx, struct ir3_register *reg) { if (reg->flags & IR3_REG_SHARED) return &ctx->reaching.shared; + else if (reg->flags & IR3_REG_PREDICATE) + return &ctx->reaching.predicate; else if (ctx->merged_regs || !(reg->flags & IR3_REG_HALF)) return &ctx->reaching.full; else return &ctx->reaching.half; } +/* Predicate RA implements spilling by cloning the instruction that produces a + * def. In that case, we might end up two different defs legitimately reaching a + * source. To support validation, the RA will store the original def in the + * instruction's data field. + */ +static struct ir3_register * +get_original_def(struct ir3_register *def) +{ + if (def == UNKNOWN || def == UNDEF || def == OVERDEF) + return def; + if (def->flags & IR3_REG_PREDICATE) + return def->instr->data; + return def; +} + static void propagate_normal_instr(struct ra_val_ctx *ctx, struct ir3_instruction *instr) { - ra_foreach_dst (dst, instr) { + foreach_dst_if (dst, instr, validate_reg_is_dst) { /* Process destinations from scalar ALU instructions that were demoted to * normal ALU instructions. For these we must treat the instruction as a * spill of itself and set the propagate state to itself. See @@ -279,9 +312,10 @@ propagate_normal_instr(struct ra_val_ctx *ctx, struct ir3_instruction *instr) struct file_state *file = ra_val_get_file(ctx, dst); physreg_t physreg = ra_reg_get_physreg(dst); + for (unsigned i = 0; i < reg_size(dst); i++) { file->regs[physreg + i] = (struct reg_state){ - .def = dst, + .def = get_original_def(dst), .offset = i, }; } @@ -554,10 +588,16 @@ dump_reg_state(struct reg_state *state) /* The analysis should always remove UNKNOWN eventually. */ assert(state->def != UNKNOWN); - fprintf(stderr, "ssa_%u:%u(%sr%u.%c) + %u", state->def->instr->serialno, + const char *prefix = "r"; + unsigned num = state->def->num / 4; + if (state->def->flags & IR3_REG_PREDICATE) { + prefix = "p"; + num = 0; + } + + fprintf(stderr, "ssa_%u:%u(%s%s%u.%c) + %u", state->def->instr->serialno, state->def->name, (state->def->flags & IR3_REG_HALF) ? "h" : "", - state->def->num / 4, "xyzw"[state->def->num % 4], - state -> offset); + prefix, num, "xyzw"[state->def->num % 4], state -> offset); } } @@ -571,7 +611,7 @@ check_reaching_src(struct ra_val_ctx *ctx, struct ir3_instruction *instr, physreg_t physreg = ra_reg_get_physreg(src); for (unsigned i = 0; i < reg_size(src); i++) { struct reg_state expected = (struct reg_state){ - .def = src->def, + .def = get_original_def(src->def), .offset = i, }; chase_definition(&expected); @@ -604,7 +644,7 @@ check_reaching_instr(struct ra_val_ctx *ctx, struct ir3_instruction *instr) return; } - ra_foreach_src (src, instr) { + foreach_src_if (src, instr, validate_reg_is_src) { check_reaching_src(ctx, instr, src); } } @@ -647,6 +687,8 @@ check_reaching_defs(struct ra_val_ctx *ctx, struct ir3 *ir) start->half.regs[i].def = UNDEF; for (unsigned i = 0; i < RA_SHARED_SIZE; i++) start->shared.regs[i].def = UNDEF; + for (unsigned i = 0; i < ctx->predicate_size; i++) + start->predicate.regs[i].def = UNDEF; bool progress; do { @@ -684,6 +726,7 @@ ir3_ra_validate(struct ir3_shader_variant *v, unsigned full_size, ctx->merged_regs = v->mergedregs; ctx->full_size = full_size; ctx->half_size = half_size; + ctx->predicate_size = v->compiler->num_predicates * 2; ctx->block_count = block_count; ctx->shared_ra = shared_ra; if (ctx->shared_ra)