From dde084d68820e50b43694029c39711492c052eb3 Mon Sep 17 00:00:00 2001 From: Vasily Khoruzhick Date: Thu, 13 Feb 2025 21:44:30 -0800 Subject: [PATCH] lima: ppir: use combiner unit for mul MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Combiner unit runs after fmul/smul/fadd/sadd units and it can consume the results that previous units wrote to the registers. So prefer placing scalar mul into combiner unit and predecessors (if any) into other units shader-db: total instructions in shared programs: 29072 -> 27698 (-4.73%) instructions in affected programs: 11237 -> 9863 (-12.23%) helped: 163 HURT: 0 helped stats (abs) min: 1 max: 42 x̄: 8.43 x̃: 4 helped stats (rel) min: 0.64% max: 30.00% x̄: 13.03% x̃: 11.76% 95% mean confidence interval for instructions value: -9.89 -6.96 95% mean confidence interval for instructions %-change: -14.09% -11.97% Instructions are helped. total loops in shared programs: 2 -> 2 (0.00%) loops in affected programs: 0 -> 0 helped: 0 HURT: 0 total spills in shared programs: 367 -> 372 (1.36%) spills in affected programs: 16 -> 21 (31.25%) helped: 1 HURT: 2 total fills in shared programs: 1208 -> 1224 (1.32%) fills in affected programs: 51 -> 67 (31.37%) helped: 2 HURT: 2 LOST: 0 GAINED: 0 Reviewed-by: Erico Nunes Signed-off-by: Vasily Khoruzhick Part-of: --- src/gallium/drivers/lima/ir/pp/instr.c | 154 +++++++++++++++++- src/gallium/drivers/lima/ir/pp/node.c | 1 + .../drivers/lima/ir/pp/node_to_instr.c | 7 + 3 files changed, 153 insertions(+), 9 deletions(-) diff --git a/src/gallium/drivers/lima/ir/pp/instr.c b/src/gallium/drivers/lima/ir/pp/instr.c index 707055c48ac..1dca40286d3 100644 --- a/src/gallium/drivers/lima/ir/pp/instr.c +++ b/src/gallium/drivers/lima/ir/pp/instr.c @@ -86,7 +86,10 @@ void ppir_instr_insert_mul_node(ppir_node *add, ppir_node *mul) if (ppir_node_target_equal(src, dest)) { src->type = ppir_target_pipeline; src->pipeline = pipeline; - } + } else + /* Do not pipeline select sources if select condition is not + * pipelined yet */ + return; if (ppir_node_target_equal(++src, dest)) { src->type = ppir_target_pipeline; @@ -211,6 +214,7 @@ bool ppir_instr_insert_node(ppir_instr *instr, ppir_node *node) int *slots = ppir_op_infos[node->op].slots; for (int i = 0; slots[i] != PPIR_INSTR_SLOT_END; i++) { int pos = slots[i]; + ppir_dest *dest = ppir_node_get_dest(node); if (instr->slots[pos]) { /* node already in this instr, i.e. load_uniform */ @@ -220,18 +224,124 @@ bool ppir_instr_insert_node(ppir_instr *instr, ppir_node *node) continue; } - /* ^fmul dests (e.g. condition for select) can only be - * scheduled to ALU_SCL_MUL */ - if (pos == PPIR_INSTR_SLOT_ALU_SCL_ADD) { - ppir_dest *dest = ppir_node_get_dest(node); - if (dest && dest->type == ppir_target_pipeline && - dest->pipeline == ppir_pipeline_reg_fmul) - continue; + if (pos == PPIR_INSTR_SLOT_ALU_VEC_MUL) { + if (dest && dest->type == ppir_target_pipeline) { + ppir_node *add = ppir_node_first_succ(node); + if (add->instr_pos == PPIR_INSTR_SLOT_ALU_SCL_ADD) + continue; + } + } + + if (pos == PPIR_INSTR_SLOT_ALU_SCL_MUL) { + if (dest && dest->type == ppir_target_pipeline) { + ppir_node *succ = ppir_node_first_succ(node); + if (succ->instr_pos == PPIR_INSTR_SLOT_ALU_VEC_ADD && succ->op != ppir_op_select) + continue; + } + + /* Do not schedule into scalar mul slot if there is a select in any add + * slot */ + ppir_node *sadd = instr->slots[PPIR_INSTR_SLOT_ALU_SCL_ADD]; + ppir_node *vadd = instr->slots[PPIR_INSTR_SLOT_ALU_VEC_ADD]; + if (dest && dest->type != ppir_target_pipeline && + ((sadd && sadd->op == ppir_op_select) || (vadd && vadd->op == ppir_op_select))) + continue; + } + + /* Don't attempt to schedule a node with pipeline reg destination + * into slots that cannot have a pipeline output */ + switch (pos) + { + case PPIR_INSTR_SLOT_ALU_SCL_ADD: + case PPIR_INSTR_SLOT_ALU_VEC_ADD: + case PPIR_INSTR_SLOT_ALU_COMBINE: + if (dest && dest->type == ppir_target_pipeline) + continue; + break; + default: + break; + } + + /* Don't schedule add node if it has a pipelined predecessor, but + * corresponding slot is already taken */ + if (pos == PPIR_INSTR_SLOT_ALU_VEC_ADD || pos == PPIR_INSTR_SLOT_ALU_SCL_ADD) { + int mul_pos = pos == PPIR_INSTR_SLOT_ALU_VEC_ADD ? + PPIR_INSTR_SLOT_ALU_VEC_MUL : + PPIR_INSTR_SLOT_ALU_SCL_ADD; + for (int i = 0; i < ppir_node_get_src_num(node); i++) { + ppir_src *src = ppir_node_get_src(node, i); + if (src->type == ppir_target_pipeline && + (src->pipeline == ppir_pipeline_reg_fmul || + src->pipeline == ppir_pipeline_reg_vmul)) + if (instr->slots[mul_pos]) + continue; + } + } + + /* Don't schedule select into add node if smul slot is already taken + */ + if (node->op == ppir_op_select && instr->slots[PPIR_INSTR_SLOT_ALU_SCL_MUL]) + break; + + /* Don't schedule to mul slot if it outputs to pipeline reg, + * but corresponding ADD slot is taken */ + if (pos == PPIR_INSTR_SLOT_ALU_VEC_MUL || pos == PPIR_INSTR_SLOT_ALU_SCL_MUL) { + ppir_node *add; + ppir_pipeline pipeline; + if (pos == PPIR_INSTR_SLOT_ALU_VEC_MUL) { + add = instr->slots[PPIR_INSTR_SLOT_ALU_VEC_ADD]; + pipeline = ppir_pipeline_reg_vmul; + } else { + add = instr->slots[PPIR_INSTR_SLOT_ALU_SCL_ADD]; + pipeline = ppir_pipeline_reg_fmul; + } + if (add) { + for (int i = 0; i < ppir_node_get_src_num(add); i++) { + ppir_src *src = ppir_node_get_src(add, i); + if (src->type == ppir_target_pipeline && src->pipeline == pipeline && + src->node != node) + continue; + } + } + } + + /* Handle select condition for vector mul */ + if (pos == PPIR_INSTR_SLOT_ALU_SCL_MUL) { + ppir_node *select = instr->slots[PPIR_INSTR_SLOT_ALU_VEC_ADD]; + if (select && select->op == ppir_op_select) { + ppir_src *src = ppir_node_get_src(select, 0); + if (src->node != node) + continue; + } + } + + if (pos == PPIR_INSTR_SLOT_ALU_COMBINE) { + /* Branch unit runs in parallel with combiner, so we cannot + * schedule into combiner slot if branch slot is used + */ + if (instr->slots[PPIR_INSTR_SLOT_BRANCH]) + continue; + + if (!ppir_target_is_scalar(dest)) + continue; + /* Combiner doesn't have pipeline destination */ + if (dest->type == ppir_target_pipeline) + continue; + + /* No modifiers on vector output */ + if (node->op == ppir_op_mul && dest->modifier != ppir_outmod_none) + continue; + + /* No modifiers on vector source on combiner */ + if (ppir_node_get_src_num(node) == 2) { + ppir_src *src = ppir_node_get_src(node, 1); + if (src->negate || src->absolute) + continue; + } } if (pos == PPIR_INSTR_SLOT_ALU_SCL_MUL || pos == PPIR_INSTR_SLOT_ALU_SCL_ADD) { - ppir_dest *dest = ppir_node_get_dest(node); if (!ppir_target_is_scalar(dest)) continue; } @@ -246,6 +356,32 @@ bool ppir_instr_insert_node(ppir_instr *instr, ppir_node *node) instr, ppir_pipeline_reg_uniform, &l->dest, NULL); } + /* Scalar add can be placed in vector slot, update pipeline source + * register if it is the case + */ + if (pos == PPIR_INSTR_SLOT_ALU_VEC_ADD) { + if (ppir_target_is_scalar(dest)) { + for (int i = 0; i < ppir_node_get_src_num(node); i++) + { + /* Don't update condition on select, it always comes from + * fmul */ + if (i == 0 && node->op == ppir_op_select) + continue; + ppir_src *src = ppir_node_get_src(node, i); + if (src->type == ppir_target_pipeline && src->pipeline == ppir_pipeline_reg_fmul) + src->pipeline = ppir_pipeline_reg_vmul; + } + } + } + + /* Scalar mul can be placed in vector slot, update pipeline dest + * register if it is the case + */ + if (pos == PPIR_INSTR_SLOT_ALU_VEC_MUL) { + if (dest->type == ppir_target_pipeline) + dest->pipeline = ppir_pipeline_reg_vmul; + } + return true; } diff --git a/src/gallium/drivers/lima/ir/pp/node.c b/src/gallium/drivers/lima/ir/pp/node.c index 3236374633a..ffcda28330c 100644 --- a/src/gallium/drivers/lima/ir/pp/node.c +++ b/src/gallium/drivers/lima/ir/pp/node.c @@ -58,6 +58,7 @@ const ppir_op_info ppir_op_infos[] = { [ppir_op_mul] = { .name = "mul", .slots = (int []) { + PPIR_INSTR_SLOT_ALU_COMBINE, PPIR_INSTR_SLOT_ALU_SCL_MUL, PPIR_INSTR_SLOT_ALU_VEC_MUL, PPIR_INSTR_SLOT_END }, diff --git a/src/gallium/drivers/lima/ir/pp/node_to_instr.c b/src/gallium/drivers/lima/ir/pp/node_to_instr.c index 577112da95a..87f60ffd519 100644 --- a/src/gallium/drivers/lima/ir/pp/node_to_instr.c +++ b/src/gallium/drivers/lima/ir/pp/node_to_instr.c @@ -111,7 +111,14 @@ static bool ppir_do_one_node_to_instr(ppir_block *block, ppir_node *node) alu->dest.ssa.num_components == 1) { node->instr_pos = PPIR_INSTR_SLOT_ALU_SCL_MUL; ppir_instr_insert_mul_node(succ, node); + } else if (succ->instr_pos == PPIR_INSTR_SLOT_ALU_COMBINE || + succ->instr_pos == PPIR_INSTR_SLOT_BRANCH) { + /* Successor is combiner, we can try scheduling ALU node + * into fmul/smul/fadd/sadd slots */ + if (succ->instr) + ppir_instr_insert_node(succ->instr, node); } + } /* can't inserted to any existing instr, create one */