diff --git a/src/gallium/drivers/lima/ir/pp/instr.c b/src/gallium/drivers/lima/ir/pp/instr.c index 707055c48ac..1dca40286d3 100644 --- a/src/gallium/drivers/lima/ir/pp/instr.c +++ b/src/gallium/drivers/lima/ir/pp/instr.c @@ -86,7 +86,10 @@ void ppir_instr_insert_mul_node(ppir_node *add, ppir_node *mul) if (ppir_node_target_equal(src, dest)) { src->type = ppir_target_pipeline; src->pipeline = pipeline; - } + } else + /* Do not pipeline select sources if select condition is not + * pipelined yet */ + return; if (ppir_node_target_equal(++src, dest)) { src->type = ppir_target_pipeline; @@ -211,6 +214,7 @@ bool ppir_instr_insert_node(ppir_instr *instr, ppir_node *node) int *slots = ppir_op_infos[node->op].slots; for (int i = 0; slots[i] != PPIR_INSTR_SLOT_END; i++) { int pos = slots[i]; + ppir_dest *dest = ppir_node_get_dest(node); if (instr->slots[pos]) { /* node already in this instr, i.e. load_uniform */ @@ -220,18 +224,124 @@ bool ppir_instr_insert_node(ppir_instr *instr, ppir_node *node) continue; } - /* ^fmul dests (e.g. condition for select) can only be - * scheduled to ALU_SCL_MUL */ - if (pos == PPIR_INSTR_SLOT_ALU_SCL_ADD) { - ppir_dest *dest = ppir_node_get_dest(node); - if (dest && dest->type == ppir_target_pipeline && - dest->pipeline == ppir_pipeline_reg_fmul) - continue; + if (pos == PPIR_INSTR_SLOT_ALU_VEC_MUL) { + if (dest && dest->type == ppir_target_pipeline) { + ppir_node *add = ppir_node_first_succ(node); + if (add->instr_pos == PPIR_INSTR_SLOT_ALU_SCL_ADD) + continue; + } + } + + if (pos == PPIR_INSTR_SLOT_ALU_SCL_MUL) { + if (dest && dest->type == ppir_target_pipeline) { + ppir_node *succ = ppir_node_first_succ(node); + if (succ->instr_pos == PPIR_INSTR_SLOT_ALU_VEC_ADD && succ->op != ppir_op_select) + continue; + } + + /* Do not schedule into scalar mul slot if there is a select in any add + * slot */ + ppir_node *sadd = instr->slots[PPIR_INSTR_SLOT_ALU_SCL_ADD]; + ppir_node *vadd = instr->slots[PPIR_INSTR_SLOT_ALU_VEC_ADD]; + if (dest && dest->type != ppir_target_pipeline && + ((sadd && sadd->op == ppir_op_select) || (vadd && vadd->op == ppir_op_select))) + continue; + } + + /* Don't attempt to schedule a node with pipeline reg destination + * into slots that cannot have a pipeline output */ + switch (pos) + { + case PPIR_INSTR_SLOT_ALU_SCL_ADD: + case PPIR_INSTR_SLOT_ALU_VEC_ADD: + case PPIR_INSTR_SLOT_ALU_COMBINE: + if (dest && dest->type == ppir_target_pipeline) + continue; + break; + default: + break; + } + + /* Don't schedule add node if it has a pipelined predecessor, but + * corresponding slot is already taken */ + if (pos == PPIR_INSTR_SLOT_ALU_VEC_ADD || pos == PPIR_INSTR_SLOT_ALU_SCL_ADD) { + int mul_pos = pos == PPIR_INSTR_SLOT_ALU_VEC_ADD ? + PPIR_INSTR_SLOT_ALU_VEC_MUL : + PPIR_INSTR_SLOT_ALU_SCL_ADD; + for (int i = 0; i < ppir_node_get_src_num(node); i++) { + ppir_src *src = ppir_node_get_src(node, i); + if (src->type == ppir_target_pipeline && + (src->pipeline == ppir_pipeline_reg_fmul || + src->pipeline == ppir_pipeline_reg_vmul)) + if (instr->slots[mul_pos]) + continue; + } + } + + /* Don't schedule select into add node if smul slot is already taken + */ + if (node->op == ppir_op_select && instr->slots[PPIR_INSTR_SLOT_ALU_SCL_MUL]) + break; + + /* Don't schedule to mul slot if it outputs to pipeline reg, + * but corresponding ADD slot is taken */ + if (pos == PPIR_INSTR_SLOT_ALU_VEC_MUL || pos == PPIR_INSTR_SLOT_ALU_SCL_MUL) { + ppir_node *add; + ppir_pipeline pipeline; + if (pos == PPIR_INSTR_SLOT_ALU_VEC_MUL) { + add = instr->slots[PPIR_INSTR_SLOT_ALU_VEC_ADD]; + pipeline = ppir_pipeline_reg_vmul; + } else { + add = instr->slots[PPIR_INSTR_SLOT_ALU_SCL_ADD]; + pipeline = ppir_pipeline_reg_fmul; + } + if (add) { + for (int i = 0; i < ppir_node_get_src_num(add); i++) { + ppir_src *src = ppir_node_get_src(add, i); + if (src->type == ppir_target_pipeline && src->pipeline == pipeline && + src->node != node) + continue; + } + } + } + + /* Handle select condition for vector mul */ + if (pos == PPIR_INSTR_SLOT_ALU_SCL_MUL) { + ppir_node *select = instr->slots[PPIR_INSTR_SLOT_ALU_VEC_ADD]; + if (select && select->op == ppir_op_select) { + ppir_src *src = ppir_node_get_src(select, 0); + if (src->node != node) + continue; + } + } + + if (pos == PPIR_INSTR_SLOT_ALU_COMBINE) { + /* Branch unit runs in parallel with combiner, so we cannot + * schedule into combiner slot if branch slot is used + */ + if (instr->slots[PPIR_INSTR_SLOT_BRANCH]) + continue; + + if (!ppir_target_is_scalar(dest)) + continue; + /* Combiner doesn't have pipeline destination */ + if (dest->type == ppir_target_pipeline) + continue; + + /* No modifiers on vector output */ + if (node->op == ppir_op_mul && dest->modifier != ppir_outmod_none) + continue; + + /* No modifiers on vector source on combiner */ + if (ppir_node_get_src_num(node) == 2) { + ppir_src *src = ppir_node_get_src(node, 1); + if (src->negate || src->absolute) + continue; + } } if (pos == PPIR_INSTR_SLOT_ALU_SCL_MUL || pos == PPIR_INSTR_SLOT_ALU_SCL_ADD) { - ppir_dest *dest = ppir_node_get_dest(node); if (!ppir_target_is_scalar(dest)) continue; } @@ -246,6 +356,32 @@ bool ppir_instr_insert_node(ppir_instr *instr, ppir_node *node) instr, ppir_pipeline_reg_uniform, &l->dest, NULL); } + /* Scalar add can be placed in vector slot, update pipeline source + * register if it is the case + */ + if (pos == PPIR_INSTR_SLOT_ALU_VEC_ADD) { + if (ppir_target_is_scalar(dest)) { + for (int i = 0; i < ppir_node_get_src_num(node); i++) + { + /* Don't update condition on select, it always comes from + * fmul */ + if (i == 0 && node->op == ppir_op_select) + continue; + ppir_src *src = ppir_node_get_src(node, i); + if (src->type == ppir_target_pipeline && src->pipeline == ppir_pipeline_reg_fmul) + src->pipeline = ppir_pipeline_reg_vmul; + } + } + } + + /* Scalar mul can be placed in vector slot, update pipeline dest + * register if it is the case + */ + if (pos == PPIR_INSTR_SLOT_ALU_VEC_MUL) { + if (dest->type == ppir_target_pipeline) + dest->pipeline = ppir_pipeline_reg_vmul; + } + return true; } diff --git a/src/gallium/drivers/lima/ir/pp/node.c b/src/gallium/drivers/lima/ir/pp/node.c index 3236374633a..ffcda28330c 100644 --- a/src/gallium/drivers/lima/ir/pp/node.c +++ b/src/gallium/drivers/lima/ir/pp/node.c @@ -58,6 +58,7 @@ const ppir_op_info ppir_op_infos[] = { [ppir_op_mul] = { .name = "mul", .slots = (int []) { + PPIR_INSTR_SLOT_ALU_COMBINE, PPIR_INSTR_SLOT_ALU_SCL_MUL, PPIR_INSTR_SLOT_ALU_VEC_MUL, PPIR_INSTR_SLOT_END }, diff --git a/src/gallium/drivers/lima/ir/pp/node_to_instr.c b/src/gallium/drivers/lima/ir/pp/node_to_instr.c index 577112da95a..87f60ffd519 100644 --- a/src/gallium/drivers/lima/ir/pp/node_to_instr.c +++ b/src/gallium/drivers/lima/ir/pp/node_to_instr.c @@ -111,7 +111,14 @@ static bool ppir_do_one_node_to_instr(ppir_block *block, ppir_node *node) alu->dest.ssa.num_components == 1) { node->instr_pos = PPIR_INSTR_SLOT_ALU_SCL_MUL; ppir_instr_insert_mul_node(succ, node); + } else if (succ->instr_pos == PPIR_INSTR_SLOT_ALU_COMBINE || + succ->instr_pos == PPIR_INSTR_SLOT_BRANCH) { + /* Successor is combiner, we can try scheduling ALU node + * into fmul/smul/fadd/sadd slots */ + if (succ->instr) + ppir_instr_insert_node(succ->instr, node); } + } /* can't inserted to any existing instr, create one */