diff --git a/src/broadcom/compiler/nir_to_vir.c b/src/broadcom/compiler/nir_to_vir.c index 548acba19fa..51dfddae391 100644 --- a/src/broadcom/compiler/nir_to_vir.c +++ b/src/broadcom/compiler/nir_to_vir.c @@ -4589,7 +4589,7 @@ ntq_emit_loop(struct v3d_compile *c, nir_loop *loop) struct qblock *save_loop_cont_block = c->loop_cont_block; struct qblock *save_loop_break_block = c->loop_break_block; - if (vir_in_nonuniform_control_flow(c) || loop->divergent) { + if (vir_in_nonuniform_control_flow(c) || nir_loop_is_divergent(loop)) { ntq_emit_nonuniform_loop(c, loop); } else { ntq_emit_uniform_loop(c, loop); diff --git a/src/compiler/nir/nir.c b/src/compiler/nir/nir.c index 6158a51e551..357f16d6293 100644 --- a/src/compiler/nir/nir.c +++ b/src/compiler/nir/nir.c @@ -645,7 +645,8 @@ nir_loop_create(nir_shader *shader) cf_init(&loop->cf_node, nir_cf_node_loop); /* Assume that loops are divergent until proven otherwise */ - loop->divergent = true; + loop->divergent_break = true; + loop->divergent_continue = true; nir_block *body = nir_block_create(shader); exec_list_make_empty(&loop->body); diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index 0db44ed5e89..a329f0db7f0 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -3338,9 +3338,23 @@ typedef struct { nir_loop_info *info; nir_loop_control control; bool partially_unrolled; - bool divergent; + + /** + * Whether some loop-active invocations might take a different control-flow path: + * divergent_continue indicates that a continue statement might be taken by + * only some of the loop-active invocations. A subsequent break is always + * considered divergent. + */ + bool divergent_continue; + bool divergent_break; } nir_loop; +static inline bool +nir_loop_is_divergent(nir_loop *loop) +{ + return loop->divergent_continue || loop->divergent_break; +} + /** * Various bits of metadata that can may be created or required by * optimization and analysis passes diff --git a/src/compiler/nir/nir_divergence_analysis.c b/src/compiler/nir/nir_divergence_analysis.c index 15171c1f92b..9f1ad0aa3b5 100644 --- a/src/compiler/nir/nir_divergence_analysis.c +++ b/src/compiler/nir/nir_divergence_analysis.c @@ -1228,7 +1228,8 @@ visit_loop(nir_loop *loop, struct divergence_state *state) progress |= visit_loop_exit_phi(phi, loop_state.divergent_loop_break); } - loop->divergent = (loop_state.divergent_loop_break || loop_state.divergent_loop_continue); + loop->divergent_continue = loop_state.divergent_loop_continue; + loop->divergent_break = loop_state.divergent_loop_break; return progress; } @@ -1331,15 +1332,14 @@ nir_update_instr_divergence(nir_shader *shader, nir_instr *instr) bool nir_has_divergent_loop(nir_shader *shader) { - bool divergent_loop = false; nir_function_impl *func = nir_shader_get_entrypoint(shader); foreach_list_typed(nir_cf_node, node, node, &func->body) { - if (node->type == nir_cf_node_loop && nir_cf_node_as_loop(node)->divergent) { - divergent_loop = true; - break; + if (node->type == nir_cf_node_loop) { + if (nir_loop_is_divergent(nir_cf_node_as_loop(node))) + return true; } } - return divergent_loop; + return false; } diff --git a/src/compiler/nir/nir_print.c b/src/compiler/nir/nir_print.c index 41e02e5fa03..41f3eae83fe 100644 --- a/src/compiler/nir/nir_print.c +++ b/src/compiler/nir/nir_print.c @@ -2209,7 +2209,7 @@ print_loop(nir_loop *loop, print_state *state, unsigned tabs) FILE *fp = state->fp; print_indentation(tabs, fp); - fprintf(fp, "%sloop {\n", divergence_status(state, loop->divergent)); + fprintf(fp, "%sloop {\n", divergence_status(state, loop->divergent_break)); foreach_list_typed(nir_cf_node, node, node, &loop->body) { print_cf_node(node, state, tabs + 1); } diff --git a/src/compiler/nir/nir_serialize.c b/src/compiler/nir/nir_serialize.c index b822d9dbe99..1fd05142e87 100644 --- a/src/compiler/nir/nir_serialize.c +++ b/src/compiler/nir/nir_serialize.c @@ -1824,7 +1824,8 @@ static void write_loop(write_ctx *ctx, nir_loop *loop) { blob_write_uint8(ctx->blob, loop->control); - blob_write_uint8(ctx->blob, loop->divergent); + blob_write_uint8(ctx->blob, loop->divergent_continue); + blob_write_uint8(ctx->blob, loop->divergent_break); bool has_continue_construct = nir_loop_has_continue_construct(loop); blob_write_uint8(ctx->blob, has_continue_construct); @@ -1842,7 +1843,8 @@ read_loop(read_ctx *ctx, struct exec_list *cf_list) nir_cf_node_insert_end(cf_list, &loop->cf_node); loop->control = blob_read_uint8(ctx->blob); - loop->divergent = blob_read_uint8(ctx->blob); + loop->divergent_continue = blob_read_uint8(ctx->blob); + loop->divergent_break = blob_read_uint8(ctx->blob); bool has_continue_construct = blob_read_uint8(ctx->blob); read_cf_list(ctx, &loop->body); diff --git a/src/nouveau/compiler/nak_nir_lower_cf.c b/src/nouveau/compiler/nak_nir_lower_cf.c index 6d5172714fe..45dbb87987e 100644 --- a/src/nouveau/compiler/nak_nir_lower_cf.c +++ b/src/nouveau/compiler/nak_nir_lower_cf.c @@ -340,15 +340,19 @@ lower_cf_list(nir_builder *b, nir_def *esc_reg, struct scope *parent_scope, * while avoiding an extra sync for the loop break is tricky at best. */ struct scope break_scope = push_scope(b, SCOPE_TYPE_LOOP_BREAK, - parent_scope, loop->divergent, - loop->divergent, break_block); + parent_scope, + nir_loop_is_divergent(loop), + nir_loop_is_divergent(loop), + break_block); nir_goto(b, head_block); push_block(b, head_block, break_scope.divergent); struct scope cont_scope = push_scope(b, SCOPE_TYPE_LOOP_CONT, - &break_scope, loop->divergent, - loop->divergent, cont_block); + &break_scope, + nir_loop_is_divergent(loop), + nir_loop_is_divergent(loop), + cont_block); lower_cf_list(b, esc_reg, &cont_scope, &loop->body); normal_exit(b, esc_reg, cont_block); diff --git a/src/nouveau/compiler/nak_nir_lower_non_uniform_ldcx.c b/src/nouveau/compiler/nak_nir_lower_non_uniform_ldcx.c index e3983281637..88073d829e8 100644 --- a/src/nouveau/compiler/nak_nir_lower_non_uniform_ldcx.c +++ b/src/nouveau/compiler/nak_nir_lower_non_uniform_ldcx.c @@ -444,7 +444,7 @@ lower_cf_list(nir_builder *b, struct exec_list *cf_list) case nir_cf_node_loop: { nir_loop *loop = nir_cf_node_as_loop(node); - if (loop->divergent) { + if (nir_loop_is_divergent(loop)) { nir_block *succ = nir_cf_node_as_block(nir_cf_node_next(node)); progress |= lower_non_uniform_cf_node(b, node, block, succ); } else {