nir/divergence: separately indicate whether loops have divergent continues or breaks

bool nir_loop_is_divergent(nir_loop *)
 replaces the previous loop->divergent indicator.

Reviewed-by: Rhys Perry <pendingchaos02@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/30787>

src/broadcom/compiler/nir_to_vir.c

@@ -4589,7 +4589,7 @@ ntq_emit_loop(struct v3d_compile *c, nir_loop *loop)
         struct qblock *save_loop_cont_block = c->loop_cont_block;
         struct qblock *save_loop_break_block = c->loop_break_block;
 
-        if (vir_in_nonuniform_control_flow(c) || loop->divergent) {
+        if (vir_in_nonuniform_control_flow(c) || nir_loop_is_divergent(loop)) {
                 ntq_emit_nonuniform_loop(c, loop);
         } else {
                 ntq_emit_uniform_loop(c, loop);

src/compiler/nir/nir.c

@@ -645,7 +645,8 @@ nir_loop_create(nir_shader *shader)
 
    cf_init(&loop->cf_node, nir_cf_node_loop);
    /* Assume that loops are divergent until proven otherwise */
-   loop->divergent = true;
+   loop->divergent_break = true;
+   loop->divergent_continue = true;
 
    nir_block *body = nir_block_create(shader);
    exec_list_make_empty(&loop->body);

src/compiler/nir/nir.h

@@ -3338,9 +3338,23 @@ typedef struct {
    nir_loop_info *info;
    nir_loop_control control;
    bool partially_unrolled;
-   bool divergent;
+
+   /**
+    * Whether some loop-active invocations might take a different control-flow path:
+    * divergent_continue indicates that a continue statement might be taken by
+    * only some of the loop-active invocations. A subsequent break is always
+    * considered divergent.
+    */
+   bool divergent_continue;
+   bool divergent_break;
 } nir_loop;
 
+static inline bool
+nir_loop_is_divergent(nir_loop *loop)
+{
+   return loop->divergent_continue || loop->divergent_break;
+}
+
 /**
  * Various bits of metadata that can may be created or required by
  * optimization and analysis passes

src/compiler/nir/nir_divergence_analysis.c

@@ -1228,7 +1228,8 @@ visit_loop(nir_loop *loop, struct divergence_state *state)
       progress |= visit_loop_exit_phi(phi, loop_state.divergent_loop_break);
    }
 
-   loop->divergent = (loop_state.divergent_loop_break || loop_state.divergent_loop_continue);
+   loop->divergent_continue = loop_state.divergent_loop_continue;
+   loop->divergent_break = loop_state.divergent_loop_break;
 
    return progress;
 }
@@ -1331,15 +1332,14 @@ nir_update_instr_divergence(nir_shader *shader, nir_instr *instr)
 bool
 nir_has_divergent_loop(nir_shader *shader)
 {
-   bool divergent_loop = false;
    nir_function_impl *func = nir_shader_get_entrypoint(shader);
 
    foreach_list_typed(nir_cf_node, node, node, &func->body) {
-      if (node->type == nir_cf_node_loop && nir_cf_node_as_loop(node)->divergent) {
-         divergent_loop = true;
-         break;
+      if (node->type == nir_cf_node_loop) {
+         if (nir_loop_is_divergent(nir_cf_node_as_loop(node)))
+            return true;
       }
    }
 
-   return divergent_loop;
+   return false;
 }

src/compiler/nir/nir_print.c

@@ -2209,7 +2209,7 @@ print_loop(nir_loop *loop, print_state *state, unsigned tabs)
    FILE *fp = state->fp;
 
    print_indentation(tabs, fp);
-   fprintf(fp, "%sloop {\n", divergence_status(state, loop->divergent));
+   fprintf(fp, "%sloop {\n", divergence_status(state, loop->divergent_break));
    foreach_list_typed(nir_cf_node, node, node, &loop->body) {
       print_cf_node(node, state, tabs + 1);
    }

src/compiler/nir/nir_serialize.c

@@ -1824,7 +1824,8 @@ static void
 write_loop(write_ctx *ctx, nir_loop *loop)
 {
    blob_write_uint8(ctx->blob, loop->control);
-   blob_write_uint8(ctx->blob, loop->divergent);
+   blob_write_uint8(ctx->blob, loop->divergent_continue);
+   blob_write_uint8(ctx->blob, loop->divergent_break);
    bool has_continue_construct = nir_loop_has_continue_construct(loop);
    blob_write_uint8(ctx->blob, has_continue_construct);
 
@@ -1842,7 +1843,8 @@ read_loop(read_ctx *ctx, struct exec_list *cf_list)
    nir_cf_node_insert_end(cf_list, &loop->cf_node);
 
    loop->control = blob_read_uint8(ctx->blob);
-   loop->divergent = blob_read_uint8(ctx->blob);
+   loop->divergent_continue = blob_read_uint8(ctx->blob);
+   loop->divergent_break = blob_read_uint8(ctx->blob);
    bool has_continue_construct = blob_read_uint8(ctx->blob);
 
    read_cf_list(ctx, &loop->body);

src/nouveau/compiler/nak_nir_lower_cf.c

@@ -340,15 +340,19 @@ lower_cf_list(nir_builder *b, nir_def *esc_reg, struct scope *parent_scope,
           * while avoiding an extra sync for the loop break is tricky at best.
           */
          struct scope break_scope = push_scope(b, SCOPE_TYPE_LOOP_BREAK,
-                                               parent_scope, loop->divergent,
-                                               loop->divergent, break_block);
+                                               parent_scope,
+                                               nir_loop_is_divergent(loop),
+                                               nir_loop_is_divergent(loop),
+                                               break_block);
 
          nir_goto(b, head_block);
          push_block(b, head_block, break_scope.divergent);
 
          struct scope cont_scope = push_scope(b, SCOPE_TYPE_LOOP_CONT,
-                                              &break_scope, loop->divergent,
-                                              loop->divergent, cont_block);
+                                              &break_scope,
+                                              nir_loop_is_divergent(loop),
+                                              nir_loop_is_divergent(loop),
+                                              cont_block);
 
          lower_cf_list(b, esc_reg, &cont_scope, &loop->body);
          normal_exit(b, esc_reg, cont_block);

src/nouveau/compiler/nak_nir_lower_non_uniform_ldcx.c

@@ -444,7 +444,7 @@ lower_cf_list(nir_builder *b, struct exec_list *cf_list)
 
       case nir_cf_node_loop: {
          nir_loop *loop = nir_cf_node_as_loop(node);
-         if (loop->divergent) {
+         if (nir_loop_is_divergent(loop)) {
             nir_block *succ = nir_cf_node_as_block(nir_cf_node_next(node));
             progress |= lower_non_uniform_cf_node(b, node, block, succ);
          } else {