gallivm: Skip inactive branches

Improves performance of shaders with a lot of control flow.

Reviewed-by: Roland Scheidegger <roland.scheidegger@broadcom.com>
Acked-by: Mike Blumenkrantz <michael.blumenkrantz@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/30347>

src/gallium/auxiliary/gallivm/lp_bld_nir.c

@@ -2810,20 +2810,37 @@ visit_block(struct lp_build_nir_context *bld_base, nir_block *block)
    }
 }
 
+static bool
+lp_should_flatten_cf_list(struct exec_list *cf_list)
+{
+   if (exec_list_is_empty(cf_list))
+      return true;
+   if (!exec_list_is_singular(cf_list))
+      return false;
+
+   struct exec_node *head = exec_list_get_head(cf_list);
+   nir_block *block = nir_cf_node_as_block(exec_node_data(nir_cf_node, head, node));
+   return exec_list_length(&block->instr_list) < 8;
+}
 
 static void
 visit_if(struct lp_build_nir_context *bld_base, nir_if *if_stmt)
 {
    LLVMValueRef cond = get_src(bld_base, if_stmt->condition);
 
-   bld_base->if_cond(bld_base, cond);
+   bool flatten_then = lp_should_flatten_cf_list(&if_stmt->then_list);
+
+   bld_base->if_cond(bld_base, cond, flatten_then);
    visit_cf_list(bld_base, &if_stmt->then_list);
 
    if (!exec_list_is_empty(&if_stmt->else_list)) {
-      bld_base->else_stmt(bld_base);
+      bool flatten_else = lp_should_flatten_cf_list(&if_stmt->else_list);
+      bld_base->else_stmt(bld_base, flatten_then, flatten_else);
       visit_cf_list(bld_base, &if_stmt->else_list);
+      bld_base->endif_stmt(bld_base, flatten_else);
+   } else {
+      bld_base->endif_stmt(bld_base, flatten_then);
    }
-   bld_base->endif_stmt(bld_base);
 }
 
 

src/gallium/auxiliary/gallivm/lp_bld_nir.h

@@ -28,6 +28,7 @@
 
 #include "gallivm/lp_bld.h"
 #include "gallivm/lp_bld_limits.h"
+#include "gallivm/lp_bld_flow.h"
 #include "lp_bld_type.h"
 
 #include "gallivm/lp_bld_tgsi.h"
@@ -93,6 +94,9 @@ struct lp_build_nir_context
    LLVMValueRef func;
    nir_shader *shader;
 
+   struct lp_build_if_state if_stack[LP_MAX_TGSI_NESTING];
+   uint32_t if_stack_size;
+
    void (*load_ubo)(struct lp_build_nir_context *bld_base,
                     unsigned nc,
                     unsigned bit_size,
@@ -218,9 +222,9 @@ struct lp_build_nir_context
 
    void (*bgnloop)(struct lp_build_nir_context *bld_base);
    void (*endloop)(struct lp_build_nir_context *bld_base);
-   void (*if_cond)(struct lp_build_nir_context *bld_base, LLVMValueRef cond);
-   void (*else_stmt)(struct lp_build_nir_context *bld_base);
-   void (*endif_stmt)(struct lp_build_nir_context *bld_base);
+   void (*if_cond)(struct lp_build_nir_context *bld_base, LLVMValueRef cond, bool flatten);
+   void (*else_stmt)(struct lp_build_nir_context *bld_base, bool flatten_then, bool flatten_else);
+   void (*endif_stmt)(struct lp_build_nir_context *bld_base, bool flatten);
    void (*break_stmt)(struct lp_build_nir_context *bld_base);
    void (*continue_stmt)(struct lp_build_nir_context *bld_base);
 

src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c

@@ -2026,21 +2026,62 @@ static void endloop(struct lp_build_nir_context *bld_base)
    lp_exec_endloop(bld_base->base.gallivm, &bld->exec_mask, bld->mask);
 }
 
-static void if_cond(struct lp_build_nir_context *bld_base, LLVMValueRef cond)
+static void lp_build_skip_branch(struct lp_build_nir_context *bld_base, bool flatten)
 {
-   LLVMBuilderRef builder = bld_base->base.gallivm->builder;
+   if (flatten)
+      return;
+
+   struct gallivm_state *gallivm = bld_base->base.gallivm;
+   LLVMBuilderRef builder = gallivm->builder;
+
+   LLVMValueRef exec_mask = mask_vec(bld_base);
+
+   LLVMValueRef bitmask = LLVMBuildICmp(builder, LLVMIntNE, exec_mask, bld_base->uint_bld.zero, "");
+   bitmask = LLVMBuildBitCast(builder, bitmask, LLVMIntTypeInContext(gallivm->context, bld_base->uint_bld.type.length), "");
+   bitmask = LLVMBuildZExt(builder, bitmask, bld_base->int_bld.elem_type, "");
+
+   LLVMValueRef any_active = LLVMBuildICmp(builder, LLVMIntNE, bitmask, lp_build_const_int32(gallivm, 0), "any_active");
+
+   assert(bld_base->if_stack_size < LP_MAX_TGSI_NESTING);
+   lp_build_if(&bld_base->if_stack[bld_base->if_stack_size], gallivm, any_active);
+   bld_base->if_stack_size++;
+}
+
+static void lp_build_skip_branch_end(struct lp_build_nir_context *bld_base, bool flatten)
+{
+   if (flatten)
+      return;
+
+   assert(bld_base->if_stack_size);
+   bld_base->if_stack_size--;
+   lp_build_endif(&bld_base->if_stack[bld_base->if_stack_size]);
+}
+
+static void if_cond(struct lp_build_nir_context *bld_base, LLVMValueRef cond, bool flatten)
+{
+   struct gallivm_state *gallivm = bld_base->base.gallivm;
+
+   LLVMBuilderRef builder = gallivm->builder;
    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
    lp_exec_mask_cond_push(&bld->exec_mask, LLVMBuildBitCast(builder, cond, bld_base->base.int_vec_type, ""));
+
+   lp_build_skip_branch(bld_base, flatten);
 }
 
-static void else_stmt(struct lp_build_nir_context *bld_base)
+static void else_stmt(struct lp_build_nir_context *bld_base, bool flatten_then, bool flatten_else)
 {
+   lp_build_skip_branch_end(bld_base, flatten_then);
+
    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
    lp_exec_mask_cond_invert(&bld->exec_mask);
+
+   lp_build_skip_branch(bld_base, flatten_else);
 }
 
-static void endif_stmt(struct lp_build_nir_context *bld_base)
+static void endif_stmt(struct lp_build_nir_context *bld_base, bool flatten)
 {
+   lp_build_skip_branch_end(bld_base, flatten);
+
    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
    lp_exec_mask_cond_pop(&bld->exec_mask);
 }

src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml

@@ -81,7 +81,7 @@ traces:
       checksum: 018418bdd7f60a186cce532613b0c7ab
   bgfx/33-pom.rdc:
     gl-vmware-llvmpipe:
-      checksum: 4d7c66e327a9e9fe3e7a2d0e7bbe152d
+      checksum: 1d85816768c32de8f82af29802484079
   bgfx/34-mvs.rdc:
     gl-vmware-llvmpipe:
       checksum: 6ad9c7d97debb7bf495b0bfca921ba9c

src/gallium/drivers/svga/ci/traces-vmware.yml

@@ -78,7 +78,7 @@ traces:
       checksum: 018418bdd7f60a186cce532613b0c7ab
   bgfx/33-pom.rdc:
     gl-vmware-llvmpipe:
-      checksum: 4d7c66e327a9e9fe3e7a2d0e7bbe152d
+      checksum: 1d85816768c32de8f82af29802484079
   bgfx/34-mvs.rdc:
     gl-vmware-llvmpipe:
       checksum: 6ad9c7d97debb7bf495b0bfca921ba9c