From 2808419f967d99e418c730f207a7995065e974a6 Mon Sep 17 00:00:00 2001 From: Konstantin Seurer Date: Sat, 16 Mar 2024 12:57:35 +0100 Subject: [PATCH] gallivm: Skip inactive branches Improves performance of shaders with a lot of control flow. Reviewed-by: Roland Scheidegger Acked-by: Mike Blumenkrantz Part-of: --- src/gallium/auxiliary/gallivm/lp_bld_nir.c | 23 +++++++-- src/gallium/auxiliary/gallivm/lp_bld_nir.h | 10 ++-- .../auxiliary/gallivm/lp_bld_nir_soa.c | 49 +++++++++++++++++-- .../drivers/llvmpipe/ci/traces-llvmpipe.yml | 2 +- src/gallium/drivers/svga/ci/traces-vmware.yml | 2 +- 5 files changed, 74 insertions(+), 12 deletions(-) diff --git a/src/gallium/auxiliary/gallivm/lp_bld_nir.c b/src/gallium/auxiliary/gallivm/lp_bld_nir.c index 24ac0569837..dba70bd1395 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_nir.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_nir.c @@ -2810,20 +2810,37 @@ visit_block(struct lp_build_nir_context *bld_base, nir_block *block) } } +static bool +lp_should_flatten_cf_list(struct exec_list *cf_list) +{ + if (exec_list_is_empty(cf_list)) + return true; + if (!exec_list_is_singular(cf_list)) + return false; + + struct exec_node *head = exec_list_get_head(cf_list); + nir_block *block = nir_cf_node_as_block(exec_node_data(nir_cf_node, head, node)); + return exec_list_length(&block->instr_list) < 8; +} static void visit_if(struct lp_build_nir_context *bld_base, nir_if *if_stmt) { LLVMValueRef cond = get_src(bld_base, if_stmt->condition); - bld_base->if_cond(bld_base, cond); + bool flatten_then = lp_should_flatten_cf_list(&if_stmt->then_list); + + bld_base->if_cond(bld_base, cond, flatten_then); visit_cf_list(bld_base, &if_stmt->then_list); if (!exec_list_is_empty(&if_stmt->else_list)) { - bld_base->else_stmt(bld_base); + bool flatten_else = lp_should_flatten_cf_list(&if_stmt->else_list); + bld_base->else_stmt(bld_base, flatten_then, flatten_else); visit_cf_list(bld_base, &if_stmt->else_list); + bld_base->endif_stmt(bld_base, flatten_else); + } else { + bld_base->endif_stmt(bld_base, flatten_then); } - bld_base->endif_stmt(bld_base); } diff --git a/src/gallium/auxiliary/gallivm/lp_bld_nir.h b/src/gallium/auxiliary/gallivm/lp_bld_nir.h index 4eff207aa25..4a731ea5d95 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_nir.h +++ b/src/gallium/auxiliary/gallivm/lp_bld_nir.h @@ -28,6 +28,7 @@ #include "gallivm/lp_bld.h" #include "gallivm/lp_bld_limits.h" +#include "gallivm/lp_bld_flow.h" #include "lp_bld_type.h" #include "gallivm/lp_bld_tgsi.h" @@ -93,6 +94,9 @@ struct lp_build_nir_context LLVMValueRef func; nir_shader *shader; + struct lp_build_if_state if_stack[LP_MAX_TGSI_NESTING]; + uint32_t if_stack_size; + void (*load_ubo)(struct lp_build_nir_context *bld_base, unsigned nc, unsigned bit_size, @@ -218,9 +222,9 @@ struct lp_build_nir_context void (*bgnloop)(struct lp_build_nir_context *bld_base); void (*endloop)(struct lp_build_nir_context *bld_base); - void (*if_cond)(struct lp_build_nir_context *bld_base, LLVMValueRef cond); - void (*else_stmt)(struct lp_build_nir_context *bld_base); - void (*endif_stmt)(struct lp_build_nir_context *bld_base); + void (*if_cond)(struct lp_build_nir_context *bld_base, LLVMValueRef cond, bool flatten); + void (*else_stmt)(struct lp_build_nir_context *bld_base, bool flatten_then, bool flatten_else); + void (*endif_stmt)(struct lp_build_nir_context *bld_base, bool flatten); void (*break_stmt)(struct lp_build_nir_context *bld_base); void (*continue_stmt)(struct lp_build_nir_context *bld_base); diff --git a/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c b/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c index a740d13d007..c907fdde688 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c @@ -2026,21 +2026,62 @@ static void endloop(struct lp_build_nir_context *bld_base) lp_exec_endloop(bld_base->base.gallivm, &bld->exec_mask, bld->mask); } -static void if_cond(struct lp_build_nir_context *bld_base, LLVMValueRef cond) +static void lp_build_skip_branch(struct lp_build_nir_context *bld_base, bool flatten) { - LLVMBuilderRef builder = bld_base->base.gallivm->builder; + if (flatten) + return; + + struct gallivm_state *gallivm = bld_base->base.gallivm; + LLVMBuilderRef builder = gallivm->builder; + + LLVMValueRef exec_mask = mask_vec(bld_base); + + LLVMValueRef bitmask = LLVMBuildICmp(builder, LLVMIntNE, exec_mask, bld_base->uint_bld.zero, ""); + bitmask = LLVMBuildBitCast(builder, bitmask, LLVMIntTypeInContext(gallivm->context, bld_base->uint_bld.type.length), ""); + bitmask = LLVMBuildZExt(builder, bitmask, bld_base->int_bld.elem_type, ""); + + LLVMValueRef any_active = LLVMBuildICmp(builder, LLVMIntNE, bitmask, lp_build_const_int32(gallivm, 0), "any_active"); + + assert(bld_base->if_stack_size < LP_MAX_TGSI_NESTING); + lp_build_if(&bld_base->if_stack[bld_base->if_stack_size], gallivm, any_active); + bld_base->if_stack_size++; +} + +static void lp_build_skip_branch_end(struct lp_build_nir_context *bld_base, bool flatten) +{ + if (flatten) + return; + + assert(bld_base->if_stack_size); + bld_base->if_stack_size--; + lp_build_endif(&bld_base->if_stack[bld_base->if_stack_size]); +} + +static void if_cond(struct lp_build_nir_context *bld_base, LLVMValueRef cond, bool flatten) +{ + struct gallivm_state *gallivm = bld_base->base.gallivm; + + LLVMBuilderRef builder = gallivm->builder; struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base; lp_exec_mask_cond_push(&bld->exec_mask, LLVMBuildBitCast(builder, cond, bld_base->base.int_vec_type, "")); + + lp_build_skip_branch(bld_base, flatten); } -static void else_stmt(struct lp_build_nir_context *bld_base) +static void else_stmt(struct lp_build_nir_context *bld_base, bool flatten_then, bool flatten_else) { + lp_build_skip_branch_end(bld_base, flatten_then); + struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base; lp_exec_mask_cond_invert(&bld->exec_mask); + + lp_build_skip_branch(bld_base, flatten_else); } -static void endif_stmt(struct lp_build_nir_context *bld_base) +static void endif_stmt(struct lp_build_nir_context *bld_base, bool flatten) { + lp_build_skip_branch_end(bld_base, flatten); + struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base; lp_exec_mask_cond_pop(&bld->exec_mask); } diff --git a/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml b/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml index c19fadf49bc..3c18eed7a6c 100644 --- a/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml +++ b/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml @@ -81,7 +81,7 @@ traces: checksum: 018418bdd7f60a186cce532613b0c7ab bgfx/33-pom.rdc: gl-vmware-llvmpipe: - checksum: 4d7c66e327a9e9fe3e7a2d0e7bbe152d + checksum: 1d85816768c32de8f82af29802484079 bgfx/34-mvs.rdc: gl-vmware-llvmpipe: checksum: 6ad9c7d97debb7bf495b0bfca921ba9c diff --git a/src/gallium/drivers/svga/ci/traces-vmware.yml b/src/gallium/drivers/svga/ci/traces-vmware.yml index 063956d4288..86477529c0e 100644 --- a/src/gallium/drivers/svga/ci/traces-vmware.yml +++ b/src/gallium/drivers/svga/ci/traces-vmware.yml @@ -78,7 +78,7 @@ traces: checksum: 018418bdd7f60a186cce532613b0c7ab bgfx/33-pom.rdc: gl-vmware-llvmpipe: - checksum: 4d7c66e327a9e9fe3e7a2d0e7bbe152d + checksum: 1d85816768c32de8f82af29802484079 bgfx/34-mvs.rdc: gl-vmware-llvmpipe: checksum: 6ad9c7d97debb7bf495b0bfca921ba9c