diff --git a/src/compiler/nir/nir_divergence_analysis.c b/src/compiler/nir/nir_divergence_analysis.c index ba73297b6b8..88f36250c10 100644 --- a/src/compiler/nir/nir_divergence_analysis.c +++ b/src/compiler/nir/nir_divergence_analysis.c @@ -46,15 +46,6 @@ struct divergence_state { nir_loop *loop; bool loop_all_invariant; - /* Whether the caller requested vertex divergence (meaning between vertices - * of the same primitive) instead of subgroup invocation divergence - * (between invocations of the same subgroup). For example, patch input - * loads are always convergent, while subgroup intrinsics are divergent - * because vertices of the same primitive can be processed by different - * subgroups. - */ - bool vertex_divergence; - /** current control flow state */ /* True if some loop-active invocations might take a different control-flow path. * A divergent break does not cause subsequent control-flow to be considered @@ -215,7 +206,7 @@ visit_intrinsic(nir_intrinsic_instr *instr, struct divergence_state *state) * subgroups, so subgroup ops are always divergent between vertices of * the same primitive. */ - is_divergent = state->vertex_divergence; + is_divergent = state->options & nir_divergence_vertex; break; /* Intrinsics which are always uniform */ @@ -397,7 +388,7 @@ visit_intrinsic(nir_intrinsic_instr *instr, struct divergence_state *state) is_divergent |= !(options & nir_divergence_single_prim_per_subgroup); } else if (stage == MESA_SHADER_TESS_EVAL) { /* Patch input loads are uniform between vertices of the same primitive. */ - if (state->vertex_divergence) + if (state->options & nir_divergence_vertex) is_divergent = false; else is_divergent |= !(options & nir_divergence_single_patch_per_tes_subgroup); @@ -487,19 +478,19 @@ visit_intrinsic(nir_intrinsic_instr *instr, struct divergence_state *state) is_divergent |= !(options & nir_divergence_single_prim_per_subgroup); break; case nir_intrinsic_load_instance_id: - is_divergent = !state->vertex_divergence; + is_divergent = !(state->options & nir_divergence_vertex); break; case nir_intrinsic_load_primitive_id: if (stage == MESA_SHADER_FRAGMENT) is_divergent = !(options & nir_divergence_single_prim_per_subgroup); else if (stage == MESA_SHADER_TESS_CTRL) - is_divergent = !state->vertex_divergence && + is_divergent = !(state->options & nir_divergence_vertex) && !(options & nir_divergence_single_patch_per_tcs_subgroup); else if (stage == MESA_SHADER_TESS_EVAL) - is_divergent = !state->vertex_divergence && + is_divergent = !(state->options & nir_divergence_vertex) && !(options & nir_divergence_single_patch_per_tes_subgroup); else if (stage == MESA_SHADER_GEOMETRY || stage == MESA_SHADER_VERTEX) - is_divergent = !state->vertex_divergence; + is_divergent = !(state->options & nir_divergence_vertex); else if (stage == MESA_SHADER_ANY_HIT || stage == MESA_SHADER_CLOSEST_HIT || stage == MESA_SHADER_INTERSECTION) @@ -536,7 +527,7 @@ visit_intrinsic(nir_intrinsic_instr *instr, struct divergence_state *state) * vertices of the same primitive because they may be in * different subgroups. */ - is_divergent = state->vertex_divergence; + is_divergent = state->options & nir_divergence_vertex; break; } FALLTHROUGH; @@ -544,7 +535,7 @@ visit_intrinsic(nir_intrinsic_instr *instr, struct divergence_state *state) case nir_intrinsic_inclusive_scan_clusters_ir3: { nir_op op = nir_intrinsic_reduction_op(instr); is_divergent = src_divergent(instr->src[0], state) || - state->vertex_divergence; + state->options & nir_divergence_vertex; if (op != nir_op_umin && op != nir_op_imin && op != nir_op_fmin && op != nir_op_umax && op != nir_op_imax && op != nir_op_fmax && op != nir_op_iand && op != nir_op_ior) @@ -556,7 +547,7 @@ visit_intrinsic(nir_intrinsic_instr *instr, struct divergence_state *state) /* This reduces the last invocations in all 8-wide clusters. It should * behave the same as reduce with cluster_size == subgroup_size. */ - is_divergent = state->vertex_divergence; + is_divergent = state->options & nir_divergence_vertex; break; case nir_intrinsic_load_ubo: @@ -1531,21 +1522,11 @@ nir_divergence_analysis(nir_shader *shader) void nir_vertex_divergence_analysis(nir_shader *shader) { - struct divergence_state state = { - .stage = shader->info.stage, - .shader = shader, - .options = shader->options->divergence_analysis_options, - .loop = NULL, - .loop_all_invariant = false, - .vertex_divergence = true, - .first_visit = true, - }; + nir_divergence_options options = + shader->options->divergence_analysis_options | nir_divergence_vertex; nir_foreach_function_impl(impl, shader) { - nir_metadata_require(impl, nir_metadata_block_index); - state.impl = impl; - visit_cf_list(&impl->body, &state); - nir_progress(true, impl, nir_metadata_all & ~nir_metadata_divergence); + nir_divergence_analysis_impl(impl, options); } } diff --git a/src/compiler/nir/nir_shader_compiler_options.h b/src/compiler/nir/nir_shader_compiler_options.h index 7aa0724586d..525fcd02a45 100644 --- a/src/compiler/nir/nir_shader_compiler_options.h +++ b/src/compiler/nir/nir_shader_compiler_options.h @@ -77,6 +77,12 @@ typedef enum { nir_divergence_uniform_load_tears = (1 << 7), /* If used, this allows phis for divergent merges with undef and a uniform source to be considered uniform */ nir_divergence_ignore_undef_if_phi_srcs = (1 << 8), + /* Whether to compute vertex divergence (meaning between vertices + * of the same primitive) instead of subgroup invocation divergence + * (between invocations of the same subgroup). For example, patch input + * loads are always convergent, while subgroup intrinsics are divergent. + */ + nir_divergence_vertex = (1 << 11), } nir_divergence_options; /** An instruction filtering callback