From 944f8f6db23d08a43665ca6ce9b6d3aa1fb2a36f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Ol=C5=A1=C3=A1k?= Date: Wed, 25 Jun 2025 17:48:32 -0400 Subject: [PATCH] nir: move nir_lower_io_vars_to_scalar into its own file Acked-by: Alyssa Rosenzweig Part-of: --- src/compiler/nir/meson.build | 1 + src/compiler/nir/nir_lower_io_to_scalar.c | 267 ---------------- .../nir/nir_lower_io_vars_to_scalar.c | 293 ++++++++++++++++++ 3 files changed, 294 insertions(+), 267 deletions(-) create mode 100644 src/compiler/nir/nir_lower_io_vars_to_scalar.c diff --git a/src/compiler/nir/meson.build b/src/compiler/nir/meson.build index 50acc601eca..5c4cb6f5c1d 100644 --- a/src/compiler/nir/meson.build +++ b/src/compiler/nir/meson.build @@ -188,6 +188,7 @@ else 'nir_lower_io_to_temporaries.c', 'nir_lower_io_to_scalar.c', 'nir_lower_io_to_vector.c', + 'nir_lower_io_vars_to_scalar.c', 'nir_lower_is_helper_invocation.c', 'nir_lower_multiview.c', 'nir_lower_mediump.c', diff --git a/src/compiler/nir/nir_lower_io_to_scalar.c b/src/compiler/nir/nir_lower_io_to_scalar.c index f1c8e0c13c2..cae61a9f475 100644 --- a/src/compiler/nir/nir_lower_io_to_scalar.c +++ b/src/compiler/nir/nir_lower_io_to_scalar.c @@ -349,270 +349,3 @@ nir_lower_io_to_scalar(nir_shader *shader, nir_variable_mode mask, nir_instr_fil nir_metadata_control_flow, &state); } - -static nir_variable ** -get_channel_variables(struct hash_table *ht, nir_variable *var) -{ - nir_variable **chan_vars; - struct hash_entry *entry = _mesa_hash_table_search(ht, var); - if (!entry) { - chan_vars = (nir_variable **)calloc(4, sizeof(nir_variable *)); - _mesa_hash_table_insert(ht, var, chan_vars); - } else { - chan_vars = (nir_variable **)entry->data; - } - - return chan_vars; -} - -/* - * Note that the src deref that we are cloning is the head of the - * chain of deref instructions from the original intrinsic, but - * the dst we are cloning to is the tail (because chains of deref - * instructions are created back to front) - */ - -static nir_deref_instr * -clone_deref_array(nir_builder *b, nir_deref_instr *dst_tail, - const nir_deref_instr *src_head) -{ - const nir_deref_instr *parent = nir_deref_instr_parent(src_head); - - if (!parent) - return dst_tail; - - assert(src_head->deref_type == nir_deref_type_array); - - dst_tail = clone_deref_array(b, dst_tail, parent); - - return nir_build_deref_array(b, dst_tail, - src_head->arr.index.ssa); -} - -static void -lower_load_to_scalar_early(nir_builder *b, nir_intrinsic_instr *intr, - nir_variable *var, struct hash_table *split_inputs, - struct hash_table *split_outputs) -{ - b->cursor = nir_before_instr(&intr->instr); - - nir_def *loads[NIR_MAX_VEC_COMPONENTS]; - - nir_variable **chan_vars; - if (var->data.mode == nir_var_shader_in) { - chan_vars = get_channel_variables(split_inputs, var); - } else { - chan_vars = get_channel_variables(split_outputs, var); - } - - for (unsigned i = 0; i < intr->num_components; i++) { - nir_variable *chan_var = chan_vars[var->data.location_frac + i]; - if (!chan_vars[var->data.location_frac + i]) { - chan_var = nir_variable_clone(var, b->shader); - chan_var->data.location_frac = var->data.location_frac + i; - chan_var->type = glsl_channel_type(chan_var->type); - - chan_vars[var->data.location_frac + i] = chan_var; - - nir_shader_add_variable(b->shader, chan_var); - } - - nir_intrinsic_instr *chan_intr = - nir_intrinsic_instr_create(b->shader, intr->intrinsic); - nir_def_init(&chan_intr->instr, &chan_intr->def, 1, - intr->def.bit_size); - chan_intr->num_components = 1; - - nir_deref_instr *deref = nir_build_deref_var(b, chan_var); - - deref = clone_deref_array(b, deref, nir_src_as_deref(intr->src[0])); - - chan_intr->src[0] = nir_src_for_ssa(&deref->def); - - if (intr->intrinsic == nir_intrinsic_interp_deref_at_offset || - intr->intrinsic == nir_intrinsic_interp_deref_at_sample || - intr->intrinsic == nir_intrinsic_interp_deref_at_vertex) - chan_intr->src[1] = nir_src_for_ssa(intr->src[1].ssa); - - nir_builder_instr_insert(b, &chan_intr->instr); - - loads[i] = &chan_intr->def; - } - - nir_def_replace(&intr->def, nir_vec(b, loads, intr->num_components)); -} - -static void -lower_store_output_to_scalar_early(nir_builder *b, nir_intrinsic_instr *intr, - nir_variable *var, - struct hash_table *split_outputs) -{ - b->cursor = nir_before_instr(&intr->instr); - - nir_def *value = intr->src[1].ssa; - - nir_variable **chan_vars = get_channel_variables(split_outputs, var); - for (unsigned i = 0; i < intr->num_components; i++) { - if (!(nir_intrinsic_write_mask(intr) & (1 << i))) - continue; - - nir_variable *chan_var = chan_vars[var->data.location_frac + i]; - if (!chan_vars[var->data.location_frac + i]) { - chan_var = nir_variable_clone(var, b->shader); - chan_var->data.location_frac = var->data.location_frac + i; - chan_var->type = glsl_channel_type(chan_var->type); - - chan_vars[var->data.location_frac + i] = chan_var; - - nir_shader_add_variable(b->shader, chan_var); - } - - nir_intrinsic_instr *chan_intr = - nir_intrinsic_instr_create(b->shader, intr->intrinsic); - chan_intr->num_components = 1; - - nir_intrinsic_set_write_mask(chan_intr, 0x1); - - nir_deref_instr *deref = nir_build_deref_var(b, chan_var); - - deref = clone_deref_array(b, deref, nir_src_as_deref(intr->src[0])); - - chan_intr->src[0] = nir_src_for_ssa(&deref->def); - chan_intr->src[1] = nir_src_for_ssa(nir_channel(b, value, i)); - - nir_builder_instr_insert(b, &chan_intr->instr); - } - - /* Remove the old store intrinsic */ - nir_instr_remove(&intr->instr); -} - -struct io_to_scalar_early_state { - struct hash_table *split_inputs, *split_outputs; - nir_variable_mode mask; -}; - -static bool -nir_lower_io_vars_to_scalar_instr(nir_builder *b, nir_instr *instr, void *data) -{ - struct io_to_scalar_early_state *state = data; - - if (instr->type != nir_instr_type_intrinsic) - return false; - - nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); - - if (intr->num_components == 1) - return false; - - if (intr->intrinsic != nir_intrinsic_load_deref && - intr->intrinsic != nir_intrinsic_store_deref && - intr->intrinsic != nir_intrinsic_interp_deref_at_centroid && - intr->intrinsic != nir_intrinsic_interp_deref_at_sample && - intr->intrinsic != nir_intrinsic_interp_deref_at_offset && - intr->intrinsic != nir_intrinsic_interp_deref_at_vertex) - return false; - - nir_deref_instr *deref = nir_src_as_deref(intr->src[0]); - if (!nir_deref_mode_is_one_of(deref, state->mask)) - return false; - - nir_variable *var = nir_deref_instr_get_variable(deref); - nir_variable_mode mode = var->data.mode; - - /* TODO: add patch support */ - if (var->data.patch) - return false; - - /* TODO: add doubles support */ - if (glsl_type_is_64bit(glsl_without_array(var->type))) - return false; - - if (!(b->shader->info.stage == MESA_SHADER_VERTEX && - mode == nir_var_shader_in) && - var->data.location < VARYING_SLOT_VAR0 && - var->data.location >= 0) - return false; - - /* Don't bother splitting if we can't opt away any unused - * components. - */ - if (var->data.always_active_io) - return false; - - if (var->data.must_be_shader_input) - return false; - - /* Skip types we cannot split */ - if (glsl_type_is_matrix(glsl_without_array(var->type)) || - glsl_type_is_struct_or_ifc(glsl_without_array(var->type))) - return false; - - switch (intr->intrinsic) { - case nir_intrinsic_interp_deref_at_centroid: - case nir_intrinsic_interp_deref_at_sample: - case nir_intrinsic_interp_deref_at_offset: - case nir_intrinsic_interp_deref_at_vertex: - case nir_intrinsic_load_deref: - if ((state->mask & nir_var_shader_in && mode == nir_var_shader_in) || - (state->mask & nir_var_shader_out && mode == nir_var_shader_out)) { - lower_load_to_scalar_early(b, intr, var, state->split_inputs, - state->split_outputs); - return true; - } - break; - case nir_intrinsic_store_deref: - if (state->mask & nir_var_shader_out && - mode == nir_var_shader_out) { - lower_store_output_to_scalar_early(b, intr, var, state->split_outputs); - return true; - } - break; - default: - break; - } - - return false; -} - -/* - * This function is intended to be called earlier than nir_lower_io_to_scalar() - * i.e. before nir_lower_io() is called. - */ -bool -nir_lower_io_vars_to_scalar(nir_shader *shader, nir_variable_mode mask) -{ - struct io_to_scalar_early_state state = { - .split_inputs = _mesa_pointer_hash_table_create(NULL), - .split_outputs = _mesa_pointer_hash_table_create(NULL), - .mask = mask - }; - - bool progress = nir_shader_instructions_pass(shader, - nir_lower_io_vars_to_scalar_instr, - nir_metadata_control_flow, - &state); - - /* Remove old input from the shaders inputs list */ - hash_table_foreach(state.split_inputs, entry) { - nir_variable *var = (nir_variable *)entry->key; - exec_node_remove(&var->node); - - free(entry->data); - } - - /* Remove old output from the shaders outputs list */ - hash_table_foreach(state.split_outputs, entry) { - nir_variable *var = (nir_variable *)entry->key; - exec_node_remove(&var->node); - - free(entry->data); - } - - _mesa_hash_table_destroy(state.split_inputs, NULL); - _mesa_hash_table_destroy(state.split_outputs, NULL); - - nir_remove_dead_derefs(shader); - - return progress; -} diff --git a/src/compiler/nir/nir_lower_io_vars_to_scalar.c b/src/compiler/nir/nir_lower_io_vars_to_scalar.c new file mode 100644 index 00000000000..8e685e1f3bc --- /dev/null +++ b/src/compiler/nir/nir_lower_io_vars_to_scalar.c @@ -0,0 +1,293 @@ +/* + * Copyright © 2016 Broadcom + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "nir.h" +#include "nir_builder.h" +#include "nir_deref.h" + +static nir_variable ** +get_channel_variables(struct hash_table *ht, nir_variable *var) +{ + nir_variable **chan_vars; + struct hash_entry *entry = _mesa_hash_table_search(ht, var); + if (!entry) { + chan_vars = (nir_variable **)calloc(4, sizeof(nir_variable *)); + _mesa_hash_table_insert(ht, var, chan_vars); + } else { + chan_vars = (nir_variable **)entry->data; + } + + return chan_vars; +} + +/* + * Note that the src deref that we are cloning is the head of the + * chain of deref instructions from the original intrinsic, but + * the dst we are cloning to is the tail (because chains of deref + * instructions are created back to front) + */ + +static nir_deref_instr * +clone_deref_array(nir_builder *b, nir_deref_instr *dst_tail, + const nir_deref_instr *src_head) +{ + const nir_deref_instr *parent = nir_deref_instr_parent(src_head); + + if (!parent) + return dst_tail; + + assert(src_head->deref_type == nir_deref_type_array); + + dst_tail = clone_deref_array(b, dst_tail, parent); + + return nir_build_deref_array(b, dst_tail, + src_head->arr.index.ssa); +} + +static void +lower_load_to_scalar_early(nir_builder *b, nir_intrinsic_instr *intr, + nir_variable *var, struct hash_table *split_inputs, + struct hash_table *split_outputs) +{ + b->cursor = nir_before_instr(&intr->instr); + + nir_def *loads[NIR_MAX_VEC_COMPONENTS]; + + nir_variable **chan_vars; + if (var->data.mode == nir_var_shader_in) { + chan_vars = get_channel_variables(split_inputs, var); + } else { + chan_vars = get_channel_variables(split_outputs, var); + } + + for (unsigned i = 0; i < intr->num_components; i++) { + nir_variable *chan_var = chan_vars[var->data.location_frac + i]; + if (!chan_vars[var->data.location_frac + i]) { + chan_var = nir_variable_clone(var, b->shader); + chan_var->data.location_frac = var->data.location_frac + i; + chan_var->type = glsl_channel_type(chan_var->type); + + chan_vars[var->data.location_frac + i] = chan_var; + + nir_shader_add_variable(b->shader, chan_var); + } + + nir_intrinsic_instr *chan_intr = + nir_intrinsic_instr_create(b->shader, intr->intrinsic); + nir_def_init(&chan_intr->instr, &chan_intr->def, 1, + intr->def.bit_size); + chan_intr->num_components = 1; + + nir_deref_instr *deref = nir_build_deref_var(b, chan_var); + + deref = clone_deref_array(b, deref, nir_src_as_deref(intr->src[0])); + + chan_intr->src[0] = nir_src_for_ssa(&deref->def); + + if (intr->intrinsic == nir_intrinsic_interp_deref_at_offset || + intr->intrinsic == nir_intrinsic_interp_deref_at_sample || + intr->intrinsic == nir_intrinsic_interp_deref_at_vertex) + chan_intr->src[1] = nir_src_for_ssa(intr->src[1].ssa); + + nir_builder_instr_insert(b, &chan_intr->instr); + + loads[i] = &chan_intr->def; + } + + nir_def_replace(&intr->def, nir_vec(b, loads, intr->num_components)); +} + +static void +lower_store_output_to_scalar_early(nir_builder *b, nir_intrinsic_instr *intr, + nir_variable *var, + struct hash_table *split_outputs) +{ + b->cursor = nir_before_instr(&intr->instr); + + nir_def *value = intr->src[1].ssa; + + nir_variable **chan_vars = get_channel_variables(split_outputs, var); + for (unsigned i = 0; i < intr->num_components; i++) { + if (!(nir_intrinsic_write_mask(intr) & (1 << i))) + continue; + + nir_variable *chan_var = chan_vars[var->data.location_frac + i]; + if (!chan_vars[var->data.location_frac + i]) { + chan_var = nir_variable_clone(var, b->shader); + chan_var->data.location_frac = var->data.location_frac + i; + chan_var->type = glsl_channel_type(chan_var->type); + + chan_vars[var->data.location_frac + i] = chan_var; + + nir_shader_add_variable(b->shader, chan_var); + } + + nir_intrinsic_instr *chan_intr = + nir_intrinsic_instr_create(b->shader, intr->intrinsic); + chan_intr->num_components = 1; + + nir_intrinsic_set_write_mask(chan_intr, 0x1); + + nir_deref_instr *deref = nir_build_deref_var(b, chan_var); + + deref = clone_deref_array(b, deref, nir_src_as_deref(intr->src[0])); + + chan_intr->src[0] = nir_src_for_ssa(&deref->def); + chan_intr->src[1] = nir_src_for_ssa(nir_channel(b, value, i)); + + nir_builder_instr_insert(b, &chan_intr->instr); + } + + /* Remove the old store intrinsic */ + nir_instr_remove(&intr->instr); +} + +struct io_to_scalar_early_state { + struct hash_table *split_inputs, *split_outputs; + nir_variable_mode mask; +}; + +static bool +nir_lower_io_vars_to_scalar_instr(nir_builder *b, nir_instr *instr, void *data) +{ + struct io_to_scalar_early_state *state = data; + + if (instr->type != nir_instr_type_intrinsic) + return false; + + nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); + + if (intr->num_components == 1) + return false; + + if (intr->intrinsic != nir_intrinsic_load_deref && + intr->intrinsic != nir_intrinsic_store_deref && + intr->intrinsic != nir_intrinsic_interp_deref_at_centroid && + intr->intrinsic != nir_intrinsic_interp_deref_at_sample && + intr->intrinsic != nir_intrinsic_interp_deref_at_offset && + intr->intrinsic != nir_intrinsic_interp_deref_at_vertex) + return false; + + nir_deref_instr *deref = nir_src_as_deref(intr->src[0]); + if (!nir_deref_mode_is_one_of(deref, state->mask)) + return false; + + nir_variable *var = nir_deref_instr_get_variable(deref); + nir_variable_mode mode = var->data.mode; + + /* TODO: add patch support */ + if (var->data.patch) + return false; + + /* TODO: add doubles support */ + if (glsl_type_is_64bit(glsl_without_array(var->type))) + return false; + + if (!(b->shader->info.stage == MESA_SHADER_VERTEX && + mode == nir_var_shader_in) && + var->data.location < VARYING_SLOT_VAR0 && + var->data.location >= 0) + return false; + + /* Don't bother splitting if we can't opt away any unused + * components. + */ + if (var->data.always_active_io) + return false; + + if (var->data.must_be_shader_input) + return false; + + /* Skip types we cannot split */ + if (glsl_type_is_matrix(glsl_without_array(var->type)) || + glsl_type_is_struct_or_ifc(glsl_without_array(var->type))) + return false; + + switch (intr->intrinsic) { + case nir_intrinsic_interp_deref_at_centroid: + case nir_intrinsic_interp_deref_at_sample: + case nir_intrinsic_interp_deref_at_offset: + case nir_intrinsic_interp_deref_at_vertex: + case nir_intrinsic_load_deref: + if ((state->mask & nir_var_shader_in && mode == nir_var_shader_in) || + (state->mask & nir_var_shader_out && mode == nir_var_shader_out)) { + lower_load_to_scalar_early(b, intr, var, state->split_inputs, + state->split_outputs); + return true; + } + break; + case nir_intrinsic_store_deref: + if (state->mask & nir_var_shader_out && + mode == nir_var_shader_out) { + lower_store_output_to_scalar_early(b, intr, var, state->split_outputs); + return true; + } + break; + default: + break; + } + + return false; +} + +/* + * This function is intended to be called earlier than nir_lower_io_to_scalar() + * i.e. before nir_lower_io() is called. + */ +bool +nir_lower_io_vars_to_scalar(nir_shader *shader, nir_variable_mode mask) +{ + struct io_to_scalar_early_state state = { + .split_inputs = _mesa_pointer_hash_table_create(NULL), + .split_outputs = _mesa_pointer_hash_table_create(NULL), + .mask = mask + }; + + bool progress = nir_shader_instructions_pass(shader, + nir_lower_io_vars_to_scalar_instr, + nir_metadata_control_flow, + &state); + + /* Remove old input from the shaders inputs list */ + hash_table_foreach(state.split_inputs, entry) { + nir_variable *var = (nir_variable *)entry->key; + exec_node_remove(&var->node); + + free(entry->data); + } + + /* Remove old output from the shaders outputs list */ + hash_table_foreach(state.split_outputs, entry) { + nir_variable *var = (nir_variable *)entry->key; + exec_node_remove(&var->node); + + free(entry->data); + } + + _mesa_hash_table_destroy(state.split_inputs, NULL); + _mesa_hash_table_destroy(state.split_outputs, NULL); + + nir_remove_dead_derefs(shader); + + return progress; +}