From bf8a1e1e71c9f6326525f50165c1e2bf0b00dddf Mon Sep 17 00:00:00 2001 From: Lionel Landwerlin Date: Tue, 14 Jan 2025 15:23:18 +0200 Subject: [PATCH] brw/elk: move internal kernel parsing out of intel_clc So it can be called internally. Signed-off-by: Lionel Landwerlin Tested-by: Matt Turner Reviewed-by: Dylan Baker Part-of: --- src/intel/compiler/brw_kernel.c | 318 ------------------------- src/intel/compiler/brw_nir.h | 4 + src/intel/compiler/brw_spirv.c | 359 +++++++++++++++++++++++++++++ src/intel/compiler/elk/elk_nir.h | 4 + src/intel/compiler/elk/elk_spirv.c | 359 +++++++++++++++++++++++++++++ src/intel/compiler/elk/meson.build | 2 + src/intel/compiler/intel_clc.c | 12 +- src/intel/compiler/meson.build | 5 +- 8 files changed, 741 insertions(+), 322 deletions(-) create mode 100644 src/intel/compiler/brw_spirv.c create mode 100644 src/intel/compiler/elk/elk_spirv.c diff --git a/src/intel/compiler/brw_kernel.c b/src/intel/compiler/brw_kernel.c index d06114316e8..a26b3a05e99 100644 --- a/src/intel/compiler/brw_kernel.c +++ b/src/intel/compiler/brw_kernel.c @@ -468,321 +468,3 @@ brw_kernel_from_spirv(struct brw_compiler *compiler, return kernel->code != NULL; } - -static nir_def * -rebuild_value_from_store(struct util_dynarray *stores, - nir_def *value, unsigned read_offset) -{ - unsigned read_size = value->num_components * value->bit_size / 8; - - util_dynarray_foreach(stores, nir_intrinsic_instr *, _store) { - nir_intrinsic_instr *store = *_store; - - unsigned write_offset = nir_src_as_uint(store->src[1]); - unsigned write_size = nir_src_num_components(store->src[0]) * - nir_src_bit_size(store->src[0]) / 8; - if (write_offset <= read_offset && - (write_offset + write_size) >= (read_offset + read_size)) { - assert(nir_block_dominates(store->instr.block, value->parent_instr->block)); - assert(write_size == read_size); - return store->src[0].ssa; - } - } - unreachable("Matching scratch store not found"); -} - -/** - * Remove temporary variables stored to scratch to be then reloaded - * immediately. Remap the load to the store SSA value. - * - * This workaround is only meant to be applied to shaders in src/intel/shaders - * were we know there should be no issue. More complex cases might not work - * with this approach. - */ -static bool -nir_remove_llvm17_scratch(nir_shader *nir) -{ - struct util_dynarray scratch_stores; - void *mem_ctx = ralloc_context(NULL); - - util_dynarray_init(&scratch_stores, mem_ctx); - - nir_foreach_function_impl(func, nir) { - nir_foreach_block(block, func) { - nir_foreach_instr(instr, block) { - if (instr->type != nir_instr_type_intrinsic) - continue; - - nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); - - if (intrin->intrinsic != nir_intrinsic_store_scratch) - continue; - - nir_const_value *offset = nir_src_as_const_value(intrin->src[1]); - if (offset != NULL) { - util_dynarray_append(&scratch_stores, nir_intrinsic_instr *, intrin); - } - } - } - } - - bool progress = false; - if (util_dynarray_num_elements(&scratch_stores, nir_intrinsic_instr *) > 0) { - nir_foreach_function_impl(func, nir) { - nir_foreach_block(block, func) { - nir_foreach_instr_safe(instr, block) { - if (instr->type != nir_instr_type_intrinsic) - continue; - - nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); - - if (intrin->intrinsic != nir_intrinsic_load_scratch) - continue; - - nir_const_value *offset = nir_src_as_const_value(intrin->src[0]); - if (offset == NULL) - continue; - - nir_def_replace(&intrin->def, - rebuild_value_from_store(&scratch_stores, &intrin->def, nir_src_as_uint(intrin->src[0]))); - - progress = true; - } - } - } - } - - util_dynarray_foreach(&scratch_stores, nir_intrinsic_instr *, _store) { - nir_intrinsic_instr *store = *_store; - nir_instr_remove(&store->instr); - } - - /* Quick sanity check */ - assert(util_dynarray_num_elements(&scratch_stores, nir_intrinsic_instr *) == 0 || - progress); - - ralloc_free(mem_ctx); - - return progress; -} - -static void -cleanup_llvm17_scratch(nir_shader *nir) -{ - { - bool progress; - do { - progress = false; - NIR_PASS(progress, nir, nir_copy_prop); - NIR_PASS(progress, nir, nir_opt_dce); - NIR_PASS(progress, nir, nir_opt_constant_folding); - NIR_PASS(progress, nir, nir_opt_cse); - NIR_PASS(progress, nir, nir_opt_algebraic); - } while (progress); - } - - nir_remove_llvm17_scratch(nir); - - { - bool progress; - do { - progress = false; - NIR_PASS(progress, nir, nir_copy_prop); - NIR_PASS(progress, nir, nir_opt_dce); - NIR_PASS(progress, nir, nir_opt_constant_folding); - NIR_PASS(progress, nir, nir_opt_cse); - NIR_PASS(progress, nir, nir_opt_algebraic); - } while (progress); - } -} - -nir_shader * -brw_nir_from_spirv(void *mem_ctx, unsigned gfx_version, const uint32_t *spirv, - size_t spirv_size, bool llvm17_wa) -{ - struct spirv_to_nir_options spirv_options = { - .environment = NIR_SPIRV_OPENCL, - .capabilities = &spirv_caps, - .printf = true, - .shared_addr_format = nir_address_format_62bit_generic, - .global_addr_format = nir_address_format_62bit_generic, - .temp_addr_format = nir_address_format_62bit_generic, - .constant_addr_format = nir_address_format_64bit_global, - .create_library = true, - }; - - assert(spirv_size % 4 == 0); - - assert(gfx_version); - const nir_shader_compiler_options *nir_options = - gfx_version >= 9 ? &brw_scalar_nir_options - : &elk_scalar_nir_options; - - nir_shader *nir = - spirv_to_nir(spirv, spirv_size / 4, NULL, 0, MESA_SHADER_KERNEL, - "library", &spirv_options, nir_options); - nir_validate_shader(nir, "after spirv_to_nir"); - ralloc_steal(mem_ctx, nir); - nir->info.name = ralloc_strdup(nir, "library"); - - if (INTEL_DEBUG(DEBUG_CS)) { - /* Re-index SSA defs so we print more sensible numbers. */ - nir_foreach_function_impl(impl, nir) { - nir_index_ssa_defs(impl); - } - - fprintf(stderr, "NIR (from SPIR-V) for kernel\n"); - nir_print_shader(nir, stderr); - } - - nir_lower_printf_options printf_opts = { - .ptr_bit_size = 64, - .use_printf_base_identifier = true, - }; - NIR_PASS_V(nir, nir_lower_printf, &printf_opts); - - NIR_PASS_V(nir, implement_intel_builtins); - NIR_PASS_V(nir, nir_link_shader_functions, spirv_options.clc_shader); - - /* We have to lower away local constant initializers right before we - * inline functions. That way they get properly initialized at the top - * of the function and not at the top of its caller. - */ - NIR_PASS_V(nir, nir_lower_variable_initializers, ~(nir_var_shader_temp | - nir_var_function_temp)); - NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_uniform | nir_var_mem_ubo | - nir_var_mem_constant | nir_var_function_temp | nir_var_image, NULL); - { - bool progress; - do - { - progress = false; - NIR_PASS(progress, nir, nir_copy_prop); - NIR_PASS(progress, nir, nir_opt_copy_prop_vars); - NIR_PASS(progress, nir, nir_opt_deref); - NIR_PASS(progress, nir, nir_opt_dce); - NIR_PASS(progress, nir, nir_opt_undef); - NIR_PASS(progress, nir, nir_opt_constant_folding); - NIR_PASS(progress, nir, nir_opt_cse); - NIR_PASS(progress, nir, nir_lower_vars_to_ssa); - NIR_PASS(progress, nir, nir_opt_algebraic); - } while (progress); - } - - NIR_PASS_V(nir, nir_lower_variable_initializers, nir_var_function_temp); - NIR_PASS_V(nir, nir_lower_returns); - NIR_PASS_V(nir, nir_inline_functions); - - assert(nir->scratch_size == 0); - NIR_PASS_V(nir, nir_lower_vars_to_explicit_types, nir_var_function_temp, glsl_get_cl_type_size_align); - - { - bool progress; - do - { - progress = false; - NIR_PASS(progress, nir, nir_copy_prop); - NIR_PASS(progress, nir, nir_opt_copy_prop_vars); - NIR_PASS(progress, nir, nir_opt_deref); - NIR_PASS(progress, nir, nir_opt_dce); - NIR_PASS(progress, nir, nir_opt_undef); - NIR_PASS(progress, nir, nir_opt_constant_folding); - NIR_PASS(progress, nir, nir_opt_cse); - NIR_PASS(progress, nir, nir_split_var_copies); - NIR_PASS(progress, nir, nir_lower_var_copies); - NIR_PASS(progress, nir, nir_lower_vars_to_ssa); - NIR_PASS(progress, nir, nir_opt_algebraic); - NIR_PASS(progress, nir, nir_opt_if, nir_opt_if_optimize_phi_true_false); - NIR_PASS(progress, nir, nir_opt_dead_cf); - NIR_PASS(progress, nir, nir_opt_remove_phis); - NIR_PASS(progress, nir, nir_opt_peephole_select, 8, true, true); - NIR_PASS(progress, nir, nir_lower_vec3_to_vec4, nir_var_mem_generic | nir_var_uniform); - NIR_PASS(progress, nir, nir_opt_memcpy); - } while (progress); - } - - NIR_PASS_V(nir, nir_scale_fdiv); - - NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_uniform | nir_var_mem_ubo | - nir_var_mem_constant | nir_var_function_temp | nir_var_image, NULL); - - - NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_mem_shared | nir_var_function_temp, NULL); - - nir->scratch_size = 0; - NIR_PASS_V(nir, nir_lower_vars_to_explicit_types, - nir_var_mem_shared | nir_var_function_temp | nir_var_shader_temp | - nir_var_mem_global | nir_var_mem_constant, - glsl_get_cl_type_size_align); - - // Lower memcpy - needs to wait until types are sized - { - bool progress; - do { - progress = false; - NIR_PASS(progress, nir, nir_opt_memcpy); - NIR_PASS(progress, nir, nir_copy_prop); - NIR_PASS(progress, nir, nir_opt_copy_prop_vars); - NIR_PASS(progress, nir, nir_opt_deref); - NIR_PASS(progress, nir, nir_opt_dce); - NIR_PASS(progress, nir, nir_split_var_copies); - NIR_PASS(progress, nir, nir_lower_var_copies); - NIR_PASS(progress, nir, nir_lower_vars_to_ssa); - NIR_PASS(progress, nir, nir_opt_constant_folding); - NIR_PASS(progress, nir, nir_opt_cse); - } while (progress); - } - NIR_PASS_V(nir, nir_lower_memcpy); - - NIR_PASS_V(nir, nir_lower_explicit_io, - nir_var_mem_shared | nir_var_function_temp | nir_var_shader_temp | nir_var_uniform, - nir_address_format_32bit_offset_as_64bit); - - NIR_PASS_V(nir, nir_lower_system_values); - - /* Hopefully we can drop this once lower_vars_to_ssa has improved to not - * lower everything to scratch. - */ - if (llvm17_wa) - cleanup_llvm17_scratch(nir); - - /* Lower again, this time after dead-variables to get more compact variable - * layouts. - */ - nir->global_mem_size = 0; - nir->scratch_size = 0; - nir->info.shared_size = 0; - NIR_PASS_V(nir, nir_lower_vars_to_explicit_types, - nir_var_mem_shared | nir_var_mem_global | nir_var_mem_constant, - glsl_get_cl_type_size_align); - if (nir->constant_data_size > 0) { - assert(nir->constant_data == NULL); - nir->constant_data = rzalloc_size(nir, nir->constant_data_size); - nir_gather_explicit_io_initializers(nir, nir->constant_data, - nir->constant_data_size, - nir_var_mem_constant); - } - - NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_constant, - nir_address_format_64bit_global); - - NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_uniform, - nir_address_format_32bit_offset_as_64bit); - - NIR_PASS_V(nir, nir_lower_explicit_io, - nir_var_shader_temp | nir_var_function_temp | - nir_var_mem_shared | nir_var_mem_global, - nir_address_format_62bit_generic); - - if (INTEL_DEBUG(DEBUG_CS)) { - /* Re-index SSA defs so we print more sensible numbers. */ - nir_foreach_function_impl(impl, nir) { - nir_index_ssa_defs(impl); - } - - fprintf(stderr, "NIR (before I/O lowering) for kernel\n"); - nir_print_shader(nir, stderr); - } - - return nir; -} diff --git a/src/intel/compiler/brw_nir.h b/src/intel/compiler/brw_nir.h index 1b2f35b9c65..5f193803ee8 100644 --- a/src/intel/compiler/brw_nir.h +++ b/src/intel/compiler/brw_nir.h @@ -293,6 +293,10 @@ brw_nir_no_indirect_mask(const struct brw_compiler *compiler, bool brw_nir_uses_inline_data(nir_shader *shader); +nir_shader * +brw_nir_from_spirv(void *mem_ctx, unsigned gfx_version, const uint32_t *spirv, + size_t spirv_size, bool llvm17_wa); + #ifdef __cplusplus } #endif diff --git a/src/intel/compiler/brw_spirv.c b/src/intel/compiler/brw_spirv.c new file mode 100644 index 00000000000..566bb8b9af3 --- /dev/null +++ b/src/intel/compiler/brw_spirv.c @@ -0,0 +1,359 @@ +/* + * Copyright © 2025 Intel Corporation + * SPDX-License-Identifier: MIT + */ + +#include "brw_nir.h" +#include "intel_nir.h" + +#include "intel_nir.h" +#include "compiler/nir/nir_builder.h" +#include "compiler/spirv/nir_spirv.h" +#include "compiler/spirv/spirv_info.h" +#include "dev/intel_debug.h" +#include "util/u_dynarray.h" + +static nir_def * +rebuild_value_from_store(struct util_dynarray *stores, + nir_def *value, unsigned read_offset) +{ + unsigned read_size = value->num_components * value->bit_size / 8; + + util_dynarray_foreach(stores, nir_intrinsic_instr *, _store) { + nir_intrinsic_instr *store = *_store; + + unsigned write_offset = nir_src_as_uint(store->src[1]); + unsigned write_size = nir_src_num_components(store->src[0]) * + nir_src_bit_size(store->src[0]) / 8; + if (write_offset <= read_offset && + (write_offset + write_size) >= (read_offset + read_size)) { + assert(nir_block_dominates(store->instr.block, value->parent_instr->block)); + assert(write_size == read_size); + return store->src[0].ssa; + } + } + unreachable("Matching scratch store not found"); +} + +/** + * Remove temporary variables stored to scratch to be then reloaded + * immediately. Remap the load to the store SSA value. + * + * This workaround is only meant to be applied to shaders in src/intel/shaders + * were we know there should be no issue. More complex cases might not work + * with this approach. + */ +static bool +nir_remove_llvm17_scratch(nir_shader *nir) +{ + struct util_dynarray scratch_stores; + void *mem_ctx = ralloc_context(NULL); + + util_dynarray_init(&scratch_stores, mem_ctx); + + nir_foreach_function_impl(func, nir) { + nir_foreach_block(block, func) { + nir_foreach_instr(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + + nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); + + if (intrin->intrinsic != nir_intrinsic_store_scratch) + continue; + + nir_const_value *offset = nir_src_as_const_value(intrin->src[1]); + if (offset != NULL) { + util_dynarray_append(&scratch_stores, nir_intrinsic_instr *, intrin); + } + } + } + } + + bool progress = false; + if (util_dynarray_num_elements(&scratch_stores, nir_intrinsic_instr *) > 0) { + nir_foreach_function_impl(func, nir) { + nir_foreach_block(block, func) { + nir_foreach_instr_safe(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + + nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); + + if (intrin->intrinsic != nir_intrinsic_load_scratch) + continue; + + nir_const_value *offset = nir_src_as_const_value(intrin->src[0]); + if (offset == NULL) + continue; + + nir_def_replace(&intrin->def, + rebuild_value_from_store(&scratch_stores, &intrin->def, nir_src_as_uint(intrin->src[0]))); + + progress = true; + } + } + } + } + + util_dynarray_foreach(&scratch_stores, nir_intrinsic_instr *, _store) { + nir_intrinsic_instr *store = *_store; + nir_instr_remove(&store->instr); + } + + /* Quick sanity check */ + assert(util_dynarray_num_elements(&scratch_stores, nir_intrinsic_instr *) == 0 || + progress); + + ralloc_free(mem_ctx); + + return progress; +} + +static void +cleanup_llvm17_scratch(nir_shader *nir) +{ + { + bool progress; + do { + progress = false; + NIR_PASS(progress, nir, nir_copy_prop); + NIR_PASS(progress, nir, nir_opt_dce); + NIR_PASS(progress, nir, nir_opt_constant_folding); + NIR_PASS(progress, nir, nir_opt_cse); + NIR_PASS(progress, nir, nir_opt_algebraic); + } while (progress); + } + + nir_remove_llvm17_scratch(nir); + + { + bool progress; + do { + progress = false; + NIR_PASS(progress, nir, nir_copy_prop); + NIR_PASS(progress, nir, nir_opt_dce); + NIR_PASS(progress, nir, nir_opt_constant_folding); + NIR_PASS(progress, nir, nir_opt_cse); + NIR_PASS(progress, nir, nir_opt_algebraic); + } while (progress); + } +} + +static const struct spirv_capabilities spirv_caps = { + .Addresses = true, + .Float16 = true, + .Float64 = true, + .Groups = true, + .StorageImageWriteWithoutFormat = true, + .Int8 = true, + .Int16 = true, + .Int64 = true, + .Int64Atomics = true, + .Kernel = true, + .Linkage = true, /* We receive linked kernel from clc */ + .DenormFlushToZero = true, + .DenormPreserve = true, + .SignedZeroInfNanPreserve = true, + .RoundingModeRTE = true, + .RoundingModeRTZ = true, + .GenericPointer = true, + .GroupNonUniform = true, + .GroupNonUniformArithmetic = true, + .GroupNonUniformClustered = true, + .GroupNonUniformBallot = true, + .GroupNonUniformQuad = true, + .GroupNonUniformShuffle = true, + .GroupNonUniformVote = true, + .SubgroupDispatch = true, +}; + +nir_shader * +brw_nir_from_spirv(void *mem_ctx, unsigned gfx_version, const uint32_t *spirv, + size_t spirv_size, bool llvm17_wa) +{ + assert(gfx_version >= 9); + + struct spirv_to_nir_options spirv_options = { + .environment = NIR_SPIRV_OPENCL, + .capabilities = &spirv_caps, + .printf = true, + .shared_addr_format = nir_address_format_62bit_generic, + .global_addr_format = nir_address_format_62bit_generic, + .temp_addr_format = nir_address_format_62bit_generic, + .constant_addr_format = nir_address_format_64bit_global, + .create_library = true, + }; + + assert(spirv_size % 4 == 0); + + const nir_shader_compiler_options *nir_options = &brw_scalar_nir_options; + + nir_shader *nir = + spirv_to_nir(spirv, spirv_size / 4, NULL, 0, MESA_SHADER_KERNEL, + "library", &spirv_options, nir_options); + nir_validate_shader(nir, "after spirv_to_nir"); + nir_validate_ssa_dominance(nir, "after spirv_to_nir"); + ralloc_steal(mem_ctx, nir); + nir->info.name = ralloc_strdup(nir, "library"); + + if (INTEL_DEBUG(DEBUG_CS)) { + /* Re-index SSA defs so we print more sensible numbers. */ + nir_foreach_function_impl(impl, nir) { + nir_index_ssa_defs(impl); + } + + fprintf(stderr, "NIR (from SPIR-V) for kernel\n"); + nir_print_shader(nir, stderr); + } + + nir_lower_printf_options printf_opts = { + .ptr_bit_size = 64, + .use_printf_base_identifier = true, + }; + NIR_PASS_V(nir, nir_lower_printf, &printf_opts); + + NIR_PASS_V(nir, nir_link_shader_functions, spirv_options.clc_shader); + + /* We have to lower away local constant initializers right before we + * inline functions. That way they get properly initialized at the top + * of the function and not at the top of its caller. + */ + NIR_PASS_V(nir, nir_lower_variable_initializers, ~(nir_var_shader_temp | + nir_var_function_temp)); + NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_uniform | nir_var_mem_ubo | + nir_var_mem_constant | nir_var_function_temp | nir_var_image, NULL); + { + bool progress; + do + { + progress = false; + NIR_PASS(progress, nir, nir_copy_prop); + NIR_PASS(progress, nir, nir_opt_copy_prop_vars); + NIR_PASS(progress, nir, nir_opt_deref); + NIR_PASS(progress, nir, nir_opt_dce); + NIR_PASS(progress, nir, nir_opt_undef); + NIR_PASS(progress, nir, nir_opt_constant_folding); + NIR_PASS(progress, nir, nir_opt_cse); + NIR_PASS(progress, nir, nir_lower_vars_to_ssa); + NIR_PASS(progress, nir, nir_opt_algebraic); + } while (progress); + } + + NIR_PASS_V(nir, nir_lower_variable_initializers, nir_var_function_temp); + NIR_PASS_V(nir, nir_lower_returns); + NIR_PASS_V(nir, nir_inline_functions); + + assert(nir->scratch_size == 0); + NIR_PASS_V(nir, nir_lower_vars_to_explicit_types, nir_var_function_temp, glsl_get_cl_type_size_align); + + { + bool progress; + do + { + progress = false; + NIR_PASS(progress, nir, nir_copy_prop); + NIR_PASS(progress, nir, nir_opt_copy_prop_vars); + NIR_PASS(progress, nir, nir_opt_deref); + NIR_PASS(progress, nir, nir_opt_dce); + NIR_PASS(progress, nir, nir_opt_undef); + NIR_PASS(progress, nir, nir_opt_constant_folding); + NIR_PASS(progress, nir, nir_opt_cse); + NIR_PASS(progress, nir, nir_split_var_copies); + NIR_PASS(progress, nir, nir_lower_var_copies); + NIR_PASS(progress, nir, nir_lower_vars_to_ssa); + NIR_PASS(progress, nir, nir_opt_algebraic); + NIR_PASS(progress, nir, nir_opt_if, nir_opt_if_optimize_phi_true_false); + NIR_PASS(progress, nir, nir_opt_dead_cf); + NIR_PASS(progress, nir, nir_opt_remove_phis); + NIR_PASS(progress, nir, nir_opt_peephole_select, 8, true, true); + NIR_PASS(progress, nir, nir_lower_vec3_to_vec4, nir_var_mem_generic | nir_var_uniform); + NIR_PASS(progress, nir, nir_opt_memcpy); + } while (progress); + } + + NIR_PASS_V(nir, nir_scale_fdiv); + + NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_uniform | nir_var_mem_ubo | + nir_var_mem_constant | nir_var_function_temp | nir_var_image, NULL); + + + NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_mem_shared | nir_var_function_temp, NULL); + + nir->scratch_size = 0; + NIR_PASS_V(nir, nir_lower_vars_to_explicit_types, + nir_var_mem_shared | nir_var_function_temp | nir_var_shader_temp | + nir_var_mem_global | nir_var_mem_constant, + glsl_get_cl_type_size_align); + + // Lower memcpy - needs to wait until types are sized + { + bool progress; + do { + progress = false; + NIR_PASS(progress, nir, nir_opt_memcpy); + NIR_PASS(progress, nir, nir_copy_prop); + NIR_PASS(progress, nir, nir_opt_copy_prop_vars); + NIR_PASS(progress, nir, nir_opt_deref); + NIR_PASS(progress, nir, nir_opt_dce); + NIR_PASS(progress, nir, nir_split_var_copies); + NIR_PASS(progress, nir, nir_lower_var_copies); + NIR_PASS(progress, nir, nir_lower_vars_to_ssa); + NIR_PASS(progress, nir, nir_opt_constant_folding); + NIR_PASS(progress, nir, nir_opt_cse); + } while (progress); + } + NIR_PASS_V(nir, nir_lower_memcpy); + + NIR_PASS_V(nir, nir_lower_explicit_io, + nir_var_mem_shared | nir_var_function_temp | nir_var_shader_temp | nir_var_uniform, + nir_address_format_32bit_offset_as_64bit); + + NIR_PASS_V(nir, nir_lower_system_values); + + /* Hopefully we can drop this once lower_vars_to_ssa has improved to not + * lower everything to scratch. + */ + if (llvm17_wa) + cleanup_llvm17_scratch(nir); + + /* Lower again, this time after dead-variables to get more compact variable + * layouts. + */ + nir->global_mem_size = 0; + nir->scratch_size = 0; + nir->info.shared_size = 0; + NIR_PASS_V(nir, nir_lower_vars_to_explicit_types, + nir_var_mem_shared | nir_var_mem_global | nir_var_mem_constant, + glsl_get_cl_type_size_align); + if (nir->constant_data_size > 0) { + assert(nir->constant_data == NULL); + nir->constant_data = rzalloc_size(nir, nir->constant_data_size); + nir_gather_explicit_io_initializers(nir, nir->constant_data, + nir->constant_data_size, + nir_var_mem_constant); + } + + NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_constant, + nir_address_format_64bit_global); + + NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_uniform, + nir_address_format_32bit_offset_as_64bit); + + NIR_PASS_V(nir, nir_lower_explicit_io, + nir_var_shader_temp | nir_var_function_temp | + nir_var_mem_shared | nir_var_mem_global, + nir_address_format_62bit_generic); + + if (INTEL_DEBUG(DEBUG_CS)) { + /* Re-index SSA defs so we print more sensible numbers. */ + nir_foreach_function_impl(impl, nir) { + nir_index_ssa_defs(impl); + } + + fprintf(stderr, "NIR (before I/O lowering) for kernel\n"); + nir_print_shader(nir, stderr); + } + + return nir; +} diff --git a/src/intel/compiler/elk/elk_nir.h b/src/intel/compiler/elk/elk_nir.h index 550bf8a556b..48c93eecacc 100644 --- a/src/intel/compiler/elk/elk_nir.h +++ b/src/intel/compiler/elk/elk_nir.h @@ -274,6 +274,10 @@ const struct glsl_type *elk_nir_get_var_type(const struct nir_shader *nir, void elk_nir_adjust_payload(nir_shader *shader); +nir_shader * +elk_nir_from_spirv(void *mem_ctx, unsigned gfx_version, const uint32_t *spirv, + size_t spirv_size, bool llvm17_wa); + #ifdef __cplusplus } #endif diff --git a/src/intel/compiler/elk/elk_spirv.c b/src/intel/compiler/elk/elk_spirv.c new file mode 100644 index 00000000000..745b824a992 --- /dev/null +++ b/src/intel/compiler/elk/elk_spirv.c @@ -0,0 +1,359 @@ +/* + * Copyright © 2025 Intel Corporation + * SPDX-License-Identifier: MIT + */ + +#include "elk_nir.h" +#include "elk_nir_options.h" + +#include "../intel_nir.h" +#include "compiler/nir/nir_builder.h" +#include "compiler/spirv/nir_spirv.h" +#include "compiler/spirv/spirv_info.h" +#include "dev/intel_debug.h" +#include "util/u_dynarray.h" + +static nir_def * +rebuild_value_from_store(struct util_dynarray *stores, + nir_def *value, unsigned read_offset) +{ + unsigned read_size = value->num_components * value->bit_size / 8; + + util_dynarray_foreach(stores, nir_intrinsic_instr *, _store) { + nir_intrinsic_instr *store = *_store; + + unsigned write_offset = nir_src_as_uint(store->src[1]); + unsigned write_size = nir_src_num_components(store->src[0]) * + nir_src_bit_size(store->src[0]) / 8; + if (write_offset <= read_offset && + (write_offset + write_size) >= (read_offset + read_size)) { + assert(nir_block_dominates(store->instr.block, value->parent_instr->block)); + assert(write_size == read_size); + return store->src[0].ssa; + } + } + unreachable("Matching scratch store not found"); +} + +/** + * Remove temporary variables stored to scratch to be then reloaded + * immediately. Remap the load to the store SSA value. + * + * This workaround is only meant to be applied to shaders in src/intel/shaders + * were we know there should be no issue. More complex cases might not work + * with this approach. + */ +static bool +nir_remove_llvm17_scratch(nir_shader *nir) +{ + struct util_dynarray scratch_stores; + void *mem_ctx = ralloc_context(NULL); + + util_dynarray_init(&scratch_stores, mem_ctx); + + nir_foreach_function_impl(func, nir) { + nir_foreach_block(block, func) { + nir_foreach_instr(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + + nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); + + if (intrin->intrinsic != nir_intrinsic_store_scratch) + continue; + + nir_const_value *offset = nir_src_as_const_value(intrin->src[1]); + if (offset != NULL) { + util_dynarray_append(&scratch_stores, nir_intrinsic_instr *, intrin); + } + } + } + } + + bool progress = false; + if (util_dynarray_num_elements(&scratch_stores, nir_intrinsic_instr *) > 0) { + nir_foreach_function_impl(func, nir) { + nir_foreach_block(block, func) { + nir_foreach_instr_safe(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + + nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); + + if (intrin->intrinsic != nir_intrinsic_load_scratch) + continue; + + nir_const_value *offset = nir_src_as_const_value(intrin->src[0]); + if (offset == NULL) + continue; + + nir_def_replace(&intrin->def, + rebuild_value_from_store(&scratch_stores, &intrin->def, nir_src_as_uint(intrin->src[0]))); + + progress = true; + } + } + } + } + + util_dynarray_foreach(&scratch_stores, nir_intrinsic_instr *, _store) { + nir_intrinsic_instr *store = *_store; + nir_instr_remove(&store->instr); + } + + /* Quick sanity check */ + assert(util_dynarray_num_elements(&scratch_stores, nir_intrinsic_instr *) == 0 || + progress); + + ralloc_free(mem_ctx); + + return progress; +} + +static void +cleanup_llvm17_scratch(nir_shader *nir) +{ + { + bool progress; + do { + progress = false; + NIR_PASS(progress, nir, nir_copy_prop); + NIR_PASS(progress, nir, nir_opt_dce); + NIR_PASS(progress, nir, nir_opt_constant_folding); + NIR_PASS(progress, nir, nir_opt_cse); + NIR_PASS(progress, nir, nir_opt_algebraic); + } while (progress); + } + + nir_remove_llvm17_scratch(nir); + + { + bool progress; + do { + progress = false; + NIR_PASS(progress, nir, nir_copy_prop); + NIR_PASS(progress, nir, nir_opt_dce); + NIR_PASS(progress, nir, nir_opt_constant_folding); + NIR_PASS(progress, nir, nir_opt_cse); + NIR_PASS(progress, nir, nir_opt_algebraic); + } while (progress); + } +} + +static const struct spirv_capabilities spirv_caps = { + .Addresses = true, + .Float16 = true, + .Float64 = true, + .Groups = true, + .StorageImageWriteWithoutFormat = true, + .Int8 = true, + .Int16 = true, + .Int64 = true, + .Int64Atomics = true, + .Kernel = true, + .Linkage = true, /* We receive linked kernel from clc */ + .DenormFlushToZero = true, + .DenormPreserve = true, + .SignedZeroInfNanPreserve = true, + .RoundingModeRTE = true, + .RoundingModeRTZ = true, + .GenericPointer = true, + .GroupNonUniform = true, + .GroupNonUniformArithmetic = true, + .GroupNonUniformClustered = true, + .GroupNonUniformBallot = true, + .GroupNonUniformQuad = true, + .GroupNonUniformShuffle = true, + .GroupNonUniformVote = true, + .SubgroupDispatch = true, +}; + +nir_shader * +elk_nir_from_spirv(void *mem_ctx, unsigned gfx_version, const uint32_t *spirv, + size_t spirv_size, bool llvm17_wa) +{ + assert(gfx_version < 9); + + struct spirv_to_nir_options spirv_options = { + .environment = NIR_SPIRV_OPENCL, + .capabilities = &spirv_caps, + .printf = true, + .shared_addr_format = nir_address_format_62bit_generic, + .global_addr_format = nir_address_format_62bit_generic, + .temp_addr_format = nir_address_format_62bit_generic, + .constant_addr_format = nir_address_format_64bit_global, + .create_library = true, + }; + + assert(spirv_size % 4 == 0); + + const nir_shader_compiler_options *nir_options = &elk_scalar_nir_options; + + nir_shader *nir = + spirv_to_nir(spirv, spirv_size / 4, NULL, 0, MESA_SHADER_KERNEL, + "library", &spirv_options, nir_options); + nir_validate_shader(nir, "after spirv_to_nir"); + nir_validate_ssa_dominance(nir, "after spirv_to_nir"); + ralloc_steal(mem_ctx, nir); + nir->info.name = ralloc_strdup(nir, "library"); + + if (INTEL_DEBUG(DEBUG_CS)) { + /* Re-index SSA defs so we print more sensible numbers. */ + nir_foreach_function_impl(impl, nir) { + nir_index_ssa_defs(impl); + } + + fprintf(stderr, "NIR (from SPIR-V) for kernel\n"); + nir_print_shader(nir, stderr); + } + + nir_lower_printf_options printf_opts = { + .ptr_bit_size = 64, + .use_printf_base_identifier = true, + }; + NIR_PASS_V(nir, nir_lower_printf, &printf_opts); + + NIR_PASS_V(nir, nir_link_shader_functions, spirv_options.clc_shader); + + /* We have to lower away local constant initializers right before we + * inline functions. That way they get properly initialized at the top + * of the function and not at the top of its caller. + */ + NIR_PASS_V(nir, nir_lower_variable_initializers, ~(nir_var_shader_temp | + nir_var_function_temp)); + NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_uniform | nir_var_mem_ubo | + nir_var_mem_constant | nir_var_function_temp | nir_var_image, NULL); + { + bool progress; + do + { + progress = false; + NIR_PASS(progress, nir, nir_copy_prop); + NIR_PASS(progress, nir, nir_opt_copy_prop_vars); + NIR_PASS(progress, nir, nir_opt_deref); + NIR_PASS(progress, nir, nir_opt_dce); + NIR_PASS(progress, nir, nir_opt_undef); + NIR_PASS(progress, nir, nir_opt_constant_folding); + NIR_PASS(progress, nir, nir_opt_cse); + NIR_PASS(progress, nir, nir_lower_vars_to_ssa); + NIR_PASS(progress, nir, nir_opt_algebraic); + } while (progress); + } + + NIR_PASS_V(nir, nir_lower_variable_initializers, nir_var_function_temp); + NIR_PASS_V(nir, nir_lower_returns); + NIR_PASS_V(nir, nir_inline_functions); + + assert(nir->scratch_size == 0); + NIR_PASS_V(nir, nir_lower_vars_to_explicit_types, nir_var_function_temp, glsl_get_cl_type_size_align); + + { + bool progress; + do + { + progress = false; + NIR_PASS(progress, nir, nir_copy_prop); + NIR_PASS(progress, nir, nir_opt_copy_prop_vars); + NIR_PASS(progress, nir, nir_opt_deref); + NIR_PASS(progress, nir, nir_opt_dce); + NIR_PASS(progress, nir, nir_opt_undef); + NIR_PASS(progress, nir, nir_opt_constant_folding); + NIR_PASS(progress, nir, nir_opt_cse); + NIR_PASS(progress, nir, nir_split_var_copies); + NIR_PASS(progress, nir, nir_lower_var_copies); + NIR_PASS(progress, nir, nir_lower_vars_to_ssa); + NIR_PASS(progress, nir, nir_opt_algebraic); + NIR_PASS(progress, nir, nir_opt_if, nir_opt_if_optimize_phi_true_false); + NIR_PASS(progress, nir, nir_opt_dead_cf); + NIR_PASS(progress, nir, nir_opt_remove_phis); + NIR_PASS(progress, nir, nir_opt_peephole_select, 8, true, true); + NIR_PASS(progress, nir, nir_lower_vec3_to_vec4, nir_var_mem_generic | nir_var_uniform); + NIR_PASS(progress, nir, nir_opt_memcpy); + } while (progress); + } + + NIR_PASS_V(nir, nir_scale_fdiv); + + NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_uniform | nir_var_mem_ubo | + nir_var_mem_constant | nir_var_function_temp | nir_var_image, NULL); + + + NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_mem_shared | nir_var_function_temp, NULL); + + nir->scratch_size = 0; + NIR_PASS_V(nir, nir_lower_vars_to_explicit_types, + nir_var_mem_shared | nir_var_function_temp | nir_var_shader_temp | + nir_var_mem_global | nir_var_mem_constant, + glsl_get_cl_type_size_align); + + // Lower memcpy - needs to wait until types are sized + { + bool progress; + do { + progress = false; + NIR_PASS(progress, nir, nir_opt_memcpy); + NIR_PASS(progress, nir, nir_copy_prop); + NIR_PASS(progress, nir, nir_opt_copy_prop_vars); + NIR_PASS(progress, nir, nir_opt_deref); + NIR_PASS(progress, nir, nir_opt_dce); + NIR_PASS(progress, nir, nir_split_var_copies); + NIR_PASS(progress, nir, nir_lower_var_copies); + NIR_PASS(progress, nir, nir_lower_vars_to_ssa); + NIR_PASS(progress, nir, nir_opt_constant_folding); + NIR_PASS(progress, nir, nir_opt_cse); + } while (progress); + } + NIR_PASS_V(nir, nir_lower_memcpy); + + NIR_PASS_V(nir, nir_lower_explicit_io, + nir_var_mem_shared | nir_var_function_temp | nir_var_shader_temp | nir_var_uniform, + nir_address_format_32bit_offset_as_64bit); + + NIR_PASS_V(nir, nir_lower_system_values); + + /* Hopefully we can drop this once lower_vars_to_ssa has improved to not + * lower everything to scratch. + */ + if (llvm17_wa) + cleanup_llvm17_scratch(nir); + + /* Lower again, this time after dead-variables to get more compact variable + * layouts. + */ + nir->global_mem_size = 0; + nir->scratch_size = 0; + nir->info.shared_size = 0; + NIR_PASS_V(nir, nir_lower_vars_to_explicit_types, + nir_var_mem_shared | nir_var_mem_global | nir_var_mem_constant, + glsl_get_cl_type_size_align); + if (nir->constant_data_size > 0) { + assert(nir->constant_data == NULL); + nir->constant_data = rzalloc_size(nir, nir->constant_data_size); + nir_gather_explicit_io_initializers(nir, nir->constant_data, + nir->constant_data_size, + nir_var_mem_constant); + } + + NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_constant, + nir_address_format_64bit_global); + + NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_uniform, + nir_address_format_32bit_offset_as_64bit); + + NIR_PASS_V(nir, nir_lower_explicit_io, + nir_var_shader_temp | nir_var_function_temp | + nir_var_mem_shared | nir_var_mem_global, + nir_address_format_62bit_generic); + + if (INTEL_DEBUG(DEBUG_CS)) { + /* Re-index SSA defs so we print more sensible numbers. */ + nir_foreach_function_impl(impl, nir) { + nir_index_ssa_defs(impl); + } + + fprintf(stderr, "NIR (before I/O lowering) for kernel\n"); + nir_print_shader(nir, stderr); + } + + return nir; +} diff --git a/src/intel/compiler/elk/meson.build b/src/intel/compiler/elk/meson.build index 310b1423385..c7fc5fbbb92 100644 --- a/src/intel/compiler/elk/meson.build +++ b/src/intel/compiler/elk/meson.build @@ -85,6 +85,7 @@ libintel_compiler_elk_files = files( 'elk_shader.cpp', 'elk_shader.h', 'elk_simd_selection.cpp', + 'elk_spirv.c', 'elk_vec4_builder.h', 'elk_vec4_cmod_propagation.cpp', 'elk_vec4_copy_propagation.cpp', @@ -139,6 +140,7 @@ idep_intel_compiler_elk = declare_dependency( link_with : [libintel_compiler_elk], dependencies : [ idep_nir, + idep_vtn, ], ) diff --git a/src/intel/compiler/intel_clc.c b/src/intel/compiler/intel_clc.c index ba72951f702..dd2ee81974a 100644 --- a/src/intel/compiler/intel_clc.c +++ b/src/intel/compiler/intel_clc.c @@ -23,6 +23,8 @@ #include "brw_compiler.h" #include "brw_kernel.h" +#include "brw_nir.h" +#include "elk/elk_nir.h" #include "compiler/brw_disasm.h" #include "compiler/clc/clc.h" #include "compiler/glsl_types.h" @@ -387,9 +389,13 @@ output_nir(const struct intel_clc_params *params, struct clc_binary *binary) spirv_library_to_nir_builder(fp, binary->data, binary->size / 4, &spirv_options); - nir_shader *nir = brw_nir_from_spirv(params->mem_ctx, params->gfx_version, - binary->data, binary->size, - params->llvm17_wa); + nir_shader *nir = params->gfx_version >= 9 ? + brw_nir_from_spirv(params->mem_ctx, params->gfx_version, + binary->data, binary->size, + params->llvm17_wa) : + elk_nir_from_spirv(params->mem_ctx, params->gfx_version, + binary->data, binary->size, + params->llvm17_wa); if (!nir) { fprintf(stderr, "Failed to generate NIR out of SPIRV\n"); fclose(fp); diff --git a/src/intel/compiler/meson.build b/src/intel/compiler/meson.build index 57d3bfed1d2..50b95b724ea 100644 --- a/src/intel/compiler/meson.build +++ b/src/intel/compiler/meson.build @@ -113,6 +113,7 @@ libintel_compiler_brw_files = files( 'brw_schedule_instructions.cpp', 'brw_shader.cpp', 'brw_simd_selection.cpp', + 'brw_spirv.c', 'brw_validate.cpp', 'brw_vue_map.c', 'brw_workaround.cpp', @@ -161,6 +162,7 @@ idep_intel_compiler_brw = declare_dependency( dependencies : [ idep_nir, idep_mesautil, + idep_vtn, ], ) @@ -178,12 +180,13 @@ elif with_intel_clc # Use just the nir_options part of ELK instead of fully linking. 'elk/elk_nir_options.h', 'elk/elk_nir_options.c', + 'elk/elk_spirv.c', ], link_with : [libisl], include_directories : [inc_include, inc_src, inc_intel], c_args : [pre_args, no_override_init_args], link_args : [ld_args_build_id], - dependencies : [idep_nir, idep_vtn, idep_mesaclc, idep_mesautil, idep_intel_dev, + dependencies : [idep_nir, idep_mesaclc, idep_mesautil, idep_intel_dev, idep_intel_compiler_brw], # If we can run host binaries directly, just build intel_clc for the host. # Most commonly this happens when doing a cross compile from an x86_64 build