diff --git a/meson.build b/meson.build index d1fc8fccce9..f7ad909d77a 100644 --- a/meson.build +++ b/meson.build @@ -816,7 +816,7 @@ with_driver_using_cl = [ with_gallium_iris, with_intel_vk, with_gallium_asahi, with_asahi_vk, with_gallium_panfrost, with_panfrost_vk, - with_nouveau_vk, + with_nouveau_vk, with_imagination_vk, ].contains(true) if get_option('mesa-clc') == 'system' diff --git a/meson.options b/meson.options index 48362ca693b..02b921454c3 100644 --- a/meson.options +++ b/meson.options @@ -241,6 +241,13 @@ option( 'vulkan driver', ) +option('imagination-uscgen-devices', + type : 'array', + value : [], + choices : ['axe-1-16m', 'bxs-4-64', 'gx6250'], + description : 'List of devices for which to pre-build USC program binaries.', +) + option( 'shader-cache', type : 'feature', diff --git a/src/imagination/.clang-format b/src/imagination/.clang-format index d660e17a4c1..abb996fd48c 100644 --- a/src/imagination/.clang-format +++ b/src/imagination/.clang-format @@ -206,6 +206,7 @@ ForEachMacros: [ 'foreach_list_typed_reverse', 'foreach_list_typed_reverse_safe', 'foreach_list_typed_safe', + 'foreach_target', 'hash_table_foreach', 'hash_table_u64_foreach', 'LIST_FOR_EACH_ENTRY', @@ -227,6 +228,8 @@ ForEachMacros: [ 'nir_foreach_block', 'nir_foreach_block_safe', 'nir_foreach_block_unstructured', + 'nir_foreach_entrypoint', + 'nir_foreach_entrypoint_safe', 'nir_foreach_function', 'nir_foreach_function_with_impl', 'nir_foreach_instr', diff --git a/src/imagination/common/meson.build b/src/imagination/common/meson.build index 4a2ece442bf..652e916f89f 100644 --- a/src/imagination/common/meson.build +++ b/src/imagination/common/meson.build @@ -1,6 +1,8 @@ # Copyright © 2022 Imagination Technologies Ltd. # SPDX-License-Identifier: MIT +pvr_device_info_dir = join_paths(meson.current_source_dir(), 'device_info') + libpowervr_common = static_library( 'powervr_common', [ diff --git a/src/imagination/meson.build b/src/imagination/meson.build index 5e7dcd95895..c715d0fb745 100644 --- a/src/imagination/meson.build +++ b/src/imagination/meson.build @@ -11,10 +11,13 @@ inc_imagination = include_directories([ 'include', ]) -if with_imagination_vk +if with_imagination_vk or with_tools.contains('imagination') subdir('common') subdir('csbgen') subdir('pco') subdir('rogue') +endif + +if with_imagination_vk subdir('vulkan') endif diff --git a/src/imagination/pco/meson.build b/src/imagination/pco/meson.build index 4f18154e56d..33d1a1cc762 100644 --- a/src/imagination/pco/meson.build +++ b/src/imagination/pco/meson.build @@ -1,6 +1,8 @@ # Copyright © 2024 Imagination Technologies Ltd. # SPDX-License-Identifier: MIT +subdir('usclib') + inc_powervr_compiler = include_directories(['.']) libpowervr_compiler_files = files( @@ -121,7 +123,9 @@ libpowervr_compiler = static_library( ], # Suppress 'parameter passing for argument of type ... changed in GCC ...' warnings. c_args : [imagination_c_args, no_override_init_args, '-Wno-psabi'], - dependencies : [idep_mesautil, idep_nir, idep_pco_pygen], + dependencies : [idep_mesautil, idep_nir, idep_pco_pygen, idep_pco_usclib], gnu_symbol_visibility : 'hidden', install : false, ) + +subdir('uscgen') diff --git a/src/imagination/pco/pco.c b/src/imagination/pco/pco.c index 2dc03cfb8f5..37076aac8d1 100644 --- a/src/imagination/pco/pco.c +++ b/src/imagination/pco/pco.c @@ -12,8 +12,10 @@ #include "compiler/list.h" #include "compiler/glsl_types.h" +#include "nir_serialize.h" #include "pco.h" #include "pco_internal.h" +#include "util/blob.h" #include "util/hash_table.h" #include "util/list.h" #include "util/macros.h" @@ -58,6 +60,13 @@ pco_ctx *pco_ctx_create(const struct pvr_device_info *dev_info, void *mem_ctx) return ctx; } +void pco_ctx_setup_usclib(pco_ctx *ctx, const void *data, unsigned size) +{ + struct blob_reader blob_reader; + blob_reader_init(&blob_reader, data, size); + ctx->usclib = nir_deserialize(ctx, pco_nir_options(), &blob_reader); +} + /** * \brief Updates the device info for a PCO compiler context. * @@ -291,3 +300,21 @@ pco_data *pco_shader_data(pco_shader *shader) { return &shader->data; } + +/** + * \brief Returns precompilation data for a shader. + * + * \param[in] shader PCO shader. + * \return The precompilation data. + */ +pco_precomp_data pco_get_precomp_data(pco_shader *shader) +{ + assert(pco_shader_binary_size(shader)); + + return (pco_precomp_data){ + .temps = shader->data.common.temps, + .vtxins = shader->data.common.vtxins, + .coeffs = shader->data.common.coeffs, + .shareds = shader->data.common.shareds, + }; +} diff --git a/src/imagination/pco/pco.h b/src/imagination/pco/pco.h index 70441c05528..bc3d7c132ac 100644 --- a/src/imagination/pco/pco.h +++ b/src/imagination/pco/pco.h @@ -27,6 +27,7 @@ typedef struct _pco_ctx pco_ctx; typedef struct _pco_data pco_data; pco_ctx *pco_ctx_create(const struct pvr_device_info *dev_info, void *mem_ctx); +void pco_ctx_setup_usclib(pco_ctx *ctx, const void *data, unsigned size); void pco_ctx_update_dev_info(pco_ctx *ctx, const struct pvr_device_info *dev_info); const struct spirv_to_nir_options *pco_spirv_options(void); diff --git a/src/imagination/pco/pco_data.h b/src/imagination/pco/pco_data.h index e4b6d991706..5e1c1dc476d 100644 --- a/src/imagination/pco/pco_data.h +++ b/src/imagination/pco/pco_data.h @@ -19,6 +19,9 @@ #include +/* Compiler-specific forward-declarations. */ +typedef struct _pco_shader pco_shader; + /** Generic range struct. */ typedef struct _pco_range { unsigned start; @@ -159,4 +162,17 @@ typedef struct _pco_data { pco_common_data common; } pco_data; +/** PCO precompiled shader data. */ +typedef struct PACKED _pco_precomp_data { + uint8_t temps; + uint8_t vtxins; + + uint16_t coeffs : 12; + uint16_t shareds : 12; + + uint8_t pad[3]; +} pco_precomp_data; +static_assert(sizeof(pco_precomp_data) == 8, "sizeof(pco_precomp_data) != 8"); + +pco_precomp_data pco_get_precomp_data(pco_shader *shader); #endif /* PCO_DATA_H */ diff --git a/src/imagination/pco/pco_internal.h b/src/imagination/pco/pco_internal.h index c77ff7423fa..a54e22e808f 100644 --- a/src/imagination/pco/pco_internal.h +++ b/src/imagination/pco/pco_internal.h @@ -43,6 +43,9 @@ typedef struct _pco_ctx { /** Device-specific SPIR-V to NIR options. */ struct spirv_to_nir_options spirv_options; + + /** USC library. */ + const nir_shader *usclib; } pco_ctx; void pco_setup_spirv_options(const struct pvr_device_info *dev_info, diff --git a/src/imagination/pco/pco_nir.c b/src/imagination/pco/pco_nir.c index d4bf942020e..d54e5052be8 100644 --- a/src/imagination/pco/pco_nir.c +++ b/src/imagination/pco/pco_nir.c @@ -627,6 +627,8 @@ void pco_rev_link_nir(pco_ctx *ctx, nir_shader *producer, nir_shader *consumer) */ void pco_lower_nir(pco_ctx *ctx, nir_shader *nir, pco_data *data) { + bool uses_usclib = false; + NIR_PASS(_, nir, nir_opt_access, @@ -710,6 +712,27 @@ void pco_lower_nir(pco_ctx *ctx, nir_shader *nir, pco_data *data) NIR_PASS(_, nir, pco_nir_pvi, &data->vs); } + if (uses_usclib) { + assert(ctx->usclib); + + nir_link_shader_functions(nir, ctx->usclib); + NIR_PASS(_, nir, nir_inline_functions); + nir_remove_non_entrypoints(nir); + NIR_PASS(_, nir, nir_opt_deref); + NIR_PASS(_, nir, nir_lower_vars_to_ssa); + NIR_PASS(_, nir, nir_remove_dead_derefs); + NIR_PASS(_, + nir, + nir_remove_dead_variables, + nir_var_function_temp | nir_var_shader_temp, + NULL); + NIR_PASS(_, + nir, + nir_lower_vars_to_explicit_types, + nir_var_shader_temp | nir_var_function_temp, + glsl_get_cl_type_size_align); + } + NIR_PASS(_, nir, nir_lower_io_to_scalar, diff --git a/src/imagination/pco/uscgen/meson.build b/src/imagination/pco/uscgen/meson.build new file mode 100644 index 00000000000..49109b07b52 --- /dev/null +++ b/src/imagination/pco/uscgen/meson.build @@ -0,0 +1,57 @@ +# Copyright © 2025 Imagination Technologies Ltd. +# based in part on intel driver which is: +# Copyright 2017 Intel Corporation +# SPDX-License-Identifier: MIT + +if get_option('precomp-compiler') == 'system' + prog_pco_clc = find_program('pco_clc', native : true) +else + prog_pco_clc = executable( + 'pco_clc', + ['pco_clc.c'], + link_with : [libpowervr_compiler, libpowervr_common], + include_directories : [inc_imagination, inc_include, inc_src], + c_args : [pre_args, no_override_init_args], + link_args : [ld_args_build_id], + dependencies : [idep_mesaclc, dep_llvm, dep_spirv_tools, idep_nir, idep_mesautil], + # If we can run host binaries directly, just build pco_clc for the host. + # Most commonly this happens when doing a cross compile from an x86_64 build + # machine to an x86 host + native : not meson.can_run_host_binaries(), + install : get_option('install-precomp-compiler'), + ) +endif + +### + +uscgen_devices = get_option('imagination-uscgen-devices') +foreach uscgen_device : uscgen_devices + device_info_path = join_paths(pvr_device_info_dir, uscgen_device + '.h') + if not fs.is_file(device_info_path) + error('Missing device info for ' + uscgen_device) + endif +endforeach + +pco_uscgen_programs = custom_target( + 'pco_uscgen_programs', + input : pco_usclib_spv, + output : ['pco_uscgen_programs.h', 'pco_uscgen_programs.c'], + command : [prog_pco_clc, pco_usclib_spv, '@OUTPUT@', uscgen_devices], + env: ['MESA_SHADER_CACHE_DISABLE=true'], +) + +idep_pco_uscgen_programs_h = declare_dependency( + sources : [pco_uscgen_programs], + include_directories : include_directories('.'), +) + +### + +libpowervr_uscgen = static_library( + 'powervr_uscgen', + [pco_uscgen_programs], + include_directories : [inc_imagination], + c_args : [no_override_init_args], + gnu_symbol_visibility : 'hidden', + dependencies: [idep_nir, idep_mesautil], +) diff --git a/src/imagination/pco/uscgen/pco_clc.c b/src/imagination/pco/uscgen/pco_clc.c new file mode 100644 index 00000000000..e8a0211ad34 --- /dev/null +++ b/src/imagination/pco/uscgen/pco_clc.c @@ -0,0 +1,562 @@ +/* + * Copyright © 2025 Imagination Technologies Ltd. + * Copyright 2023 Alyssa Rosenzweig + * Copyright 2020 Intel Corporation + * SPDX-License-Identifier: MIT + */ + +#include "common/pvr_device_info.h" +#include "compiler/glsl_types.h" +#include "compiler/shader_enums.h" +#include "compiler/spirv/nir_spirv.h" +#include "nir/nir.h" +#include "nir/nir_builder.h" +#include "nir/nir_builder_opcodes.h" +#include "nir/nir_intrinsics.h" +#include "nir/nir_precompiled.h" +#include "pco/pco.h" +#include "pco/pco_data.h" +#include "util/macros.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define CLC_PREFIX "pco_usclib" + +#define VS_PREFIX "vs_" +#define FS_PREFIX "fs_" +#define CS_PREFIX "cs_" +#define COMMON_SUFFIX "_common" +#define COMMON_DEVICE "common" + +static const struct pvr_device_info pvr_device_info_common = { + .ident = + (struct pvr_device_ident){ + .device_id = 0, + .series_name = COMMON_DEVICE, + .public_name = COMMON_DEVICE, + }, + + .features = + (struct pvr_device_features){ + .has_common_store_size_in_dwords = true, + .has_compute = true, + .has_ipf_creq_pf = true, + .has_isp_max_tiles_in_flight = true, + .has_isp_samples_per_pixel = true, + .has_max_instances_per_pds_task = true, + .has_max_multisample = true, + .has_max_partitions = true, + .has_max_usc_tasks = true, + .has_num_clusters = true, + .has_num_raster_pipes = true, + .has_pbe2_in_xe = true, + .has_pbe_filterable_f16 = true, + .has_pbe_yuv = true, + .has_roguexe = true, + .has_screen_size8K = true, + .has_simple_internal_parameter_format = true, + .has_simple_internal_parameter_format_v2 = true, + .has_simple_parameter_format_version = true, + .has_slc_cache_line_size_bits = true, + .has_tile_size_16x16 = true, + .has_tile_size_x = true, + .has_tile_size_y = true, + .has_tpu_border_colour_enhanced = true, + .has_tpu_extended_integer_lookup = true, + .has_tpu_image_state_v2 = true, + .has_unified_store_depth = true, + .has_usc_f16sop_u8 = true, + .has_usc_min_output_registers_per_pix = true, + .has_usc_pixel_partition_mask = true, + .has_usc_slots = true, + .has_uvs_banks = true, + .has_uvs_pba_entries = true, + .has_uvs_vtx_entries = true, + .has_vdm_cam_size = true, + .has_vdm_degenerate_culling = true, + + .common_store_size_in_dwords = 512U * 4U * 4U, + .isp_max_tiles_in_flight = 1U, + .isp_samples_per_pixel = 1U, + .max_instances_per_pds_task = 32U, + .max_multisample = 4U, + .max_partitions = 4U, + .max_usc_tasks = 24U, + .num_clusters = 1U, + .num_raster_pipes = 1U, + .simple_parameter_format_version = 2U, + .slc_cache_line_size_bits = 512U, + .tile_size_x = 16U, + .tile_size_y = 16U, + .unified_store_depth = 64U, + .usc_min_output_registers_per_pix = 1U, + .usc_slots = 14U, + .uvs_banks = 2U, + .uvs_pba_entries = 320U, + .uvs_vtx_entries = 288U, + .vdm_cam_size = 32U, + + .has_s8xe = true, + .has_usc_itr_parallel_instances = true, + + .usc_itr_parallel_instances = 4U, + }, + + .enhancements = (struct pvr_device_enhancements){ 0 }, + .quirks = (struct pvr_device_quirks){ 0 }, +}; + +/* Standard optimization loop */ +static void optimize(nir_shader *nir) +{ + bool progress; + do { + progress = false; + + NIR_PASS(progress, nir, nir_split_var_copies); + NIR_PASS(progress, nir, nir_split_struct_vars, nir_var_function_temp); + NIR_PASS(progress, nir, nir_lower_var_copies); + NIR_PASS(progress, nir, nir_lower_vars_to_ssa); + + NIR_PASS(progress, nir, nir_copy_prop); + NIR_PASS(progress, nir, nir_opt_remove_phis); + NIR_PASS(progress, nir, nir_lower_all_phis_to_scalar); + NIR_PASS(progress, nir, nir_opt_dce); + NIR_PASS(progress, nir, nir_opt_dead_cf); + NIR_PASS(progress, nir, nir_opt_cse); + nir_opt_peephole_select_options peep_opts = { + .limit = 64, + .expensive_alu_ok = true, + }; + NIR_PASS(progress, nir, nir_opt_peephole_select, &peep_opts); + NIR_PASS(progress, nir, nir_opt_phi_precision); + NIR_PASS(progress, nir, nir_opt_algebraic); + NIR_PASS(progress, nir, nir_opt_constant_folding); + + NIR_PASS(progress, nir, nir_opt_deref); + NIR_PASS(progress, nir, nir_opt_copy_prop_vars); + NIR_PASS(progress, nir, nir_opt_undef); + NIR_PASS(progress, nir, nir_lower_undef_to_zero); + + NIR_PASS(progress, nir, nir_opt_shrink_vectors, true); + NIR_PASS(progress, nir, nir_opt_loop_unroll); + + } while (progress); +} + +static nir_shader * +spv_to_nir(void *mem_ctx, uint32_t *spirv_map, unsigned spirv_len) +{ + static const struct spirv_to_nir_options precomp_spirv_options = { + .environment = NIR_SPIRV_OPENCL, + .shared_addr_format = nir_address_format_62bit_generic, + .global_addr_format = nir_address_format_62bit_generic, + .temp_addr_format = nir_address_format_62bit_generic, + .constant_addr_format = nir_address_format_64bit_global, + .create_library = true, + }; + + nir_shader *nir = spirv_to_nir(spirv_map, + spirv_len / 4, + NULL, + 0, + MESA_SHADER_KERNEL, + "library", + &precomp_spirv_options, + pco_nir_options()); + + nir_validate_shader(nir, "after spirv_to_nir"); + nir_validate_ssa_dominance(nir, "after spirv_to_nir"); + ralloc_steal(mem_ctx, nir); + + nir_fixup_is_exported(nir); + + NIR_PASS(_, nir, nir_lower_system_values); + NIR_PASS(_, nir, nir_lower_calls_to_builtins); + + nir_lower_compute_system_values_options cs = { .global_id_is_32bit = true }; + NIR_PASS(_, nir, nir_lower_compute_system_values, &cs); + + /* We have to lower away local constant initializers right before we + * inline functions. That way they get properly initialized at the top + * of the function and not at the top of its caller. + */ + NIR_PASS(_, nir, nir_lower_variable_initializers, nir_var_function_temp); + NIR_PASS(_, nir, nir_lower_returns); + NIR_PASS(_, nir, nir_inline_functions); + nir_remove_non_exported(nir); + NIR_PASS(_, nir, nir_copy_prop); + NIR_PASS(_, nir, nir_opt_deref); + + /* We can't deal with constant data, get rid of it */ + nir_lower_constant_to_temp(nir); + + /* We can go ahead and lower the rest of the constant initializers. We do + * this here so that nir_remove_dead_variables and + * split_per_member_structs below see the corresponding stores. + */ + NIR_PASS(_, nir, nir_lower_variable_initializers, ~0); + + /* LLVM loves to take advantage of the fact that vec3s in OpenCL are 16B + * aligned and so it can just read/write them as vec4s. This results in a + * LOT of vec4->vec3 casts on loads and stores. One solution to this + * problem is to get rid of all vec3 variables. + */ + NIR_PASS(_, + nir, + nir_lower_vec3_to_vec4, + nir_var_shader_temp | nir_var_function_temp | nir_var_mem_shared | + nir_var_mem_global | nir_var_mem_constant); + + /* We assign explicit types early so that the optimizer can take advantage + * of that information and hopefully get rid of some of our memcpys. + */ + NIR_PASS(_, + nir, + nir_lower_vars_to_explicit_types, + nir_var_uniform | nir_var_shader_temp | nir_var_function_temp | + nir_var_mem_shared | nir_var_mem_global, + glsl_get_cl_type_size_align); + + optimize(nir); + + NIR_PASS(_, nir, nir_remove_dead_variables, nir_var_all, NULL); + + /* Lower again, this time after dead-variables to get more compact + * variable layouts. + */ + NIR_PASS(_, + nir, + nir_lower_vars_to_explicit_types, + nir_var_shader_temp | nir_var_function_temp | nir_var_mem_shared | + nir_var_mem_global | nir_var_mem_constant, + glsl_get_cl_type_size_align); + assert(nir->constant_data_size == 0); + + NIR_PASS(_, nir, nir_lower_memcpy); + + NIR_PASS(_, + nir, + nir_lower_explicit_io, + nir_var_mem_constant, + nir_address_format_64bit_global); + + NIR_PASS(_, + nir, + nir_lower_explicit_io, + nir_var_uniform, + nir_address_format_32bit_offset_as_64bit); + + /* Note: we cannot lower explicit I/O here, because we need derefs intact + * for function calls into the library to work. + */ + + NIR_PASS(_, nir, nir_lower_convert_alu_types, NULL); + NIR_PASS(_, nir, nir_opt_if, 0); + NIR_PASS(_, nir, nir_opt_idiv_const, 16); + + optimize(nir); + + return nir; +} + +static nir_def *load_kernel_input(nir_builder *b, + unsigned num_components, + unsigned bit_size, + unsigned base) +{ + return nir_load_preamble(b, num_components, bit_size, .base = base); +} + +/** + * Common function to build a NIR shader and export the binary. + * + * \param ctx PCO context. + * \param nir NIR shader. + * \param data Shader data. + * \return The finalized PCO shader. + */ +static pco_shader *build_shader(pco_ctx *ctx, nir_shader *nir, pco_data *data) +{ + pco_preprocess_nir(ctx, nir); + pco_lower_nir(ctx, nir, data); + pco_postprocess_nir(ctx, nir, data); + + pco_shader *shader = pco_trans_nir(ctx, nir, data, NULL); + pco_process_ir(ctx, shader); + pco_encode_ir(ctx, shader); + + return shader; +} + +static inline mesa_shader_stage get_shader_stage(const char *name) +{ + if (!strncmp(VS_PREFIX, name, strlen(VS_PREFIX))) + return MESA_SHADER_VERTEX; + + if (!strncmp(FS_PREFIX, name, strlen(FS_PREFIX))) + return MESA_SHADER_FRAGMENT; + + if (!strncmp(CS_PREFIX, name, strlen(CS_PREFIX))) + return MESA_SHADER_COMPUTE; + + UNREACHABLE(""); +} + +static inline bool is_shader_common(const char *name) +{ + unsigned name_len = strlen(name); + unsigned suffix_len = strlen(COMMON_SUFFIX); + + if (name_len < suffix_len) + return false; + + return !strcmp(&name[name_len - suffix_len], COMMON_SUFFIX); +} + +static const char * +remap_variant(nir_function *func, UNUSED unsigned variant, const char *target) +{ + return is_shader_common(func->name) ? COMMON_DEVICE : target; +} + +int main(int argc, char *argv[argc]) +{ + if (argc < 4) { + fprintf( + stderr, + "Usage: %s [device(s...)]\n", + argv[0]); + + return 1; + } + + void *mem_ctx = ralloc_context(NULL); + + char *spv_file = argv[1]; + char *hdr_file = argv[2]; + char *src_file = argv[3]; + + unsigned num_devices = argc - 4; + const char **devices = + ralloc_array_size(mem_ctx, sizeof(*devices), num_devices + 1); + char **device_names = + ralloc_array_size(mem_ctx, sizeof(*device_names), num_devices + 1); + uint64_t *device_ids = + ralloc_array_size(mem_ctx, sizeof(*device_ids), num_devices + 1); + struct pvr_device_info *device_infos = + ralloc_array_size(mem_ctx, sizeof(*device_infos), num_devices + 1); + + for (unsigned d = 0; d < num_devices; ++d) { + devices[d] = ralloc_strdup(devices, argv[4 + d]); + device_names[d] = ralloc_strdup(device_names, devices[d]); + for (unsigned u = 0; u < strlen(device_names[d]); ++u) + if (device_names[d][u] == '-') + device_names[d][u] = '_'; + + pvr_device_info_init_public_name(&device_infos[d], devices[d]); + device_ids[d] = pvr_get_packed_bvnc(&device_infos[d]); + } + + /* Common device. */ + devices[num_devices] = ralloc_strdup(devices, COMMON_DEVICE); + device_names[num_devices] = ralloc_strdup(device_names, COMMON_DEVICE); + device_ids[num_devices] = 0U; + memcpy(&device_infos[num_devices], + &pvr_device_info_common, + sizeof(pvr_device_info_common)); + ++num_devices; + + int fd = open(spv_file, O_RDONLY); + if (fd < 0) { + fprintf(stderr, "Failed to open %s\n", spv_file); + goto err_free_mem_ctx; + } + + off_t spirv_len = lseek(fd, 0, SEEK_END); + assert(spirv_len % 4 == 0); + + void *spirv_map = mmap(NULL, spirv_len, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + + if (spirv_map == MAP_FAILED) { + fprintf(stderr, + "Failed to mmap the file: errno=%d, %s\n", + errno, + strerror(errno)); + goto err_free_mem_ctx; + } + + FILE *fp_hdr = fopen(hdr_file, "w"); + if (!fp_hdr) { + fprintf(stderr, "Failed to open %s\n", hdr_file); + goto err_unmap_spirv; + } + + FILE *fp_src = fopen(src_file, "w"); + if (!fp_src) { + fprintf(stderr, "Failed to open %s\n", src_file); + goto err_close_hdr; + } + + nir_precomp_print_header(fp_src, + fp_hdr, + "Imagination Technologies Ltd.", + basename(hdr_file)); + + pco_ctx *ctx = pco_ctx_create(NULL, mem_ctx); + nir_shader *nir = spv_to_nir(mem_ctx, spirv_map, spirv_len); + struct nir_precomp_opts opts = { 0 }; + + nir_precomp_print_target_enum_map(fp_src, + fp_hdr, + CLC_PREFIX, + num_devices, + (const char **)device_names, + device_ids); + + nir_precomp_print_program_enum(fp_hdr, nir, CLC_PREFIX); + nir_precomp_print_dispatch_macros(fp_hdr, &opts, nir); + + nir_foreach_entrypoint (func, nir) { + unsigned num_variants = nir_precomp_nr_variants(func); + + nir_precomp_print_layout_struct(fp_hdr, &opts, func); + + mesa_shader_stage stage = get_shader_stage(func->name); + bool is_common = is_shader_common(func->name); + + for (unsigned variant = 0; variant < num_variants; ++variant) { + nir_shader *s = nir_precompiled_build_variant(func, + stage, + variant, + pco_nir_options(), + &opts, + load_kernel_input); + + nir_link_shader_functions(s, nir); + NIR_PASS(_, s, nir_inline_functions); + nir_remove_non_entrypoints(s); + NIR_PASS(_, s, nir_opt_deref); + NIR_PASS(_, s, nir_lower_vars_to_ssa); + NIR_PASS(_, s, nir_remove_dead_derefs); + NIR_PASS(_, + s, + nir_remove_dead_variables, + nir_var_function_temp | nir_var_shader_temp, + NULL); + NIR_PASS(_, + s, + nir_lower_vars_to_explicit_types, + nir_var_shader_temp | nir_var_function_temp, + glsl_get_cl_type_size_align); + + NIR_PASS(_, + s, + nir_lower_vars_to_explicit_types, + nir_var_mem_shared, + glsl_get_cl_type_size_align); + + NIR_PASS(_, + s, + nir_lower_explicit_io, + nir_var_shader_temp | nir_var_function_temp | + nir_var_mem_shared | nir_var_mem_global, + nir_address_format_62bit_generic); + + /* Unroll loops before lowering indirects */ + bool progress; + do { + progress = false; + NIR_PASS(progress, s, nir_opt_loop); + } while (progress); + + for (unsigned d = 0; d < num_devices; ++d) { + if (is_common && d != (num_devices - 1)) + continue; + + pco_ctx_update_dev_info(ctx, &device_infos[d]); + + nir_shader *clone = nir_shader_clone(NULL, s); + + pco_data data = { 0 }; + pco_shader *shader = build_shader(ctx, clone, &data); + pco_precomp_data precomp_data = pco_get_precomp_data(shader); + + unsigned binary_size = + pco_shader_binary_size(shader) + sizeof(precomp_data); + assert(!(binary_size % sizeof(uint32_t))); + + uint32_t *binary_data = malloc(binary_size); + memcpy(binary_data, &precomp_data, sizeof(precomp_data)); + memcpy((uint8_t *)binary_data + sizeof(precomp_data), + pco_shader_binary_data(shader), + pco_shader_binary_size(shader)); + + nir_precomp_print_blob(fp_src, + func->name, + device_names[d], + variant, + binary_data, + binary_size, + true); + + free(binary_data); + ralloc_free(shader); + ralloc_free(clone); + } + + ralloc_free(s); + } + } + + for (unsigned d = 0; d < num_devices; ++d) { + nir_precomp_print_extern_binary_map(fp_hdr, CLC_PREFIX, device_names[d]); + nir_precomp_print_binary_map(fp_src, + nir, + CLC_PREFIX, + device_names[d], + remap_variant); + } + + nir_precomp_print_target_binary_map(fp_src, + fp_hdr, + CLC_PREFIX, + num_devices, + (const char **)device_names); + + /* Remove common shaders - no need to preserve their NIR. */ + nir_foreach_entrypoint_safe (func, nir) { + if (!is_shader_common(func->name)) + continue; + + exec_node_remove(&func->node); + } + + nir_precomp_print_nir(fp_src, fp_hdr, nir, CLC_PREFIX, "nir"); + + ralloc_free(mem_ctx); + munmap(spirv_map, spirv_len); + fclose(fp_src); + fclose(fp_hdr); + + return 0; + +err_close_hdr: + fclose(fp_hdr); + +err_unmap_spirv: + munmap(spirv_map, spirv_len); + +err_free_mem_ctx: + ralloc_free(mem_ctx); + + return 1; +} diff --git a/src/imagination/pco/usclib/common.cl b/src/imagination/pco/usclib/common.cl new file mode 100644 index 00000000000..baf6128c12b --- /dev/null +++ b/src/imagination/pco/usclib/common.cl @@ -0,0 +1,24 @@ +/* + * Copyright © 2025 Imagination Technologies Ltd. + * SPDX-License-Identifier: MIT + */ + +#include "libcl.h" + +KERNEL(1) +vs_nop_common(void) +{ + return; +} + +KERNEL(1) +fs_nop_common(void) +{ + return; +} + +KERNEL(1) +cs_nop_common(void) +{ + return; +} diff --git a/src/imagination/pco/usclib/libcl.h b/src/imagination/pco/usclib/libcl.h new file mode 100644 index 00000000000..21da40b5225 --- /dev/null +++ b/src/imagination/pco/usclib/libcl.h @@ -0,0 +1,43 @@ +/* + * Copyright © 2025 Imagination Technologies Ltd. + * SPDX-License-Identifier: MIT + */ + +#include "compiler/libcl/libcl.h" +#include "compiler/shader_enums.h" + +enum pco_mutex_id { + PCO_MUTEX_ID_ATOMIC_EMU, + PCO_MUTEX_ID_BARRIER, + + _PCO_MUTEX_ID_COUNT, +}; +static_assert(_PCO_MUTEX_ID_COUNT <= 16, "Too many mutex IDs."); + +enum pco_mutex_op { + PCO_MUTEX_OP_RELEASE, + PCO_MUTEX_OP_RELEASE_SLEEP, + PCO_MUTEX_OP_RELEASE_WAKEUP, + PCO_MUTEX_OP_LOCK, +}; + +#define ROGUE_MAX_INSTANCES_PER_TASK 32 + +void nir_mutex_pco(enum pco_mutex_id mutex_id, enum pco_mutex_op mutex_op); +uint32_t nir_load_instance_num_pco(void); + +uint32_t nir_load_ssbo(uint2 buffer_index, + uint offset, + enum gl_access_qualifier access, + uint align_mul, + uint align_offset, + uint offset_shift); + +void nir_store_ssbo(uint32_t value, + uint2 block_index, + uint offset, + uint write_mask, + enum gl_access_qualifier access, + uint align_mul, + uint align_offset, + uint offset_shift); diff --git a/src/imagination/pco/usclib/meson.build b/src/imagination/pco/usclib/meson.build new file mode 100644 index 00000000000..58d6e041ccd --- /dev/null +++ b/src/imagination/pco/usclib/meson.build @@ -0,0 +1,41 @@ +# Copyright © 2025 Imagination Technologies Ltd. +# SPDX-License-Identifier: MIT + +pco_usclib_shader_files = files( + 'common.cl', +) + +# We need to set -fmacro-prefix-map properly for reproducability. +fs = import('fs') +relative_dir = fs.relative_to(meson.global_source_root(), meson.global_build_root()) + '/' + +pco_usclib_spv = custom_target( + 'pco_usclib.spv', + input : pco_usclib_shader_files, + output : 'pco_usclib.spv', + command : [ + prog_mesa_clc, + '-o', '@OUTPUT@', '--depfile', '@DEPFILE@', + pco_usclib_shader_files, '--', + '-cl-std=cl2.0', '-D__OPENCL_VERSION__=200', + '-I' + join_paths(meson.project_source_root(), 'include'), + '-I' + join_paths(meson.project_source_root(), 'src/compiler/libcl'), + '-I' + join_paths(meson.current_source_dir(), '.'), + '-I' + join_paths(meson.current_source_dir(), '../../../'), + '-fmacro-prefix-map=@0@='.format(relative_dir), + ], + env: ['MESA_SHADER_CACHE_DISABLE=true'], + depfile : 'pco_usclib.h.d', +) + +pco_usclib = custom_target( + 'pco_usclib.h', + input : pco_usclib_spv, + output : ['pco_usclib.cpp', 'pco_usclib.h'], + command : [prog_vtn_bindgen2, pco_usclib_spv, '@OUTPUT0@', '@OUTPUT1@'], +) + +idep_pco_usclib = declare_dependency( + sources : [pco_usclib], + include_directories : include_directories('.'), +) diff --git a/src/imagination/vulkan/meson.build b/src/imagination/vulkan/meson.build index 3b5dbcb0b18..f0eaa632d5a 100644 --- a/src/imagination/vulkan/meson.build +++ b/src/imagination/vulkan/meson.build @@ -53,6 +53,7 @@ pvr_files = files( 'pvr_shader.c', 'pvr_spm.c', 'pvr_tex_state.c', + 'pvr_usc.c', 'pvr_wsi.c', 'usc/pvr_uscgen.c', @@ -69,6 +70,7 @@ pvr_deps = [ dep_libdrm, dep_valgrind, idep_mesautil, + idep_pco_uscgen_programs_h, idep_vulkan_runtime, idep_vulkan_util, idep_vulkan_wsi, @@ -109,6 +111,7 @@ libvulkan_powervr_mesa = shared_library( libpowervr_compiler, libpowervr_pds, libpowervr_rogue, + libpowervr_uscgen, libvulkan_wsi, ], dependencies : [ diff --git a/src/imagination/vulkan/pvr_device.c b/src/imagination/vulkan/pvr_device.c index 5312c6815b1..0a0a54bb7ac 100644 --- a/src/imagination/vulkan/pvr_device.c +++ b/src/imagination/vulkan/pvr_device.c @@ -42,6 +42,7 @@ #include "git_sha1.h" #include "hwdef/rogue_hw_utils.h" #include "pco/pco.h" +#include "pco_uscgen_programs.h" #include "pvr_bo.h" #include "pvr_border.h" #include "pvr_clear.h" @@ -834,6 +835,9 @@ static VkResult pvr_physical_device_init(struct pvr_physical_device *pdevice, "Failed to initialize PCO compiler context"); goto err_free_compiler; } + pco_ctx_setup_usclib(pdevice->pco_ctx, + pco_usclib_0_nir, + sizeof(pco_usclib_0_nir)); result = pvr_wsi_init(pdevice); if (result != VK_SUCCESS) { diff --git a/src/imagination/vulkan/pvr_usc.c b/src/imagination/vulkan/pvr_usc.c new file mode 100644 index 00000000000..99d1ddacbf2 --- /dev/null +++ b/src/imagination/vulkan/pvr_usc.c @@ -0,0 +1,86 @@ +/* + * Copyright © 2025 Imagination Technologies Ltd. + * + * SPDX-License-Identifier: MIT + */ + +/** + * \file pvr_usc.c + * + * \brief USC internal shader generation. + */ + +#include "nir/nir.h" +#include "nir/nir_builder.h" +#include "pco/pco.h" +#include "pco/pco_data.h" +#include "pco_uscgen_programs.h" +#include "pvr_usc.h" +#include "util/macros.h" + +/** + * Common function to build a NIR shader and export the binary. + * + * \param ctx PCO context. + * \param nir NIR shader. + * \param data Shader data. + * \return The finalized PCO shader. + */ +static pco_shader *build_shader(pco_ctx *ctx, nir_shader *nir, pco_data *data) +{ + pco_preprocess_nir(ctx, nir); + pco_lower_nir(ctx, nir, data); + pco_postprocess_nir(ctx, nir, data); + + pco_shader *shader = pco_trans_nir(ctx, nir, data, NULL); + ralloc_steal(shader, nir); + pco_process_ir(ctx, shader); + pco_encode_ir(ctx, shader); + + return shader; +} + +/** + * Generate a nop (empty) shader. + * + * \param ctx PCO context. + * \param stage Shader stage. ++ * \return The nop shader. + */ +pco_shader *pvr_usc_nop(pco_ctx *ctx, mesa_shader_stage stage) +{ + nir_builder b = + nir_builder_init_simple_shader(stage, + pco_nir_options(), + "nop (%s)", + _mesa_shader_stage_to_string(stage)); + + /* Just return. */ + nir_jump(&b, nir_jump_return); + + return build_shader(ctx, b.shader, &(pco_data){ 0 }); +} + +/** + * Generate an end-of-tile shader. + * + * \param ctx PCO context. + * \param props End of tile shader properties. + * \return The end-of-tile shader. + */ +pco_shader *pvr_usc_eot(pco_ctx *ctx, struct pvr_eot_props *props) +{ + UNREACHABLE("finishme: pvr_usc_eot"); +} + +/** + * Generate a transfer queue shader. + * + * \param ctx PCO context. + * \param props Transfer queue shader properties. + * \return The transfer queue shader. + */ +pco_shader *pvr_usc_tq(pco_ctx *ctx, struct pvr_tq_props *props) +{ + UNREACHABLE("finishme: pvr_usc_tq"); +} diff --git a/src/imagination/vulkan/pvr_usc.h b/src/imagination/vulkan/pvr_usc.h new file mode 100644 index 00000000000..ccfa6479239 --- /dev/null +++ b/src/imagination/vulkan/pvr_usc.h @@ -0,0 +1,34 @@ +/* + * Copyright © 2025 Imagination Technologies Ltd. + * + * SPDX-License-Identifier: MIT + */ + +#ifndef PVR_USC_H +#define PVR_USC_H + +/** + * \file pvr_usc.h + * + * \brief USC internal shader generation header. + */ + +#include "compiler/shader_enums.h" +#include "pco/pco.h" + +/* NOP shader generation. */ +pco_shader *pvr_usc_nop(pco_ctx *ctx, mesa_shader_stage stage); + +/* EOT shader generation. */ +struct pvr_eot_props { +}; + +pco_shader *pvr_usc_eot(pco_ctx *ctx, struct pvr_eot_props *props); + +/* Transfer queue shader generation. */ +struct pvr_tq_props { +}; + +pco_shader *pvr_usc_tq(pco_ctx *ctx, struct pvr_tq_props *props); + +#endif /* PVR_USC_H */ diff --git a/src/imagination/vulkan/pvr_uscgen.c b/src/imagination/vulkan/pvr_uscgen.c deleted file mode 100644 index 6b7f31c7f73..00000000000 --- a/src/imagination/vulkan/pvr_uscgen.c +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright © 2024 Imagination Technologies Ltd. - * - * SPDX-License-Identifier: MIT - */ - -/** - * \file pvr_uscgen.c - * - * \brief USC shader generation. - */ - -#include "nir/nir.h" -#include "nir/nir_builder.h" -#include "pco/pco.h" -#include "pvr_uscgen.h" -#include "util/macros.h" - -/** - * Common function to build a NIR shader and export the binary. - * - * \param ctx PCO context. - * \param nir NIR shader. - * \param binary Output shader binary. - */ -static void build_shader(pco_ctx *ctx, nir_shader *nir, pco_binary **binary) -{ - pco_preprocess_nir(ctx, nir); - pco_lower_nir(ctx, nir); - pco_postprocess_nir(ctx, nir); - - pco_shader *shader = pco_trans_nir(ctx, nir); - pco_process_ir(ctx, shader); - - pco_binary *bin = pco_encode_ir(ctx, shader); - ralloc_free(shader); - - pco_binary_finalize(ctx, bin); - *binary = bin; -} - -/** - * Generate a nop (empty) shader. - * - * \param ctx PCO context. - * \param stage Shader stage. - * \param binary Output shader binary. - */ -void pvr_uscgen_nop(pco_ctx *ctx, mesa_shader_stage stage, pco_binary **binary) -{ - UNREACHABLE("finishme: pvr_uscgen_nop"); -} - -/** - * Generate an end-of-tile shader. - * - * \param ctx PCO context. - * \param props End of tile shader properties. - * \param binary Output shader binary. - */ -void pvr_uscgen_eot(pco_ctx *ctx, - struct pvr_eot_props *props, - pco_binary **binary) -{ - UNREACHABLE("finishme: pvr_uscgen_eot"); -} - -/** - * Generate a transfer queue shader. - * - * \param ctx PCO context. - * \param props Transfer queue shader properties. - * \param binary Output shader binary. - */ -void pvr_uscgen_tq(pco_ctx *ctx, - struct pvr_tq_props *props, - pco_binary **binary) -{ - UNREACHABLE("finishme: pvr_uscgen_tq"); -} diff --git a/src/imagination/vulkan/pvr_uscgen.h b/src/imagination/vulkan/pvr_uscgen.h deleted file mode 100644 index 6773e93021d..00000000000 --- a/src/imagination/vulkan/pvr_uscgen.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright © 2024 Imagination Technologies Ltd. - * - * SPDX-License-Identifier: MIT - */ - -#ifndef PVR_USCGEN_H -#define PVR_USCGEN_H - -/** - * \file pvr_uscgen.h - * - * \brief USC shader generation header. - */ - -#include "compiler/shader_enums.h" -#include "pco/pco.h" - -/* NOP shader generation. */ -void pvr_uscgen_nop(pco_ctx *ctx, mesa_shader_stage stage, pco_binary **binary); - -/* EOT shader generation. */ -struct pvr_eot_props { -}; - -void pvr_uscgen_eot(pco_ctx *ctx, - struct pvr_eot_props *props, - pco_binary **binary); - -/* Transfer queue shader generation. */ -struct pvr_tq_props { -}; - -void pvr_uscgen_tq(pco_ctx *ctx, - struct pvr_tq_props *props, - pco_binary **binary); - -#endif /* PVR_USCGEN_H */ diff --git a/src/meson.build b/src/meson.build index 7da2b6ae1bd..fdfb2dc246f 100644 --- a/src/meson.build +++ b/src/meson.build @@ -86,7 +86,7 @@ endif if with_gallium_freedreno or with_freedreno_vk or with_tools.contains('freedreno') subdir('freedreno') endif -if with_imagination_vk +if with_imagination_vk or with_tools.contains('imagination') subdir('imagination') endif if with_gallium_panfrost or with_gallium_lima or with_panfrost_vk or with_tools.contains('panfrost')