diff --git a/src/amd/vulkan/radv_shader.c b/src/amd/vulkan/radv_shader.c index 7fbe4b0d91b..e510ac05076 100644 --- a/src/amd/vulkan/radv_shader.c +++ b/src/amd/vulkan/radv_shader.c @@ -97,7 +97,6 @@ get_nir_options_for_stage(struct radv_physical_device *device, gl_shader_stage s .lower_iadd_sat = device->rad_info.gfx_level <= GFX8, .lower_hadd = true, .lower_mul_32x16 = true, - .lower_uclz = true, .has_bfe = true, .has_bfm = true, .has_bitfield_select = true, diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index 63aeabe9ac5..ad35fb95293 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -3419,14 +3419,10 @@ typedef struct nir_shader_compiler_options { bool lower_bitfield_reverse; /** Lowers bit_count to shifts. */ bool lower_bit_count; - /** Lowers ifind_msb to compare and ufind_msb */ + /** Lowers ifind_msb. */ bool lower_ifind_msb; - /** Lowers ifind_msb and ufind_msb to reverse variants */ - bool lower_find_msb_to_reverse; - /** Lowers ifind_msb to uclz and logic ops*/ - bool lower_ifind_msb_to_uclz; - /** Lowers ufind_msb to 31-uclz */ - bool lower_ufind_msb_to_uclz; + /** Lowers ufind_msb. */ + bool lower_ufind_msb; /** Lowers find_lsb to ufind_msb and logic ops */ bool lower_find_lsb; bool lower_uadd_carry; @@ -3629,11 +3625,6 @@ typedef struct nir_shader_compiler_options { */ bool lower_mul_32x16; - /** - * Set if uclz should be lowered to find_msb_rev. - */ - bool lower_uclz; - /** * Should IO be re-vectorized? Some scalar ISAs still operate on vec4's * for IO purposes and would prefer loads/stores be vectorized. @@ -3754,6 +3745,9 @@ typedef struct nir_shader_compiler_options { /** Backend supports bitfield_select. */ bool has_bitfield_select; + /** Backend supports uclz. */ + bool has_uclz; + /** * Is this the Intel vec4 backend? * diff --git a/src/compiler/nir/nir_opt_algebraic.py b/src/compiler/nir/nir_opt_algebraic.py index 6e594ea451a..9208bc63111 100644 --- a/src/compiler/nir/nir_opt_algebraic.py +++ b/src/compiler/nir/nir_opt_algebraic.py @@ -1855,18 +1855,18 @@ optimizations.extend([ (('bcsel', ('ine', 'a@32', 0), ('iadd', 31, ('ineg', ('ufind_msb', a))), ('ufind_msb', a)), ('ufind_msb_rev', a), 'options->has_find_msb_rev'), (('bcsel', ('ieq', 'a@32', 0), ('ufind_msb', a), ('iadd', 31, ('ineg', ('ufind_msb', a)))), ('ufind_msb_rev', a), 'options->has_find_msb_rev'), - (('bcsel', ('ine', ('ifind_msb_rev', 'a@32'), -1), ('iadd', 31, ('ineg', ('ifind_msb_rev', a))), -1), ('ifind_msb', a), '!options->lower_find_msb_to_reverse'), - (('bcsel', ('ine', ('ufind_msb_rev', 'a@32'), -1), ('iadd', 31, ('ineg', ('ufind_msb_rev', a))), -1), ('ufind_msb', a), '!options->lower_find_msb_to_reverse'), - (('bcsel', ('ieq', ('ifind_msb_rev', 'a@32'), -1), -1, ('iadd', 31, ('ineg', ('ifind_msb_rev', a)))), ('ifind_msb', a), '!options->lower_find_msb_to_reverse'), - (('bcsel', ('ieq', ('ufind_msb_rev', 'a@32'), -1), -1, ('iadd', 31, ('ineg', ('ufind_msb_rev', a)))), ('ufind_msb', a), '!options->lower_find_msb_to_reverse'), - (('bcsel', ('ine', ('ifind_msb_rev', 'a@32'), -1), ('iadd', 31, ('ineg', ('ifind_msb_rev', a))), ('ifind_msb_rev', a)), ('ifind_msb', a), '!options->lower_find_msb_to_reverse'), - (('bcsel', ('ine', ('ufind_msb_rev', 'a@32'), -1), ('iadd', 31, ('ineg', ('ufind_msb_rev', a))), ('ufind_msb_rev', a)), ('ufind_msb', a), '!options->lower_find_msb_to_reverse'), - (('bcsel', ('ieq', ('ifind_msb_rev', 'a@32'), -1), ('ifind_msb_rev', a), ('iadd', 31, ('ineg', ('ifind_msb_rev', a)))), ('ifind_msb', a), '!options->lower_find_msb_to_reverse'), - (('bcsel', ('ieq', ('ufind_msb_rev', 'a@32'), -1), ('ufind_msb_rev', a), ('iadd', 31, ('ineg', ('ufind_msb_rev', a)))), ('ufind_msb', a), '!options->lower_find_msb_to_reverse'), - (('bcsel', ('ine', 'a@32', 0), ('iadd', 31, ('ineg', ('ufind_msb_rev', a))), -1), ('ufind_msb', a), '!options->lower_find_msb_to_reverse'), - (('bcsel', ('ieq', 'a@32', 0), -1, ('iadd', 31, ('ineg', ('ufind_msb_rev', a)))), ('ufind_msb', a), '!options->lower_find_msb_to_reverse'), - (('bcsel', ('ine', 'a@32', 0), ('iadd', 31, ('ineg', ('ufind_msb_rev', a))), ('ufind_msb_rev', a)), ('ufind_msb', a), '!options->lower_find_msb_to_reverse'), - (('bcsel', ('ieq', 'a@32', 0), ('ufind_msb_rev', a), ('iadd', 31, ('ineg', ('ufind_msb_rev', a)))), ('ufind_msb', a), '!options->lower_find_msb_to_reverse'), + (('bcsel', ('ine', ('ifind_msb_rev', 'a@32'), -1), ('iadd', 31, ('ineg', ('ifind_msb_rev', a))), -1), ('ifind_msb', a), '!options->lower_ifind_msb'), + (('bcsel', ('ine', ('ufind_msb_rev', 'a@32'), -1), ('iadd', 31, ('ineg', ('ufind_msb_rev', a))), -1), ('ufind_msb', a), '!options->lower_ufind_msb'), + (('bcsel', ('ieq', ('ifind_msb_rev', 'a@32'), -1), -1, ('iadd', 31, ('ineg', ('ifind_msb_rev', a)))), ('ifind_msb', a), '!options->lower_ifind_msb'), + (('bcsel', ('ieq', ('ufind_msb_rev', 'a@32'), -1), -1, ('iadd', 31, ('ineg', ('ufind_msb_rev', a)))), ('ufind_msb', a), '!options->lower_ufind_msb'), + (('bcsel', ('ine', ('ifind_msb_rev', 'a@32'), -1), ('iadd', 31, ('ineg', ('ifind_msb_rev', a))), ('ifind_msb_rev', a)), ('ifind_msb', a), '!options->lower_ifind_msb'), + (('bcsel', ('ine', ('ufind_msb_rev', 'a@32'), -1), ('iadd', 31, ('ineg', ('ufind_msb_rev', a))), ('ufind_msb_rev', a)), ('ufind_msb', a), '!options->lower_ufind_msb'), + (('bcsel', ('ieq', ('ifind_msb_rev', 'a@32'), -1), ('ifind_msb_rev', a), ('iadd', 31, ('ineg', ('ifind_msb_rev', a)))), ('ifind_msb', a), '!options->lower_ifind_msb'), + (('bcsel', ('ieq', ('ufind_msb_rev', 'a@32'), -1), ('ufind_msb_rev', a), ('iadd', 31, ('ineg', ('ufind_msb_rev', a)))), ('ufind_msb', a), '!options->lower_ufind_msb'), + (('bcsel', ('ine', 'a@32', 0), ('iadd', 31, ('ineg', ('ufind_msb_rev', a))), -1), ('ufind_msb', a), '!options->lower_ufind_msb'), + (('bcsel', ('ieq', 'a@32', 0), -1, ('iadd', 31, ('ineg', ('ufind_msb_rev', a)))), ('ufind_msb', a), '!options->lower_ufind_msb'), + (('bcsel', ('ine', 'a@32', 0), ('iadd', 31, ('ineg', ('ufind_msb_rev', a))), ('ufind_msb_rev', a)), ('ufind_msb', a), '!options->lower_ufind_msb'), + (('bcsel', ('ieq', 'a@32', 0), ('ufind_msb_rev', a), ('iadd', 31, ('ineg', ('ufind_msb_rev', a)))), ('ufind_msb', a), '!options->lower_ufind_msb'), # This is safe. Both ufind_msb_rev and bitfield_reverse can only have # 32-bit sources, so the transformation can only generate correct NIR. @@ -2068,13 +2068,13 @@ optimizations.extend([ (('ifind_msb', 'value'), ('ufind_msb', ('bcsel', ('ilt', 'value', 0), ('inot', 'value'), 'value')), - 'options->lower_ifind_msb'), + 'options->lower_ifind_msb && !options->has_find_msb_rev && !options->has_uclz'), (('ifind_msb', 'value'), ('bcsel', ('ige', ('ifind_msb_rev', 'value'), 0), ('isub', 31, ('ifind_msb_rev', 'value')), ('ifind_msb_rev', 'value')), - 'options->lower_find_msb_to_reverse'), + 'options->lower_ifind_msb && options->has_find_msb_rev'), # uclz of an absolute value source almost always does the right thing. # There are a couple problem values: @@ -2096,19 +2096,19 @@ optimizations.extend([ # be achieved by (x ^ (x >> 31)). (('ifind_msb', 'value'), ('isub', 31, ('uclz', ('ixor', 'value', ('ishr', 'value', 31)))), - 'options->lower_ifind_msb_to_uclz'), + 'options->lower_ifind_msb && options->has_uclz'), (('ufind_msb', 'value@32'), ('bcsel', ('ige', ('ufind_msb_rev', 'value'), 0), ('isub', 31, ('ufind_msb_rev', 'value')), ('ufind_msb_rev', 'value')), - 'options->lower_find_msb_to_reverse'), + 'options->lower_ufind_msb && options->has_find_msb_rev'), (('ufind_msb', 'value@32'), ('isub', 31, ('uclz', 'value')), - 'options->lower_ufind_msb_to_uclz'), + 'options->lower_ufind_msb && options->has_uclz'), - (('uclz', a), ('umin', 32, ('ufind_msb_rev', a)), 'options->lower_uclz'), + (('uclz', a), ('umin', 32, ('ufind_msb_rev', a)), '!options->has_uclz && options->has_find_msb_rev'), (('find_lsb', 'value@64'), ('ufind_msb', ('iand', 'value', ('ineg', 'value'))), diff --git a/src/gallium/drivers/etnaviv/etnaviv_compiler.c b/src/gallium/drivers/etnaviv/etnaviv_compiler.c index 46b93de0cf4..35b1ef45dbb 100644 --- a/src/gallium/drivers/etnaviv/etnaviv_compiler.c +++ b/src/gallium/drivers/etnaviv/etnaviv_compiler.c @@ -75,8 +75,9 @@ etna_compiler_create(const char *renderer, const struct etna_specs *specs) .lower_unpack_32_2x16_split = true, .lower_unpack_64_2x32_split = true, .lower_find_lsb = true, - .lower_ifind_msb_to_uclz = true, - .lower_ufind_msb_to_uclz = true, + .lower_ifind_msb = true, + .lower_ufind_msb = true, + .has_uclz = true, }; compiler->regs = etna_ra_setup(compiler); diff --git a/src/gallium/drivers/r600/r600_pipe_common.c b/src/gallium/drivers/r600/r600_pipe_common.c index c0a34156703..1a5b4b89663 100644 --- a/src/gallium/drivers/r600/r600_pipe_common.c +++ b/src/gallium/drivers/r600/r600_pipe_common.c @@ -1360,11 +1360,13 @@ bool r600_common_screen_init(struct r600_common_screen *rscreen, .use_interpolated_input_intrinsics = true, .has_fsub = true, .has_isub = true, + .has_find_msb_rev = true, .lower_iabs = true, .lower_uadd_sat = true, .lower_usub_sat = true, .has_fused_comp_and_csel = true, - .lower_find_msb_to_reverse = true, + .lower_ifind_msb = true, + .lower_ufind_msb = true, .lower_to_scalar = true, .lower_to_scalar_filter = r600_lower_to_scalar_instr_filter, .linker_ignore_precision = true, diff --git a/src/intel/compiler/brw_compiler.c b/src/intel/compiler/brw_compiler.c index 54c11acaa1f..4fbcf5ad880 100644 --- a/src/intel/compiler/brw_compiler.c +++ b/src/intel/compiler/brw_compiler.c @@ -29,11 +29,12 @@ #include "util/u_debug.h" #define COMMON_OPTIONS \ + .has_uclz = true, \ .lower_fdiv = true, \ .lower_scmp = true, \ .lower_flrp16 = true, \ .lower_fmod = true, \ - .lower_ufind_msb_to_uclz = true, \ + .lower_ufind_msb = true, \ .lower_uadd_carry = true, \ .lower_usub_borrow = true, \ .lower_flrp64 = true, \ @@ -191,7 +192,7 @@ brw_compiler_create(void *mem_ctx, const struct intel_device_info *devinfo) nir_options->lower_rotate = devinfo->ver < 11; nir_options->lower_bitfield_reverse = devinfo->ver < 7; nir_options->lower_find_lsb = devinfo->ver < 7; - nir_options->lower_ifind_msb_to_uclz = devinfo->ver < 7; + nir_options->lower_ifind_msb = devinfo->ver < 7; nir_options->has_iadd3 = devinfo->verx10 >= 125; nir_options->has_sdot_4x8 = devinfo->ver >= 12; diff --git a/src/microsoft/compiler/nir_to_dxil.c b/src/microsoft/compiler/nir_to_dxil.c index 002b65b217b..36a04d86d44 100644 --- a/src/microsoft/compiler/nir_to_dxil.c +++ b/src/microsoft/compiler/nir_to_dxil.c @@ -104,7 +104,8 @@ nir_options = { .lower_flrp32 = true, .lower_flrp64 = true, .lower_bitfield_extract = true, - .lower_find_msb_to_reverse = true, + .lower_ifind_msb = true, + .lower_ufind_msb = true, .lower_extract_word = true, .lower_extract_byte = true, .lower_insert_word = true, @@ -138,6 +139,7 @@ nir_options = { .has_fsub = true, .has_isub = true, .has_bfe = true, + .has_find_msb_rev = true, .vertex_id_zero_based = true, .lower_base_vertex = true, .lower_helper_invocation = true,