diff --git a/src/broadcom/vulkan/v3dv_pipeline.c b/src/broadcom/vulkan/v3dv_pipeline.c index 3e5cdaad994..ddd8fbfacce 100644 --- a/src/broadcom/vulkan/v3dv_pipeline.c +++ b/src/broadcom/vulkan/v3dv_pipeline.c @@ -194,7 +194,8 @@ static const struct spirv_to_nir_options default_spirv_options = { }; const nir_shader_compiler_options v3dv_nir_options = { - .lower_add_sat = true, + .lower_uadd_sat = true, + .lower_iadd_sat = true, .lower_all_io_to_temps = true, .lower_extract_byte = true, .lower_extract_word = true, diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index fbd7cfda291..1e3bb82ab11 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -3615,7 +3615,7 @@ typedef struct nir_shader_compiler_options { bool lower_hadd64; /** - * Set if nir_op_add_sat and nir_op_usub_sat should be lowered to simple + * Set if nir_op_uadd_sat and nir_op_usub_sat should be lowered to simple * arithmetic. * * If this flag is set, the lowering will be applied to all bit-sizes of @@ -3623,7 +3623,7 @@ typedef struct nir_shader_compiler_options { * * \sa ::lower_usub_sat64 */ - bool lower_add_sat; + bool lower_uadd_sat; /** * Set if only 64-bit nir_op_usub_sat should be lowered to simple @@ -3633,6 +3633,15 @@ typedef struct nir_shader_compiler_options { */ bool lower_usub_sat64; + /** + * Set if nir_op_iadd_sat and nir_op_isub_sat should be lowered to simple + * arithmetic. + * + * If this flag is set, the lowering will be applied to all bit-sizes of + * these instructions. + */ + bool lower_iadd_sat; + /** * Should IO be re-vectorized? Some scalar ISAs still operate on vec4's * for IO purposes and would prefer loads/stores be vectorized. diff --git a/src/compiler/nir/nir_opt_algebraic.py b/src/compiler/nir/nir_opt_algebraic.py index 140b92c2dec..1d9b1b2d1fb 100644 --- a/src/compiler/nir/nir_opt_algebraic.py +++ b/src/compiler/nir/nir_opt_algebraic.py @@ -218,9 +218,9 @@ optimizations = [ (('sdot_4x8_iadd', '#a', '#b', 'c(is_not_const)'), ('iadd', ('sdot_4x8_iadd', a, b, 0), c)), (('udot_4x8_uadd', '#a', '#b', 'c(is_not_const)'), ('iadd', ('udot_4x8_uadd', a, b, 0), c)), (('sudot_4x8_iadd', '#a', '#b', 'c(is_not_const)'), ('iadd', ('sudot_4x8_iadd', a, b, 0), c)), - (('sdot_4x8_iadd_sat', '#a', '#b', 'c(is_not_const)'), ('iadd_sat', ('sdot_4x8_iadd', a, b, 0), c), '!options->lower_add_sat'), - (('udot_4x8_uadd_sat', '#a', '#b', 'c(is_not_const)'), ('uadd_sat', ('udot_4x8_uadd', a, b, 0), c), '!options->lower_add_sat'), - (('sudot_4x8_iadd_sat', '#a', '#b', 'c(is_not_const)'), ('iadd_sat', ('sudot_4x8_iadd', a, b, 0), c), '!options->lower_add_sat'), + (('sdot_4x8_iadd_sat', '#a', '#b', 'c(is_not_const)'), ('iadd_sat', ('sdot_4x8_iadd', a, b, 0), c), '!options->lower_iadd_sat'), + (('udot_4x8_uadd_sat', '#a', '#b', 'c(is_not_const)'), ('uadd_sat', ('udot_4x8_uadd', a, b, 0), c), '!options->lower_uadd_sat'), + (('sudot_4x8_iadd_sat', '#a', '#b', 'c(is_not_const)'), ('iadd_sat', ('sudot_4x8_iadd', a, b, 0), c), '!options->lower_iadd_sat'), ] # Shorthand for the expansion of just the dot product part of the [iu]dp4a @@ -1538,9 +1538,9 @@ optimizations.extend([ (('irhadd@64', a, b), ('isub', ('ior', a, b), ('ishr', ('ixor', a, b), 1)), 'options->lower_hadd64 || (options->lower_int64_options & nir_lower_iadd64) != 0'), (('urhadd@64', a, b), ('isub', ('ior', a, b), ('ushr', ('ixor', a, b), 1)), 'options->lower_hadd64 || (options->lower_int64_options & nir_lower_iadd64) != 0'), - (('uadd_sat@64', a, b), ('bcsel', ('ult', ('iadd', a, b), a), -1, ('iadd', a, b)), 'options->lower_add_sat || (options->lower_int64_options & nir_lower_iadd64) != 0'), - (('uadd_sat', a, b), ('bcsel', ('ult', ('iadd', a, b), a), -1, ('iadd', a, b)), 'options->lower_add_sat'), - (('usub_sat', a, b), ('bcsel', ('ult', a, b), 0, ('isub', a, b)), 'options->lower_add_sat'), + (('uadd_sat@64', a, b), ('bcsel', ('ult', ('iadd', a, b), a), -1, ('iadd', a, b)), 'options->lower_uadd_sat || (options->lower_int64_options & nir_lower_iadd64) != 0'), + (('uadd_sat', a, b), ('bcsel', ('ult', ('iadd', a, b), a), -1, ('iadd', a, b)), 'options->lower_uadd_sat'), + (('usub_sat', a, b), ('bcsel', ('ult', a, b), 0, ('isub', a, b)), 'options->lower_uadd_sat'), (('usub_sat@64', a, b), ('bcsel', ('ult', a, b), 0, ('isub', a, b)), 'options->lower_usub_sat64 || (options->lower_int64_options & nir_lower_iadd64) != 0'), # int64_t sum = a + b; @@ -1860,10 +1860,10 @@ for bit_size in [8, 16, 32, 64]: optimizations += [ (('iadd_sat@' + str(bit_size), a, b), ('bcsel', ('ige', b, 1), ('bcsel', ('ilt', ('iadd', a, b), a), intmax, ('iadd', a, b)), - ('bcsel', ('ilt', a, ('iadd', a, b)), intmin, ('iadd', a, b))), 'options->lower_add_sat'), + ('bcsel', ('ilt', a, ('iadd', a, b)), intmin, ('iadd', a, b))), 'options->lower_iadd_sat'), (('isub_sat@' + str(bit_size), a, b), ('bcsel', ('ilt', b, 0), ('bcsel', ('ilt', ('isub', a, b), a), intmax, ('isub', a, b)), - ('bcsel', ('ilt', a, ('isub', a, b)), intmin, ('isub', a, b))), 'options->lower_add_sat'), + ('bcsel', ('ilt', a, ('isub', a, b)), intmin, ('isub', a, b))), 'options->lower_iadd_sat'), ] invert = OrderedDict([('feq', 'fneu'), ('fneu', 'feq')]) diff --git a/src/gallium/drivers/i915/i915_screen.c b/src/gallium/drivers/i915/i915_screen.c index 62c9f34d237..345dbdcee95 100644 --- a/src/gallium/drivers/i915/i915_screen.c +++ b/src/gallium/drivers/i915/i915_screen.c @@ -134,7 +134,8 @@ static const struct nir_shader_compiler_options gallivm_nir_options = { .lower_ffma64 = true, .lower_fmod = true, .lower_hadd = true, - .lower_add_sat = true, + .lower_uadd_sat = true, + .lower_iadd_sat = true, .lower_ldexp = true, .lower_pack_snorm_2x16 = true, .lower_pack_snorm_4x8 = true, diff --git a/src/gallium/drivers/llvmpipe/lp_screen.c b/src/gallium/drivers/llvmpipe/lp_screen.c index 9a406d19bd5..981d711cadf 100644 --- a/src/gallium/drivers/llvmpipe/lp_screen.c +++ b/src/gallium/drivers/llvmpipe/lp_screen.c @@ -568,7 +568,8 @@ static const struct nir_shader_compiler_options gallivm_nir_options = { .lower_ffma64 = true, .lower_fmod = true, .lower_hadd = true, - .lower_add_sat = true, + .lower_uadd_sat = true, + .lower_iadd_sat = true, .lower_ldexp = true, .lower_pack_snorm_2x16 = true, .lower_pack_snorm_4x8 = true, diff --git a/src/gallium/drivers/nouveau/codegen/nv50_ir_from_nir.cpp b/src/gallium/drivers/nouveau/codegen/nv50_ir_from_nir.cpp index a7a6c6a92f5..ff835fdb773 100644 --- a/src/gallium/drivers/nouveau/codegen/nv50_ir_from_nir.cpp +++ b/src/gallium/drivers/nouveau/codegen/nv50_ir_from_nir.cpp @@ -3286,7 +3286,8 @@ nvir_nir_shader_compiler_options(int chipset) op.lower_device_index_to_zero = false; // TODO op.lower_wpos_pntc = false; // TODO op.lower_hadd = true; // TODO - op.lower_add_sat = true; // TODO + op.lower_uadd_sat = true; // TODO + op.lower_iadd_sat = true; // TODO op.vectorize_io = false; op.lower_to_scalar = false; op.unify_interfaces = false; diff --git a/src/gallium/drivers/v3d/v3d_screen.c b/src/gallium/drivers/v3d/v3d_screen.c index 3dbf121dc21..38b155fd961 100644 --- a/src/gallium/drivers/v3d/v3d_screen.c +++ b/src/gallium/drivers/v3d/v3d_screen.c @@ -645,7 +645,8 @@ v3d_screen_is_format_supported(struct pipe_screen *pscreen, } static const nir_shader_compiler_options v3d_nir_options = { - .lower_add_sat = true, + .lower_uadd_sat = true, + .lower_iadd_sat = true, .lower_all_io_to_temps = true, .lower_extract_byte = true, .lower_extract_word = true, diff --git a/src/microsoft/compiler/nir_to_dxil.c b/src/microsoft/compiler/nir_to_dxil.c index cc33fabbbbb..d5f666ddce9 100644 --- a/src/microsoft/compiler/nir_to_dxil.c +++ b/src/microsoft/compiler/nir_to_dxil.c @@ -95,7 +95,8 @@ nir_options = { .lower_all_io_to_elements = true, .lower_all_io_to_temps = true, .lower_hadd = true, - .lower_add_sat = true, + .lower_uadd_sat = true, + .lower_iadd_sat = true, .lower_uadd_carry = true, .lower_mul_high = true, .lower_rotate = true,