From f90d71055be5ff6381479095448a606ec4018b93 Mon Sep 17 00:00:00 2001 From: Ian Romanick Date: Wed, 2 Feb 2022 18:49:25 -0800 Subject: [PATCH] intel/compiler: Add and use a pass to generate imul_32x16 instructions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gfx8 and Gfx9 platforms are helped for cycles because now many instructions like mul(8) g12<1>D g10<8,8,1>D 6D become mul(8) g12<1>D g10<8,8,1>D 6W It is the same number of instructions, but the 32x16 multiply is a little faster. v2: Fix transposed hi and lo in "(hi >= INT16_MIN && lo <= INT16_MAX)". Noticed by Caio. Use nir_src_is_const instead of open coding it. Suggested by Caio. Broadwell and Skylake had similar results. (Skylake shown) total cycles in shared programs: 845748380 -> 845145547 (-0.07%) cycles in affected programs: 446346348 -> 445743515 (-0.14%) helped: 6017 HURT: 0 helped stats (abs) min: 2 max: 7380 x̄: 100.19 x̃: 8 helped stats (rel) min: <.01% max: 3.72% x̄: 0.41% x̃: 0.39% 95% mean confidence interval for cycles value: -113.37 -87.00 95% mean confidence interval for cycles %-change: -0.42% -0.41% Cycles are helped. Skylake Cycles in all programs: 8844820715 -> 8828897462 (-0.2%) Cycles helped: 47914 Cycles hurt: 1 No shader-db or fossil-db changes on any other Intel platform. Reviewed-by: Caio Oliveira Part-of: --- src/intel/compiler/brw_nir.c | 3 + src/intel/compiler/brw_nir.h | 2 + .../compiler/brw_nir_opt_peephole_imul32x16.c | 120 ++++++++++++++++++ src/intel/compiler/meson.build | 1 + 4 files changed, 126 insertions(+) create mode 100644 src/intel/compiler/brw_nir_opt_peephole_imul32x16.c diff --git a/src/intel/compiler/brw_nir.c b/src/intel/compiler/brw_nir.c index 8cbf39ab9dd..b9d46d0065b 100644 --- a/src/intel/compiler/brw_nir.c +++ b/src/intel/compiler/brw_nir.c @@ -1286,6 +1286,9 @@ brw_postprocess_nir(nir_shader *nir, const struct brw_compiler *compiler, OPT(brw_nir_opt_peephole_ffma); } + if (devinfo->ver >= 7 && is_scalar) + OPT(brw_nir_opt_peephole_imul32x16); + if (OPT(nir_opt_comparison_pre)) { OPT(nir_copy_prop); OPT(nir_opt_dce); diff --git a/src/intel/compiler/brw_nir.h b/src/intel/compiler/brw_nir.h index 98f6c946c11..91fa8209915 100644 --- a/src/intel/compiler/brw_nir.h +++ b/src/intel/compiler/brw_nir.h @@ -176,6 +176,8 @@ void brw_nir_analyze_ubo_ranges(const struct brw_compiler *compiler, bool brw_nir_opt_peephole_ffma(nir_shader *shader); +bool brw_nir_opt_peephole_imul32x16(nir_shader *shader); + void brw_nir_optimize(nir_shader *nir, const struct brw_compiler *compiler, bool is_scalar, diff --git a/src/intel/compiler/brw_nir_opt_peephole_imul32x16.c b/src/intel/compiler/brw_nir_opt_peephole_imul32x16.c new file mode 100644 index 00000000000..39c3fa630dd --- /dev/null +++ b/src/intel/compiler/brw_nir_opt_peephole_imul32x16.c @@ -0,0 +1,120 @@ +/* + * Copyright © 2022 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "brw_nir.h" +#include "compiler/nir/nir_builder.h" + +/** + * Implement a peephole pass to convert integer multiplications to imul32x16. + */ + +static void +replace_imul_instr(nir_builder *b, nir_alu_instr *imul, unsigned small_val, + nir_op new_opcode) +{ + assert(small_val == 0 || small_val == 1); + + b->cursor = nir_before_instr(&imul->instr); + + nir_alu_instr *imul_32x16 = nir_alu_instr_create(b->shader, new_opcode); + imul_32x16->dest.saturate = imul->dest.saturate; + imul_32x16->dest.write_mask = imul->dest.write_mask; + + nir_alu_src_copy(&imul_32x16->src[0], &imul->src[1 - small_val], imul_32x16); + nir_alu_src_copy(&imul_32x16->src[1], &imul->src[small_val], imul_32x16); + + nir_ssa_dest_init(&imul_32x16->instr, &imul_32x16->dest.dest, + imul->dest.dest.ssa.num_components, + 32, NULL); + + nir_ssa_def_rewrite_uses(&imul->dest.dest.ssa, + &imul_32x16->dest.dest.ssa); + + nir_builder_instr_insert(b, &imul_32x16->instr); + + nir_instr_remove(&imul->instr); + nir_instr_free(&imul->instr); +} + +static bool +brw_nir_opt_peephole_imul32x16_instr(nir_builder *b, + nir_instr *instr, + UNUSED void *cb_data) +{ + if (instr->type != nir_instr_type_alu) + return false; + + nir_alu_instr *imul = nir_instr_as_alu(instr); + if (imul->op != nir_op_imul) + return false; + + if (imul->dest.dest.ssa.bit_size != 32) + return false; + + nir_op new_opcode = nir_num_opcodes; + + unsigned i; + for (i = 0; i < 2; i++) { + if (!nir_src_is_const(imul->src[i].src)) + continue; + + int64_t lo = INT64_MAX; + int64_t hi = INT64_MIN; + + for (unsigned comp = 0; comp < imul->dest.dest.ssa.num_components; comp++) { + int64_t v = nir_src_comp_as_int(imul->src[i].src, comp); + + if (v < lo) + lo = v; + + if (v > hi) + hi = v; + } + + if (lo >= INT16_MIN && hi <= INT16_MAX) { + new_opcode = nir_op_imul_32x16; + break; + } else if (lo >= 0 && hi <= UINT16_MAX) { + new_opcode = nir_op_umul_32x16; + break; + } + } + + if (new_opcode != nir_num_opcodes) { + replace_imul_instr(b, imul, i, new_opcode); + return true; + } + + return false; +} + +bool +brw_nir_opt_peephole_imul32x16(nir_shader *shader) +{ + return nir_shader_instructions_pass(shader, + brw_nir_opt_peephole_imul32x16_instr, + nir_metadata_block_index | + nir_metadata_dominance, + NULL); +} + diff --git a/src/intel/compiler/meson.build b/src/intel/compiler/meson.build index 2110afd3f4a..5b5baa4799f 100644 --- a/src/intel/compiler/meson.build +++ b/src/intel/compiler/meson.build @@ -98,6 +98,7 @@ libintel_compiler_files = files( 'brw_nir_lower_shading_rate_output.c', 'brw_nir_lower_storage_image.c', 'brw_nir_opt_peephole_ffma.c', + 'brw_nir_opt_peephole_imul32x16.c', 'brw_nir_rt.h', 'brw_nir_rt.c', 'brw_nir_rt_builder.h',