From e2d9ff8004586214b38214b6f80a87a40fa2758b Mon Sep 17 00:00:00 2001 From: Kenneth Graunke Date: Wed, 21 Feb 2024 03:13:14 -0800 Subject: [PATCH] intel/brw: Handle scratch address swizzling of constants Pass in the nir_src and check if it's constant, handling it via CPU-side arithmetic instead of emitting instructions. While we can constant fold these via our optimization passes, we have to do opt_algebraic to fold the binary operation with constant sources into a MOV of an immediate, then opt_copy_propagation to put it in the next expression, and so on, until the entire expression is folded. This can take several iterations of the optimization loop, which is inefficient. For example, gfxbench5/aztec-ruins/normal/7 has load/store_scratch intrinsics with constant sources, and this patch removes a number of optimization passes according to INTEL_DEBUG=optimizer. Reviewed-by: Caio Oliveira Part-of: --- src/intel/compiler/brw_fs_nir.cpp | 67 +++++++++++++++++++------------ 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/src/intel/compiler/brw_fs_nir.cpp b/src/intel/compiler/brw_fs_nir.cpp index 2ac9220e4ac..26d1244f5eb 100644 --- a/src/intel/compiler/brw_fs_nir.cpp +++ b/src/intel/compiler/brw_fs_nir.cpp @@ -4796,22 +4796,43 @@ get_nir_buffer_intrinsic_index(nir_to_brw_state &ntb, const brw::fs_builder &bld static fs_reg swizzle_nir_scratch_addr(nir_to_brw_state &ntb, const brw::fs_builder &bld, - const fs_reg &nir_addr, + const nir_src &nir_addr_src, bool in_dwords) { fs_visitor &s = ntb.s; - fs_reg nir_addr_ud = retype(nir_addr, BRW_TYPE_UD); - const fs_reg &chan_index = ntb.system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION]; const unsigned chan_index_bits = ffs(s.dispatch_width) - 1; + if (nir_src_is_const(nir_addr_src)) { + unsigned nir_addr = nir_src_as_uint(nir_addr_src); + if (in_dwords) { + /* In this case, we know the address is aligned to a DWORD and we want + * the final address in DWORDs. + */ + return bld.OR(chan_index, + brw_imm_ud(nir_addr << (chan_index_bits - 2))); + } else { + /* This case is substantially more annoying because we have to pay + * attention to those pesky two bottom bits. + */ + unsigned addr_hi = (nir_addr & ~0x3u) << chan_index_bits; + unsigned addr_lo = (nir_addr & 0x3u); + + return bld.OR(bld.SHL(chan_index, brw_imm_ud(2)), + brw_imm_ud(addr_lo | addr_hi)); + } + } + + const fs_reg nir_addr = + retype(get_nir_src(ntb, nir_addr_src), BRW_TYPE_UD); + if (in_dwords) { /* In this case, we know the address is aligned to a DWORD and we want * the final address in DWORDs. */ - return bld.OR(bld.SHL(nir_addr_ud, brw_imm_ud(chan_index_bits - 2)), + return bld.OR(bld.SHL(nir_addr, brw_imm_ud(chan_index_bits - 2)), chan_index); } else { /* This case substantially more annoying because we have to pay @@ -4819,8 +4840,8 @@ swizzle_nir_scratch_addr(nir_to_brw_state &ntb, */ fs_reg chan_addr = bld.SHL(chan_index, brw_imm_ud(2)); fs_reg addr_bits = - bld.OR(bld.AND(nir_addr_ud, brw_imm_ud(0x3u)), - bld.SHL(bld.AND(nir_addr_ud, brw_imm_ud(~0x3u)), + bld.OR(bld.AND(nir_addr, brw_imm_ud(0x3u)), + bld.SHL(bld.AND(nir_addr, brw_imm_ud(~0x3u)), brw_imm_ud(chan_index_bits))); return bld.OR(addr_bits, chan_addr); } @@ -6734,7 +6755,13 @@ fs_nir_emit_intrinsic(nir_to_brw_state &ntb, srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size); srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(0); - const fs_reg nir_addr = get_nir_src(ntb, instr->src[0]); + + /* The offset for a DWORD scattered message is in dwords. */ + bool addr_in_dwords = devinfo->verx10 < 125 && + bit_size == 32 && nir_intrinsic_align(instr) >= 4; + + srcs[SURFACE_LOGICAL_SRC_ADDRESS] = + swizzle_nir_scratch_addr(ntb, bld, instr->src[0], addr_in_dwords); /* Make dest unsigned because that's what the temporary will be */ dest.type = brw_type_with_size(BRW_TYPE_UD, bit_size); @@ -6749,24 +6776,15 @@ fs_nir_emit_intrinsic(nir_to_brw_state &ntb, assert(bit_size == 32 && nir_intrinsic_align(instr) >= 4); - srcs[SURFACE_LOGICAL_SRC_ADDRESS] = - swizzle_nir_scratch_addr(ntb, bld, nir_addr, false); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(1); bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); } else { - /* The offset for a DWORD scattered message is in dwords. */ - srcs[SURFACE_LOGICAL_SRC_ADDRESS] = - swizzle_nir_scratch_addr(ntb, bld, nir_addr, true); - bld.emit(SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); } } else { - srcs[SURFACE_LOGICAL_SRC_ADDRESS] = - swizzle_nir_scratch_addr(ntb, bld, nir_addr, false); - fs_reg read_result = bld.vgrf(BRW_TYPE_UD); bld.emit(SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL, read_result, srcs, SURFACE_LOGICAL_NUM_SRCS); @@ -6804,7 +6822,13 @@ fs_nir_emit_intrinsic(nir_to_brw_state &ntb, * they should not have different behaviour in the helper invocations. */ srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(0); - const fs_reg nir_addr = get_nir_src(ntb, instr->src[1]); + + /* The offset for a DWORD scattered message is in dwords. */ + bool addr_in_dwords = devinfo->verx10 < 125 && + bit_size == 32 && nir_intrinsic_align(instr) >= 4; + + srcs[SURFACE_LOGICAL_SRC_ADDRESS] = + swizzle_nir_scratch_addr(ntb, bld, instr->src[1], addr_in_dwords); fs_reg data = get_nir_src(ntb, instr->src[0]); data.type = brw_type_with_size(BRW_TYPE_UD, bit_size); @@ -6818,8 +6842,6 @@ fs_nir_emit_intrinsic(nir_to_brw_state &ntb, if (devinfo->verx10 >= 125) { srcs[SURFACE_LOGICAL_SRC_DATA] = data; - srcs[SURFACE_LOGICAL_SRC_ADDRESS] = - swizzle_nir_scratch_addr(ntb, bld, nir_addr, false); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(1); bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL, @@ -6827,10 +6849,6 @@ fs_nir_emit_intrinsic(nir_to_brw_state &ntb, } else { srcs[SURFACE_LOGICAL_SRC_DATA] = data; - /* The offset for a DWORD scattered message is in dwords. */ - srcs[SURFACE_LOGICAL_SRC_ADDRESS] = - swizzle_nir_scratch_addr(ntb, bld, nir_addr, true); - bld.emit(SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL, fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS); } @@ -6838,9 +6856,6 @@ fs_nir_emit_intrinsic(nir_to_brw_state &ntb, srcs[SURFACE_LOGICAL_SRC_DATA] = bld.vgrf(BRW_TYPE_UD); bld.MOV(srcs[SURFACE_LOGICAL_SRC_DATA], data); - srcs[SURFACE_LOGICAL_SRC_ADDRESS] = - swizzle_nir_scratch_addr(ntb, bld, nir_addr, false); - bld.emit(SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL, fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS); }