diff --git a/src/intel/compiler/brw_eu_defines.h b/src/intel/compiler/brw_eu_defines.h index 8dd3960268c..9ae89a8be84 100644 --- a/src/intel/compiler/brw_eu_defines.h +++ b/src/intel/compiler/brw_eu_defines.h @@ -323,63 +323,9 @@ enum opcode { */ FS_OPCODE_PACK, - /** - * Typed and untyped surface access opcodes. - * - * LOGICAL opcodes are eventually translated to the matching non-LOGICAL - * opcode but instead of taking a single payload blob they expect their - * arguments separately as individual sources: - * - * Source 0: [required] Surface coordinates. - * Source 1: [optional] Operation source. - * Source 2: [required] Surface index. - * Source 3: [required] Number of coordinate components (as UD immediate). - * Source 4: [required] Opcode-specific control immediate, same as source 2 - * of the matching non-LOGICAL opcode. - */ - SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL, - SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL, - SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL, - - SHADER_OPCODE_UNALIGNED_OWORD_BLOCK_READ_LOGICAL, - SHADER_OPCODE_OWORD_BLOCK_WRITE_LOGICAL, - - /** - * Untyped A64 surface access opcodes. - * - * Source 0: 64-bit address - * Source 1: Operational source - * Source 2: [required] Opcode-specific control immediate, same as source 2 - * of the matching non-LOGICAL opcode. - */ - SHADER_OPCODE_A64_UNTYPED_READ_LOGICAL, - SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL, - SHADER_OPCODE_A64_BYTE_SCATTERED_READ_LOGICAL, - SHADER_OPCODE_A64_BYTE_SCATTERED_WRITE_LOGICAL, - SHADER_OPCODE_A64_OWORD_BLOCK_READ_LOGICAL, - SHADER_OPCODE_A64_UNALIGNED_OWORD_BLOCK_READ_LOGICAL, - SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL, - SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL, - - SHADER_OPCODE_TYPED_ATOMIC_LOGICAL, - SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL, - SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL, - SHADER_OPCODE_RND_MODE, SHADER_OPCODE_FLOAT_CONTROL_MODE, - /** - * Byte scattered write/read opcodes. - * - * LOGICAL opcodes are eventually translated to the matching non-LOGICAL - * opcode, but instead of taking a single payload blog they expect their - * arguments separately as individual sources, like untyped write/read. - */ - SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL, - SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL, - SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL, - SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL, - /** * Memory fence messages. * @@ -690,43 +636,6 @@ enum memory_flags { MEMORY_FLAG_INCLUDE_HELPERS = 1 << 1, }; -enum surface_logical_srcs { - /** Surface binding table index */ - SURFACE_LOGICAL_SRC_SURFACE, - /** Surface bindless handle */ - SURFACE_LOGICAL_SRC_SURFACE_HANDLE, - /** Surface address; could be multi-dimensional for typed opcodes */ - SURFACE_LOGICAL_SRC_ADDRESS, - /** Data to be written or used in an atomic op */ - SURFACE_LOGICAL_SRC_DATA, - /** Surface number of dimensions. Affects the size of ADDRESS */ - SURFACE_LOGICAL_SRC_IMM_DIMS, - /** Per-opcode immediate argument. For atomics, this is the atomic opcode */ - SURFACE_LOGICAL_SRC_IMM_ARG, - /** - * Some instructions with side-effects should not be predicated on - * sample mask, e.g. lowered stores to scratch. - */ - SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK, - - SURFACE_LOGICAL_NUM_SRCS -}; - -enum a64_logical_srcs { - /** Address the A64 message operates on */ - A64_LOGICAL_ADDRESS, - /** Source for the operation (unused of LOAD ops) */ - A64_LOGICAL_SRC, - /** Per-opcode immediate argument. Number of dwords, bit size, or atomic op. */ - A64_LOGICAL_ARG, - /** - * Some instructions do want to run on helper lanes (like ray queries). - */ - A64_LOGICAL_ENABLE_HELPERS, - - A64_LOGICAL_NUM_SRCS -}; - enum rt_logical_srcs { /** Address of the globals */ RT_LOGICAL_SRC_GLOBALS, @@ -1414,11 +1323,6 @@ enum brw_message_target { #define GFX8_BTI_STATELESS_NON_COHERENT 253 #define GFX9_BTI_BINDLESS 252 -/* This ID doesn't map anything HW related value. It exists to inform the - * lowering code to not use the bindless heap. - */ -#define GFX125_NON_BINDLESS (1u << 16) - /* Dataport atomic operations for Untyped Atomic Integer Operation message * (and others). */ diff --git a/src/intel/compiler/brw_fs.cpp b/src/intel/compiler/brw_fs.cpp index ae9aa5d24db..c3650df960c 100644 --- a/src/intel/compiler/brw_fs.cpp +++ b/src/intel/compiler/brw_fs.cpp @@ -720,103 +720,6 @@ fs_inst::components_read(unsigned i) const else return 1; - case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL: - case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL: - assert(src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == IMM); - /* Surface coordinates. */ - if (i == SURFACE_LOGICAL_SRC_ADDRESS) - return src[SURFACE_LOGICAL_SRC_IMM_DIMS].ud; - /* Surface operation source (ignored for reads). */ - else if (i == SURFACE_LOGICAL_SRC_DATA) - return 0; - else - return 1; - - case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL: - case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL: - assert(src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == IMM && - src[SURFACE_LOGICAL_SRC_IMM_ARG].file == IMM); - /* Surface coordinates. */ - if (i == SURFACE_LOGICAL_SRC_ADDRESS) - return src[SURFACE_LOGICAL_SRC_IMM_DIMS].ud; - /* Surface operation source. */ - else if (i == SURFACE_LOGICAL_SRC_DATA) - return src[SURFACE_LOGICAL_SRC_IMM_ARG].ud; - else - return 1; - - case SHADER_OPCODE_A64_UNTYPED_READ_LOGICAL: - case SHADER_OPCODE_A64_OWORD_BLOCK_READ_LOGICAL: - case SHADER_OPCODE_A64_UNALIGNED_OWORD_BLOCK_READ_LOGICAL: - assert(src[A64_LOGICAL_ARG].file == IMM); - return 1; - - case SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL: - assert(src[A64_LOGICAL_ARG].file == IMM); - if (i == A64_LOGICAL_SRC) { /* data to write */ - const unsigned comps = src[A64_LOGICAL_ARG].ud / exec_size; - assert(comps > 0); - return comps; - } else { - return 1; - } - - case SHADER_OPCODE_UNALIGNED_OWORD_BLOCK_READ_LOGICAL: - assert(src[SURFACE_LOGICAL_SRC_IMM_ARG].file == IMM); - return 1; - - case SHADER_OPCODE_OWORD_BLOCK_WRITE_LOGICAL: - assert(src[SURFACE_LOGICAL_SRC_IMM_ARG].file == IMM); - if (i == SURFACE_LOGICAL_SRC_DATA) { - const unsigned comps = src[SURFACE_LOGICAL_SRC_IMM_ARG].ud / exec_size; - assert(comps > 0); - return comps; - } else { - return 1; - } - - case SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL: - assert(src[A64_LOGICAL_ARG].file == IMM); - return i == A64_LOGICAL_SRC ? src[A64_LOGICAL_ARG].ud : 1; - - case SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL: - assert(src[A64_LOGICAL_ARG].file == IMM); - return i == A64_LOGICAL_SRC ? - lsc_op_num_data_values(src[A64_LOGICAL_ARG].ud) : 1; - - case SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL: - case SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL: - /* Scattered logical opcodes use the following params: - * src[0] Surface coordinates - * src[1] Surface operation source (ignored for reads) - * src[2] Surface - * src[3] IMM with always 1 dimension. - * src[4] IMM with arg bitsize for scattered read/write 8, 16, 32 - */ - assert(src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == IMM && - src[SURFACE_LOGICAL_SRC_IMM_ARG].file == IMM); - return i == SURFACE_LOGICAL_SRC_DATA ? 0 : 1; - - case SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL: - case SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL: - assert(src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == IMM && - src[SURFACE_LOGICAL_SRC_IMM_ARG].file == IMM); - return 1; - - case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: - case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL: { - assert(src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == IMM && - src[SURFACE_LOGICAL_SRC_IMM_ARG].file == IMM); - const unsigned op = src[SURFACE_LOGICAL_SRC_IMM_ARG].ud; - /* Surface coordinates. */ - if (i == SURFACE_LOGICAL_SRC_ADDRESS) - return src[SURFACE_LOGICAL_SRC_IMM_DIMS].ud; - /* Surface operation source. */ - else if (i == SURFACE_LOGICAL_SRC_DATA) - return lsc_op_num_data_values(op); - else - return 1; - } case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET: return (i == 0 ? 2 : 1); diff --git a/src/intel/compiler/brw_fs_copy_propagation.cpp b/src/intel/compiler/brw_fs_copy_propagation.cpp index 4fe6f853604..c22a74d9f7d 100644 --- a/src/intel/compiler/brw_fs_copy_propagation.cpp +++ b/src/intel/compiler/brw_fs_copy_propagation.cpp @@ -1207,14 +1207,6 @@ try_constant_propagate_value(brw_reg val, brw_reg_type dst_type, case SHADER_OPCODE_TG4_OFFSET_BIAS_LOGICAL: case SHADER_OPCODE_SAMPLEINFO_LOGICAL: case SHADER_OPCODE_IMAGE_SIZE_LOGICAL: - case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: - case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL: - case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL: - case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL: - case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL: - case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL: - case SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL: - case SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL: case SHADER_OPCODE_MEMORY_LOAD_LOGICAL: case SHADER_OPCODE_MEMORY_STORE_LOGICAL: case SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL: @@ -1229,17 +1221,6 @@ try_constant_propagate_value(brw_reg val, brw_reg_type dst_type, progress = true; break; - case SHADER_OPCODE_UNALIGNED_OWORD_BLOCK_READ_LOGICAL: - /* The address of the send message cannot be immediate (see the - * assertions in brw_set_src0 in brw_eu_emit.c). There is no mechanism - * to legalize it later, so do not generate the invalid thing here. - */ - if (arg != SURFACE_LOGICAL_SRC_ADDRESS) { - inst->src[arg] = val; - progress = true; - } - break; - default: break; } diff --git a/src/intel/compiler/brw_fs_dead_code_eliminate.cpp b/src/intel/compiler/brw_fs_dead_code_eliminate.cpp index 1ce70359eed..d22d5ba21d4 100644 --- a/src/intel/compiler/brw_fs_dead_code_eliminate.cpp +++ b/src/intel/compiler/brw_fs_dead_code_eliminate.cpp @@ -56,9 +56,6 @@ static bool can_omit_write(const fs_inst *inst) { switch (inst->opcode) { - case SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL: - case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: - case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL: case SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL: return true; default: diff --git a/src/intel/compiler/brw_fs_lower_simd_width.cpp b/src/intel/compiler/brw_fs_lower_simd_width.cpp index 921e8c9241f..b751d56dca9 100644 --- a/src/intel/compiler/brw_fs_lower_simd_width.cpp +++ b/src/intel/compiler/brw_fs_lower_simd_width.cpp @@ -407,38 +407,6 @@ brw_fs_get_lowered_simd_width(const fs_visitor *shader, const fs_inst *inst) */ return devinfo->ver < 20 ? 8 : 16; - case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL: - case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL: - case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL: - return devinfo->ver < 20 ? 8 : inst->exec_size; - - case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: - case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL: - case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL: - case SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL: - case SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL: - case SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL: - case SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL: - case SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL: - case SHADER_OPCODE_A64_UNTYPED_READ_LOGICAL: - case SHADER_OPCODE_A64_BYTE_SCATTERED_WRITE_LOGICAL: - case SHADER_OPCODE_A64_BYTE_SCATTERED_READ_LOGICAL: - return devinfo->ver < 20 ? - MIN2(16, inst->exec_size) : - inst->exec_size; - - case SHADER_OPCODE_A64_OWORD_BLOCK_READ_LOGICAL: - case SHADER_OPCODE_A64_UNALIGNED_OWORD_BLOCK_READ_LOGICAL: - case SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL: - return devinfo->ver < 20 ? - MIN2(16, inst->exec_size) : - inst->exec_size; - - case SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL: - return devinfo->ver < 20 ? - devinfo->has_lsc ? MIN2(16, inst->exec_size) : 8 : - inst->exec_size; - case SHADER_OPCODE_URB_READ_LOGICAL: case SHADER_OPCODE_URB_WRITE_LOGICAL: return MIN2(devinfo->ver < 20 ? 8 : 16, inst->exec_size); diff --git a/src/intel/compiler/brw_lower_logical_sends.cpp b/src/intel/compiler/brw_lower_logical_sends.cpp index 8d254bc04f1..3c3588dbf27 100644 --- a/src/intel/compiler/brw_lower_logical_sends.cpp +++ b/src/intel/compiler/brw_lower_logical_sends.cpp @@ -1568,7 +1568,31 @@ lower_lsc_memory_logical_send(const fs_builder &bld, fs_inst *inst) } static brw_reg -emit_a64_oword_block_header(const fs_builder &bld, const brw_reg &addr); +emit_a64_oword_block_header(const fs_builder &bld, const brw_reg &addr) +{ + const fs_builder ubld = bld.exec_all().group(8, 0); + + assert(brw_type_size_bytes(addr.type) == 8 && addr.stride == 0); + + brw_reg expanded_addr = addr; + if (addr.file == UNIFORM) { + /* We can't do stride 1 with the UNIFORM file, it requires stride 0 */ + expanded_addr = ubld.vgrf(BRW_TYPE_UQ); + expanded_addr.stride = 0; + ubld.MOV(expanded_addr, retype(addr, BRW_TYPE_UQ)); + } + + brw_reg header = ubld.vgrf(BRW_TYPE_UD); + ubld.MOV(header, brw_imm_ud(0)); + + /* Use a 2-wide MOV to fill out the address */ + brw_reg addr_vec2 = expanded_addr; + addr_vec2.type = BRW_TYPE_UD; + addr_vec2.stride = 1; + ubld.group(2, 0).MOV(header, addr_vec2); + + return header; +} static void lower_hdc_memory_logical_send(const fs_builder &bld, fs_inst *inst) @@ -1855,911 +1879,6 @@ lower_hdc_memory_logical_send(const fs_builder &bld, fs_inst *inst) inst->src[3] = payload2; } -static void -lower_surface_logical_send(const fs_builder &bld, fs_inst *inst) -{ - const brw_compiler *compiler = bld.shader->compiler; - const intel_device_info *devinfo = bld.shader->devinfo; - - /* Get the logical send arguments. */ - const brw_reg addr = inst->src[SURFACE_LOGICAL_SRC_ADDRESS]; - const brw_reg src = inst->src[SURFACE_LOGICAL_SRC_DATA]; - const brw_reg surface = inst->src[SURFACE_LOGICAL_SRC_SURFACE]; - const brw_reg surface_handle = inst->src[SURFACE_LOGICAL_SRC_SURFACE_HANDLE]; - const UNUSED brw_reg dims = inst->src[SURFACE_LOGICAL_SRC_IMM_DIMS]; - const brw_reg arg = inst->src[SURFACE_LOGICAL_SRC_IMM_ARG]; - const brw_reg allow_sample_mask = - inst->src[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK]; - assert(arg.file == IMM); - assert(allow_sample_mask.file == IMM); - - /* Calculate the total number of components of the payload. */ - const unsigned addr_sz = inst->components_read(SURFACE_LOGICAL_SRC_ADDRESS); - const unsigned src_sz = inst->components_read(SURFACE_LOGICAL_SRC_DATA); - - const bool is_typed_access = - inst->opcode == SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL || - inst->opcode == SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL || - inst->opcode == SHADER_OPCODE_TYPED_ATOMIC_LOGICAL; - - const bool is_surface_access = is_typed_access || - inst->opcode == SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL || - inst->opcode == SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL || - inst->opcode == SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL; - - const bool is_stateless = - surface.file == IMM && (surface.ud == BRW_BTI_STATELESS || - surface.ud == GFX8_BTI_STATELESS_NON_COHERENT); - - const bool has_side_effects = inst->has_side_effects(); - - brw_reg sample_mask = allow_sample_mask.ud ? brw_sample_mask_reg(bld) : - brw_reg(brw_imm_ud(0xffffffff)); - - brw_reg header; - if (is_stateless) { - assert(!is_surface_access); - fs_builder ubld = bld.exec_all().group(8, 0); - header = ubld.vgrf(BRW_TYPE_UD); - ubld.emit(SHADER_OPCODE_SCRATCH_HEADER, header, brw_ud8_grf(0, 0)); - } - const unsigned header_sz = header.file != BAD_FILE ? 1 : 0; - - brw_reg payload, payload2; - unsigned mlen, ex_mlen = 0; - if (src.file == BAD_FILE || header.file == BAD_FILE) { - /* We have split sends on gfx9 and above */ - if (header.file == BAD_FILE) { - payload = bld.move_to_vgrf(addr, addr_sz); - payload2 = bld.move_to_vgrf(src, src_sz); - mlen = addr_sz * (inst->exec_size / 8); - ex_mlen = src_sz * (inst->exec_size / 8); - } else { - assert(src.file == BAD_FILE); - payload = header; - payload2 = bld.move_to_vgrf(addr, addr_sz); - mlen = header_sz; - ex_mlen = addr_sz * (inst->exec_size / 8); - } - } else { - /* Allocate space for the payload. */ - const unsigned sz = header_sz + addr_sz + src_sz; - payload = bld.vgrf(BRW_TYPE_UD, sz); - brw_reg *const components = new brw_reg[sz]; - unsigned n = 0; - - /* Construct the payload. */ - if (header.file != BAD_FILE) - components[n++] = header; - - for (unsigned i = 0; i < addr_sz; i++) - components[n++] = offset(addr, bld, i); - - for (unsigned i = 0; i < src_sz; i++) - components[n++] = offset(src, bld, i); - - bld.LOAD_PAYLOAD(payload, components, sz, header_sz); - mlen = header_sz + (addr_sz + src_sz) * inst->exec_size / 8; - - delete[] components; - } - - /* Predicate the instruction on the sample mask if no header is - * provided. - */ - if ((header.file == BAD_FILE || !is_surface_access) && - sample_mask.file != BAD_FILE && sample_mask.file != IMM) - brw_emit_predicate_on_sample_mask(bld, inst); - - uint32_t sfid; - switch (inst->opcode) { - case SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL: - case SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL: - /* Byte scattered opcodes go through the normal data cache */ - sfid = GFX7_SFID_DATAPORT_DATA_CACHE; - break; - - case SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL: - case SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL: - sfid = GFX7_SFID_DATAPORT_DATA_CACHE; - break; - - case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL: - case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL: - case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: - /* Untyped Surface messages go through the data cache but the SFID value - * changed on Haswell. - */ - sfid = HSW_SFID_DATAPORT_DATA_CACHE_1; - break; - - case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL: - case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL: - case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL: - /* Typed surface messages go through the render cache on IVB and the - * data cache on HSW+. - */ - sfid = HSW_SFID_DATAPORT_DATA_CACHE_1; - break; - - default: - unreachable("Unsupported surface opcode"); - } - - uint32_t desc; - switch (inst->opcode) { - case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL: - desc = brw_dp_untyped_surface_rw_desc(devinfo, inst->exec_size, - arg.ud, /* num_channels */ - false /* write */); - break; - - case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL: - desc = brw_dp_untyped_surface_rw_desc(devinfo, inst->exec_size, - arg.ud, /* num_channels */ - true /* write */); - break; - - case SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL: - desc = brw_dp_byte_scattered_rw_desc(devinfo, inst->exec_size, - arg.ud, /* bit_size */ - false /* write */); - break; - - case SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL: - desc = brw_dp_byte_scattered_rw_desc(devinfo, inst->exec_size, - arg.ud, /* bit_size */ - true /* write */); - break; - - case SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL: - assert(arg.ud == 32); /* bit_size */ - desc = brw_dp_dword_scattered_rw_desc(devinfo, inst->exec_size, - false /* write */); - break; - - case SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL: - assert(arg.ud == 32); /* bit_size */ - desc = brw_dp_dword_scattered_rw_desc(devinfo, inst->exec_size, - true /* write */); - break; - - case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: - if (lsc_opcode_is_atomic_float((enum lsc_opcode) arg.ud)) { - desc = brw_dp_untyped_atomic_float_desc(devinfo, inst->exec_size, - lsc_op_to_legacy_atomic(arg.ud), - !inst->dst.is_null()); - } else { - desc = brw_dp_untyped_atomic_desc(devinfo, inst->exec_size, - lsc_op_to_legacy_atomic(arg.ud), - !inst->dst.is_null()); - } - break; - - case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL: - desc = brw_dp_typed_surface_rw_desc(devinfo, inst->exec_size, inst->group, - arg.ud, /* num_channels */ - false /* write */); - break; - - case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL: - desc = brw_dp_typed_surface_rw_desc(devinfo, inst->exec_size, inst->group, - arg.ud, /* num_channels */ - true /* write */); - break; - - case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL: - desc = brw_dp_typed_atomic_desc(devinfo, inst->exec_size, inst->group, - lsc_op_to_legacy_atomic(arg.ud), - !inst->dst.is_null()); - break; - - default: - unreachable("Unknown surface logical instruction"); - } - - /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; - inst->mlen = mlen; - inst->ex_mlen = ex_mlen; - inst->header_size = header_sz; - inst->send_has_side_effects = has_side_effects; - inst->send_is_volatile = !has_side_effects; - inst->send_ex_bso = surface_handle.file != BAD_FILE && - compiler->extended_bindless_surface_offset; - - /* Set up SFID and descriptors */ - inst->sfid = sfid; - setup_surface_descriptors(bld, inst, desc, surface, surface_handle); - - inst->resize_sources(4); - - /* Finally, the payload */ - inst->src[2] = payload; - inst->src[3] = payload2; -} - -static enum lsc_data_size -lsc_bits_to_data_size(unsigned bit_size) -{ - switch (bit_size / 8) { - case 1: return LSC_DATA_SIZE_D8U32; - case 2: return LSC_DATA_SIZE_D16U32; - case 4: return LSC_DATA_SIZE_D32; - case 8: return LSC_DATA_SIZE_D64; - default: - unreachable("Unsupported data size."); - } -} - -static void -lower_lsc_surface_logical_send(bblock_t *block, const fs_builder &bld, - fs_inst *inst) -{ - const brw_compiler *compiler = bld.shader->compiler; - const intel_device_info *devinfo = bld.shader->devinfo; - assert(devinfo->has_lsc); - - /* Get the logical send arguments. */ - const brw_reg addr = inst->src[SURFACE_LOGICAL_SRC_ADDRESS]; - const brw_reg src = inst->src[SURFACE_LOGICAL_SRC_DATA]; - const brw_reg surface = inst->src[SURFACE_LOGICAL_SRC_SURFACE]; - const brw_reg surface_handle = inst->src[SURFACE_LOGICAL_SRC_SURFACE_HANDLE]; - const brw_reg dims = inst->src[SURFACE_LOGICAL_SRC_IMM_DIMS]; - const brw_reg arg = inst->src[SURFACE_LOGICAL_SRC_IMM_ARG]; - const brw_reg allow_sample_mask = - inst->src[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK]; - assert(arg.file == IMM); - assert(allow_sample_mask.file == IMM); - assert(dims.file == IMM); - - /* Calculate the total number of components of the payload. */ - const unsigned addr_sz = inst->components_read(SURFACE_LOGICAL_SRC_ADDRESS); - const unsigned src_comps = inst->components_read(SURFACE_LOGICAL_SRC_DATA); - const unsigned src_sz = brw_type_size_bytes(src.type); - const unsigned dst_sz = brw_type_size_bytes(inst->dst.type); - - const bool has_side_effects = inst->has_side_effects(); - - const bool is_typed_access = - inst->opcode == SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL || - inst->opcode == SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL || - inst->opcode == SHADER_OPCODE_TYPED_ATOMIC_LOGICAL; - - unsigned num_components = 0; - - unsigned ex_mlen = 0; - brw_reg payload, payload2; - payload = bld.move_to_vgrf(addr, addr_sz); - if (src.file != BAD_FILE) { - payload2 = bld.move_to_vgrf(src, src_comps); - ex_mlen = (src_comps * src_sz * inst->exec_size) / REG_SIZE; - } - - /* Predicate the instruction on the sample mask if needed */ - brw_reg sample_mask = allow_sample_mask.ud ? brw_sample_mask_reg(bld) : - brw_reg(brw_imm_ud(0xffffffff)); - if (sample_mask.file != BAD_FILE && sample_mask.file != IMM) - brw_emit_predicate_on_sample_mask(bld, inst); - - if (surface.file == IMM && surface.ud == GFX7_BTI_SLM) - inst->sfid = GFX12_SFID_SLM; - else if (is_typed_access) - inst->sfid = GFX12_SFID_TGM; - else - inst->sfid = GFX12_SFID_UGM; - - /* Dimensions should always be 1 for UGM/UGML/SLM or - * between 1 and 4 for TGM - */ - assert(dims.ud == 1 || - (inst->sfid == GFX12_SFID_TGM && - dims.ud >= 1 && dims.ud <= 4)); - - /* We should have exactly one of surface and surface_handle. For scratch - * messages generated by brw_fs_nir.cpp we also allow a special value to - * know what heap base we should use in STATE_BASE_ADDRESS (SS = Surface - * State Offset, or BSS = Bindless Surface State Offset). - */ - bool non_bindless = surface.file == IMM && surface.ud == GFX125_NON_BINDLESS; - assert((surface.file == BAD_FILE) != (surface_handle.file == BAD_FILE) || - (non_bindless && surface_handle.file != BAD_FILE)); - - enum lsc_addr_surface_type surf_type; - if (surface_handle.file != BAD_FILE) { - if (surface.file == BAD_FILE) { - assert(!non_bindless); - surf_type = LSC_ADDR_SURFTYPE_BSS; - } else { - assert(surface.file == IMM && - (surface.ud == 0 || surface.ud == GFX125_NON_BINDLESS)); - surf_type = non_bindless ? LSC_ADDR_SURFTYPE_SS : LSC_ADDR_SURFTYPE_BSS; - } - } else if (surface.file == IMM && surface.ud == GFX7_BTI_SLM) - surf_type = LSC_ADDR_SURFTYPE_FLAT; - else - surf_type = LSC_ADDR_SURFTYPE_BTI; - - switch (inst->opcode) { - case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL: - num_components = arg.ud; - inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD_CMASK, - surf_type, LSC_ADDR_SIZE_A32, - LSC_DATA_SIZE_D32, - BITSET_MASK(num_components), - false /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS)); - break; - case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL: - num_components = arg.ud; - inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD, - surf_type, LSC_ADDR_SIZE_A32, - LSC_DATA_SIZE_D32, num_components, - false /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS)); - break; - case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL: - num_components = arg.ud; - inst->desc = lsc_msg_desc(devinfo, LSC_OP_STORE_CMASK, - surf_type, LSC_ADDR_SIZE_A32, - LSC_DATA_SIZE_D32, - BITSET_MASK(num_components), - false /* transpose */, - LSC_CACHE(devinfo, STORE, L1STATE_L3MOCS)); - break; - case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL: - num_components = arg.ud; - inst->desc = lsc_msg_desc(devinfo, LSC_OP_STORE, - surf_type, LSC_ADDR_SIZE_A32, - LSC_DATA_SIZE_D32, num_components, - false /* transpose */, - LSC_CACHE(devinfo, STORE, L1STATE_L3MOCS)); - break; - case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL: - case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: { - /* Bspec: Atomic instruction -> Cache section: - * - * Atomic messages are always forced to "un-cacheable" in the L1 - * cache. - */ - enum lsc_opcode opcode = (enum lsc_opcode) arg.ud; - - num_components = 1; - inst->desc = lsc_msg_desc(devinfo, opcode, - surf_type, LSC_ADDR_SIZE_A32, - lsc_bits_to_data_size(dst_sz * 8), - num_components, - false /* transpose */, - LSC_CACHE(devinfo, STORE, L1UC_L3WB)); - break; - } - case SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL: - case SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL: - num_components = 1; - inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD, - surf_type, LSC_ADDR_SIZE_A32, - lsc_bits_to_data_size(arg.ud), - num_components, - false /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS)); - break; - case SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL: - case SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL: - num_components = 1; - inst->desc = lsc_msg_desc(devinfo, LSC_OP_STORE, - surf_type, LSC_ADDR_SIZE_A32, - lsc_bits_to_data_size(arg.ud), - num_components, - false /* transpose */, - LSC_CACHE(devinfo, STORE, L1STATE_L3MOCS)); - break; - default: - unreachable("Unknown surface logical instruction"); - } - - /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; - inst->mlen = lsc_msg_addr_len(devinfo, LSC_ADDR_SIZE_A32, inst->exec_size * dims.ud); - inst->ex_mlen = ex_mlen; - inst->header_size = 0; - inst->send_has_side_effects = has_side_effects; - inst->send_is_volatile = !has_side_effects; - inst->send_ex_bso = surf_type == LSC_ADDR_SURFTYPE_BSS && - compiler->extended_bindless_surface_offset; - - /* Messages with destination datatypes narrower than a dword use a - * D*32 LSC data size, update the destination to use a temporary of - * the raw (UD) return payload datatype. - */ - if (dst_sz < 4) { - assert(lsc_data_size_bytes(lsc_bits_to_data_size(dst_sz * 8)) == 4); - assert(inst->size_written == inst->dst.component_size(inst->exec_size)); - const brw_reg dest32 = bld.vgrf(BRW_TYPE_UD); - const brw_reg_type t = brw_int_type(dst_sz, false); - bld.at(block, inst->next).MOV(retype(inst->dst, t), dest32); - inst->dst = dest32; - inst->size_written = inst->dst.component_size(inst->exec_size); - } - - inst->resize_sources(4); - - if (non_bindless) { - inst->src[0] = brw_imm_ud(0); /* desc */ - inst->src[1] = surface_handle; /* ex_desc */ - } else { - setup_lsc_surface_descriptors(bld, inst, inst->desc, - surface.file != BAD_FILE ? - surface : surface_handle); - } - - /* Finally, the payload */ - inst->src[2] = payload; - inst->src[3] = payload2; -} - -static void -lower_lsc_block_logical_send(const fs_builder &bld, fs_inst *inst) -{ - const brw_compiler *compiler = bld.shader->compiler; - const intel_device_info *devinfo = bld.shader->devinfo; - assert(devinfo->has_lsc); - - /* Get the logical send arguments. */ - const brw_reg addr = inst->src[SURFACE_LOGICAL_SRC_ADDRESS]; - const brw_reg src = inst->src[SURFACE_LOGICAL_SRC_DATA]; - const brw_reg surface = inst->src[SURFACE_LOGICAL_SRC_SURFACE]; - const brw_reg surface_handle = inst->src[SURFACE_LOGICAL_SRC_SURFACE_HANDLE]; - const brw_reg arg = inst->src[SURFACE_LOGICAL_SRC_IMM_ARG]; - assert(arg.file == IMM); - assert(inst->src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == BAD_FILE); - assert(inst->src[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK].file == BAD_FILE); - - const bool is_stateless = - surface.file == IMM && (surface.ud == BRW_BTI_STATELESS || - surface.ud == GFX8_BTI_STATELESS_NON_COHERENT); - - const bool has_side_effects = inst->has_side_effects(); - - const bool write = inst->opcode == SHADER_OPCODE_OWORD_BLOCK_WRITE_LOGICAL; - - fs_builder ubld = bld.exec_all().group(1, 0); - brw_reg stateless_ex_desc; - if (is_stateless) { - stateless_ex_desc = ubld.vgrf(BRW_TYPE_UD); - ubld.AND(stateless_ex_desc, - retype(brw_vec1_grf(0, 5), BRW_TYPE_UD), - brw_imm_ud(INTEL_MASK(31, 10))); - if (devinfo->ver >= 20) - ubld.SHR(stateless_ex_desc, stateless_ex_desc, brw_imm_ud(4)); - } - - brw_reg data; - if (write) { - const unsigned src_sz = inst->components_read(SURFACE_LOGICAL_SRC_DATA); - data = retype(bld.move_to_vgrf(src, src_sz), BRW_TYPE_UD); - } - - inst->opcode = SHADER_OPCODE_SEND; - if (surface.file == IMM && surface.ud == GFX7_BTI_SLM) - inst->sfid = GFX12_SFID_SLM; - else - inst->sfid = GFX12_SFID_UGM; - const enum lsc_addr_surface_type surf_type = - inst->sfid == GFX12_SFID_SLM ? - LSC_ADDR_SURFTYPE_FLAT : - surface.file == BAD_FILE ? - LSC_ADDR_SURFTYPE_BSS : LSC_ADDR_SURFTYPE_BTI; - inst->desc = lsc_msg_desc(devinfo, - write ? LSC_OP_STORE : LSC_OP_LOAD, - surf_type, - LSC_ADDR_SIZE_A32, - LSC_DATA_SIZE_D32, - arg.ud /* num_channels */, - true /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS)); - - inst->mlen = lsc_msg_addr_len(devinfo, LSC_ADDR_SIZE_A32, 1); - inst->exec_size = 1; - inst->ex_mlen = write ? DIV_ROUND_UP(arg.ud, 8) : 0; - inst->header_size = 0; - inst->send_has_side_effects = has_side_effects; - inst->send_is_volatile = !has_side_effects; - inst->send_ex_bso = surf_type == LSC_ADDR_SURFTYPE_BSS && - compiler->extended_bindless_surface_offset; - - inst->resize_sources(4); - - if (stateless_ex_desc.file != BAD_FILE) { - inst->src[0] = brw_imm_ud(0); /* desc */ - inst->src[1] = stateless_ex_desc; /* ex_desc */ - } else { - setup_lsc_surface_descriptors(bld, inst, inst->desc, - surface.file != BAD_FILE ? - surface : surface_handle); - } - inst->src[2] = addr; /* payload */ - inst->src[3] = data; /* payload2 */ -} - -static void -lower_surface_block_logical_send(const fs_builder &bld, fs_inst *inst) -{ - const intel_device_info *devinfo = bld.shader->devinfo; - - /* Get the logical send arguments. */ - const brw_reg addr = inst->src[SURFACE_LOGICAL_SRC_ADDRESS]; - const brw_reg src = inst->src[SURFACE_LOGICAL_SRC_DATA]; - const brw_reg surface = inst->src[SURFACE_LOGICAL_SRC_SURFACE]; - const brw_reg surface_handle = inst->src[SURFACE_LOGICAL_SRC_SURFACE_HANDLE]; - const brw_reg arg = inst->src[SURFACE_LOGICAL_SRC_IMM_ARG]; - assert(arg.file == IMM); - assert(inst->src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == BAD_FILE); - assert(inst->src[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK].file == BAD_FILE); - - const bool is_stateless = - surface.file == IMM && (surface.ud == BRW_BTI_STATELESS || - surface.ud == GFX8_BTI_STATELESS_NON_COHERENT); - - const bool has_side_effects = inst->has_side_effects(); - - /* SLM block reads must use the 16B-aligned OWord Block Read messages, - * as the unaligned message doesn't exist for SLM. However, we still - * use SHADER_OPCODE_UNALIGNED_OWORD_BLOCK_READ_LOGICAL in that case - * (to avoid adding more opcodes), but only emit it with 16B alignment. - */ - const bool align_16B = - inst->opcode != SHADER_OPCODE_UNALIGNED_OWORD_BLOCK_READ_LOGICAL || - (surface.file == IMM && surface.ud == GFX7_BTI_SLM); - - const bool write = inst->opcode == SHADER_OPCODE_OWORD_BLOCK_WRITE_LOGICAL; - - /* The address is stored in the header. See MH_A32_GO and MH_BTS_GO. */ - fs_builder ubld = bld.exec_all().group(8, 0); - brw_reg header = ubld.vgrf(BRW_TYPE_UD); - - if (is_stateless) - ubld.emit(SHADER_OPCODE_SCRATCH_HEADER, header, brw_ud8_grf(0, 0)); - else - ubld.MOV(header, brw_imm_d(0)); - - /* Address in OWord units when aligned to OWords. */ - if (align_16B) - ubld.group(1, 0).SHR(component(header, 2), addr, brw_imm_ud(4)); - else - ubld.group(1, 0).MOV(component(header, 2), addr); - - brw_reg data; - unsigned ex_mlen = 0; - if (write) { - const unsigned src_sz = inst->components_read(SURFACE_LOGICAL_SRC_DATA); - data = retype(bld.move_to_vgrf(src, src_sz), BRW_TYPE_UD); - ex_mlen = src_sz * brw_type_size_bytes(src.type) * inst->exec_size / REG_SIZE; - } - - inst->opcode = SHADER_OPCODE_SEND; - inst->mlen = 1; - inst->ex_mlen = ex_mlen; - inst->header_size = 1; - inst->send_has_side_effects = has_side_effects; - inst->send_is_volatile = !has_side_effects; - - inst->sfid = GFX7_SFID_DATAPORT_DATA_CACHE; - - const uint32_t desc = brw_dp_oword_block_rw_desc(devinfo, align_16B, - arg.ud, write); - setup_surface_descriptors(bld, inst, desc, surface, surface_handle); - - inst->resize_sources(4); - - inst->src[2] = header; - inst->src[3] = data; -} - -static brw_reg -emit_a64_oword_block_header(const fs_builder &bld, const brw_reg &addr) -{ - const fs_builder ubld = bld.exec_all().group(8, 0); - - assert(brw_type_size_bytes(addr.type) == 8 && addr.stride == 0); - - brw_reg expanded_addr = addr; - if (addr.file == UNIFORM) { - /* We can't do stride 1 with the UNIFORM file, it requires stride 0 */ - expanded_addr = ubld.vgrf(BRW_TYPE_UQ); - expanded_addr.stride = 0; - ubld.MOV(expanded_addr, retype(addr, BRW_TYPE_UQ)); - } - - brw_reg header = ubld.vgrf(BRW_TYPE_UD); - ubld.MOV(header, brw_imm_ud(0)); - - /* Use a 2-wide MOV to fill out the address */ - brw_reg addr_vec2 = expanded_addr; - addr_vec2.type = BRW_TYPE_UD; - addr_vec2.stride = 1; - ubld.group(2, 0).MOV(header, addr_vec2); - - return header; -} - -static void -emit_fragment_mask(const fs_builder &bld, fs_inst *inst) -{ - assert(inst->src[A64_LOGICAL_ENABLE_HELPERS].file == IMM); - const bool enable_helpers = inst->src[A64_LOGICAL_ENABLE_HELPERS].ud; - - /* If we're a fragment shader, we have to predicate with the sample mask to - * avoid helper invocations to avoid helper invocations in instructions - * with side effects, unless they are explicitly required. - * - * There are also special cases when we actually want to run on helpers - * (ray queries). - */ - assert(bld.shader->stage == MESA_SHADER_FRAGMENT); - if (enable_helpers) - emit_predicate_on_vector_mask(bld, inst); - else if (inst->has_side_effects()) - brw_emit_predicate_on_sample_mask(bld, inst); -} - -static void -lower_lsc_a64_logical_send(bblock_t *block, const fs_builder &bld, fs_inst *inst) -{ - const intel_device_info *devinfo = bld.shader->devinfo; - - /* Get the logical send arguments. */ - const brw_reg addr = inst->src[A64_LOGICAL_ADDRESS]; - const brw_reg src = inst->src[A64_LOGICAL_SRC]; - const unsigned src_sz = brw_type_size_bytes(src.type); - const unsigned dst_sz = brw_type_size_bytes(inst->dst.type); - - const unsigned src_comps = inst->components_read(1); - assert(inst->src[A64_LOGICAL_ARG].file == IMM); - const unsigned arg = inst->src[A64_LOGICAL_ARG].ud; - const bool has_side_effects = inst->has_side_effects(); - - brw_reg payload = retype(bld.move_to_vgrf(addr, 1), BRW_TYPE_UD); - brw_reg payload2 = retype(bld.move_to_vgrf(src, src_comps), BRW_TYPE_UD); - unsigned ex_mlen = src_comps * src_sz * inst->exec_size / REG_SIZE; - unsigned num_components = 0; - - switch (inst->opcode) { - case SHADER_OPCODE_A64_UNTYPED_READ_LOGICAL: - num_components = arg; - inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD, - LSC_ADDR_SURFTYPE_FLAT, LSC_ADDR_SIZE_A64, - LSC_DATA_SIZE_D32, num_components, - false /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS)); - break; - case SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL: - num_components = arg; - inst->desc = lsc_msg_desc(devinfo, LSC_OP_STORE, - LSC_ADDR_SURFTYPE_FLAT, LSC_ADDR_SIZE_A64, - LSC_DATA_SIZE_D32, num_components, - false /* transpose */, - LSC_CACHE(devinfo, STORE, L1STATE_L3MOCS)); - break; - case SHADER_OPCODE_A64_BYTE_SCATTERED_READ_LOGICAL: - num_components = 1; - inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD, - LSC_ADDR_SURFTYPE_FLAT, LSC_ADDR_SIZE_A64, - lsc_bits_to_data_size(arg), - num_components, - false /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS)); - break; - case SHADER_OPCODE_A64_BYTE_SCATTERED_WRITE_LOGICAL: - num_components = 1; - inst->desc = lsc_msg_desc(devinfo, LSC_OP_STORE, - LSC_ADDR_SURFTYPE_FLAT, LSC_ADDR_SIZE_A64, - lsc_bits_to_data_size(arg), - num_components, - false /* transpose */, - LSC_CACHE(devinfo, STORE, L1STATE_L3MOCS)); - break; - case SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL: { - /* Bspec: Atomic instruction -> Cache section: - * - * Atomic messages are always forced to "un-cacheable" in the L1 - * cache. - */ - enum lsc_opcode opcode = (enum lsc_opcode) arg; - num_components = 1; - inst->desc = lsc_msg_desc(devinfo, opcode, - LSC_ADDR_SURFTYPE_FLAT, LSC_ADDR_SIZE_A64, - lsc_bits_to_data_size(dst_sz * 8), - num_components, - false /* transpose */, - LSC_CACHE(devinfo, STORE, L1UC_L3WB)); - break; - } - case SHADER_OPCODE_A64_OWORD_BLOCK_READ_LOGICAL: - case SHADER_OPCODE_A64_UNALIGNED_OWORD_BLOCK_READ_LOGICAL: - num_components = arg; - inst->exec_size = 1; - inst->desc = lsc_msg_desc(devinfo, - LSC_OP_LOAD, - LSC_ADDR_SURFTYPE_FLAT, - LSC_ADDR_SIZE_A64, - LSC_DATA_SIZE_D32, - num_components, - true /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS)); - break; - case SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL: - num_components = arg; - inst->exec_size = 1; - inst->desc = lsc_msg_desc(devinfo, - LSC_OP_STORE, - LSC_ADDR_SURFTYPE_FLAT, - LSC_ADDR_SIZE_A64, - LSC_DATA_SIZE_D32, - num_components, - true /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS)); - - break; - default: - unreachable("Unknown A64 logical instruction"); - } - - if (bld.shader->stage == MESA_SHADER_FRAGMENT) - emit_fragment_mask(bld, inst); - - /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; - inst->mlen = lsc_msg_addr_len(devinfo, LSC_ADDR_SIZE_A64, inst->exec_size); - inst->ex_mlen = ex_mlen; - inst->header_size = 0; - inst->send_has_side_effects = has_side_effects; - inst->send_is_volatile = !has_side_effects; - - /* Messages with destination datatypes narrower than a dword use a - * D*32 LSC data size, update the destination to use a temporary of - * the raw (UD) return payload datatype. - */ - if (dst_sz < 4) { - assert(lsc_data_size_bytes(lsc_bits_to_data_size(dst_sz * 8)) == 4); - assert(inst->size_written == inst->dst.component_size(inst->exec_size)); - const brw_reg dest32 = bld.vgrf(BRW_TYPE_UD); - const brw_reg_type t = brw_int_type(dst_sz, false); - bld.at(block, inst->next).MOV(retype(inst->dst, t), dest32); - inst->dst = dest32; - inst->size_written = inst->dst.component_size(inst->exec_size); - } - - /* Set up SFID and descriptors */ - inst->sfid = GFX12_SFID_UGM; - inst->resize_sources(4); - inst->src[0] = brw_imm_ud(0); /* desc */ - inst->src[1] = brw_imm_ud(0); /* ex_desc */ - inst->src[2] = payload; - inst->src[3] = payload2; -} - -static void -lower_a64_logical_send(const fs_builder &bld, fs_inst *inst) -{ - const intel_device_info *devinfo = bld.shader->devinfo; - - const brw_reg addr = inst->src[A64_LOGICAL_ADDRESS]; - const brw_reg src = inst->src[A64_LOGICAL_SRC]; - const unsigned src_comps = inst->components_read(1); - assert(inst->src[A64_LOGICAL_ARG].file == IMM); - const unsigned arg = inst->src[A64_LOGICAL_ARG].ud; - const bool has_side_effects = inst->has_side_effects(); - - brw_reg payload, payload2; - unsigned mlen, ex_mlen = 0, header_size = 0; - if (inst->opcode == SHADER_OPCODE_A64_OWORD_BLOCK_READ_LOGICAL || - inst->opcode == SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL || - inst->opcode == SHADER_OPCODE_A64_UNALIGNED_OWORD_BLOCK_READ_LOGICAL) { - - /* OWORD messages only take a scalar address in a header */ - mlen = 1; - header_size = 1; - payload = emit_a64_oword_block_header(bld, addr); - - if (inst->opcode == SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL) { - ex_mlen = src_comps * brw_type_size_bytes(src.type) * inst->exec_size / REG_SIZE; - payload2 = retype(bld.move_to_vgrf(src, src_comps), BRW_TYPE_UD); - } - } else { - /* On Skylake and above, we have SENDS */ - mlen = 2 * (inst->exec_size / 8); - ex_mlen = src_comps * brw_type_size_bytes(src.type) * inst->exec_size / REG_SIZE; - payload = retype(bld.move_to_vgrf(addr, 1), BRW_TYPE_UD); - payload2 = retype(bld.move_to_vgrf(src, src_comps), BRW_TYPE_UD); - } - - uint32_t desc; - switch (inst->opcode) { - case SHADER_OPCODE_A64_UNTYPED_READ_LOGICAL: - desc = brw_dp_a64_untyped_surface_rw_desc(devinfo, inst->exec_size, - arg, /* num_channels */ - false /* write */); - break; - - case SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL: - desc = brw_dp_a64_untyped_surface_rw_desc(devinfo, inst->exec_size, - arg, /* num_channels */ - true /* write */); - break; - - case SHADER_OPCODE_A64_OWORD_BLOCK_READ_LOGICAL: - desc = brw_dp_a64_oword_block_rw_desc(devinfo, - true, /* align_16B */ - arg, /* num_dwords */ - false /* write */); - break; - - case SHADER_OPCODE_A64_UNALIGNED_OWORD_BLOCK_READ_LOGICAL: - desc = brw_dp_a64_oword_block_rw_desc(devinfo, - false, /* align_16B */ - arg, /* num_dwords */ - false /* write */); - break; - - case SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL: - desc = brw_dp_a64_oword_block_rw_desc(devinfo, - true, /* align_16B */ - arg, /* num_dwords */ - true /* write */); - break; - - case SHADER_OPCODE_A64_BYTE_SCATTERED_READ_LOGICAL: - desc = brw_dp_a64_byte_scattered_rw_desc(devinfo, inst->exec_size, - arg, /* bit_size */ - false /* write */); - break; - - case SHADER_OPCODE_A64_BYTE_SCATTERED_WRITE_LOGICAL: - desc = brw_dp_a64_byte_scattered_rw_desc(devinfo, inst->exec_size, - arg, /* bit_size */ - true /* write */); - break; - - case SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL: - if (lsc_opcode_is_atomic_float((enum lsc_opcode) arg)) { - desc = - brw_dp_a64_untyped_atomic_float_desc(devinfo, inst->exec_size, - brw_type_size_bits(inst->dst.type), - lsc_op_to_legacy_atomic(arg), - !inst->dst.is_null()); - } else { - desc = brw_dp_a64_untyped_atomic_desc(devinfo, inst->exec_size, - brw_type_size_bits(inst->dst.type), - lsc_op_to_legacy_atomic(arg), - !inst->dst.is_null()); - } - break; - - default: - unreachable("Unknown A64 logical instruction"); - } - - if (bld.shader->stage == MESA_SHADER_FRAGMENT) - emit_fragment_mask(bld, inst); - - /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; - inst->mlen = mlen; - inst->ex_mlen = ex_mlen; - inst->header_size = header_size; - inst->send_has_side_effects = has_side_effects; - inst->send_is_volatile = !has_side_effects; - - /* Set up SFID and descriptors */ - inst->sfid = HSW_SFID_DATAPORT_DATA_CACHE_1; - inst->desc = desc; - inst->resize_sources(4); - inst->src[0] = brw_imm_ud(0); /* desc */ - inst->src[1] = brw_imm_ud(0); /* ex_desc */ - inst->src[2] = payload; - inst->src[3] = payload2; -} - static void lower_lsc_varying_pull_constant_logical_send(const fs_builder &bld, fs_inst *inst) @@ -3293,51 +2412,6 @@ brw_fs_lower_logical_sends(fs_visitor &s) lower_hdc_memory_logical_send(ibld, inst); break; - case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL: - case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL: - case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: - case SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL: - case SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL: - case SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL: - case SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL: - if (devinfo->has_lsc) - lower_lsc_surface_logical_send(block, ibld, inst); - else - lower_surface_logical_send(ibld, inst); - break; - - case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL: - case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL: - case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL: - devinfo->ver >= 20 && devinfo->has_lsc ? - lower_lsc_surface_logical_send(block, ibld, inst) : - lower_surface_logical_send(ibld, inst); - break; - - case SHADER_OPCODE_UNALIGNED_OWORD_BLOCK_READ_LOGICAL: - case SHADER_OPCODE_OWORD_BLOCK_WRITE_LOGICAL: - if (devinfo->has_lsc) { - lower_lsc_block_logical_send(ibld, inst); - break; - } - lower_surface_block_logical_send(ibld, inst); - break; - - case SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL: - case SHADER_OPCODE_A64_UNTYPED_READ_LOGICAL: - case SHADER_OPCODE_A64_BYTE_SCATTERED_WRITE_LOGICAL: - case SHADER_OPCODE_A64_BYTE_SCATTERED_READ_LOGICAL: - case SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL: - case SHADER_OPCODE_A64_OWORD_BLOCK_READ_LOGICAL: - case SHADER_OPCODE_A64_UNALIGNED_OWORD_BLOCK_READ_LOGICAL: - case SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL: - if (devinfo->has_lsc) { - lower_lsc_a64_logical_send(block, ibld, inst); - break; - } - lower_a64_logical_send(ibld, inst); - break; - case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_LOGICAL: if (devinfo->has_lsc && !s.compiler->indirect_ubos_use_sampler) lower_lsc_varying_pull_constant_logical_send(ibld, inst); diff --git a/src/intel/compiler/brw_print.cpp b/src/intel/compiler/brw_print.cpp index 771037e6f05..3c3ff26ebc0 100644 --- a/src/intel/compiler/brw_print.cpp +++ b/src/intel/compiler/brw_print.cpp @@ -181,38 +181,6 @@ brw_instruction_name(const struct brw_isa_info *isa, enum opcode op) case SHADER_OPCODE_IMAGE_SIZE_LOGICAL: return "image_size_logical"; - case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: - return "untyped_atomic_logical"; - case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL: - return "untyped_surface_read_logical"; - case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL: - return "untyped_surface_write_logical"; - case SHADER_OPCODE_UNALIGNED_OWORD_BLOCK_READ_LOGICAL: - return "unaligned_oword_block_read_logical"; - case SHADER_OPCODE_OWORD_BLOCK_WRITE_LOGICAL: - return "oword_block_write_logical"; - case SHADER_OPCODE_A64_UNTYPED_READ_LOGICAL: - return "a64_untyped_read_logical"; - case SHADER_OPCODE_A64_OWORD_BLOCK_READ_LOGICAL: - return "a64_oword_block_read_logical"; - case SHADER_OPCODE_A64_UNALIGNED_OWORD_BLOCK_READ_LOGICAL: - return "a64_unaligned_oword_block_read_logical"; - case SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL: - return "a64_oword_block_write_logical"; - case SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL: - return "a64_untyped_write_logical"; - case SHADER_OPCODE_A64_BYTE_SCATTERED_READ_LOGICAL: - return "a64_byte_scattered_read_logical"; - case SHADER_OPCODE_A64_BYTE_SCATTERED_WRITE_LOGICAL: - return "a64_byte_scattered_write_logical"; - case SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL: - return "a64_untyped_atomic_logical"; - case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL: - return "typed_atomic_logical"; - case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL: - return "typed_surface_read_logical"; - case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL: - return "typed_surface_write_logical"; case SHADER_OPCODE_MEMORY_FENCE: return "memory_fence"; case FS_OPCODE_SCHEDULING_FENCE: @@ -221,15 +189,6 @@ brw_instruction_name(const struct brw_isa_info *isa, enum opcode op) /* For an interlock we actually issue a memory fence via sendc. */ return "interlock"; - case SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL: - return "byte_scattered_read_logical"; - case SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL: - return "byte_scattered_write_logical"; - case SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL: - return "dword_scattered_read_logical"; - case SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL: - return "dword_scattered_write_logical"; - case SHADER_OPCODE_LOAD_PAYLOAD: return "load_payload"; case FS_OPCODE_PACK: diff --git a/src/intel/compiler/brw_shader.cpp b/src/intel/compiler/brw_shader.cpp index 52e2c9931f1..8ea5f518c44 100644 --- a/src/intel/compiler/brw_shader.cpp +++ b/src/intel/compiler/brw_shader.cpp @@ -462,15 +462,6 @@ fs_inst::has_side_effects() const case BRW_OPCODE_SYNC: case SHADER_OPCODE_MEMORY_STORE_LOGICAL: case SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL: - case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: - case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL: - case SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL: - case SHADER_OPCODE_A64_BYTE_SCATTERED_WRITE_LOGICAL: - case SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL: - case SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL: - case SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL: - case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL: - case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL: case SHADER_OPCODE_MEMORY_FENCE: case SHADER_OPCODE_INTERLOCK: case SHADER_OPCODE_URB_WRITE_LOGICAL: @@ -479,8 +470,6 @@ fs_inst::has_side_effects() const case SHADER_OPCODE_RND_MODE: case SHADER_OPCODE_FLOAT_CONTROL_MODE: case FS_OPCODE_SCHEDULING_FENCE: - case SHADER_OPCODE_OWORD_BLOCK_WRITE_LOGICAL: - case SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL: case SHADER_OPCODE_BTD_SPAWN_LOGICAL: case SHADER_OPCODE_BTD_RETIRE_LOGICAL: case RT_OPCODE_TRACE_RAY_LOGICAL: @@ -493,21 +482,8 @@ fs_inst::has_side_effects() const bool fs_inst::is_volatile() const { - switch (opcode) { - case SHADER_OPCODE_SEND: - return send_is_volatile; - - case SHADER_OPCODE_MEMORY_LOAD_LOGICAL: - case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL: - case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL: - case SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL: - case SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL: - case SHADER_OPCODE_A64_UNTYPED_READ_LOGICAL: - case SHADER_OPCODE_A64_BYTE_SCATTERED_READ_LOGICAL: - return true; - default: - return false; - } + return opcode == SHADER_OPCODE_MEMORY_LOAD_LOGICAL || + (opcode == SHADER_OPCODE_SEND && send_is_volatile); } #ifndef NDEBUG