diff --git a/src/intel/compiler/brw_eu_defines.h b/src/intel/compiler/brw_eu_defines.h index 382e5f89b3f..8dd3960268c 100644 --- a/src/intel/compiler/brw_eu_defines.h +++ b/src/intel/compiler/brw_eu_defines.h @@ -539,6 +539,10 @@ enum opcode { SHADER_OPCODE_LOAD_SUBGROUP_INVOCATION, RT_OPCODE_TRACE_RAY_LOGICAL, + + SHADER_OPCODE_MEMORY_LOAD_LOGICAL, + SHADER_OPCODE_MEMORY_STORE_LOGICAL, + SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL, }; enum fb_write_logical_srcs { @@ -625,6 +629,67 @@ enum get_buffer_size_srcs { GET_BUFFER_SIZE_SRCS }; +enum memory_logical_mode { + MEMORY_MODE_TYPED, + MEMORY_MODE_UNTYPED, + MEMORY_MODE_SHARED_LOCAL, + MEMORY_MODE_SCRATCH, +}; + +enum memory_logical_srcs { + /** enum lsc_opcode (as UD immediate) */ + MEMORY_LOGICAL_OPCODE, + + /** enum memory_logical_mode (as UD immediate) */ + MEMORY_LOGICAL_MODE, + + /** enum lsc_addr_surface_type (as UD immediate) */ + MEMORY_LOGICAL_BINDING_TYPE, + + /** + * Where to find the surface state. Depends on BINDING_TYPE above: + * + * - SS: pointer to surface state (relative to surface base address) + * - BSS: pointer to surface state (relative to bindless surface base) + * - BTI: binding table index + * - FLAT: This should should be BAD_FILE + */ + MEMORY_LOGICAL_BINDING, + + /** Coordinate/address/offset for where to access memory */ + MEMORY_LOGICAL_ADDRESS, + + /** Dimensionality of the "address" source (as UD immediate) */ + MEMORY_LOGICAL_COORD_COMPONENTS, + + /** Required alignment of address in bytes; 0 for natural alignment */ + MEMORY_LOGICAL_ALIGNMENT, + + /** Bit-size in the form of enum lsc_data_size (as UD immediate) */ + MEMORY_LOGICAL_DATA_SIZE, + + /** Number of vector components (as UD immediate) */ + MEMORY_LOGICAL_COMPONENTS, + + /** memory_flags bitfield (as UD immediate) */ + MEMORY_LOGICAL_FLAGS, + + /** Data to write for stores or the first operand for atomics */ + MEMORY_LOGICAL_DATA0, + + /** Second operand for two-source atomics */ + MEMORY_LOGICAL_DATA1, + + MEMORY_LOGICAL_NUM_SRCS +}; + +enum memory_flags { + /** Whether this is a transposed (i.e. block) memory access */ + MEMORY_FLAG_TRANSPOSE = 1 << 0, + /** Whether this operation should fire for helper invocations */ + MEMORY_FLAG_INCLUDE_HELPERS = 1 << 1, +}; + enum surface_logical_srcs { /** Surface binding table index */ SURFACE_LOGICAL_SRC_SURFACE, diff --git a/src/intel/compiler/brw_fs.cpp b/src/intel/compiler/brw_fs.cpp index b60e494fb0c..ae9aa5d24db 100644 --- a/src/intel/compiler/brw_fs.cpp +++ b/src/intel/compiler/brw_fs.cpp @@ -241,6 +241,14 @@ fs_inst::is_control_source(unsigned arg) const case SHADER_OPCODE_SEND: return arg == 0 || arg == 1; + case SHADER_OPCODE_MEMORY_LOAD_LOGICAL: + case SHADER_OPCODE_MEMORY_STORE_LOGICAL: + case SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL: + return arg != MEMORY_LOGICAL_BINDING && + arg != MEMORY_LOGICAL_ADDRESS && + arg != MEMORY_LOGICAL_DATA0 && + arg != MEMORY_LOGICAL_DATA1; + default: return false; } @@ -696,6 +704,22 @@ fs_inst::components_read(unsigned i) const } else return 1; + case SHADER_OPCODE_MEMORY_LOAD_LOGICAL: + if (i == MEMORY_LOGICAL_DATA0 || i == MEMORY_LOGICAL_DATA0) + return 0; + /* fallthrough */ + case SHADER_OPCODE_MEMORY_STORE_LOGICAL: + if (i == MEMORY_LOGICAL_DATA1) + return 0; + /* fallthrough */ + case SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL: + if (i == MEMORY_LOGICAL_DATA0 || i == MEMORY_LOGICAL_DATA1) + return src[MEMORY_LOGICAL_COMPONENTS].ud; + else if (i == MEMORY_LOGICAL_ADDRESS) + return src[MEMORY_LOGICAL_COORD_COMPONENTS].ud; + else + return 1; + case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL: case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL: assert(src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == IMM); diff --git a/src/intel/compiler/brw_fs_copy_propagation.cpp b/src/intel/compiler/brw_fs_copy_propagation.cpp index 0e624c5743b..4fe6f853604 100644 --- a/src/intel/compiler/brw_fs_copy_propagation.cpp +++ b/src/intel/compiler/brw_fs_copy_propagation.cpp @@ -1215,6 +1215,9 @@ try_constant_propagate_value(brw_reg val, brw_reg_type dst_type, case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL: case SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL: case SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL: + case SHADER_OPCODE_MEMORY_LOAD_LOGICAL: + case SHADER_OPCODE_MEMORY_STORE_LOGICAL: + case SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL: case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD: case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_LOGICAL: case SHADER_OPCODE_BROADCAST: diff --git a/src/intel/compiler/brw_fs_dead_code_eliminate.cpp b/src/intel/compiler/brw_fs_dead_code_eliminate.cpp index ae12d4749d2..1ce70359eed 100644 --- a/src/intel/compiler/brw_fs_dead_code_eliminate.cpp +++ b/src/intel/compiler/brw_fs_dead_code_eliminate.cpp @@ -59,6 +59,7 @@ can_omit_write(const fs_inst *inst) case SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL: case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL: + case SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL: return true; default: /* We can eliminate the destination write for ordinary instructions, diff --git a/src/intel/compiler/brw_fs_lower_simd_width.cpp b/src/intel/compiler/brw_fs_lower_simd_width.cpp index 7f73f44812b..921e8c9241f 100644 --- a/src/intel/compiler/brw_fs_lower_simd_width.cpp +++ b/src/intel/compiler/brw_fs_lower_simd_width.cpp @@ -377,6 +377,24 @@ brw_fs_get_lowered_simd_width(const fs_visitor *shader, const fs_inst *inst) case SHADER_OPCODE_TXS_LOGICAL: return get_sampler_lowered_simd_width(devinfo, inst); + case SHADER_OPCODE_MEMORY_LOAD_LOGICAL: + case SHADER_OPCODE_MEMORY_STORE_LOGICAL: + case SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL: + if (devinfo->ver >= 20) + return inst->exec_size; + + if (inst->src[MEMORY_LOGICAL_MODE].ud == MEMORY_MODE_TYPED) + return 8; + + /* HDC A64 atomics are limited to SIMD8 */ + if (!devinfo->has_lsc && + inst->src[MEMORY_LOGICAL_BINDING_TYPE].ud == LSC_ADDR_SURFTYPE_FLAT + && lsc_opcode_is_atomic((enum lsc_opcode) + inst->src[MEMORY_LOGICAL_OPCODE].ud)) + return 8; + + return MIN2(16, inst->exec_size); + /* On gfx12 parameters are fixed to 16-bit values and therefore they all * always fit regardless of the execution size. */ diff --git a/src/intel/compiler/brw_fs_validate.cpp b/src/intel/compiler/brw_fs_validate.cpp index b3544108411..a38cf32b00c 100644 --- a/src/intel/compiler/brw_fs_validate.cpp +++ b/src/intel/compiler/brw_fs_validate.cpp @@ -29,6 +29,7 @@ #include "brw_fs.h" #include "brw_cfg.h" +#include "brw_eu.h" #define fsv_assert(assertion) \ { \ @@ -87,6 +88,92 @@ } #ifndef NDEBUG +static inline bool +is_ud_imm(const brw_reg ®) +{ + return reg.file == IMM && reg.type == BRW_TYPE_UD; +} + +static void +validate_memory_logical(const fs_visitor &s, const fs_inst *inst) +{ + const intel_device_info *devinfo = s.devinfo; + + fsv_assert(is_ud_imm(inst->src[MEMORY_LOGICAL_OPCODE])); + fsv_assert(is_ud_imm(inst->src[MEMORY_LOGICAL_MODE])); + fsv_assert(is_ud_imm(inst->src[MEMORY_LOGICAL_BINDING_TYPE])); + fsv_assert(is_ud_imm(inst->src[MEMORY_LOGICAL_COORD_COMPONENTS])); + fsv_assert(is_ud_imm(inst->src[MEMORY_LOGICAL_ALIGNMENT])); + fsv_assert(is_ud_imm(inst->src[MEMORY_LOGICAL_DATA_SIZE])); + fsv_assert(is_ud_imm(inst->src[MEMORY_LOGICAL_COMPONENTS])); + fsv_assert(is_ud_imm(inst->src[MEMORY_LOGICAL_FLAGS])); + + enum lsc_data_size data_size = + (enum lsc_data_size) inst->src[MEMORY_LOGICAL_DATA_SIZE].ud; + unsigned data_size_B = lsc_data_size_bytes(data_size); + + if (!devinfo->has_lsc) { + fsv_assert(data_size == LSC_DATA_SIZE_D8U32 || + data_size == LSC_DATA_SIZE_D16U32 || + data_size == LSC_DATA_SIZE_D32 || + data_size == LSC_DATA_SIZE_D64); + } + + enum lsc_opcode op = (enum lsc_opcode) inst->src[MEMORY_LOGICAL_OPCODE].ud; + enum memory_flags flags = (memory_flags)inst->src[MEMORY_LOGICAL_FLAGS].ud; + bool transpose = flags & MEMORY_FLAG_TRANSPOSE; + bool include_helpers = flags & MEMORY_FLAG_INCLUDE_HELPERS; + + fsv_assert(!transpose || !include_helpers); + fsv_assert(!transpose || lsc_opcode_has_transpose(op)); + + if (inst->src[MEMORY_LOGICAL_BINDING_TYPE].ud == LSC_ADDR_SURFTYPE_FLAT) + fsv_assert(inst->src[MEMORY_LOGICAL_BINDING].file == BAD_FILE); + + if (inst->src[MEMORY_LOGICAL_DATA1].file != BAD_FILE) { + fsv_assert(inst->src[MEMORY_LOGICAL_COMPONENTS].ud == + inst->components_read(MEMORY_LOGICAL_DATA1)); + + fsv_assert(inst->src[MEMORY_LOGICAL_DATA0].type == + inst->src[MEMORY_LOGICAL_DATA1].type); + } + + if (inst->src[MEMORY_LOGICAL_DATA0].file != BAD_FILE) { + fsv_assert(inst->src[MEMORY_LOGICAL_COMPONENTS].ud == + inst->components_read(MEMORY_LOGICAL_DATA0)); + + fsv_assert(brw_type_size_bytes(inst->src[MEMORY_LOGICAL_DATA0].type) == + data_size_B); + } + + if (inst->dst.file != BAD_FILE) + fsv_assert(brw_type_size_bytes(inst->dst.type) == data_size_B); + + switch (inst->opcode) { + case SHADER_OPCODE_MEMORY_LOAD_LOGICAL: + fsv_assert(op == LSC_OP_LOAD || op == LSC_OP_LOAD_CMASK); + fsv_assert(inst->src[MEMORY_LOGICAL_DATA0].file == BAD_FILE); + fsv_assert(inst->src[MEMORY_LOGICAL_DATA1].file == BAD_FILE); + break; + case SHADER_OPCODE_MEMORY_STORE_LOGICAL: + fsv_assert(lsc_opcode_is_store(op)); + fsv_assert(inst->src[MEMORY_LOGICAL_DATA0].file != BAD_FILE); + fsv_assert(inst->src[MEMORY_LOGICAL_DATA1].file == BAD_FILE); + break; + case SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL: + fsv_assert(lsc_opcode_is_atomic(op)); + fsv_assert((inst->src[MEMORY_LOGICAL_DATA0].file == BAD_FILE) + == (lsc_op_num_data_values(op) < 1)); + fsv_assert((inst->src[MEMORY_LOGICAL_DATA1].file == BAD_FILE) + == (lsc_op_num_data_values(op) < 2)); + fsv_assert(inst->src[MEMORY_LOGICAL_COMPONENTS].ud == 1); + fsv_assert(!include_helpers); + break; + default: + unreachable("invalid opcode"); + } +} + void brw_fs_validate(const fs_visitor &s) { @@ -104,6 +191,12 @@ brw_fs_validate(const fs_visitor &s) fsv_assert(inst->sources == 1); break; + case SHADER_OPCODE_MEMORY_LOAD_LOGICAL: + case SHADER_OPCODE_MEMORY_STORE_LOGICAL: + case SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL: + validate_memory_logical(s, inst); + break; + default: break; } diff --git a/src/intel/compiler/brw_lower_logical_sends.cpp b/src/intel/compiler/brw_lower_logical_sends.cpp index 3e6a7e1ba02..c877aeaaa2a 100644 --- a/src/intel/compiler/brw_lower_logical_sends.cpp +++ b/src/intel/compiler/brw_lower_logical_sends.cpp @@ -1415,6 +1415,18 @@ setup_lsc_surface_descriptors(const fs_builder &bld, fs_inst *inst, } } +static void +lower_lsc_memory_logical_send(const fs_builder &bld, fs_inst *inst) +{ + unreachable("Not implemented yet"); +} + +static void +lower_hdc_memory_logical_send(const fs_builder &bld, fs_inst *inst) +{ + unreachable("Not implemented yet"); +} + static void lower_surface_logical_send(const fs_builder &bld, fs_inst *inst) { @@ -2842,6 +2854,17 @@ brw_fs_lower_logical_sends(fs_visitor &s) lower_get_buffer_size(ibld, inst); break; + case SHADER_OPCODE_MEMORY_LOAD_LOGICAL: + case SHADER_OPCODE_MEMORY_STORE_LOGICAL: + case SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL: + if (devinfo->ver >= 20 || + (devinfo->has_lsc && + inst->src[MEMORY_LOGICAL_MODE].ud != MEMORY_MODE_TYPED)) + lower_lsc_memory_logical_send(ibld, inst); + else + lower_hdc_memory_logical_send(ibld, inst); + break; + case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL: case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL: case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: diff --git a/src/intel/compiler/brw_print.cpp b/src/intel/compiler/brw_print.cpp index d1ff925f571..94bd5e2c981 100644 --- a/src/intel/compiler/brw_print.cpp +++ b/src/intel/compiler/brw_print.cpp @@ -325,6 +325,12 @@ brw_instruction_name(const struct brw_isa_info *isa, enum opcode op) return "read_arch_reg"; case SHADER_OPCODE_LOAD_SUBGROUP_INVOCATION: return "load_subgroup_invocation"; + case SHADER_OPCODE_MEMORY_LOAD_LOGICAL: + return "memory_load_logical"; + case SHADER_OPCODE_MEMORY_STORE_LOGICAL: + return "memory_store_logical"; + case SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL: + return "memory_atomic_logical"; } unreachable("not reached"); diff --git a/src/intel/compiler/brw_shader.cpp b/src/intel/compiler/brw_shader.cpp index 59cb909a4ec..52e2c9931f1 100644 --- a/src/intel/compiler/brw_shader.cpp +++ b/src/intel/compiler/brw_shader.cpp @@ -460,6 +460,8 @@ fs_inst::has_side_effects() const return send_has_side_effects; case BRW_OPCODE_SYNC: + case SHADER_OPCODE_MEMORY_STORE_LOGICAL: + case SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL: case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL: case SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL: @@ -495,6 +497,7 @@ fs_inst::is_volatile() const case SHADER_OPCODE_SEND: return send_is_volatile; + case SHADER_OPCODE_MEMORY_LOAD_LOGICAL: case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL: case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL: case SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL: