diff --git a/src/intel/compiler/elk/elk_disasm.c b/src/intel/compiler/elk/elk_disasm.c index df8aff2e2c4..392bb998913 100644 --- a/src/intel/compiler/elk/elk_disasm.c +++ b/src/intel/compiler/elk/elk_disasm.c @@ -313,9 +313,6 @@ static const char *const gfx6_sfid[16] = { [GFX7_SFID_PIXEL_INTERPOLATOR] = "pixel interp", [HSW_SFID_DATAPORT_DATA_CACHE_1] = "dp data 1", [HSW_SFID_CRE] = "cre", - [GFX12_SFID_SLM] = "slm", - [GFX12_SFID_TGM] = "tgm", - [GFX12_SFID_UGM] = "ugm", }; static const char *const gfx7_gateway_subfuncid[8] = { @@ -685,37 +682,12 @@ static const char *const lsc_addr_surface_type[] = { [LSC_ADDR_SURFTYPE_BTI] = "bti", }; -static const char* const lsc_fence_scope[] = { - [LSC_FENCE_THREADGROUP] = "threadgroup", - [LSC_FENCE_LOCAL] = "local", - [LSC_FENCE_TILE] = "tile", - [LSC_FENCE_GPU] = "gpu", - [LSC_FENCE_ALL_GPU] = "all_gpu", - [LSC_FENCE_SYSTEM_RELEASE] = "system_release", - [LSC_FENCE_SYSTEM_ACQUIRE] = "system_acquire", -}; - -static const char* const lsc_flush_type[] = { - [LSC_FLUSH_TYPE_NONE] = "none", - [LSC_FLUSH_TYPE_EVICT] = "evict", - [LSC_FLUSH_TYPE_INVALIDATE] = "invalidate", - [LSC_FLUSH_TYPE_DISCARD] = "discard", - [LSC_FLUSH_TYPE_CLEAN] = "clean", - [LSC_FLUSH_TYPE_L3ONLY] = "l3only", - [LSC_FLUSH_TYPE_NONE_6] = "none_6", -}; - static const char* const lsc_addr_size[] = { [LSC_ADDR_SIZE_A16] = "a16", [LSC_ADDR_SIZE_A32] = "a32", [LSC_ADDR_SIZE_A64] = "a64", }; -static const char* const lsc_backup_fence_routing[] = { - [LSC_NORMAL_ROUTING] = "normal_routing", - [LSC_ROUTE_TO_LSC] = "route_to_lsc", -}; - static const char* const lsc_data_size[] = { [LSC_DATA_SIZE_D8] = "d8", [LSC_DATA_SIZE_D16] = "d16", @@ -777,30 +749,6 @@ static const char* const lsc_cache_store[] = { [LSC_CACHE_STORE_L1WB_L3WB] = "L1WB_L3WB", }; -static const char* const xe2_lsc_cache_load[] = { - [XE2_LSC_CACHE_LOAD_L1STATE_L3MOCS] = "L1STATE_L3MOCS", - [XE2_LSC_CACHE_LOAD_L1UC_L3UC] = "L1UC_L3UC", - [XE2_LSC_CACHE_LOAD_L1UC_L3C] = "L1UC_L3C", - [XE2_LSC_CACHE_LOAD_L1UC_L3CC] = "L1UC_L3CC", - [XE2_LSC_CACHE_LOAD_L1C_L3UC] = "L1C_L3UC", - [XE2_LSC_CACHE_LOAD_L1C_L3C] = "L1C_L3C", - [XE2_LSC_CACHE_LOAD_L1C_L3CC] = "L1C_L3CC", - [XE2_LSC_CACHE_LOAD_L1S_L3UC] = "L1S_L3UC", - [XE2_LSC_CACHE_LOAD_L1S_L3C] = "L1S_L3C", - [XE2_LSC_CACHE_LOAD_L1IAR_L3IAR] = "L1IAR_L3IAR", -}; - -static const char* const xe2_lsc_cache_store[] = { - [XE2_LSC_CACHE_STORE_L1STATE_L3MOCS] = "L1STATE_L3MOCS", - [XE2_LSC_CACHE_STORE_L1UC_L3UC] = "L1UC_L3UC", - [XE2_LSC_CACHE_STORE_L1UC_L3WB] = "L1UC_L3WB", - [XE2_LSC_CACHE_STORE_L1WT_L3UC] = "L1WT_L3UC", - [XE2_LSC_CACHE_STORE_L1WT_L3WB] = "L1WT_L3WB", - [XE2_LSC_CACHE_STORE_L1S_L3UC] = "L1S_L3UC", - [XE2_LSC_CACHE_STORE_L1S_L3WB] = "L1S_L3WB", - [XE2_LSC_CACHE_STORE_L1WB_L3WB] = "L1WB_L3WB", -}; - static int column; static int @@ -1732,21 +1680,6 @@ lsc_disassemble_ex_desc(const struct intel_device_info *devinfo, } } -static inline bool -elk_sfid_is_lsc(unsigned sfid) -{ - switch (sfid) { - case GFX12_SFID_UGM: - case GFX12_SFID_SLM: - case GFX12_SFID_TGM: - return true; - default: - break; - } - - return false; -} - int elk_disassemble_inst(FILE *file, const struct elk_isa_info *isa, const elk_inst *inst, bool is_compacted, @@ -2144,83 +2077,6 @@ elk_disassemble_inst(FILE *file, const struct elk_isa_info *isa, gfx7_gateway_subfuncid[elk_inst_gateway_subfuncid(devinfo, inst)]); break; - case GFX12_SFID_SLM: - case GFX12_SFID_TGM: - case GFX12_SFID_UGM: { - assert(devinfo->has_lsc); - format(file, " ("); - const enum elk_lsc_opcode op = lsc_msg_desc_opcode(devinfo, imm_desc); - err |= control(file, "operation", lsc_operation, - op, &space); - format(file, ","); - err |= control(file, "addr_size", lsc_addr_size, - lsc_msg_desc_addr_size(devinfo, imm_desc), - &space); - - if (op == LSC_OP_FENCE) { - format(file, ","); - err |= control(file, "scope", lsc_fence_scope, - lsc_fence_msg_desc_scope(devinfo, imm_desc), - &space); - format(file, ","); - err |= control(file, "flush_type", lsc_flush_type, - lsc_fence_msg_desc_flush_type(devinfo, imm_desc), - &space); - format(file, ","); - err |= control(file, "backup_mode_fence_routing", - lsc_backup_fence_routing, - lsc_fence_msg_desc_backup_routing(devinfo, imm_desc), - &space); - } else { - format(file, ","); - err |= control(file, "data_size", lsc_data_size, - lsc_msg_desc_data_size(devinfo, imm_desc), - &space); - format(file, ","); - if (elk_lsc_opcode_has_cmask(op)) { - err |= control(file, "component_mask", - lsc_cmask_str, - lsc_msg_desc_cmask(devinfo, imm_desc), - &space); - } else { - err |= control(file, "vector_size", - lsc_vect_size_str, - lsc_msg_desc_vect_size(devinfo, imm_desc), - &space); - if (lsc_msg_desc_transpose(devinfo, imm_desc)) - format(file, ", transpose"); - } - switch(op) { - case LSC_OP_LOAD_CMASK: - case LSC_OP_LOAD: - format(file, ","); - err |= control(file, "cache_load", - devinfo->ver >= 20 ? - xe2_lsc_cache_load : - lsc_cache_load, - lsc_msg_desc_cache_ctrl(devinfo, imm_desc), - &space); - break; - default: - format(file, ","); - err |= control(file, "cache_store", - devinfo->ver >= 20 ? - xe2_lsc_cache_store : - lsc_cache_store, - lsc_msg_desc_cache_ctrl(devinfo, imm_desc), - &space); - break; - } - } - format(file, " dst_len = %u,", lsc_msg_desc_dest_len(devinfo, imm_desc)); - format(file, " src0_len = %u,", lsc_msg_desc_src0_len(devinfo, imm_desc)); - - err |= control(file, "address_type", lsc_addr_surface_type, - lsc_msg_desc_addr_type(devinfo, imm_desc), &space); - format(file, " )"); - break; - } - case GFX7_SFID_DATAPORT_DATA_CACHE: if (devinfo->ver >= 7) { format(file, " ("); @@ -2340,19 +2196,14 @@ elk_disassemble_inst(FILE *file, const struct elk_isa_info *isa, if (space) string(file, " "); } - if (elk_sfid_is_lsc(sfid) || - (sfid == ELK_SFID_URB && devinfo->ver >= 20)) { - lsc_disassemble_ex_desc(devinfo, imm_desc, imm_ex_desc, file); - } else { - if (has_imm_desc) - format(file, " mlen %u", elk_message_desc_mlen(devinfo, imm_desc)); - if (has_imm_ex_desc) { - format(file, " ex_mlen %u", - elk_message_ex_desc_ex_mlen(devinfo, imm_ex_desc)); - } - if (has_imm_desc) - format(file, " rlen %u", elk_message_desc_rlen(devinfo, imm_desc)); + if (has_imm_desc) + format(file, " mlen %u", elk_message_desc_mlen(devinfo, imm_desc)); + if (has_imm_ex_desc) { + format(file, " ex_mlen %u", + elk_message_ex_desc_ex_mlen(devinfo, imm_ex_desc)); } + if (has_imm_desc) + format(file, " rlen %u", elk_message_desc_rlen(devinfo, imm_desc)); } pad(file, 64); if (opcode != ELK_OPCODE_NOP && opcode != ELK_OPCODE_NENOP) { diff --git a/src/intel/compiler/elk/elk_eu_defines.h b/src/intel/compiler/elk/elk_eu_defines.h index b5671db07c8..808433c2777 100644 --- a/src/intel/compiler/elk/elk_eu_defines.h +++ b/src/intel/compiler/elk/elk_eu_defines.h @@ -530,10 +530,6 @@ enum elk_message_target { GFX7_SFID_PIXEL_INTERPOLATOR = 11, HSW_SFID_DATAPORT_DATA_CACHE_1 = 12, HSW_SFID_CRE = 13, - - GFX12_SFID_TGM = 13, /* Typed Global Memory */ - GFX12_SFID_SLM = 14, /* Shared Local Memory */ - GFX12_SFID_UGM = 15, /* Untyped Global Memory */ }; #define GFX7_MESSAGE_TARGET_DP_DATA_CACHE 10 diff --git a/src/intel/compiler/elk/elk_eu_emit.c b/src/intel/compiler/elk/elk_eu_emit.c index 89ef83b7f9d..1ea83db4dc8 100644 --- a/src/intel/compiler/elk/elk_eu_emit.c +++ b/src/intel/compiler/elk/elk_eu_emit.c @@ -2933,53 +2933,6 @@ elk_set_memory_fence_message(struct elk_codegen *p, elk_inst_set_binding_table_index(devinfo, insn, bti); } -static void -gfx12_set_memory_fence_message(struct elk_codegen *p, - struct elk_inst *insn, - enum elk_message_target sfid, - uint32_t desc) -{ - const unsigned mlen = 1 * reg_unit(p->devinfo); /* g0 header */ - /* Completion signaled by write to register. No data returned. */ - const unsigned rlen = 1 * reg_unit(p->devinfo); - - elk_inst_set_sfid(p->devinfo, insn, sfid); - - if (sfid == ELK_SFID_URB && p->devinfo->ver < 20) { - elk_set_desc(p, insn, elk_urb_fence_desc(p->devinfo) | - elk_message_desc(p->devinfo, mlen, rlen, true)); - } else { - enum lsc_fence_scope scope = lsc_fence_msg_desc_scope(p->devinfo, desc); - enum lsc_flush_type flush_type = lsc_fence_msg_desc_flush_type(p->devinfo, desc); - - if (sfid == GFX12_SFID_TGM) { - scope = LSC_FENCE_TILE; - flush_type = LSC_FLUSH_TYPE_EVICT; - } - - /* Wa_14012437816: - * - * "For any fence greater than local scope, always set flush type to - * at least invalidate so that fence goes on properly." - * - * "The bug is if flush_type is 'None', the scope is always downgraded - * to 'local'." - * - * Here set scope to NONE_6 instead of NONE, which has the same effect - * as NONE but avoids the downgrade to scope LOCAL. - */ - if (intel_needs_workaround(p->devinfo, 14012437816) && - scope > LSC_FENCE_LOCAL && - flush_type == LSC_FLUSH_TYPE_NONE) { - flush_type = LSC_FLUSH_TYPE_NONE_6; - } - - elk_set_desc(p, insn, lsc_fence_msg_desc(p->devinfo, scope, - flush_type, false) | - elk_message_desc(p->devinfo, mlen, rlen, false)); - } -} - void elk_memory_fence(struct elk_codegen *p, struct elk_reg dst, @@ -3004,11 +2957,7 @@ elk_memory_fence(struct elk_codegen *p, elk_set_dest(p, insn, dst); elk_set_src0(p, insn, src); - /* All DG2 hardware requires LSC for fence messages, even A-step */ - if (devinfo->has_lsc) - gfx12_set_memory_fence_message(p, insn, sfid, desc); - else - elk_set_memory_fence_message(p, insn, sfid, commit_enable, bti); + elk_set_memory_fence_message(p, insn, sfid, commit_enable, bti); } void diff --git a/src/intel/compiler/elk/elk_eu_validate.c b/src/intel/compiler/elk/elk_eu_validate.c index 92192762274..708d2c51a08 100644 --- a/src/intel/compiler/elk/elk_eu_validate.c +++ b/src/intel/compiler/elk/elk_eu_validate.c @@ -2314,28 +2314,6 @@ send_descriptor_restrictions(const struct elk_isa_info *isa, return error_msg; } - const uint32_t desc = elk_inst_send_desc(devinfo, inst); - - switch (elk_inst_sfid(devinfo, inst)) { - case ELK_SFID_URB: - if (devinfo->ver < 20) - break; - FALLTHROUGH; - case GFX12_SFID_TGM: - case GFX12_SFID_SLM: - case GFX12_SFID_UGM: - ERROR_IF(!devinfo->has_lsc, "Platform does not support LSC"); - - ERROR_IF(elk_lsc_opcode_has_transpose(lsc_msg_desc_opcode(devinfo, desc)) && - lsc_msg_desc_transpose(devinfo, desc) && - elk_inst_exec_size(devinfo, inst) != ELK_EXECUTE_1, - "Transposed vectors are restricted to Exec_Mask = 1."); - break; - - default: - break; - } - if (elk_inst_sfid(devinfo, inst) == ELK_SFID_URB && devinfo->ver < 20) { /* Gfx4 doesn't have a "header present" bit in the SEND message. */ ERROR_IF(devinfo->ver > 4 && !elk_inst_header_present(devinfo, inst), diff --git a/src/intel/compiler/elk/elk_fs.cpp b/src/intel/compiler/elk/elk_fs.cpp index 32ba6cacadd..ac0f4af4de1 100644 --- a/src/intel/compiler/elk/elk_fs.cpp +++ b/src/intel/compiler/elk/elk_fs.cpp @@ -1290,63 +1290,6 @@ elk_fs_visitor::assign_curb_setup() */ assert(devinfo->verx10 >= 125); assert(uniform_push_length <= reg_unit(devinfo)); - } else if (is_compute && devinfo->verx10 >= 125) { - assert(devinfo->has_lsc); - fs_builder ubld = fs_builder(this, 1).exec_all().at( - cfg->first_block(), cfg->first_block()->start()); - - /* The base offset for our push data is passed in as R0.0[31:6]. We have - * to mask off the bottom 6 bits. - */ - elk_fs_reg base_addr = ubld.vgrf(ELK_REGISTER_TYPE_UD); - ubld.AND(base_addr, - retype(elk_vec1_grf(0, 0), ELK_REGISTER_TYPE_UD), - elk_imm_ud(INTEL_MASK(31, 6))); - - /* On Gfx12-HP we load constants at the start of the program using A32 - * stateless messages. - */ - for (unsigned i = 0; i < uniform_push_length;) { - /* Limit ourselves to LSC HW limit of 8 GRFs (256bytes D32V64). */ - unsigned num_regs = MIN2(uniform_push_length - i, 8); - assert(num_regs > 0); - num_regs = 1 << util_logbase2(num_regs); - - elk_fs_reg addr = ubld.vgrf(ELK_REGISTER_TYPE_UD); - ubld.ADD(addr, base_addr, elk_imm_ud(i * REG_SIZE)); - - elk_fs_reg srcs[4] = { - elk_imm_ud(0), /* desc */ - elk_imm_ud(0), /* ex_desc */ - addr, /* payload */ - elk_fs_reg(), /* payload2 */ - }; - - elk_fs_reg dest = retype(elk_vec8_grf(payload().num_regs + i, 0), - ELK_REGISTER_TYPE_UD); - elk_fs_inst *send = ubld.emit(ELK_SHADER_OPCODE_SEND, dest, srcs, 4); - - send->sfid = GFX12_SFID_UGM; - send->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD, - 1 /* exec_size */, - LSC_ADDR_SURFTYPE_FLAT, - LSC_ADDR_SIZE_A32, - 1 /* num_coordinates */, - LSC_DATA_SIZE_D32, - num_regs * 8 /* num_channels */, - true /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS), - true /* has_dest */); - send->header_size = 0; - send->mlen = lsc_msg_desc_src0_len(devinfo, send->desc); - send->size_written = - lsc_msg_desc_dest_len(devinfo, send->desc) * REG_SIZE; - send->send_is_volatile = true; - - i += num_regs; - } - - invalidate_analysis(DEPENDENCY_INSTRUCTIONS); } /* Map the offsets in the UNIFORM file to fixed HW regs. */ @@ -6009,41 +5952,6 @@ elk_fs_visitor::fixup_3src_null_dest() DEPENDENCY_VARIABLES); } -static bool -needs_dummy_fence(const intel_device_info *devinfo, elk_fs_inst *inst) -{ - /* This workaround is about making sure that any instruction writing - * through UGM has completed before we hit EOT. - */ - if (inst->sfid != GFX12_SFID_UGM) - return false; - - /* Any UGM, non-Scratch-surface Stores (not including Atomic) messages, - * where the L1-cache override is NOT among {WB, WS, WT} - */ - enum elk_lsc_opcode opcode = lsc_msg_desc_opcode(devinfo, inst->desc); - if (elk_lsc_opcode_is_store(opcode)) { - switch (lsc_msg_desc_cache_ctrl(devinfo, inst->desc)) { - case LSC_CACHE_STORE_L1STATE_L3MOCS: - case LSC_CACHE_STORE_L1WB_L3WB: - case LSC_CACHE_STORE_L1S_L3UC: - case LSC_CACHE_STORE_L1S_L3WB: - case LSC_CACHE_STORE_L1WT_L3UC: - case LSC_CACHE_STORE_L1WT_L3WB: - return false; - - default: - return true; - } - } - - /* Any UGM Atomic message WITHOUT return value */ - if (elk_lsc_opcode_is_atomic(opcode) && inst->dst.file == BAD_FILE) - return true; - - return false; -} - /* Wa_14015360517 * * The first instruction of any kernel should have non-zero emask. @@ -6073,56 +5981,6 @@ elk_fs_visitor::emit_dummy_mov_instruction() invalidate_analysis(DEPENDENCY_INSTRUCTIONS | DEPENDENCY_VARIABLES); } -/* Wa_22013689345 - * - * We need to emit UGM fence message before EOT, if shader has any UGM write - * or atomic message. - * - * TODO/FINISHME: According to Curro we could avoid the fence in some cases. - * We probably need a better criteria in needs_dummy_fence(). - */ -void -elk_fs_visitor::emit_dummy_memory_fence_before_eot() -{ - bool progress = false; - bool has_ugm_write_or_atomic = false; - - if (!intel_needs_workaround(devinfo, 22013689345)) - return; - - foreach_block_and_inst_safe (block, elk_fs_inst, inst, cfg) { - if (!inst->eot) { - if (needs_dummy_fence(devinfo, inst)) - has_ugm_write_or_atomic = true; - continue; - } - - if (!has_ugm_write_or_atomic) - break; - - const fs_builder ibld(this, block, inst); - const fs_builder ubld = ibld.exec_all().group(1, 0); - - elk_fs_reg dst = ubld.vgrf(ELK_REGISTER_TYPE_UD); - elk_fs_inst *dummy_fence = ubld.emit(ELK_SHADER_OPCODE_MEMORY_FENCE, - dst, elk_vec8_grf(0, 0), - /* commit enable */ elk_imm_ud(1), - /* bti */ elk_imm_ud(0)); - dummy_fence->sfid = GFX12_SFID_UGM; - dummy_fence->desc = lsc_fence_msg_desc(devinfo, LSC_FENCE_TILE, - LSC_FLUSH_TYPE_NONE_6, false); - ubld.emit(ELK_FS_OPCODE_SCHEDULING_FENCE, ubld.null_reg_ud(), dst); - progress = true; - /* TODO: remove this break if we ever have shader with multiple EOT. */ - break; - } - - if (progress) { - invalidate_analysis(DEPENDENCY_INSTRUCTIONS | - DEPENDENCY_VARIABLES); - } -} - /** * Find the first instruction in the program that might start a region of * divergent control flow due to a HALT jump. There is no @@ -6508,7 +6366,6 @@ elk_fs_visitor::run_vs() assign_vs_urb_setup(); fixup_3src_null_dest(); - emit_dummy_memory_fence_before_eot(); /* Wa_14015360517 */ emit_dummy_mov_instruction(); @@ -6638,7 +6495,6 @@ elk_fs_visitor::run_tcs() assign_tcs_urb_setup(); fixup_3src_null_dest(); - emit_dummy_memory_fence_before_eot(); /* Wa_14015360517 */ emit_dummy_mov_instruction(); @@ -6670,7 +6526,6 @@ elk_fs_visitor::run_tes() assign_tes_urb_setup(); fixup_3src_null_dest(); - emit_dummy_memory_fence_before_eot(); /* Wa_14015360517 */ emit_dummy_mov_instruction(); @@ -6719,7 +6574,6 @@ elk_fs_visitor::run_gs() assign_gs_urb_setup(); fixup_3src_null_dest(); - emit_dummy_memory_fence_before_eot(); /* Wa_14015360517 */ emit_dummy_mov_instruction(); @@ -6827,7 +6681,6 @@ elk_fs_visitor::run_fs(bool allow_spilling, bool do_rep_send) assign_urb_setup(); fixup_3src_null_dest(); - emit_dummy_memory_fence_before_eot(); /* Wa_14015360517 */ emit_dummy_mov_instruction(); @@ -6868,7 +6721,6 @@ elk_fs_visitor::run_cs(bool allow_spilling) assign_curb_setup(); fixup_3src_null_dest(); - emit_dummy_memory_fence_before_eot(); /* Wa_14015360517 */ emit_dummy_mov_instruction(); diff --git a/src/intel/compiler/elk/elk_fs.h b/src/intel/compiler/elk/elk_fs.h index 571b3086103..97482aba9a3 100644 --- a/src/intel/compiler/elk/elk_fs.h +++ b/src/intel/compiler/elk/elk_fs.h @@ -213,7 +213,6 @@ public: void allocate_registers(bool allow_spilling); uint32_t compute_max_register_pressure(); void fixup_3src_null_dest(); - void emit_dummy_memory_fence_before_eot(); void emit_dummy_mov_instruction(); bool fixup_nomask_control_flow(); void assign_curb_setup(); diff --git a/src/intel/compiler/elk/elk_fs_nir.cpp b/src/intel/compiler/elk/elk_fs_nir.cpp index e61b048acc8..fcaf79829ae 100644 --- a/src/intel/compiler/elk/elk_fs_nir.cpp +++ b/src/intel/compiler/elk/elk_fs_nir.cpp @@ -4889,39 +4889,6 @@ emit_fence(const fs_builder &bld, enum elk_opcode opcode, return dst; } -static uint32_t -lsc_fence_descriptor_for_intrinsic(const struct intel_device_info *devinfo, - nir_intrinsic_instr *instr) -{ - assert(devinfo->has_lsc); - - enum lsc_fence_scope scope = LSC_FENCE_LOCAL; - enum lsc_flush_type flush_type = LSC_FLUSH_TYPE_NONE; - - if (nir_intrinsic_has_memory_scope(instr)) { - switch (nir_intrinsic_memory_scope(instr)) { - case SCOPE_DEVICE: - case SCOPE_QUEUE_FAMILY: - scope = LSC_FENCE_TILE; - flush_type = LSC_FLUSH_TYPE_EVICT; - break; - case SCOPE_WORKGROUP: - scope = LSC_FENCE_THREADGROUP; - break; - case SCOPE_SHADER_CALL: - case SCOPE_INVOCATION: - case SCOPE_SUBGROUP: - case SCOPE_NONE: - break; - } - } else { - /* No scope defined. */ - scope = LSC_FENCE_TILE; - flush_type = LSC_FLUSH_TYPE_EVICT; - } - return lsc_fence_msg_desc(devinfo, scope, flush_type, true); -} - /** * Create a MOV to read the timestamp register. */ @@ -5244,40 +5211,7 @@ fs_nir_emit_intrinsic(nir_to_elk_state &ntb, const fs_builder ubld = bld.group(8, 0); - if (devinfo->has_lsc) { - assert(devinfo->verx10 >= 125); - uint32_t desc = - lsc_fence_descriptor_for_intrinsic(devinfo, instr); - if (ugm_fence) { - fence_regs[fence_regs_count++] = - emit_fence(ubld, opcode, GFX12_SFID_UGM, desc, - true /* commit_enable */, - 0 /* bti; ignored for LSC */); - } - - if (tgm_fence) { - fence_regs[fence_regs_count++] = - emit_fence(ubld, opcode, GFX12_SFID_TGM, desc, - true /* commit_enable */, - 0 /* bti; ignored for LSC */); - } - - if (slm_fence) { - assert(opcode == ELK_SHADER_OPCODE_MEMORY_FENCE); - fence_regs[fence_regs_count++] = - emit_fence(ubld, opcode, GFX12_SFID_SLM, desc, - true /* commit_enable */, - 0 /* BTI; ignored for LSC */); - } - - if (urb_fence) { - assert(opcode == ELK_SHADER_OPCODE_MEMORY_FENCE); - fence_regs[fence_regs_count++] = - emit_fence(ubld, opcode, ELK_SFID_URB, desc, - true /* commit_enable */, - 0 /* BTI; ignored for LSC */); - } - } else if (devinfo->ver >= 11) { + if (devinfo->ver >= 11) { if (tgm_fence || ugm_fence || urb_fence) { fence_regs[fence_regs_count++] = emit_fence(ubld, opcode, GFX7_SFID_DATAPORT_DATA_CACHE, 0, diff --git a/src/intel/compiler/elk/elk_fs_reg_allocate.cpp b/src/intel/compiler/elk/elk_fs_reg_allocate.cpp index 7cfb591124f..df236a6238a 100644 --- a/src/intel/compiler/elk/elk_fs_reg_allocate.cpp +++ b/src/intel/compiler/elk/elk_fs_reg_allocate.cpp @@ -827,53 +827,7 @@ elk_fs_reg_alloc::emit_unspill(const fs_builder &bld, ++stats->fill_count; elk_fs_inst *unspill_inst; - if (devinfo->verx10 >= 125) { - /* LSC is limited to SIMD16 load/store but we can load more using - * transpose messages. - */ - const bool use_transpose = bld.dispatch_width() > 16; - const fs_builder ubld = use_transpose ? bld.exec_all().group(1, 0) : bld; - elk_fs_reg offset; - if (use_transpose) { - offset = build_single_offset(ubld, spill_offset, ip); - } else { - offset = build_lane_offsets(ubld, spill_offset, ip); - } - /* We leave the extended descriptor empty and flag the instruction to - * ask the generated to insert the extended descriptor in the address - * register. That way we don't need to burn an additional register - * for register allocation spill/fill. - */ - elk_fs_reg srcs[] = { - elk_imm_ud(0), /* desc */ - elk_imm_ud(0), /* ex_desc */ - offset, /* payload */ - elk_fs_reg(), /* payload2 */ - }; - - unspill_inst = ubld.emit(ELK_SHADER_OPCODE_SEND, dst, - srcs, ARRAY_SIZE(srcs)); - unspill_inst->sfid = GFX12_SFID_UGM; - unspill_inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD, - unspill_inst->exec_size, - LSC_ADDR_SURFTYPE_SS, - LSC_ADDR_SIZE_A32, - 1 /* num_coordinates */, - LSC_DATA_SIZE_D32, - use_transpose ? reg_size * 8 : 1 /* num_channels */, - use_transpose, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS), - true /* has_dest */); - unspill_inst->header_size = 0; - unspill_inst->mlen = - lsc_msg_desc_src0_len(devinfo, unspill_inst->desc); - unspill_inst->ex_mlen = 0; - unspill_inst->size_written = - lsc_msg_desc_dest_len(devinfo, unspill_inst->desc) * REG_SIZE; - unspill_inst->send_has_side_effects = false; - unspill_inst->send_is_volatile = true; - unspill_inst->send_ex_desc_scratch = true; - } else if (devinfo->ver >= 9) { + if (devinfo->ver >= 9) { elk_fs_reg header = this->scratch_header; fs_builder ubld = bld.exec_all().group(1, 0); assert(spill_offset % 16 == 0); @@ -935,40 +889,7 @@ elk_fs_reg_alloc::emit_spill(const fs_builder &bld, ++stats->spill_count; elk_fs_inst *spill_inst; - if (devinfo->verx10 >= 125) { - elk_fs_reg offset = build_lane_offsets(bld, spill_offset, ip); - /* We leave the extended descriptor empty and flag the instruction - * relocate the extended descriptor. That way the surface offset is - * directly put into the instruction and we don't need to use a - * register to hold it. - */ - elk_fs_reg srcs[] = { - elk_imm_ud(0), /* desc */ - elk_imm_ud(0), /* ex_desc */ - offset, /* payload */ - src, /* payload2 */ - }; - spill_inst = bld.emit(ELK_SHADER_OPCODE_SEND, bld.null_reg_f(), - srcs, ARRAY_SIZE(srcs)); - spill_inst->sfid = GFX12_SFID_UGM; - spill_inst->desc = lsc_msg_desc(devinfo, LSC_OP_STORE, - bld.dispatch_width(), - LSC_ADDR_SURFTYPE_SS, - LSC_ADDR_SIZE_A32, - 1 /* num_coordinates */, - LSC_DATA_SIZE_D32, - 1 /* num_channels */, - false /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS), - false /* has_dest */); - spill_inst->header_size = 0; - spill_inst->mlen = lsc_msg_desc_src0_len(devinfo, spill_inst->desc); - spill_inst->ex_mlen = reg_size; - spill_inst->size_written = 0; - spill_inst->send_has_side_effects = true; - spill_inst->send_is_volatile = false; - spill_inst->send_ex_desc_scratch = true; - } else if (devinfo->ver >= 9) { + if (devinfo->ver >= 9) { elk_fs_reg header = this->scratch_header; fs_builder ubld = bld.exec_all().group(1, 0); assert(spill_offset % 16 == 0); diff --git a/src/intel/compiler/elk/elk_gram.y b/src/intel/compiler/elk/elk_gram.y index 50b5c701132..d53406a072f 100644 --- a/src/intel/compiler/elk/elk_gram.y +++ b/src/intel/compiler/elk/elk_gram.y @@ -1029,9 +1029,6 @@ sharedfunction: | CRE { $$ = HSW_SFID_CRE; } | SAMPLER { $$ = ELK_SFID_SAMPLER; } | DP_SAMPLER { $$ = GFX6_SFID_DATAPORT_SAMPLER_CACHE; } - | SLM { $$ = GFX12_SFID_SLM; } - | TGM { $$ = GFX12_SFID_TGM; } - | UGM { $$ = GFX12_SFID_UGM; } ; exp2: diff --git a/src/intel/compiler/elk/elk_ir_performance.cpp b/src/intel/compiler/elk/elk_ir_performance.cpp index 43a651899f6..256f3d9cd92 100644 --- a/src/intel/compiler/elk/elk_ir_performance.cpp +++ b/src/intel/compiler/elk/elk_ir_performance.cpp @@ -935,9 +935,6 @@ namespace { case ELK_SFID_URB: case GFX7_SFID_DATAPORT_DATA_CACHE: - case GFX12_SFID_SLM: - case GFX12_SFID_TGM: - case GFX12_SFID_UGM: case HSW_SFID_DATAPORT_DATA_CACHE_1: if (devinfo->ver >= 7) return calculate_desc(info, EU_UNIT_DP_DC, 2, 0, 0, 30 /* XXX */, 0, @@ -1103,47 +1100,6 @@ namespace { else abort(); - case GFX12_SFID_UGM: - case GFX12_SFID_TGM: - case GFX12_SFID_SLM: - switch (lsc_msg_desc_opcode(devinfo, info.desc)) { - case LSC_OP_LOAD: - case LSC_OP_STORE: - case LSC_OP_LOAD_CMASK: - case LSC_OP_STORE_CMASK: - return calculate_desc(info, EU_UNIT_DP_DC, 2, 0, 0, - 0, 20 /* XXX */, - 10 /* XXX */, 100 /* XXX */, 0, 0, - 0, 0); - - case LSC_OP_FENCE: - case LSC_OP_ATOMIC_INC: - case LSC_OP_ATOMIC_DEC: - case LSC_OP_ATOMIC_LOAD: - case LSC_OP_ATOMIC_STORE: - case LSC_OP_ATOMIC_ADD: - case LSC_OP_ATOMIC_SUB: - case LSC_OP_ATOMIC_MIN: - case LSC_OP_ATOMIC_MAX: - case LSC_OP_ATOMIC_UMIN: - case LSC_OP_ATOMIC_UMAX: - case LSC_OP_ATOMIC_CMPXCHG: - case LSC_OP_ATOMIC_FADD: - case LSC_OP_ATOMIC_FSUB: - case LSC_OP_ATOMIC_FMIN: - case LSC_OP_ATOMIC_FMAX: - case LSC_OP_ATOMIC_FCMPXCHG: - case LSC_OP_ATOMIC_AND: - case LSC_OP_ATOMIC_OR: - case LSC_OP_ATOMIC_XOR: - return calculate_desc(info, EU_UNIT_DP_DC, 2, 0, 0, - 30 /* XXX */, 400 /* XXX */, - 10 /* XXX */, 100 /* XXX */, 0, 0, - 0, 400 /* XXX */); - default: - abort(); - } - case ELK_SFID_URB: return calculate_desc(info, EU_UNIT_URB, 2, 0, 0, 0, 6 /* XXX */, 32 /* XXX */, 200 /* XXX */, 0, 0, 0, 0); diff --git a/src/intel/compiler/elk/elk_lower_logical_sends.cpp b/src/intel/compiler/elk/elk_lower_logical_sends.cpp index 218004566bf..461b590efec 100644 --- a/src/intel/compiler/elk/elk_lower_logical_sends.cpp +++ b/src/intel/compiler/elk/elk_lower_logical_sends.cpp @@ -1611,49 +1611,6 @@ setup_surface_descriptors(const fs_builder &bld, elk_fs_inst *inst, uint32_t des } } -static void -setup_lsc_surface_descriptors(const fs_builder &bld, elk_fs_inst *inst, - uint32_t desc, const elk_fs_reg &surface) -{ - const ASSERTED intel_device_info *devinfo = bld.shader->devinfo; - const elk_compiler *compiler = bld.shader->compiler; - - inst->src[0] = elk_imm_ud(0); /* desc */ - - enum lsc_addr_surface_type surf_type = lsc_msg_desc_addr_type(devinfo, desc); - switch (surf_type) { - case LSC_ADDR_SURFTYPE_BSS: - inst->send_ex_bso = compiler->extended_bindless_surface_offset; - /* fall-through */ - case LSC_ADDR_SURFTYPE_SS: - assert(surface.file != BAD_FILE); - /* We assume that the driver provided the handle in the top 20 bits so - * we can use the surface handle directly as the extended descriptor. - */ - inst->src[1] = retype(surface, ELK_REGISTER_TYPE_UD); - break; - - case LSC_ADDR_SURFTYPE_BTI: - assert(surface.file != BAD_FILE); - if (surface.file == IMM) { - inst->src[1] = elk_imm_ud(lsc_bti_ex_desc(devinfo, surface.ud)); - } else { - const fs_builder ubld = bld.exec_all().group(1, 0); - elk_fs_reg tmp = ubld.vgrf(ELK_REGISTER_TYPE_UD); - ubld.SHL(tmp, surface, elk_imm_ud(24)); - inst->src[1] = component(tmp, 0); - } - break; - - case LSC_ADDR_SURFTYPE_FLAT: - inst->src[1] = elk_imm_ud(0); - break; - - default: - unreachable("Invalid LSC surface address type"); - } -} - static void lower_surface_logical_send(const fs_builder &bld, elk_fs_inst *inst) { @@ -1903,262 +1860,6 @@ lower_surface_logical_send(const fs_builder &bld, elk_fs_inst *inst) inst->src[3] = payload2; } -static enum lsc_data_size -lsc_bits_to_data_size(unsigned bit_size) -{ - switch (bit_size / 8) { - case 1: return LSC_DATA_SIZE_D8U32; - case 2: return LSC_DATA_SIZE_D16U32; - case 4: return LSC_DATA_SIZE_D32; - case 8: return LSC_DATA_SIZE_D64; - default: - unreachable("Unsupported data size."); - } -} - -static void -lower_lsc_surface_logical_send(const fs_builder &bld, elk_fs_inst *inst) -{ - const elk_compiler *compiler = bld.shader->compiler; - const intel_device_info *devinfo = bld.shader->devinfo; - assert(devinfo->has_lsc); - - /* Get the logical send arguments. */ - const elk_fs_reg addr = inst->src[SURFACE_LOGICAL_SRC_ADDRESS]; - const elk_fs_reg src = inst->src[SURFACE_LOGICAL_SRC_DATA]; - const elk_fs_reg surface = inst->src[SURFACE_LOGICAL_SRC_SURFACE]; - const elk_fs_reg surface_handle = inst->src[SURFACE_LOGICAL_SRC_SURFACE_HANDLE]; - const UNUSED elk_fs_reg dims = inst->src[SURFACE_LOGICAL_SRC_IMM_DIMS]; - const elk_fs_reg arg = inst->src[SURFACE_LOGICAL_SRC_IMM_ARG]; - const elk_fs_reg allow_sample_mask = - inst->src[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK]; - assert(arg.file == IMM); - assert(allow_sample_mask.file == IMM); - - /* Calculate the total number of components of the payload. */ - const unsigned addr_sz = inst->components_read(SURFACE_LOGICAL_SRC_ADDRESS); - const unsigned src_comps = inst->components_read(SURFACE_LOGICAL_SRC_DATA); - const unsigned src_sz = type_sz(src.type); - const unsigned dst_sz = type_sz(inst->dst.type); - - const bool has_side_effects = inst->has_side_effects(); - - unsigned ex_mlen = 0; - elk_fs_reg payload, payload2; - payload = bld.move_to_vgrf(addr, addr_sz); - if (src.file != BAD_FILE) { - payload2 = bld.move_to_vgrf(src, src_comps); - ex_mlen = (src_comps * src_sz * inst->exec_size) / REG_SIZE; - } - - /* Predicate the instruction on the sample mask if needed */ - elk_fs_reg sample_mask = allow_sample_mask.ud ? elk_sample_mask_reg(bld) : - elk_fs_reg(elk_imm_ud(0xffffffff)); - if (sample_mask.file != BAD_FILE && sample_mask.file != IMM) - elk_emit_predicate_on_sample_mask(bld, inst); - - if (surface.file == IMM && surface.ud == GFX7_BTI_SLM) - inst->sfid = GFX12_SFID_SLM; - else - inst->sfid = GFX12_SFID_UGM; - - /* We should have exactly one of surface and surface_handle. For scratch - * messages generated by elk_fs_nir.cpp we also allow a special value to - * know what heap base we should use in STATE_BASE_ADDRESS (SS = Surface - * State Offset, or BSS = Bindless Surface State Offset). - */ - bool non_bindless = surface.file == IMM && surface.ud == GFX125_NON_BINDLESS; - assert((surface.file == BAD_FILE) != (surface_handle.file == BAD_FILE) || - (non_bindless && surface_handle.file != BAD_FILE)); - - enum lsc_addr_surface_type surf_type; - if (surface_handle.file != BAD_FILE) { - if (surface.file == BAD_FILE) { - assert(!non_bindless); - surf_type = LSC_ADDR_SURFTYPE_BSS; - } else { - assert(surface.file == IMM && - (surface.ud == 0 || surface.ud == GFX125_NON_BINDLESS)); - surf_type = non_bindless ? LSC_ADDR_SURFTYPE_SS : LSC_ADDR_SURFTYPE_BSS; - } - } else if (surface.file == IMM && surface.ud == GFX7_BTI_SLM) - surf_type = LSC_ADDR_SURFTYPE_FLAT; - else - surf_type = LSC_ADDR_SURFTYPE_BTI; - - switch (inst->opcode) { - case ELK_SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL: - inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD_CMASK, inst->exec_size, - surf_type, LSC_ADDR_SIZE_A32, - 1 /* num_coordinates */, - LSC_DATA_SIZE_D32, arg.ud /* num_channels */, - false /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS), - true /* has_dest */); - break; - case ELK_SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL: - inst->desc = lsc_msg_desc(devinfo, LSC_OP_STORE_CMASK, inst->exec_size, - surf_type, LSC_ADDR_SIZE_A32, - 1 /* num_coordinates */, - LSC_DATA_SIZE_D32, arg.ud /* num_channels */, - false /* transpose */, - LSC_CACHE(devinfo, STORE, L1STATE_L3MOCS), - false /* has_dest */); - break; - case ELK_SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: { - /* Bspec: Atomic instruction -> Cache section: - * - * Atomic messages are always forced to "un-cacheable" in the L1 - * cache. - */ - enum elk_lsc_opcode opcode = (enum elk_lsc_opcode) arg.ud; - - inst->desc = lsc_msg_desc(devinfo, opcode, inst->exec_size, - surf_type, LSC_ADDR_SIZE_A32, - 1 /* num_coordinates */, - lsc_bits_to_data_size(dst_sz * 8), - 1 /* num_channels */, - false /* transpose */, - LSC_CACHE(devinfo, STORE, L1UC_L3WB), - !inst->dst.is_null()); - break; - } - case ELK_SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL: - inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD, inst->exec_size, - surf_type, LSC_ADDR_SIZE_A32, - 1 /* num_coordinates */, - lsc_bits_to_data_size(arg.ud), - 1 /* num_channels */, - false /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS), - true /* has_dest */); - break; - case ELK_SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL: - inst->desc = lsc_msg_desc(devinfo, LSC_OP_STORE, inst->exec_size, - surf_type, LSC_ADDR_SIZE_A32, - 1 /* num_coordinates */, - lsc_bits_to_data_size(arg.ud), - 1 /* num_channels */, - false /* transpose */, - LSC_CACHE(devinfo, STORE, L1STATE_L3MOCS), - false /* has_dest */); - break; - default: - unreachable("Unknown surface logical instruction"); - } - - /* Update the original instruction. */ - inst->opcode = ELK_SHADER_OPCODE_SEND; - inst->mlen = lsc_msg_desc_src0_len(devinfo, inst->desc); - inst->ex_mlen = ex_mlen; - inst->header_size = 0; - inst->send_has_side_effects = has_side_effects; - inst->send_is_volatile = !has_side_effects; - inst->send_ex_bso = surf_type == LSC_ADDR_SURFTYPE_BSS && - compiler->extended_bindless_surface_offset; - - inst->resize_sources(4); - - if (non_bindless) { - inst->src[0] = elk_imm_ud(0); /* desc */ - inst->src[1] = surface_handle; /* ex_desc */ - } else { - setup_lsc_surface_descriptors(bld, inst, inst->desc, - surface.file != BAD_FILE ? - surface : surface_handle); - } - - /* Finally, the payload */ - inst->src[2] = payload; - inst->src[3] = payload2; -} - -static void -lower_lsc_block_logical_send(const fs_builder &bld, elk_fs_inst *inst) -{ - const elk_compiler *compiler = bld.shader->compiler; - const intel_device_info *devinfo = bld.shader->devinfo; - assert(devinfo->has_lsc); - - /* Get the logical send arguments. */ - const elk_fs_reg addr = inst->src[SURFACE_LOGICAL_SRC_ADDRESS]; - const elk_fs_reg src = inst->src[SURFACE_LOGICAL_SRC_DATA]; - const elk_fs_reg surface = inst->src[SURFACE_LOGICAL_SRC_SURFACE]; - const elk_fs_reg surface_handle = inst->src[SURFACE_LOGICAL_SRC_SURFACE_HANDLE]; - const elk_fs_reg arg = inst->src[SURFACE_LOGICAL_SRC_IMM_ARG]; - assert(arg.file == IMM); - assert(inst->src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == BAD_FILE); - assert(inst->src[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK].file == BAD_FILE); - - const bool is_stateless = - surface.file == IMM && (surface.ud == ELK_BTI_STATELESS || - surface.ud == GFX8_BTI_STATELESS_NON_COHERENT); - - const bool has_side_effects = inst->has_side_effects(); - - const bool write = inst->opcode == ELK_SHADER_OPCODE_OWORD_BLOCK_WRITE_LOGICAL; - - fs_builder ubld = bld.exec_all().group(1, 0); - elk_fs_reg stateless_ex_desc; - if (is_stateless) { - stateless_ex_desc = ubld.vgrf(ELK_REGISTER_TYPE_UD); - ubld.AND(stateless_ex_desc, - retype(elk_vec1_grf(0, 5), ELK_REGISTER_TYPE_UD), - elk_imm_ud(INTEL_MASK(31, 10))); - } - - elk_fs_reg data; - if (write) { - const unsigned src_sz = inst->components_read(SURFACE_LOGICAL_SRC_DATA); - data = retype(bld.move_to_vgrf(src, src_sz), ELK_REGISTER_TYPE_UD); - } - - inst->opcode = ELK_SHADER_OPCODE_SEND; - if (surface.file == IMM && surface.ud == GFX7_BTI_SLM) - inst->sfid = GFX12_SFID_SLM; - else - inst->sfid = GFX12_SFID_UGM; - const enum lsc_addr_surface_type surf_type = - inst->sfid == GFX12_SFID_SLM ? - LSC_ADDR_SURFTYPE_FLAT : - surface.file == BAD_FILE ? - LSC_ADDR_SURFTYPE_BSS : LSC_ADDR_SURFTYPE_BTI; - inst->desc = lsc_msg_desc(devinfo, - write ? LSC_OP_STORE : LSC_OP_LOAD, - 1 /* exec_size */, - surf_type, - LSC_ADDR_SIZE_A32, - 1 /* num_coordinates */, - LSC_DATA_SIZE_D32, - arg.ud /* num_channels */, - true /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS), - !write /* has_dest */); - - inst->mlen = lsc_msg_desc_src0_len(devinfo, inst->desc); - inst->size_written = lsc_msg_desc_dest_len(devinfo, inst->desc) * REG_SIZE; - inst->exec_size = 1; - inst->ex_mlen = write ? DIV_ROUND_UP(arg.ud, 8) : 0; - inst->header_size = 0; - inst->send_has_side_effects = has_side_effects; - inst->send_is_volatile = !has_side_effects; - inst->send_ex_bso = surf_type == LSC_ADDR_SURFTYPE_BSS && - compiler->extended_bindless_surface_offset; - - inst->resize_sources(4); - - if (stateless_ex_desc.file != BAD_FILE) { - inst->src[0] = elk_imm_ud(0); /* desc */ - inst->src[1] = stateless_ex_desc; /* ex_desc */ - } else { - setup_lsc_surface_descriptors(bld, inst, inst->desc, - surface.file != BAD_FILE ? - surface : surface_handle); - } - inst->src[2] = addr; /* payload */ - inst->src[3] = data; /* payload2 */ -} - static void lower_surface_block_logical_send(const fs_builder &bld, elk_fs_inst *inst) { @@ -2275,137 +1976,6 @@ emit_fragment_mask(const fs_builder &bld, elk_fs_inst *inst) elk_emit_predicate_on_sample_mask(bld, inst); } -static void -lower_lsc_a64_logical_send(const fs_builder &bld, elk_fs_inst *inst) -{ - const intel_device_info *devinfo = bld.shader->devinfo; - - /* Get the logical send arguments. */ - const elk_fs_reg addr = inst->src[A64_LOGICAL_ADDRESS]; - const elk_fs_reg src = inst->src[A64_LOGICAL_SRC]; - const unsigned src_sz = type_sz(src.type); - const unsigned dst_sz = type_sz(inst->dst.type); - - const unsigned src_comps = inst->components_read(1); - assert(inst->src[A64_LOGICAL_ARG].file == IMM); - const unsigned arg = inst->src[A64_LOGICAL_ARG].ud; - const bool has_side_effects = inst->has_side_effects(); - - elk_fs_reg payload = retype(bld.move_to_vgrf(addr, 1), ELK_REGISTER_TYPE_UD); - elk_fs_reg payload2 = retype(bld.move_to_vgrf(src, src_comps), - ELK_REGISTER_TYPE_UD); - unsigned ex_mlen = src_comps * src_sz * inst->exec_size / REG_SIZE; - - switch (inst->opcode) { - case ELK_SHADER_OPCODE_A64_UNTYPED_READ_LOGICAL: - inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD_CMASK, inst->exec_size, - LSC_ADDR_SURFTYPE_FLAT, LSC_ADDR_SIZE_A64, - 1 /* num_coordinates */, - LSC_DATA_SIZE_D32, arg /* num_channels */, - false /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS), - true /* has_dest */); - break; - case ELK_SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL: - inst->desc = lsc_msg_desc(devinfo, LSC_OP_STORE_CMASK, inst->exec_size, - LSC_ADDR_SURFTYPE_FLAT, LSC_ADDR_SIZE_A64, - 1 /* num_coordinates */, - LSC_DATA_SIZE_D32, arg /* num_channels */, - false /* transpose */, - LSC_CACHE(devinfo, STORE, L1STATE_L3MOCS), - false /* has_dest */); - break; - case ELK_SHADER_OPCODE_A64_BYTE_SCATTERED_READ_LOGICAL: - inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD, inst->exec_size, - LSC_ADDR_SURFTYPE_FLAT, LSC_ADDR_SIZE_A64, - 1 /* num_coordinates */, - lsc_bits_to_data_size(arg), - 1 /* num_channels */, - false /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS), - true /* has_dest */); - break; - case ELK_SHADER_OPCODE_A64_BYTE_SCATTERED_WRITE_LOGICAL: - inst->desc = lsc_msg_desc(devinfo, LSC_OP_STORE, inst->exec_size, - LSC_ADDR_SURFTYPE_FLAT, LSC_ADDR_SIZE_A64, - 1 /* num_coordinates */, - lsc_bits_to_data_size(arg), - 1 /* num_channels */, - false /* transpose */, - LSC_CACHE(devinfo, STORE, L1STATE_L3MOCS), - false /* has_dest */); - break; - case ELK_SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL: { - /* Bspec: Atomic instruction -> Cache section: - * - * Atomic messages are always forced to "un-cacheable" in the L1 - * cache. - */ - enum elk_lsc_opcode opcode = (enum elk_lsc_opcode) arg; - inst->desc = lsc_msg_desc(devinfo, opcode, inst->exec_size, - LSC_ADDR_SURFTYPE_FLAT, LSC_ADDR_SIZE_A64, - 1 /* num_coordinates */, - lsc_bits_to_data_size(dst_sz * 8), - 1 /* num_channels */, - false /* transpose */, - LSC_CACHE(devinfo, STORE, L1UC_L3WB), - !inst->dst.is_null()); - break; - } - case ELK_SHADER_OPCODE_A64_OWORD_BLOCK_READ_LOGICAL: - case ELK_SHADER_OPCODE_A64_UNALIGNED_OWORD_BLOCK_READ_LOGICAL: - inst->exec_size = 1; - inst->desc = lsc_msg_desc(devinfo, - LSC_OP_LOAD, - 1 /* exec_size */, - LSC_ADDR_SURFTYPE_FLAT, - LSC_ADDR_SIZE_A64, - 1 /* num_coordinates */, - LSC_DATA_SIZE_D32, - arg /* num_channels */, - true /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS), - true /* has_dest */); - break; - case ELK_SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL: - inst->exec_size = 1; - inst->desc = lsc_msg_desc(devinfo, - LSC_OP_STORE, - 1 /* exec_size */, - LSC_ADDR_SURFTYPE_FLAT, - LSC_ADDR_SIZE_A64, - 1 /* num_coordinates */, - LSC_DATA_SIZE_D32, - arg /* num_channels */, - true /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS), - false /* has_dest */); - - break; - default: - unreachable("Unknown A64 logical instruction"); - } - - if (bld.shader->stage == MESA_SHADER_FRAGMENT) - emit_fragment_mask(bld, inst); - - /* Update the original instruction. */ - inst->opcode = ELK_SHADER_OPCODE_SEND; - inst->mlen = lsc_msg_desc_src0_len(devinfo, inst->desc); - inst->ex_mlen = ex_mlen; - inst->header_size = 0; - inst->send_has_side_effects = has_side_effects; - inst->send_is_volatile = !has_side_effects; - - /* Set up SFID and descriptors */ - inst->sfid = GFX12_SFID_UGM; - inst->resize_sources(4); - inst->src[0] = elk_imm_ud(0); /* desc */ - inst->src[1] = elk_imm_ud(0); /* ex_desc */ - inst->src[2] = payload; - inst->src[3] = payload2; -} - static void lower_a64_logical_send(const fs_builder &bld, elk_fs_inst *inst) { @@ -2545,97 +2115,6 @@ lower_a64_logical_send(const fs_builder &bld, elk_fs_inst *inst) inst->src[3] = payload2; } -static void -lower_lsc_varying_pull_constant_logical_send(const fs_builder &bld, - elk_fs_inst *inst) -{ - const intel_device_info *devinfo = bld.shader->devinfo; - ASSERTED const elk_compiler *compiler = bld.shader->compiler; - - elk_fs_reg surface = inst->src[PULL_VARYING_CONSTANT_SRC_SURFACE]; - elk_fs_reg surface_handle = inst->src[PULL_VARYING_CONSTANT_SRC_SURFACE_HANDLE]; - elk_fs_reg offset_B = inst->src[PULL_VARYING_CONSTANT_SRC_OFFSET]; - elk_fs_reg alignment_B = inst->src[PULL_VARYING_CONSTANT_SRC_ALIGNMENT]; - - /* We are switching the instruction from an ALU-like instruction to a - * send-from-grf instruction. Since sends can't handle strides or - * source modifiers, we have to make a copy of the offset source. - */ - elk_fs_reg ubo_offset = bld.move_to_vgrf(offset_B, 1); - - enum lsc_addr_surface_type surf_type = - surface_handle.file == BAD_FILE ? - LSC_ADDR_SURFTYPE_BTI : LSC_ADDR_SURFTYPE_BSS; - - assert(alignment_B.file == ELK_IMMEDIATE_VALUE); - unsigned alignment = alignment_B.ud; - - inst->opcode = ELK_SHADER_OPCODE_SEND; - inst->sfid = GFX12_SFID_UGM; - inst->resize_sources(3); - inst->send_ex_bso = surf_type == LSC_ADDR_SURFTYPE_BSS && - compiler->extended_bindless_surface_offset; - - assert(!compiler->indirect_ubos_use_sampler); - - inst->src[0] = elk_imm_ud(0); - inst->src[2] = ubo_offset; /* payload */ - - if (alignment >= 4) { - inst->desc = - lsc_msg_desc(devinfo, LSC_OP_LOAD_CMASK, inst->exec_size, - surf_type, LSC_ADDR_SIZE_A32, - 1 /* num_coordinates */, - LSC_DATA_SIZE_D32, - 4 /* num_channels */, - false /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS), - true /* has_dest */); - inst->mlen = lsc_msg_desc_src0_len(devinfo, inst->desc); - - setup_lsc_surface_descriptors(bld, inst, inst->desc, - surface.file != BAD_FILE ? - surface : surface_handle); - } else { - inst->desc = - lsc_msg_desc(devinfo, LSC_OP_LOAD, inst->exec_size, - surf_type, LSC_ADDR_SIZE_A32, - 1 /* num_coordinates */, - LSC_DATA_SIZE_D32, - 1 /* num_channels */, - false /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS), - true /* has_dest */); - inst->mlen = lsc_msg_desc_src0_len(devinfo, inst->desc); - - setup_lsc_surface_descriptors(bld, inst, inst->desc, - surface.file != BAD_FILE ? - surface : surface_handle); - - /* The byte scattered messages can only read one dword at a time so - * we have to duplicate the message 4 times to read the full vec4. - * Hopefully, dead code will clean up the mess if some of them aren't - * needed. - */ - assert(inst->size_written == 16 * inst->exec_size); - inst->size_written /= 4; - for (unsigned c = 1; c < 4; c++) { - /* Emit a copy of the instruction because we're about to modify - * it. Because this loop starts at 1, we will emit copies for the - * first 3 and the final one will be the modified instruction. - */ - bld.emit(*inst); - - /* Offset the source */ - inst->src[2] = bld.vgrf(ELK_REGISTER_TYPE_UD); - bld.ADD(inst->src[2], ubo_offset, elk_imm_ud(c * 4)); - - /* Offset the destination */ - inst->dst = offset(inst->dst, bld, 1); - } - } -} - static void lower_varying_pull_constant_logical_send(const fs_builder &bld, elk_fs_inst *inst) { @@ -3018,10 +2497,6 @@ elk_fs_visitor::lower_logical_sends() case ELK_SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: case ELK_SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL: case ELK_SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL: - if (devinfo->has_lsc) { - lower_lsc_surface_logical_send(ibld, inst); - break; - } case ELK_SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL: case ELK_SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL: case ELK_SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL: @@ -3032,10 +2507,6 @@ elk_fs_visitor::lower_logical_sends() case ELK_SHADER_OPCODE_UNALIGNED_OWORD_BLOCK_READ_LOGICAL: case ELK_SHADER_OPCODE_OWORD_BLOCK_WRITE_LOGICAL: - if (devinfo->has_lsc) { - lower_lsc_block_logical_send(ibld, inst); - break; - } lower_surface_block_logical_send(ibld, inst); break; @@ -3047,18 +2518,11 @@ elk_fs_visitor::lower_logical_sends() case ELK_SHADER_OPCODE_A64_OWORD_BLOCK_READ_LOGICAL: case ELK_SHADER_OPCODE_A64_UNALIGNED_OWORD_BLOCK_READ_LOGICAL: case ELK_SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL: - if (devinfo->has_lsc) { - lower_lsc_a64_logical_send(ibld, inst); - break; - } lower_a64_logical_send(ibld, inst); break; case ELK_FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_LOGICAL: - if (devinfo->has_lsc && !compiler->indirect_ubos_use_sampler) - lower_lsc_varying_pull_constant_logical_send(ibld, inst); - else - lower_varying_pull_constant_logical_send(ibld, inst); + lower_varying_pull_constant_logical_send(ibld, inst); break; case ELK_SHADER_OPCODE_RCP: @@ -3155,48 +2619,7 @@ elk_fs_visitor::lower_uniform_pull_constant_loads() assert(offset_B.file == IMM); assert(size_B.file == IMM); - if (devinfo->has_lsc) { - const fs_builder ubld = - fs_builder(this, block, inst).group(8, 0).exec_all(); - - const elk_fs_reg payload = ubld.vgrf(ELK_REGISTER_TYPE_UD); - ubld.MOV(payload, offset_B); - - inst->sfid = GFX12_SFID_UGM; - inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD, - 1 /* simd_size */, - surface_handle.file == BAD_FILE ? - LSC_ADDR_SURFTYPE_BTI : - LSC_ADDR_SURFTYPE_BSS, - LSC_ADDR_SIZE_A32, - 1 /* num_coordinates */, - LSC_DATA_SIZE_D32, - inst->size_written / 4, - true /* transpose */, - LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS), - true /* has_dest */); - - /* Update the original instruction. */ - inst->opcode = ELK_SHADER_OPCODE_SEND; - inst->mlen = lsc_msg_desc_src0_len(devinfo, inst->desc); - inst->send_ex_bso = surface_handle.file != BAD_FILE && - compiler->extended_bindless_surface_offset; - inst->ex_mlen = 0; - inst->header_size = 0; - inst->send_has_side_effects = false; - inst->send_is_volatile = true; - inst->exec_size = 1; - - /* Finally, the payload */ - - inst->resize_sources(3); - setup_lsc_surface_descriptors(ubld, inst, inst->desc, - surface.file != BAD_FILE ? - surface : surface_handle); - inst->src[2] = payload; - - invalidate_analysis(DEPENDENCY_INSTRUCTIONS | DEPENDENCY_VARIABLES); - } else if (devinfo->ver >= 7) { + if (devinfo->ver >= 7) { const fs_builder ubld = fs_builder(this, block, inst).exec_all(); elk_fs_reg header = fs_builder(this, 8).exec_all().vgrf(ELK_REGISTER_TYPE_UD); diff --git a/src/intel/compiler/elk/elk_schedule_instructions.cpp b/src/intel/compiler/elk/elk_schedule_instructions.cpp index 175010c257b..3db06a4eaed 100644 --- a/src/intel/compiler/elk/elk_schedule_instructions.cpp +++ b/src/intel/compiler/elk/elk_schedule_instructions.cpp @@ -571,43 +571,6 @@ elk_schedule_node::set_latency_gfx7(const struct elk_isa_info *isa) latency = 50; /* TODO */ break; - case GFX12_SFID_UGM: - case GFX12_SFID_TGM: - case GFX12_SFID_SLM: - switch (lsc_msg_desc_opcode(isa->devinfo, inst->desc)) { - case LSC_OP_LOAD: - case LSC_OP_STORE: - case LSC_OP_LOAD_CMASK: - case LSC_OP_STORE_CMASK: - latency = 300; - break; - case LSC_OP_FENCE: - case LSC_OP_ATOMIC_INC: - case LSC_OP_ATOMIC_DEC: - case LSC_OP_ATOMIC_LOAD: - case LSC_OP_ATOMIC_STORE: - case LSC_OP_ATOMIC_ADD: - case LSC_OP_ATOMIC_SUB: - case LSC_OP_ATOMIC_MIN: - case LSC_OP_ATOMIC_MAX: - case LSC_OP_ATOMIC_UMIN: - case LSC_OP_ATOMIC_UMAX: - case LSC_OP_ATOMIC_CMPXCHG: - case LSC_OP_ATOMIC_FADD: - case LSC_OP_ATOMIC_FSUB: - case LSC_OP_ATOMIC_FMIN: - case LSC_OP_ATOMIC_FMAX: - case LSC_OP_ATOMIC_FCMPXCHG: - case LSC_OP_ATOMIC_AND: - case LSC_OP_ATOMIC_OR: - case LSC_OP_ATOMIC_XOR: - latency = 1400; - break; - default: - unreachable("unsupported new data port message instruction"); - } - break; - case ELK_SFID_URB: latency = 200; break;