AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

intel/elk: Remove Gfx12 SFIDs and related LSC code

Browse Source 
Reviewed-by: Lionel Landwerlin <lionel.g.landwerlin@intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/27629>
This commit is contained in:
Caio Oliveira
2024-02-10 08:32:30 -08:00
committed by Marge Bot
parent 43b2261ab5
commit 9b709e31cb
12 changed files with 13 additions and 1194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/intel/compiler/elk/elk_disasm.c
+7 -156
View File
@@ -313,9 +313,6 @@ static const char *const gfx6_sfid[16] = {
[GFX7_SFID_PIXEL_INTERPOLATOR] = "pixel interp",
[HSW_SFID_DATAPORT_DATA_CACHE_1] = "dp data 1",
[HSW_SFID_CRE] = "cre",
[GFX12_SFID_SLM] = "slm",
[GFX12_SFID_TGM] = "tgm",
[GFX12_SFID_UGM] = "ugm",
};
static const char *const gfx7_gateway_subfuncid[8] = {
@@ -685,37 +682,12 @@ static const char *const lsc_addr_surface_type[] = {
[LSC_ADDR_SURFTYPE_BTI] = "bti",
};
static const char* const lsc_fence_scope[] = {
[LSC_FENCE_THREADGROUP] = "threadgroup",
[LSC_FENCE_LOCAL] = "local",
[LSC_FENCE_TILE] = "tile",
[LSC_FENCE_GPU] = "gpu",
[LSC_FENCE_ALL_GPU] = "all_gpu",
[LSC_FENCE_SYSTEM_RELEASE] = "system_release",
[LSC_FENCE_SYSTEM_ACQUIRE] = "system_acquire",
};
static const char* const lsc_flush_type[] = {
[LSC_FLUSH_TYPE_NONE] = "none",
[LSC_FLUSH_TYPE_EVICT] = "evict",
[LSC_FLUSH_TYPE_INVALIDATE] = "invalidate",
[LSC_FLUSH_TYPE_DISCARD] = "discard",
[LSC_FLUSH_TYPE_CLEAN] = "clean",
[LSC_FLUSH_TYPE_L3ONLY] = "l3only",
[LSC_FLUSH_TYPE_NONE_6] = "none_6",
};
static const char* const lsc_addr_size[] = {
[LSC_ADDR_SIZE_A16] = "a16",
[LSC_ADDR_SIZE_A32] = "a32",
[LSC_ADDR_SIZE_A64] = "a64",
};
static const char* const lsc_backup_fence_routing[] = {
[LSC_NORMAL_ROUTING] = "normal_routing",
[LSC_ROUTE_TO_LSC] = "route_to_lsc",
};
static const char* const lsc_data_size[] = {
[LSC_DATA_SIZE_D8] = "d8",
[LSC_DATA_SIZE_D16] = "d16",
@@ -777,30 +749,6 @@ static const char* const lsc_cache_store[] = {
[LSC_CACHE_STORE_L1WB_L3WB] = "L1WB_L3WB",
};
static const char* const xe2_lsc_cache_load[] = {
[XE2_LSC_CACHE_LOAD_L1STATE_L3MOCS] = "L1STATE_L3MOCS",
[XE2_LSC_CACHE_LOAD_L1UC_L3UC] = "L1UC_L3UC",
[XE2_LSC_CACHE_LOAD_L1UC_L3C] = "L1UC_L3C",
[XE2_LSC_CACHE_LOAD_L1UC_L3CC] = "L1UC_L3CC",
[XE2_LSC_CACHE_LOAD_L1C_L3UC] = "L1C_L3UC",
[XE2_LSC_CACHE_LOAD_L1C_L3C] = "L1C_L3C",
[XE2_LSC_CACHE_LOAD_L1C_L3CC] = "L1C_L3CC",
[XE2_LSC_CACHE_LOAD_L1S_L3UC] = "L1S_L3UC",
[XE2_LSC_CACHE_LOAD_L1S_L3C] = "L1S_L3C",
[XE2_LSC_CACHE_LOAD_L1IAR_L3IAR] = "L1IAR_L3IAR",
};
static const char* const xe2_lsc_cache_store[] = {
[XE2_LSC_CACHE_STORE_L1STATE_L3MOCS] = "L1STATE_L3MOCS",
[XE2_LSC_CACHE_STORE_L1UC_L3UC] = "L1UC_L3UC",
[XE2_LSC_CACHE_STORE_L1UC_L3WB] = "L1UC_L3WB",
[XE2_LSC_CACHE_STORE_L1WT_L3UC] = "L1WT_L3UC",
[XE2_LSC_CACHE_STORE_L1WT_L3WB] = "L1WT_L3WB",
[XE2_LSC_CACHE_STORE_L1S_L3UC] = "L1S_L3UC",
[XE2_LSC_CACHE_STORE_L1S_L3WB] = "L1S_L3WB",
[XE2_LSC_CACHE_STORE_L1WB_L3WB] = "L1WB_L3WB",
};
static int column;
static int
@@ -1732,21 +1680,6 @@ lsc_disassemble_ex_desc(const struct intel_device_info *devinfo,
}
}
static inline bool
elk_sfid_is_lsc(unsigned sfid)
{
switch (sfid) {
case GFX12_SFID_UGM:
case GFX12_SFID_SLM:
case GFX12_SFID_TGM:
return true;
default:
break;
}
return false;
}
int
elk_disassemble_inst(FILE *file, const struct elk_isa_info *isa,
const elk_inst *inst, bool is_compacted,
@@ -2144,83 +2077,6 @@ elk_disassemble_inst(FILE *file, const struct elk_isa_info *isa,
gfx7_gateway_subfuncid[elk_inst_gateway_subfuncid(devinfo, inst)]);
break;
case GFX12_SFID_SLM:
case GFX12_SFID_TGM:
case GFX12_SFID_UGM: {
assert(devinfo->has_lsc);
format(file, " (");
const enum elk_lsc_opcode op = lsc_msg_desc_opcode(devinfo, imm_desc);
err |= control(file, "operation", lsc_operation,
op, &space);
format(file, ",");
err |= control(file, "addr_size", lsc_addr_size,
lsc_msg_desc_addr_size(devinfo, imm_desc),
&space);
if (op == LSC_OP_FENCE) {
format(file, ",");
err |= control(file, "scope", lsc_fence_scope,
lsc_fence_msg_desc_scope(devinfo, imm_desc),
&space);
format(file, ",");
err |= control(file, "flush_type", lsc_flush_type,
lsc_fence_msg_desc_flush_type(devinfo, imm_desc),
&space);
format(file, ",");
err |= control(file, "backup_mode_fence_routing",
lsc_backup_fence_routing,
lsc_fence_msg_desc_backup_routing(devinfo, imm_desc),
&space);
} else {
format(file, ",");
err |= control(file, "data_size", lsc_data_size,
lsc_msg_desc_data_size(devinfo, imm_desc),
&space);
format(file, ",");
if (elk_lsc_opcode_has_cmask(op)) {
err |= control(file, "component_mask",
lsc_cmask_str,
lsc_msg_desc_cmask(devinfo, imm_desc),
&space);
} else {
err |= control(file, "vector_size",
lsc_vect_size_str,
lsc_msg_desc_vect_size(devinfo, imm_desc),
&space);
if (lsc_msg_desc_transpose(devinfo, imm_desc))
format(file, ", transpose");
}
switch(op) {
case LSC_OP_LOAD_CMASK:
case LSC_OP_LOAD:
format(file, ",");
err |= control(file, "cache_load",
devinfo->ver >= 20 ?
xe2_lsc_cache_load :
lsc_cache_load,
lsc_msg_desc_cache_ctrl(devinfo, imm_desc),
&space);
break;
default:
format(file, ",");
err |= control(file, "cache_store",
devinfo->ver >= 20 ?
xe2_lsc_cache_store :
lsc_cache_store,
lsc_msg_desc_cache_ctrl(devinfo, imm_desc),
&space);
break;
}
}
format(file, " dst_len = %u,", lsc_msg_desc_dest_len(devinfo, imm_desc));
format(file, " src0_len = %u,", lsc_msg_desc_src0_len(devinfo, imm_desc));
err |= control(file, "address_type", lsc_addr_surface_type,
lsc_msg_desc_addr_type(devinfo, imm_desc), &space);
format(file, " )");
break;
}
case GFX7_SFID_DATAPORT_DATA_CACHE:
if (devinfo->ver >= 7) {
format(file, " (");
@@ -2340,19 +2196,14 @@ elk_disassemble_inst(FILE *file, const struct elk_isa_info *isa,
if (space)
string(file, " ");
}
if (elk_sfid_is_lsc(sfid) ||
(sfid == ELK_SFID_URB && devinfo->ver >= 20)) {
lsc_disassemble_ex_desc(devinfo, imm_desc, imm_ex_desc, file);
} else {
if (has_imm_desc)
format(file, " mlen %u", elk_message_desc_mlen(devinfo, imm_desc));
if (has_imm_ex_desc) {
format(file, " ex_mlen %u",
elk_message_ex_desc_ex_mlen(devinfo, imm_ex_desc));
}
if (has_imm_desc)
format(file, " rlen %u", elk_message_desc_rlen(devinfo, imm_desc));
if (has_imm_desc)
format(file, " mlen %u", elk_message_desc_mlen(devinfo, imm_desc));
if (has_imm_ex_desc) {
format(file, " ex_mlen %u",
elk_message_ex_desc_ex_mlen(devinfo, imm_ex_desc));
}
if (has_imm_desc)
format(file, " rlen %u", elk_message_desc_rlen(devinfo, imm_desc));
}
pad(file, 64);
if (opcode != ELK_OPCODE_NOP && opcode != ELK_OPCODE_NENOP) {
src/intel/compiler/elk/elk_eu_defines.h
-4
View File
@@ -530,10 +530,6 @@ enum elk_message_target {
GFX7_SFID_PIXEL_INTERPOLATOR = 11,
HSW_SFID_DATAPORT_DATA_CACHE_1 = 12,
HSW_SFID_CRE = 13,
GFX12_SFID_TGM = 13, /* Typed Global Memory */
GFX12_SFID_SLM = 14, /* Shared Local Memory */
GFX12_SFID_UGM = 15, /* Untyped Global Memory */
};
#define GFX7_MESSAGE_TARGET_DP_DATA_CACHE 10
src/intel/compiler/elk/elk_eu_emit.c
+1 -52
View File
@@ -2933,53 +2933,6 @@ elk_set_memory_fence_message(struct elk_codegen *p,
elk_inst_set_binding_table_index(devinfo, insn, bti);
}
static void
gfx12_set_memory_fence_message(struct elk_codegen *p,
struct elk_inst *insn,
enum elk_message_target sfid,
uint32_t desc)
{
const unsigned mlen = 1 * reg_unit(p->devinfo); /* g0 header */
/* Completion signaled by write to register. No data returned. */
const unsigned rlen = 1 * reg_unit(p->devinfo);
elk_inst_set_sfid(p->devinfo, insn, sfid);
if (sfid == ELK_SFID_URB && p->devinfo->ver < 20) {
elk_set_desc(p, insn, elk_urb_fence_desc(p->devinfo) |
elk_message_desc(p->devinfo, mlen, rlen, true));
} else {
enum lsc_fence_scope scope = lsc_fence_msg_desc_scope(p->devinfo, desc);
enum lsc_flush_type flush_type = lsc_fence_msg_desc_flush_type(p->devinfo, desc);
if (sfid == GFX12_SFID_TGM) {
scope = LSC_FENCE_TILE;
flush_type = LSC_FLUSH_TYPE_EVICT;
}
/* Wa_14012437816:
*
* "For any fence greater than local scope, always set flush type to
* at least invalidate so that fence goes on properly."
*
* "The bug is if flush_type is 'None', the scope is always downgraded
* to 'local'."
*
* Here set scope to NONE_6 instead of NONE, which has the same effect
* as NONE but avoids the downgrade to scope LOCAL.
*/
if (intel_needs_workaround(p->devinfo, 14012437816) &&
scope > LSC_FENCE_LOCAL &&
flush_type == LSC_FLUSH_TYPE_NONE) {
flush_type = LSC_FLUSH_TYPE_NONE_6;
}
elk_set_desc(p, insn, lsc_fence_msg_desc(p->devinfo, scope,
flush_type, false) |
elk_message_desc(p->devinfo, mlen, rlen, false));
}
}
void
elk_memory_fence(struct elk_codegen *p,
struct elk_reg dst,
@@ -3004,11 +2957,7 @@ elk_memory_fence(struct elk_codegen *p,
elk_set_dest(p, insn, dst);
elk_set_src0(p, insn, src);
/* All DG2 hardware requires LSC for fence messages, even A-step */
if (devinfo->has_lsc)
gfx12_set_memory_fence_message(p, insn, sfid, desc);
else
elk_set_memory_fence_message(p, insn, sfid, commit_enable, bti);
elk_set_memory_fence_message(p, insn, sfid, commit_enable, bti);
}
void
src/intel/compiler/elk/elk_eu_validate.c
-22
View File
@@ -2314,28 +2314,6 @@ send_descriptor_restrictions(const struct elk_isa_info *isa,
return error_msg;
}
const uint32_t desc = elk_inst_send_desc(devinfo, inst);
switch (elk_inst_sfid(devinfo, inst)) {
case ELK_SFID_URB:
if (devinfo->ver < 20)
break;
FALLTHROUGH;
case GFX12_SFID_TGM:
case GFX12_SFID_SLM:
case GFX12_SFID_UGM:
ERROR_IF(!devinfo->has_lsc, "Platform does not support LSC");
ERROR_IF(elk_lsc_opcode_has_transpose(lsc_msg_desc_opcode(devinfo, desc)) &&
lsc_msg_desc_transpose(devinfo, desc) &&
elk_inst_exec_size(devinfo, inst) != ELK_EXECUTE_1,
"Transposed vectors are restricted to Exec_Mask = 1.");
break;
default:
break;
}
if (elk_inst_sfid(devinfo, inst) == ELK_SFID_URB && devinfo->ver < 20) {
/* Gfx4 doesn't have a "header present" bit in the SEND message. */
ERROR_IF(devinfo->ver > 4 && !elk_inst_header_present(devinfo, inst),
src/intel/compiler/elk/elk_fs.cpp
-148
View File
@@ -1290,63 +1290,6 @@ elk_fs_visitor::assign_curb_setup()
*/
assert(devinfo->verx10 >= 125);
assert(uniform_push_length <= reg_unit(devinfo));
} else if (is_compute && devinfo->verx10 >= 125) {
assert(devinfo->has_lsc);
fs_builder ubld = fs_builder(this, 1).exec_all().at(
cfg->first_block(), cfg->first_block()->start());
/* The base offset for our push data is passed in as R0.0[31:6]. We have
* to mask off the bottom 6 bits.
*/
elk_fs_reg base_addr = ubld.vgrf(ELK_REGISTER_TYPE_UD);
ubld.AND(base_addr,
retype(elk_vec1_grf(0, 0), ELK_REGISTER_TYPE_UD),
elk_imm_ud(INTEL_MASK(31, 6)));
/* On Gfx12-HP we load constants at the start of the program using A32
* stateless messages.
*/
for (unsigned i = 0; i < uniform_push_length;) {
/* Limit ourselves to LSC HW limit of 8 GRFs (256bytes D32V64). */
unsigned num_regs = MIN2(uniform_push_length - i, 8);
assert(num_regs > 0);
num_regs = 1 << util_logbase2(num_regs);
elk_fs_reg addr = ubld.vgrf(ELK_REGISTER_TYPE_UD);
ubld.ADD(addr, base_addr, elk_imm_ud(i * REG_SIZE));
elk_fs_reg srcs[4] = {
elk_imm_ud(0), /* desc */
elk_imm_ud(0), /* ex_desc */
addr, /* payload */
elk_fs_reg(), /* payload2 */
};
elk_fs_reg dest = retype(elk_vec8_grf(payload().num_regs + i, 0),
ELK_REGISTER_TYPE_UD);
elk_fs_inst *send = ubld.emit(ELK_SHADER_OPCODE_SEND, dest, srcs, 4);
send->sfid = GFX12_SFID_UGM;
send->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD,
1 /* exec_size */,
LSC_ADDR_SURFTYPE_FLAT,
LSC_ADDR_SIZE_A32,
1 /* num_coordinates */,
LSC_DATA_SIZE_D32,
num_regs * 8 /* num_channels */,
true /* transpose */,
LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS),
true /* has_dest */);
send->header_size = 0;
send->mlen = lsc_msg_desc_src0_len(devinfo, send->desc);
send->size_written =
lsc_msg_desc_dest_len(devinfo, send->desc) * REG_SIZE;
send->send_is_volatile = true;
i += num_regs;
}
invalidate_analysis(DEPENDENCY_INSTRUCTIONS);
}
/* Map the offsets in the UNIFORM file to fixed HW regs. */
@@ -6009,41 +5952,6 @@ elk_fs_visitor::fixup_3src_null_dest()
DEPENDENCY_VARIABLES);
}
static bool
needs_dummy_fence(const intel_device_info *devinfo, elk_fs_inst *inst)
{
/* This workaround is about making sure that any instruction writing
* through UGM has completed before we hit EOT.
*/
if (inst->sfid != GFX12_SFID_UGM)
return false;
/* Any UGM, non-Scratch-surface Stores (not including Atomic) messages,
* where the L1-cache override is NOT among {WB, WS, WT}
*/
enum elk_lsc_opcode opcode = lsc_msg_desc_opcode(devinfo, inst->desc);
if (elk_lsc_opcode_is_store(opcode)) {
switch (lsc_msg_desc_cache_ctrl(devinfo, inst->desc)) {
case LSC_CACHE_STORE_L1STATE_L3MOCS:
case LSC_CACHE_STORE_L1WB_L3WB:
case LSC_CACHE_STORE_L1S_L3UC:
case LSC_CACHE_STORE_L1S_L3WB:
case LSC_CACHE_STORE_L1WT_L3UC:
case LSC_CACHE_STORE_L1WT_L3WB:
return false;
default:
return true;
}
}
/* Any UGM Atomic message WITHOUT return value */
if (elk_lsc_opcode_is_atomic(opcode) && inst->dst.file == BAD_FILE)
return true;
return false;
}
/* Wa_14015360517
*
* The first instruction of any kernel should have non-zero emask.
@@ -6073,56 +5981,6 @@ elk_fs_visitor::emit_dummy_mov_instruction()
invalidate_analysis(DEPENDENCY_INSTRUCTIONS | DEPENDENCY_VARIABLES);
}
/* Wa_22013689345
*
* We need to emit UGM fence message before EOT, if shader has any UGM write
* or atomic message.
*
* TODO/FINISHME: According to Curro we could avoid the fence in some cases.
* We probably need a better criteria in needs_dummy_fence().
*/
void
elk_fs_visitor::emit_dummy_memory_fence_before_eot()
{
bool progress = false;
bool has_ugm_write_or_atomic = false;
if (!intel_needs_workaround(devinfo, 22013689345))
return;
foreach_block_and_inst_safe (block, elk_fs_inst, inst, cfg) {
if (!inst->eot) {
if (needs_dummy_fence(devinfo, inst))
has_ugm_write_or_atomic = true;
continue;
}
if (!has_ugm_write_or_atomic)
break;
const fs_builder ibld(this, block, inst);
const fs_builder ubld = ibld.exec_all().group(1, 0);
elk_fs_reg dst = ubld.vgrf(ELK_REGISTER_TYPE_UD);
elk_fs_inst *dummy_fence = ubld.emit(ELK_SHADER_OPCODE_MEMORY_FENCE,
dst, elk_vec8_grf(0, 0),
/* commit enable */ elk_imm_ud(1),
/* bti */ elk_imm_ud(0));
dummy_fence->sfid = GFX12_SFID_UGM;
dummy_fence->desc = lsc_fence_msg_desc(devinfo, LSC_FENCE_TILE,
LSC_FLUSH_TYPE_NONE_6, false);
ubld.emit(ELK_FS_OPCODE_SCHEDULING_FENCE, ubld.null_reg_ud(), dst);
progress = true;
/* TODO: remove this break if we ever have shader with multiple EOT. */
break;
}
if (progress) {
invalidate_analysis(DEPENDENCY_INSTRUCTIONS |
DEPENDENCY_VARIABLES);
}
}
/**
* Find the first instruction in the program that might start a region of
* divergent control flow due to a HALT jump. There is no
@@ -6508,7 +6366,6 @@ elk_fs_visitor::run_vs()
assign_vs_urb_setup();
fixup_3src_null_dest();
emit_dummy_memory_fence_before_eot();
/* Wa_14015360517 */
emit_dummy_mov_instruction();
@@ -6638,7 +6495,6 @@ elk_fs_visitor::run_tcs()
assign_tcs_urb_setup();
fixup_3src_null_dest();
emit_dummy_memory_fence_before_eot();
/* Wa_14015360517 */
emit_dummy_mov_instruction();
@@ -6670,7 +6526,6 @@ elk_fs_visitor::run_tes()
assign_tes_urb_setup();
fixup_3src_null_dest();
emit_dummy_memory_fence_before_eot();
/* Wa_14015360517 */
emit_dummy_mov_instruction();
@@ -6719,7 +6574,6 @@ elk_fs_visitor::run_gs()
assign_gs_urb_setup();
fixup_3src_null_dest();
emit_dummy_memory_fence_before_eot();
/* Wa_14015360517 */
emit_dummy_mov_instruction();
@@ -6827,7 +6681,6 @@ elk_fs_visitor::run_fs(bool allow_spilling, bool do_rep_send)
assign_urb_setup();
fixup_3src_null_dest();
emit_dummy_memory_fence_before_eot();
/* Wa_14015360517 */
emit_dummy_mov_instruction();
@@ -6868,7 +6721,6 @@ elk_fs_visitor::run_cs(bool allow_spilling)
assign_curb_setup();
fixup_3src_null_dest();
emit_dummy_memory_fence_before_eot();
/* Wa_14015360517 */
emit_dummy_mov_instruction();
src/intel/compiler/elk/elk_fs.h
-1
View File
@@ -213,7 +213,6 @@ public:
void allocate_registers(bool allow_spilling);
uint32_t compute_max_register_pressure();
void fixup_3src_null_dest();
void emit_dummy_memory_fence_before_eot();
void emit_dummy_mov_instruction();
bool fixup_nomask_control_flow();
void assign_curb_setup();
src/intel/compiler/elk/elk_fs_nir.cpp
+1 -67
View File
@@ -4889,39 +4889,6 @@ emit_fence(const fs_builder &bld, enum elk_opcode opcode,
return dst;
}
static uint32_t
lsc_fence_descriptor_for_intrinsic(const struct intel_device_info *devinfo,
nir_intrinsic_instr *instr)
{
assert(devinfo->has_lsc);
enum lsc_fence_scope scope = LSC_FENCE_LOCAL;
enum lsc_flush_type flush_type = LSC_FLUSH_TYPE_NONE;
if (nir_intrinsic_has_memory_scope(instr)) {
switch (nir_intrinsic_memory_scope(instr)) {
case SCOPE_DEVICE:
case SCOPE_QUEUE_FAMILY:
scope = LSC_FENCE_TILE;
flush_type = LSC_FLUSH_TYPE_EVICT;
break;
case SCOPE_WORKGROUP:
scope = LSC_FENCE_THREADGROUP;
break;
case SCOPE_SHADER_CALL:
case SCOPE_INVOCATION:
case SCOPE_SUBGROUP:
case SCOPE_NONE:
break;
}
} else {
/* No scope defined. */
scope = LSC_FENCE_TILE;
flush_type = LSC_FLUSH_TYPE_EVICT;
}
return lsc_fence_msg_desc(devinfo, scope, flush_type, true);
}
/**
* Create a MOV to read the timestamp register.
*/
@@ -5244,40 +5211,7 @@ fs_nir_emit_intrinsic(nir_to_elk_state &ntb,
const fs_builder ubld = bld.group(8, 0);
if (devinfo->has_lsc) {
assert(devinfo->verx10 >= 125);
uint32_t desc =
lsc_fence_descriptor_for_intrinsic(devinfo, instr);
if (ugm_fence) {
fence_regs[fence_regs_count++] =
emit_fence(ubld, opcode, GFX12_SFID_UGM, desc,
true /* commit_enable */,
0 /* bti; ignored for LSC */);
}
if (tgm_fence) {
fence_regs[fence_regs_count++] =
emit_fence(ubld, opcode, GFX12_SFID_TGM, desc,
true /* commit_enable */,
0 /* bti; ignored for LSC */);
}
if (slm_fence) {
assert(opcode == ELK_SHADER_OPCODE_MEMORY_FENCE);
fence_regs[fence_regs_count++] =
emit_fence(ubld, opcode, GFX12_SFID_SLM, desc,
true /* commit_enable */,
0 /* BTI; ignored for LSC */);
}
if (urb_fence) {
assert(opcode == ELK_SHADER_OPCODE_MEMORY_FENCE);
fence_regs[fence_regs_count++] =
emit_fence(ubld, opcode, ELK_SFID_URB, desc,
true /* commit_enable */,
0 /* BTI; ignored for LSC */);
}
} else if (devinfo->ver >= 11) {
if (devinfo->ver >= 11) {
if (tgm_fence || ugm_fence || urb_fence) {
fence_regs[fence_regs_count++] =
emit_fence(ubld, opcode, GFX7_SFID_DATAPORT_DATA_CACHE, 0,
src/intel/compiler/elk/elk_fs_reg_allocate.cpp
+2 -81
View File
@@ -827,53 +827,7 @@ elk_fs_reg_alloc::emit_unspill(const fs_builder &bld,
++stats->fill_count;
elk_fs_inst *unspill_inst;
if (devinfo->verx10 >= 125) {
/* LSC is limited to SIMD16 load/store but we can load more using
* transpose messages.
*/
const bool use_transpose = bld.dispatch_width() > 16;
const fs_builder ubld = use_transpose ? bld.exec_all().group(1, 0) : bld;
elk_fs_reg offset;
if (use_transpose) {
offset = build_single_offset(ubld, spill_offset, ip);
} else {
offset = build_lane_offsets(ubld, spill_offset, ip);
}
/* We leave the extended descriptor empty and flag the instruction to
* ask the generated to insert the extended descriptor in the address
* register. That way we don't need to burn an additional register
* for register allocation spill/fill.
*/
elk_fs_reg srcs[] = {
elk_imm_ud(0), /* desc */
elk_imm_ud(0), /* ex_desc */
offset, /* payload */
elk_fs_reg(), /* payload2 */
};
unspill_inst = ubld.emit(ELK_SHADER_OPCODE_SEND, dst,
srcs, ARRAY_SIZE(srcs));
unspill_inst->sfid = GFX12_SFID_UGM;
unspill_inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD,
unspill_inst->exec_size,
LSC_ADDR_SURFTYPE_SS,
LSC_ADDR_SIZE_A32,
1 /* num_coordinates */,
LSC_DATA_SIZE_D32,
use_transpose ? reg_size * 8 : 1 /* num_channels */,
use_transpose,
LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS),
true /* has_dest */);
unspill_inst->header_size = 0;
unspill_inst->mlen =
lsc_msg_desc_src0_len(devinfo, unspill_inst->desc);
unspill_inst->ex_mlen = 0;
unspill_inst->size_written =
lsc_msg_desc_dest_len(devinfo, unspill_inst->desc) * REG_SIZE;
unspill_inst->send_has_side_effects = false;
unspill_inst->send_is_volatile = true;
unspill_inst->send_ex_desc_scratch = true;
} else if (devinfo->ver >= 9) {
if (devinfo->ver >= 9) {
elk_fs_reg header = this->scratch_header;
fs_builder ubld = bld.exec_all().group(1, 0);
assert(spill_offset % 16 == 0);
@@ -935,40 +889,7 @@ elk_fs_reg_alloc::emit_spill(const fs_builder &bld,
++stats->spill_count;
elk_fs_inst *spill_inst;
if (devinfo->verx10 >= 125) {
elk_fs_reg offset = build_lane_offsets(bld, spill_offset, ip);
/* We leave the extended descriptor empty and flag the instruction
* relocate the extended descriptor. That way the surface offset is
* directly put into the instruction and we don't need to use a
* register to hold it.
*/
elk_fs_reg srcs[] = {
elk_imm_ud(0), /* desc */
elk_imm_ud(0), /* ex_desc */
offset, /* payload */
src, /* payload2 */
};
spill_inst = bld.emit(ELK_SHADER_OPCODE_SEND, bld.null_reg_f(),
srcs, ARRAY_SIZE(srcs));
spill_inst->sfid = GFX12_SFID_UGM;
spill_inst->desc = lsc_msg_desc(devinfo, LSC_OP_STORE,
bld.dispatch_width(),
LSC_ADDR_SURFTYPE_SS,
LSC_ADDR_SIZE_A32,
1 /* num_coordinates */,
LSC_DATA_SIZE_D32,
1 /* num_channels */,
false /* transpose */,
LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS),
false /* has_dest */);
spill_inst->header_size = 0;
spill_inst->mlen = lsc_msg_desc_src0_len(devinfo, spill_inst->desc);
spill_inst->ex_mlen = reg_size;
spill_inst->size_written = 0;
spill_inst->send_has_side_effects = true;
spill_inst->send_is_volatile = false;
spill_inst->send_ex_desc_scratch = true;
} else if (devinfo->ver >= 9) {
if (devinfo->ver >= 9) {
elk_fs_reg header = this->scratch_header;
fs_builder ubld = bld.exec_all().group(1, 0);
assert(spill_offset % 16 == 0);
src/intel/compiler/elk/elk_gram.y
-3
View File
@@ -1029,9 +1029,6 @@ sharedfunction:
| CRE { $$ = HSW_SFID_CRE; }
| SAMPLER { $$ = ELK_SFID_SAMPLER; }
| DP_SAMPLER { $$ = GFX6_SFID_DATAPORT_SAMPLER_CACHE; }
| SLM { $$ = GFX12_SFID_SLM; }
| TGM { $$ = GFX12_SFID_TGM; }
| UGM { $$ = GFX12_SFID_UGM; }
;
exp2:
src/intel/compiler/elk/elk_ir_performance.cpp
-44
View File
@@ -935,9 +935,6 @@ namespace {
case ELK_SFID_URB:
case GFX7_SFID_DATAPORT_DATA_CACHE:
case GFX12_SFID_SLM:
case GFX12_SFID_TGM:
case GFX12_SFID_UGM:
case HSW_SFID_DATAPORT_DATA_CACHE_1:
if (devinfo->ver >= 7)
return calculate_desc(info, EU_UNIT_DP_DC, 2, 0, 0, 30 /* XXX */, 0,
@@ -1103,47 +1100,6 @@ namespace {
else
abort();
case GFX12_SFID_UGM:
case GFX12_SFID_TGM:
case GFX12_SFID_SLM:
switch (lsc_msg_desc_opcode(devinfo, info.desc)) {
case LSC_OP_LOAD:
case LSC_OP_STORE:
case LSC_OP_LOAD_CMASK:
case LSC_OP_STORE_CMASK:
return calculate_desc(info, EU_UNIT_DP_DC, 2, 0, 0,
0, 20 /* XXX */,
10 /* XXX */, 100 /* XXX */, 0, 0,
0, 0);
case LSC_OP_FENCE:
case LSC_OP_ATOMIC_INC:
case LSC_OP_ATOMIC_DEC:
case LSC_OP_ATOMIC_LOAD:
case LSC_OP_ATOMIC_STORE:
case LSC_OP_ATOMIC_ADD:
case LSC_OP_ATOMIC_SUB:
case LSC_OP_ATOMIC_MIN:
case LSC_OP_ATOMIC_MAX:
case LSC_OP_ATOMIC_UMIN:
case LSC_OP_ATOMIC_UMAX:
case LSC_OP_ATOMIC_CMPXCHG:
case LSC_OP_ATOMIC_FADD:
case LSC_OP_ATOMIC_FSUB:
case LSC_OP_ATOMIC_FMIN:
case LSC_OP_ATOMIC_FMAX:
case LSC_OP_ATOMIC_FCMPXCHG:
case LSC_OP_ATOMIC_AND:
case LSC_OP_ATOMIC_OR:
case LSC_OP_ATOMIC_XOR:
return calculate_desc(info, EU_UNIT_DP_DC, 2, 0, 0,
30 /* XXX */, 400 /* XXX */,
10 /* XXX */, 100 /* XXX */, 0, 0,
0, 400 /* XXX */);
default:
abort();
}
case ELK_SFID_URB:
return calculate_desc(info, EU_UNIT_URB, 2, 0, 0, 0, 6 /* XXX */,
32 /* XXX */, 200 /* XXX */, 0, 0, 0, 0);
src/intel/compiler/elk/elk_lower_logical_sends.cpp
+2 -579
View File
@@ -1611,49 +1611,6 @@ setup_surface_descriptors(const fs_builder &bld, elk_fs_inst *inst, uint32_t des
}
}
static void
setup_lsc_surface_descriptors(const fs_builder &bld, elk_fs_inst *inst,
uint32_t desc, const elk_fs_reg &surface)
{
const ASSERTED intel_device_info *devinfo = bld.shader->devinfo;
const elk_compiler *compiler = bld.shader->compiler;
inst->src[0] = elk_imm_ud(0); /* desc */
enum lsc_addr_surface_type surf_type = lsc_msg_desc_addr_type(devinfo, desc);
switch (surf_type) {
case LSC_ADDR_SURFTYPE_BSS:
inst->send_ex_bso = compiler->extended_bindless_surface_offset;
/* fall-through */
case LSC_ADDR_SURFTYPE_SS:
assert(surface.file != BAD_FILE);
/* We assume that the driver provided the handle in the top 20 bits so
* we can use the surface handle directly as the extended descriptor.
*/
inst->src[1] = retype(surface, ELK_REGISTER_TYPE_UD);
break;
case LSC_ADDR_SURFTYPE_BTI:
assert(surface.file != BAD_FILE);
if (surface.file == IMM) {
inst->src[1] = elk_imm_ud(lsc_bti_ex_desc(devinfo, surface.ud));
} else {
const fs_builder ubld = bld.exec_all().group(1, 0);
elk_fs_reg tmp = ubld.vgrf(ELK_REGISTER_TYPE_UD);
ubld.SHL(tmp, surface, elk_imm_ud(24));
inst->src[1] = component(tmp, 0);
}
break;
case LSC_ADDR_SURFTYPE_FLAT:
inst->src[1] = elk_imm_ud(0);
break;
default:
unreachable("Invalid LSC surface address type");
}
}
static void
lower_surface_logical_send(const fs_builder &bld, elk_fs_inst *inst)
{
@@ -1903,262 +1860,6 @@ lower_surface_logical_send(const fs_builder &bld, elk_fs_inst *inst)
inst->src[3] = payload2;
}
static enum lsc_data_size
lsc_bits_to_data_size(unsigned bit_size)
{
switch (bit_size / 8) {
case 1: return LSC_DATA_SIZE_D8U32;
case 2: return LSC_DATA_SIZE_D16U32;
case 4: return LSC_DATA_SIZE_D32;
case 8: return LSC_DATA_SIZE_D64;
default:
unreachable("Unsupported data size.");
}
}
static void
lower_lsc_surface_logical_send(const fs_builder &bld, elk_fs_inst *inst)
{
const elk_compiler *compiler = bld.shader->compiler;
const intel_device_info *devinfo = bld.shader->devinfo;
assert(devinfo->has_lsc);
/* Get the logical send arguments. */
const elk_fs_reg addr = inst->src[SURFACE_LOGICAL_SRC_ADDRESS];
const elk_fs_reg src = inst->src[SURFACE_LOGICAL_SRC_DATA];
const elk_fs_reg surface = inst->src[SURFACE_LOGICAL_SRC_SURFACE];
const elk_fs_reg surface_handle = inst->src[SURFACE_LOGICAL_SRC_SURFACE_HANDLE];
const UNUSED elk_fs_reg dims = inst->src[SURFACE_LOGICAL_SRC_IMM_DIMS];
const elk_fs_reg arg = inst->src[SURFACE_LOGICAL_SRC_IMM_ARG];
const elk_fs_reg allow_sample_mask =
inst->src[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK];
assert(arg.file == IMM);
assert(allow_sample_mask.file == IMM);
/* Calculate the total number of components of the payload. */
const unsigned addr_sz = inst->components_read(SURFACE_LOGICAL_SRC_ADDRESS);
const unsigned src_comps = inst->components_read(SURFACE_LOGICAL_SRC_DATA);
const unsigned src_sz = type_sz(src.type);
const unsigned dst_sz = type_sz(inst->dst.type);
const bool has_side_effects = inst->has_side_effects();
unsigned ex_mlen = 0;
elk_fs_reg payload, payload2;
payload = bld.move_to_vgrf(addr, addr_sz);
if (src.file != BAD_FILE) {
payload2 = bld.move_to_vgrf(src, src_comps);
ex_mlen = (src_comps * src_sz * inst->exec_size) / REG_SIZE;
}
/* Predicate the instruction on the sample mask if needed */
elk_fs_reg sample_mask = allow_sample_mask.ud ? elk_sample_mask_reg(bld) :
elk_fs_reg(elk_imm_ud(0xffffffff));
if (sample_mask.file != BAD_FILE && sample_mask.file != IMM)
elk_emit_predicate_on_sample_mask(bld, inst);
if (surface.file == IMM && surface.ud == GFX7_BTI_SLM)
inst->sfid = GFX12_SFID_SLM;
else
inst->sfid = GFX12_SFID_UGM;
/* We should have exactly one of surface and surface_handle. For scratch
* messages generated by elk_fs_nir.cpp we also allow a special value to
* know what heap base we should use in STATE_BASE_ADDRESS (SS = Surface
* State Offset, or BSS = Bindless Surface State Offset).
*/
bool non_bindless = surface.file == IMM && surface.ud == GFX125_NON_BINDLESS;
assert((surface.file == BAD_FILE) != (surface_handle.file == BAD_FILE) ||
(non_bindless && surface_handle.file != BAD_FILE));
enum lsc_addr_surface_type surf_type;
if (surface_handle.file != BAD_FILE) {
if (surface.file == BAD_FILE) {
assert(!non_bindless);
surf_type = LSC_ADDR_SURFTYPE_BSS;
} else {
assert(surface.file == IMM &&
(surface.ud == 0 || surface.ud == GFX125_NON_BINDLESS));
surf_type = non_bindless ? LSC_ADDR_SURFTYPE_SS : LSC_ADDR_SURFTYPE_BSS;
}
} else if (surface.file == IMM && surface.ud == GFX7_BTI_SLM)
surf_type = LSC_ADDR_SURFTYPE_FLAT;
else
surf_type = LSC_ADDR_SURFTYPE_BTI;
switch (inst->opcode) {
case ELK_SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL:
inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD_CMASK, inst->exec_size,
surf_type, LSC_ADDR_SIZE_A32,
1 /* num_coordinates */,
LSC_DATA_SIZE_D32, arg.ud /* num_channels */,
false /* transpose */,
LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS),
true /* has_dest */);
break;
case ELK_SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL:
inst->desc = lsc_msg_desc(devinfo, LSC_OP_STORE_CMASK, inst->exec_size,
surf_type, LSC_ADDR_SIZE_A32,
1 /* num_coordinates */,
LSC_DATA_SIZE_D32, arg.ud /* num_channels */,
false /* transpose */,
LSC_CACHE(devinfo, STORE, L1STATE_L3MOCS),
false /* has_dest */);
break;
case ELK_SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL: {
/* Bspec: Atomic instruction -> Cache section:
*
* Atomic messages are always forced to "un-cacheable" in the L1
* cache.
*/
enum elk_lsc_opcode opcode = (enum elk_lsc_opcode) arg.ud;
inst->desc = lsc_msg_desc(devinfo, opcode, inst->exec_size,
surf_type, LSC_ADDR_SIZE_A32,
1 /* num_coordinates */,
lsc_bits_to_data_size(dst_sz * 8),
1 /* num_channels */,
false /* transpose */,
LSC_CACHE(devinfo, STORE, L1UC_L3WB),
!inst->dst.is_null());
break;
}
case ELK_SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL:
inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD, inst->exec_size,
surf_type, LSC_ADDR_SIZE_A32,
1 /* num_coordinates */,
lsc_bits_to_data_size(arg.ud),
1 /* num_channels */,
false /* transpose */,
LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS),
true /* has_dest */);
break;
case ELK_SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL:
inst->desc = lsc_msg_desc(devinfo, LSC_OP_STORE, inst->exec_size,
surf_type, LSC_ADDR_SIZE_A32,
1 /* num_coordinates */,
lsc_bits_to_data_size(arg.ud),
1 /* num_channels */,
false /* transpose */,
LSC_CACHE(devinfo, STORE, L1STATE_L3MOCS),
false /* has_dest */);
break;
default:
unreachable("Unknown surface logical instruction");
}
/* Update the original instruction. */
inst->opcode = ELK_SHADER_OPCODE_SEND;
inst->mlen = lsc_msg_desc_src0_len(devinfo, inst->desc);
inst->ex_mlen = ex_mlen;
inst->header_size = 0;
inst->send_has_side_effects = has_side_effects;
inst->send_is_volatile = !has_side_effects;
inst->send_ex_bso = surf_type == LSC_ADDR_SURFTYPE_BSS &&
compiler->extended_bindless_surface_offset;
inst->resize_sources(4);
if (non_bindless) {
inst->src[0] = elk_imm_ud(0); /* desc */
inst->src[1] = surface_handle; /* ex_desc */
} else {
setup_lsc_surface_descriptors(bld, inst, inst->desc,
surface.file != BAD_FILE ?
surface : surface_handle);
}
/* Finally, the payload */
inst->src[2] = payload;
inst->src[3] = payload2;
}
static void
lower_lsc_block_logical_send(const fs_builder &bld, elk_fs_inst *inst)
{
const elk_compiler *compiler = bld.shader->compiler;
const intel_device_info *devinfo = bld.shader->devinfo;
assert(devinfo->has_lsc);
/* Get the logical send arguments. */
const elk_fs_reg addr = inst->src[SURFACE_LOGICAL_SRC_ADDRESS];
const elk_fs_reg src = inst->src[SURFACE_LOGICAL_SRC_DATA];
const elk_fs_reg surface = inst->src[SURFACE_LOGICAL_SRC_SURFACE];
const elk_fs_reg surface_handle = inst->src[SURFACE_LOGICAL_SRC_SURFACE_HANDLE];
const elk_fs_reg arg = inst->src[SURFACE_LOGICAL_SRC_IMM_ARG];
assert(arg.file == IMM);
assert(inst->src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == BAD_FILE);
assert(inst->src[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK].file == BAD_FILE);
const bool is_stateless =
surface.file == IMM && (surface.ud == ELK_BTI_STATELESS ||
surface.ud == GFX8_BTI_STATELESS_NON_COHERENT);
const bool has_side_effects = inst->has_side_effects();
const bool write = inst->opcode == ELK_SHADER_OPCODE_OWORD_BLOCK_WRITE_LOGICAL;
fs_builder ubld = bld.exec_all().group(1, 0);
elk_fs_reg stateless_ex_desc;
if (is_stateless) {
stateless_ex_desc = ubld.vgrf(ELK_REGISTER_TYPE_UD);
ubld.AND(stateless_ex_desc,
retype(elk_vec1_grf(0, 5), ELK_REGISTER_TYPE_UD),
elk_imm_ud(INTEL_MASK(31, 10)));
}
elk_fs_reg data;
if (write) {
const unsigned src_sz = inst->components_read(SURFACE_LOGICAL_SRC_DATA);
data = retype(bld.move_to_vgrf(src, src_sz), ELK_REGISTER_TYPE_UD);
}
inst->opcode = ELK_SHADER_OPCODE_SEND;
if (surface.file == IMM && surface.ud == GFX7_BTI_SLM)
inst->sfid = GFX12_SFID_SLM;
else
inst->sfid = GFX12_SFID_UGM;
const enum lsc_addr_surface_type surf_type =
inst->sfid == GFX12_SFID_SLM ?
LSC_ADDR_SURFTYPE_FLAT :
surface.file == BAD_FILE ?
LSC_ADDR_SURFTYPE_BSS : LSC_ADDR_SURFTYPE_BTI;
inst->desc = lsc_msg_desc(devinfo,
write ? LSC_OP_STORE : LSC_OP_LOAD,
1 /* exec_size */,
surf_type,
LSC_ADDR_SIZE_A32,
1 /* num_coordinates */,
LSC_DATA_SIZE_D32,
arg.ud /* num_channels */,
true /* transpose */,
LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS),
!write /* has_dest */);
inst->mlen = lsc_msg_desc_src0_len(devinfo, inst->desc);
inst->size_written = lsc_msg_desc_dest_len(devinfo, inst->desc) * REG_SIZE;
inst->exec_size = 1;
inst->ex_mlen = write ? DIV_ROUND_UP(arg.ud, 8) : 0;
inst->header_size = 0;
inst->send_has_side_effects = has_side_effects;
inst->send_is_volatile = !has_side_effects;
inst->send_ex_bso = surf_type == LSC_ADDR_SURFTYPE_BSS &&
compiler->extended_bindless_surface_offset;
inst->resize_sources(4);
if (stateless_ex_desc.file != BAD_FILE) {
inst->src[0] = elk_imm_ud(0); /* desc */
inst->src[1] = stateless_ex_desc; /* ex_desc */
} else {
setup_lsc_surface_descriptors(bld, inst, inst->desc,
surface.file != BAD_FILE ?
surface : surface_handle);
}
inst->src[2] = addr; /* payload */
inst->src[3] = data; /* payload2 */
}
static void
lower_surface_block_logical_send(const fs_builder &bld, elk_fs_inst *inst)
{
@@ -2275,137 +1976,6 @@ emit_fragment_mask(const fs_builder &bld, elk_fs_inst *inst)
elk_emit_predicate_on_sample_mask(bld, inst);
}
static void
lower_lsc_a64_logical_send(const fs_builder &bld, elk_fs_inst *inst)
{
const intel_device_info *devinfo = bld.shader->devinfo;
/* Get the logical send arguments. */
const elk_fs_reg addr = inst->src[A64_LOGICAL_ADDRESS];
const elk_fs_reg src = inst->src[A64_LOGICAL_SRC];
const unsigned src_sz = type_sz(src.type);
const unsigned dst_sz = type_sz(inst->dst.type);
const unsigned src_comps = inst->components_read(1);
assert(inst->src[A64_LOGICAL_ARG].file == IMM);
const unsigned arg = inst->src[A64_LOGICAL_ARG].ud;
const bool has_side_effects = inst->has_side_effects();
elk_fs_reg payload = retype(bld.move_to_vgrf(addr, 1), ELK_REGISTER_TYPE_UD);
elk_fs_reg payload2 = retype(bld.move_to_vgrf(src, src_comps),
ELK_REGISTER_TYPE_UD);
unsigned ex_mlen = src_comps * src_sz * inst->exec_size / REG_SIZE;
switch (inst->opcode) {
case ELK_SHADER_OPCODE_A64_UNTYPED_READ_LOGICAL:
inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD_CMASK, inst->exec_size,
LSC_ADDR_SURFTYPE_FLAT, LSC_ADDR_SIZE_A64,
1 /* num_coordinates */,
LSC_DATA_SIZE_D32, arg /* num_channels */,
false /* transpose */,
LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS),
true /* has_dest */);
break;
case ELK_SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL:
inst->desc = lsc_msg_desc(devinfo, LSC_OP_STORE_CMASK, inst->exec_size,
LSC_ADDR_SURFTYPE_FLAT, LSC_ADDR_SIZE_A64,
1 /* num_coordinates */,
LSC_DATA_SIZE_D32, arg /* num_channels */,
false /* transpose */,
LSC_CACHE(devinfo, STORE, L1STATE_L3MOCS),
false /* has_dest */);
break;
case ELK_SHADER_OPCODE_A64_BYTE_SCATTERED_READ_LOGICAL:
inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD, inst->exec_size,
LSC_ADDR_SURFTYPE_FLAT, LSC_ADDR_SIZE_A64,
1 /* num_coordinates */,
lsc_bits_to_data_size(arg),
1 /* num_channels */,
false /* transpose */,
LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS),
true /* has_dest */);
break;
case ELK_SHADER_OPCODE_A64_BYTE_SCATTERED_WRITE_LOGICAL:
inst->desc = lsc_msg_desc(devinfo, LSC_OP_STORE, inst->exec_size,
LSC_ADDR_SURFTYPE_FLAT, LSC_ADDR_SIZE_A64,
1 /* num_coordinates */,
lsc_bits_to_data_size(arg),
1 /* num_channels */,
false /* transpose */,
LSC_CACHE(devinfo, STORE, L1STATE_L3MOCS),
false /* has_dest */);
break;
case ELK_SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL: {
/* Bspec: Atomic instruction -> Cache section:
*
* Atomic messages are always forced to "un-cacheable" in the L1
* cache.
*/
enum elk_lsc_opcode opcode = (enum elk_lsc_opcode) arg;
inst->desc = lsc_msg_desc(devinfo, opcode, inst->exec_size,
LSC_ADDR_SURFTYPE_FLAT, LSC_ADDR_SIZE_A64,
1 /* num_coordinates */,
lsc_bits_to_data_size(dst_sz * 8),
1 /* num_channels */,
false /* transpose */,
LSC_CACHE(devinfo, STORE, L1UC_L3WB),
!inst->dst.is_null());
break;
}
case ELK_SHADER_OPCODE_A64_OWORD_BLOCK_READ_LOGICAL:
case ELK_SHADER_OPCODE_A64_UNALIGNED_OWORD_BLOCK_READ_LOGICAL:
inst->exec_size = 1;
inst->desc = lsc_msg_desc(devinfo,
LSC_OP_LOAD,
1 /* exec_size */,
LSC_ADDR_SURFTYPE_FLAT,
LSC_ADDR_SIZE_A64,
1 /* num_coordinates */,
LSC_DATA_SIZE_D32,
arg /* num_channels */,
true /* transpose */,
LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS),
true /* has_dest */);
break;
case ELK_SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL:
inst->exec_size = 1;
inst->desc = lsc_msg_desc(devinfo,
LSC_OP_STORE,
1 /* exec_size */,
LSC_ADDR_SURFTYPE_FLAT,
LSC_ADDR_SIZE_A64,
1 /* num_coordinates */,
LSC_DATA_SIZE_D32,
arg /* num_channels */,
true /* transpose */,
LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS),
false /* has_dest */);
break;
default:
unreachable("Unknown A64 logical instruction");
}
if (bld.shader->stage == MESA_SHADER_FRAGMENT)
emit_fragment_mask(bld, inst);
/* Update the original instruction. */
inst->opcode = ELK_SHADER_OPCODE_SEND;
inst->mlen = lsc_msg_desc_src0_len(devinfo, inst->desc);
inst->ex_mlen = ex_mlen;
inst->header_size = 0;
inst->send_has_side_effects = has_side_effects;
inst->send_is_volatile = !has_side_effects;
/* Set up SFID and descriptors */
inst->sfid = GFX12_SFID_UGM;
inst->resize_sources(4);
inst->src[0] = elk_imm_ud(0); /* desc */
inst->src[1] = elk_imm_ud(0); /* ex_desc */
inst->src[2] = payload;
inst->src[3] = payload2;
}
static void
lower_a64_logical_send(const fs_builder &bld, elk_fs_inst *inst)
{
@@ -2545,97 +2115,6 @@ lower_a64_logical_send(const fs_builder &bld, elk_fs_inst *inst)
inst->src[3] = payload2;
}
static void
lower_lsc_varying_pull_constant_logical_send(const fs_builder &bld,
elk_fs_inst *inst)
{
const intel_device_info *devinfo = bld.shader->devinfo;
ASSERTED const elk_compiler *compiler = bld.shader->compiler;
elk_fs_reg surface = inst->src[PULL_VARYING_CONSTANT_SRC_SURFACE];
elk_fs_reg surface_handle = inst->src[PULL_VARYING_CONSTANT_SRC_SURFACE_HANDLE];
elk_fs_reg offset_B = inst->src[PULL_VARYING_CONSTANT_SRC_OFFSET];
elk_fs_reg alignment_B = inst->src[PULL_VARYING_CONSTANT_SRC_ALIGNMENT];
/* We are switching the instruction from an ALU-like instruction to a
* send-from-grf instruction. Since sends can't handle strides or
* source modifiers, we have to make a copy of the offset source.
*/
elk_fs_reg ubo_offset = bld.move_to_vgrf(offset_B, 1);
enum lsc_addr_surface_type surf_type =
surface_handle.file == BAD_FILE ?
LSC_ADDR_SURFTYPE_BTI : LSC_ADDR_SURFTYPE_BSS;
assert(alignment_B.file == ELK_IMMEDIATE_VALUE);
unsigned alignment = alignment_B.ud;
inst->opcode = ELK_SHADER_OPCODE_SEND;
inst->sfid = GFX12_SFID_UGM;
inst->resize_sources(3);
inst->send_ex_bso = surf_type == LSC_ADDR_SURFTYPE_BSS &&
compiler->extended_bindless_surface_offset;
assert(!compiler->indirect_ubos_use_sampler);
inst->src[0] = elk_imm_ud(0);
inst->src[2] = ubo_offset; /* payload */
if (alignment >= 4) {
inst->desc =
lsc_msg_desc(devinfo, LSC_OP_LOAD_CMASK, inst->exec_size,
surf_type, LSC_ADDR_SIZE_A32,
1 /* num_coordinates */,
LSC_DATA_SIZE_D32,
4 /* num_channels */,
false /* transpose */,
LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS),
true /* has_dest */);
inst->mlen = lsc_msg_desc_src0_len(devinfo, inst->desc);
setup_lsc_surface_descriptors(bld, inst, inst->desc,
surface.file != BAD_FILE ?
surface : surface_handle);
} else {
inst->desc =
lsc_msg_desc(devinfo, LSC_OP_LOAD, inst->exec_size,
surf_type, LSC_ADDR_SIZE_A32,
1 /* num_coordinates */,
LSC_DATA_SIZE_D32,
1 /* num_channels */,
false /* transpose */,
LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS),
true /* has_dest */);
inst->mlen = lsc_msg_desc_src0_len(devinfo, inst->desc);
setup_lsc_surface_descriptors(bld, inst, inst->desc,
surface.file != BAD_FILE ?
surface : surface_handle);
/* The byte scattered messages can only read one dword at a time so
* we have to duplicate the message 4 times to read the full vec4.
* Hopefully, dead code will clean up the mess if some of them aren't
* needed.
*/
assert(inst->size_written == 16 * inst->exec_size);
inst->size_written /= 4;
for (unsigned c = 1; c < 4; c++) {
/* Emit a copy of the instruction because we're about to modify
* it. Because this loop starts at 1, we will emit copies for the
* first 3 and the final one will be the modified instruction.
*/
bld.emit(*inst);
/* Offset the source */
inst->src[2] = bld.vgrf(ELK_REGISTER_TYPE_UD);
bld.ADD(inst->src[2], ubo_offset, elk_imm_ud(c * 4));
/* Offset the destination */
inst->dst = offset(inst->dst, bld, 1);
}
}
}
static void
lower_varying_pull_constant_logical_send(const fs_builder &bld, elk_fs_inst *inst)
{
@@ -3018,10 +2497,6 @@ elk_fs_visitor::lower_logical_sends()
case ELK_SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL:
case ELK_SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL:
case ELK_SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL:
if (devinfo->has_lsc) {
lower_lsc_surface_logical_send(ibld, inst);
break;
}
case ELK_SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL:
case ELK_SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL:
case ELK_SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL:
@@ -3032,10 +2507,6 @@ elk_fs_visitor::lower_logical_sends()
case ELK_SHADER_OPCODE_UNALIGNED_OWORD_BLOCK_READ_LOGICAL:
case ELK_SHADER_OPCODE_OWORD_BLOCK_WRITE_LOGICAL:
if (devinfo->has_lsc) {
lower_lsc_block_logical_send(ibld, inst);
break;
}
lower_surface_block_logical_send(ibld, inst);
break;
@@ -3047,18 +2518,11 @@ elk_fs_visitor::lower_logical_sends()
case ELK_SHADER_OPCODE_A64_OWORD_BLOCK_READ_LOGICAL:
case ELK_SHADER_OPCODE_A64_UNALIGNED_OWORD_BLOCK_READ_LOGICAL:
case ELK_SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL:
if (devinfo->has_lsc) {
lower_lsc_a64_logical_send(ibld, inst);
break;
}
lower_a64_logical_send(ibld, inst);
break;
case ELK_FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_LOGICAL:
if (devinfo->has_lsc && !compiler->indirect_ubos_use_sampler)
lower_lsc_varying_pull_constant_logical_send(ibld, inst);
else
lower_varying_pull_constant_logical_send(ibld, inst);
lower_varying_pull_constant_logical_send(ibld, inst);
break;
case ELK_SHADER_OPCODE_RCP:
@@ -3155,48 +2619,7 @@ elk_fs_visitor::lower_uniform_pull_constant_loads()
assert(offset_B.file == IMM);
assert(size_B.file == IMM);
if (devinfo->has_lsc) {
const fs_builder ubld =
fs_builder(this, block, inst).group(8, 0).exec_all();
const elk_fs_reg payload = ubld.vgrf(ELK_REGISTER_TYPE_UD);
ubld.MOV(payload, offset_B);
inst->sfid = GFX12_SFID_UGM;
inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD,
1 /* simd_size */,
surface_handle.file == BAD_FILE ?
LSC_ADDR_SURFTYPE_BTI :
LSC_ADDR_SURFTYPE_BSS,
LSC_ADDR_SIZE_A32,
1 /* num_coordinates */,
LSC_DATA_SIZE_D32,
inst->size_written / 4,
true /* transpose */,
LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS),
true /* has_dest */);
/* Update the original instruction. */
inst->opcode = ELK_SHADER_OPCODE_SEND;
inst->mlen = lsc_msg_desc_src0_len(devinfo, inst->desc);
inst->send_ex_bso = surface_handle.file != BAD_FILE &&
compiler->extended_bindless_surface_offset;
inst->ex_mlen = 0;
inst->header_size = 0;
inst->send_has_side_effects = false;
inst->send_is_volatile = true;
inst->exec_size = 1;
/* Finally, the payload */
inst->resize_sources(3);
setup_lsc_surface_descriptors(ubld, inst, inst->desc,
surface.file != BAD_FILE ?
surface : surface_handle);
inst->src[2] = payload;
invalidate_analysis(DEPENDENCY_INSTRUCTIONS | DEPENDENCY_VARIABLES);
} else if (devinfo->ver >= 7) {
if (devinfo->ver >= 7) {
const fs_builder ubld = fs_builder(this, block, inst).exec_all();
elk_fs_reg header = fs_builder(this, 8).exec_all().vgrf(ELK_REGISTER_TYPE_UD);
src/intel/compiler/elk/elk_schedule_instructions.cpp
-37
View File
@@ -571,43 +571,6 @@ elk_schedule_node::set_latency_gfx7(const struct elk_isa_info *isa)
latency = 50; /* TODO */
break;
case GFX12_SFID_UGM:
case GFX12_SFID_TGM:
case GFX12_SFID_SLM:
switch (lsc_msg_desc_opcode(isa->devinfo, inst->desc)) {
case LSC_OP_LOAD:
case LSC_OP_STORE:
case LSC_OP_LOAD_CMASK:
case LSC_OP_STORE_CMASK:
latency = 300;
break;
case LSC_OP_FENCE:
case LSC_OP_ATOMIC_INC:
case LSC_OP_ATOMIC_DEC:
case LSC_OP_ATOMIC_LOAD:
case LSC_OP_ATOMIC_STORE:
case LSC_OP_ATOMIC_ADD:
case LSC_OP_ATOMIC_SUB:
case LSC_OP_ATOMIC_MIN:
case LSC_OP_ATOMIC_MAX:
case LSC_OP_ATOMIC_UMIN:
case LSC_OP_ATOMIC_UMAX:
case LSC_OP_ATOMIC_CMPXCHG:
case LSC_OP_ATOMIC_FADD:
case LSC_OP_ATOMIC_FSUB:
case LSC_OP_ATOMIC_FMIN:
case LSC_OP_ATOMIC_FMAX:
case LSC_OP_ATOMIC_FCMPXCHG:
case LSC_OP_ATOMIC_AND:
case LSC_OP_ATOMIC_OR:
case LSC_OP_ATOMIC_XOR:
latency = 1400;
break;
default:
unreachable("unsupported new data port message instruction");
}
break;
case ELK_SFID_URB:
latency = 200;
break;