From 90967e7b1633d4aadce87361e76a0455577a200c Mon Sep 17 00:00:00 2001 From: Caio Oliveira Date: Fri, 22 Aug 2025 00:08:22 -0700 Subject: [PATCH] brw: Add brw_load_payload_inst Reviewed-by: Lionel Landwerlin Reviewed-by: Kenneth Graunke Part-of: --- src/intel/compiler/brw_builder.h | 15 ++++---- src/intel/compiler/brw_inst.cpp | 6 +++- src/intel/compiler/brw_inst.h | 13 +++++-- src/intel/compiler/brw_lower.cpp | 34 ++++++++++--------- .../compiler/brw_lower_logical_sends.cpp | 4 --- src/intel/compiler/brw_lower_simd_width.cpp | 16 +++++---- src/intel/compiler/brw_opt.cpp | 14 +++++--- .../compiler/brw_opt_copy_propagation.cpp | 31 +++++++++-------- src/intel/compiler/brw_opt_cse.cpp | 21 ++++++++++-- .../compiler/brw_opt_register_coalesce.cpp | 14 ++++---- 10 files changed, 102 insertions(+), 66 deletions(-) diff --git a/src/intel/compiler/brw_builder.h b/src/intel/compiler/brw_builder.h index 6a4cf801f0e..3d48b6fd2e0 100644 --- a/src/intel/compiler/brw_builder.h +++ b/src/intel/compiler/brw_builder.h @@ -765,19 +765,20 @@ public: /** * Collect a number of registers in a contiguous range of registers. */ - brw_inst * + brw_load_payload_inst * LOAD_PAYLOAD(const brw_reg &dst, const brw_reg *src, unsigned sources, unsigned header_size) const { - brw_inst *inst = emit(SHADER_OPCODE_LOAD_PAYLOAD, dst, src, sources); - inst->header_size = header_size; - inst->size_written = header_size * REG_SIZE; + brw_load_payload_inst *lp = + emit(SHADER_OPCODE_LOAD_PAYLOAD, dst, src, sources)->as_load_payload(); + lp->header_size = header_size; + lp->size_written = header_size * REG_SIZE; for (unsigned i = header_size; i < sources; i++) { - inst->size_written += dispatch_width() * brw_type_size_bytes(src[i].type) * - dst.stride; + lp->size_written += dispatch_width() * brw_type_size_bytes(src[i].type) * + dst.stride; } - return inst; + return lp; } brw_inst * diff --git a/src/intel/compiler/brw_inst.cpp b/src/intel/compiler/brw_inst.cpp index 3a58f30378e..e666d850dc7 100644 --- a/src/intel/compiler/brw_inst.cpp +++ b/src/intel/compiler/brw_inst.cpp @@ -17,6 +17,7 @@ brw_inst_kind_size(brw_inst_kind kind) STATIC_ASSERT(sizeof(brw_send_inst) >= sizeof(brw_tex_inst)); STATIC_ASSERT(sizeof(brw_send_inst) >= sizeof(brw_mem_inst)); STATIC_ASSERT(sizeof(brw_send_inst) >= sizeof(brw_dpas_inst)); + STATIC_ASSERT(sizeof(brw_send_inst) >= sizeof(brw_load_payload_inst)); /* TODO: Temporarily here to ensure all instructions can be converted to * SEND. Once all new kinds are added, change so that BASE allocate only @@ -188,6 +189,9 @@ brw_inst_kind_for_opcode(enum opcode opcode) case BRW_OPCODE_DPAS: return BRW_KIND_DPAS; + case SHADER_OPCODE_LOAD_PAYLOAD: + return BRW_KIND_LOAD_PAYLOAD; + default: return BRW_KIND_BASE; } @@ -549,7 +553,7 @@ brw_inst::size_read(const struct intel_device_info *devinfo, int arg) const break; case SHADER_OPCODE_LOAD_PAYLOAD: - if (arg < this->header_size) + if (arg < as_load_payload()->header_size) return retype(src[arg], BRW_TYPE_UD).component_size(8); break; diff --git a/src/intel/compiler/brw_inst.h b/src/intel/compiler/brw_inst.h index d755c569f0c..7c40a4c8be2 100644 --- a/src/intel/compiler/brw_inst.h +++ b/src/intel/compiler/brw_inst.h @@ -45,6 +45,7 @@ enum ENUM_PACKED brw_inst_kind { BRW_KIND_TEX, BRW_KIND_MEM, BRW_KIND_DPAS, + BRW_KIND_LOAD_PAYLOAD, }; brw_inst_kind brw_inst_kind_for_opcode(enum opcode opcode); @@ -74,6 +75,7 @@ struct brw_inst : brw_exec_node { KIND_HELPERS(as_tex, brw_tex_inst, BRW_KIND_TEX); KIND_HELPERS(as_mem, brw_mem_inst, BRW_KIND_MEM); KIND_HELPERS(as_dpas, brw_dpas_inst, BRW_KIND_DPAS); + KIND_HELPERS(as_load_payload, brw_load_payload_inst, BRW_KIND_LOAD_PAYLOAD); #undef KIND_HELPERS @@ -168,9 +170,6 @@ struct brw_inst : brw_exec_node { */ uint8_t group; - /** The number of hardware registers used for a message header. */ - uint8_t header_size; - uint32_t offset; /**< spill/unspill offset or texture offset bitfield */ uint16_t size_written; /**< Data written to the destination register in bytes. */ @@ -238,6 +237,9 @@ struct brw_send_inst : brw_inst { uint8_t ex_mlen; uint8_t sfid; + /** The number of hardware registers used for a message header. */ + uint8_t header_size; + union { struct { /** @@ -296,6 +298,11 @@ struct brw_dpas_inst : brw_inst { uint8_t rcount; }; +struct brw_load_payload_inst : brw_inst { + /** The number of hardware registers used for a message header. */ + uint8_t header_size; +}; + /** * Make the execution of \p inst dependent on the evaluation of a possibly * inverted predicate. diff --git a/src/intel/compiler/brw_lower.cpp b/src/intel/compiler/brw_lower.cpp index 20c92f32c31..a1e82968eb1 100644 --- a/src/intel/compiler/brw_lower.cpp +++ b/src/intel/compiler/brw_lower.cpp @@ -56,41 +56,43 @@ brw_lower_load_payload(brw_shader &s) if (inst->opcode != SHADER_OPCODE_LOAD_PAYLOAD) continue; - assert(inst->dst.file == VGRF); - assert(inst->saturate == false); - brw_reg dst = inst->dst; + brw_load_payload_inst *lp = inst->as_load_payload(); - const brw_builder ibld(inst); + assert(lp->dst.file == VGRF); + assert(lp->saturate == false); + brw_reg dst = lp->dst; + + const brw_builder ibld(lp); const brw_builder ubld = ibld.exec_all(); - for (uint8_t i = 0; i < inst->header_size;) { + for (uint8_t i = 0; i < lp->header_size;) { /* Number of header GRFs to initialize at once with a single MOV * instruction. */ const unsigned n = - (i + 1 < inst->header_size && - (inst->src[i].file == IMM || - (inst->src[i].is_contiguous() && - inst->src[i + 1].equals(byte_offset(inst->src[i], REG_SIZE))))) ? + (i + 1 < lp->header_size && + (lp->src[i].file == IMM || + (lp->src[i].is_contiguous() && + lp->src[i + 1].equals(byte_offset(lp->src[i], REG_SIZE))))) ? 2 : 1; - if (inst->src[i].file != BAD_FILE) + if (lp->src[i].file != BAD_FILE) ubld.group(8 * n, 0).MOV(retype(dst, BRW_TYPE_UD), - retype(inst->src[i], BRW_TYPE_UD)); + retype(lp->src[i], BRW_TYPE_UD)); dst = byte_offset(dst, n * REG_SIZE); i += n; } - for (uint8_t i = inst->header_size; i < inst->sources; i++) { - dst.type = inst->src[i].type; - if (inst->src[i].file != BAD_FILE) { - ibld.MOV(dst, inst->src[i]); + for (uint8_t i = lp->header_size; i < lp->sources; i++) { + dst.type = lp->src[i].type; + if (lp->src[i].file != BAD_FILE) { + ibld.MOV(dst, lp->src[i]); } dst = offset(dst, ibld, 1); } - inst->remove(); + lp->remove(); progress = true; } diff --git a/src/intel/compiler/brw_lower_logical_sends.cpp b/src/intel/compiler/brw_lower_logical_sends.cpp index 5981ddb6981..32c1174c5e7 100644 --- a/src/intel/compiler/brw_lower_logical_sends.cpp +++ b/src/intel/compiler/brw_lower_logical_sends.cpp @@ -45,7 +45,6 @@ lower_urb_read_logical_send(const brw_builder &bld, brw_inst *inst) inst->src[URB_LOGICAL_SRC_PER_SLOT_OFFSETS].file != BAD_FILE; assert(inst->size_written % REG_SIZE == 0); - assert(inst->header_size == 0); brw_reg payload_sources[2]; unsigned header_size = 0; @@ -86,7 +85,6 @@ lower_urb_read_logical_send_xe2(const brw_builder &bld, brw_inst *inst) assert(devinfo->has_lsc); assert(inst->size_written % (REG_SIZE * reg_unit(devinfo)) == 0); - assert(inst->header_size == 0); /* Get the logical send arguments. */ const brw_reg handle = inst->src[URB_LOGICAL_SRC_HANDLE]; @@ -146,8 +144,6 @@ lower_urb_write_logical_send(const brw_builder &bld, brw_inst *inst) const bool channel_mask_present = inst->src[URB_LOGICAL_SRC_CHANNEL_MASK].file != BAD_FILE; - assert(inst->header_size == 0); - const unsigned length = 1 + per_slot_present + channel_mask_present + inst->components_read(URB_LOGICAL_SRC_DATA); diff --git a/src/intel/compiler/brw_lower_simd_width.cpp b/src/intel/compiler/brw_lower_simd_width.cpp index 63ae2b71dcd..243f930670e 100644 --- a/src/intel/compiler/brw_lower_simd_width.cpp +++ b/src/intel/compiler/brw_lower_simd_width.cpp @@ -437,8 +437,10 @@ brw_get_lowered_simd_width(const brw_shader *shader, const brw_inst *inst) } case SHADER_OPCODE_LOAD_PAYLOAD: { + const brw_load_payload_inst *lp = inst->as_load_payload(); + const unsigned reg_count = - DIV_ROUND_UP(inst->dst.component_size(inst->exec_size), + DIV_ROUND_UP(lp->dst.component_size(lp->exec_size), REG_SIZE * reg_unit(devinfo)); if (reg_count > 2) { @@ -446,14 +448,14 @@ brw_get_lowered_simd_width(const brw_shader *shader, const brw_inst *inst) * can be easily lowered (which excludes headers and heterogeneous * types). */ - assert(!inst->header_size); - for (unsigned i = 0; i < inst->sources; i++) - assert(brw_type_size_bits(inst->dst.type) == brw_type_size_bits(inst->src[i].type) || - inst->src[i].file == BAD_FILE); + assert(!lp->header_size); + for (unsigned i = 0; i < lp->sources; i++) + assert(brw_type_size_bits(lp->dst.type) == brw_type_size_bits(lp->src[i].type) || + lp->src[i].file == BAD_FILE); - return inst->exec_size / DIV_ROUND_UP(reg_count, 2); + return lp->exec_size / DIV_ROUND_UP(reg_count, 2); } else { - return inst->exec_size; + return lp->exec_size; } } default: diff --git a/src/intel/compiler/brw_opt.cpp b/src/intel/compiler/brw_opt.cpp index 58ddf2b02c4..88964c6c601 100644 --- a/src/intel/compiler/brw_opt.cpp +++ b/src/intel/compiler/brw_opt.cpp @@ -234,7 +234,7 @@ brw_optimize(brw_shader &s) } static unsigned -load_payload_sources_read_for_size(brw_inst *lp, unsigned size_read) +load_payload_sources_read_for_size(brw_load_payload_inst *lp, unsigned size_read) { assert(lp->opcode == SHADER_OPCODE_LOAD_PAYLOAD); assert(size_read >= lp->header_size * REG_SIZE); @@ -283,11 +283,13 @@ brw_opt_zero_samples(brw_shader &s) if (send->ex_mlen > 0) continue; - brw_inst *lp = (brw_inst *) send->prev; + brw_inst *prev = (brw_inst *) send->prev; - if (lp->is_head_sentinel() || lp->opcode != SHADER_OPCODE_LOAD_PAYLOAD) + if (prev->is_head_sentinel() || prev->opcode != SHADER_OPCODE_LOAD_PAYLOAD) continue; + brw_load_payload_inst *lp = prev->as_load_payload(); + /* How much of the payload are actually read by this SEND. */ const unsigned params = load_payload_sources_read_for_size(lp, send->mlen * REG_SIZE); @@ -353,11 +355,13 @@ brw_opt_split_sends(brw_shader &s) continue; /* Currently don't split sends that reuse a previously used payload. */ - brw_inst *lp = (brw_inst *) send->prev; + brw_inst *prev = (brw_inst *) send->prev; - if (lp->is_head_sentinel() || lp->opcode != SHADER_OPCODE_LOAD_PAYLOAD) + if (prev->is_head_sentinel() || prev->opcode != SHADER_OPCODE_LOAD_PAYLOAD) continue; + brw_load_payload_inst *lp = prev->as_load_payload(); + if (lp->dst.file != send->src[SEND_SRC_PAYLOAD1].file || lp->dst.nr != send->src[SEND_SRC_PAYLOAD1].nr) continue; diff --git a/src/intel/compiler/brw_opt_copy_propagation.cpp b/src/intel/compiler/brw_opt_copy_propagation.cpp index 0d1bf4aeaa9..9edd1146770 100644 --- a/src/intel/compiler/brw_opt_copy_propagation.cpp +++ b/src/intel/compiler/brw_opt_copy_propagation.cpp @@ -1482,25 +1482,26 @@ opt_copy_propagation_local(brw_shader &s, linear_ctx *lin_ctx, acp.add(entry); } else if (inst->opcode == SHADER_OPCODE_LOAD_PAYLOAD && inst->dst.file == VGRF) { + brw_load_payload_inst *lp = inst->as_load_payload(); int offset = 0; - for (int i = 0; i < inst->sources; i++) { - int effective_width = i < inst->header_size ? 8 : inst->exec_size; + for (int i = 0; i < lp->sources; i++) { + int effective_width = i < lp->header_size ? 8 : lp->exec_size; const unsigned size_written = - effective_width * brw_type_size_bytes(inst->src[i].type); - if (inst->src[i].file == VGRF || - (inst->src[i].file == FIXED_GRF && - inst->src[i].is_contiguous())) { - const brw_reg_type t = i < inst->header_size ? - BRW_TYPE_UD : inst->src[i].type; - brw_reg dst = byte_offset(retype(inst->dst, t), offset); - if (!dst.equals(inst->src[i])) { + effective_width * brw_type_size_bytes(lp->src[i].type); + if (lp->src[i].file == VGRF || + (lp->src[i].file == FIXED_GRF && + lp->src[i].is_contiguous())) { + const brw_reg_type t = i < lp->header_size ? + BRW_TYPE_UD : lp->src[i].type; + brw_reg dst = byte_offset(retype(lp->dst, t), offset); + if (!dst.equals(lp->src[i])) { acp_entry *entry = linear_zalloc(lin_ctx, acp_entry); entry->dst = dst; - entry->src = retype(inst->src[i], t); + entry->src = retype(lp->src[i], t); entry->size_written = size_written; - entry->size_read = inst->size_read(devinfo, i); - entry->opcode = inst->opcode; - entry->force_writemask_all = inst->force_writemask_all; + entry->size_read = lp->size_read(devinfo, i); + entry->opcode = lp->opcode; + entry->force_writemask_all = lp->force_writemask_all; acp.add(entry); } } @@ -1866,7 +1867,7 @@ find_value_for_offset(brw_inst *def, const brw_reg &src, unsigned src_size) break; case SHADER_OPCODE_LOAD_PAYLOAD: { unsigned offset = 0; - for (int i = def->header_size; i < def->sources; i++) { + for (int i = def->as_load_payload()->header_size; i < def->sources; i++) { /* Ignore the source splat if the source is a scalar. In that case * always use just the first component. */ diff --git a/src/intel/compiler/brw_opt_cse.cpp b/src/intel/compiler/brw_opt_cse.cpp index 38bf2b61890..5473df8b401 100644 --- a/src/intel/compiler/brw_opt_cse.cpp +++ b/src/intel/compiler/brw_opt_cse.cpp @@ -246,6 +246,7 @@ send_inst_match(brw_send_inst *a, brw_send_inst *b) return a->mlen == b->mlen && a->ex_mlen == b->ex_mlen && a->sfid == b->sfid && + a->header_size == b->header_size && a->desc == b->desc && a->ex_desc == b->ex_desc && a->send_bits == b->send_bits; @@ -280,6 +281,12 @@ dpas_inst_match(brw_dpas_inst *a, brw_dpas_inst *b) a->rcount == b->rcount; } +static bool +load_payload_inst_match(brw_load_payload_inst *a, brw_load_payload_inst *b) +{ + return a->header_size == b->header_size; +} + static bool instructions_match(brw_inst *a, brw_inst *b, bool *negate) { @@ -290,6 +297,8 @@ instructions_match(brw_inst *a, brw_inst *b, bool *negate) (a->kind != BRW_KIND_TEX || tex_inst_match(a->as_tex(), b->as_tex())) && (a->kind != BRW_KIND_MEM || mem_inst_match(a->as_mem(), b->as_mem())) && (a->kind != BRW_KIND_DPAS || dpas_inst_match(a->as_dpas(), b->as_dpas())) && + (a->kind != BRW_KIND_LOAD_PAYLOAD || + load_payload_inst_match(a->as_load_payload(), b->as_load_payload())) && a->exec_size == b->exec_size && a->group == b->group && a->predicate == b->predicate && @@ -297,7 +306,6 @@ instructions_match(brw_inst *a, brw_inst *b, bool *negate) a->dst.type == b->dst.type && a->offset == b->offset && a->size_written == b->size_written && - a->header_size == b->header_size && a->sources == b->sources && a->bits == b->bits && operands_match(a, b, negate); @@ -339,7 +347,6 @@ hash_inst(const void *v) inst->sources, inst->exec_size, inst->group, - inst->header_size, inst->conditional_mod, inst->predicate, @@ -364,6 +371,7 @@ hash_inst(const void *v) send->ex_mlen, send->sfid, send->send_bits, + send->header_size, }; const uint32_t send_u32data[] = { send->desc, @@ -415,6 +423,15 @@ hash_inst(const void *v) break; } + case BRW_KIND_LOAD_PAYLOAD: { + const brw_load_payload_inst *lp = inst->as_load_payload(); + const uint8_t lp_u8data[] = { + lp->header_size, + }; + hash = HASH(hash, lp_u8data); + break; + } + case BRW_KIND_BASE: /* Nothing else to do. */ break; diff --git a/src/intel/compiler/brw_opt_register_coalesce.cpp b/src/intel/compiler/brw_opt_register_coalesce.cpp index f5442d50a84..15e383e46c0 100644 --- a/src/intel/compiler/brw_opt_register_coalesce.cpp +++ b/src/intel/compiler/brw_opt_register_coalesce.cpp @@ -48,14 +48,16 @@ static bool is_nop_mov(const brw_inst *inst) { if (inst->opcode == SHADER_OPCODE_LOAD_PAYLOAD) { - brw_reg dst = inst->dst; - for (int i = 0; i < inst->sources; i++) { - if (!dst.equals(inst->src[i])) { + const brw_load_payload_inst *lp = inst->as_load_payload(); + + brw_reg dst = lp->dst; + for (int i = 0; i < lp->sources; i++) { + if (!dst.equals(lp->src[i])) { return false; } - dst.offset += (i < inst->header_size ? REG_SIZE : - inst->exec_size * dst.stride * - brw_type_size_bytes(inst->src[i].type)); + dst.offset += (i < lp->header_size ? REG_SIZE : + lp->exec_size * dst.stride * + brw_type_size_bytes(lp->src[i].type)); } return true; } else if (inst->opcode == BRW_OPCODE_MOV) {