From 018611364099258b14bf774863b66639cff17c3b Mon Sep 17 00:00:00 2001 From: Rohan Garg Date: Tue, 22 Oct 2024 17:00:13 +0200 Subject: [PATCH] brw: encode the offset into the message descriptor for Xe2 Signed-off-by: Rohan Garg Reviewed-by: Kenneth Graunke Reviewed-by: Lionel Landwerlin Part-of: --- src/intel/compiler/brw_eu.h | 32 ++++++++- src/intel/compiler/brw_eu_emit.c | 14 ++++ src/intel/compiler/brw_generator.cpp | 11 +++- src/intel/compiler/brw_inst.h | 8 ++- .../compiler/brw_lower_logical_sends.cpp | 66 ++++++++++++++----- src/intel/compiler/brw_print.cpp | 3 + 6 files changed, 113 insertions(+), 21 deletions(-) diff --git a/src/intel/compiler/brw_eu.h b/src/intel/compiler/brw_eu.h index 724f34aa505..c54e68efeaa 100644 --- a/src/intel/compiler/brw_eu.h +++ b/src/intel/compiler/brw_eu.h @@ -1295,11 +1295,12 @@ lsc_fence_msg_desc_backup_routing(UNUSED const struct intel_device_info *devinfo } static inline uint32_t -lsc_bti_ex_desc(const struct intel_device_info *devinfo, unsigned bti) +lsc_bti_ex_desc(const struct intel_device_info *devinfo, unsigned bti, + unsigned base_offset) { assert(devinfo->has_lsc); return SET_BITS(bti, 31, 24) | - SET_BITS(0, 23, 12); /* base offset */ + SET_BITS(base_offset, 23, 12); /* base offset */ } static inline unsigned @@ -1318,6 +1319,14 @@ lsc_bti_ex_desc_index(const struct intel_device_info *devinfo, return GET_BITS(ex_desc, 31, 24); } +static inline unsigned +lsc_flat_ex_desc(const struct intel_device_info *devinfo, + uint32_t base_offset) +{ + assert(devinfo->has_lsc); + return SET_BITS(base_offset, 31, 12); +} + static inline unsigned lsc_flat_ex_desc_base_offset(const struct intel_device_info *devinfo, uint32_t ex_desc) @@ -1460,6 +1469,7 @@ brw_send_indirect_split_message(struct brw_codegen *p, struct brw_reg payload1, struct brw_reg desc, struct brw_reg ex_desc, + uint32_t ex_desc_imm_inst, unsigned ex_mlen, bool ex_bso, bool eot, @@ -1612,6 +1622,24 @@ next_offset(struct brw_codegen *p, void *store, int offset) /** Maximum SEND message length */ #define BRW_MAX_MSG_LENGTH 15 +/** Offset encoding signed size limits (top bit is the sign) */ +#define LSC_ADDRESS_OFFSET_FLAT_BITS 20 +#define LSC_ADDRESS_OFFSET_SS_BITS 17 +#define LSC_ADDRESS_OFFSET_BTI_BITS 12 + +static inline unsigned +brw_max_immediate_offset_bits(enum lsc_addr_surface_type binding_type) +{ + static const unsigned max_bits[] = { + [LSC_ADDR_SURFTYPE_FLAT] = LSC_ADDRESS_OFFSET_FLAT_BITS, + [LSC_ADDR_SURFTYPE_BSS] = LSC_ADDRESS_OFFSET_SS_BITS, + [LSC_ADDR_SURFTYPE_SS] = LSC_ADDRESS_OFFSET_SS_BITS, + [LSC_ADDR_SURFTYPE_BTI] = LSC_ADDRESS_OFFSET_BTI_BITS, + }; + assert(binding_type <= LSC_ADDR_SURFTYPE_BTI); + return max_bits[binding_type]; +} + #ifdef __cplusplus } #endif diff --git a/src/intel/compiler/brw_eu_emit.c b/src/intel/compiler/brw_eu_emit.c index 6f3434ae2ed..b052d884ecb 100644 --- a/src/intel/compiler/brw_eu_emit.c +++ b/src/intel/compiler/brw_eu_emit.c @@ -1490,6 +1490,7 @@ brw_send_indirect_split_message(struct brw_codegen *p, struct brw_reg payload1, struct brw_reg desc, struct brw_reg ex_desc, + uint32_t ex_desc_imm_inst, unsigned ex_mlen, bool ex_bso, bool eot, @@ -1517,6 +1518,7 @@ brw_send_indirect_split_message(struct brw_codegen *p, } if (ex_desc.file == IMM) { + assert(ex_desc_imm_inst == 0); brw_eu_inst_set_send_sel_reg32_ex_desc(devinfo, send, 0); brw_eu_inst_set_sends_ex_desc(devinfo, send, ex_desc.ud, gather); } else { @@ -1525,6 +1527,18 @@ brw_send_indirect_split_message(struct brw_codegen *p, brw_eu_inst_set_send_sel_reg32_ex_desc(devinfo, send, 1); brw_eu_inst_set_send_ex_desc_ia_subreg_nr(devinfo, send, phys_subnr(devinfo, ex_desc) >> 2); + if (ex_desc_imm_inst) { + /* Write the immediate extended descriptor immediate value, but only + * the part used for encoding an offset. This matches to bits + * 12:15-19:31 as described in BSpec 70586. + */ + assert(devinfo->ver >= 20); + brw_eu_inst_set_bits(send, 127, 124, GET_BITS(ex_desc_imm_inst, 31, 28)); + brw_eu_inst_set_bits(send, 97, 96, GET_BITS(ex_desc_imm_inst, 27, 26)); + brw_eu_inst_set_bits(send, 65, 64, GET_BITS(ex_desc_imm_inst, 25, 24)); + brw_eu_inst_set_bits(send, 47, 43, GET_BITS(ex_desc_imm_inst, 23, 19)); + brw_eu_inst_set_bits(send, 39, 36, GET_BITS(ex_desc_imm_inst, 15, 12)); + } if (devinfo->ver >= 20 && sfid == BRW_SFID_UGM) brw_eu_inst_set_bits(send, 103, 99, ex_mlen / reg_unit(devinfo)); } diff --git a/src/intel/compiler/brw_generator.cpp b/src/intel/compiler/brw_generator.cpp index e6b42570ce9..260b7d7de97 100644 --- a/src/intel/compiler/brw_generator.cpp +++ b/src/intel/compiler/brw_generator.cpp @@ -177,6 +177,11 @@ brw_generator::generate_send(brw_inst *inst, } if (ex_desc.file == IMM && ex_desc.ud == 0) { + /* An immediate extended descriptor value only happens when the extended + * descriptor is written indirectly (it already contains a SS/BSS + * surface handle) + */ + assert(!inst->send_ex_desc_imm); brw_send_indirect_message(p, inst->sfid, dst, payload, desc, inst->eot, gather); if (inst->check_tdr) brw_eu_inst_set_opcode(p->isa, brw_last_inst, BRW_OPCODE_SENDC); @@ -185,8 +190,10 @@ brw_generator::generate_send(brw_inst *inst, * also covers the dual-payload case because ex_mlen goes in ex_desc. */ brw_send_indirect_split_message(p, inst->sfid, dst, payload, payload2, - desc, ex_desc, inst->ex_mlen, - inst->send_ex_bso, inst->eot, gather); + desc, ex_desc, + inst->send_ex_desc_imm ? inst->offset : 0, + inst->ex_mlen, inst->send_ex_bso, + inst->eot, gather); if (inst->check_tdr) brw_eu_inst_set_opcode(p->isa, brw_last_inst, devinfo->ver >= 12 ? BRW_OPCODE_SENDC : BRW_OPCODE_SENDSC); diff --git a/src/intel/compiler/brw_inst.h b/src/intel/compiler/brw_inst.h index 10aff1917c9..30b2b9d8624 100644 --- a/src/intel/compiler/brw_inst.h +++ b/src/intel/compiler/brw_inst.h @@ -190,7 +190,7 @@ public: */ unsigned rcount:4; - unsigned pad:5; + unsigned pad:4; bool predicate_inverse:1; bool writes_accumulator:1; /**< instruction implicitly writes accumulator */ @@ -205,6 +205,12 @@ public: * bindless surface offset (26bits instead of * 20bits) */ + /** + * Only for SHADER_OPCODE_SEND, @offset field contains an immediate + * part of the extended descriptor that must be encoded in the + * instruction. + */ + bool send_ex_desc_imm:1; /** * The predication mask applied to this instruction is guaranteed to * be uniform and a superset of the execution mask of the present block. diff --git a/src/intel/compiler/brw_lower_logical_sends.cpp b/src/intel/compiler/brw_lower_logical_sends.cpp index 473305d10fd..a361c0afc68 100644 --- a/src/intel/compiler/brw_lower_logical_sends.cpp +++ b/src/intel/compiler/brw_lower_logical_sends.cpp @@ -29,6 +29,8 @@ #include "brw_shader.h" #include "brw_builder.h" +#include "util/bitpack_helpers.h" + static void lower_urb_read_logical_send(const brw_builder &bld, brw_inst *inst) { @@ -1396,14 +1398,26 @@ setup_surface_descriptors(const brw_builder &bld, brw_inst *inst, uint32_t desc, static void setup_lsc_surface_descriptors(const brw_builder &bld, brw_inst *inst, - uint32_t desc, const brw_reg &surface) + uint32_t desc, const brw_reg &surface, + int32_t base_offset) { const ASSERTED intel_device_info *devinfo = bld.shader->devinfo; const brw_compiler *compiler = bld.shader->compiler; + assert(base_offset == 0 || devinfo->ver >= 20); + inst->src[0] = brw_imm_ud(0); /* desc */ + inst->src[1] = brw_imm_ud(0); enum lsc_addr_surface_type surf_type = lsc_msg_desc_addr_type(devinfo, desc); + + unsigned max_imm_bits = brw_max_immediate_offset_bits(surf_type); + assert(base_offset >= u_intN_min(max_imm_bits)); + assert(base_offset <= u_intN_max(max_imm_bits)); + + const unsigned base_offset_bits = + util_bitpack_sint(base_offset, 0, max_imm_bits - 1); + switch (surf_type) { case LSC_ADDR_SURFTYPE_BSS: inst->send_ex_bso = compiler->extended_bindless_surface_offset; @@ -1417,22 +1431,40 @@ setup_lsc_surface_descriptors(const brw_builder &bld, brw_inst *inst, /* Gfx20+ assumes ExBSO with UGM */ if (devinfo->ver >= 20 && inst->sfid == BRW_SFID_UGM) inst->send_ex_bso = true; + + /* We're already using the extended descriptor to hold the surface + * handle. But now the immediate extended descriptor bits in the + * instruction are unused, so this is where the HW design team thought + * it would be a good idea to store the immediate offset. + * + * This doesn't play well with the rest of our compiler that considers + * there is only one value for the extended descriptor. So here we stash + * the base offset in brw_inst::offset and flag the instruction for the + * generator to do the right thing with it. + */ + if (base_offset) { + inst->send_ex_desc_imm = true; + inst->offset = SET_BITS(GET_BITS(base_offset_bits, 16, 4), 31, 19) | + SET_BITS(GET_BITS(base_offset_bits, 3, 0), 15, 12); + } break; case LSC_ADDR_SURFTYPE_BTI: assert(surface.file != BAD_FILE); if (surface.file == IMM) { - inst->src[1] = brw_imm_ud(lsc_bti_ex_desc(devinfo, surface.ud)); + inst->src[1] = brw_imm_ud(lsc_bti_ex_desc(devinfo, surface.ud, + base_offset_bits)); } else { const brw_builder ubld = bld.uniform(); - brw_reg tmp = ubld.vgrf(BRW_TYPE_UD); - ubld.SHL(tmp, surface, brw_imm_ud(24)); + brw_reg tmp = ubld.OR( + ubld.SHL(surface, brw_imm_ud(24)), + brw_imm_ud(base_offset << 12)); inst->src[1] = component(tmp, 0); } break; case LSC_ADDR_SURFTYPE_FLAT: - inst->src[1] = brw_imm_ud(0); + inst->src[1] = brw_imm_ud(lsc_flat_ex_desc(devinfo, base_offset_bits)); break; default: @@ -1490,11 +1522,14 @@ lower_lsc_memory_logical_send(const brw_builder &bld, brw_inst *inst) brw_type_with_size(data0.type, data_size_B * 8); const enum lsc_addr_size addr_size = lsc_addr_size_for_type(addr.type); + assert(inst->src[MEMORY_LOGICAL_ADDRESS_OFFSET].file == IMM); + const int32_t base_offset = inst->src[MEMORY_LOGICAL_ADDRESS_OFFSET].d; - const brw_reg base_offset = - retype(inst->src[MEMORY_LOGICAL_ADDRESS_OFFSET], BRW_TYPE_UD); - /* TODO: setup the offset */ - assert(base_offset.ud == 0); + /** + * TGM messages cannot have a base offset + */ + if (mode == MEMORY_MODE_TYPED) + assert(base_offset == 0); brw_reg payload = addr; @@ -1596,7 +1631,7 @@ lower_lsc_memory_logical_send(const brw_builder &bld, brw_inst *inst) transpose, cache_mode); /* Set up extended descriptors, fills src[0] and src[1]. */ - setup_lsc_surface_descriptors(bld, inst, inst->desc, binding); + setup_lsc_surface_descriptors(bld, inst, inst->desc, binding, base_offset); inst->opcode = SHADER_OPCODE_SEND; inst->mlen = lsc_msg_addr_len(devinfo, addr_size, @@ -1678,9 +1713,8 @@ lower_hdc_memory_logical_send(const brw_builder &bld, brw_inst *inst) const brw_reg data1 = inst->src[MEMORY_LOGICAL_DATA1]; const bool has_side_effects = inst->has_side_effects(); const bool has_dest = inst->dst.file != BAD_FILE && !inst->dst.is_null(); - const brw_reg base_offset = - retype(inst->src[MEMORY_LOGICAL_ADDRESS_OFFSET], BRW_TYPE_UD); - assert(base_offset.ud == 0); + assert(inst->src[MEMORY_LOGICAL_ADDRESS_OFFSET].file == IMM && + inst->src[MEMORY_LOGICAL_ADDRESS_OFFSET].d == 0); /* Don't predicate scratch writes on the sample mask. Otherwise, * FS helper invocations would load undefined values from scratch memory. @@ -1986,7 +2020,7 @@ lower_lsc_varying_pull_constant_logical_send(const brw_builder &bld, setup_lsc_surface_descriptors(bld, inst, inst->desc, surface.file != BAD_FILE ? - surface : surface_handle); + surface : surface_handle, 0); } else { inst->desc = lsc_msg_desc(devinfo, LSC_OP_LOAD, @@ -1999,7 +2033,7 @@ lower_lsc_varying_pull_constant_logical_send(const brw_builder &bld, setup_lsc_surface_descriptors(bld, inst, inst->desc, surface.file != BAD_FILE ? - surface : surface_handle); + surface : surface_handle, 0); /* The byte scattered messages can only read one dword at a time so * we have to duplicate the message 4 times to read the full vec4. @@ -2746,7 +2780,7 @@ brw_lower_uniform_pull_constant_loads(brw_shader &s) inst->resize_sources(3); setup_lsc_surface_descriptors(ubld, inst, inst->desc, surface.file != BAD_FILE ? - surface : surface_handle); + surface : surface_handle, 0); inst->src[2] = payload; s.invalidate_analysis(BRW_DEPENDENCY_INSTRUCTIONS | diff --git a/src/intel/compiler/brw_print.cpp b/src/intel/compiler/brw_print.cpp index 4752ac66127..5c876c4497c 100644 --- a/src/intel/compiler/brw_print.cpp +++ b/src/intel/compiler/brw_print.cpp @@ -671,6 +671,9 @@ brw_print_instruction(const brw_shader &s, const brw_inst *inst, FILE *file, con if (is_send && inst->ex_desc) fprintf(file, "ExDesc 0x%08x ", inst->ex_desc); + if (is_send && inst->send_ex_desc_imm) + fprintf(file, "ExDescImmInst 0x%08x ", inst->offset); + if (inst->sched.regdist || inst->sched.mode) { fprintf(file, "{ "); brw_print_swsb(file, s.devinfo, inst->sched);