diff --git a/src/intel/compiler/brw_analysis_performance.cpp b/src/intel/compiler/brw_analysis_performance.cpp index 6e78f1d5661..3c130055f92 100644 --- a/src/intel/compiler/brw_analysis_performance.cpp +++ b/src/intel/compiler/brw_analysis_performance.cpp @@ -173,7 +173,8 @@ namespace { brw_type_size_bytes(inst->src[0].type) == brw_type_size_bytes(inst->src[1].type)) tx = brw_int_type(8, tx == BRW_TYPE_D); - rcount = inst->opcode == BRW_OPCODE_DPAS ? inst->rcount : 0; + const brw_dpas_inst *dpas = inst->as_dpas(); + rcount = dpas ? dpas->rcount : 0; } /** ISA encoding information */ diff --git a/src/intel/compiler/brw_builder.h b/src/intel/compiler/brw_builder.h index 3ac97f91010..6a4cf801f0e 100644 --- a/src/intel/compiler/brw_builder.h +++ b/src/intel/compiler/brw_builder.h @@ -805,7 +805,7 @@ public: return inst; } - brw_inst * + brw_dpas_inst * DPAS(const brw_reg &dst, const brw_reg &src0, const brw_reg &src1, const brw_reg &src2, unsigned sdepth, unsigned rcount) const { @@ -813,15 +813,15 @@ public: assert(sdepth == 8); assert(rcount == 1 || rcount == 2 || rcount == 4 || rcount == 8); - brw_inst *inst = emit(BRW_OPCODE_DPAS, dst, src0, src1, src2); - inst->sdepth = sdepth; - inst->rcount = rcount; + brw_dpas_inst *dpas = emit(BRW_OPCODE_DPAS, dst, src0, src1, src2)->as_dpas(); + dpas->sdepth = sdepth; + dpas->rcount = rcount; unsigned type_size = brw_type_size_bytes(dst.type); assert(type_size == 4 || type_size == 2); - inst->size_written = rcount * reg_unit(shader->devinfo) * 8 * type_size; + dpas->size_written = rcount * reg_unit(shader->devinfo) * 8 * type_size; - return inst; + return dpas; } void diff --git a/src/intel/compiler/brw_generator.cpp b/src/intel/compiler/brw_generator.cpp index 33ce7865f95..c17b178433f 100644 --- a/src/intel/compiler/brw_generator.cpp +++ b/src/intel/compiler/brw_generator.cpp @@ -926,11 +926,13 @@ brw_generator::generate_code(const brw_shader &s, brw_LINE(p, dst, src[0], src[1]); break; - case BRW_OPCODE_DPAS: + case BRW_OPCODE_DPAS: { assert(devinfo->verx10 >= 125); - brw_DPAS(p, translate_systolic_depth(inst->sdepth), inst->rcount, + const brw_dpas_inst *dpas = inst->as_dpas(); + brw_DPAS(p, translate_systolic_depth(dpas->sdepth), dpas->rcount, dst, src[0], src[1], src[2]); break; + } case BRW_OPCODE_MAD: if (devinfo->ver < 10) diff --git a/src/intel/compiler/brw_inst.cpp b/src/intel/compiler/brw_inst.cpp index 7b76e5af02e..3a58f30378e 100644 --- a/src/intel/compiler/brw_inst.cpp +++ b/src/intel/compiler/brw_inst.cpp @@ -16,6 +16,7 @@ brw_inst_kind_size(brw_inst_kind kind) { STATIC_ASSERT(sizeof(brw_send_inst) >= sizeof(brw_tex_inst)); STATIC_ASSERT(sizeof(brw_send_inst) >= sizeof(brw_mem_inst)); + STATIC_ASSERT(sizeof(brw_send_inst) >= sizeof(brw_dpas_inst)); /* TODO: Temporarily here to ensure all instructions can be converted to * SEND. Once all new kinds are added, change so that BASE allocate only @@ -184,6 +185,9 @@ brw_inst_kind_for_opcode(enum opcode opcode) case SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL: return BRW_KIND_MEM; + case BRW_OPCODE_DPAS: + return BRW_KIND_DPAS; + default: return BRW_KIND_BASE; } @@ -569,13 +573,14 @@ brw_inst::size_read(const struct intel_device_info *devinfo, int arg) const */ const unsigned reg_unit = this->exec_size / 8; const unsigned type_size = brw_type_size_bytes(src[arg].type); + const brw_dpas_inst *dpas = as_dpas(); switch (arg) { case 0: assert(type_size == 4 || type_size == 2); - return rcount * reg_unit * 8 * type_size; + return dpas->rcount * reg_unit * 8 * type_size; case 1: - return sdepth * reg_unit * REG_SIZE; + return dpas->sdepth * reg_unit * REG_SIZE; case 2: /* This is simpler than the formula described in the Bspec, but it * covers all of the cases that we support. Each inner sdepth @@ -584,7 +589,7 @@ brw_inst::size_read(const struct intel_device_info *devinfo, int arg) const * currently supportable through Vulkan. This is independent of * reg_unit. */ - return rcount * sdepth * 4; + return dpas->rcount * dpas->sdepth * 4; default: UNREACHABLE("Invalid source number."); } diff --git a/src/intel/compiler/brw_inst.h b/src/intel/compiler/brw_inst.h index 071b20a6f21..d755c569f0c 100644 --- a/src/intel/compiler/brw_inst.h +++ b/src/intel/compiler/brw_inst.h @@ -44,6 +44,7 @@ enum ENUM_PACKED brw_inst_kind { BRW_KIND_SEND, BRW_KIND_TEX, BRW_KIND_MEM, + BRW_KIND_DPAS, }; brw_inst_kind brw_inst_kind_for_opcode(enum opcode opcode); @@ -72,6 +73,7 @@ struct brw_inst : brw_exec_node { KIND_HELPERS(as_send, brw_send_inst, BRW_KIND_SEND); KIND_HELPERS(as_tex, brw_tex_inst, BRW_KIND_TEX); KIND_HELPERS(as_mem, brw_mem_inst, BRW_KIND_MEM); + KIND_HELPERS(as_dpas, brw_dpas_inst, BRW_KIND_DPAS); #undef KIND_HELPERS @@ -184,17 +186,7 @@ struct brw_inst : brw_exec_node { /* Chooses which flag subregister (f0.0 to f3.1) is used for * conditional mod and predication. */ - unsigned flag_subreg:3; - - /** - * Systolic depth used by DPAS instruction. - */ - unsigned sdepth:4; - - /** - * Repeat count used by DPAS instruction. - */ - unsigned rcount:4; + uint8_t flag_subreg:3; bool predicate_inverse:1; bool writes_accumulator:1; /**< instruction implicitly writes accumulator */ @@ -219,9 +211,9 @@ struct brw_inst : brw_exec_node { */ bool has_no_mask_send_params:1; - unsigned pad:13; + uint8_t pad:5; }; - uint32_t bits; + uint16_t bits; }; brw_reg dst; @@ -296,6 +288,14 @@ struct brw_mem_inst : brw_inst { int32_t address_offset; }; +struct brw_dpas_inst : brw_inst { + /** Systolic depth. */ + uint8_t sdepth; + + /** Repeat count. */ + uint8_t rcount; +}; + /** * Make the execution of \p inst dependent on the evaluation of a possibly * inverted predicate. diff --git a/src/intel/compiler/brw_lower_dpas.cpp b/src/intel/compiler/brw_lower_dpas.cpp index 522772adf0c..6b9d4fc18bc 100644 --- a/src/intel/compiler/brw_lower_dpas.cpp +++ b/src/intel/compiler/brw_lower_dpas.cpp @@ -7,34 +7,34 @@ #include "brw_builder.h" static void -f16_using_mac(const brw_builder &bld, brw_inst *inst) +f16_using_mac(const brw_builder &bld, brw_dpas_inst *dpas) { /* We only intend to support configurations where the destination and * accumulator have the same type. */ - if (!inst->src[0].is_null()) - assert(inst->dst.type == inst->src[0].type); + if (!dpas->src[0].is_null()) + assert(dpas->dst.type == dpas->src[0].type); - assert(inst->src[1].type == BRW_TYPE_HF); - assert(inst->src[2].type == BRW_TYPE_HF); + assert(dpas->src[1].type == BRW_TYPE_HF); + assert(dpas->src[2].type == BRW_TYPE_HF); - const brw_reg_type src0_type = inst->dst.type; + const brw_reg_type src0_type = dpas->dst.type; const brw_reg_type src1_type = BRW_TYPE_HF; const brw_reg_type src2_type = BRW_TYPE_HF; - const brw_reg dest = inst->dst; - brw_reg src0 = inst->src[0]; - const brw_reg src1 = retype(inst->src[1], src1_type); - const brw_reg src2 = retype(inst->src[2], src2_type); + const brw_reg dest = dpas->dst; + brw_reg src0 = dpas->src[0]; + const brw_reg src1 = retype(dpas->src[1], src1_type); + const brw_reg src2 = retype(dpas->src[2], src2_type); const unsigned dest_stride = dest.type == BRW_TYPE_HF ? REG_SIZE / 2 : REG_SIZE; - for (unsigned r = 0; r < inst->rcount; r++) { + for (unsigned r = 0; r < dpas->rcount; r++) { brw_reg temp = bld.vgrf(BRW_TYPE_HF); for (unsigned subword = 0; subword < 2; subword++) { - for (unsigned s = 0; s < inst->sdepth; s++) { + for (unsigned s = 0; s < dpas->sdepth; s++) { /* The first multiply of the dot-product operation has to * explicitly write the accumulator register. The successive MAC * instructions will implicitly read *and* write the @@ -48,8 +48,8 @@ f16_using_mac(const brw_builder &bld, brw_inst *inst) */ if (s == 0 && subword == 0) { const unsigned acc_width = 8; - brw_reg acc = suboffset(retype(brw_acc_reg(inst->exec_size), BRW_TYPE_UD), - inst->group % acc_width); + brw_reg acc = suboffset(retype(brw_acc_reg(dpas->exec_size), BRW_TYPE_UD), + dpas->group % acc_width); if (bld.shader->devinfo->verx10 >= 125) { acc = subscript(acc, BRW_TYPE_HF, subword); @@ -75,7 +75,7 @@ f16_using_mac(const brw_builder &bld, brw_inst *inst) * instruction, so only write the result register on the final * MAC in the sequence. */ - if ((s + 1) == inst->sdepth && subword == 1) + if ((s + 1) == dpas->sdepth && subword == 1) result = temp; else result = retype(bld.null_reg_ud(), BRW_TYPE_HF); @@ -113,33 +113,33 @@ f16_using_mac(const brw_builder &bld, brw_inst *inst) } static void -int8_using_dp4a(const brw_builder &bld, brw_inst *inst) +int8_using_dp4a(const brw_builder &bld, brw_dpas_inst *dpas) { /* We only intend to support configurations where the destination and * accumulator have the same type. */ - if (!inst->src[0].is_null()) - assert(inst->dst.type == inst->src[0].type); + if (!dpas->src[0].is_null()) + assert(dpas->dst.type == dpas->src[0].type); - assert(inst->src[1].type == BRW_TYPE_B || - inst->src[1].type == BRW_TYPE_UB); - assert(inst->src[2].type == BRW_TYPE_B || - inst->src[2].type == BRW_TYPE_UB); + assert(dpas->src[1].type == BRW_TYPE_B || + dpas->src[1].type == BRW_TYPE_UB); + assert(dpas->src[2].type == BRW_TYPE_B || + dpas->src[2].type == BRW_TYPE_UB); - const brw_reg_type src1_type = inst->src[1].type == BRW_TYPE_UB + const brw_reg_type src1_type = dpas->src[1].type == BRW_TYPE_UB ? BRW_TYPE_UD : BRW_TYPE_D; - const brw_reg_type src2_type = inst->src[2].type == BRW_TYPE_UB + const brw_reg_type src2_type = dpas->src[2].type == BRW_TYPE_UB ? BRW_TYPE_UD : BRW_TYPE_D; - brw_reg dest = inst->dst; - brw_reg src0 = inst->src[0]; - const brw_reg src1 = retype(inst->src[1], src1_type); - const brw_reg src2 = retype(inst->src[2], src2_type); + brw_reg dest = dpas->dst; + brw_reg src0 = dpas->src[0]; + const brw_reg src1 = retype(dpas->src[1], src1_type); + const brw_reg src2 = retype(dpas->src[2], src2_type); const unsigned dest_stride = reg_unit(bld.shader->devinfo) * REG_SIZE; - for (unsigned r = 0; r < inst->rcount; r++) { + for (unsigned r = 0; r < dpas->rcount; r++) { if (!src0.is_null()) { bld.MOV(dest, src0); src0 = byte_offset(src0, dest_stride); @@ -147,12 +147,12 @@ int8_using_dp4a(const brw_builder &bld, brw_inst *inst) bld.MOV(dest, retype(brw_imm_d(0), dest.type)); } - for (unsigned s = 0; s < inst->sdepth; s++) { + for (unsigned s = 0; s < dpas->sdepth; s++) { bld.DP4A(dest, dest, - byte_offset(src1, s * inst->exec_size * 4), - component(byte_offset(src2, r * inst->sdepth * 4), s)) - ->saturate = inst->saturate; + byte_offset(src1, s * dpas->exec_size * 4), + component(byte_offset(src2, r * dpas->sdepth * 4), s)) + ->saturate = dpas->saturate; } dest = byte_offset(dest, dest_stride); @@ -160,35 +160,35 @@ int8_using_dp4a(const brw_builder &bld, brw_inst *inst) } static void -int8_using_mul_add(const brw_builder &bld, brw_inst *inst) +int8_using_mul_add(const brw_builder &bld, brw_dpas_inst *dpas) { /* We only intend to support configurations where the destination and * accumulator have the same type. */ - if (!inst->src[0].is_null()) - assert(inst->dst.type == inst->src[0].type); + if (!dpas->src[0].is_null()) + assert(dpas->dst.type == dpas->src[0].type); - assert(inst->src[1].type == BRW_TYPE_B || - inst->src[1].type == BRW_TYPE_UB); - assert(inst->src[2].type == BRW_TYPE_B || - inst->src[2].type == BRW_TYPE_UB); + assert(dpas->src[1].type == BRW_TYPE_B || + dpas->src[1].type == BRW_TYPE_UB); + assert(dpas->src[2].type == BRW_TYPE_B || + dpas->src[2].type == BRW_TYPE_UB); - const brw_reg_type src0_type = inst->dst.type; + const brw_reg_type src0_type = dpas->dst.type; - const brw_reg_type src1_type = inst->src[1].type == BRW_TYPE_UB + const brw_reg_type src1_type = dpas->src[1].type == BRW_TYPE_UB ? BRW_TYPE_UD : BRW_TYPE_D; - const brw_reg_type src2_type = inst->src[2].type == BRW_TYPE_UB + const brw_reg_type src2_type = dpas->src[2].type == BRW_TYPE_UB ? BRW_TYPE_UD : BRW_TYPE_D; - brw_reg dest = inst->dst; - brw_reg src0 = inst->src[0]; - const brw_reg src1 = retype(inst->src[1], src1_type); - const brw_reg src2 = retype(inst->src[2], src2_type); + brw_reg dest = dpas->dst; + brw_reg src0 = dpas->src[0]; + const brw_reg src1 = retype(dpas->src[1], src1_type); + const brw_reg src2 = retype(dpas->src[2], src2_type); const unsigned dest_stride = REG_SIZE; - for (unsigned r = 0; r < inst->rcount; r++) { + for (unsigned r = 0; r < dpas->rcount; r++) { if (!src0.is_null()) { bld.MOV(dest, src0); src0 = byte_offset(src0, dest_stride); @@ -196,13 +196,13 @@ int8_using_mul_add(const brw_builder &bld, brw_inst *inst) bld.MOV(dest, retype(brw_imm_d(0), dest.type)); } - for (unsigned s = 0; s < inst->sdepth; s++) { + for (unsigned s = 0; s < dpas->sdepth; s++) { brw_reg temp1 = bld.vgrf(BRW_TYPE_UD); brw_reg temp2 = bld.vgrf(BRW_TYPE_UD); brw_reg temp3 = bld.vgrf(BRW_TYPE_UD, 2); const brw_reg_type temp_type = - (inst->src[1].type == BRW_TYPE_B || - inst->src[2].type == BRW_TYPE_B) + (dpas->src[1].type == BRW_TYPE_B || + dpas->src[2].type == BRW_TYPE_B) ? BRW_TYPE_W : BRW_TYPE_UW; /* Expand 8 dwords of packed bytes into 16 dwords of packed @@ -214,12 +214,12 @@ int8_using_mul_add(const brw_builder &bld, brw_inst *inst) */ bld.group(32, 0).MOV(retype(temp3, temp_type), retype(byte_offset(src2, r * REG_SIZE), - inst->src[2].type)); + dpas->src[2].type)); bld.MUL(subscript(temp1, temp_type, 0), subscript(retype(byte_offset(src1, s * REG_SIZE), BRW_TYPE_UD), - inst->src[1].type, 0), + dpas->src[1].type, 0), subscript(component(retype(temp3, BRW_TYPE_UD), s * 2), temp_type, 0)); @@ -227,7 +227,7 @@ int8_using_mul_add(const brw_builder &bld, brw_inst *inst) bld.MUL(subscript(temp1, temp_type, 1), subscript(retype(byte_offset(src1, s * REG_SIZE), BRW_TYPE_UD), - inst->src[1].type, 1), + dpas->src[1].type, 1), subscript(component(retype(temp3, BRW_TYPE_UD), s * 2), temp_type, 1)); @@ -235,7 +235,7 @@ int8_using_mul_add(const brw_builder &bld, brw_inst *inst) bld.MUL(subscript(temp2, temp_type, 0), subscript(retype(byte_offset(src1, s * REG_SIZE), BRW_TYPE_UD), - inst->src[1].type, 2), + dpas->src[1].type, 2), subscript(component(retype(temp3, BRW_TYPE_UD), s * 2 + 1), temp_type, 0)); @@ -243,7 +243,7 @@ int8_using_mul_add(const brw_builder &bld, brw_inst *inst) bld.MUL(subscript(temp2, temp_type, 1), subscript(retype(byte_offset(src1, s * REG_SIZE), BRW_TYPE_UD), - inst->src[1].type, 3), + dpas->src[1].type, 3), subscript(component(retype(temp3, BRW_TYPE_UD), s * 2 + 1), temp_type, 1)); @@ -261,7 +261,7 @@ int8_using_mul_add(const brw_builder &bld, brw_inst *inst) retype(temp2, src0_type)); bld.ADD(dest, dest, retype(temp1, src0_type)) - ->saturate = inst->saturate; + ->saturate = dpas->saturate; } dest = byte_offset(dest, dest_stride); @@ -277,20 +277,21 @@ brw_lower_dpas(brw_shader &v) if (inst->opcode != BRW_OPCODE_DPAS) continue; + brw_dpas_inst *dpas = inst->as_dpas(); const unsigned exec_size = v.devinfo->ver >= 20 ? 16 : 8; - const brw_builder bld = brw_builder(inst).group(exec_size, 0).exec_all(); + const brw_builder bld = brw_builder(dpas).group(exec_size, 0).exec_all(); - if (brw_type_is_float(inst->dst.type)) { - f16_using_mac(bld, inst); + if (brw_type_is_float(dpas->dst.type)) { + f16_using_mac(bld, dpas); } else { if (v.devinfo->ver >= 12) { - int8_using_dp4a(bld, inst); + int8_using_dp4a(bld, dpas); } else { - int8_using_mul_add(bld, inst); + int8_using_mul_add(bld, dpas); } } - inst->remove(); + dpas->remove(); progress = true; } diff --git a/src/intel/compiler/brw_opt_cse.cpp b/src/intel/compiler/brw_opt_cse.cpp index b187b32a45e..38bf2b61890 100644 --- a/src/intel/compiler/brw_opt_cse.cpp +++ b/src/intel/compiler/brw_opt_cse.cpp @@ -273,6 +273,13 @@ mem_inst_match(brw_mem_inst *a, brw_mem_inst *b) a->address_offset == b->address_offset; } +static bool +dpas_inst_match(brw_dpas_inst *a, brw_dpas_inst *b) +{ + return a->sdepth == b->sdepth && + a->rcount == b->rcount; +} + static bool instructions_match(brw_inst *a, brw_inst *b, bool *negate) { @@ -282,6 +289,7 @@ instructions_match(brw_inst *a, brw_inst *b, bool *negate) (a->kind != BRW_KIND_SEND || send_inst_match(a->as_send(), b->as_send())) && (a->kind != BRW_KIND_TEX || tex_inst_match(a->as_tex(), b->as_tex())) && (a->kind != BRW_KIND_MEM || mem_inst_match(a->as_mem(), b->as_mem())) && + (a->kind != BRW_KIND_DPAS || dpas_inst_match(a->as_dpas(), b->as_dpas())) && a->exec_size == b->exec_size && a->group == b->group && a->predicate == b->predicate && @@ -397,6 +405,16 @@ hash_inst(const void *v) break; } + case BRW_KIND_DPAS: { + const brw_dpas_inst *dpas = inst->as_dpas(); + const uint8_t dpas_u8data[] = { + dpas->sdepth, + dpas->rcount, + }; + hash = HASH(hash, dpas_u8data); + break; + } + case BRW_KIND_BASE: /* Nothing else to do. */ break; diff --git a/src/intel/compiler/brw_reg_allocate.cpp b/src/intel/compiler/brw_reg_allocate.cpp index 49fa0e303c4..b23376c9713 100644 --- a/src/intel/compiler/brw_reg_allocate.cpp +++ b/src/intel/compiler/brw_reg_allocate.cpp @@ -504,7 +504,7 @@ brw_inst_has_source_and_destination_hazard(const struct intel_device_info *devin * There may be some advantage to properly modeling this, but for now, * be overly conservative. */ - return inst->rcount > 1; + return inst->as_dpas()->rcount > 1; default: /* The SIMD16 compressed instruction * diff --git a/src/intel/compiler/brw_schedule_instructions.cpp b/src/intel/compiler/brw_schedule_instructions.cpp index dfc6cd59fc9..8c025270bc4 100644 --- a/src/intel/compiler/brw_schedule_instructions.cpp +++ b/src/intel/compiler/brw_schedule_instructions.cpp @@ -560,7 +560,7 @@ schedule_node::set_latency(const struct brw_isa_info *isa) } case BRW_OPCODE_DPAS: - switch (inst->rcount) { + switch (inst->as_dpas()->rcount) { case 1: latency = 21; break;