From c1d5c318bc07665566582a7611fe976d18ccd8e5 Mon Sep 17 00:00:00 2001 From: Danylo Piliaiev Date: Wed, 24 Nov 2021 14:57:03 +0200 Subject: [PATCH] ir3: New cat3 instructions * shrm - (src2 >> src1) & src3 * shlm - (src2 << src1) & src3 * shrg - (src2 >> src1) | src3 * shlg - (src2 << src1) | src3 * andg - (src2 & src1) | src3 * dp2acc - dot product of two {i,u}8vec2 packed into SRC1 and SRC2, added to 32b SRC3 * dp4acc - dot product of two {i,u}8vec4 packed into SRC1 and SRC2, added to 32b SRC3 * wmm - vec4(x_1, x_2, x_3, x_4) * (y_1 + y_2 + y_3 + y_4), which is duplicated (1 << (SRC3 / 32)) times starting from DST register * wmm.accu - same as wmm but result is added to DST registers, however the first reg in each vec4 result is overwritten instead of accumulating. Signed-off-by: Danylo Piliaiev Part-of: --- src/freedreno/ir3/disasm-a3xx.c | 10 +- src/freedreno/ir3/instr-a3xx.h | 10 +- src/freedreno/ir3/ir3.c | 23 ++- src/freedreno/ir3/ir3.h | 27 +++- src/freedreno/ir3/ir3_lexer.l | 15 +- src/freedreno/ir3/ir3_parser.y | 40 ++++- src/freedreno/ir3/tests/disasm.c | 17 +- src/freedreno/isa/ir3-cat3.xml | 263 ++++++++++++++++++++++++++----- 8 files changed, 352 insertions(+), 53 deletions(-) diff --git a/src/freedreno/ir3/disasm-a3xx.c b/src/freedreno/ir3/disasm-a3xx.c index 362db68b763..c0feeb13730 100644 --- a/src/freedreno/ir3/disasm-a3xx.c +++ b/src/freedreno/ir3/disasm-a3xx.c @@ -258,7 +258,15 @@ static const struct opc_info { OPC(3, OPC_SEL_F32, sel.f32), OPC(3, OPC_SAD_S16, sad.s16), OPC(3, OPC_SAD_S32, sad.s32), - OPC(3, OPC_SHLG_B16, shlg.b16), + OPC(3, OPC_SHRM, shrm), + OPC(3, OPC_SHLM, shlm), + OPC(3, OPC_SHRG, shrg), + OPC(3, OPC_SHLG, shlg), + OPC(3, OPC_ANDG, andg), + OPC(3, OPC_DP2ACC, dp2acc), + OPC(3, OPC_DP4ACC, dp4acc), + OPC(3, OPC_WMM, wmm), + OPC(3, OPC_WMM_ACCU, wmm.accu), /* category 4: */ OPC(4, OPC_RCP, rcp), diff --git a/src/freedreno/ir3/instr-a3xx.h b/src/freedreno/ir3/instr-a3xx.h index cffe78b1442..069324845ae 100644 --- a/src/freedreno/ir3/instr-a3xx.h +++ b/src/freedreno/ir3/instr-a3xx.h @@ -198,7 +198,15 @@ typedef enum { OPC_SEL_F32 = _OPC(3, 13), OPC_SAD_S16 = _OPC(3, 14), OPC_SAD_S32 = _OPC(3, 15), - OPC_SHLG_B16 = _OPC(3, 16), + OPC_SHRM = _OPC(3, 16), + OPC_SHLM = _OPC(3, 17), + OPC_SHRG = _OPC(3, 18), + OPC_SHLG = _OPC(3, 19), + OPC_ANDG = _OPC(3, 20), + OPC_DP2ACC = _OPC(3, 21), + OPC_DP4ACC = _OPC(3, 22), + OPC_WMM = _OPC(3, 23), + OPC_WMM_ACCU = _OPC(3, 24), /* category 4: */ OPC_RCP = _OPC(4, 0), diff --git a/src/freedreno/ir3/ir3.c b/src/freedreno/ir3/ir3.c index 7d7a07eaa4b..17fd4c2aa4b 100644 --- a/src/freedreno/ir3/ir3.c +++ b/src/freedreno/ir3/ir3.c @@ -922,12 +922,29 @@ ir3_valid_flags(struct ir3_instruction *instr, unsigned n, unsigned flags) valid_flags = ir3_cat3_absneg(instr->opc) | IR3_REG_RELATIV | IR3_REG_SHARED; - if (instr->opc == OPC_SHLG_B16) { + switch (instr->opc) { + case OPC_SHRM: + case OPC_SHLM: + case OPC_SHRG: + case OPC_SHLG: + case OPC_ANDG: { valid_flags |= IR3_REG_IMMED; - /* shlg.b16 can be RELATIV+CONST but not CONST: */ + /* Can be RELATIV+CONST but not CONST: */ if (flags & IR3_REG_RELATIV) valid_flags |= IR3_REG_CONST; - } else { + break; + } + case OPC_WMM: + case OPC_WMM_ACCU: { + valid_flags = IR3_REG_SHARED; + if (n == 2) + valid_flags = IR3_REG_CONST; + break; + } + case OPC_DP2ACC: + case OPC_DP4ACC: + break; + default: valid_flags |= IR3_REG_CONST; } diff --git a/src/freedreno/ir3/ir3.h b/src/freedreno/ir3/ir3.h index 932f85a94a5..5ef1eaeeeaf 100644 --- a/src/freedreno/ir3/ir3.h +++ b/src/freedreno/ir3/ir3.h @@ -329,6 +329,16 @@ struct ir3_instruction { IR3_COND_NE = 5, } condition; } cat2; + struct { + enum { + IR3_SRC_UNSIGNED = 0, + IR3_SRC_MIXED = 1, + } signedness; + enum { + IR3_SRC_PACKED_LOW = 0, + IR3_SRC_PACKED_HIGH = 1, + } packed; + } cat3; struct { unsigned samp, tex; unsigned tex_base : 3; @@ -1383,7 +1393,13 @@ ir3_cat3_absneg(opc_t opc) case OPC_SEL_B16: case OPC_SEL_B32: - case OPC_SHLG_B16: + case OPC_SHRM: + case OPC_SHLM: + case OPC_SHRG: + case OPC_SHLG: + case OPC_ANDG: + case OPC_WMM: + case OPC_WMM_ACCU: default: return 0; @@ -1407,6 +1423,8 @@ ir3_output_conv_type(struct ir3_instruction *instr, bool *can_fold) case OPC_BARY_F: case OPC_MAD_F32: case OPC_MAD_F16: + case OPC_WMM: + case OPC_WMM_ACCU: return TYPE_F32; case OPC_ADD_U: @@ -1423,6 +1441,11 @@ ir3_output_conv_type(struct ir3_instruction *instr, bool *can_fold) case OPC_SHR_B: case OPC_ASHR_B: case OPC_MAD_U24: + case OPC_SHRM: + case OPC_SHLM: + case OPC_SHRG: + case OPC_SHLG: + case OPC_ANDG: /* Comparison ops zero-extend/truncate their results, so consider them as * unsigned here. */ @@ -2213,6 +2236,8 @@ INSTR3(MAD_U24) INSTR3(MAD_S24) INSTR3(MAD_F16) INSTR3(MAD_F32) +INSTR3(DP2ACC) +INSTR3(DP4ACC) /* NOTE: SEL_B32 checks for zero vs nonzero */ INSTR3(SEL_B16) INSTR3(SEL_B32) diff --git a/src/freedreno/ir3/ir3_lexer.l b/src/freedreno/ir3/ir3_lexer.l index 1a496e2a791..04bff6ab4af 100644 --- a/src/freedreno/ir3/ir3_lexer.l +++ b/src/freedreno/ir3/ir3_lexer.l @@ -244,7 +244,15 @@ static int parse_reg(const char *str) "sel.f32" return TOKEN(T_OP_SEL_F32); "sad.s16" return TOKEN(T_OP_SAD_S16); "sad.s32" return TOKEN(T_OP_SAD_S32); -"shlg.b16" return TOKEN(T_OP_SHLG_B16); +"shrm" return TOKEN(T_OP_SHRM); +"shlm" return TOKEN(T_OP_SHLM); +"shrg" return TOKEN(T_OP_SHRG); +"shlg" return TOKEN(T_OP_SHLG); +"andg" return TOKEN(T_OP_ANDG); +"dp2acc" return TOKEN(T_OP_DP2ACC); +"dp4acc" return TOKEN(T_OP_DP4ACC); +"wmm" return TOKEN(T_OP_WMM); +"wmm.accu" return TOKEN(T_OP_WMM_ACCU); /* category 4: */ "rcp" return TOKEN(T_OP_RCP); @@ -383,6 +391,11 @@ static int parse_reg(const char *str) "untyped" return TOKEN(T_UNTYPED); "typed" return TOKEN(T_TYPED); +"unsigned" return TOKEN(T_UNSIGNED); +"mixed" return TOKEN(T_MIXED); +"low" return TOKEN(T_LOW); +"high" return TOKEN(T_HIGH); + "1d" return TOKEN(T_1D); "2d" return TOKEN(T_2D); "3d" return TOKEN(T_3D); diff --git a/src/freedreno/ir3/ir3_parser.y b/src/freedreno/ir3/ir3_parser.y index a2469dfbfec..5c5d9ea5d21 100644 --- a/src/freedreno/ir3/ir3_parser.y +++ b/src/freedreno/ir3/ir3_parser.y @@ -484,7 +484,15 @@ static void print_token(FILE *file, int type, YYSTYPE value) %token T_OP_SEL_F32 %token T_OP_SAD_S16 %token T_OP_SAD_S32 -%token T_OP_SHLG_B16 +%token T_OP_SHRM +%token T_OP_SHLM +%token T_OP_SHRG +%token T_OP_SHLG +%token T_OP_ANDG +%token T_OP_DP2ACC +%token T_OP_DP4ACC +%token T_OP_WMM +%token T_OP_WMM_ACCU /* category 4: */ %token T_OP_RCP @@ -623,6 +631,11 @@ static void print_token(FILE *file, int type, YYSTYPE value) %token T_UNTYPED %token T_TYPED +%token T_MIXED +%token T_UNSIGNED +%token T_LOW +%token T_HIGH + %token T_1D %token T_2D %token T_3D @@ -949,6 +962,12 @@ cat2_instr: cat2_opc_1src dst_reg ',' src_reg_or_const_or_rel_or_imm | cat2_opc_2src_cnd '.' cond dst_reg ',' src_reg_or_const_or_rel_or_imm ',' src_reg_or_const_or_rel_or_imm | cat2_opc_2src dst_reg ',' src_reg_or_const_or_rel_or_imm ',' src_reg_or_const_or_rel_or_imm +cat3_dp_signedness:'.' T_MIXED { instr->cat3.signedness = IR3_SRC_MIXED; } +| '.' T_UNSIGNED{ instr->cat3.signedness = IR3_SRC_UNSIGNED; } + +cat3_dp_pack: '.' T_LOW { instr->cat3.packed = IR3_SRC_PACKED_LOW; } +| '.' T_HIGH { instr->cat3.packed = IR3_SRC_PACKED_HIGH; } + cat3_opc: T_OP_MAD_U16 { new_instr(OPC_MAD_U16); } | T_OP_MADSH_U16 { new_instr(OPC_MADSH_U16); } | T_OP_MAD_S16 { new_instr(OPC_MAD_S16); } @@ -966,8 +985,22 @@ cat3_opc: T_OP_MAD_U16 { new_instr(OPC_MAD_U16); } | T_OP_SAD_S16 { new_instr(OPC_SAD_S16); } | T_OP_SAD_S32 { new_instr(OPC_SAD_S32); } +cat3_imm_reg_opc: T_OP_SHRM { new_instr(OPC_SHRM); } +| T_OP_SHLM { new_instr(OPC_SHLM); } +| T_OP_SHRG { new_instr(OPC_SHRG); } +| T_OP_SHLG { new_instr(OPC_SHLG); } +| T_OP_ANDG { new_instr(OPC_ANDG); } + +cat3_wmm: T_OP_WMM { new_instr(OPC_WMM); } +| T_OP_WMM_ACCU { new_instr(OPC_WMM_ACCU); } + +cat3_dp: T_OP_DP2ACC { new_instr(OPC_DP2ACC); } +| T_OP_DP4ACC { new_instr(OPC_DP4ACC); } + cat3_instr: cat3_opc dst_reg ',' src_reg_or_const_or_rel ',' src_reg_or_const ',' src_reg_or_const_or_rel -| T_OP_SHLG_B16 { new_instr(OPC_SHLG_B16); } dst_reg ',' src_reg_or_rel_or_imm ',' src_reg_or_const ',' src_reg_or_rel_or_imm +| cat3_imm_reg_opc dst_reg ',' src_reg_or_rel_or_imm ',' src_reg_or_const ',' src_reg_or_rel_or_imm +| cat3_wmm dst_reg ',' src_reg_gpr ',' src_reg ',' immediate +| cat3_dp cat3_dp_signedness cat3_dp_pack dst_reg ',' src_reg_or_rel_or_imm ',' src_reg_or_const ',' src_reg_or_rel_or_imm cat4_opc: T_OP_RCP { new_instr(OPC_RCP); } | T_OP_RSQ { new_instr(OPC_RSQ); } @@ -1267,6 +1300,9 @@ src_reg_flags: src_reg_flag src_reg: src | src_reg_flags src +src_reg_gpr: src_reg +| relative_gpr_src + src_const: const | src_reg_flags const diff --git a/src/freedreno/ir3/tests/disasm.c b/src/freedreno/ir3/tests/disasm.c index 325cd70d4fa..ee1544c3b42 100644 --- a/src/freedreno/ir3/tests/disasm.c +++ b/src/freedreno/ir3/tests/disasm.c @@ -127,9 +127,20 @@ static const struct test { INSTR_6XX(63820005_10315030, "mad.f32 r1.y, (neg)c12.x, r1.x, c12.y"), INSTR_6XX(62050009_00091000, "mad.u24 r2.y, c0.x, r2.z, r2.y"), INSTR_6XX(61828008_00081033, "madsh.m16 r2.x, c12.w, r1.y, r2.x"), - INSTR_6XX(65900820_100cb008, "(nop3) shlg.b16 hr8.x, 8, hr8.x, 12"), /* (nop3) shlg.b16 hr8.x, (r)8, (r)hr8.x, 12; */ - INSTR_6XX(65ae085c_0002a001, "(nop3) shlg.b16 hr23.x, hr0.y, hr23.x, hr0.z"), /* not seen in blob */ - INSTR_6XX(65900820_0c0aac05, "(nop3) shlg.b16 hr8.x, hc, hr8.x, hc"), /* not seen in blob */ + INSTR_6XX(65900820_100cb008, "(nop3) shlg hr8.x, 8, hr8.x, 12"), /* (nop3) shlg.b16 hr8.x, (r)8, (r)hr8.x, 12; */ + INSTR_6XX(65ae085c_0002a001, "(nop3) shlg hr23.x, hr0.y, hr23.x, hr0.z"), /* not seen in blob */ + INSTR_6XX(65900820_0c0aac05, "(nop3) shlg hr8.x, hc, hr8.x, hc"), /* not seen in blob */ + INSTR_6XX(65ae0c5c_0002a001, "(nop3) shlg r23.x, r0.y, r23.x, r0.z"), /* (nop3) shlg.b32 r23.x, (r)r0.y, (r)r23.x, r0.z */ + INSTR_6XX(64018802_0002e003, "(nop3) shrm hr0.z, (neg)hr0.w, hr0.w, hr0.z"), + INSTR_6XX(64818802_0002e003, "(nop3) shlm hr0.z, (neg)hr0.w, hr0.w, hr0.z"), + INSTR_6XX(65018802_0002e003, "(nop3) shrg hr0.z, (neg)hr0.w, hr0.w, hr0.z"), + INSTR_6XX(66018802_0002e003, "(nop3) andg hr0.z, (neg)hr0.w, hr0.w, hr0.z"), + INSTR_6XX(67018802_1002e003, "(nop3) wmm hr0.z, (neg)hr0.w, hr0.w, 2"), /* (nop3) wmm.f16f16 hr0.z, (abs)(r)hr0.w, (r)hr0.w, 2 */ + INSTR_6XX(67018c02_1002e003, "(nop3) wmm.accu hr0.z, (neg)hr0.w, hr0.w, 2"), + INSTR_6XX(6701c802_9002a003, "(nop3) wmm r0.z, r0.w, r0.w, 2"), /* (nop3) wmm.f32f32 r0.z, (r)r0.w, (r)r0.w, 2 */ + /* custom test with qcom_dot8 function from cl_qcom_dot_product8 */ + INSTR_6XX(66818c02_0002e003, "(sat)(nop3) dp2acc.mixed.low r0.z, r0.w, r0.w, r0.z"), /* (nop3) dp2acc (sat)r0.z, (signed)(low)(r)r0.w, (low)(r)r0.w, r0.z */ + INSTR_6XX(6681c802_8002a003, "(nop3) dp4acc.unsigned.low r0.z, r0.w, r0.w, (neg)r0.z"), /* (nop3) dp4acc r0.z, (unsigned)(r)r0.w, (r)r0.w, (neg)r0.z */ /* cat4 */ INSTR_6XX(8010000a_00000003, "rcp r2.z, r0.w"), diff --git a/src/freedreno/isa/ir3-cat3.xml b/src/freedreno/isa/ir3-cat3.xml index 18491e2a1af..d3bb351ee3b 100644 --- a/src/freedreno/isa/ir3-cat3.xml +++ b/src/freedreno/isa/ir3-cat3.xml @@ -106,24 +106,13 @@ SOFTWARE. {SY}{SS}{JP}{SAT}{REPEAT}{UL}{NAME} {DST_HALF}{DST}, {SRC1_NEG}{SRC1_R}{SRC1}, {SRC2_NEG}{SRC2_R}{HALF}{SRC2}, {SRC3_NEG}{SRC3_R}{SRC3} - - - - - - - The source precision is determined by the instruction - opcode. If {DST_CONV} the result is widened/narrowed - to the opposite precision. - - @@ -139,6 +128,38 @@ SOFTWARE. !!(src->srcs[1]->flags & (IR3_REG_FNEG | IR3_REG_SNEG | IR3_REG_BNOT)) !!(src->srcs[2]->flags & (IR3_REG_FNEG | IR3_REG_SNEG | IR3_REG_BNOT)) src->srcs[0] + + + + + + + + + + + + + 0 + + + + + + + + + + + + + The source precision is determined by the instruction + opcode. If {DST_CONV} the result is widened/narrowed + to the opposite precision. + + + + ((src->dsts[0]->num >> 2) == 62) ? 0 : !!((src->srcs[0]->flags ^ src->dsts[0]->flags) & IR3_REG_HALF) @@ -146,21 +167,7 @@ SOFTWARE. - - 0 - - - - - - - - - - - - - + The difference is that this cat3 version does not support plain @@ -168,38 +175,39 @@ SOFTWARE. On the other hand it still supports relative gpr and consts. - 1 + + + + + 1 + + + + + + src->srcs[2] - false + !(src->srcs[1]->flags & IR3_REG_HALF) + + ((src->dsts[0]->num >> 2) == 62) ? 0 : + !!((src->srcs[1]->flags ^ src->dsts[0]->flags) & IR3_REG_HALF) + - - - - (src2 << src1) | src3 - - - - - - 1011 - - - 0000 @@ -280,4 +288,177 @@ SOFTWARE. + + + (src2 >> src1) & src3 + + + 1000 + + + + + (src2 << src1) & src3 + + + 1001 + + + + + (src2 >> src1) | src3 + + + 1010 + + + + + (src2 << src1) | src3 + + + 1011 + + + + + (src2 & src1) | src3 + + + 1100 + + + + + + + + + + + + + + + + + {SY}{SS}{JP}{SAT}(nop{NOP}) {UL}{NAME}{SRC_SIGN}{SRC_PACK} {DST}, {SRC1}, {SRC2}, {SRC3_NEG}{SRC3} + + + + + + + + + + 1 + + + + + + + + + + src->srcs[2] + src->cat3.signedness + src->cat3.packed + + + + + + Given: + SRC1 is a i8vec2 or u8vec2 + SRC2 is a u8vec2 + SRC1 and SRC2 are packed into low or high halves of the registers. + SRC3 is a int32_t or uint32_t + Do: + DST = dot(SRC1, SRC2) + SRC3 + + + 0 + 1101 + + + + + Same a dp2acc but for vec4 instead of vec2. + Corresponds to packed variantes of OpUDotKHR and OpSUDotKHR. + + + 1 + 1101 + + + + (!{DST_FULL}) + + + + + + + + + + + + + + + 1 + + + + + + + + + + + + + src->srcs[2] + !(src->srcs[0]->flags & IR3_REG_HALF) + + ((src->dsts[0]->num >> 2) == 62) ? 1 : + !(src->dsts[0]->flags & IR3_REG_HALF) + + + + + + + Given: + SRC1 = (x_1, x_2, x_3, x_4) - 4 consecutive registers + SRC2 = (y_1, y_2, y_3, y_4) - 4 consecutive registers + SRC3 is an immediate in range of [0, 160] + + Do: + float y_sum = y_1 + y_2 + y_3 + y_4 + vec4 result = (x_1 * y_sum, x_2 * y_sum, x_3 * y_sum, x_4 * y_sum) + + Starting from DST reg duplicate *result* into consecutive registers + (1 << (SRC3 / 32)) times. + + + 0 + 1110 + + + + + Same as wmm but instead of overwriting DST - the result is + added to DST registers, however the first reg of the result + is always overwritten. + + + 1 + 1110 + +