panfrost: implement 16-bit pack/unpack intrinsics

This significantly improves codegen for the 16-bit ldexp2 lowering.

Signed-off-by: Benjamin Lee <benjamin.lee@collabora.com>
Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/33781>

src/panfrost/compiler/bifrost_compile.c

@@ -2607,6 +2607,23 @@ bi_emit_alu(bi_builder *b, nir_alu_instr *instr)
       return;
    }
 
+   case nir_op_unpack_32_2x16_split_x:
+   case nir_op_unpack_32_2x16_split_y: {
+      assert(comps <= 2);
+
+      bi_index idx = bi_src_index(&instr->src[0].src);
+      bi_index srcs[2] = {idx, idx};
+
+      unsigned offset = instr->op == nir_op_unpack_32_2x16_split_x ? 0 : 1;
+      unsigned channels[2] = {
+         comps > 0 ? instr->src[0].swizzle[0] * 2 + offset : 0,
+         comps > 1 ? instr->src[0].swizzle[1] * 2 + offset : 0,
+      };
+
+      bi_make_vec_to(b, dst, srcs, channels, comps, 16);
+      return;
+   }
+
    case nir_op_unpack_64_2x32_split_x: {
       unsigned chan = (instr->src[0].swizzle[0] * 2) + 0;
       bi_mov_i32_to(b, dst,
@@ -2691,6 +2708,18 @@ bi_emit_alu(bi_builder *b, nir_alu_instr *instr)
       return;
    }
 
+   case nir_op_pack_32_2x16_split: {
+      assert(comps == 1);
+
+      bi_index srcs[2] = {bi_src_index(&instr->src[0].src),
+                          bi_src_index(&instr->src[1].src)};
+      unsigned channels[2] = {instr->src[0].swizzle[0],
+                              instr->src[1].swizzle[0]};
+
+      bi_make_vec_to(b, dst, srcs, channels, 2, 16);
+      return;
+   }
+
    case nir_op_f2f16:
    case nir_op_f2f16_rtz:
    case nir_op_f2f16_rtne: {
@@ -2989,6 +3018,23 @@ bi_emit_alu(bi_builder *b, nir_alu_instr *instr)
       break;
    }
 
+   case nir_op_pack_half_2x16_split:
+      /* On v11+, V2F32_TO_V2F16 is gone. This should be lowered in
+       * bifrost_nir_lower_algebraic_late. */
+      assert(b->shader->arch < 11);
+
+      bi_v2f32_to_v2f16_to(b, dst, s0, s1);
+      break;
+
+   case nir_op_unpack_half_2x16_split_x:
+      assert(comps == 1);
+      bi_f16_to_f32_to(b, dst, bi_half(s0, false));
+      break;
+   case nir_op_unpack_half_2x16_split_y:
+      assert(comps == 1);
+      bi_f16_to_f32_to(b, dst, bi_half(s0, true));
+      break;
+
    case nir_op_ishl:
       bi_lshift_or_to(b, sz, dst, s0, bi_zero(), bi_byte(s1, 0));
       break;

src/panfrost/compiler/bifrost_compile.h

@@ -120,7 +120,6 @@ void bifrost_compile_shader_nir(nir_shader *nir,
       .lower_unpack_snorm_2x16 = true,                                         \
       .lower_unpack_unorm_4x8 = true,                                          \
       .lower_unpack_snorm_4x8 = true,                                          \
-      .lower_pack_split = true,                                                \
                                                                                \
       .lower_doubles_options =                                                 \
          nir_lower_dmod, /* TODO: Don't lower supported 64-bit operations */   \

src/panfrost/compiler/bifrost_nir_algebraic.py

@@ -94,6 +94,9 @@ algebraic_late = [
     (('f2i16', 'a@16'), ('f2i16', ('f2f32', a)), 'gpu_arch >= 11'),
     (('f2u16', 'a@16'), ('f2u16', ('f2f32', a)), 'gpu_arch >= 11'),
 
+    # On v11+, V2F32_TO_V2S16 is gone
+    (('pack_half_2x16_split', a, b), ('pack_32_2x16_split', ('f2f16', a), ('f2f16', b)), 'gpu_arch >= 11'),
+
     # On v11+, F16_TO_S32/F16_TO_U32 is gone but we still have F32_TO_S32/F32_TO_U32
     (('f2i32', 'a@16'), ('f2i32', ('f2f32', a)), 'gpu_arch >= 11'),
     (('f2u32', 'a@16'), ('f2u32', ('f2f32', a)), 'gpu_arch >= 11'),