agx: Add block_image_store instruction

This hw instruction writes out an entire block from the tilebuffer to an
attached render target (PBE descriptor). It is used (only?) in end-of-tile
shaders to implement write out. We need to handle it in the compiler as a
prerequisite to compiling end-of-tile shaders ourselves, instead of hardcoding.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/19871>

src/asahi/compiler/agx_compile.c

@@ -635,6 +635,49 @@ agx_emit_store_preamble(agx_builder *b, nir_intrinsic_instr *instr)
    return agx_uniform_store(b, value, offset);
 }
 
+static enum agx_dim
+agx_tex_dim(enum glsl_sampler_dim dim, bool array)
+{
+   switch (dim) {
+   case GLSL_SAMPLER_DIM_1D:
+   case GLSL_SAMPLER_DIM_BUF:
+      return array ? AGX_DIM_1D_ARRAY : AGX_DIM_1D;
+
+   case GLSL_SAMPLER_DIM_2D:
+   case GLSL_SAMPLER_DIM_RECT:
+   case GLSL_SAMPLER_DIM_EXTERNAL:
+      return array ? AGX_DIM_2D_ARRAY : AGX_DIM_2D;
+
+   case GLSL_SAMPLER_DIM_MS:
+      return array ? AGX_DIM_2D_MS_ARRAY : AGX_DIM_2D_MS;
+
+   case GLSL_SAMPLER_DIM_3D:
+      assert(!array && "3D arrays unsupported");
+      return AGX_DIM_3D;
+
+   case GLSL_SAMPLER_DIM_CUBE:
+      return array ? AGX_DIM_CUBE_ARRAY : AGX_DIM_CUBE;
+
+   default:
+      unreachable("Invalid sampler dim\n");
+   }
+}
+
+static agx_instr *
+agx_emit_block_image_store(agx_builder *b, nir_intrinsic_instr *instr)
+{
+   unsigned image = nir_src_as_uint(instr->src[0]);
+   agx_index offset = agx_src_index(&instr->src[1]);
+   enum agx_format format = agx_format_for_pipe(nir_intrinsic_format(instr));
+   enum agx_dim dim = agx_tex_dim(nir_intrinsic_image_dim(instr), false);
+
+   // XXX: how does this possibly work
+   if (format == AGX_FORMAT_F16)
+      format = AGX_FORMAT_I16;
+
+   return agx_block_image_store(b, agx_immediate(image), offset, format, dim);
+}
+
 /*
  * Emit code to generate gl_FragCoord. The xy components are calculated from
  * special registers, whereas the zw components are interpolated varyings.
@@ -768,6 +811,9 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
   case nir_intrinsic_store_preamble:
      return agx_emit_store_preamble(b, instr);
 
+  case nir_intrinsic_block_image_store_agx:
+     return agx_emit_block_image_store(b, instr);
+
   case nir_intrinsic_load_blend_const_color_r_float: return agx_blend_const(b, dst, 0);
   case nir_intrinsic_load_blend_const_color_g_float: return agx_blend_const(b, dst, 1);
   case nir_intrinsic_load_blend_const_color_b_float: return agx_blend_const(b, dst, 2);
@@ -1081,35 +1127,6 @@ agx_emit_alu(agx_builder *b, nir_alu_instr *instr)
    }
 }
 
-static enum agx_dim
-agx_tex_dim(enum glsl_sampler_dim dim, bool array)
-{
-   switch (dim) {
-   case GLSL_SAMPLER_DIM_1D:
-   case GLSL_SAMPLER_DIM_BUF:
-      return array ? AGX_DIM_1D_ARRAY : AGX_DIM_1D;
-
-   case GLSL_SAMPLER_DIM_2D:
-   case GLSL_SAMPLER_DIM_RECT:
-   case GLSL_SAMPLER_DIM_EXTERNAL:
-      return array ? AGX_DIM_2D_ARRAY : AGX_DIM_2D;
-
-   case GLSL_SAMPLER_DIM_MS:
-      assert(!array && "multisampled arrays unsupported");
-      return AGX_DIM_2D_MS;
-
-   case GLSL_SAMPLER_DIM_3D:
-      assert(!array && "3D arrays unsupported");
-      return AGX_DIM_3D;
-
-   case GLSL_SAMPLER_DIM_CUBE:
-      return array ? AGX_DIM_CUBE_ARRAY : AGX_DIM_CUBE;
-
-   default:
-      unreachable("Invalid sampler dim\n");
-   }
-}
-
 static enum agx_lod_mode
 agx_lod_mode_for_nir(nir_texop op)
 {

src/asahi/compiler/agx_opcodes.py

@@ -285,6 +285,11 @@ op("stop", (0x88, 0xFFFF, 2, _), dests = 0, can_eliminate = False)
 op("trap", (0x08, 0xFFFF, 2, _), dests = 0, can_eliminate = False)
 op("writeout", (0x48, 0xFF, 4, _), dests = 0, imms = [WRITEOUT], can_eliminate = False)
 
+# Sources are the image and the offset within shared memory
+# TODO: Do we need the short encoding?
+op("block_image_store", (0xB1, 0xFF, 10, _), dests = 0, srcs = 2,
+   imms = [FORMAT, DIM], can_eliminate = False)
+
 # Convenient aliases.
 op("mov", _, srcs = 1)
 op("not", _, srcs = 1)

src/asahi/compiler/agx_optimizer.c

@@ -180,6 +180,7 @@ agx_optimizer_copyprop(agx_instr **defs, agx_instr *I)
            I->op == AGX_OPCODE_PHI ||
            I->op == AGX_OPCODE_ST_TILE ||
            I->op == AGX_OPCODE_LD_TILE ||
+           I->op == AGX_OPCODE_BLOCK_IMAGE_STORE ||
            /*I->op == AGX_OPCODE_DEVICE_STORE ||*/
            I->op == AGX_OPCODE_UNIFORM_STORE ||
            I->op == AGX_OPCODE_ST_VARY))
@@ -216,7 +217,8 @@ agx_optimizer_forward(agx_context *ctx)
       /* Inline immediates if we can. TODO: systematic */
       if (I->op != AGX_OPCODE_ST_VARY && I->op != AGX_OPCODE_ST_TILE &&
           I->op != AGX_OPCODE_COLLECT && I->op != AGX_OPCODE_TEXTURE_SAMPLE &&
-          I->op != AGX_OPCODE_TEXTURE_LOAD && I->op != AGX_OPCODE_UNIFORM_STORE)
+          I->op != AGX_OPCODE_TEXTURE_LOAD && I->op != AGX_OPCODE_UNIFORM_STORE &&
+          I->op != AGX_OPCODE_BLOCK_IMAGE_STORE)
          agx_optimizer_inline_imm(defs, I, info.nr_srcs, info.is_float);
    }
 

src/asahi/compiler/agx_pack.c

@@ -660,6 +660,53 @@ agx_pack_instr(struct util_dynarray *emission, struct util_dynarray *fixups, agx
       break;
    }
 
+   case AGX_OPCODE_BLOCK_IMAGE_STORE:
+   {
+      enum agx_format F = I->format;
+      assert(F < 0x10);
+
+      unsigned Tt = 0;
+      assert(Tt < 0x4);
+
+      unsigned T = agx_pack_texture(I->src[0], &Tt);
+      assert(T < 0x100);
+
+      agx_index offset = I->src[1];
+      assert(offset.type == AGX_INDEX_REGISTER);
+      assert(offset.size == AGX_SIZE_16);
+      unsigned R = offset.value;
+
+      assert(I->dim == AGX_DIM_2D || I->dim == AGX_DIM_2D_MS);
+      bool msaa = (I->dim == AGX_DIM_2D_MS);
+
+      bool unk1 = true;
+      unsigned unk2 = msaa ? 38 : 37; /* XXX */
+      unsigned unk3 = 1;
+
+      uint32_t word0 =
+         agx_opcodes_info[I->op].encoding.exact |
+         (1 << 15) /* we always set length bit for now */ |
+         ((F & 1) << 8) |
+         ((R & BITFIELD_MASK(6)) << 9) |
+         (unk1 ? (1 << 31) : 0);
+
+      uint32_t word1 =
+         (T & BITFIELD_MASK(6)) |
+         (Tt << 2) |
+         (unk2 << 9) |
+         ((R >> 6) << 28);
+
+      uint32_t word2 =
+         (F >> 1) |
+         (unk3 ? (1 << 3) : 0) |
+         ((T >> 6) << 14);
+
+      memcpy(util_dynarray_grow_bytes(emission, 1, 4), &word0, 4);
+      memcpy(util_dynarray_grow_bytes(emission, 1, 4), &word1, 4);
+      memcpy(util_dynarray_grow_bytes(emission, 1, 2), &word2, 2);
+      break;
+   }
+
    case AGX_OPCODE_JMP_EXEC_ANY:
    case AGX_OPCODE_JMP_EXEC_NONE:
    {