ir3: Implement sparse residency check

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/32671>

src/freedreno/ir3/ir3_a6xx.c

@@ -290,15 +290,19 @@ emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr,
                           struct ir3_instruction **dst)
 {
    struct ir3_builder *b = &ctx->build;
-   struct ir3_instruction *ldib;
+   struct ir3_instruction *ldib, *rck;
    struct ir3_instruction *const *coords = ir3_get_src(ctx, &intr->src[1]);
    unsigned ncoords = ir3_get_image_coords(intr, NULL);
+   unsigned num_components = intr->num_components;
+   if (intr->intrinsic == nir_intrinsic_image_sparse_load ||
+       intr->intrinsic == nir_intrinsic_bindless_image_sparse_load)
+      num_components--;
 
    ldib = ir3_LDIB(b, ir3_image_to_ibo(ctx, intr->src[0]), 0,
                    ir3_create_collect(b, coords, ncoords), 0,
                    create_immed(b, 0), 0);
-   ldib->dsts[0]->wrmask = MASK(intr->num_components);
-   ldib->cat6.iim_val = intr->num_components;
+   ldib->dsts[0]->wrmask = MASK(num_components);
+   ldib->cat6.iim_val = num_components;
    ldib->cat6.d = ncoords;
    ldib->cat6.type = ir3_get_type_for_image_intrinsic(intr);
    ldib->cat6.typed = true;
@@ -307,6 +311,23 @@ emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr,
    ir3_handle_bindless_cat6(ldib, intr->src[0]);
    ir3_handle_nonuniform(ldib, intr);
 
+   if (intr->intrinsic == nir_intrinsic_image_sparse_load ||
+       intr->intrinsic == nir_intrinsic_bindless_image_sparse_load) {
+      rck = ir3_LDIB(b, ir3_image_to_ibo(ctx, intr->src[0]), 0,
+                      ir3_create_collect(b, coords, ncoords), 0,
+                      create_immed(b, 0), 0);
+      rck->dsts[0]->wrmask = 0b1;
+      rck->cat6.iim_val = 1;
+      rck->cat6.d = ncoords;
+      rck->cat6.type = TYPE_U32;
+      rck->cat6.typed = true;
+      rck->flags |= IR3_INSTR_RCK;
+      ir3_handle_bindless_cat6(rck, intr->src[0]);
+      ir3_handle_nonuniform(rck, intr);
+
+      dst[num_components] = rck;
+   }
+
    ir3_split_dest(b, dst, ldib, 0, intr->num_components);
 }
 

src/freedreno/ir3/ir3_compiler_nir.c

@@ -1767,7 +1767,7 @@ emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr,
 
    struct ir3_builder *b = &ctx->build;
    struct tex_src_info info = get_image_ssbo_samp_tex_src(ctx, &intr->src[0], true);
-   struct ir3_instruction *sam;
+   struct ir3_instruction *sam, *rck;
    struct ir3_instruction *const *src0 = ir3_get_src(ctx, &intr->src[1]);
    struct ir3_instruction *coords[4];
    unsigned flags, ncoords = ir3_get_image_coords(intr, &flags);
@@ -1799,6 +1799,17 @@ emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr,
    sam->barrier_conflict = IR3_BARRIER_IMAGE_W;
 
    ir3_split_dest(b, dst, sam, 0, 4);
+
+   if (intr->intrinsic == nir_intrinsic_image_sparse_load ||
+       intr->intrinsic == nir_intrinsic_bindless_image_sparse_load) {
+      /* The results of the residency check cannot change within the shader, so
+       * it doesn't need any barriers.
+       */
+      rck = emit_sam(ctx, OPC_ISAM, info, type, 0b1,
+                     ir3_create_collect(b, coords, ncoords), NULL);
+      rck->flags |= IR3_INSTR_RCK;
+      dst[4] = rck;
+   }
 }
 
 /* A4xx version of image_size, see ir3_a6xx.c for newer resinfo version. */
@@ -2901,6 +2912,8 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr)
       break;
    case nir_intrinsic_image_load:
    case nir_intrinsic_bindless_image_load:
+   case nir_intrinsic_image_sparse_load:
+   case nir_intrinsic_bindless_image_sparse_load:
       emit_intrinsic_load_image(ctx, intr, dst);
       break;
    case nir_intrinsic_image_store:
@@ -3604,6 +3617,12 @@ emit_tex(struct ir3_context *ctx, nir_tex_instr *tex)
 
    dst = ir3_get_def(ctx, &tex->def, ncomp);
 
+   /* For sparse residency check, the last component is a residency code that is
+    * emitted separately.
+    */
+   if (tex->is_sparse)
+      ncomp--;
+
    for (unsigned i = 0; i < tex->num_srcs; i++) {
       switch (tex->src[i].src_type) {
       case nir_tex_src_coord:
@@ -3921,6 +3940,14 @@ emit_tex(struct ir3_context *ctx, nir_tex_instr *tex)
       sam = emit_sam(ctx, opc, info, type, MASK(ncomp), col0, col1);
    }
 
+   if (tex->is_sparse) {
+      info.flags |= flags;
+      struct ir3_instruction *rck =
+         emit_sam(ctx, opc, info, TYPE_U32, MASK(1), col0, col1);
+      rck->flags |= IR3_INSTR_RCK;
+      dst[ncomp] = rck;
+   }
+
    if (tg4_swizzle_fixup) {
       /* TODO: fix-up for ASTC when alpha is selected? */
       array_insert(ctx->ir, ctx->ir->tg4, sam);

src/freedreno/ir3/ir3_image.c

@@ -102,7 +102,9 @@ ir3_get_type_for_image_intrinsic(const nir_intrinsic_instr *instr)
    nir_alu_type type = nir_type_uint;
    switch (instr->intrinsic) {
    case nir_intrinsic_image_load:
+   case nir_intrinsic_image_sparse_load:
    case nir_intrinsic_bindless_image_load:
+   case nir_intrinsic_bindless_image_sparse_load:
       type = nir_alu_type_get_base_type(nir_intrinsic_dest_type(instr));
       /* SpvOpAtomicLoad doesn't have dest type */
       if (type == nir_type_invalid)

src/freedreno/ir3/ir3_nir.c

@@ -596,6 +596,36 @@ ir3_nir_lower_shader_clock(nir_shader *shader, uint64_t uche_trap_base)
                                      nir_metadata_none, &uche_trap_base);
 }
 
+static bool
+ir3_nir_lower_sparse_residency_cb(nir_builder *b, nir_intrinsic_instr *instr,
+                                  void *data)
+{
+   b->cursor = nir_before_instr(&instr->instr);
+
+   nir_def *def;
+   switch (instr->intrinsic) {
+   case nir_intrinsic_is_sparse_texels_resident:
+      def = nir_ieq_imm(b, instr->src[0].ssa, 0);
+      break;
+   case nir_intrinsic_sparse_residency_code_and:
+      def = nir_ior(b, instr->src[0].ssa, instr->src[1].ssa);
+      break;
+   default:
+      return false;
+   }
+
+   nir_def_rewrite_uses(&instr->def, def);
+   return true;
+}
+
+static bool
+ir3_nir_lower_sparse_residency(nir_shader *shader)
+{
+   return nir_shader_intrinsics_pass(
+      shader, ir3_nir_lower_sparse_residency_cb,
+      nir_metadata_control_flow, NULL);
+}
+
 void
 ir3_finalize_nir(struct ir3_compiler *compiler,
                  const struct ir3_shader_nir_options *options,
@@ -635,6 +665,8 @@ ir3_finalize_nir(struct ir3_compiler *compiler,
    OPT(s, nir_lower_tex, &tex_options);
    OPT(s, nir_lower_load_const_to_scalar);
 
+   NIR_PASS(_, s, ir3_nir_lower_sparse_residency);
+
    if (compiler->array_index_add_half)
       OPT(s, ir3_nir_lower_array_sampler);
 

src/freedreno/ir3/ir3_nir_lower_tex_prefetch.c

@@ -178,7 +178,8 @@ lower_tex_prefetch_block(nir_block *block, ir3_prefetch_state *state)
          continue;
 
       /* only prefetch for simple 2d tex fetch case */
-      if (tex->sampler_dim != GLSL_SAMPLER_DIM_2D || tex->is_array)
+      if (tex->sampler_dim != GLSL_SAMPLER_DIM_2D || tex->is_array ||
+          tex->is_sparse)
          continue;
 
       if (!ok_tex_samp(tex))

src/freedreno/ir3/ir3_print.c

@@ -192,6 +192,8 @@ print_instr_name(struct log_stream *stream, struct ir3_instruction *instr,
          mesa_log_stream_printf(stream, ".a1en");
       if (instr->flags & IR3_INSTR_U)
          mesa_log_stream_printf(stream, ".u");
+      if (instr->flags & IR3_INSTR_RCK)
+         mesa_log_stream_printf(stream, ".rck");
       if (instr->opc == OPC_LDC)
          mesa_log_stream_printf(stream, ".offset%d", instr->cat6.d);
       if (instr->opc == OPC_LDC_K)

src/freedreno/vulkan/tu_shader.cc

@@ -502,6 +502,7 @@ lower_intrinsic(nir_builder *b, nir_intrinsic_instr *instr,
       return lower_ssbo_ubo_intrinsic(dev, b, instr);
 
    case nir_intrinsic_image_deref_load:
+   case nir_intrinsic_image_deref_sparse_load:
    case nir_intrinsic_image_deref_store:
    case nir_intrinsic_image_deref_atomic:
    case nir_intrinsic_image_deref_atomic_swap: