freedreno/a5xx: Add private mem support.

A bunch of our piglit fails were due to failing to compile shaders due to
a lack of spilling support.  I used a simple shader with a large local
array with tunable size to determine the MEMSIZEPERITEM increment and the
location of HWSTACKOFFSET (matching a3xx locations).  Unfortunately
fibers_per_sp I had to guess by taking a big spilling shader and cranking
it up until it rendered correctly.  The value I found made HWSTACKOFFSET's
shift value match a6xx's, as a bit of confirmation.

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/24358>

src/freedreno/ci/freedreno-a530-fails.txt

@@ -47,38 +47,6 @@ dEQP-GLES3.functional.polygon_offset.float32_displacement_with_units,Fail
 dEQP-GLES3.functional.shaders.texture_functions.textureprojgradoffset.sampler3d_float_vertex,Fail
 dEQP-GLES3.functional.texture.specification.texstorage3d.size.3d_2x2x2_2_levels,Fail
 
-# MESA: error: ir3_ra() failed!
-dEQP-GLES31.functional.atomic_counter.inc_dec.8_counters_5_calls_1_thread,Fail
-dEQP-GLES31.functional.atomic_counter.get_dec.8_counters_5_calls_10_threads,Fail
-dEQP-GLES31.functional.atomic_counter.get_dec.8_counters_5_calls_1_thread,Fail
-dEQP-GLES31.functional.atomic_counter.get_inc.8_counters_5_calls_10_threads,Fail
-dEQP-GLES31.functional.atomic_counter.get_inc.8_counters_5_calls_1_thread,Fail
-dEQP-GLES31.functional.atomic_counter.get_inc_dec.4_counters_5_calls_10_threads,Fail
-dEQP-GLES31.functional.atomic_counter.get_inc_dec.4_counters_5_calls_1_thread,Fail
-dEQP-GLES31.functional.atomic_counter.get_inc_dec.8_counters_5_calls_10_threads,Fail
-dEQP-GLES31.functional.atomic_counter.get_inc_dec.8_counters_5_calls_1_thread,Fail
-dEQP-GLES31.functional.atomic_counter.inc_dec.8_counters_5_calls_10_threads,Fail
-dEQP-GLES31.functional.atomic_counter.layout.default_offset_set.get_inc.8_counters_5_calls_1_thread,Fail
-dEQP-GLES31.functional.atomic_counter.layout.default_offset_set.get_dec.8_counters_5_calls_1000_threads,Fail
-dEQP-GLES31.functional.atomic_counter.layout.default_offset_set.get_dec.8_counters_5_calls_1_thread,Fail
-dEQP-GLES31.functional.atomic_counter.layout.default_offset_set.inc_dec.8_counters_5_calls_1000_threads,Fail
-dEQP-GLES31.functional.atomic_counter.layout.default_offset_set.inc_dec.8_counters_5_calls_1_thread,Fail
-dEQP-GLES31.functional.atomic_counter.layout.first_offset_set.get_dec.8_counters_5_calls_1000_threads,Fail
-dEQP-GLES31.functional.atomic_counter.layout.first_offset_set.get_dec.8_counters_5_calls_1_thread,Fail
-dEQP-GLES31.functional.atomic_counter.layout.first_offset_set.get_inc.8_counters_5_calls_1_thread,Fail
-dEQP-GLES31.functional.atomic_counter.layout.first_offset_set.inc_dec.8_counters_5_calls_1000_threads,Fail
-dEQP-GLES31.functional.atomic_counter.layout.first_offset_set.inc_dec.8_counters_5_calls_1_thread,Fail
-dEQP-GLES31.functional.atomic_counter.layout.reset_default_offset.get_dec.8_counters_5_calls_1000_threads,Fail
-dEQP-GLES31.functional.atomic_counter.layout.reset_default_offset.get_dec.8_counters_5_calls_1_thread,Fail
-dEQP-GLES31.functional.atomic_counter.layout.reset_default_offset.inc_dec.8_counters_5_calls_1000_threads,Fail
-dEQP-GLES31.functional.atomic_counter.layout.reset_default_offset.inc_dec.8_counters_5_calls_1_thread,Fail
-dEQP-GLES31.functional.atomic_counter.layout.reverse_offset.get_dec.8_counters_5_calls_1000_threads,Fail
-dEQP-GLES31.functional.atomic_counter.layout.reverse_offset.get_dec.8_counters_5_calls_1_thread,Fail
-dEQP-GLES31.functional.atomic_counter.layout.reverse_offset.get_inc.8_counters_5_calls_1000_threads,Fail
-dEQP-GLES31.functional.atomic_counter.layout.reverse_offset.get_inc.8_counters_5_calls_1_thread,Fail
-dEQP-GLES31.functional.atomic_counter.layout.reverse_offset.inc_dec.8_counters_5_calls_1000_threads,Fail
-dEQP-GLES31.functional.atomic_counter.layout.reverse_offset.inc_dec.8_counters_5_calls_1_thread,Fail
-
 # drm:a5xx_irq] *ERROR* gpu fault ring 2 fence 15595 status F700B1C1 rb 004a/004a ib1 000000000BFEA000/0000 ib2 0000000002F46000/0000
 dEQP-GLES31.functional.ubo.random.all_per_block_buffers.20,Fail
 
@@ -88,22 +56,11 @@ KHR-GLES2.core.internalformat.copy_tex_image.alpha,Fail
 KHR-GLES3.core.internalformat.copy_tex_image.alpha,Fail
 KHR-GLES31.core.internalformat.copy_tex_image.alpha,Fail
 
-# "Invalid array size was returned. at es31cArrayOfArraysTests.cpp:4779"
-# msm 900000.mdss: [drm:a5xx_irq] *ERROR* gpu fault ring 0 fence 2c54ef status E40801C1 rb 0162/0162 ib1 000000000104B000/0000 ib2 000000000104C000/0000
-KHR-GLES31.core.arrays_of_arrays.InteractionFunctionCalls1,Fail
-KHR-GLES31.core.arrays_of_arrays.InteractionFunctionCalls2,Fail
-
 KHR-GLES31.core.draw_indirect.advanced-twoPass-transformFeedback-arrays,Fail
 KHR-GLES31.core.draw_indirect.advanced-twoPass-transformFeedback-elements,Fail
 
 KHR-GLES31.core.layout_binding.buffer_layout_binding_atomicAdd_FragmentShader,Fail
 
-KHR-GLES31.core.shader_atomic_counters.advanced-usage-multidim-array-large,Fail
-
-
-# Several "Byte at index <n> is 0x00 should be 0x<ab>"
-KHR-GLES31.core.shader_storage_buffer_object.basic-stdLayout_UBO_SSBO-case2-cs,Fail
-
 KHR-GLES31.core.shader_storage_buffer_object.basic-syntax-cs,Fail
 
 # msm 900000.mdss: [drm:a5xx_irq] *ERROR* gpu fault ring 0 fence 2c54ef status E40801C1 rb 0162/0162 ib1 000000000104B000/0000 ib2 000000000104C000/0000
@@ -179,7 +136,7 @@ glx@glx_ext_import_context@make current- multi process,Fail
 glx@glx_ext_import_context@make current- single process,Fail
 glx@glx_ext_import_context@query context info,Fail
 shaders@glsl-bug-110796,Fail
-shaders@glsl-predication-on-large-array,Fail
+
 shaders@point-vertex-id divisor,Crash
 shaders@point-vertex-id gl_instanceid,Crash
 shaders@point-vertex-id gl_instanceid divisor,Crash
@@ -360,10 +317,6 @@ spec@ext_transform_feedback@tessellation triangles monochrome,Fail
 spec@ext_transform_feedback@tessellation triangles smooth,Fail
 spec@ext_transform_feedback@tessellation triangles wireframe,Fail
 
-# MESA: error: ir3_ra() failed!
-spec@glsl-1.10@execution@temp-array-indexing@glsl-fs-giant-temp-array,Fail
-spec@glsl-1.10@execution@temp-array-indexing@glsl-vs-giant-temp-array,Fail
-
 spec@glsl-1.30@execution@texelfetch fs sampler3d 1x129x9-98x129x9,Fail
 spec@glsl-1.30@execution@texelfetch fs sampler3d 98x129x1-98x129x9,Fail
 spec@glsl-1.30@execution@texelfetch fs sampler3d 98x1x9-98x129x9,Fail
@@ -386,9 +339,7 @@ spec@khr_texture_compression_astc@miptree-gl srgb-sd,Fail
 
 spec@nv_copy_image@nv_copy_image-formats,Fail
 spec@nv_copy_image@nv_copy_image-formats@Source: GL_DEPTH32F_STENCIL8/Destination: GL_DEPTH32F_STENCIL8,Fail
-spec@glsl-1.30@execution@fs-large-local-array-vec2,Fail
-spec@glsl-1.30@execution@fs-large-local-array-vec3,Fail
-spec@glsl-1.30@execution@fs-large-local-array-vec4,Fail
+
 spec@arb_texture_buffer_object@texture-buffer-size-clamp,Fail
 
 # https://gitlab.freedesktop.org/mesa/mesa/-/issues/7159
@@ -415,7 +366,6 @@ dEQP-GLES31.functional.ssbo.layout.random.all_per_block_buffers.11,Fail
 dEQP-GLES31.functional.ssbo.layout.random.all_per_block_buffers.19,Fail
 dEQP-GLES31.functional.ssbo.layout.random.all_per_block_buffers.23,Fail
 dEQP-GLES31.functional.ssbo.layout.random.all_per_block_buffers.24,Fail
-dEQP-GLES31.functional.ssbo.layout.random.all_per_block_buffers.45,Fail
 dEQP-GLES31.functional.ssbo.layout.random.basic_arrays.10,Fail
 dEQP-GLES31.functional.ssbo.layout.random.basic_arrays.18,Fail
 dEQP-GLES31.functional.ssbo.layout.random.basic_instance_arrays.1,Fail
@@ -423,20 +373,10 @@ dEQP-GLES31.functional.ssbo.layout.random.basic_instance_arrays.19,Fail
 dEQP-GLES31.functional.ssbo.layout.random.basic_instance_arrays.23,Fail
 dEQP-GLES31.functional.ssbo.layout.random.basic_types.15,Fail
 dEQP-GLES31.functional.ssbo.layout.random.nested_structs_arrays_instance_arrays.13,Fail
-dEQP-GLES31.functional.ssbo.layout.random.nested_structs_arrays_instance_arrays.22,Fail
 dEQP-GLES31.functional.ssbo.layout.random.nested_structs.7,Fail
 dEQP-GLES31.functional.ssbo.layout.random.nested_structs.9,Fail
 dEQP-GLES31.functional.ssbo.layout.random.unsized_arrays.23,Fail
 dEQP-GLES31.functional.ssbo.layout.random.vector_types.3,Fail
-dEQP-GLES31.functional.ssbo.layout.single_struct_array.per_block_buffer.packed_instance_array,Fail
-dEQP-GLES31.functional.ssbo.layout.single_struct_array.per_block_buffer.std430_instance_array,Fail
-dEQP-GLES31.functional.ssbo.layout.single_struct_array.single_buffer.packed_instance_array,Fail
-dEQP-GLES31.functional.ssbo.layout.single_struct_array.single_buffer.shared_instance_array,Fail
-dEQP-GLES31.functional.ssbo.layout.single_struct_array.single_buffer.std430_instance_array,Fail
-dEQP-GLES31.functional.ssbo.layout.unsized_struct_array.per_block_buffer.shared_instance_array,Fail
-dEQP-GLES31.functional.ssbo.layout.unsized_struct_array.per_block_buffer.std430_instance_array,Fail
-dEQP-GLES31.functional.ssbo.layout.unsized_struct_array.single_buffer.packed_instance_array,Fail
-dEQP-GLES31.functional.ssbo.layout.unsized_struct_array.single_buffer.std140_instance_array,Fail
 
 KHR-GLES31.core.shader_image_load_store.basic-allTargets-loadStoreCS,Fail
 KHR-GLES31.core.shader_storage_buffer_object.advanced-usage-operators-cs,Fail

src/freedreno/common/freedreno_devices.py

@@ -274,7 +274,7 @@ add_gpus([
         cs_shared_mem_size = 32 * 1024,
         num_sp_cores = 4,
         wave_granularity = 2,
-        fibers_per_sp = 0, # TODO
+        fibers_per_sp = 64 * 16, # Lowest number that didn't fault on spillall fs-varying-array-mat4-col-row-rd.
     ))
 
 # a6xx can be divided into distinct sub-generations, where certain device-

src/freedreno/ir3/ir3_compiler.c

@@ -188,9 +188,6 @@ ir3_compiler_create(struct fd_device *dev, const struct fd_dev_id *dev_id,
       /* TODO: implement clip+cull distances on earlier gen's */
       compiler->has_clip_cull = true;
 
-      /* TODO: implement private memory on earlier gen's */
-      compiler->has_pvtmem = true;
-
       compiler->has_preamble = true;
 
       compiler->tess_use_shared = dev_info->a6xx.tess_use_shared;
@@ -219,6 +216,8 @@ ir3_compiler_create(struct fd_device *dev, const struct fd_dev_id *dev_id,
 
    /* This is just a guess for a4xx. */
    compiler->pvtmem_per_fiber_align = compiler->gen >= 4 ? 512 : 128;
+   /* TODO: implement private memory on earlier gen's */
+   compiler->has_pvtmem = compiler->gen >= 5;
 
    if (compiler->gen >= 6) {
       compiler->reg_size_vec4 = dev_info->a6xx.reg_size_vec4;

src/freedreno/registers/adreno/a5xx.xml

@@ -2518,10 +2518,31 @@ bit 7 for RECTLIST (clear) when z32s8 (used for clear of depth32?  not set
 	<reg32 offset="0xe5ab" name="UNKNOWN_E5AB"/>
 	<reg32 offset="0xe5ac" name="SP_VS_OBJ_START_LO"/>
 	<reg32 offset="0xe5ad" name="SP_VS_OBJ_START_HI"/>
+
+	<bitset name="a5xx_sp_xs_pvt_mem_param" inline="yes">
+		<bitfield name="MEMSIZEPERITEM" low="0" high="7" shr="9">
+			<doc>The size of memory that ldp/stp can address.</doc>
+		</bitfield>
+		<bitfield name="HWSTACKOFFSET" low="8" high="23" shr="11" type="uint"/>
+		<bitfield name="HWSTACKSIZEPERTHREAD" low="24" high="31">
+                        <doc>Guessing that this is the same as a3xx/a6xx.</doc>
+		</bitfield>
+	</bitset>
+
+	<bitset name="a5xx_sp_xs_pvt_mem_size" inline="yes">
+		<bitfield name="TOTALPVTMEMSIZE" low="0" high="17" shr="12"/>
+	</bitset>
+
+	<reg32 offset="0xe5ae" name="SP_VS_PVT_MEM_PARAM" type="a5xx_sp_xs_pvt_mem_param"/>
+	<reg64 offset="0xe5af" name="SP_VS_PVT_MEM_ADDR" type="waddress" align="32"/>
+	<reg32 offset="0xe5b1" name="SP_VS_PVT_MEM_SIZE" type="a5xx_sp_xs_pvt_mem_size"/>
 	<reg32 offset="0xe5c0" name="SP_FS_CTRL_REG0" type="a5xx_sp_xs_ctrl_reg0"/>
 	<reg32 offset="0xe5c2" name="UNKNOWN_E5C2"/>
 	<reg32 offset="0xe5c3" name="SP_FS_OBJ_START_LO"/>
 	<reg32 offset="0xe5c4" name="SP_FS_OBJ_START_HI"/>
+	<reg32 offset="0xe5c5" name="SP_FS_PVT_MEM_PARAM" type="a5xx_sp_xs_pvt_mem_param"/>
+	<reg64 offset="0xe5c6" name="SP_FS_PVT_MEM_ADDR" type="waddress" align="32"/>
+	<reg32 offset="0xe5c8" name="SP_FS_PVT_MEM_SIZE" type="a5xx_sp_xs_pvt_mem_size"/>
 	<reg32 offset="0xe5c9" name="SP_BLEND_CNTL">
 		<!-- per-mrt enable bit -->
 		<bitfield name="ENABLE_BLEND" low="0" high="7"/>
@@ -2558,20 +2579,33 @@ bit 7 for RECTLIST (clear) when z32s8 (used for clear of depth32?  not set
 	<reg32 offset="0xe5f2" name="UNKNOWN_E5F2"/>
 	<reg32 offset="0xe5f3" name="SP_CS_OBJ_START_LO"/>
 	<reg32 offset="0xe5f4" name="SP_CS_OBJ_START_HI"/>
+	<reg32 offset="0xe5f5" name="SP_CS_PVT_MEM_PARAM" type="a5xx_sp_xs_pvt_mem_param"/>
+	<reg64 offset="0xe5f6" name="SP_CS_PVT_MEM_ADDR" type="waddress" align="32"/>
+	<reg32 offset="0xe5f8" name="SP_CS_PVT_MEM_SIZE" type="a5xx_sp_xs_pvt_mem_size"/>
+
 	<!-- e5f9 something compute related.. seems to change when HLSQ_CS_CNTL_1 changes -->
 
 	<reg32 offset="0xe600" name="SP_HS_CTRL_REG0" type="a5xx_sp_xs_ctrl_reg0"/>
 	<reg32 offset="0xe602" name="UNKNOWN_E602"/>
 	<reg32 offset="0xe603" name="SP_HS_OBJ_START_LO"/>
 	<reg32 offset="0xe604" name="SP_HS_OBJ_START_HI"/>
+	<reg32 offset="0xe605" name="SP_HS_PVT_MEM_PARAM" type="a5xx_sp_xs_pvt_mem_param"/>
+	<reg64 offset="0xe606" name="SP_HS_PVT_MEM_ADDR" type="waddress" align="32"/>
+	<reg32 offset="0xe608" name="SP_HS_PVT_MEM_SIZE" type="a5xx_sp_xs_pvt_mem_size"/>
 	<reg32 offset="0xe610" name="SP_DS_CTRL_REG0" type="a5xx_sp_xs_ctrl_reg0"/>
 	<reg32 offset="0xe62b" name="UNKNOWN_E62B"/>
 	<reg32 offset="0xe62c" name="SP_DS_OBJ_START_LO"/>
 	<reg32 offset="0xe62d" name="SP_DS_OBJ_START_HI"/>
+	<reg32 offset="0xe62e" name="SP_DS_PVT_MEM_PARAM" type="a5xx_sp_xs_pvt_mem_param"/>
+	<reg64 offset="0xe62f" name="SP_DS_PVT_MEM_ADDR" type="waddress" align="32"/>
+	<reg32 offset="0xe631" name="SP_DS_PVT_MEM_SIZE" type="a5xx_sp_xs_pvt_mem_size"/>
 	<reg32 offset="0xe640" name="SP_GS_CTRL_REG0" type="a5xx_sp_xs_ctrl_reg0"/>
 	<reg32 offset="0xe65b" name="UNKNOWN_E65B"/>
 	<reg32 offset="0xe65c" name="SP_GS_OBJ_START_LO"/>
 	<reg32 offset="0xe65d" name="SP_GS_OBJ_START_HI"/>
+	<reg32 offset="0xe65e" name="SP_GS_PVT_MEM_PARAM" type="a5xx_sp_xs_pvt_mem_param"/>
+	<reg64 offset="0xe65f" name="SP_GS_PVT_MEM_ADDR" type="waddress" align="32"/>
+	<reg32 offset="0xe661" name="SP_GS_PVT_MEM_SIZE" type="a5xx_sp_xs_pvt_mem_size"/>
 
 	<reg32 offset="0xe704" name="TPL1_TP_RAS_MSAA_CNTL">
 		<bitfield name="SAMPLES" low="0" high="1" type="a3xx_msaa_samples"/>

src/gallium/drivers/freedreno/a5xx/fd5_compute.c

@@ -34,7 +34,7 @@
 
 /* maybe move to fd5_program? */
 static void
-cs_program_emit(struct fd_ringbuffer *ring, struct ir3_shader_variant *v)
+cs_program_emit(struct fd_context *ctx, struct fd_ringbuffer *ring, struct ir3_shader_variant *v) assert_dt
 {
    const struct ir3_info *i = &v->info;
    enum a3xx_threadsize thrsz = i->double_threadsize ? FOUR_QUADS : TWO_QUADS;
@@ -83,7 +83,7 @@ cs_program_emit(struct fd_ringbuffer *ring, struct ir3_shader_variant *v)
    OUT_RING(ring, constlen); /* HLSQ_CS_CONSTLEN */
    OUT_RING(ring, instrlen); /* HLSQ_CS_INSTRLEN */
 
-   fd5_emit_shader_obj(ring, v, REG_A5XX_SP_CS_OBJ_START_LO);
+   fd5_emit_shader_obj(ctx, ring, v, REG_A5XX_SP_CS_OBJ_START_LO);
 
    OUT_PKT4(ring, REG_A5XX_HLSQ_UPDATE_CNTL, 1);
    OUT_RING(ring, 0x1f00000);
@@ -119,7 +119,7 @@ fd5_launch_grid(struct fd_context *ctx,
       return;
 
    if (ctx->dirty_shader[PIPE_SHADER_COMPUTE] & FD_DIRTY_SHADER_PROG)
-      cs_program_emit(ring, v);
+      cs_program_emit(ctx, ring, v);
 
    fd5_emit_cs_state(ctx, ring, v);
    fd5_emit_cs_consts(v, ring, ctx, info);

src/gallium/drivers/freedreno/a5xx/fd5_program.c

@@ -82,11 +82,27 @@ fd5_emit_shader(struct fd_ringbuffer *ring, const struct ir3_shader_variant *so)
 }
 
 void
-fd5_emit_shader_obj(struct fd_ringbuffer *ring,
-                    const struct ir3_shader_variant *so, uint32_t shader_obj_reg)
+fd5_emit_shader_obj(struct fd_context *ctx, struct fd_ringbuffer *ring,
+                    const struct ir3_shader_variant *so,
+                    uint32_t shader_obj_reg)
 {
-   OUT_PKT4(ring, shader_obj_reg, 2);
+   ir3_get_private_mem(ctx, so);
+
+   OUT_PKT4(ring, shader_obj_reg, 6);
    OUT_RELOC(ring, so->bo, 0, 0, 0); /* SP_VS_OBJ_START_LO/HI */
+
+   uint32_t per_sp_size = ctx->pvtmem[so->pvtmem_per_wave].per_sp_size;
+   OUT_RING(ring, A5XX_SP_VS_PVT_MEM_PARAM_MEMSIZEPERITEM(
+                     ctx->pvtmem[so->pvtmem_per_wave].per_fiber_size) |
+                     A5XX_SP_VS_PVT_MEM_PARAM_HWSTACKOFFSET(per_sp_size));
+   if (so->pvtmem_size > 0) { /* SP_xS_PVT_MEM_ADDR */
+      OUT_RELOC(ring, ctx->pvtmem[so->pvtmem_per_wave].bo, 0, 0, 0);
+      fd_ringbuffer_attach_bo(ring, ctx->pvtmem[so->pvtmem_per_wave].bo);
+   } else {
+      OUT_RING(ring, 0);
+      OUT_RING(ring, 0);
+   }
+   OUT_RING(ring, A5XX_SP_VS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(per_sp_size));
 }
 
 /* TODO maybe some of this we could pre-compute once rather than having
@@ -495,7 +511,7 @@ fd5_program_emit(struct fd_context *ctx, struct fd_ringbuffer *ring,
       OUT_RING(ring, reg);
    }
 
-   fd5_emit_shader_obj(ring, s[VS].v, REG_A5XX_SP_VS_OBJ_START_LO);
+   fd5_emit_shader_obj(ctx, ring, s[VS].v, REG_A5XX_SP_VS_OBJ_START_LO);
 
    if (s[VS].instrlen)
       fd5_emit_shader(ring, s[VS].v);
@@ -519,7 +535,7 @@ fd5_program_emit(struct fd_context *ctx, struct fd_ringbuffer *ring,
       OUT_RING(ring, 0x00000000); /* SP_FS_OBJ_START_LO */
       OUT_RING(ring, 0x00000000); /* SP_FS_OBJ_START_HI */
    } else {
-      fd5_emit_shader_obj(ring, s[FS].v, REG_A5XX_SP_FS_OBJ_START_LO);
+      fd5_emit_shader_obj(ctx, ring, s[FS].v, REG_A5XX_SP_FS_OBJ_START_LO);
    }
 
    OUT_PKT4(ring, REG_A5XX_HLSQ_CONTROL_0_REG, 5);

src/gallium/drivers/freedreno/a5xx/fd5_program.h

@@ -51,12 +51,12 @@ fd5_program_state(struct ir3_program_state *state)
 void fd5_emit_shader(struct fd_ringbuffer *ring,
                      const struct ir3_shader_variant *so);
 
-void fd5_emit_shader_obj(struct fd_ringbuffer *ring,
+void fd5_emit_shader_obj(struct fd_context *ctx, struct fd_ringbuffer *ring,
                          const struct ir3_shader_variant *so,
-                         uint32_t shader_obj_reg);
+                         uint32_t shader_obj_reg) assert_dt;
 
 void fd5_program_emit(struct fd_context *ctx, struct fd_ringbuffer *ring,
-                      struct fd5_emit *emit);
+                      struct fd5_emit *emit) assert_dt;
 
 void fd5_prog_init(struct pipe_context *pctx);