freedreno/batch: Add a global epilogue

Rename the existing one to make it clear that it is per-tile, and add a
new one that runs after all the tile passes.  Will be needed in the next
commit.

Signed-off-by: Rob Clark <robdclark@chromium.org>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/19400>

src/gallium/drivers/freedreno/a6xx/fd6_gmem.c

@@ -1437,8 +1437,8 @@ fd6_emit_tile(struct fd_batch *batch, const struct fd_tile *tile)
       emit_conditional_ib(batch, tile, batch->draw);
    }
 
-   if (batch->epilogue)
-      fd6_emit_ib(batch->gmem, batch->epilogue);
+   if (batch->tile_epilogue)
+      fd6_emit_ib(batch->gmem, batch->tile_epilogue);
 }
 
 static void
@@ -1446,6 +1446,9 @@ fd6_emit_tile_gmem2mem(struct fd_batch *batch, const struct fd_tile *tile)
 {
    struct fd_ringbuffer *ring = batch->gmem;
 
+   if (batch->epilogue)
+      fd6_emit_ib(batch->gmem, batch->epilogue);
+
    if (use_hw_binning(batch)) {
       OUT_PKT7(ring, CP_SET_MARKER, 1);
       OUT_RING(ring, A6XX_CP_SET_MARKER_0_MODE(RM6_ENDVIS));
@@ -1633,6 +1636,9 @@ fd6_emit_sysmem_fini(struct fd_batch *batch) assert_dt
 
    emit_common_fini(batch);
 
+   if (batch->tile_epilogue)
+      fd6_emit_ib(batch->gmem, batch->tile_epilogue);
+
    if (batch->epilogue)
       fd6_emit_ib(batch->gmem, batch->epilogue);
 

src/gallium/drivers/freedreno/a6xx/fd6_query.c

@@ -106,7 +106,7 @@ occlusion_pause(struct fd_acc_query *aq, struct fd_batch *batch) assert_dt
    /* To avoid stalling in the draw buffer, emit code the code to compute the
     * counter delta in the epilogue ring.
     */
-   struct fd_ringbuffer *epilogue = fd_batch_get_epilogue(batch);
+   struct fd_ringbuffer *epilogue = fd_batch_get_tile_epilogue(batch);
 
    OUT_PKT7(epilogue, CP_WAIT_REG_MEM, 6);
    OUT_RING(epilogue, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_NE) |

src/gallium/drivers/freedreno/freedreno_batch.c

@@ -163,6 +163,11 @@ cleanup_submit(struct fd_batch *batch)
       batch->prologue = NULL;
    }
 
+   if (batch->tile_epilogue) {
+      fd_ringbuffer_del(batch->tile_epilogue);
+      batch->tile_epilogue = NULL;
+   }
+
    if (batch->epilogue) {
       fd_ringbuffer_del(batch->epilogue);
       batch->epilogue = NULL;

src/gallium/drivers/freedreno/freedreno_batch.h

@@ -210,6 +210,9 @@ struct fd_batch {
    struct fd_ringbuffer *prologue;
 
    /** epilogue cmdstream (executed after each tile): */
+   struct fd_ringbuffer *tile_epilogue;
+
+   /** epilogue cmdstream (executed after all tiles): */
    struct fd_ringbuffer *epilogue;
 
    struct fd_ringbuffer *tile_setup;
@@ -402,6 +405,18 @@ fd_event_write(struct fd_batch *batch, struct fd_ringbuffer *ring,
 
 /* Get per-tile epilogue */
 static inline struct fd_ringbuffer *
+fd_batch_get_tile_epilogue(struct fd_batch *batch)
+{
+   if (batch->tile_epilogue == NULL) {
+      batch->tile_epilogue = fd_submit_new_ringbuffer(batch->submit, 0x1000,
+                                                 FD_RINGBUFFER_GROWABLE);
+   }
+
+   return batch->tile_epilogue;
+}
+
+/* Get epilogue run after all tiles*/
+static inline struct fd_ringbuffer *
 fd_batch_get_epilogue(struct fd_batch *batch)
 {
    if (batch->epilogue == NULL) {