From 7e2b5ce722453157f2c8c891c5c7863700d7aaf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Ol=C5=A1=C3=A1k?= Date: Fri, 19 Mar 2021 23:20:48 -0400 Subject: [PATCH] radeonsi: set compute/cpdma sync flags in the outermost caller This allows us to control syncing everywhere. Reviewed-by: Pierre-Eric Pelloux-Prayer Part-of: --- src/gallium/drivers/radeonsi/si_blit.c | 8 ++-- src/gallium/drivers/radeonsi/si_buffer.c | 6 +-- src/gallium/drivers/radeonsi/si_clear.c | 14 +++--- .../drivers/radeonsi/si_compute_blit.c | 43 ++++++++++--------- src/gallium/drivers/radeonsi/si_pipe.c | 3 +- src/gallium/drivers/radeonsi/si_pipe.h | 11 ++--- src/gallium/drivers/radeonsi/si_test_blit.c | 2 +- src/gallium/drivers/radeonsi/si_texture.c | 18 ++++---- 8 files changed, 55 insertions(+), 50 deletions(-) diff --git a/src/gallium/drivers/radeonsi/si_blit.c b/src/gallium/drivers/radeonsi/si_blit.c index 76c1f1eec29..6e573aa532f 100644 --- a/src/gallium/drivers/radeonsi/si_blit.c +++ b/src/gallium/drivers/radeonsi/si_blit.c @@ -876,7 +876,7 @@ void si_resource_copy_region(struct pipe_context *ctx, struct pipe_resource *dst /* Handle buffers first. */ if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) { - si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width); + si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width, SI_OP_SYNC_BEFORE_AFTER); return; } @@ -889,7 +889,7 @@ void si_resource_copy_region(struct pipe_context *ctx, struct pipe_resource *dst !(dst->target != src->target && (src->target == PIPE_TEXTURE_1D_ARRAY || dst->target == PIPE_TEXTURE_1D_ARRAY))) { si_compute_copy_image(sctx, dst, dst_level, src, src_level, dstx, dsty, dstz, - src_box, false); + src_box, false, SI_OP_SYNC_BEFORE_AFTER); return; } @@ -1334,7 +1334,7 @@ void si_decompress_dcc(struct si_context *sctx, struct si_texture *tex) u_minify(ptex->height0, level), util_num_layers(ptex, level), &box); si_compute_copy_image(sctx, ptex, level, ptex, level, 0, 0, 0, &box, - true); + true, SI_OP_SYNC_BEFORE_AFTER); } /* Now clear DCC metadata to uncompressed. @@ -1346,7 +1346,7 @@ void si_decompress_dcc(struct si_context *sctx, struct si_texture *tex) */ uint32_t clear_value = DCC_UNCOMPRESSED; si_clear_buffer(sctx, ptex, tex->surface.dcc_offset, - tex->surface.dcc_size, &clear_value, 4, + tex->surface.dcc_size, &clear_value, 4, SI_OP_SYNC_BEFORE_AFTER, SI_COHERENCY_CB_META, SI_COMPUTE_CLEAR_METHOD); } } diff --git a/src/gallium/drivers/radeonsi/si_buffer.c b/src/gallium/drivers/radeonsi/si_buffer.c index 33c3d6ad390..ba9489d0138 100644 --- a/src/gallium/drivers/radeonsi/si_buffer.c +++ b/src/gallium/drivers/radeonsi/si_buffer.c @@ -213,7 +213,7 @@ bool si_alloc_resource(struct si_screen *sscreen, struct si_resource *res) } if (res->b.b.flags & SI_RESOURCE_FLAG_CLEAR) - si_screen_clear_buffer(sscreen, &res->b.b, 0, res->bo_size, 0); + si_screen_clear_buffer(sscreen, &res->b.b, 0, res->bo_size, 0, SI_OP_SYNC_BEFORE_AFTER); return true; } @@ -447,7 +447,7 @@ static void *si_buffer_transfer_map(struct pipe_context *ctx, struct pipe_resour if (staging) { /* Copy the VRAM buffer to the staging buffer. */ si_copy_buffer(sctx, &staging->b.b, resource, box->x % SI_MAP_BUFFER_ALIGNMENT, - box->x, box->width); + box->x, box->width, SI_OP_SYNC_BEFORE_AFTER); data = si_buffer_map(sctx, staging, usage & ~PIPE_MAP_UNSYNCHRONIZED); if (!data) { @@ -484,7 +484,7 @@ static void si_buffer_do_flush_region(struct pipe_context *ctx, struct pipe_tran /* Copy the staging buffer into the original one. */ si_copy_buffer(sctx, transfer->resource, &stransfer->staging->b.b, box->x, src_offset, - box->width); + box->width, SI_OP_SYNC_BEFORE_AFTER); } util_range_add(&buf->b.b, &buf->valid_buffer_range, box->x, box->x + box->width); diff --git a/src/gallium/drivers/radeonsi/si_clear.c b/src/gallium/drivers/radeonsi/si_clear.c index 236e180401e..08c02418c05 100644 --- a/src/gallium/drivers/radeonsi/si_clear.c +++ b/src/gallium/drivers/radeonsi/si_clear.c @@ -261,8 +261,8 @@ bool vi_dcc_clear_level(struct si_context *sctx, struct si_texture *tex, unsigne clear_size = tex->surface.u.legacy.level[level].dcc_fast_clear_size * num_layers; } - si_clear_buffer(sctx, dcc_buffer, dcc_offset, clear_size, &clear_value, 4, SI_COHERENCY_CB_META, - SI_AUTO_SELECT_CLEAR_METHOD); + si_clear_buffer(sctx, dcc_buffer, dcc_offset, clear_size, &clear_value, 4, + SI_OP_SYNC_BEFORE_AFTER, SI_COHERENCY_CB_META, SI_AUTO_SELECT_CLEAR_METHOD); return true; } @@ -504,8 +504,8 @@ static void si_do_fast_color_clear(struct si_context *sctx, unsigned *buffers, if (tex->buffer.b.b.nr_samples >= 2 && tex->cmask_buffer) { uint32_t clear_value = 0xCCCCCCCC; si_clear_buffer(sctx, &tex->cmask_buffer->b.b, tex->surface.cmask_offset, - tex->surface.cmask_size, &clear_value, 4, SI_COHERENCY_CB_META, - SI_AUTO_SELECT_CLEAR_METHOD); + tex->surface.cmask_size, &clear_value, 4, SI_OP_SYNC_BEFORE_AFTER, + SI_COHERENCY_CB_META, SI_AUTO_SELECT_CLEAR_METHOD); fmask_decompress_needed = true; } } else { @@ -533,8 +533,8 @@ static void si_do_fast_color_clear(struct si_context *sctx, unsigned *buffers, /* Do the fast clear. */ uint32_t clear_value = 0; si_clear_buffer(sctx, &tex->cmask_buffer->b.b, tex->surface.cmask_offset, - tex->surface.cmask_size, &clear_value, 4, SI_COHERENCY_CB_META, - SI_AUTO_SELECT_CLEAR_METHOD); + tex->surface.cmask_size, &clear_value, 4, SI_OP_SYNC_BEFORE_AFTER, + SI_COHERENCY_CB_META, SI_AUTO_SELECT_CLEAR_METHOD); eliminate_needed = true; } @@ -624,7 +624,7 @@ static void si_clear(struct pipe_context *ctx, unsigned buffers, sctx->chip_class == GFX8 ? 0xfffff30f : 0xfffc000f; si_clear_buffer(sctx, &zstex->buffer.b.b, zstex->surface.htile_offset, zstex->surface.htile_size, &clear_value, 4, - SI_COHERENCY_DB_META, SI_AUTO_SELECT_CLEAR_METHOD); + SI_OP_SYNC_BEFORE_AFTER, SI_COHERENCY_DB_META, SI_AUTO_SELECT_CLEAR_METHOD); } /* TC-compatible HTILE only supports depth clears to 0 or 1. */ diff --git a/src/gallium/drivers/radeonsi/si_compute_blit.c b/src/gallium/drivers/radeonsi/si_compute_blit.c index cf77c018c3a..59c03b91688 100644 --- a/src/gallium/drivers/radeonsi/si_compute_blit.c +++ b/src/gallium/drivers/radeonsi/si_compute_blit.c @@ -115,7 +115,8 @@ void si_launch_grid_internal(struct si_context *sctx, struct pipe_grid_info *inf static void si_compute_clear_12bytes_buffer(struct si_context *sctx, struct pipe_resource *dst, unsigned dst_offset, unsigned size, - const uint32_t *clear_value, enum si_coherency coher) + const uint32_t *clear_value, unsigned flags, + enum si_coherency coher) { struct pipe_context *ctx = &sctx->b; @@ -166,7 +167,7 @@ static void si_compute_clear_12bytes_buffer(struct si_context *sctx, struct pipe info.grid[1] = 1; info.grid[2] = 1; - si_launch_grid_internal(sctx, &info, saved_cs, SI_OP_SYNC_BEFORE_AFTER); + si_launch_grid_internal(sctx, &info, saved_cs, flags); ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, 1, &saved_sb, saved_writable_mask); ctx->set_constant_buffer(ctx, PIPE_SHADER_COMPUTE, 0, true, &saved_cb); @@ -178,7 +179,7 @@ static void si_compute_do_clear_or_copy(struct si_context *sctx, struct pipe_res unsigned dst_offset, struct pipe_resource *src, unsigned src_offset, unsigned size, const uint32_t *clear_value, unsigned clear_value_size, - enum si_coherency coher) + unsigned flags, enum si_coherency coher) { struct pipe_context *ctx = &sctx->b; @@ -260,7 +261,7 @@ static void si_compute_do_clear_or_copy(struct si_context *sctx, struct pipe_res ctx->bind_compute_state(ctx, sctx->cs_clear_buffer); } - si_launch_grid_internal(sctx, &info, saved_cs, SI_OP_SYNC_BEFORE_AFTER); + si_launch_grid_internal(sctx, &info, saved_cs, flags); enum si_cache_policy cache_policy = get_cache_policy(sctx, coher, size); sctx->flags |= cache_policy == L2_BYPASS ? SI_CONTEXT_WB_L2 : 0; @@ -274,8 +275,9 @@ static void si_compute_do_clear_or_copy(struct si_context *sctx, struct pipe_res pipe_resource_reference(&saved_sb[i].buffer, NULL); } -void si_clear_buffer(struct si_context *sctx, struct pipe_resource *dst, uint64_t offset, - uint64_t size, uint32_t *clear_value, uint32_t clear_value_size, +void si_clear_buffer(struct si_context *sctx, struct pipe_resource *dst, + uint64_t offset, uint64_t size, uint32_t *clear_value, + uint32_t clear_value_size, unsigned flags, enum si_coherency coher, enum si_clear_method method) { if (!size) @@ -319,7 +321,7 @@ void si_clear_buffer(struct si_context *sctx, struct pipe_resource *dst, uint64_ } if (clear_value_size == 12) { - si_compute_clear_12bytes_buffer(sctx, dst, offset, size, clear_value, coher); + si_compute_clear_12bytes_buffer(sctx, dst, offset, size, clear_value, flags, coher); return; } @@ -355,12 +357,11 @@ void si_clear_buffer(struct si_context *sctx, struct pipe_resource *dst, uint64_ } if (method == SI_COMPUTE_CLEAR_METHOD) { si_compute_do_clear_or_copy(sctx, dst, offset, NULL, 0, aligned_size, clear_value, - clear_value_size, coher); + clear_value_size, flags, coher); } else { assert(clear_value_size == 4); si_cp_dma_clear_buffer(sctx, &sctx->gfx_cs, dst, offset, aligned_size, *clear_value, - SI_OP_SYNC_BEFORE_AFTER, coher, - get_cache_policy(sctx, coher, size)); + flags, coher, get_cache_policy(sctx, coher, size)); } offset += aligned_size; @@ -378,13 +379,13 @@ void si_clear_buffer(struct si_context *sctx, struct pipe_resource *dst, uint64_ } void si_screen_clear_buffer(struct si_screen *sscreen, struct pipe_resource *dst, uint64_t offset, - uint64_t size, unsigned value) + uint64_t size, unsigned value, unsigned flags) { struct si_context *ctx = (struct si_context *)sscreen->aux_context; simple_mtx_lock(&sscreen->aux_context_lock); - si_clear_buffer(ctx, dst, offset, size, &value, 4, SI_COHERENCY_SHADER, - SI_AUTO_SELECT_CLEAR_METHOD); + si_clear_buffer(ctx, dst, offset, size, &value, 4, flags, + SI_COHERENCY_SHADER, SI_AUTO_SELECT_CLEAR_METHOD); sscreen->aux_context->flush(sscreen->aux_context, NULL, 0); simple_mtx_unlock(&sscreen->aux_context_lock); } @@ -394,11 +395,12 @@ static void si_pipe_clear_buffer(struct pipe_context *ctx, struct pipe_resource int clear_value_size) { si_clear_buffer((struct si_context *)ctx, dst, offset, size, (uint32_t *)clear_value, - clear_value_size, SI_COHERENCY_SHADER, SI_AUTO_SELECT_CLEAR_METHOD); + clear_value_size, SI_OP_SYNC_BEFORE_AFTER, SI_COHERENCY_SHADER, + SI_AUTO_SELECT_CLEAR_METHOD); } void si_copy_buffer(struct si_context *sctx, struct pipe_resource *dst, struct pipe_resource *src, - uint64_t dst_offset, uint64_t src_offset, unsigned size) + uint64_t dst_offset, uint64_t src_offset, unsigned size, unsigned flags) { if (!size) return; @@ -428,17 +430,18 @@ void si_copy_buffer(struct si_context *sctx, struct pipe_resource *dst, struct p if (sctx->screen->info.has_dedicated_vram && si_resource(dst)->domains & RADEON_DOMAIN_VRAM && si_resource(src)->domains & RADEON_DOMAIN_VRAM && size > compute_min_size && dst_offset % 4 == 0 && src_offset % 4 == 0 && size % 4 == 0) { - si_compute_do_clear_or_copy(sctx, dst, dst_offset, src, src_offset, size, NULL, 0, coher); + si_compute_do_clear_or_copy(sctx, dst, dst_offset, src, src_offset, size, NULL, 0, + flags, coher); } else { si_cp_dma_copy_buffer(sctx, dst, src, dst_offset, src_offset, size, - SI_OP_SYNC_BEFORE_AFTER, coher, cache_policy); + flags, coher, cache_policy); } } void si_compute_copy_image(struct si_context *sctx, struct pipe_resource *dst, unsigned dst_level, struct pipe_resource *src, unsigned src_level, unsigned dstx, unsigned dsty, unsigned dstz, const struct pipe_box *src_box, - bool is_dcc_decompress) + bool is_dcc_decompress, unsigned flags) { struct pipe_context *ctx = &sctx->b; unsigned width = src_box->width; @@ -612,7 +615,7 @@ void si_compute_copy_image(struct si_context *sctx, struct pipe_resource *dst, u info.grid[2] = depth; } - si_launch_grid_internal(sctx, &info, saved_cs, SI_OP_SYNC_BEFORE_AFTER | SI_OP_CS_IMAGE); + si_launch_grid_internal(sctx, &info, saved_cs, flags | SI_OP_CS_IMAGE); ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 2, 0, saved_image); for (int i = 0; i < 2; i++) @@ -768,7 +771,7 @@ void si_compute_expand_fmask(struct pipe_context *ctx, struct pipe_resource *tex struct si_texture *stex = (struct si_texture *)tex; si_clear_buffer(sctx, tex, stex->surface.fmask_offset, stex->surface.fmask_size, (uint32_t *)&fmask_expand_values[log_fragments][log_samples - 1], - log_fragments >= 2 && log_samples == 4 ? 8 : 4, + log_fragments >= 2 && log_samples == 4 ? 8 : 4, SI_OP_SYNC_BEFORE_AFTER, SI_COHERENCY_SHADER, SI_AUTO_SELECT_CLEAR_METHOD); } diff --git a/src/gallium/drivers/radeonsi/si_pipe.c b/src/gallium/drivers/radeonsi/si_pipe.c index c68519a2acf..f9281879a98 100644 --- a/src/gallium/drivers/radeonsi/si_pipe.c +++ b/src/gallium/drivers/radeonsi/si_pipe.c @@ -734,7 +734,8 @@ static struct pipe_context *si_create_context(struct pipe_screen *screen, unsign */ uint32_t clear_value = 0; si_clear_buffer(sctx, sctx->null_const_buf.buffer, 0, sctx->null_const_buf.buffer->width0, - &clear_value, 4, SI_COHERENCY_SHADER, SI_CP_DMA_CLEAR_METHOD); + &clear_value, 4, SI_OP_SYNC_BEFORE_AFTER, SI_COHERENCY_SHADER, + SI_CP_DMA_CLEAR_METHOD); } if (!(flags & SI_CONTEXT_FLAG_AUX)) diff --git a/src/gallium/drivers/radeonsi/si_pipe.h b/src/gallium/drivers/radeonsi/si_pipe.h index 115d4887a77..90f4f4dce52 100644 --- a/src/gallium/drivers/radeonsi/si_pipe.h +++ b/src/gallium/drivers/radeonsi/si_pipe.h @@ -1381,17 +1381,18 @@ enum si_clear_method { SI_COMPUTE_CLEAR_METHOD, SI_AUTO_SELECT_CLEAR_METHOD }; -void si_clear_buffer(struct si_context *sctx, struct pipe_resource *dst, uint64_t offset, - uint64_t size, uint32_t *clear_value, uint32_t clear_value_size, +void si_clear_buffer(struct si_context *sctx, struct pipe_resource *dst, + uint64_t offset, uint64_t size, uint32_t *clear_value, + uint32_t clear_value_size, unsigned flags, enum si_coherency coher, enum si_clear_method method); void si_screen_clear_buffer(struct si_screen *sscreen, struct pipe_resource *dst, uint64_t offset, - uint64_t size, unsigned value); + uint64_t size, unsigned value, unsigned flags); void si_copy_buffer(struct si_context *sctx, struct pipe_resource *dst, struct pipe_resource *src, - uint64_t dst_offset, uint64_t src_offset, unsigned size); + uint64_t dst_offset, uint64_t src_offset, unsigned size, unsigned flags); void si_compute_copy_image(struct si_context *sctx, struct pipe_resource *dst, unsigned dst_level, struct pipe_resource *src, unsigned src_level, unsigned dstx, unsigned dsty, unsigned dstz, const struct pipe_box *src_box, - bool is_dcc_decompress); + bool is_dcc_decompress, unsigned flags); void si_compute_clear_render_target(struct pipe_context *ctx, struct pipe_surface *dstsurf, const union pipe_color_union *color, unsigned dstx, unsigned dsty, unsigned width, unsigned height, diff --git a/src/gallium/drivers/radeonsi/si_test_blit.c b/src/gallium/drivers/radeonsi/si_test_blit.c index e7222b9127e..7f011e8c4a5 100644 --- a/src/gallium/drivers/radeonsi/si_test_blit.c +++ b/src/gallium/drivers/radeonsi/si_test_blit.c @@ -293,7 +293,7 @@ void si_test_blit(struct si_screen *sscreen) /* clear dst pixels */ uint32_t zero = 0; - si_clear_buffer(sctx, dst, 0, sdst->surface.surf_size, &zero, 4, + si_clear_buffer(sctx, dst, 0, sdst->surface.surf_size, &zero, 4, SI_OP_SYNC_BEFORE_AFTER, SI_COHERENCY_SHADER, SI_AUTO_SELECT_CLEAR_METHOD); memset(dst_cpu.ptr, 0, dst_cpu.layer_stride * tdst.array_size); diff --git a/src/gallium/drivers/radeonsi/si_texture.c b/src/gallium/drivers/radeonsi/si_texture.c index 00b5c512a1d..07e58fb322f 100644 --- a/src/gallium/drivers/radeonsi/si_texture.c +++ b/src/gallium/drivers/radeonsi/si_texture.c @@ -1012,7 +1012,7 @@ static struct si_texture *si_texture_create_object(struct pipe_screen *screen, if (tex->cmask_buffer) { /* Initialize the cmask to 0xCC (= compressed state). */ si_screen_clear_buffer(sscreen, &tex->cmask_buffer->b.b, tex->surface.cmask_offset, - tex->surface.cmask_size, 0xCCCCCCCC); + tex->surface.cmask_size, 0xCCCCCCCC, SI_OP_SYNC_BEFORE_AFTER); } if (tex->surface.htile_offset) { uint32_t clear_value = 0; @@ -1021,7 +1021,7 @@ static struct si_texture *si_texture_create_object(struct pipe_screen *screen, clear_value = 0x0000030F; si_screen_clear_buffer(sscreen, &tex->buffer.b.b, tex->surface.htile_offset, - tex->surface.htile_size, clear_value); + tex->surface.htile_size, clear_value, SI_OP_SYNC_BEFORE_AFTER); } /* Initialize DCC only if the texture is not being imported. */ @@ -1035,17 +1035,17 @@ static struct si_texture *si_texture_create_object(struct pipe_screen *screen, tex->buffer.b.b.nr_samples <= 2) { /* Simple case - all tiles have DCC enabled. */ si_screen_clear_buffer(sscreen, &tex->buffer.b.b, tex->surface.dcc_offset, - tex->surface.dcc_size, DCC_CLEAR_COLOR_0000); + tex->surface.dcc_size, DCC_CLEAR_COLOR_0000, SI_OP_SYNC_BEFORE_AFTER); } else if (sscreen->info.chip_class >= GFX9) { /* Clear to uncompressed. Clearing this to black is complicated. */ si_screen_clear_buffer(sscreen, &tex->buffer.b.b, tex->surface.dcc_offset, - tex->surface.dcc_size, DCC_UNCOMPRESSED); + tex->surface.dcc_size, DCC_UNCOMPRESSED, SI_OP_SYNC_BEFORE_AFTER); } else { /* GFX8: Initialize mipmap levels and multisamples separately. */ if (tex->buffer.b.b.nr_samples >= 2) { /* Clearing this to black is complicated. */ si_screen_clear_buffer(sscreen, &tex->buffer.b.b, tex->surface.dcc_offset, - tex->surface.dcc_size, DCC_UNCOMPRESSED); + tex->surface.dcc_size, DCC_UNCOMPRESSED, SI_OP_SYNC_BEFORE_AFTER); } else { /* Clear the enabled mipmap levels to black. */ unsigned size = 0; @@ -1061,12 +1061,12 @@ static struct si_texture *si_texture_create_object(struct pipe_screen *screen, /* Mipmap levels with DCC. */ if (size) { si_screen_clear_buffer(sscreen, &tex->buffer.b.b, tex->surface.dcc_offset, size, - DCC_CLEAR_COLOR_0000); + DCC_CLEAR_COLOR_0000, SI_OP_SYNC_BEFORE_AFTER); } /* Mipmap levels without DCC. */ if (size != tex->surface.dcc_size) { si_screen_clear_buffer(sscreen, &tex->buffer.b.b, tex->surface.dcc_offset + size, - tex->surface.dcc_size - size, DCC_UNCOMPRESSED); + tex->surface.dcc_size - size, DCC_UNCOMPRESSED, SI_OP_SYNC_BEFORE_AFTER); } } } @@ -1078,7 +1078,7 @@ static struct si_texture *si_texture_create_object(struct pipe_screen *screen, /* Uninitialized DCC can hang the display hw. * Clear to white to indicate that. */ si_screen_clear_buffer(sscreen, &tex->buffer.b.b, tex->surface.display_dcc_offset, - tex->surface.u.gfx9.display_dcc_size, DCC_CLEAR_COLOR_1111); + tex->surface.u.gfx9.display_dcc_size, DCC_CLEAR_COLOR_1111, SI_OP_SYNC_BEFORE_AFTER); } /* Upload the DCC retile map. @@ -1105,7 +1105,7 @@ static struct si_texture *si_texture_create_object(struct pipe_screen *screen, simple_mtx_lock(&sscreen->aux_context_lock); si_copy_buffer(sctx, &tex->dcc_retile_buffer->b.b, &buf->b.b, 0, - 0, buf->b.b.width0); + 0, buf->b.b.width0, SI_OP_SYNC_BEFORE_AFTER); sscreen->aux_context->flush(sscreen->aux_context, NULL, 0); simple_mtx_unlock(&sscreen->aux_context_lock);