From 1d5f4341085db88b2ef87098aa7996640e8bea98 Mon Sep 17 00:00:00 2001 From: Samuel Pitoiset Date: Wed, 3 Apr 2024 15:04:26 +0200 Subject: [PATCH] radv: move CP DMA related code to radv_cp_dma.c/h Signed-off-by: Samuel Pitoiset Part-of: --- src/amd/vulkan/meson.build | 2 + src/amd/vulkan/meta/radv_meta_buffer.c | 1 + src/amd/vulkan/radv_cmd_buffer.c | 1 + src/amd/vulkan/radv_cp_dma.c | 373 +++++++++++++++++++++++++ src/amd/vulkan/radv_cp_dma.h | 49 ++++ src/amd/vulkan/radv_private.h | 7 - src/amd/vulkan/si_cmd_buffer.c | 338 ---------------------- 7 files changed, 426 insertions(+), 345 deletions(-) create mode 100644 src/amd/vulkan/radv_cp_dma.c create mode 100644 src/amd/vulkan/radv_cp_dma.h diff --git a/src/amd/vulkan/meson.build b/src/amd/vulkan/meson.build index 73fa575ea3d..2d91bcbe010 100644 --- a/src/amd/vulkan/meson.build +++ b/src/amd/vulkan/meson.build @@ -112,6 +112,8 @@ libradv_files = files( 'radv_buffer_view.h', 'radv_cmd_buffer.c', 'radv_cmd_buffer.h', + 'radv_cp_dma.c', + 'radv_cp_dma.h', 'radv_cp_reg_shadowing.c', 'radv_cp_reg_shadowing.h', 'radv_cs.h', diff --git a/src/amd/vulkan/meta/radv_meta_buffer.c b/src/amd/vulkan/meta/radv_meta_buffer.c index e82e9781ff1..8fe14bbe204 100644 --- a/src/amd/vulkan/meta/radv_meta_buffer.c +++ b/src/amd/vulkan/meta/radv_meta_buffer.c @@ -1,4 +1,5 @@ #include "nir/nir_builder.h" +#include "radv_cp_dma.h" #include "radv_debug.h" #include "radv_meta.h" #include "radv_sdma.h" diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c index 30a58d0b954..83e99efac13 100644 --- a/src/amd/vulkan/radv_cmd_buffer.c +++ b/src/amd/vulkan/radv_cmd_buffer.c @@ -27,6 +27,7 @@ #include "radv_cmd_buffer.h" #include "meta/radv_meta.h" +#include "radv_cp_dma.h" #include "radv_cs.h" #include "radv_debug.h" #include "radv_device_generated_commands.h" diff --git a/src/amd/vulkan/radv_cp_dma.c b/src/amd/vulkan/radv_cp_dma.c new file mode 100644 index 00000000000..1db29263626 --- /dev/null +++ b/src/amd/vulkan/radv_cp_dma.c @@ -0,0 +1,373 @@ +/* + * Copyright © 2016 Red Hat. + * Copyright © 2016 Bas Nieuwenhuizen + * + * based on si_state.c + * Copyright © 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "radv_cp_dma.h" +#include "radv_buffer.h" +#include "radv_cs.h" +#include "radv_debug.h" +#include "radv_private.h" +#include "radv_shader.h" +#include "radv_sqtt.h" +#include "sid.h" + +/* Set this if you want the 3D engine to wait until CP DMA is done. + * It should be set on the last CP DMA packet. */ +#define CP_DMA_SYNC (1 << 0) + +/* Set this if the source data was used as a destination in a previous CP DMA + * packet. It's for preventing a read-after-write (RAW) hazard between two + * CP DMA packets. */ +#define CP_DMA_RAW_WAIT (1 << 1) +#define CP_DMA_USE_L2 (1 << 2) +#define CP_DMA_CLEAR (1 << 3) + +/* Alignment for optimal performance. */ +#define SI_CPDMA_ALIGNMENT 32 + +/* The max number of bytes that can be copied per packet. */ +static inline unsigned +cp_dma_max_byte_count(enum amd_gfx_level gfx_level) +{ + unsigned max = gfx_level >= GFX11 ? 32767 + : gfx_level >= GFX9 ? S_415_BYTE_COUNT_GFX9(~0u) + : S_415_BYTE_COUNT_GFX6(~0u); + + /* make it aligned for optimal performance */ + return max & ~(SI_CPDMA_ALIGNMENT - 1); +} + +/* Emit a CP DMA packet to do a copy from one buffer to another, or to clear + * a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit + * clear value. + */ +static void +radv_cs_emit_cp_dma(struct radv_device *device, struct radeon_cmdbuf *cs, bool predicating, uint64_t dst_va, + uint64_t src_va, unsigned size, unsigned flags) +{ + const struct radv_physical_device *pdev = radv_device_physical(device); + uint32_t header = 0, command = 0; + + assert(size <= cp_dma_max_byte_count(pdev->info.gfx_level)); + + radeon_check_space(device->ws, cs, 9); + if (pdev->info.gfx_level >= GFX9) + command |= S_415_BYTE_COUNT_GFX9(size); + else + command |= S_415_BYTE_COUNT_GFX6(size); + + /* Sync flags. */ + if (flags & CP_DMA_SYNC) + header |= S_411_CP_SYNC(1); + + if (flags & CP_DMA_RAW_WAIT) + command |= S_415_RAW_WAIT(1); + + /* Src and dst flags. */ + if (pdev->info.gfx_level >= GFX9 && !(flags & CP_DMA_CLEAR) && src_va == dst_va) + header |= S_411_DST_SEL(V_411_NOWHERE); /* prefetch only */ + else if (flags & CP_DMA_USE_L2) + header |= S_411_DST_SEL(V_411_DST_ADDR_TC_L2); + + if (flags & CP_DMA_CLEAR) + header |= S_411_SRC_SEL(V_411_DATA); + else if (flags & CP_DMA_USE_L2) + header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2); + + if (pdev->info.gfx_level >= GFX7) { + radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, predicating)); + radeon_emit(cs, header); + radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */ + radeon_emit(cs, src_va >> 32); /* SRC_ADDR_HI [31:0] */ + radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */ + radeon_emit(cs, dst_va >> 32); /* DST_ADDR_HI [31:0] */ + radeon_emit(cs, command); + } else { + assert(!(flags & CP_DMA_USE_L2)); + header |= S_411_SRC_ADDR_HI(src_va >> 32); + radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, predicating)); + radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */ + radeon_emit(cs, header); /* SRC_ADDR_HI [15:0] + flags. */ + radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */ + radeon_emit(cs, (dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */ + radeon_emit(cs, command); + } +} + +static void +radv_emit_cp_dma(struct radv_cmd_buffer *cmd_buffer, uint64_t dst_va, uint64_t src_va, unsigned size, unsigned flags) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + bool predicating = cmd_buffer->state.predicating; + + radv_cs_emit_cp_dma(device, cs, predicating, dst_va, src_va, size, flags); + + /* CP DMA is executed in ME, but index buffers are read by PFP. + * This ensures that ME (CP DMA) is idle before PFP starts fetching + * indices. If we wanted to execute CP DMA in PFP, this packet + * should precede it. + */ + if (flags & CP_DMA_SYNC) { + if (cmd_buffer->qf == RADV_QUEUE_GENERAL) { + radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); + radeon_emit(cs, 0); + } + + /* CP will see the sync flag and wait for all DMAs to complete. */ + cmd_buffer->state.dma_is_busy = false; + } + + if (radv_device_fault_detection_enabled(device)) + radv_cmd_buffer_trace_emit(cmd_buffer); +} + +void +radv_cs_cp_dma_prefetch(const struct radv_device *device, struct radeon_cmdbuf *cs, uint64_t va, unsigned size, + bool predicating) +{ + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_winsys *ws = device->ws; + enum amd_gfx_level gfx_level = pdev->info.gfx_level; + uint32_t header = 0, command = 0; + + if (gfx_level >= GFX11) + size = MIN2(size, 32768 - SI_CPDMA_ALIGNMENT); + + assert(size <= cp_dma_max_byte_count(gfx_level)); + + radeon_check_space(ws, cs, 9); + + uint64_t aligned_va = va & ~(SI_CPDMA_ALIGNMENT - 1); + uint64_t aligned_size = ((va + size + SI_CPDMA_ALIGNMENT - 1) & ~(SI_CPDMA_ALIGNMENT - 1)) - aligned_va; + + if (gfx_level >= GFX9) { + command |= S_415_BYTE_COUNT_GFX9(aligned_size) | S_415_DISABLE_WR_CONFIRM_GFX9(1); + header |= S_411_DST_SEL(V_411_NOWHERE); + } else { + command |= S_415_BYTE_COUNT_GFX6(aligned_size) | S_415_DISABLE_WR_CONFIRM_GFX6(1); + header |= S_411_DST_SEL(V_411_DST_ADDR_TC_L2); + } + + header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2); + + radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, predicating)); + radeon_emit(cs, header); + radeon_emit(cs, aligned_va); /* SRC_ADDR_LO [31:0] */ + radeon_emit(cs, aligned_va >> 32); /* SRC_ADDR_HI [31:0] */ + radeon_emit(cs, aligned_va); /* DST_ADDR_LO [31:0] */ + radeon_emit(cs, aligned_va >> 32); /* DST_ADDR_HI [31:0] */ + radeon_emit(cs, command); +} + +void +radv_cp_dma_prefetch(struct radv_cmd_buffer *cmd_buffer, uint64_t va, unsigned size) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + + radv_cs_cp_dma_prefetch(device, cmd_buffer->cs, va, size, cmd_buffer->state.predicating); + + if (radv_device_fault_detection_enabled(device)) + radv_cmd_buffer_trace_emit(cmd_buffer); +} + +static void +radv_cp_dma_prepare(struct radv_cmd_buffer *cmd_buffer, uint64_t byte_count, uint64_t remaining_size, unsigned *flags) +{ + + /* Flush the caches for the first copy only. + * Also wait for the previous CP DMA operations. + */ + if (cmd_buffer->state.flush_bits) { + radv_emit_cache_flush(cmd_buffer); + *flags |= CP_DMA_RAW_WAIT; + } + + /* Do the synchronization after the last dma, so that all data + * is written to memory. + */ + if (byte_count == remaining_size) + *flags |= CP_DMA_SYNC; +} + +static void +radv_cp_dma_realign_engine(struct radv_cmd_buffer *cmd_buffer, unsigned size) +{ + uint64_t va; + uint32_t offset; + unsigned dma_flags = 0; + unsigned buf_size = SI_CPDMA_ALIGNMENT * 2; + void *ptr; + + assert(size < SI_CPDMA_ALIGNMENT); + + radv_cmd_buffer_upload_alloc(cmd_buffer, buf_size, &offset, &ptr); + + va = radv_buffer_get_va(cmd_buffer->upload.upload_bo); + va += offset; + + radv_cp_dma_prepare(cmd_buffer, size, size, &dma_flags); + + radv_emit_cp_dma(cmd_buffer, va, va + SI_CPDMA_ALIGNMENT, size, dma_flags); +} + +void +radv_cp_dma_buffer_copy(struct radv_cmd_buffer *cmd_buffer, uint64_t src_va, uint64_t dest_va, uint64_t size) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + enum amd_gfx_level gfx_level = pdev->info.gfx_level; + uint64_t main_src_va, main_dest_va; + uint64_t skipped_size = 0, realign_size = 0; + + /* Assume that we are not going to sync after the last DMA operation. */ + cmd_buffer->state.dma_is_busy = true; + + if (pdev->info.family <= CHIP_CARRIZO || pdev->info.family == CHIP_STONEY) { + /* If the size is not aligned, we must add a dummy copy at the end + * just to align the internal counter. Otherwise, the DMA engine + * would slow down by an order of magnitude for following copies. + */ + if (size % SI_CPDMA_ALIGNMENT) + realign_size = SI_CPDMA_ALIGNMENT - (size % SI_CPDMA_ALIGNMENT); + + /* If the copy begins unaligned, we must start copying from the next + * aligned block and the skipped part should be copied after everything + * else has been copied. Only the src alignment matters, not dst. + */ + if (src_va % SI_CPDMA_ALIGNMENT) { + skipped_size = SI_CPDMA_ALIGNMENT - (src_va % SI_CPDMA_ALIGNMENT); + /* The main part will be skipped if the size is too small. */ + skipped_size = MIN2(skipped_size, size); + size -= skipped_size; + } + } + main_src_va = src_va + skipped_size; + main_dest_va = dest_va + skipped_size; + + while (size) { + unsigned dma_flags = 0; + unsigned byte_count = MIN2(size, cp_dma_max_byte_count(gfx_level)); + + if (pdev->info.gfx_level >= GFX9) { + /* DMA operations via L2 are coherent and faster. + * TODO: GFX7-GFX8 should also support this but it + * requires tests/benchmarks. + * + * Also enable on GFX9 so we can use L2 at rest on GFX9+. On Raven + * this didn't seem to be worse. + * + * Note that we only use CP DMA for sizes < RADV_BUFFER_OPS_CS_THRESHOLD, + * which is 4k at the moment, so this is really unlikely to cause + * significant thrashing. + */ + dma_flags |= CP_DMA_USE_L2; + } + + radv_cp_dma_prepare(cmd_buffer, byte_count, size + skipped_size + realign_size, &dma_flags); + + dma_flags &= ~CP_DMA_SYNC; + + radv_emit_cp_dma(cmd_buffer, main_dest_va, main_src_va, byte_count, dma_flags); + + size -= byte_count; + main_src_va += byte_count; + main_dest_va += byte_count; + } + + if (skipped_size) { + unsigned dma_flags = 0; + + radv_cp_dma_prepare(cmd_buffer, skipped_size, size + skipped_size + realign_size, &dma_flags); + + radv_emit_cp_dma(cmd_buffer, dest_va, src_va, skipped_size, dma_flags); + } + if (realign_size) + radv_cp_dma_realign_engine(cmd_buffer, realign_size); +} + +void +radv_cp_dma_clear_buffer(struct radv_cmd_buffer *cmd_buffer, uint64_t va, uint64_t size, unsigned value) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + + if (!size) + return; + + assert(va % 4 == 0 && size % 4 == 0); + + enum amd_gfx_level gfx_level = pdev->info.gfx_level; + + /* Assume that we are not going to sync after the last DMA operation. */ + cmd_buffer->state.dma_is_busy = true; + + while (size) { + unsigned byte_count = MIN2(size, cp_dma_max_byte_count(gfx_level)); + unsigned dma_flags = CP_DMA_CLEAR; + + if (pdev->info.gfx_level >= GFX9) { + /* DMA operations via L2 are coherent and faster. + * TODO: GFX7-GFX8 should also support this but it + * requires tests/benchmarks. + * + * Also enable on GFX9 so we can use L2 at rest on GFX9+. + */ + dma_flags |= CP_DMA_USE_L2; + } + + radv_cp_dma_prepare(cmd_buffer, byte_count, size, &dma_flags); + + /* Emit the clear packet. */ + radv_emit_cp_dma(cmd_buffer, va, value, byte_count, dma_flags); + + size -= byte_count; + va += byte_count; + } +} + +void +radv_cp_dma_wait_for_idle(struct radv_cmd_buffer *cmd_buffer) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + + if (pdev->info.gfx_level < GFX7) + return; + + if (!cmd_buffer->state.dma_is_busy) + return; + + /* Issue a dummy DMA that copies zero bytes. + * + * The DMA engine will see that there's no work to do and skip this + * DMA request, however, the CP will see the sync flag and still wait + * for all DMAs to complete. + */ + radv_emit_cp_dma(cmd_buffer, 0, 0, 0, CP_DMA_SYNC); + + cmd_buffer->state.dma_is_busy = false; +} diff --git a/src/amd/vulkan/radv_cp_dma.h b/src/amd/vulkan/radv_cp_dma.h new file mode 100644 index 00000000000..10cd399c976 --- /dev/null +++ b/src/amd/vulkan/radv_cp_dma.h @@ -0,0 +1,49 @@ +/* + * Copyright © 2016 Red Hat. + * Copyright © 2016 Bas Nieuwenhuizen + * + * based in part on anv driver which is: + * Copyright © 2015 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#ifndef RADV_CP_DMA_H +#define RADV_CP_DMA_H + +#include +#include + +struct radv_device; +struct radeon_cmdbuf; +struct radv_cmd_buffer; + +void radv_cs_cp_dma_prefetch(const struct radv_device *device, struct radeon_cmdbuf *cs, uint64_t va, unsigned size, + bool predicating); + +void radv_cp_dma_prefetch(struct radv_cmd_buffer *cmd_buffer, uint64_t va, unsigned size); + +void radv_cp_dma_buffer_copy(struct radv_cmd_buffer *cmd_buffer, uint64_t src_va, uint64_t dest_va, uint64_t size); + +void radv_cp_dma_clear_buffer(struct radv_cmd_buffer *cmd_buffer, uint64_t va, uint64_t size, unsigned value); + +void radv_cp_dma_wait_for_idle(struct radv_cmd_buffer *cmd_buffer); + +#endif /* RADV_CP_DMA_H */ diff --git a/src/amd/vulkan/radv_private.h b/src/amd/vulkan/radv_private.h index 0504d7c67d9..ff1b6a09e24 100644 --- a/src/amd/vulkan/radv_private.h +++ b/src/amd/vulkan/radv_private.h @@ -229,13 +229,6 @@ void radv_emit_set_predication_state(struct radv_cmd_buffer *cmd_buffer, bool dr uint64_t va); void radv_emit_cond_exec(const struct radv_device *device, struct radeon_cmdbuf *cs, uint64_t va, uint32_t count); -void radv_cp_dma_buffer_copy(struct radv_cmd_buffer *cmd_buffer, uint64_t src_va, uint64_t dest_va, uint64_t size); -void radv_cs_cp_dma_prefetch(const struct radv_device *device, struct radeon_cmdbuf *cs, uint64_t va, unsigned size, - bool predicating); -void radv_cp_dma_prefetch(struct radv_cmd_buffer *cmd_buffer, uint64_t va, unsigned size); -void radv_cp_dma_clear_buffer(struct radv_cmd_buffer *cmd_buffer, uint64_t va, uint64_t size, unsigned value); -void radv_cp_dma_wait_for_idle(struct radv_cmd_buffer *cmd_buffer); - void radv_emit_default_sample_locations(struct radeon_cmdbuf *cs, int nr_samples); unsigned radv_get_default_max_sample_dist(int log_samples); void radv_device_init_msaa(struct radv_device *device); diff --git a/src/amd/vulkan/si_cmd_buffer.c b/src/amd/vulkan/si_cmd_buffer.c index 59f64731db7..381d31ea464 100644 --- a/src/amd/vulkan/si_cmd_buffer.c +++ b/src/amd/vulkan/si_cmd_buffer.c @@ -1592,344 +1592,6 @@ radv_emit_cond_exec(const struct radv_device *device, struct radeon_cmdbuf *cs, } } -/* Set this if you want the 3D engine to wait until CP DMA is done. - * It should be set on the last CP DMA packet. */ -#define CP_DMA_SYNC (1 << 0) - -/* Set this if the source data was used as a destination in a previous CP DMA - * packet. It's for preventing a read-after-write (RAW) hazard between two - * CP DMA packets. */ -#define CP_DMA_RAW_WAIT (1 << 1) -#define CP_DMA_USE_L2 (1 << 2) -#define CP_DMA_CLEAR (1 << 3) - -/* Alignment for optimal performance. */ -#define SI_CPDMA_ALIGNMENT 32 - -/* The max number of bytes that can be copied per packet. */ -static inline unsigned -cp_dma_max_byte_count(enum amd_gfx_level gfx_level) -{ - unsigned max = gfx_level >= GFX11 ? 32767 - : gfx_level >= GFX9 ? S_415_BYTE_COUNT_GFX9(~0u) - : S_415_BYTE_COUNT_GFX6(~0u); - - /* make it aligned for optimal performance */ - return max & ~(SI_CPDMA_ALIGNMENT - 1); -} - -/* Emit a CP DMA packet to do a copy from one buffer to another, or to clear - * a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit - * clear value. - */ -static void -radv_cs_emit_cp_dma(struct radv_device *device, struct radeon_cmdbuf *cs, bool predicating, uint64_t dst_va, - uint64_t src_va, unsigned size, unsigned flags) -{ - const struct radv_physical_device *pdev = radv_device_physical(device); - uint32_t header = 0, command = 0; - - assert(size <= cp_dma_max_byte_count(pdev->info.gfx_level)); - - radeon_check_space(device->ws, cs, 9); - if (pdev->info.gfx_level >= GFX9) - command |= S_415_BYTE_COUNT_GFX9(size); - else - command |= S_415_BYTE_COUNT_GFX6(size); - - /* Sync flags. */ - if (flags & CP_DMA_SYNC) - header |= S_411_CP_SYNC(1); - - if (flags & CP_DMA_RAW_WAIT) - command |= S_415_RAW_WAIT(1); - - /* Src and dst flags. */ - if (pdev->info.gfx_level >= GFX9 && !(flags & CP_DMA_CLEAR) && src_va == dst_va) - header |= S_411_DST_SEL(V_411_NOWHERE); /* prefetch only */ - else if (flags & CP_DMA_USE_L2) - header |= S_411_DST_SEL(V_411_DST_ADDR_TC_L2); - - if (flags & CP_DMA_CLEAR) - header |= S_411_SRC_SEL(V_411_DATA); - else if (flags & CP_DMA_USE_L2) - header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2); - - if (pdev->info.gfx_level >= GFX7) { - radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, predicating)); - radeon_emit(cs, header); - radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */ - radeon_emit(cs, src_va >> 32); /* SRC_ADDR_HI [31:0] */ - radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */ - radeon_emit(cs, dst_va >> 32); /* DST_ADDR_HI [31:0] */ - radeon_emit(cs, command); - } else { - assert(!(flags & CP_DMA_USE_L2)); - header |= S_411_SRC_ADDR_HI(src_va >> 32); - radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, predicating)); - radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */ - radeon_emit(cs, header); /* SRC_ADDR_HI [15:0] + flags. */ - radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */ - radeon_emit(cs, (dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */ - radeon_emit(cs, command); - } -} - -static void -radv_emit_cp_dma(struct radv_cmd_buffer *cmd_buffer, uint64_t dst_va, uint64_t src_va, unsigned size, unsigned flags) -{ - struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); - struct radeon_cmdbuf *cs = cmd_buffer->cs; - bool predicating = cmd_buffer->state.predicating; - - radv_cs_emit_cp_dma(device, cs, predicating, dst_va, src_va, size, flags); - - /* CP DMA is executed in ME, but index buffers are read by PFP. - * This ensures that ME (CP DMA) is idle before PFP starts fetching - * indices. If we wanted to execute CP DMA in PFP, this packet - * should precede it. - */ - if (flags & CP_DMA_SYNC) { - if (cmd_buffer->qf == RADV_QUEUE_GENERAL) { - radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); - radeon_emit(cs, 0); - } - - /* CP will see the sync flag and wait for all DMAs to complete. */ - cmd_buffer->state.dma_is_busy = false; - } - - if (radv_device_fault_detection_enabled(device)) - radv_cmd_buffer_trace_emit(cmd_buffer); -} - -void -radv_cs_cp_dma_prefetch(const struct radv_device *device, struct radeon_cmdbuf *cs, uint64_t va, unsigned size, - bool predicating) -{ - const struct radv_physical_device *pdev = radv_device_physical(device); - struct radeon_winsys *ws = device->ws; - enum amd_gfx_level gfx_level = pdev->info.gfx_level; - uint32_t header = 0, command = 0; - - if (gfx_level >= GFX11) - size = MIN2(size, 32768 - SI_CPDMA_ALIGNMENT); - - assert(size <= cp_dma_max_byte_count(gfx_level)); - - radeon_check_space(ws, cs, 9); - - uint64_t aligned_va = va & ~(SI_CPDMA_ALIGNMENT - 1); - uint64_t aligned_size = ((va + size + SI_CPDMA_ALIGNMENT - 1) & ~(SI_CPDMA_ALIGNMENT - 1)) - aligned_va; - - if (gfx_level >= GFX9) { - command |= S_415_BYTE_COUNT_GFX9(aligned_size) | S_415_DISABLE_WR_CONFIRM_GFX9(1); - header |= S_411_DST_SEL(V_411_NOWHERE); - } else { - command |= S_415_BYTE_COUNT_GFX6(aligned_size) | S_415_DISABLE_WR_CONFIRM_GFX6(1); - header |= S_411_DST_SEL(V_411_DST_ADDR_TC_L2); - } - - header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2); - - radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, predicating)); - radeon_emit(cs, header); - radeon_emit(cs, aligned_va); /* SRC_ADDR_LO [31:0] */ - radeon_emit(cs, aligned_va >> 32); /* SRC_ADDR_HI [31:0] */ - radeon_emit(cs, aligned_va); /* DST_ADDR_LO [31:0] */ - radeon_emit(cs, aligned_va >> 32); /* DST_ADDR_HI [31:0] */ - radeon_emit(cs, command); -} - -void -radv_cp_dma_prefetch(struct radv_cmd_buffer *cmd_buffer, uint64_t va, unsigned size) -{ - struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); - - radv_cs_cp_dma_prefetch(device, cmd_buffer->cs, va, size, cmd_buffer->state.predicating); - - if (radv_device_fault_detection_enabled(device)) - radv_cmd_buffer_trace_emit(cmd_buffer); -} - -static void -radv_cp_dma_prepare(struct radv_cmd_buffer *cmd_buffer, uint64_t byte_count, uint64_t remaining_size, unsigned *flags) -{ - - /* Flush the caches for the first copy only. - * Also wait for the previous CP DMA operations. - */ - if (cmd_buffer->state.flush_bits) { - radv_emit_cache_flush(cmd_buffer); - *flags |= CP_DMA_RAW_WAIT; - } - - /* Do the synchronization after the last dma, so that all data - * is written to memory. - */ - if (byte_count == remaining_size) - *flags |= CP_DMA_SYNC; -} - -static void -radv_cp_dma_realign_engine(struct radv_cmd_buffer *cmd_buffer, unsigned size) -{ - uint64_t va; - uint32_t offset; - unsigned dma_flags = 0; - unsigned buf_size = SI_CPDMA_ALIGNMENT * 2; - void *ptr; - - assert(size < SI_CPDMA_ALIGNMENT); - - radv_cmd_buffer_upload_alloc(cmd_buffer, buf_size, &offset, &ptr); - - va = radv_buffer_get_va(cmd_buffer->upload.upload_bo); - va += offset; - - radv_cp_dma_prepare(cmd_buffer, size, size, &dma_flags); - - radv_emit_cp_dma(cmd_buffer, va, va + SI_CPDMA_ALIGNMENT, size, dma_flags); -} - -void -radv_cp_dma_buffer_copy(struct radv_cmd_buffer *cmd_buffer, uint64_t src_va, uint64_t dest_va, uint64_t size) -{ - struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); - const struct radv_physical_device *pdev = radv_device_physical(device); - enum amd_gfx_level gfx_level = pdev->info.gfx_level; - uint64_t main_src_va, main_dest_va; - uint64_t skipped_size = 0, realign_size = 0; - - /* Assume that we are not going to sync after the last DMA operation. */ - cmd_buffer->state.dma_is_busy = true; - - if (pdev->info.family <= CHIP_CARRIZO || pdev->info.family == CHIP_STONEY) { - /* If the size is not aligned, we must add a dummy copy at the end - * just to align the internal counter. Otherwise, the DMA engine - * would slow down by an order of magnitude for following copies. - */ - if (size % SI_CPDMA_ALIGNMENT) - realign_size = SI_CPDMA_ALIGNMENT - (size % SI_CPDMA_ALIGNMENT); - - /* If the copy begins unaligned, we must start copying from the next - * aligned block and the skipped part should be copied after everything - * else has been copied. Only the src alignment matters, not dst. - */ - if (src_va % SI_CPDMA_ALIGNMENT) { - skipped_size = SI_CPDMA_ALIGNMENT - (src_va % SI_CPDMA_ALIGNMENT); - /* The main part will be skipped if the size is too small. */ - skipped_size = MIN2(skipped_size, size); - size -= skipped_size; - } - } - main_src_va = src_va + skipped_size; - main_dest_va = dest_va + skipped_size; - - while (size) { - unsigned dma_flags = 0; - unsigned byte_count = MIN2(size, cp_dma_max_byte_count(gfx_level)); - - if (pdev->info.gfx_level >= GFX9) { - /* DMA operations via L2 are coherent and faster. - * TODO: GFX7-GFX8 should also support this but it - * requires tests/benchmarks. - * - * Also enable on GFX9 so we can use L2 at rest on GFX9+. On Raven - * this didn't seem to be worse. - * - * Note that we only use CP DMA for sizes < RADV_BUFFER_OPS_CS_THRESHOLD, - * which is 4k at the moment, so this is really unlikely to cause - * significant thrashing. - */ - dma_flags |= CP_DMA_USE_L2; - } - - radv_cp_dma_prepare(cmd_buffer, byte_count, size + skipped_size + realign_size, &dma_flags); - - dma_flags &= ~CP_DMA_SYNC; - - radv_emit_cp_dma(cmd_buffer, main_dest_va, main_src_va, byte_count, dma_flags); - - size -= byte_count; - main_src_va += byte_count; - main_dest_va += byte_count; - } - - if (skipped_size) { - unsigned dma_flags = 0; - - radv_cp_dma_prepare(cmd_buffer, skipped_size, size + skipped_size + realign_size, &dma_flags); - - radv_emit_cp_dma(cmd_buffer, dest_va, src_va, skipped_size, dma_flags); - } - if (realign_size) - radv_cp_dma_realign_engine(cmd_buffer, realign_size); -} - -void -radv_cp_dma_clear_buffer(struct radv_cmd_buffer *cmd_buffer, uint64_t va, uint64_t size, unsigned value) -{ - struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); - const struct radv_physical_device *pdev = radv_device_physical(device); - - if (!size) - return; - - assert(va % 4 == 0 && size % 4 == 0); - - enum amd_gfx_level gfx_level = pdev->info.gfx_level; - - /* Assume that we are not going to sync after the last DMA operation. */ - cmd_buffer->state.dma_is_busy = true; - - while (size) { - unsigned byte_count = MIN2(size, cp_dma_max_byte_count(gfx_level)); - unsigned dma_flags = CP_DMA_CLEAR; - - if (pdev->info.gfx_level >= GFX9) { - /* DMA operations via L2 are coherent and faster. - * TODO: GFX7-GFX8 should also support this but it - * requires tests/benchmarks. - * - * Also enable on GFX9 so we can use L2 at rest on GFX9+. - */ - dma_flags |= CP_DMA_USE_L2; - } - - radv_cp_dma_prepare(cmd_buffer, byte_count, size, &dma_flags); - - /* Emit the clear packet. */ - radv_emit_cp_dma(cmd_buffer, va, value, byte_count, dma_flags); - - size -= byte_count; - va += byte_count; - } -} - -void -radv_cp_dma_wait_for_idle(struct radv_cmd_buffer *cmd_buffer) -{ - struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); - const struct radv_physical_device *pdev = radv_device_physical(device); - - if (pdev->info.gfx_level < GFX7) - return; - - if (!cmd_buffer->state.dma_is_busy) - return; - - /* Issue a dummy DMA that copies zero bytes. - * - * The DMA engine will see that there's no work to do and skip this - * DMA request, however, the CP will see the sync flag and still wait - * for all DMAs to complete. - */ - radv_emit_cp_dma(cmd_buffer, 0, 0, 0, CP_DMA_SYNC); - - cmd_buffer->state.dma_is_busy = false; -} - /* For MSAA sample positions. */ #define FILL_SREG(s0x, s0y, s1x, s1y, s2x, s2y, s3x, s3y) \ ((((unsigned)(s0x)&0xf) << 0) | (((unsigned)(s0y)&0xf) << 4) | (((unsigned)(s1x)&0xf) << 8) | \