AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvk/upload_queue: Use an nvk_mem_stream internally

Browse Source 
Reviewed-by: Mel Henning <mhenning@darkrefraction.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/35356>
This commit is contained in:
Faith Ekstrand
2025-06-04 18:49:33 -04:00
committed by Marge Bot
parent e206962409
commit fdec18dfa5
3 changed files with 68 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/nouveau/vulkan/nvk_queue.c
+1 -1
View File
@@ -274,7 +274,7 @@ nvk_queue_submit_exec(struct nvk_queue *queue,
if (upload_time_point > 0) {
struct vk_sync_wait wait = {
.sync = dev->upload.sync,
.sync = dev->upload.stream.sync,
.stage_mask = ~0,
.wait_value = upload_time_point,
};
src/nouveau/vulkan/nvk_upload_queue.c
+58 -194
View File
@@ -13,56 +13,10 @@
#include "nv_push.h"
#include "nv_push_cl90b5.h"
#define NVK_UPLOAD_MEM_SIZE 64*1024
struct nvk_upload_mem {
struct nvkmd_mem *mem;
/** Link in nvk_upload_queue::recycle */
struct list_head link;
/** Time point at which point this BO will be idle */
uint64_t idle_time_point;
};
static VkResult
nvk_upload_mem_create(struct nvk_device *dev,
struct nvk_upload_mem **mem_out)
{
struct nvk_upload_mem *mem;
VkResult result;
mem = vk_zalloc(&dev->vk.alloc, sizeof(*mem), 8,
VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
if (mem == NULL)
return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);
result = nvkmd_dev_alloc_mapped_mem(dev->nvkmd, &dev->vk.base,
NVK_UPLOAD_MEM_SIZE, 0, NVKMD_MEM_GART,
NVKMD_MEM_MAP_WR, &mem->mem);
if (result != VK_SUCCESS) {
vk_free(&dev->vk.alloc, mem);
return result;
}
*mem_out = mem;
return VK_SUCCESS;
}
static void
nvk_upload_mem_destroy(struct nvk_device *dev,
struct nvk_upload_mem *mem)
{
nvkmd_mem_unref(mem->mem);
vk_free(&dev->vk.alloc, mem);
}
VkResult
nvk_upload_queue_init(struct nvk_device *dev,
struct nvk_upload_queue *queue)
{
const struct nvk_physical_device *pdev = nvk_device_physical(dev);
VkResult result;
memset(queue, 0, sizeof(*queue));
@@ -74,15 +28,11 @@ nvk_upload_queue_init(struct nvk_device *dev,
if (result != VK_SUCCESS)
goto fail_mutex;
const struct vk_sync_type *sync_type = pdev->nvkmd->sync_types[0];
assert(sync_type->features & VK_SYNC_FEATURE_TIMELINE);
result = vk_sync_create(&dev->vk, sync_type, VK_SYNC_IS_TIMELINE,
0, &queue->sync);
result = nvk_mem_stream_init(dev, &queue->stream);
if (result != VK_SUCCESS)
goto fail_ctx;
list_inithead(&queue->recycle);
nv_push_init(&queue->push, queue->push_data, ARRAY_SIZE(queue->push_data));
return VK_SUCCESS;
@@ -98,13 +48,7 @@ void
nvk_upload_queue_finish(struct nvk_device *dev,
struct nvk_upload_queue *queue)
{
list_for_each_entry_safe(struct nvk_upload_mem, mem, &queue->recycle, link)
nvk_upload_mem_destroy(dev, mem);
if (queue->mem != NULL)
nvk_upload_mem_destroy(dev, queue->mem);
vk_sync_destroy(&dev->vk, queue->sync);
nvk_mem_stream_finish(dev, &queue->stream);
nvkmd_ctx_destroy(queue->ctx);
simple_mtx_destroy(&queue->mutex);
}
@@ -116,44 +60,27 @@ nvk_upload_queue_flush_locked(struct nvk_device *dev,
{
VkResult result;
if (queue->mem == NULL || queue->mem_push_start == queue->mem_push_end) {
if (time_point_out != NULL)
*time_point_out = queue->last_time_point;
if (time_point_out != NULL)
*time_point_out = queue->last_time_point;
if (nv_push_dw_count(&queue->push) == 0)
return VK_SUCCESS;
}
uint64_t time_point = queue->last_time_point + 1;
if (time_point == UINT64_MAX)
abort();
const struct nvkmd_ctx_exec exec = {
.addr = queue->mem->mem->va->addr + queue->mem_push_start,
.size_B = queue->mem_push_end - queue->mem_push_start,
};
result = nvkmd_ctx_exec(queue->ctx, &dev->vk.base, 1, &exec);
if (result != VK_SUCCESS)
return result;
const struct vk_sync_signal signal = {
.sync = queue->sync,
.stage_mask = ~0,
.signal_value = time_point,
};
result = nvkmd_ctx_signal(queue->ctx, &dev->vk.base, 1, &signal);
if (result != VK_SUCCESS)
return result;
/* Wait until now to update last_time_point so that, if we do fail and lose
* the device, nvk_upload_queue_sync won't wait forever on a time point
* that will never signal.
/* nvk_mem_stream_flush() will not update last_time_point if it fails. If
* we do fail and lose the device, nvk_upload_queue_sync won't wait forever
* on a time point that will never signal.
*/
queue->last_time_point = time_point;
result = nvk_mem_stream_push(dev, &queue->stream, queue->ctx,
queue->push_data,
nv_push_dw_count(&queue->push),
&queue->last_time_point);
if (result != VK_SUCCESS)
return result;
queue->mem->idle_time_point = time_point;
queue->mem_push_start = queue->mem_push_end;
nv_push_init(&queue->push, queue->push_data, ARRAY_SIZE(queue->push_data));
if (time_point_out != NULL)
*time_point_out = time_point;
*time_point_out = queue->last_time_point;
return VK_SUCCESS;
}
@@ -185,7 +112,7 @@ nvk_upload_queue_sync_locked(struct nvk_device *dev,
if (queue->last_time_point == 0)
return VK_SUCCESS;
return vk_sync_wait(&dev->vk, queue->sync, queue->last_time_point,
return vk_sync_wait(&dev->vk, queue->stream.sync, queue->last_time_point,
VK_SYNC_WAIT_COMPLETE, UINT64_MAX);
}
@@ -202,53 +129,6 @@ nvk_upload_queue_sync(struct nvk_device *dev,
return result;
}
static VkResult
nvk_upload_queue_reserve(struct nvk_device *dev,
struct nvk_upload_queue *queue,
uint32_t min_mem_size)
{
VkResult result;
assert(min_mem_size <= NVK_UPLOAD_MEM_SIZE);
assert(queue->mem_push_end <= queue->mem_data_start);
if (queue->mem != NULL) {
if (queue->mem_data_start - queue->mem_push_end >= min_mem_size)
return VK_SUCCESS;
/* Not enough room in the BO. Flush and add it to the recycle list */
result = nvk_upload_queue_flush_locked(dev, queue, NULL);
if (result != VK_SUCCESS)
return result;
assert(queue->mem_push_start == queue->mem_push_end);
list_addtail(&queue->mem->link, &queue->recycle);
queue->mem = NULL;
}
assert(queue->mem == NULL);
queue->mem_push_start = queue->mem_push_end = 0;
queue->mem_data_start = NVK_UPLOAD_MEM_SIZE;
/* Try to pop an idle BO off the recycle list */
if (!list_is_empty(&queue->recycle)) {
uint64_t time_point_passed = 0;
result = vk_sync_get_value(&dev->vk, queue->sync, &time_point_passed);
if (result != VK_SUCCESS)
return result;
struct nvk_upload_mem *mem =
list_first_entry(&queue->recycle, struct nvk_upload_mem, link);
if (time_point_passed >= mem->idle_time_point) {
list_del(&mem->link);
queue->mem = mem;
return VK_SUCCESS;
}
}
return nvk_upload_mem_create(dev, &queue->mem);
}
static VkResult
nvk_upload_queue_upload_locked(struct nvk_device *dev,
struct nvk_upload_queue *queue,
@@ -262,45 +142,37 @@ nvk_upload_queue_upload_locked(struct nvk_device *dev,
while (size > 0) {
const uint32_t cmd_size_dw = 12;
const uint32_t cmd_size = cmd_size_dw * 4;
if (queue->push.end + cmd_size_dw > queue->push.limit) {
result = nvk_upload_queue_flush_locked(dev, queue, NULL);
if (result != VK_SUCCESS)
return result;
}
struct nv_push *p = &queue->push;
/* Don't split the upload for stmall stuff. If it's under 1KB and we
* can't fit it in the current buffer, just get another.
*/
const uint32_t min_size = cmd_size + MIN2(size, 1024);
result = nvk_upload_queue_reserve(dev, queue, min_size);
const uint32_t data_size = MIN2(size, NVK_MEM_STREAM_MAX_ALLOC_SIZE);
uint64_t data_addr;
void *data_map;
result = nvk_mem_stream_alloc(dev, &queue->stream, data_size, 4,
&data_addr, &data_map);
if (result != VK_SUCCESS)
return result;
assert(queue->mem != NULL);
assert(queue->mem_data_start > queue->mem_push_end);
const uint32_t avail = queue->mem_data_start - queue->mem_push_end;
assert(avail >= min_size);
const uint32_t data_size = MIN2(size, avail - cmd_size);
const uint32_t data_mem_offset = queue->mem_data_start - data_size;
assert(queue->mem_push_end + cmd_size <= data_mem_offset);
const uint64_t data_addr = queue->mem->mem->va->addr + data_mem_offset;
memcpy(queue->mem->mem->map + data_mem_offset, src, data_size);
queue->mem_data_start = data_mem_offset;
struct nv_push p;
nv_push_init(&p, queue->mem->mem->map + queue->mem_push_end, cmd_size_dw);
memcpy(data_map, src, data_size);
assert(data_size <= (1 << 17));
P_MTHD(&p, NV90B5, OFFSET_IN_UPPER);
P_NV90B5_OFFSET_IN_UPPER(&p, data_addr >> 32);
P_NV90B5_OFFSET_IN_LOWER(&p, data_addr & 0xffffffff);
P_NV90B5_OFFSET_OUT_UPPER(&p, dst_addr >> 32);
P_NV90B5_OFFSET_OUT_LOWER(&p, dst_addr & 0xffffffff);
P_NV90B5_PITCH_IN(&p, data_size);
P_NV90B5_PITCH_OUT(&p, data_size);
P_NV90B5_LINE_LENGTH_IN(&p, data_size);
P_NV90B5_LINE_COUNT(&p, 1);
P_MTHD(p, NV90B5, OFFSET_IN_UPPER);
P_NV90B5_OFFSET_IN_UPPER(p, data_addr >> 32);
P_NV90B5_OFFSET_IN_LOWER(p, data_addr & 0xffffffff);
P_NV90B5_OFFSET_OUT_UPPER(p, dst_addr >> 32);
P_NV90B5_OFFSET_OUT_LOWER(p, dst_addr & 0xffffffff);
P_NV90B5_PITCH_IN(p, data_size);
P_NV90B5_PITCH_OUT(p, data_size);
P_NV90B5_LINE_LENGTH_IN(p, data_size);
P_NV90B5_LINE_COUNT(p, 1);
P_IMMD(&p, NV90B5, LAUNCH_DMA, {
P_IMMD(p, NV90B5, LAUNCH_DMA, {
.data_transfer_type = DATA_TRANSFER_TYPE_NON_PIPELINED,
.multi_line_enable = MULTI_LINE_ENABLE_FALSE,
.flush_enable = FLUSH_ENABLE_TRUE,
@@ -308,9 +180,6 @@ nvk_upload_queue_upload_locked(struct nvk_device *dev,
.dst_memory_layout = DST_MEMORY_LAYOUT_PITCH,
});
assert(nv_push_dw_count(&p) <= cmd_size_dw);
queue->mem_push_end += nv_push_dw_count(&p) * 4;
dst_addr += data_size;
src += data_size;
size -= data_size;
@@ -346,11 +215,12 @@ nvk_upload_queue_fill_locked(struct nvk_device *dev,
while (size > 0) {
const uint32_t cmd_size_dw = 14;
const uint32_t cmd_size = cmd_size_dw * 4;
result = nvk_upload_queue_reserve(dev, queue, cmd_size);
if (result != VK_SUCCESS)
return result;
if (queue->push.end + cmd_size_dw > queue->push.limit) {
result = nvk_upload_queue_flush_locked(dev, queue, NULL);
if (result != VK_SUCCESS)
return result;
}
struct nv_push *p = &queue->push;
const uint32_t max_dim = 1 << 17;
uint32_t width_B, height;
@@ -363,19 +233,16 @@ nvk_upload_queue_fill_locked(struct nvk_device *dev,
}
assert(width_B * height <= size);
struct nv_push p;
nv_push_init(&p, queue->mem->mem->map + queue->mem_push_end, cmd_size_dw);
P_MTHD(p, NV90B5, OFFSET_OUT_UPPER);
P_NV90B5_OFFSET_OUT_UPPER(p, dst_addr >> 32);
P_NV90B5_OFFSET_OUT_LOWER(p, dst_addr & 0xffffffff);
P_NV90B5_PITCH_IN(p, width_B);
P_NV90B5_PITCH_OUT(p, width_B);
P_NV90B5_LINE_LENGTH_IN(p, width_B / 4);
P_NV90B5_LINE_COUNT(p, height);
P_MTHD(&p, NV90B5, OFFSET_OUT_UPPER);
P_NV90B5_OFFSET_OUT_UPPER(&p, dst_addr >> 32);
P_NV90B5_OFFSET_OUT_LOWER(&p, dst_addr & 0xffffffff);
P_NV90B5_PITCH_IN(&p, width_B);
P_NV90B5_PITCH_OUT(&p, width_B);
P_NV90B5_LINE_LENGTH_IN(&p, width_B / 4);
P_NV90B5_LINE_COUNT(&p, height);
P_IMMD(&p, NV90B5, SET_REMAP_CONST_A, data);
P_IMMD(&p, NV90B5, SET_REMAP_COMPONENTS, {
P_IMMD(p, NV90B5, SET_REMAP_CONST_A, data);
P_IMMD(p, NV90B5, SET_REMAP_COMPONENTS, {
.dst_x = DST_X_CONST_A,
.dst_y = DST_Y_CONST_A,
.dst_z = DST_Z_CONST_A,
@@ -385,7 +252,7 @@ nvk_upload_queue_fill_locked(struct nvk_device *dev,
.num_dst_components = NUM_DST_COMPONENTS_ONE,
});
P_IMMD(&p, NV90B5, LAUNCH_DMA, {
P_IMMD(p, NV90B5, LAUNCH_DMA, {
.data_transfer_type = DATA_TRANSFER_TYPE_NON_PIPELINED,
.multi_line_enable = height > 1,
.flush_enable = FLUSH_ENABLE_TRUE,
@@ -394,9 +261,6 @@ nvk_upload_queue_fill_locked(struct nvk_device *dev,
.remap_enable = REMAP_ENABLE_TRUE,
});
assert(nv_push_dw_count(&p) <= cmd_size_dw);
queue->mem_push_end += nv_push_dw_count(&p) * 4;
dst_addr += width_B * height;
size -= width_B * height;
}
src/nouveau/vulkan/nvk_upload_queue.h
+9 -21
View File
@@ -2,43 +2,31 @@
* Copyright © 2024 Collabora Ltd. and Red Hat Inc.
* SPDX-License-Identifier: MIT
*/
#ifndef NVK_DMA_QUEUE_H
#define NVK_DMA_QUEUE_H 1
#ifndef NVK_UPLOAD_QUEUE_H
#define NVK_UPLOAD_QUEUE_H 1
#include "nvk_private.h"
#include "nvk_mem_stream.h"
#include "util/list.h"
#include "nv_push.h"
#include "util/simple_mtx.h"
struct nvk_device;
struct nvk_upload_mem;
struct nvkmd_mem;
struct nvkmd_ctx;
struct vk_sync;
struct nvk_upload_queue {
simple_mtx_t mutex;
struct nvkmd_ctx *ctx;
struct vk_sync *sync;
struct nvk_mem_stream stream;
uint64_t last_time_point;
struct nvk_upload_mem *mem;
/* We grow the buffer from both ends. Pushbuf data goes at the start of
* the buffer and upload data at the tail.
*/
uint32_t mem_push_start;
uint32_t mem_push_end;
uint32_t mem_data_start;
/* list of nvk_upload_mem */
struct list_head recycle;
uint32_t push_data[4096];
struct nv_push push;
};
VkResult nvk_upload_queue_init(struct nvk_device *dev,
struct nvk_upload_queue *queue);
void nvk_upload_queue_finish(struct nvk_device *dev,
struct nvk_upload_queue *queue);
@@ -58,4 +46,4 @@ VkResult nvk_upload_queue_fill(struct nvk_device *dev,
struct nvk_upload_queue *queue,
uint64_t dst_addr, uint32_t data, size_t size);
#endif /* NVK_DMA_QUEUE_H */
#endif /* NVK_UPLOAD_QUEUE_H */