From 2c18b0c6aa1ec909b1732f4b1153f7feeebff3ce Mon Sep 17 00:00:00 2001 From: Sergio Lopez Date: Tue, 15 Oct 2024 09:59:45 +0200 Subject: [PATCH] hk: limit the number of free BOs in each cmd pool Some clients like to have long living command pools, each one with multiple command buffers, and trigger bursts of BO allocations on them. This leads to the an ever increasing number of BOs allocated on each command pool, easily reaching over a couple thousand BOs per pool. The visible effect is an ever increasing amount of VRAM consumption, which has been observed to easily get over 700MB for a workload consuming around 4GB of heap. Here we limit the number of free BOs in each pool to 32. The rest are dereferenced when the command buffer is reset. In practice, with workloads triggering bursts of BO allocations on command buffers, those BOs aren't really released, keeping them in the BO cache and getting reused shortly after. This has the nice size effect of doing bookkeeping on the BO cache, trimming it up when needed. With this change, the tested workloads present a significant reduction of VRAM, combined with an stabilization of BO allocations. Signed-off-by: Sergio Lopez Part-of: --- src/asahi/vulkan/hk_cmd_pool.c | 34 ++++++++++++++++++++++++++-------- src/asahi/vulkan/hk_cmd_pool.h | 5 ++++- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/asahi/vulkan/hk_cmd_pool.c b/src/asahi/vulkan/hk_cmd_pool.c index 8834b5543d2..d2e68eddad6 100644 --- a/src/asahi/vulkan/hk_cmd_pool.c +++ b/src/asahi/vulkan/hk_cmd_pool.c @@ -52,8 +52,8 @@ hk_CreateCommandPool(VkDevice _device, VK_FROM_HANDLE(hk_device, device, _device); struct hk_cmd_pool *pool; - pool = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*pool), 8, - VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + pool = vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*pool), 8, + VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (pool == NULL) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); @@ -92,11 +92,15 @@ hk_cmd_pool_alloc_bo(struct hk_cmd_pool *pool, bool usc, { struct hk_cmd_bo *bo = NULL; if (usc) { - if (!list_is_empty(&pool->free_usc_bos)) + if (!list_is_empty(&pool->free_usc_bos)) { bo = list_first_entry(&pool->free_usc_bos, struct hk_cmd_bo, link); + pool->num_free_usc_bos--; + } } else { - if (!list_is_empty(&pool->free_bos)) + if (!list_is_empty(&pool->free_bos)) { bo = list_first_entry(&pool->free_bos, struct hk_cmd_bo, link); + pool->num_free_bos--; + } } if (bo) { list_del(&bo->link); @@ -110,15 +114,29 @@ hk_cmd_pool_alloc_bo(struct hk_cmd_pool *pool, bool usc, void hk_cmd_pool_free_bo_list(struct hk_cmd_pool *pool, struct list_head *bos) { - list_splicetail(bos, &pool->free_bos); - list_inithead(bos); + list_for_each_entry_safe(struct hk_cmd_bo, bo, bos, link) { + list_del(&bo->link); + if (pool->num_free_bos > HK_CMD_POOL_BO_MAX) { + hk_cmd_bo_destroy(pool, bo); + } else { + list_addtail(&bo->link, &pool->free_bos); + pool->num_free_bos++; + } + } } void hk_cmd_pool_free_usc_bo_list(struct hk_cmd_pool *pool, struct list_head *bos) { - list_splicetail(bos, &pool->free_usc_bos); - list_inithead(bos); + list_for_each_entry_safe(struct hk_cmd_bo, bo, bos, link) { + list_del(&bo->link); + if (pool->num_free_usc_bos > HK_CMD_POOL_BO_MAX) { + hk_cmd_bo_destroy(pool, bo); + } else { + list_addtail(&bo->link, &pool->free_usc_bos); + pool->num_free_usc_bos++; + } + } } VKAPI_ATTR void VKAPI_CALL diff --git a/src/asahi/vulkan/hk_cmd_pool.h b/src/asahi/vulkan/hk_cmd_pool.h index 2e485d25f0b..2b60701c24d 100644 --- a/src/asahi/vulkan/hk_cmd_pool.h +++ b/src/asahi/vulkan/hk_cmd_pool.h @@ -11,7 +11,8 @@ #include "vk_command_pool.h" -#define HK_CMD_BO_SIZE 1024 * 128 +#define HK_CMD_BO_SIZE 1024 * 128 +#define HK_CMD_POOL_BO_MAX 32 /* Recyclable command buffer BO, used for both push buffers and upload */ struct hk_cmd_bo { @@ -29,6 +30,8 @@ struct hk_cmd_pool { /** List of hk_cmd_bo */ struct list_head free_bos; struct list_head free_usc_bos; + uint32_t num_free_bos; + uint32_t num_free_usc_bos; }; VK_DEFINE_NONDISP_HANDLE_CASTS(hk_cmd_pool, vk.base, VkCommandPool,