hk: limit the number of free BOs in each cmd pool

Some clients like to have long living command pools, each one with
multiple command buffers, and trigger bursts of BO allocations on them.
This leads to the an ever increasing number of BOs allocated on each
command pool, easily reaching over a couple thousand BOs per pool.

The visible effect is an ever increasing amount of VRAM consumption,
which has been observed to easily get over 700MB for a workload
consuming around 4GB of heap.

Here we limit the number of free BOs in each pool to 32. The rest are
dereferenced when the command buffer is reset. In practice, with
workloads triggering bursts of BO allocations on command buffers, those
BOs aren't really released, keeping them in the BO cache and getting
reused shortly after. This has the nice size effect of doing bookkeeping
on the BO cache, trimming it up when needed.

With this change, the tested workloads present a significant reduction
of VRAM, combined with an stabilization of BO allocations.

Signed-off-by: Sergio Lopez <slp@redhat.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/31908>

src/asahi/vulkan/hk_cmd_pool.c

@@ -52,8 +52,8 @@ hk_CreateCommandPool(VkDevice _device,
    VK_FROM_HANDLE(hk_device, device, _device);
    struct hk_cmd_pool *pool;
 
-   pool = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*pool), 8,
-                    VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
+   pool = vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*pool), 8,
+                     VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
    if (pool == NULL)
       return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
 
@@ -92,11 +92,15 @@ hk_cmd_pool_alloc_bo(struct hk_cmd_pool *pool, bool usc,
 {
    struct hk_cmd_bo *bo = NULL;
    if (usc) {
-      if (!list_is_empty(&pool->free_usc_bos))
+      if (!list_is_empty(&pool->free_usc_bos)) {
          bo = list_first_entry(&pool->free_usc_bos, struct hk_cmd_bo, link);
+         pool->num_free_usc_bos--;
+      }
    } else {
-      if (!list_is_empty(&pool->free_bos))
+      if (!list_is_empty(&pool->free_bos)) {
          bo = list_first_entry(&pool->free_bos, struct hk_cmd_bo, link);
+         pool->num_free_bos--;
+      }
    }
    if (bo) {
       list_del(&bo->link);
@@ -110,15 +114,29 @@ hk_cmd_pool_alloc_bo(struct hk_cmd_pool *pool, bool usc,
 void
 hk_cmd_pool_free_bo_list(struct hk_cmd_pool *pool, struct list_head *bos)
 {
-   list_splicetail(bos, &pool->free_bos);
-   list_inithead(bos);
+   list_for_each_entry_safe(struct hk_cmd_bo, bo, bos, link) {
+      list_del(&bo->link);
+      if (pool->num_free_bos > HK_CMD_POOL_BO_MAX) {
+         hk_cmd_bo_destroy(pool, bo);
+      } else {
+         list_addtail(&bo->link, &pool->free_bos);
+         pool->num_free_bos++;
+      }
+   }
 }
 
 void
 hk_cmd_pool_free_usc_bo_list(struct hk_cmd_pool *pool, struct list_head *bos)
 {
-   list_splicetail(bos, &pool->free_usc_bos);
-   list_inithead(bos);
+   list_for_each_entry_safe(struct hk_cmd_bo, bo, bos, link) {
+      list_del(&bo->link);
+      if (pool->num_free_usc_bos > HK_CMD_POOL_BO_MAX) {
+         hk_cmd_bo_destroy(pool, bo);
+      } else {
+         list_addtail(&bo->link, &pool->free_usc_bos);
+         pool->num_free_usc_bos++;
+      }
+   }
 }
 
 VKAPI_ATTR void VKAPI_CALL

src/asahi/vulkan/hk_cmd_pool.h

@@ -11,7 +11,8 @@
 
 #include "vk_command_pool.h"
 
-#define HK_CMD_BO_SIZE 1024 * 128
+#define HK_CMD_BO_SIZE     1024 * 128
+#define HK_CMD_POOL_BO_MAX 32
 
 /* Recyclable command buffer BO, used for both push buffers and upload */
 struct hk_cmd_bo {
@@ -29,6 +30,8 @@ struct hk_cmd_pool {
    /** List of hk_cmd_bo */
    struct list_head free_bos;
    struct list_head free_usc_bos;
+   uint32_t num_free_bos;
+   uint32_t num_free_usc_bos;
 };
 
 VK_DEFINE_NONDISP_HANDLE_CASTS(hk_cmd_pool, vk.base, VkCommandPool,