diff --git a/src/broadcom/vulkan/v3dv_descriptor_set.c b/src/broadcom/vulkan/v3dv_descriptor_set.c
index 52edcf11e2d..d3430d0e894 100644
--- a/src/broadcom/vulkan/v3dv_descriptor_set.c
+++ b/src/broadcom/vulkan/v3dv_descriptor_set.c
@@ -515,6 +515,8 @@ v3dv_CreateDescriptorPool(VkDevice _device,
       pool->bo = NULL;
    }
 
+   list_inithead(&pool->set_list);
+
    *pDescriptorPool = v3dv_descriptor_pool_to_handle(pool);
 
    return VK_SUCCESS;
@@ -532,8 +534,6 @@ descriptor_set_destroy(struct v3dv_device *device,
 {
    assert(!pool->host_memory_base);
 
-   v3dv_descriptor_set_layout_unref(device, set->layout);
-
    if (free_bo && !pool->host_memory_base) {
       for (uint32_t i = 0; i < pool->entry_count; i++) {
          if (pool->entries[i].set == set) {
@@ -558,6 +558,11 @@ v3dv_DestroyDescriptorPool(VkDevice _device,
    if (!pool)
       return;
 
+   list_for_each_entry_safe(struct v3dv_descriptor_set, set,
+                            &pool->set_list, pool_link) {
+      v3dv_descriptor_set_layout_unref(device, set->layout);
+   }
+
    if (!pool->host_memory_base) {
       for(int i = 0; i < pool->entry_count; ++i) {
          descriptor_set_destroy(device, pool, pool->entries[i].set, false);
@@ -580,6 +585,12 @@ v3dv_ResetDescriptorPool(VkDevice _device,
    V3DV_FROM_HANDLE(v3dv_device, device, _device);
    V3DV_FROM_HANDLE(v3dv_descriptor_pool, pool, descriptorPool);
 
+   list_for_each_entry_safe(struct v3dv_descriptor_set, set,
+                            &pool->set_list, pool_link) {
+      v3dv_descriptor_set_layout_unref(device, set->layout);
+   }
+   list_inithead(&pool->set_list);
+
    if (!pool->host_memory_base) {
       for(int i = 0; i < pool->entry_count; ++i) {
          descriptor_set_destroy(device, pool, pool->entries[i].set, false);
@@ -909,6 +920,8 @@ descriptor_set_create(struct v3dv_device *device,
    }
 
    v3dv_descriptor_set_layout_ref(layout);
+   list_addtail(&set->pool_link, &pool->set_list);
+
    *out_set = set;
 
    return VK_SUCCESS;
@@ -959,8 +972,13 @@ v3dv_FreeDescriptorSets(VkDevice _device,
 
    for (uint32_t i = 0; i < count; i++) {
       V3DV_FROM_HANDLE(v3dv_descriptor_set, set, pDescriptorSets[i]);
-      if (set && !pool->host_memory_base)
-         descriptor_set_destroy(device, pool, set, true);
+
+      if (set) {
+         v3dv_descriptor_set_layout_unref(device, set->layout);
+         list_del(&set->pool_link);
+         if (!pool->host_memory_base)
+            descriptor_set_destroy(device, pool, set, true);
+      }
    }
 
    return VK_SUCCESS;
diff --git a/src/broadcom/vulkan/v3dv_private.h b/src/broadcom/vulkan/v3dv_private.h
index c61c278db82..5e34d4a471b 100644
--- a/src/broadcom/vulkan/v3dv_private.h
+++ b/src/broadcom/vulkan/v3dv_private.h
@@ -1729,6 +1729,9 @@ struct v3dv_descriptor_pool_entry
 struct v3dv_descriptor_pool {
    struct vk_object_base base;
 
+   /* A list with all descriptor sets allocated from the pool. */
+   struct list_head set_list;
+
    /* If this descriptor pool has been allocated for the driver for internal
     * use, typically to implement meta operations.
     */
@@ -1758,6 +1761,9 @@ struct v3dv_descriptor_pool {
 struct v3dv_descriptor_set {
    struct vk_object_base base;
 
+   /* List link into the list of all sets allocated from the pool */
+   struct list_head pool_link;
+
    struct v3dv_descriptor_pool *pool;
 
    struct v3dv_descriptor_set_layout *layout;