diff --git a/src/freedreno/vulkan/tu_descriptor_set.cc b/src/freedreno/vulkan/tu_descriptor_set.cc
index abe040d04ed..534f76dd618 100644
--- a/src/freedreno/vulkan/tu_descriptor_set.cc
+++ b/src/freedreno/vulkan/tu_descriptor_set.cc
@@ -617,58 +617,27 @@ tu_descriptor_set_create(struct tu_device *device,
          return vk_error(device, VK_ERROR_OUT_OF_POOL_MEMORY);
       }
 
-      /* try to allocate linearly first, so that we don't spend
-       * time looking for gaps if the app only allocates &
-       * resets via the pool. */
-      if (pool->current_offset + layout_size <= pool->size) {
-         set->mapped_ptr = (uint32_t*)(pool_base(pool) + pool->current_offset);
-         set->va = pool->host_bo ? 0 : pool->bo->iova + pool->current_offset;
+      uint64_t current_offset = pool->current_offset;
 
-         if (!pool->host_memory_base) {
-            pool->entries[pool->entry_count].offset = pool->current_offset;
-            pool->entries[pool->entry_count].size = layout_size;
-            pool->entries[pool->entry_count].set = set;
-            pool->entry_count++;
-         }
-         pool->current_offset += layout_size;
-      } else if (!pool->host_memory_base) {
-         uint64_t offset = 0;
-         int index;
+      if (!pool->host_memory_base) {
+         uint64_t pool_vma_offset =
+            util_vma_heap_alloc(&pool->bo_heap, set->size, 1);
+         if (!pool_vma_offset)
+            return vk_error(device, VK_ERROR_FRAGMENTED_POOL);
 
-         for (index = 0; index < pool->entry_count; ++index) {
-            if (pool->entries[index].size == 0)
-               continue;
-
-            if (pool->entries[index].offset - offset >= layout_size)
-               break;
-            offset = pool->entries[index].offset + pool->entries[index].size;
-         }
-
-         if (pool->size - offset < layout_size) {
-            vk_object_free(&device->vk, NULL, set);
+         assert(pool_vma_offset >= TU_POOL_HEAP_OFFSET &&
+                pool_vma_offset <= pool->size + TU_POOL_HEAP_OFFSET);
+         set->offset = pool_vma_offset - TU_POOL_HEAP_OFFSET;
+         current_offset = set->offset;
+      } else {
+         if (current_offset + set->size > pool->size)
             return vk_error(device, VK_ERROR_OUT_OF_POOL_MEMORY);
-         }
 
-         set->mapped_ptr = (uint32_t*)(pool_base(pool) + offset);
-         set->va = pool->host_bo ? 0 : pool->bo->iova + offset;
+         pool->current_offset += set->size;
+      }
 
-         memmove(&pool->entries[index + 1], &pool->entries[index],
-            sizeof(pool->entries[0]) * (pool->entry_count - index));
-         pool->entries[index].offset = offset;
-         pool->entries[index].size = layout_size;
-         pool->entries[index].set = set;
-         pool->entry_count++;
-      } else
-         return vk_error(device, VK_ERROR_OUT_OF_POOL_MEMORY);
-   } else if (!pool->host_memory_base) {
-      /* Also keep track of zero sized descriptor sets, such as descriptor
-       * sets with just dynamic descriptors, so that we can free the sets on
-       * vkDestroyDescriptorPool().
-       */
-      pool->entries[pool->entry_count].offset = ~0;
-      pool->entries[pool->entry_count].size = 0;
-      pool->entries[pool->entry_count].set = set;
-      pool->entry_count++;
+      set->mapped_ptr = (uint32_t*)(pool_base(pool) + current_offset);
+      set->va = pool->host_bo ? 0 : pool->bo->iova + current_offset;
    }
 
    if (layout->has_immutable_samplers) {
@@ -693,6 +662,7 @@ tu_descriptor_set_create(struct tu_device *device,
 
    vk_descriptor_set_layout_ref(&layout->vk);
    list_addtail(&set->pool_link, &pool->desc_sets);
+   pool->entry_count++;
 
    *out_set = set;
    return VK_SUCCESS;
@@ -701,30 +671,17 @@ tu_descriptor_set_create(struct tu_device *device,
 static void
 tu_descriptor_set_destroy(struct tu_device *device,
              struct tu_descriptor_pool *pool,
-             struct tu_descriptor_set *set,
-             bool free_bo)
+             struct tu_descriptor_set *set)
 {
    assert(!pool->host_memory_base);
 
-   if (free_bo && !pool->host_memory_base) {
-      for (int i = 0; i < pool->entry_count; ++i) {
-         if (pool->entries[i].set == set) {
-            if (set->size) {
-               ASSERTED uint32_t offset =
-                  (uint8_t *) set->mapped_ptr - pool_base(pool);
-               assert(pool->entries[i].offset == offset);
-            } else {
-               assert(pool->entries[i].size == 0);
-            }
-
-            memmove(&pool->entries[i], &pool->entries[i+1],
-               sizeof(pool->entries[i]) * (pool->entry_count - i - 1));
-            --pool->entry_count;
-            break;
-         }
-      }
+   if (set->size) {
+      util_vma_heap_free(&pool->bo_heap,
+                         (uint64_t) set->offset + TU_POOL_HEAP_OFFSET,
+                         set->size);
    }
 
+   pool->entry_count--;
    vk_object_free(&device->vk, NULL, set);
 }
 
@@ -793,8 +750,6 @@ tu_CreateDescriptorPool(VkDevice _device,
       uint64_t host_size = pCreateInfo->maxSets * sizeof(struct tu_descriptor_set);
       host_size += dynamic_size;
       size += host_size;
-   } else {
-      size += sizeof(struct tu_descriptor_pool_entry) * pCreateInfo->maxSets;
    }
 
    pool = (struct tu_descriptor_pool *) vk_object_zalloc(
@@ -806,6 +761,12 @@ tu_CreateDescriptorPool(VkDevice _device,
       pool->host_memory_base = (uint8_t*)pool + sizeof(struct tu_descriptor_pool);
       pool->host_memory_ptr = pool->host_memory_base;
       pool->host_memory_end = (uint8_t*)pool + size;
+   } else {
+      if (bo_size) {
+         util_vma_heap_init(&pool->bo_heap, TU_POOL_HEAP_OFFSET,
+                            bo_size + TU_POOL_HEAP_OFFSET);
+         pool->bo_heap.alloc_high = false;
+      }
    }
 
    if (bo_size) {
@@ -845,6 +806,20 @@ fail_alloc:
    return ret;
 }
 
+static void
+tu_destroy_descriptor_pool_entries(struct tu_device *device,
+                                   struct tu_descriptor_pool *pool)
+{
+   list_for_each_entry_safe (struct tu_descriptor_set, set, &pool->desc_sets,
+                             pool_link) {
+      vk_descriptor_set_layout_unref(&device->vk, &set->layout->vk);
+      if (!pool->host_memory_base)
+         tu_descriptor_set_destroy(device, pool, set);
+   }
+
+   list_inithead(&pool->desc_sets);
+}
+
 VKAPI_ATTR void VKAPI_CALL
 tu_DestroyDescriptorPool(VkDevice _device,
                          VkDescriptorPool _pool,
@@ -858,18 +833,12 @@ tu_DestroyDescriptorPool(VkDevice _device,
 
    TU_RMV(resource_destroy, device, pool);
 
-   list_for_each_entry_safe(struct tu_descriptor_set, set,
-                            &pool->desc_sets, pool_link) {
-      vk_descriptor_set_layout_unref(&device->vk, &set->layout->vk);
-   }
-
-   if (!pool->host_memory_base) {
-      for(int i = 0; i < pool->entry_count; ++i) {
-         tu_descriptor_set_destroy(device, pool, pool->entries[i].set, false);
-      }
-   }
+   tu_destroy_descriptor_pool_entries(device, pool);
 
    if (pool->size) {
+      if (!pool->host_memory_base)
+         util_vma_heap_finish(&pool->bo_heap);
+
       if (pool->host_bo)
          vk_free2(&device->vk.alloc, pAllocator, pool->host_bo);
       else
@@ -887,19 +856,15 @@ tu_ResetDescriptorPool(VkDevice _device,
    VK_FROM_HANDLE(tu_device, device, _device);
    VK_FROM_HANDLE(tu_descriptor_pool, pool, descriptorPool);
 
-   list_for_each_entry_safe(struct tu_descriptor_set, set,
-                            &pool->desc_sets, pool_link) {
-      vk_descriptor_set_layout_unref(&device->vk, &set->layout->vk);
-   }
-   list_inithead(&pool->desc_sets);
+   tu_destroy_descriptor_pool_entries(device, pool);
 
-   if (!pool->host_memory_base) {
-      for(int i = 0; i < pool->entry_count; ++i) {
-         tu_descriptor_set_destroy(device, pool, pool->entries[i].set, false);
-      }
-      pool->entry_count = 0;
+   if (!pool->host_memory_base && pool->size) {
+      util_vma_heap_finish(&pool->bo_heap);
+      util_vma_heap_init(&pool->bo_heap, TU_POOL_HEAP_OFFSET,
+                         pool->size + TU_POOL_HEAP_OFFSET);
    }
 
+   pool->entry_count = 0;
    pool->current_offset = 0;
    pool->host_memory_ptr = pool->host_memory_base;
 
@@ -967,7 +932,7 @@ tu_FreeDescriptorSets(VkDevice _device,
       }
 
       if (set && !pool->host_memory_base)
-         tu_descriptor_set_destroy(device, pool, set, true);
+         tu_descriptor_set_destroy(device, pool, set);
    }
    return VK_SUCCESS;
 }
diff --git a/src/freedreno/vulkan/tu_descriptor_set.h b/src/freedreno/vulkan/tu_descriptor_set.h
index d15b3580080..903a554937d 100644
--- a/src/freedreno/vulkan/tu_descriptor_set.h
+++ b/src/freedreno/vulkan/tu_descriptor_set.h
@@ -11,6 +11,7 @@
 #include "vk_descriptor_set_layout.h"
 
 #include "tu_sampler.h"
+#include "util/vma.h"
 
 /* The hardware supports up to 8 descriptor sets since A7XX.
  * Note: This is the maximum across generations, not the maximum for a
@@ -26,6 +27,12 @@
  */
 #define MAX_SET_SIZE ((1 << 28) * 4)
 
+/* The vma heap reserves 0 to mean NULL; we have to offset by some amount to
+ * ensure we can allocate the entire BO without hitting zero. The actual
+ * amount doesn't matter.
+ */
+#define TU_POOL_HEAP_OFFSET 32
+
 struct tu_descriptor_set_binding_layout
 {
    VkDescriptorType type;
@@ -116,6 +123,7 @@ struct tu_descriptor_set
 
    struct tu_descriptor_set_layout *layout;
    struct tu_descriptor_pool *pool;
+   uint32_t offset;
    uint32_t size;
 
    uint64_t va;
@@ -132,13 +140,6 @@ struct tu_descriptor_set
 VK_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_set, base, VkDescriptorSet,
                                VK_OBJECT_TYPE_DESCRIPTOR_SET)
 
-struct tu_descriptor_pool_entry
-{
-   uint32_t offset;
-   uint32_t size;
-   struct tu_descriptor_set *set;
-};
-
 struct tu_descriptor_pool
 {
    struct vk_object_base base;
@@ -154,9 +155,10 @@ struct tu_descriptor_pool
 
    struct list_head desc_sets;
 
+   struct util_vma_heap bo_heap;
+
    uint32_t entry_count;
    uint32_t max_entry_count;
-   struct tu_descriptor_pool_entry entries[0];
 };
 VK_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_pool, base, VkDescriptorPool,
                                VK_OBJECT_TYPE_DESCRIPTOR_POOL)