nvk: Add support for dynamic buffers

These are recorded in the CPU portion of the descriptor set and combined
with the dynamic offset at vkCmdBindDescriptorSets time and placed into
the root descriptor table.

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/24326>

src/nouveau/vulkan/nvk_cmd_buffer.c

@@ -1,8 +1,10 @@
 #include "nvk_cmd_buffer.h"
 
 #include "nvk_descriptor_set.h"
+#include "nvk_descriptor_set_layout.h"
 #include "nvk_device.h"
 #include "nvk_pipeline.h"
+#include "nvk_pipeline_layout.h"
 #include "nvk_physical_device.h"
 
 #include "nouveau_push.h"
@@ -379,7 +381,7 @@ nvk_CmdBindPipeline(VkCommandBuffer commandBuffer,
 VKAPI_ATTR void VKAPI_CALL
 nvk_CmdBindDescriptorSets(VkCommandBuffer commandBuffer,
                           VkPipelineBindPoint pipelineBindPoint,
-                          VkPipelineLayout _layout,
+                          VkPipelineLayout layout,
                           uint32_t firstSet,
                           uint32_t descriptorSetCount,
                           const VkDescriptorSet *pDescriptorSets,
@@ -387,12 +389,16 @@ nvk_CmdBindDescriptorSets(VkCommandBuffer commandBuffer,
                           const uint32_t *pDynamicOffsets)
 {
    VK_FROM_HANDLE(nvk_cmd_buffer, cmd, commandBuffer);
+   VK_FROM_HANDLE(nvk_pipeline_layout, pipeline_layout, layout);
    struct nvk_descriptor_state *desc =
       nvk_get_descriptors_state(cmd, pipelineBindPoint);
 
-   for (unsigned i = 0; i < descriptorSetCount; ++i) {
+   uint32_t next_dyn_offset = 0;
+   for (uint32_t i = 0; i < descriptorSetCount; ++i) {
       unsigned set_idx = i + firstSet;
       VK_FROM_HANDLE(nvk_descriptor_set, set, pDescriptorSets[i]);
+      const struct nvk_descriptor_set_layout *set_layout =
+         pipeline_layout->set[set_idx].layout;
 
       if (desc->sets[set_idx] != set) {
          nvk_push_descriptor_set_ref(cmd->push, set);
@@ -401,6 +407,17 @@ nvk_CmdBindDescriptorSets(VkCommandBuffer commandBuffer,
          desc->sets_dirty |= BITFIELD_BIT(set_idx);
       }
 
-      /* TODO: Dynamic buffers */
+      if (set_layout->dynamic_buffer_count > 0) {
+         const uint32_t dynamic_buffer_start =
+            pipeline_layout->set[set_idx].dynamic_buffer_start;
+
+         for (uint32_t j = 0; j < set_layout->dynamic_buffer_count; j++) {
+            struct nvk_buffer_address addr = set->dynamic_buffers[j];
+            addr.base_addr += pDynamicOffsets[next_dyn_offset + j];
+            desc->root.dynamic_buffers[dynamic_buffer_start + j] = addr;
+         }
+         next_dyn_offset += set->layout->dynamic_buffer_count;
+      }
    }
+   assert(next_dyn_offset <= dynamicOffsetCount);
 }

src/nouveau/vulkan/nvk_cmd_buffer.h

@@ -4,6 +4,7 @@
 #include "nvk_private.h"
 
 #include "nouveau_push.h"
+#include "nvk_descriptor_set.h"
 
 #include "vulkan/runtime/vk_command_buffer.h"
 #include "vulkan/runtime/vk_command_pool.h"
@@ -38,6 +39,7 @@ struct nvk_root_descriptor_table {
    uint64_t sets[NVK_MAX_SETS];
 
    /* TODO: Dynamic buffer bindings */
+   struct nvk_buffer_address dynamic_buffers[NVK_MAX_DYNAMIC_BUFFERS];
 };
 
 struct nvk_descriptor_state {

src/nouveau/vulkan/nvk_descriptor_set.c

@@ -82,6 +82,23 @@ write_buffer_desc(struct nvk_descriptor_set *set,
    write_desc(set, binding, elem, &desc, sizeof(desc));
 }
 
+static void
+write_dynamic_buffer_desc(struct nvk_descriptor_set *set,
+                          const VkDescriptorBufferInfo *const info,
+                          uint32_t binding, uint32_t elem)
+{
+   VK_FROM_HANDLE(nvk_buffer, buffer, info->buffer);
+   const struct nvk_descriptor_set_binding_layout *binding_layout =
+      &set->layout->binding[binding];
+
+   struct nvk_buffer_address *desc =
+      &set->dynamic_buffers[binding_layout->dynamic_buffer_index + elem];
+   *desc = (struct nvk_buffer_address){
+      .base_addr = nvk_buffer_address(buffer, info->offset),
+      .size = vk_buffer_range(&buffer->vk, info->offset, info->range),
+   };
+}
+
 static void
 write_buffer_view_desc(struct nvk_descriptor_set *set,
                        const VkBufferView bufferView,
@@ -164,7 +181,12 @@ nvk_UpdateDescriptorSets(VkDevice device,
 
       case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
       case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
-         unreachable("Dynamic buffers not yet supported");
+         for (uint32_t j = 0; j < write->descriptorCount; j++) {
+            write_dynamic_buffer_desc(set, write->pBufferInfo + j,
+                                      write->dstBinding,
+                                      write->dstArrayElement + j);
+         }
+         break;
 
       case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK: {
          const VkWriteDescriptorSetInlineUniformBlock *write_inline =
@@ -308,8 +330,11 @@ nvk_descriptor_set_create(struct nvk_device *device, struct nvk_descriptor_pool
                           struct nvk_descriptor_set **out_set)
 {
    struct nvk_descriptor_set *set;
-   uint32_t mem_size = sizeof(struct nvk_descriptor_set);
-   set = vk_zalloc2(&device->vk.alloc, NULL, mem_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
+
+   uint32_t mem_size = sizeof(struct nvk_descriptor_set) +
+      layout->dynamic_buffer_count * sizeof(struct nvk_buffer_address);
+   set = vk_zalloc2(&device->vk.alloc, NULL, mem_size, 8,
+                    VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
    if (!set)
       return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
 

src/nouveau/vulkan/nvk_descriptor_set.h

@@ -51,6 +51,8 @@ struct nvk_descriptor_set {
    uint32_t bo_offset;
    struct nouveau_ws_bo *bo;
    void *mapped_ptr;
+
+   struct nvk_buffer_address dynamic_buffers[];
 };
 
 VK_DEFINE_HANDLE_CASTS(nvk_descriptor_set, base, VkDescriptorSet,

src/nouveau/vulkan/nvk_descriptor_set_layout.c

@@ -38,9 +38,12 @@ nvk_descriptor_stride_align_for_type(VkDescriptorType type,
 
    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
+      *stride = *align = sizeof(struct nvk_buffer_address);
+      break;
+
    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
-      *stride = *align = sizeof(struct nvk_buffer_address);
+      *stride = *align = 0; /* These don't take up buffer space */
       break;
 
    case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK:
@@ -132,6 +135,7 @@ nvk_CreateDescriptorSetLayout(VkDevice _device,
                            MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE);
 
    uint32_t buffer_size = 0;
+   uint8_t dynamic_buffer_count = 0;
    for (uint32_t b = 0; b < num_bindings; b++) {
       /* We stashed the pCreateInfo->pBindings[] index (plus one) in the
      * immutable_samplers pointer.  Check for NULL (empty binding) and then
@@ -158,6 +162,16 @@ nvk_CreateDescriptorSetLayout(VkDevice _device,
 
       layout->binding[b].array_size = binding->descriptorCount;
 
+      switch (binding->descriptorType) {
+      case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
+      case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
+         layout->binding[b].dynamic_buffer_index = dynamic_buffer_count;
+         dynamic_buffer_count += binding->descriptorCount;
+         break;
+      default:
+         break;
+      }
+
       const VkMutableDescriptorTypeListVALVE *type_list = NULL;
       if (binding->descriptorType == VK_DESCRIPTOR_TYPE_MUTABLE_VALVE) {
          assert(mutable_info != NULL);
@@ -169,6 +183,7 @@ nvk_CreateDescriptorSetLayout(VkDevice _device,
       nvk_descriptor_stride_align_for_type(binding->descriptorType, type_list,
                                            &stride, &align);
 
+      assert(stride <= UINT8_MAX);
       layout->binding[b].offset = ALIGN_POT(buffer_size, align);
       layout->binding[b].stride = stride;
       buffer_size += stride * binding->descriptorCount;
@@ -184,12 +199,14 @@ nvk_CreateDescriptorSetLayout(VkDevice _device,
    }
 
    layout->descriptor_buffer_size = buffer_size;
+   layout->dynamic_buffer_count = dynamic_buffer_count;
 
    struct mesa_sha1 sha1_ctx;
    _mesa_sha1_init(&sha1_ctx);
 
 #define SHA1_UPDATE_VALUE(x) _mesa_sha1_update(&sha1_ctx, &(x), sizeof(x));
    SHA1_UPDATE_VALUE(layout->descriptor_buffer_size);
+   SHA1_UPDATE_VALUE(layout->dynamic_buffer_count);
    SHA1_UPDATE_VALUE(layout->binding_count);
 
    for (uint32_t b = 0; b < num_bindings; b++) {
@@ -198,6 +215,7 @@ nvk_CreateDescriptorSetLayout(VkDevice _device,
       SHA1_UPDATE_VALUE(layout->binding[b].array_size);
       SHA1_UPDATE_VALUE(layout->binding[b].offset);
       SHA1_UPDATE_VALUE(layout->binding[b].stride);
+      SHA1_UPDATE_VALUE(layout->binding[b].dynamic_buffer_index);
       /* Immutable samplers are ignored for now */
    }
 #undef SHA1_UPDATE_VALUE

src/nouveau/vulkan/nvk_descriptor_set_layout.h

@@ -24,7 +24,10 @@ struct nvk_descriptor_set_binding_layout {
    uint32_t offset;
 
    /* Stride between array elements in the descriptor buffer */
-   uint32_t stride;
+   uint8_t stride;
+
+   /* Index into the dynamic buffer binding array */
+   uint8_t dynamic_buffer_index;
 
    /* Immutable samplers (or NULL if no immutable samplers) */
    struct nvk_sampler **immutable_samplers;
@@ -40,6 +43,9 @@ struct nvk_descriptor_set_layout {
    /* Size of the descriptor buffer for this descriptor set */
    uint32_t descriptor_buffer_size;
 
+   /* Number of dynamic UBO bindings in this set */
+   uint8_t dynamic_buffer_count;
+
    /* Number of bindings in this descriptor set */
    uint32_t binding_count;
 

src/nouveau/vulkan/nvk_nir_lower_descriptors.c

@@ -38,6 +38,21 @@ load_descriptor(nir_builder *b, unsigned num_components, unsigned bit_size,
    if (ctx->clamp_desc_array_bounds)
       index = nir_umin(b, index, nir_imm_int(b, binding_layout->array_size - 1));
 
+   if (binding_layout->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
+       binding_layout->type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
+      /* Get the index in the root descriptor table dynamic_buffers array. */
+      index = nir_iadd_imm(b, index,
+                           ctx->layout->set[set].dynamic_buffer_start +
+                           binding_layout->dynamic_buffer_index);
+
+      nir_ssa_def *root_desc_offset =
+         nir_iadd_imm(b, nir_imul_imm(b, index, sizeof(struct nvk_buffer_address)),
+                      offsetof(struct nvk_root_descriptor_table, dynamic_buffers));
+
+      return nir_load_ubo(b, 4, 32, nir_imm_int(b, 0), root_desc_offset,
+                          .align_mul = 16, .align_offset = 0, .range = ~0);
+   }
+
    assert(binding_layout->stride > 0);
    nir_ssa_def *desc_ubo_offset =
       nir_iadd_imm(b, nir_imul_imm(b, index, binding_layout->stride),

src/nouveau/vulkan/nvk_pipeline_layout.c

@@ -21,10 +21,13 @@ nvk_CreatePipelineLayout(VkDevice _device,
 
    layout->num_sets = pCreateInfo->setLayoutCount;
 
+   uint8_t dynamic_buffer_count = 0;
    for (uint32_t s = 0; s < pCreateInfo->setLayoutCount; s++) {
       VK_FROM_HANDLE(nvk_descriptor_set_layout, set_layout,
                      pCreateInfo->pSetLayouts[s]);
       layout->set[s].layout = nvk_descriptor_set_layout_ref(set_layout);
+      layout->set[s].dynamic_buffer_start = dynamic_buffer_count;
+      dynamic_buffer_count += set_layout->dynamic_buffer_count;
    }
 
    struct mesa_sha1 sha1_ctx;

src/nouveau/vulkan/nvk_pipeline_layout.h

@@ -16,6 +16,7 @@ struct nvk_pipeline_layout {
 
    struct {
       struct nvk_descriptor_set_layout *layout;
+      uint8_t dynamic_buffer_start;
    } set[NVK_MAX_SETS];
 };
 

src/nouveau/vulkan/nvk_private.h

@@ -15,6 +15,7 @@
 #include <xf86drm.h>
 
 #define NVK_MAX_SETS 8
+#define NVK_MAX_DYNAMIC_BUFFERS 64
 #define NVK_MIN_UBO_ALIGNMENT 16
 
 /**