nvk: Add a VK_EXT_descriptor_buffer buffer view cache

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/30580>

src/nouveau/vulkan/meson.build

@@ -28,6 +28,8 @@ nvk_files = files(
   'nvk_device.h',
   'nvk_device_memory.c',
   'nvk_device_memory.h',
+  'nvk_edb_bview_cache.c',
+  'nvk_edb_bview_cache.h',
   'nvk_event.c',
   'nvk_event.h',
   'nvk_format.c',

src/nouveau/vulkan/nvk_descriptor_types.h

@@ -44,6 +44,23 @@ PRAGMA_DIAGNOSTIC_POP
 static_assert(sizeof(struct nvk_buffer_view_descriptor) == 4,
               "nvk_buffer_view_descriptor has no holes");
 
+PRAGMA_DIAGNOSTIC_PUSH
+PRAGMA_DIAGNOSTIC_ERROR(-Wpadded)
+/** See also nvk_edb_bview_cache */
+struct nvk_edb_buffer_view_descriptor {
+   /** Index of the HW descriptor in the texture/image table */
+   uint32_t index;
+   /** Offset into the HW descriptor in surface elements */
+   uint32_t offset_el;
+   /** Size of the virtual descriptor in surface elements */
+   uint32_t size_el;
+   /** Value returned in the alpha channel for OOB buffer access */
+   uint32_t oob_alpha;
+};
+PRAGMA_DIAGNOSTIC_POP
+static_assert(sizeof(struct nvk_edb_buffer_view_descriptor) == 16,
+              "nvk_edb_buffer_view_descriptor has no holes");
+
 PRAGMA_DIAGNOSTIC_PUSH
 PRAGMA_DIAGNOSTIC_ERROR(-Wpadded)
 struct nvk_bindless_cbuf {

src/nouveau/vulkan/nvk_device.c

@@ -189,6 +189,12 @@ nvk_CreateDevice(VkPhysicalDevice physicalDevice,
    if (result != VK_SUCCESS)
       goto fail_images;
 
+   if (dev->vk.enabled_features.descriptorBuffer) {
+      result = nvk_edb_bview_cache_init(dev, &dev->edb_bview_cache);
+      if (result != VK_SUCCESS)
+         goto fail_samplers;
+   }
+
    /* If we have a full BAR, go ahead and do shader uploads on the CPU.
     * Otherwise, we fall back to doing shader uploads via the upload queue.
     *
@@ -203,7 +209,7 @@ nvk_CreateDevice(VkPhysicalDevice physicalDevice,
                           2048 /* overalloc */,
                           pdev->info.cls_eng3d < VOLTA_A);
    if (result != VK_SUCCESS)
-      goto fail_samplers;
+      goto fail_edb_bview_cache;
 
    result = nvk_heap_init(dev, &dev->event_heap,
                           NVKMD_MEM_LOCAL, NVKMD_MEM_MAP_WR,
@@ -257,6 +263,8 @@ fail_slm:
    nvk_heap_finish(dev, &dev->event_heap);
 fail_shader_heap:
    nvk_heap_finish(dev, &dev->shader_heap);
+fail_edb_bview_cache:
+   nvk_edb_bview_cache_finish(dev, &dev->edb_bview_cache);
 fail_samplers:
    nvk_descriptor_table_finish(dev, &dev->samplers);
 fail_images:
@@ -296,6 +304,7 @@ nvk_DestroyDevice(VkDevice _device, const VkAllocationCallbacks *pAllocator)
    nvk_slm_area_finish(&dev->slm);
    nvk_heap_finish(dev, &dev->event_heap);
    nvk_heap_finish(dev, &dev->shader_heap);
+   nvk_edb_bview_cache_finish(dev, &dev->edb_bview_cache);
    nvk_descriptor_table_finish(dev, &dev->samplers);
    nvk_descriptor_table_finish(dev, &dev->images);
    nvkmd_mem_unref(dev->zero_page);

src/nouveau/vulkan/nvk_device.h

@@ -7,6 +7,7 @@
 
 #include "nvk_private.h"
 
+#include "nvk_edb_bview_cache.h"
 #include "nvk_descriptor_table.h"
 #include "nvk_heap.h"
 #include "nvk_queue.h"
@@ -42,6 +43,7 @@ struct nvk_device {
    struct nvkmd_mem *zero_page;
    struct nvk_descriptor_table images;
    struct nvk_descriptor_table samplers;
+   struct nvk_edb_bview_cache edb_bview_cache;
    struct nvk_heap shader_heap;
    struct nvk_heap event_heap;
    struct nvk_slm_area slm;

src/nouveau/vulkan/nvk_edb_bview_cache.c

@@ -0,0 +1,259 @@
+/*
+ * Copyright © 2022 Collabora Ltd. and Red Hat Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nvk_edb_bview_cache.h"
+
+#include "nil.h"
+#include "nvk_device.h"
+#include "nvk_descriptor_types.h"
+#include "nvk_physical_device.h"
+
+#include "util/format/u_format.h"
+#include "util/hash_table.h"
+
+PRAGMA_DIAGNOSTIC_PUSH
+PRAGMA_DIAGNOSTIC_ERROR(-Wpadded)
+struct bvdesc_key {
+   uint16_t format;
+   uint16_t chunk : 12;
+   uint16_t rgb_offset : 4;
+};
+PRAGMA_DIAGNOSTIC_POP
+static_assert(sizeof(struct bvdesc_key) == 4, "bvdesc_key has no holes");
+
+static uint64_t
+view_size_B(enum pipe_format format)
+{
+   const uint8_t el_size_B = util_format_get_blocksize(format);
+   if (util_is_power_of_two_nonzero(el_size_B)) {
+      return 4ull << 30;
+   } else {
+      /* On Ampere (but not Turing or Maxwell for some reason), we're limited
+       * to 3GB for RGB32 buffers.
+       */
+      assert(util_format_get_nr_components(format) == 3);
+      return 3ull << 30;
+   }
+}
+
+/* Stride in VA between views */
+static uint64_t
+view_stride_B(enum pipe_format format)
+{
+   return view_size_B(format) / 2;
+}
+
+static uint32_t
+view_size_el(enum pipe_format format)
+{
+   /* If someone uses the last element of this chunk, then they're a max-sized
+    * client view which starts at the middle of this chunk and therefore
+    * should be in the next chunk.
+    */
+   return (view_size_B(format) / util_format_get_blocksize(format)) - 1;
+}
+
+static uint64_t
+base_addr_for_chunk(struct nvk_device *dev, uint16_t chunk,
+                    enum pipe_format format)
+{
+   return dev->nvkmd->va_start + chunk * view_stride_B(format);
+}
+
+static uint64_t
+chunk_for_addr(struct nvk_device *dev, uint64_t addr, enum pipe_format format)
+{
+   assert(addr >= dev->nvkmd->va_start);
+   return (addr - dev->nvkmd->va_start) / view_stride_B(format);
+}
+
+static VkResult
+nvk_edb_bview_cache_add_bview(struct nvk_device *dev,
+                              struct nvk_edb_bview_cache *cache,
+                              struct bvdesc_key key)
+{
+   void *void_key = NULL;
+   STATIC_ASSERT(sizeof(key) <= sizeof(void_key));
+   memcpy(&void_key, &key, sizeof(key));
+
+   const uint64_t base_addr =
+      base_addr_for_chunk(dev, key.chunk, key.format) + key.rgb_offset;
+
+   uint32_t size_el = view_size_el(key.format);
+
+   const uint8_t el_size_B = util_format_get_blocksize(key.format);
+   if (base_addr + (uint64_t)size_el * el_size_B > dev->nvkmd->va_end) {
+      const uint64_t size_B = dev->nvkmd->va_end - base_addr;
+      size_el = size_B / el_size_B;
+   }
+
+   uint32_t desc[8];
+   nil_buffer_fill_tic(&nvk_device_physical(dev)->info, base_addr,
+                       nil_format(key.format), size_el, &desc);
+
+   uint32_t index;
+   VkResult result = nvk_descriptor_table_add(dev, &dev->images,
+                                              desc, sizeof(desc), &index);
+   if (result != VK_SUCCESS)
+      return result;
+
+   _mesa_hash_table_insert(cache->cache, void_key, (void *)(uintptr_t)index);
+
+   return VK_SUCCESS;
+}
+
+static uint32_t
+nvk_edb_bview_cache_lookup_bview(struct nvk_device *dev,
+                                 struct nvk_edb_bview_cache *cache,
+                                 struct bvdesc_key key)
+{
+   void *void_key = NULL;
+   STATIC_ASSERT(sizeof(key) <= sizeof(void_key));
+   memcpy(&void_key, &key, sizeof(key));
+
+   struct hash_entry *entry = _mesa_hash_table_search(cache->cache, void_key);
+   if (entry != NULL) {
+      return (uintptr_t)entry->data;
+   } else {
+      return 0;
+   }
+}
+
+VkResult
+nvk_edb_bview_cache_init(struct nvk_device *dev,
+                         struct nvk_edb_bview_cache *cache)
+{
+   struct nvk_physical_device *pdev = nvk_device_physical(dev);
+   VkResult result;
+
+   cache->cache = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
+                                          _mesa_key_pointer_equal);
+   if (cache->cache == NULL)
+      return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);
+
+   for (uint32_t format = 0; format < PIPE_FORMAT_COUNT; format++) {
+      if (!nil_format_supports_buffer(&pdev->info, format))
+         continue;
+
+      const uint8_t el_size_B = util_format_get_blocksize(format);
+
+      for (uint16_t chunk = 0;; chunk++) {
+         if (base_addr_for_chunk(dev, chunk, format) >= dev->nvkmd->va_end)
+            break;
+
+         assert(format <= UINT16_MAX);
+         assert(chunk < (1u << 12));
+
+         if (!util_is_power_of_two_nonzero(el_size_B)) {
+            assert(util_format_get_nr_components(format) == 3);
+            assert(el_size_B % 3 == 0);
+            const uint8_t chan_size_B = el_size_B / 3;
+            for (uint8_t chan = 0; chan < 3; chan++) {
+               struct bvdesc_key key = {
+                  .format = format,
+                  .chunk = chunk,
+                  .rgb_offset = chan * chan_size_B,
+               };
+               result = nvk_edb_bview_cache_add_bview(dev, cache, key);
+               if (result != VK_SUCCESS)
+                  goto fail;
+            }
+         } else {
+            struct bvdesc_key key = {
+               .format = format,
+               .chunk = chunk,
+            };
+            result = nvk_edb_bview_cache_add_bview(dev, cache, key);
+            if (result != VK_SUCCESS)
+               goto fail;
+         }
+      }
+   }
+
+   return VK_SUCCESS;
+
+fail:
+   _mesa_hash_table_destroy(cache->cache, NULL);
+   return result;
+}
+
+void
+nvk_edb_bview_cache_finish(struct nvk_device *dev,
+                           struct nvk_edb_bview_cache *cache)
+{
+   /* We don't bother freeing the descriptors as those will be cleaned up
+    * automatically when the device is destroyed.
+    */
+   if (cache->cache)
+      _mesa_hash_table_destroy(cache->cache, NULL);
+}
+
+struct nvk_edb_buffer_view_descriptor
+nvk_edb_bview_cache_get_descriptor(struct nvk_device *dev,
+                                   struct nvk_edb_bview_cache *cache,
+                                   uint64_t base_addr, uint64_t size_B,
+                                   enum pipe_format format)
+{
+   /* The actual hardware limit for buffer image/texture descriptors is 4GB
+    * regardless of format.  This cache works by covering the address space
+    * with 4GB buffer descriptors at 2GB offsets.  In order for this to work
+    * properly, the size if the client's buffer view must be at most 2 GB.
+    */
+   assert(size_B <= view_stride_B(format));
+
+   const uint8_t el_size_B = util_format_get_blocksize(format);
+   const uint64_t size_el = size_B / el_size_B;
+
+   const uint64_t chunk = chunk_for_addr(dev, base_addr, format);
+   const uint64_t desc_base_addr = base_addr_for_chunk(dev, chunk, format);
+   const uint32_t offset_B = base_addr - desc_base_addr;
+
+   const uint32_t offset_el = offset_B / el_size_B;
+
+   uint16_t rgb_offset = 0;
+   if (!util_is_power_of_two_nonzero(el_size_B)) {
+      assert(util_format_get_nr_components(format) == 3);
+      assert(el_size_B % 3 == 0);
+      rgb_offset = offset_B % el_size_B;
+   } else {
+      assert(offset_B % el_size_B == 0);
+   }
+
+   assert(offset_el + size_el > offset_el);
+   assert(offset_el + size_el <= view_size_el(format));
+
+   assert(format <= UINT16_MAX);
+   assert(chunk < (1u << 12));
+   assert(rgb_offset < (1u << 4));
+   const struct bvdesc_key key = {
+      .format = format,
+      .chunk = chunk,
+      .rgb_offset = rgb_offset,
+   };
+   uint32_t index = nvk_edb_bview_cache_lookup_bview(dev, cache, key);
+
+   uint32_t oob_alpha;
+   if (util_format_has_alpha(format)) {
+      /* OOB reads as if it read 0 texture data so an RGBA format reads
+       * (0, 0, 0, 0) out-of-bounds.
+       */
+      oob_alpha = 0;
+   } else if (util_format_is_pure_integer(format)) {
+      /* OOB reads 0 texture data but then gets extended by (0, 0, 0, 1) */
+      oob_alpha = 1;
+   } else {
+      /* OOB reads 0 texture data but then gets extended by
+       * (0.0, 0.0, 0.0, 1.0)
+       */
+      oob_alpha = 0x3f800000;
+   }
+
+   return (struct nvk_edb_buffer_view_descriptor) {
+      .index = index,
+      .offset_el = offset_el,
+      .size_el = size_el,
+      .oob_alpha = oob_alpha,
+   };
+}

src/nouveau/vulkan/nvk_edb_bview_cache.h

@@ -0,0 +1,67 @@
+/*
+ * Copyright © 2022 Collabora Ltd. and Red Hat Inc.
+ * SPDX-License-Identifier: MIT
+ */
+#ifndef NVK_EDB_BVIEW_CACHE_H
+#define NVK_EDB_BVIEW_CACHE_H 1
+
+#include "nvk_private.h"
+
+#include "nvk_descriptor_types.h"
+
+#include "util/format/u_formats.h"
+
+struct hash_table;
+struct nvk_device;
+
+/** A cache of VK_EXT_descriptor_buffer BufferViews
+ *
+ * VK_EXT_descriptor_buffer effectively removes the concept of a VkBufferView
+ * object.  Instead of allocating a view object and passing that into
+ * vkGetDescriptorEXT() like you do for image views, typed buffers work more
+ * like untyped UBOs or SSBOs and you just pass a base address, size (in
+ * bytes) and format to vkGetDescriptorEXT().  On NVIDIA hardware, this is
+ * annoying because it means we no longer have an object to help us manage the
+ * life cycle of the descriptor on the heap.
+ *
+ * The solution is nvk_edb_bview_cache.  This cache stores enough typed buffer
+ * descriptors to cover the entire address space.  For each buffer format, we
+ * allocate 512 4 GiB buffer views, spaced at 2 GiB intervals.  This ensures
+ * that every client buffer view will live entirely inside one of these views.
+ * The descriptor we return from vkGetDescriptorEXT() contains the descriptor
+ * index to the HW descriptor as well as an offset and size (both in surface
+ * elements) and the alpha value to expect for OOB writes.
+ *
+ * For RGB32 formats, we place 3 3 GiB buffer views every 1.5 GiB in the
+ * address space.  We need 3 per chunk because RGB32 buffer views only have a
+ * minimum alignment of 4B but the offsetting we do in the shader is in terms
+ * of surface elements.  For offsetting by 1 or 2 components, we need a
+ * different view.  The reason why it's 3 GiB instead of 4 GiB is because
+ * Ampere reduced the maximum size of an RGB32 buffer view to 3 GiB.
+ *
+ * In nvk_nir_lower_descriptors(), we lower all texture or image buffer access
+ * to an access through one of these HW descriptors.  Bounds checkinig is done
+ * in software and the offset is applied to ensure that we only ever read from
+ * the memory range specified by the client.  The HW descriptor only exists to
+ * help with format conversion.
+ */
+struct nvk_edb_bview_cache {
+   struct hash_table *cache;
+};
+
+VkResult
+nvk_edb_bview_cache_init(struct nvk_device *dev,
+                         struct nvk_edb_bview_cache *cache);
+
+/* It's safe to call this function on a zeroed nvk_edb_bview_cache */
+void
+nvk_edb_bview_cache_finish(struct nvk_device *dev,
+                           struct nvk_edb_bview_cache *cache);
+
+struct nvk_edb_buffer_view_descriptor
+nvk_edb_bview_cache_get_descriptor(struct nvk_device *dev,
+                                   struct nvk_edb_bview_cache *cache,
+                                   uint64_t base_addr, uint64_t size_B,
+                                   enum pipe_format format);
+
+#endif /* NVK_EDB_BVIEW_CACHE_H */