From 93b30bb35388e16f7fea6a735e76a3d8cc6a9054 Mon Sep 17 00:00:00 2001 From: Faith Ekstrand Date: Wed, 7 Aug 2024 21:20:40 -0500 Subject: [PATCH] nvk: Add a VK_EXT_descriptor_buffer buffer view cache Part-of: --- src/nouveau/vulkan/meson.build | 2 + src/nouveau/vulkan/nvk_descriptor_types.h | 17 ++ src/nouveau/vulkan/nvk_device.c | 11 +- src/nouveau/vulkan/nvk_device.h | 2 + src/nouveau/vulkan/nvk_edb_bview_cache.c | 259 ++++++++++++++++++++++ src/nouveau/vulkan/nvk_edb_bview_cache.h | 67 ++++++ 6 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 src/nouveau/vulkan/nvk_edb_bview_cache.c create mode 100644 src/nouveau/vulkan/nvk_edb_bview_cache.h diff --git a/src/nouveau/vulkan/meson.build b/src/nouveau/vulkan/meson.build index 6f39c661603..c4cdf80047c 100644 --- a/src/nouveau/vulkan/meson.build +++ b/src/nouveau/vulkan/meson.build @@ -28,6 +28,8 @@ nvk_files = files( 'nvk_device.h', 'nvk_device_memory.c', 'nvk_device_memory.h', + 'nvk_edb_bview_cache.c', + 'nvk_edb_bview_cache.h', 'nvk_event.c', 'nvk_event.h', 'nvk_format.c', diff --git a/src/nouveau/vulkan/nvk_descriptor_types.h b/src/nouveau/vulkan/nvk_descriptor_types.h index 9162662284a..bfbc62b0484 100644 --- a/src/nouveau/vulkan/nvk_descriptor_types.h +++ b/src/nouveau/vulkan/nvk_descriptor_types.h @@ -44,6 +44,23 @@ PRAGMA_DIAGNOSTIC_POP static_assert(sizeof(struct nvk_buffer_view_descriptor) == 4, "nvk_buffer_view_descriptor has no holes"); +PRAGMA_DIAGNOSTIC_PUSH +PRAGMA_DIAGNOSTIC_ERROR(-Wpadded) +/** See also nvk_edb_bview_cache */ +struct nvk_edb_buffer_view_descriptor { + /** Index of the HW descriptor in the texture/image table */ + uint32_t index; + /** Offset into the HW descriptor in surface elements */ + uint32_t offset_el; + /** Size of the virtual descriptor in surface elements */ + uint32_t size_el; + /** Value returned in the alpha channel for OOB buffer access */ + uint32_t oob_alpha; +}; +PRAGMA_DIAGNOSTIC_POP +static_assert(sizeof(struct nvk_edb_buffer_view_descriptor) == 16, + "nvk_edb_buffer_view_descriptor has no holes"); + PRAGMA_DIAGNOSTIC_PUSH PRAGMA_DIAGNOSTIC_ERROR(-Wpadded) struct nvk_bindless_cbuf { diff --git a/src/nouveau/vulkan/nvk_device.c b/src/nouveau/vulkan/nvk_device.c index ef654e96589..ebd98b599b0 100644 --- a/src/nouveau/vulkan/nvk_device.c +++ b/src/nouveau/vulkan/nvk_device.c @@ -189,6 +189,12 @@ nvk_CreateDevice(VkPhysicalDevice physicalDevice, if (result != VK_SUCCESS) goto fail_images; + if (dev->vk.enabled_features.descriptorBuffer) { + result = nvk_edb_bview_cache_init(dev, &dev->edb_bview_cache); + if (result != VK_SUCCESS) + goto fail_samplers; + } + /* If we have a full BAR, go ahead and do shader uploads on the CPU. * Otherwise, we fall back to doing shader uploads via the upload queue. * @@ -203,7 +209,7 @@ nvk_CreateDevice(VkPhysicalDevice physicalDevice, 2048 /* overalloc */, pdev->info.cls_eng3d < VOLTA_A); if (result != VK_SUCCESS) - goto fail_samplers; + goto fail_edb_bview_cache; result = nvk_heap_init(dev, &dev->event_heap, NVKMD_MEM_LOCAL, NVKMD_MEM_MAP_WR, @@ -257,6 +263,8 @@ fail_slm: nvk_heap_finish(dev, &dev->event_heap); fail_shader_heap: nvk_heap_finish(dev, &dev->shader_heap); +fail_edb_bview_cache: + nvk_edb_bview_cache_finish(dev, &dev->edb_bview_cache); fail_samplers: nvk_descriptor_table_finish(dev, &dev->samplers); fail_images: @@ -296,6 +304,7 @@ nvk_DestroyDevice(VkDevice _device, const VkAllocationCallbacks *pAllocator) nvk_slm_area_finish(&dev->slm); nvk_heap_finish(dev, &dev->event_heap); nvk_heap_finish(dev, &dev->shader_heap); + nvk_edb_bview_cache_finish(dev, &dev->edb_bview_cache); nvk_descriptor_table_finish(dev, &dev->samplers); nvk_descriptor_table_finish(dev, &dev->images); nvkmd_mem_unref(dev->zero_page); diff --git a/src/nouveau/vulkan/nvk_device.h b/src/nouveau/vulkan/nvk_device.h index 03dd43a219d..ac946c8698e 100644 --- a/src/nouveau/vulkan/nvk_device.h +++ b/src/nouveau/vulkan/nvk_device.h @@ -7,6 +7,7 @@ #include "nvk_private.h" +#include "nvk_edb_bview_cache.h" #include "nvk_descriptor_table.h" #include "nvk_heap.h" #include "nvk_queue.h" @@ -42,6 +43,7 @@ struct nvk_device { struct nvkmd_mem *zero_page; struct nvk_descriptor_table images; struct nvk_descriptor_table samplers; + struct nvk_edb_bview_cache edb_bview_cache; struct nvk_heap shader_heap; struct nvk_heap event_heap; struct nvk_slm_area slm; diff --git a/src/nouveau/vulkan/nvk_edb_bview_cache.c b/src/nouveau/vulkan/nvk_edb_bview_cache.c new file mode 100644 index 00000000000..cf569ee866a --- /dev/null +++ b/src/nouveau/vulkan/nvk_edb_bview_cache.c @@ -0,0 +1,259 @@ +/* + * Copyright © 2022 Collabora Ltd. and Red Hat Inc. + * SPDX-License-Identifier: MIT + */ + +#include "nvk_edb_bview_cache.h" + +#include "nil.h" +#include "nvk_device.h" +#include "nvk_descriptor_types.h" +#include "nvk_physical_device.h" + +#include "util/format/u_format.h" +#include "util/hash_table.h" + +PRAGMA_DIAGNOSTIC_PUSH +PRAGMA_DIAGNOSTIC_ERROR(-Wpadded) +struct bvdesc_key { + uint16_t format; + uint16_t chunk : 12; + uint16_t rgb_offset : 4; +}; +PRAGMA_DIAGNOSTIC_POP +static_assert(sizeof(struct bvdesc_key) == 4, "bvdesc_key has no holes"); + +static uint64_t +view_size_B(enum pipe_format format) +{ + const uint8_t el_size_B = util_format_get_blocksize(format); + if (util_is_power_of_two_nonzero(el_size_B)) { + return 4ull << 30; + } else { + /* On Ampere (but not Turing or Maxwell for some reason), we're limited + * to 3GB for RGB32 buffers. + */ + assert(util_format_get_nr_components(format) == 3); + return 3ull << 30; + } +} + +/* Stride in VA between views */ +static uint64_t +view_stride_B(enum pipe_format format) +{ + return view_size_B(format) / 2; +} + +static uint32_t +view_size_el(enum pipe_format format) +{ + /* If someone uses the last element of this chunk, then they're a max-sized + * client view which starts at the middle of this chunk and therefore + * should be in the next chunk. + */ + return (view_size_B(format) / util_format_get_blocksize(format)) - 1; +} + +static uint64_t +base_addr_for_chunk(struct nvk_device *dev, uint16_t chunk, + enum pipe_format format) +{ + return dev->nvkmd->va_start + chunk * view_stride_B(format); +} + +static uint64_t +chunk_for_addr(struct nvk_device *dev, uint64_t addr, enum pipe_format format) +{ + assert(addr >= dev->nvkmd->va_start); + return (addr - dev->nvkmd->va_start) / view_stride_B(format); +} + +static VkResult +nvk_edb_bview_cache_add_bview(struct nvk_device *dev, + struct nvk_edb_bview_cache *cache, + struct bvdesc_key key) +{ + void *void_key = NULL; + STATIC_ASSERT(sizeof(key) <= sizeof(void_key)); + memcpy(&void_key, &key, sizeof(key)); + + const uint64_t base_addr = + base_addr_for_chunk(dev, key.chunk, key.format) + key.rgb_offset; + + uint32_t size_el = view_size_el(key.format); + + const uint8_t el_size_B = util_format_get_blocksize(key.format); + if (base_addr + (uint64_t)size_el * el_size_B > dev->nvkmd->va_end) { + const uint64_t size_B = dev->nvkmd->va_end - base_addr; + size_el = size_B / el_size_B; + } + + uint32_t desc[8]; + nil_buffer_fill_tic(&nvk_device_physical(dev)->info, base_addr, + nil_format(key.format), size_el, &desc); + + uint32_t index; + VkResult result = nvk_descriptor_table_add(dev, &dev->images, + desc, sizeof(desc), &index); + if (result != VK_SUCCESS) + return result; + + _mesa_hash_table_insert(cache->cache, void_key, (void *)(uintptr_t)index); + + return VK_SUCCESS; +} + +static uint32_t +nvk_edb_bview_cache_lookup_bview(struct nvk_device *dev, + struct nvk_edb_bview_cache *cache, + struct bvdesc_key key) +{ + void *void_key = NULL; + STATIC_ASSERT(sizeof(key) <= sizeof(void_key)); + memcpy(&void_key, &key, sizeof(key)); + + struct hash_entry *entry = _mesa_hash_table_search(cache->cache, void_key); + if (entry != NULL) { + return (uintptr_t)entry->data; + } else { + return 0; + } +} + +VkResult +nvk_edb_bview_cache_init(struct nvk_device *dev, + struct nvk_edb_bview_cache *cache) +{ + struct nvk_physical_device *pdev = nvk_device_physical(dev); + VkResult result; + + cache->cache = _mesa_hash_table_create(NULL, _mesa_hash_pointer, + _mesa_key_pointer_equal); + if (cache->cache == NULL) + return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY); + + for (uint32_t format = 0; format < PIPE_FORMAT_COUNT; format++) { + if (!nil_format_supports_buffer(&pdev->info, format)) + continue; + + const uint8_t el_size_B = util_format_get_blocksize(format); + + for (uint16_t chunk = 0;; chunk++) { + if (base_addr_for_chunk(dev, chunk, format) >= dev->nvkmd->va_end) + break; + + assert(format <= UINT16_MAX); + assert(chunk < (1u << 12)); + + if (!util_is_power_of_two_nonzero(el_size_B)) { + assert(util_format_get_nr_components(format) == 3); + assert(el_size_B % 3 == 0); + const uint8_t chan_size_B = el_size_B / 3; + for (uint8_t chan = 0; chan < 3; chan++) { + struct bvdesc_key key = { + .format = format, + .chunk = chunk, + .rgb_offset = chan * chan_size_B, + }; + result = nvk_edb_bview_cache_add_bview(dev, cache, key); + if (result != VK_SUCCESS) + goto fail; + } + } else { + struct bvdesc_key key = { + .format = format, + .chunk = chunk, + }; + result = nvk_edb_bview_cache_add_bview(dev, cache, key); + if (result != VK_SUCCESS) + goto fail; + } + } + } + + return VK_SUCCESS; + +fail: + _mesa_hash_table_destroy(cache->cache, NULL); + return result; +} + +void +nvk_edb_bview_cache_finish(struct nvk_device *dev, + struct nvk_edb_bview_cache *cache) +{ + /* We don't bother freeing the descriptors as those will be cleaned up + * automatically when the device is destroyed. + */ + if (cache->cache) + _mesa_hash_table_destroy(cache->cache, NULL); +} + +struct nvk_edb_buffer_view_descriptor +nvk_edb_bview_cache_get_descriptor(struct nvk_device *dev, + struct nvk_edb_bview_cache *cache, + uint64_t base_addr, uint64_t size_B, + enum pipe_format format) +{ + /* The actual hardware limit for buffer image/texture descriptors is 4GB + * regardless of format. This cache works by covering the address space + * with 4GB buffer descriptors at 2GB offsets. In order for this to work + * properly, the size if the client's buffer view must be at most 2 GB. + */ + assert(size_B <= view_stride_B(format)); + + const uint8_t el_size_B = util_format_get_blocksize(format); + const uint64_t size_el = size_B / el_size_B; + + const uint64_t chunk = chunk_for_addr(dev, base_addr, format); + const uint64_t desc_base_addr = base_addr_for_chunk(dev, chunk, format); + const uint32_t offset_B = base_addr - desc_base_addr; + + const uint32_t offset_el = offset_B / el_size_B; + + uint16_t rgb_offset = 0; + if (!util_is_power_of_two_nonzero(el_size_B)) { + assert(util_format_get_nr_components(format) == 3); + assert(el_size_B % 3 == 0); + rgb_offset = offset_B % el_size_B; + } else { + assert(offset_B % el_size_B == 0); + } + + assert(offset_el + size_el > offset_el); + assert(offset_el + size_el <= view_size_el(format)); + + assert(format <= UINT16_MAX); + assert(chunk < (1u << 12)); + assert(rgb_offset < (1u << 4)); + const struct bvdesc_key key = { + .format = format, + .chunk = chunk, + .rgb_offset = rgb_offset, + }; + uint32_t index = nvk_edb_bview_cache_lookup_bview(dev, cache, key); + + uint32_t oob_alpha; + if (util_format_has_alpha(format)) { + /* OOB reads as if it read 0 texture data so an RGBA format reads + * (0, 0, 0, 0) out-of-bounds. + */ + oob_alpha = 0; + } else if (util_format_is_pure_integer(format)) { + /* OOB reads 0 texture data but then gets extended by (0, 0, 0, 1) */ + oob_alpha = 1; + } else { + /* OOB reads 0 texture data but then gets extended by + * (0.0, 0.0, 0.0, 1.0) + */ + oob_alpha = 0x3f800000; + } + + return (struct nvk_edb_buffer_view_descriptor) { + .index = index, + .offset_el = offset_el, + .size_el = size_el, + .oob_alpha = oob_alpha, + }; +} diff --git a/src/nouveau/vulkan/nvk_edb_bview_cache.h b/src/nouveau/vulkan/nvk_edb_bview_cache.h new file mode 100644 index 00000000000..a764d1025e7 --- /dev/null +++ b/src/nouveau/vulkan/nvk_edb_bview_cache.h @@ -0,0 +1,67 @@ +/* + * Copyright © 2022 Collabora Ltd. and Red Hat Inc. + * SPDX-License-Identifier: MIT + */ +#ifndef NVK_EDB_BVIEW_CACHE_H +#define NVK_EDB_BVIEW_CACHE_H 1 + +#include "nvk_private.h" + +#include "nvk_descriptor_types.h" + +#include "util/format/u_formats.h" + +struct hash_table; +struct nvk_device; + +/** A cache of VK_EXT_descriptor_buffer BufferViews + * + * VK_EXT_descriptor_buffer effectively removes the concept of a VkBufferView + * object. Instead of allocating a view object and passing that into + * vkGetDescriptorEXT() like you do for image views, typed buffers work more + * like untyped UBOs or SSBOs and you just pass a base address, size (in + * bytes) and format to vkGetDescriptorEXT(). On NVIDIA hardware, this is + * annoying because it means we no longer have an object to help us manage the + * life cycle of the descriptor on the heap. + * + * The solution is nvk_edb_bview_cache. This cache stores enough typed buffer + * descriptors to cover the entire address space. For each buffer format, we + * allocate 512 4 GiB buffer views, spaced at 2 GiB intervals. This ensures + * that every client buffer view will live entirely inside one of these views. + * The descriptor we return from vkGetDescriptorEXT() contains the descriptor + * index to the HW descriptor as well as an offset and size (both in surface + * elements) and the alpha value to expect for OOB writes. + * + * For RGB32 formats, we place 3 3 GiB buffer views every 1.5 GiB in the + * address space. We need 3 per chunk because RGB32 buffer views only have a + * minimum alignment of 4B but the offsetting we do in the shader is in terms + * of surface elements. For offsetting by 1 or 2 components, we need a + * different view. The reason why it's 3 GiB instead of 4 GiB is because + * Ampere reduced the maximum size of an RGB32 buffer view to 3 GiB. + * + * In nvk_nir_lower_descriptors(), we lower all texture or image buffer access + * to an access through one of these HW descriptors. Bounds checkinig is done + * in software and the offset is applied to ensure that we only ever read from + * the memory range specified by the client. The HW descriptor only exists to + * help with format conversion. + */ +struct nvk_edb_bview_cache { + struct hash_table *cache; +}; + +VkResult +nvk_edb_bview_cache_init(struct nvk_device *dev, + struct nvk_edb_bview_cache *cache); + +/* It's safe to call this function on a zeroed nvk_edb_bview_cache */ +void +nvk_edb_bview_cache_finish(struct nvk_device *dev, + struct nvk_edb_bview_cache *cache); + +struct nvk_edb_buffer_view_descriptor +nvk_edb_bview_cache_get_descriptor(struct nvk_device *dev, + struct nvk_edb_bview_cache *cache, + uint64_t base_addr, uint64_t size_B, + enum pipe_format format); + +#endif /* NVK_EDB_BVIEW_CACHE_H */