diff --git a/src/vulkan/runtime/meson.build b/src/vulkan/runtime/meson.build index af134efb04e..196997490d1 100644 --- a/src/vulkan/runtime/meson.build +++ b/src/vulkan/runtime/meson.build @@ -54,6 +54,8 @@ vulkan_runtime_files = files( 'vk_nir.h', 'vk_object.c', 'vk_object.h', + 'vk_pipeline_cache.c', + 'vk_pipeline_cache.h', 'vk_physical_device.c', 'vk_physical_device.h', 'vk_queue.c', diff --git a/src/vulkan/runtime/vk_physical_device.c b/src/vulkan/runtime/vk_physical_device.c index 8915e73fa7c..2b747a0cb60 100644 --- a/src/vulkan/runtime/vk_physical_device.c +++ b/src/vulkan/runtime/vk_physical_device.c @@ -45,6 +45,9 @@ vk_physical_device_init(struct vk_physical_device *pdevice, vk_physical_device_dispatch_table_from_entrypoints( &pdevice->dispatch_table, &vk_common_physical_device_entrypoints, false); + /* TODO */ + pdevice->disk_cache = NULL; + return VK_SUCCESS; } diff --git a/src/vulkan/runtime/vk_physical_device.h b/src/vulkan/runtime/vk_physical_device.h index f8be3deaf4c..eac023df917 100644 --- a/src/vulkan/runtime/vk_physical_device.h +++ b/src/vulkan/runtime/vk_physical_device.h @@ -31,8 +31,10 @@ extern "C" { #endif +struct disk_cache; struct wsi_device; struct vk_sync_type; +struct vk_pipeline_cache_object_ops; /** Base struct for all VkPhysicalDevice implementations */ @@ -57,6 +59,9 @@ struct vk_physical_device { /** Physical-device-level dispatch table */ struct vk_physical_device_dispatch_table dispatch_table; + /** Disk cache, or NULL */ + struct disk_cache *disk_cache; + /** WSI device, or NULL */ struct wsi_device *wsi_device; @@ -70,6 +75,17 @@ struct vk_physical_device { * considered just one more criterion. */ const struct vk_sync_type *const *supported_sync_types; + + /** A null-terminated array of supported pipeline cache object types + * + * The common implementation of VkPipelineCache uses this to remember the + * type of objects stored in the cache and deserialize them immediately + * when importing the cache. If an object type isn't in this list, then it + * will be loaded as a raw data object and then deserialized when we first + * look it up. Deserializing immediately avoids a copy but may be more + * expensive for objects that aren't hit. + */ + const struct vk_pipeline_cache_object_ops *const *pipeline_cache_import_ops; }; VK_DEFINE_HANDLE_CASTS(vk_physical_device, base, VkPhysicalDevice, diff --git a/src/vulkan/runtime/vk_pipeline_cache.c b/src/vulkan/runtime/vk_pipeline_cache.c new file mode 100644 index 00000000000..2214b27751c --- /dev/null +++ b/src/vulkan/runtime/vk_pipeline_cache.c @@ -0,0 +1,731 @@ +/* + * Copyright © 2021 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "vk_pipeline_cache.h" + +#include "vk_alloc.h" +#include "vk_common_entrypoints.h" +#include "vk_device.h" +#include "vk_log.h" +#include "vk_physical_device.h" + +#include "util/blob.h" +#include "util/debug.h" +#include "util/disk_cache.h" +#include "util/hash_table.h" +#include "util/set.h" + +struct raw_data_object { + struct vk_pipeline_cache_object base; + + const void *data; + size_t data_size; +}; + +static bool +raw_data_object_serialize(struct vk_pipeline_cache_object *object, + struct blob *blob) +{ + struct raw_data_object *data_obj = + container_of(object, struct raw_data_object, base); + + blob_write_bytes(blob, data_obj->data, data_obj->data_size); + + return true; +} + +static void +raw_data_object_destroy(struct vk_pipeline_cache_object *object) +{ + struct raw_data_object *data_obj = + container_of(object, struct raw_data_object, base); + + vk_free(&data_obj->base.device->alloc, data_obj); +} + +static const struct vk_pipeline_cache_object_ops raw_data_object_ops = { + .serialize = raw_data_object_serialize, + .destroy = raw_data_object_destroy, +}; + +static struct raw_data_object * +raw_data_object_create(struct vk_device *device, + const void *key_data, size_t key_size, + const void *data, size_t data_size) +{ + VK_MULTIALLOC(ma); + VK_MULTIALLOC_DECL(&ma, struct raw_data_object, data_obj, 1); + VK_MULTIALLOC_DECL_SIZE(&ma, void, obj_key_data, key_size); + VK_MULTIALLOC_DECL_SIZE(&ma, void, obj_data, data_size); + + if (!vk_multialloc_alloc(&ma, &device->alloc, + VK_SYSTEM_ALLOCATION_SCOPE_DEVICE)) + return NULL; + + vk_pipeline_cache_object_init(device, &data_obj->base, + &raw_data_object_ops, + obj_key_data, key_size); + data_obj->data = obj_data; + data_obj->data_size = data_size; + + memcpy(obj_key_data, key_data, key_size); + memcpy(obj_data, data, data_size); + + return data_obj; +} + +static bool +object_keys_equal(const void *void_a, const void *void_b) +{ + const struct vk_pipeline_cache_object *a = void_a, *b = void_b; + if (a->key_size != b->key_size) + return false; + + return memcmp(a->key_data, b->key_data, a->key_size) == 0; +} + +static uint32_t +object_key_hash(const void *void_object) +{ + const struct vk_pipeline_cache_object *object = void_object; + return _mesa_hash_data(object->key_data, object->key_size); +} + +static void +vk_pipeline_cache_lock(struct vk_pipeline_cache *cache) +{ + + if (!(cache->flags & VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT_EXT)) + simple_mtx_lock(&cache->lock); +} + +static void +vk_pipeline_cache_unlock(struct vk_pipeline_cache *cache) +{ + if (!(cache->flags & VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT_EXT)) + simple_mtx_unlock(&cache->lock); +} + +static void +vk_pipeline_cache_remove_object(struct vk_pipeline_cache *cache, + uint32_t hash, + struct vk_pipeline_cache_object *object) +{ + vk_pipeline_cache_lock(cache); + struct set_entry *entry = + _mesa_set_search_pre_hashed(cache->object_cache, hash, object); + if (entry && entry->key == (const void *)object) { + /* Drop the reference owned by the cache */ + vk_pipeline_cache_object_unref(object); + + _mesa_set_remove(cache->object_cache, entry); + } + vk_pipeline_cache_unlock(cache); + + /* Drop our reference */ + vk_pipeline_cache_object_unref(object); +} + +/* Consumes references to both search and replace and produces a reference */ +static struct vk_pipeline_cache_object * +vk_pipeline_cache_replace_object(struct vk_pipeline_cache *cache, + uint32_t hash, + struct vk_pipeline_cache_object *search, + struct vk_pipeline_cache_object *replace) +{ + assert(object_keys_equal(search, replace)); + + vk_pipeline_cache_lock(cache); + struct set_entry *entry = + _mesa_set_search_pre_hashed(cache->object_cache, hash, search); + + struct vk_pipeline_cache_object *found = NULL; + if (entry) { + if (entry->key == (const void *)search) { + /* Drop the reference owned by the cache */ + vk_pipeline_cache_object_unref(search); + + entry->key = vk_pipeline_cache_object_ref(replace); + } else { + found = vk_pipeline_cache_object_ref((void *)entry->key); + } + } else { + /* I guess the object was purged? Re-add it to the cache */ + vk_pipeline_cache_object_ref(replace); + _mesa_set_add_pre_hashed(cache->object_cache, hash, replace); + } + vk_pipeline_cache_unlock(cache); + + vk_pipeline_cache_object_unref(search); + + if (found) { + vk_pipeline_cache_object_unref(replace); + return found; + } else { + return replace; + } +} + +static bool +vk_pipeline_cache_object_serialize(struct vk_pipeline_cache *cache, + struct vk_pipeline_cache_object *object, + struct blob *blob, uint32_t *data_size) +{ + if (object->ops->serialize == NULL) + return false; + + assert(blob->size == align64(blob->size, VK_PIPELINE_CACHE_BLOB_ALIGN)); + size_t start = blob->size; + + /* Special case for if we're writing to a NULL blob (just to get the size) + * and we already know the data size of the allocation. This should make + * the first GetPipelineCacheData() call to get the data size faster in the + * common case where a bunch of our objects were loaded from a previous + * cache or where we've already serialized the cache once. + */ + if (blob->data == NULL && blob->fixed_allocation) { + *data_size = p_atomic_read(&object->data_size); + if (*data_size > 0) { + blob_write_bytes(blob, NULL, *data_size); + return true; + } + } + + if (!object->ops->serialize(object, blob)) { + vk_logw(VK_LOG_OBJS(cache), + "Failed to serialize pipeline cache object"); + return false; + } + + size_t size = blob->size - start; + if (size > UINT32_MAX) { + vk_logw(VK_LOG_OBJS(cache), + "Skipping giant (4 GiB or larger) object"); + return false; + } + + if (blob->out_of_memory) { + vk_logw(VK_LOG_OBJS(cache), + "Insufficient memory for pipeline cache data"); + return false; + } + + *data_size = (uint32_t)size; + p_atomic_set(&object->data_size, *data_size); + + return true; +} + +static struct vk_pipeline_cache_object * +vk_pipeline_cache_object_deserialize(struct vk_pipeline_cache *cache, + const void *key_data, uint32_t key_size, + const void *data, size_t data_size, + const struct vk_pipeline_cache_object_ops *ops) +{ + if (unlikely(ops->deserialize == NULL)) { + vk_logw(VK_LOG_OBJS(cache), + "Pipeline cache object cannot be deserialized"); + return NULL; + } + + struct blob_reader reader; + blob_reader_init(&reader, data, data_size); + + struct vk_pipeline_cache_object *object = + ops->deserialize(cache->base.device, key_data, key_size, &reader); + + if (object == NULL) { + vk_logw(VK_LOG_OBJS(cache), + "Deserializing pipeline cache object failed"); + return NULL; + } + + assert(reader.current == reader.end && !reader.overrun); + assert(object->device == cache->base.device); + assert(object->ops == ops); + assert(object->ref_cnt == 1); + assert(object->key_size == key_size); + assert(memcmp(object->key_data, key_data, key_size) == 0); + + return object; +} + +struct vk_pipeline_cache_object * +vk_pipeline_cache_lookup_object(struct vk_pipeline_cache *cache, + const void *key_data, size_t key_size, + const struct vk_pipeline_cache_object_ops *ops, + bool *cache_hit) +{ + assert(key_size <= UINT32_MAX); + assert(ops != NULL); + + if (cache_hit != NULL) + *cache_hit = false; + + struct vk_pipeline_cache_object key = { + .key_data = key_data, + .key_size = key_size, + }; + uint32_t hash = object_key_hash(&key); + + struct vk_pipeline_cache_object *object = NULL; + + if (cache != NULL && cache->object_cache != NULL) { + vk_pipeline_cache_lock(cache); + struct set_entry *entry = + _mesa_set_search_pre_hashed(cache->object_cache, hash, &key); + if (entry) { + object = vk_pipeline_cache_object_ref((void *)entry->key); + if (cache_hit != NULL) + *cache_hit = true; + } + vk_pipeline_cache_unlock(cache); + } + + if (object == NULL) { +#ifdef ENABLE_SHADER_CACHE + struct disk_cache *disk_cache = cache->base.device->physical->disk_cache; + if (disk_cache != NULL) { + cache_key cache_key; + disk_cache_compute_key(disk_cache, key_data, key_size, cache_key); + + size_t data_size; + uint8_t *data = disk_cache_get(disk_cache, cache_key, &data_size); + if (data) { + object = vk_pipeline_cache_object_deserialize(cache, + key_data, key_size, + data, data_size, + ops); + free(data); + if (object != NULL) + return vk_pipeline_cache_add_object(cache, object); + } + } +#endif + + /* No disk cache or not found in the disk cache */ + return NULL; + } + + if (object->ops == &raw_data_object_ops) { + /* The object isn't fully formed yet and we need to deserialize it into + * a real object before it can be used. + */ + struct raw_data_object *data_obj = + container_of(object, struct raw_data_object, base); + + assert(ops != &raw_data_object_ops); + + struct vk_pipeline_cache_object *real_object = + vk_pipeline_cache_object_deserialize(cache, + data_obj->base.key_data, + data_obj->base.key_size, + data_obj->data, + data_obj->data_size, ops); + if (real_object == NULL) { + vk_pipeline_cache_remove_object(cache, hash, object); + return NULL; + } + + object = vk_pipeline_cache_replace_object(cache, hash, object, + real_object); + } + + assert(object->ops == ops); + + return object; +} + +struct vk_pipeline_cache_object * +vk_pipeline_cache_add_object(struct vk_pipeline_cache *cache, + struct vk_pipeline_cache_object *object) +{ + assert(object->ops != NULL); + + if (cache->object_cache == NULL) + return object; + + uint32_t hash = object_key_hash(object); + + vk_pipeline_cache_lock(cache); + bool found = false; + struct set_entry *entry = + _mesa_set_search_or_add_pre_hashed(cache->object_cache, + hash, object, &found); + + struct vk_pipeline_cache_object *found_object = NULL; + if (found) { + found_object = vk_pipeline_cache_object_ref((void *)entry->key); + } else { + /* The cache now owns a reference */ + vk_pipeline_cache_object_ref(object); + } + vk_pipeline_cache_unlock(cache); + + if (found) { + vk_pipeline_cache_object_unref(object); + return found_object; + } else { + /* If it wasn't in the object cache, it might not be in the disk cache + * either. Better try and add it. + */ + +#ifdef ENABLE_SHADER_CACHE + struct disk_cache *disk_cache = cache->base.device->physical->disk_cache; + if (object->ops->serialize != NULL && disk_cache) { + struct blob blob; + blob_init(&blob); + + if (object->ops->serialize(object, &blob) && !blob.out_of_memory) { + cache_key cache_key; + disk_cache_compute_key(disk_cache, object->key_data, + object->key_size, cache_key); + + disk_cache_put(disk_cache, cache_key, blob.data, blob.size, NULL); + } + + blob_finish(&blob); + } +#endif + + return object; + } +} + +static int32_t +find_type_for_ops(const struct vk_physical_device *pdevice, + const struct vk_pipeline_cache_object_ops *ops) +{ + const struct vk_pipeline_cache_object_ops *const *import_ops = + pdevice->pipeline_cache_import_ops; + + if (import_ops == NULL) + return -1; + + for (int32_t i = 0; import_ops[i]; i++) { + if (import_ops[i] == ops) + return i; + } + + return -1; +} + +static const struct vk_pipeline_cache_object_ops * +find_ops_for_type(const struct vk_physical_device *pdevice, + int32_t type) +{ + const struct vk_pipeline_cache_object_ops *const *import_ops = + pdevice->pipeline_cache_import_ops; + + if (import_ops == NULL || type < 0) + return NULL; + + return import_ops[type]; +} + +static void +vk_pipeline_cache_load(struct vk_pipeline_cache *cache, + const void *data, size_t size) +{ + struct blob_reader blob; + blob_reader_init(&blob, data, size); + + struct vk_pipeline_cache_header header; + blob_copy_bytes(&blob, &header, sizeof(header)); + uint32_t count = blob_read_uint32(&blob); + if (blob.overrun) + return; + + if (memcmp(&header, &cache->header, sizeof(header)) != 0) + return; + + for (uint32_t i = 0; i < count; i++) { + int32_t type = blob_read_uint32(&blob); + uint32_t key_size = blob_read_uint32(&blob); + uint32_t data_size = blob_read_uint32(&blob); + const void *key_data = blob_read_bytes(&blob, key_size); + blob_reader_align(&blob, VK_PIPELINE_CACHE_BLOB_ALIGN); + const void *data = blob_read_bytes(&blob, data_size); + if (blob.overrun) + break; + + const struct vk_pipeline_cache_object_ops *ops = + find_ops_for_type(cache->base.device->physical, type); + + struct vk_pipeline_cache_object *object; + if (ops != NULL) { + object = vk_pipeline_cache_object_deserialize(cache, + key_data, key_size, + data, data_size, ops); + } else { + struct raw_data_object *data_obj = + raw_data_object_create(cache->base.device, + key_data, key_size, + data, data_size); + object = data_obj != NULL ? &data_obj->base : NULL; + } + if (object == NULL) + continue; + + object = vk_pipeline_cache_add_object(cache, object); + vk_pipeline_cache_object_unref(object); + } +} + +struct vk_pipeline_cache * +vk_pipeline_cache_create(struct vk_device *device, + const struct vk_pipeline_cache_create_info *info, + const VkAllocationCallbacks *pAllocator) +{ + static const struct VkPipelineCacheCreateInfo default_create_info = { + .sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, + }; + struct vk_pipeline_cache *cache; + + const struct VkPipelineCacheCreateInfo *pCreateInfo = + info->pCreateInfo != NULL ? info->pCreateInfo : &default_create_info; + + assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO); + assert(pCreateInfo->flags == 0); + + cache = vk_object_alloc(device, pAllocator, sizeof(*cache), + VK_OBJECT_TYPE_PIPELINE_CACHE); + if (cache == NULL) + return NULL; + + cache->flags = pCreateInfo->flags; + + struct VkPhysicalDeviceProperties pdevice_props; + device->physical->dispatch_table.GetPhysicalDeviceProperties( + vk_physical_device_to_handle(device->physical), &pdevice_props); + + cache->header = (struct vk_pipeline_cache_header) { + .header_size = sizeof(struct vk_pipeline_cache_header), + .header_version = VK_PIPELINE_CACHE_HEADER_VERSION_ONE, + .vendor_id = pdevice_props.vendorID, + .device_id = pdevice_props.deviceID, + }; + memcpy(cache->header.uuid, pdevice_props.pipelineCacheUUID, VK_UUID_SIZE); + + simple_mtx_init(&cache->lock, mtx_plain); + + if (info->force_enable || + env_var_as_boolean("VK_ENABLE_PIPELINE_CACHE", true)) { + cache->object_cache = _mesa_set_create(NULL, object_key_hash, + object_keys_equal); + } + + if (cache->object_cache && pCreateInfo->initialDataSize > 0) { + vk_pipeline_cache_load(cache, pCreateInfo->pInitialData, + pCreateInfo->initialDataSize); + } + + return cache; +} + +static void +object_unref_cb(struct set_entry *entry) +{ + vk_pipeline_cache_object_unref((void *)entry->key); +} + +void +vk_pipeline_cache_destroy(struct vk_pipeline_cache *cache, + const VkAllocationCallbacks *pAllocator) +{ + if (cache->object_cache) + _mesa_set_destroy(cache->object_cache, object_unref_cb); + simple_mtx_destroy(&cache->lock); + vk_object_free(cache->base.device, pAllocator, cache); +} + +VKAPI_ATTR VkResult VKAPI_CALL +vk_common_CreatePipelineCache(VkDevice _device, + const VkPipelineCacheCreateInfo *pCreateInfo, + const VkAllocationCallbacks *pAllocator, + VkPipelineCache *pPipelineCache) +{ + VK_FROM_HANDLE(vk_device, device, _device); + struct vk_pipeline_cache *cache; + + struct vk_pipeline_cache_create_info info = { + .pCreateInfo = pCreateInfo, + }; + cache = vk_pipeline_cache_create(device, &info, pAllocator); + if (cache == NULL) + return VK_ERROR_OUT_OF_HOST_MEMORY; + + *pPipelineCache = vk_pipeline_cache_to_handle(cache); + + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL +vk_common_DestroyPipelineCache(VkDevice device, + VkPipelineCache pipelineCache, + const VkAllocationCallbacks *pAllocator) +{ + VK_FROM_HANDLE(vk_pipeline_cache, cache, pipelineCache); + + if (cache == NULL) + return; + + assert(cache->base.device == vk_device_from_handle(device)); + vk_pipeline_cache_destroy(cache, pAllocator); +} + +VKAPI_ATTR VkResult VKAPI_CALL +vk_common_GetPipelineCacheData(VkDevice _device, + VkPipelineCache pipelineCache, + size_t *pDataSize, + void *pData) +{ + VK_FROM_HANDLE(vk_device, device, _device); + VK_FROM_HANDLE(vk_pipeline_cache, cache, pipelineCache); + + struct blob blob; + if (pData) { + blob_init_fixed(&blob, pData, *pDataSize); + } else { + blob_init_fixed(&blob, NULL, SIZE_MAX); + } + + blob_write_bytes(&blob, &cache->header, sizeof(cache->header)); + + uint32_t count = 0; + intptr_t count_offset = blob_reserve_uint32(&blob); + if (count_offset < 0) { + *pDataSize = 0; + blob_finish(&blob); + return VK_INCOMPLETE; + } + + vk_pipeline_cache_lock(cache); + + VkResult result = VK_SUCCESS; + if (cache->object_cache != NULL) { + set_foreach(cache->object_cache, entry) { + struct vk_pipeline_cache_object *object = (void *)entry->key; + + if (object->ops->serialize == NULL) + continue; + + size_t blob_size_save = blob.size; + + int32_t type = find_type_for_ops(device->physical, object->ops); + blob_write_uint32(&blob, type); + blob_write_uint32(&blob, object->key_size); + intptr_t data_size_resv = blob_reserve_uint32(&blob); + blob_write_bytes(&blob, object->key_data, object->key_size); + + blob_align(&blob, VK_PIPELINE_CACHE_BLOB_ALIGN); + + uint32_t data_size; + if (!vk_pipeline_cache_object_serialize(cache, object, + &blob, &data_size)) { + blob.size = blob_size_save; + if (blob.out_of_memory) { + result = VK_INCOMPLETE; + break; + } + + /* Failed for some other reason; keep going */ + continue; + } + + /* vk_pipeline_cache_object_serialize should have failed */ + assert(!blob.out_of_memory); + + assert(data_size_resv >= 0); + blob_overwrite_uint32(&blob, data_size_resv, data_size); + } + } + + vk_pipeline_cache_unlock(cache); + + blob_overwrite_uint32(&blob, count_offset, count); + + *pDataSize = blob.size; + + blob_finish(&blob); + + return result; +} + +VKAPI_ATTR VkResult VKAPI_CALL +vk_common_MergePipelineCaches(VkDevice device, + VkPipelineCache dstCache, + uint32_t srcCacheCount, + const VkPipelineCache *pSrcCaches) +{ + VK_FROM_HANDLE(vk_pipeline_cache, dst, dstCache); + + if (!dst->object_cache) + return VK_SUCCESS; + + vk_pipeline_cache_lock(dst); + + for (uint32_t i = 0; i < srcCacheCount; i++) { + VK_FROM_HANDLE(vk_pipeline_cache, src, pSrcCaches[i]); + + if (!src->object_cache) + continue; + + assert(src != dst); + if (src == dst) + continue; + + vk_pipeline_cache_lock(src); + + set_foreach(src->object_cache, src_entry) { + struct vk_pipeline_cache_object *src_object = (void *)src_entry->key; + + bool found_in_dst = false; + struct set_entry *dst_entry = + _mesa_set_search_or_add_pre_hashed(dst->object_cache, + src_entry->hash, + src_object, &found_in_dst); + if (found_in_dst) { + struct vk_pipeline_cache_object *dst_object = (void *)dst_entry->key; + if (dst_object->ops == &raw_data_object_ops && + src_object->ops != &raw_data_object_ops) { + /* Even though dst has the object, it only has the blob version + * which isn't as useful. Replace it with the real object. + */ + vk_pipeline_cache_object_unref(dst_object); + dst_entry->key = vk_pipeline_cache_object_ref(src_object); + } + } else { + /* We inserted src_object in dst so it needs a reference */ + assert(dst_entry->key == (const void *)src_object); + vk_pipeline_cache_object_ref(src_object); + } + } + + vk_pipeline_cache_unlock(src); + } + + vk_pipeline_cache_unlock(dst); + + return VK_SUCCESS; +} diff --git a/src/vulkan/runtime/vk_pipeline_cache.h b/src/vulkan/runtime/vk_pipeline_cache.h new file mode 100644 index 00000000000..2340a2c12d1 --- /dev/null +++ b/src/vulkan/runtime/vk_pipeline_cache.h @@ -0,0 +1,243 @@ +/* + * Copyright © 2021 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#ifndef VK_PIPELINE_CACHE_H +#define VK_PIPELINE_CACHE_H + +#include "vk_object.h" +#include "vk_util.h" + +#include "util/simple_mtx.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* #include "util/blob.h" */ +struct blob; +struct blob_reader; + +/* #include "util/set.h" */ +struct set; + +struct vk_pipeline_cache; +struct vk_pipeline_cache_object; + +#define VK_PIPELINE_CACHE_BLOB_ALIGN 8 + +struct vk_pipeline_cache_object_ops { + /** Writes this cache object to the given blob + * + * Because the cache works with both raw blob data and driver object data + * and can't always tell the difference between the two, we have to be very + * careful about alignments when [de]serializing. When serialize() is + * called, the blob will be aligned to VK_PIPELINE_CACHE_BLOB_ALIGN. The + * driver must be careful to not [de]serialize any data types which require + * a higher alignment. When deserialize() is called, the blob_reader is + * also guaranteed to be aligned to VK_PIPELINE_CACHE_BLOB_ALIGN. + * + * Returns true on success + * + * This function is optional. Objects without [de]serialization support + * will still be cached in memory but will not be placed in the disk cache + * and will not exported to the client when vkGetPipelineCacheData() is + * called. + */ + bool (*serialize)(struct vk_pipeline_cache_object *object, + struct blob *blob); + + /** Constructs an object from cached data + * + * See serialize() for details about data alignment. + * + * returns the created object + * + * This function is optional. + */ + struct vk_pipeline_cache_object *(*deserialize)(struct vk_device *device, + const void *key_data, + size_t key_size, + struct blob_reader *blob); + + /** Destroys the object + * + * Called when vk_pipeline_cache_object.ref_cnt hits 0. + */ + void (*destroy)(struct vk_pipeline_cache_object *object); +}; + +/** Base struct for cached objects + * + * A vk_pipeline_cache stores any number of vk_pipeline_cache_object's, each + * of which has an associated key of arbitrary size. Cached objects are + * reference counted so that they can exist in multiple caches (for example, + * when vkMergePipelineCaches() is called) and so that they can persist after + * the pipeline cache is destroyed. Each object also has a pointer to a + * vk_pipeline_cache_object_ops table which the pipeline cache uses to + * [de]serialize the object and clean it up when the reference count hits 0. + * + * The rest of the details of any given object are entirely up to the driver. + * The driver may even have multiple types of objects (distinguished by their + * vk_pipeline_cache_object_ops table) in the cache so long as it guarantees + * it never has two objects of different types with the same key. + */ +struct vk_pipeline_cache_object { + struct vk_device *device; + const struct vk_pipeline_cache_object_ops *ops; + uint32_t ref_cnt; + + uint32_t data_size; + const void *key_data; + uint32_t key_size; +}; + +static inline void +vk_pipeline_cache_object_init(struct vk_device *device, + struct vk_pipeline_cache_object *object, + const struct vk_pipeline_cache_object_ops *ops, + const void *key_data, uint32_t key_size) +{ + object->device = device; + object->ops = ops; + p_atomic_set(&object->ref_cnt, 1); + object->data_size = 0; /* Unknown */ + object->key_data = key_data; + object->key_size = key_size; +} + +static inline void +vk_pipeline_cache_object_finish(struct vk_pipeline_cache_object *object) +{ + assert(p_atomic_read(&object->ref_cnt) <= 1); +} + +static inline struct vk_pipeline_cache_object * +vk_pipeline_cache_object_ref(struct vk_pipeline_cache_object *object) +{ + assert(object && p_atomic_read(&object->ref_cnt) >= 1); + p_atomic_inc(&object->ref_cnt); + return object; +} + +static inline void +vk_pipeline_cache_object_unref(struct vk_pipeline_cache_object *object) +{ + assert(object && p_atomic_read(&object->ref_cnt) >= 1); + if (p_atomic_dec_zero(&object->ref_cnt)) + object->ops->destroy(object); +} + +/** A generic implementation of VkPipelineCache */ +struct vk_pipeline_cache { + struct vk_object_base base; + + /* pCreateInfo::flags */ + VkPipelineCacheCreateFlags flags; + + struct vk_pipeline_cache_header header; + + /** Protects object_cache */ + simple_mtx_t lock; + + struct set *object_cache; +}; + +VK_DEFINE_NONDISP_HANDLE_CASTS(vk_pipeline_cache, base, VkPipelineCache, + VK_OBJECT_TYPE_PIPELINE_CACHE) + +struct vk_pipeline_cache_create_info { + /* The pCreateInfo for this pipeline cache, if any. + * + * For driver-internal caches, this is allowed to be NULL. + */ + const VkPipelineCacheCreateInfo *pCreateInfo; + + /** If true, ignore VK_ENABLE_PIPELINE_CACHE and enable anyway */ + bool force_enable; +}; + +struct vk_pipeline_cache * +vk_pipeline_cache_create(struct vk_device *device, + const struct vk_pipeline_cache_create_info *info, + const VkAllocationCallbacks *pAllocator); +void +vk_pipeline_cache_destroy(struct vk_pipeline_cache *cache, + const VkAllocationCallbacks *pAllocator); + +/** Attempts to look up an object in the cache by key + * + * If an object is found in the cache matching the given key, *cache_hit is + * set to true and a reference to that object is returned. + * + * If the driver sets vk_device.disk_cache, we attempt to look up any missing + * objects in the disk cache before declaring failure. If an object is found + * in the disk cache but not the in-memory cache, *cache_hit is set to false. + * + * The deserialization of pipeline cache objects found in the cache data + * provided via VkPipelineCacheCreateInfo::pInitialData happens during + * vk_pipeline_cache_lookup() rather than during vkCreatePipelineCache(). + * Prior to the first vk_pipeline_cache_lookup() of a given object, it is + * stored as an internal raw data object with the same hash. This allows us + * to avoid any complex object type tagging in the serialized cache. It does, + * however, mean that drivers need to be careful to ensure that objects with + * different types (ops) have different keys. + * + * Returns a reference to the object, if found + */ +struct vk_pipeline_cache_object * MUST_CHECK +vk_pipeline_cache_lookup_object(struct vk_pipeline_cache *cache, + const void *key_data, size_t key_size, + const struct vk_pipeline_cache_object_ops *ops, + bool *cache_hit); + +/** Adds an object to the pipeline cache + * + * This function adds the given object to the pipeline cache. We do not + * specify a key here because the key is part of the object. See also + * vk_pipeline_cache_object_init(). + * + * This function consumes a reference to the object and returns a reference to + * the (possibly different) object in the cache. The intended usage pattern + * is as follows: + * + * key = compute_key(); + * struct vk_pipeline_cache_object *object = + * vk_pipeline_cache_lookup_object(cache, &key, sizeof(key), + * &driver_type_ops, &cache_hit); + * if (object != NULL) + * return container_of(object, driver_type, base); + * + * object = do_compile(); + * assert(object != NULL); + * + * object = vk_pipeline_cache_add_object(cache, object); + * return container_of(object, driver_type, base); + */ +struct vk_pipeline_cache_object * MUST_CHECK +vk_pipeline_cache_add_object(struct vk_pipeline_cache *cache, + struct vk_pipeline_cache_object *object); + +#ifdef __cplusplus +} +#endif + +#endif /* VK_PIPELINE_CACHE_H */