diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c
index 1dfe04552d5..f5a1913a96d 100644
--- a/src/amd/vulkan/radv_pipeline.c
+++ b/src/amd/vulkan/radv_pipeline.c
@@ -86,6 +86,9 @@ void
 radv_pipeline_destroy(struct radv_device *device, struct radv_pipeline *pipeline,
                       const VkAllocationCallbacks *allocator)
 {
+   if (pipeline->cache_object)
+      vk_pipeline_cache_object_unref(&device->vk, pipeline->cache_object);
+
    switch (pipeline->type) {
    case RADV_PIPELINE_GRAPHICS:
       radv_destroy_graphics_pipeline(device, radv_pipeline_to_graphics(pipeline));
diff --git a/src/amd/vulkan/radv_pipeline_cache.c b/src/amd/vulkan/radv_pipeline_cache.c
index 6deedf4764c..4e6ebba00b5 100644
--- a/src/amd/vulkan/radv_pipeline_cache.c
+++ b/src/amd/vulkan/radv_pipeline_cache.c
@@ -396,7 +396,7 @@ radv_pipeline_cache_search(struct radv_device *device, struct vk_pipeline_cache
          radv_pipeline_to_graphics_lib(pipeline)->base.ps_epilog = ps_epilog;
    }
 
-   vk_pipeline_cache_object_unref(&device->vk, object);
+   pipeline->cache_object = object;
    return true;
 }
 
@@ -447,8 +447,7 @@ radv_pipeline_cache_insert(struct radv_device *device, struct vk_pipeline_cache
    }
 
    /* Add the object to the cache */
-   struct vk_pipeline_cache_object *object = vk_pipeline_cache_add_object(cache, &pipeline_obj->base);
-   vk_pipeline_cache_object_unref(&device->vk, object);
+   pipeline->cache_object = vk_pipeline_cache_add_object(cache, &pipeline_obj->base);
 }
 
 bool
@@ -502,6 +501,7 @@ radv_ray_tracing_pipeline_cache_search(struct radv_device *device, struct vk_pip
             VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT;
    }
 
+   pipeline->base.base.cache_object = object;
    return complete;
 }
 
@@ -516,6 +516,13 @@ radv_ray_tracing_pipeline_cache_insert(struct radv_device *device, struct vk_pip
    if (!cache)
       cache = device->mem_cache;
 
+   /* Skip insertion on cache hit.
+    * This branch can be triggered if a cache_object was found but not all NIR shaders could be
+    * looked up. The cache_object is already complete in that case.
+    */
+   if (pipeline->base.base.cache_object)
+      return;
+
    /* Count compiled shaders excl. library shaders */
    unsigned num_shaders = pipeline->base.base.shaders[MESA_SHADER_INTERSECTION] ? 1 : 0;
    for (unsigned i = 0; i < num_stages; ++i)
@@ -540,8 +547,7 @@ radv_ray_tracing_pipeline_cache_insert(struct radv_device *device, struct vk_pip
       stack_sizes[i] = pipeline->stages[i].stack_size;
 
    /* Add the object to the cache */
-   struct vk_pipeline_cache_object *object = vk_pipeline_cache_add_object(cache, &pipeline_obj->base);
-   vk_pipeline_cache_object_unref(&device->vk, object);
+   pipeline->base.base.cache_object = vk_pipeline_cache_add_object(cache, &pipeline_obj->base);
 }
 
 struct vk_pipeline_cache_object *
diff --git a/src/amd/vulkan/radv_private.h b/src/amd/vulkan/radv_private.h
index 40d8a48db47..2ff9c3d9499 100644
--- a/src/amd/vulkan/radv_private.h
+++ b/src/amd/vulkan/radv_private.h
@@ -2200,6 +2200,8 @@ struct radv_pipeline {
    struct vk_object_base base;
    enum radv_pipeline_type type;
 
+   struct vk_pipeline_cache_object *cache_object;
+
    bool is_internal;
    bool need_indirect_descriptor_sets;
    struct radv_shader *shaders[MESA_VULKAN_SHADER_STAGES];