diff --git a/src/intel/vulkan/anv_cmd_buffer.c b/src/intel/vulkan/anv_cmd_buffer.c
index 3ebaa480ce0..1a98c178faa 100644
--- a/src/intel/vulkan/anv_cmd_buffer.c
+++ b/src/intel/vulkan/anv_cmd_buffer.c
@@ -941,11 +941,7 @@ anv_cmd_buffer_bind_descriptor_set(struct anv_cmd_buffer *cmd_buffer,
       if (bind_point == VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR) {
          struct anv_push_constants *push = &pipe_state->push_constants;
 
-         struct anv_address set_addr = {
-            .bo = set->pool->bo,
-            .offset = set->desc_mem.offset,
-         };
-         push->desc_sets[set_index] = anv_address_physical(set_addr);
+         push->desc_sets[set_index] = anv_address_physical(set->desc_addr);
 
          anv_reloc_list_add_bo(cmd_buffer->batch.relocs,
                                cmd_buffer->batch.alloc,
@@ -1409,7 +1405,7 @@ anv_cmd_buffer_push_descriptor_set(struct anv_cmd_buffer *cmd_buffer,
       }
       set->desc_mem = desc_mem;
 
-      struct anv_address addr = {
+      set->desc_addr = (struct anv_address) {
          .bo = cmd_buffer->dynamic_state_stream.state_pool->block_pool.bo,
          .offset = set->desc_mem.offset,
       };
@@ -1425,7 +1421,8 @@ anv_cmd_buffer_push_descriptor_set(struct anv_cmd_buffer *cmd_buffer,
       anv_fill_buffer_surface_state(cmd_buffer->device,
                                     set->desc_surface_state, format,
                                     ISL_SURF_USAGE_CONSTANT_BUFFER_BIT,
-                                    addr, layout->descriptor_buffer_size, 1);
+                                    set->desc_addr,
+                                    layout->descriptor_buffer_size, 1);
    }
 
    return set;
diff --git a/src/intel/vulkan/anv_descriptor_set.c b/src/intel/vulkan/anv_descriptor_set.c
index 5ac21e2db5b..cab5402e51b 100644
--- a/src/intel/vulkan/anv_descriptor_set.c
+++ b/src/intel/vulkan/anv_descriptor_set.c
@@ -1079,6 +1079,11 @@ anv_descriptor_set_create(struct anv_device *device,
       set->desc_mem.alloc_size = descriptor_buffer_size;
       set->desc_mem.map = pool->bo->map + set->desc_mem.offset;
 
+      set->desc_addr = (struct anv_address) {
+         .bo = pool->bo,
+         .offset = set->desc_mem.offset,
+      };
+
       enum isl_format format =
          anv_isl_format_for_descriptor_type(device,
                                             VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER);
@@ -1086,13 +1091,11 @@ anv_descriptor_set_create(struct anv_device *device,
       set->desc_surface_state = anv_descriptor_pool_alloc_state(pool);
       anv_fill_buffer_surface_state(device, set->desc_surface_state, format,
                                     ISL_SURF_USAGE_CONSTANT_BUFFER_BIT,
-                                    (struct anv_address) {
-                                       .bo = pool->bo,
-                                       .offset = set->desc_mem.offset,
-                                    },
+                                    set->desc_addr,
                                     descriptor_buffer_size, 1);
    } else {
       set->desc_mem = ANV_STATE_NULL;
+      set->desc_addr = (struct anv_address) { .bo = NULL, .offset = 0 };
       set->desc_surface_state = ANV_STATE_NULL;
    }
 
diff --git a/src/intel/vulkan/anv_private.h b/src/intel/vulkan/anv_private.h
index 0aa21b43749..7b133f77235 100644
--- a/src/intel/vulkan/anv_private.h
+++ b/src/intel/vulkan/anv_private.h
@@ -2047,6 +2047,9 @@ struct anv_descriptor_set {
    /* Surface state for the descriptor buffer */
    struct anv_state desc_surface_state;
 
+   /* Descriptor set address. */
+   struct anv_address desc_addr;
+
    uint32_t buffer_view_count;
    struct anv_buffer_view *buffer_views;
 
diff --git a/src/intel/vulkan/genX_cmd_buffer.c b/src/intel/vulkan/genX_cmd_buffer.c
index abb66c0de56..217e19e48b5 100644
--- a/src/intel/vulkan/genX_cmd_buffer.c
+++ b/src/intel/vulkan/genX_cmd_buffer.c
@@ -2551,25 +2551,16 @@ static struct anv_address
 anv_descriptor_set_address(struct anv_cmd_buffer *cmd_buffer,
                            struct anv_descriptor_set *set)
 {
-   if (set->pool) {
-      /* This is a normal descriptor set */
-      return (struct anv_address) {
-         .bo = set->pool->bo,
-         .offset = set->desc_mem.offset,
-      };
-   } else {
+   if (set->pool == NULL) {
       /* This is a push descriptor set.  We have to flag it as used on the GPU
        * so that the next time we push descriptors, we grab a new memory.
        */
       struct anv_push_descriptor_set *push_set =
          (struct anv_push_descriptor_set *)set;
       push_set->set_used_on_gpu = true;
-
-      return (struct anv_address) {
-         .bo = cmd_buffer->dynamic_state_stream.state_pool->block_pool.bo,
-         .offset = set->desc_mem.offset,
-      };
    }
+
+   return set->desc_addr;
 }
 
 static VkResult