diff --git a/src/broadcom/vulkan/v3dv_bo.c b/src/broadcom/vulkan/v3dv_bo.c
index 459032990b2..71679ceec27 100644
--- a/src/broadcom/vulkan/v3dv_bo.c
+++ b/src/broadcom/vulkan/v3dv_bo.c
@@ -184,6 +184,7 @@ v3dv_bo_init(struct v3dv_bo *bo,
              bool private)
 {
    bo->handle = handle;
+   bo->handle_bit = 1ull << (handle % 64);
    bo->size = size;
    bo->offset = offset;
    bo->map = NULL;
diff --git a/src/broadcom/vulkan/v3dv_bo.h b/src/broadcom/vulkan/v3dv_bo.h
index fd6754c489f..ab2b8c7356d 100644
--- a/src/broadcom/vulkan/v3dv_bo.h
+++ b/src/broadcom/vulkan/v3dv_bo.h
@@ -30,6 +30,7 @@ struct v3dv_bo {
    struct list_head list_link;
 
    uint32_t handle;
+   uint64_t handle_bit;
    uint32_t size;
    uint32_t offset;
 
diff --git a/src/broadcom/vulkan/v3dv_cmd_buffer.c b/src/broadcom/vulkan/v3dv_cmd_buffer.c
index a4ace79257c..6a4e4476431 100644
--- a/src/broadcom/vulkan/v3dv_cmd_buffer.c
+++ b/src/broadcom/vulkan/v3dv_cmd_buffer.c
@@ -65,11 +65,14 @@ v3dv_job_add_bo(struct v3dv_job *job, struct v3dv_bo *bo)
    if (!bo)
       return;
 
-   if (_mesa_set_search(job->bos, bo))
-      return;
+   if (job->bo_handle_mask & bo->handle_bit) {
+      if (_mesa_set_search(job->bos, bo))
+         return;
+   }
 
    _mesa_set_add(job->bos, bo);
    job->bo_count++;
+   job->bo_handle_mask |= bo->handle_bit;
 }
 
 void
@@ -78,6 +81,7 @@ v3dv_job_add_bo_unchecked(struct v3dv_job *job, struct v3dv_bo *bo)
    assert(bo);
    _mesa_set_add(job->bos, bo);
    job->bo_count++;
+   job->bo_handle_mask |= bo->handle_bit;
 }
 
 static void
diff --git a/src/broadcom/vulkan/v3dv_private.h b/src/broadcom/vulkan/v3dv_private.h
index 67cbc244d60..5a9f0e2dac9 100644
--- a/src/broadcom/vulkan/v3dv_private.h
+++ b/src/broadcom/vulkan/v3dv_private.h
@@ -924,6 +924,7 @@ struct v3dv_job {
     */
    struct set *bos;
    uint32_t bo_count;
+   uint64_t bo_handle_mask;
 
    struct v3dv_bo *tile_alloc;
    struct v3dv_bo *tile_state;