diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c
index 9e6aa64e40a..2c15abad19d 100644
--- a/src/asahi/compiler/agx_compile.c
+++ b/src/asahi/compiler/agx_compile.c
@@ -2270,6 +2270,7 @@ agx_emit_tex(agx_builder *b, nir_tex_instr *instr)
          texture = index;
          break;
       case nir_tex_src_sampler_handle:
+         b->shader->out->uses_sampler_heap = true;
          sampler = index;
          break;
 
diff --git a/src/asahi/compiler/agx_compile.h b/src/asahi/compiler/agx_compile.h
index 33eec97482f..5ddef140280 100644
--- a/src/asahi/compiler/agx_compile.h
+++ b/src/asahi/compiler/agx_compile.h
@@ -140,6 +140,9 @@ struct agx_shader_info {
    /* Uses txf and hence needs a txf sampler mapped */
    bool uses_txf;
 
+   /* Potentially uses the sampler heap (conservative) */
+   bool uses_sampler_heap;
+
    /* Number of texture/sampler state registers pushed by the preamble. */
    uint8_t texture_state_count, sampler_state_count;
 
diff --git a/src/asahi/vulkan/hk_cmd_buffer.c b/src/asahi/vulkan/hk_cmd_buffer.c
index ab1204bc454..a1e5aad4e71 100644
--- a/src/asahi/vulkan/hk_cmd_buffer.c
+++ b/src/asahi/vulkan/hk_cmd_buffer.c
@@ -630,6 +630,9 @@ hk_reserve_scratch(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
    uint32_t max_scratch_size =
       MAX2(s->b.info.scratch_size, s->b.info.preamble_scratch_size);
 
+   /* Not scratch but this is the most convenient place for this... */
+   cs->uses_sampler_heap |= s->b.info.uses_sampler_heap;
+
    if (max_scratch_size == 0)
       return;
 
diff --git a/src/asahi/vulkan/hk_cmd_buffer.h b/src/asahi/vulkan/hk_cmd_buffer.h
index 5c88b1bf95e..00ee252faa6 100644
--- a/src/asahi/vulkan/hk_cmd_buffer.h
+++ b/src/asahi/vulkan/hk_cmd_buffer.h
@@ -360,6 +360,12 @@ struct hk_cs {
    /* Whether there is more than just the root chunk */
    bool stream_linked;
 
+   /* Whether the sampler heap is required. Although we always must maintain the
+    * heap for correctness, it's often not necessary since we can push lots of
+    * samplers (especially for GL/DX11-era engines).
+    */
+   bool uses_sampler_heap;
+
    /* Scratch requirements */
    struct {
       union {
@@ -428,6 +434,7 @@ hk_cs_merge_cdm(struct hk_cs *a, const struct hk_cs *b)
    a->current = b->current;
    a->stream_linked = true;
 
+   a->uses_sampler_heap |= b->uses_sampler_heap;
    a->scratch.cs.main |= b->scratch.cs.main;
    a->scratch.cs.preamble |= b->scratch.cs.preamble;
 
diff --git a/src/asahi/vulkan/hk_queue.c b/src/asahi/vulkan/hk_queue.c
index 917cba119e0..d86a3b34a6f 100644
--- a/src/asahi/vulkan/hk_queue.c
+++ b/src/asahi/vulkan/hk_queue.c
@@ -86,13 +86,15 @@ asahi_fill_cdm_command(struct hk_device *dev, struct hk_cs *cs,
       .cdm_ctrl_stream_base = cs->addr,
       .cdm_ctrl_stream_end = cs->addr + len,
 
-      .sampler_heap = dev->samplers.table.bo->va->addr,
-      .sampler_count = dev->samplers.table.alloc,
-
       .ts.end.handle = cs->timestamp.end.handle,
       .ts.end.offset = cs->timestamp.end.offset_B,
    };
 
+   if (cs->uses_sampler_heap) {
+      cmd->sampler_heap = dev->samplers.table.bo->va->addr;
+      cmd->sampler_count = dev->samplers.table.alloc;
+   }
+
    if (cs->scratch.cs.main || cs->scratch.cs.preamble) {
       cmd->helper.data = dev->scratch.cs.buf->va->addr;
       cmd->helper.cfg = cs->scratch.cs.preamble ? (1 << 16) : 0;
@@ -199,8 +201,10 @@ asahi_fill_vdm_command(struct hk_device *dev, struct hk_cs *cs,
    c->isp_scissor_base = cs->uploaded_scissor;
    c->isp_dbias_base = cs->uploaded_zbias;
 
-   c->sampler_heap = dev->samplers.table.bo->va->addr;
-   c->sampler_count = dev->samplers.table.alloc;
+   if (cs->uses_sampler_heap) {
+      c->sampler_heap = dev->samplers.table.bo->va->addr;
+      c->sampler_count = dev->samplers.table.alloc;
+   }
 
    c->isp_oclqry_base = dev->occlusion_queries.bo->va->addr;