diff --git a/src/util/u_idalloc.c b/src/util/u_idalloc.c
index 1a0e83b6df5..4abe478a9c8 100644
--- a/src/util/u_idalloc.c
+++ b/src/util/u_idalloc.c
@@ -144,8 +144,11 @@ ret:
 void
 util_idalloc_free(struct util_idalloc *buf, unsigned id)
 {
-   assert(id / 32 < buf->num_elements);
    unsigned idx = id / 32;
+
+   if (idx >= buf->num_elements)
+       return;
+
    buf->lowest_free_idx = MIN2(idx, buf->lowest_free_idx);
    buf->data[idx] &= ~(1 << (id % 32));
 }