diff --git a/src/freedreno/drm/freedreno_bo.c b/src/freedreno/drm/freedreno_bo.c index 9719c0674f2..ab99d317533 100644 --- a/src/freedreno/drm/freedreno_bo.c +++ b/src/freedreno/drm/freedreno_bo.c @@ -31,8 +31,6 @@ simple_mtx_t table_lock = SIMPLE_MTX_INITIALIZER; simple_mtx_t fence_lock = SIMPLE_MTX_INITIALIZER; -void bo_del(struct fd_bo *bo); -void bo_del_flush(struct fd_device *dev); /* set buffer name, and add to table, call w/ table_lock held: */ static void @@ -268,7 +266,10 @@ fd_bo_ref(struct fd_bo *bo) return bo; } -static void +static uint32_t bo_del(struct fd_bo *bo); +static void close_handles(struct fd_device *dev, uint32_t *handles, unsigned cnt); + +static uint32_t bo_del_or_recycle(struct fd_bo *bo) { struct fd_device *dev = bo->dev; @@ -277,13 +278,13 @@ bo_del_or_recycle(struct fd_bo *bo) if ((bo->bo_reuse == BO_CACHE) && (fd_bo_cache_free(&dev->bo_cache, bo) == 0)) - return; + return 0; if ((bo->bo_reuse == RING_CACHE) && (fd_bo_cache_free(&dev->ring_cache, bo) == 0)) - return; + return 0; - bo_del(bo); + return bo_del(bo); } void @@ -296,8 +297,9 @@ fd_bo_del_locked(struct fd_bo *bo) struct fd_device *dev = bo->dev; - bo_del_or_recycle(bo); - bo_del_flush(dev); + uint32_t handle = bo_del_or_recycle(bo); + if (handle) + close_handles(dev, &handle, 1); } void @@ -309,8 +311,9 @@ fd_bo_del(struct fd_bo *bo) struct fd_device *dev = bo->dev; simple_mtx_lock(&table_lock); - bo_del_or_recycle(bo); - bo_del_flush(dev); + uint32_t handle = bo_del_or_recycle(bo); + if (handle) + close_handles(dev, &handle, 1); simple_mtx_unlock(&table_lock); } @@ -321,23 +324,62 @@ fd_bo_del_array(struct fd_bo **bos, unsigned count) return; struct fd_device *dev = bos[0]->dev; + uint32_t handles[64]; + unsigned cnt = 0; simple_mtx_lock(&table_lock); for (unsigned i = 0; i < count; i++) { if (!p_atomic_dec_zero(&bos[i]->refcnt)) continue; - bo_del_or_recycle(bos[i]); + if (cnt == ARRAY_SIZE(handles)) { + close_handles(dev, handles, cnt); + cnt = 0; + } + handles[cnt] = bo_del_or_recycle(bos[i]); + if (handles[cnt]) + cnt++; } - bo_del_flush(dev); + close_handles(dev, handles, cnt); simple_mtx_unlock(&table_lock); } -/* Called under table_lock, bo_del_flush() *must* be called before - * table_lock is released (but bo_del() can be called multiple times - * before bo_del_flush(), as long as table_lock is held the entire - * time) +/** + * Special interface for fd_bo_cache to batch delete a list of handles. + * Similar to fd_bo_del_array() but bypasses the BO cache (since it is + * called from the BO cache to expire a list of BOs). */ void +fd_bo_del_list_nocache(struct list_head *list) +{ + simple_mtx_assert_locked(&table_lock); + + if (list_is_empty(list)) + return; + + struct fd_device *dev = first_bo(list)->dev; + uint32_t handles[64]; + unsigned cnt = 0; + + foreach_bo_safe (bo, list) { + assert(bo->refcnt == 0); + if (cnt == ARRAY_SIZE(handles)) { + close_handles(dev, handles, cnt); + cnt = 0; + } + handles[cnt] = bo_del(bo); + if (handles[cnt]) + cnt++; + } + + close_handles(dev, handles, cnt); +} + +/** + * The returned handle must be closed via a call to close_handles() + * + * Called under table_lock + */ +static uint32_t bo_del(struct fd_bo *bo) { struct fd_device *dev = bo->dev; @@ -364,31 +406,24 @@ bo_del(struct fd_bo *bo) bo->funcs->destroy(bo); - if (handle) { - if (dev->num_deferred_handles == ARRAY_SIZE(dev->deferred_handles)) - bo_del_flush(dev); - dev->deferred_handles[dev->num_deferred_handles++] = handle; - } + return handle; } -/* Called under table_lock */ -void -bo_del_flush(struct fd_device *dev) +static void +close_handles(struct fd_device *dev, uint32_t *handles, unsigned cnt) { - if (!dev->num_deferred_handles) + if (!cnt) return; if (dev->funcs->flush) dev->funcs->flush(dev); - for (unsigned i = 0; i < dev->num_deferred_handles; i++) { + for (unsigned i = 0; i < cnt; i++) { struct drm_gem_close req = { - .handle = dev->deferred_handles[i], + .handle = handles[i], }; drmIoctl(dev->fd, DRM_IOCTL_GEM_CLOSE, &req); } - - dev->num_deferred_handles = 0; } static void diff --git a/src/freedreno/drm/freedreno_bo_cache.c b/src/freedreno/drm/freedreno_bo_cache.c index 2977a30ce52..b74a4c53b79 100644 --- a/src/freedreno/drm/freedreno_bo_cache.c +++ b/src/freedreno/drm/freedreno_bo_cache.c @@ -35,9 +35,6 @@ } \ } while (0) -void bo_del(struct fd_bo *bo); -void bo_del_flush(struct fd_device *dev); - static void bo_remove_from_bucket(struct fd_bo_bucket *bucket, struct fd_bo *bo) { @@ -145,7 +142,6 @@ fd_bo_cache_init(struct fd_bo_cache *cache, int coarse, const char *name) void fd_bo_cache_cleanup(struct fd_bo_cache *cache, time_t time) { - struct fd_device *dev = NULL; int i, cnt = 0; simple_mtx_assert_locked(&table_lock); @@ -169,8 +165,6 @@ fd_bo_cache_cleanup(struct fd_bo_cache *cache, time_t time) if (time && ((time - bo->free_time) <= 1)) break; - dev = bo->dev; - if (cnt == 0) { BO_CACHE_LOG(cache, "cache cleanup"); dump_cache_stats(cache); @@ -186,16 +180,7 @@ fd_bo_cache_cleanup(struct fd_bo_cache *cache, time_t time) } simple_mtx_unlock(&cache->lock); - foreach_bo_safe (bo, &freelist) { - list_del(&bo->node); - bo_del(bo); - } - - /* Note: when called in bo_del_or_recycle() -> fd_bo_cache_free() path, - * the caller will handle bo_del_flush(). - */ - if (time && dev) - bo_del_flush(dev); + fd_bo_del_list_nocache(&freelist); if (cnt > 0) { BO_CACHE_LOG(cache, "cache cleaned %u BOs", cnt); @@ -259,6 +244,10 @@ fd_bo_cache_alloc(struct fd_bo_cache *cache, uint32_t *size, uint32_t flags) *size = align(*size, 4096); bucket = get_bucket(cache, *size); + struct list_head freelist; + + list_inithead(&freelist); + /* see if we can be green and recycle: */ retry: if (bucket) { @@ -270,11 +259,7 @@ retry: VG_BO_OBTAIN(bo); if (bo->funcs->madvise(bo, true) <= 0) { /* we've lost the backing pages, delete and try again: */ - struct fd_device *dev = bo->dev; - simple_mtx_lock(&table_lock); - bo_del(bo); - bo_del_flush(dev); - simple_mtx_unlock(&table_lock); + list_addtail(&bo->node, &freelist); goto retry; } p_atomic_set(&bo->refcnt, 1); @@ -285,6 +270,10 @@ retry: bucket->misses++; } + simple_mtx_lock(&table_lock); + fd_bo_del_list_nocache(&freelist); + simple_mtx_unlock(&table_lock); + BO_CACHE_LOG(cache, "miss on size=%u, flags=0x%x, bucket=%u", *size, flags, bucket ? bucket->size : 0); dump_cache_stats(cache); diff --git a/src/freedreno/drm/freedreno_drmif.h b/src/freedreno/drm/freedreno_drmif.h index 244b4314403..4abef176249 100644 --- a/src/freedreno/drm/freedreno_drmif.h +++ b/src/freedreno/drm/freedreno_drmif.h @@ -282,6 +282,7 @@ fd_bo_get_iova(struct fd_bo *bo) struct fd_bo *fd_bo_ref(struct fd_bo *bo); void fd_bo_del(struct fd_bo *bo); void fd_bo_del_array(struct fd_bo **bos, unsigned count); +void fd_bo_del_list_nocache(struct list_head *list); int fd_bo_get_name(struct fd_bo *bo, uint32_t *name); uint32_t fd_bo_handle(struct fd_bo *bo); int fd_bo_dmabuf(struct fd_bo *bo); diff --git a/src/freedreno/drm/freedreno_priv.h b/src/freedreno/drm/freedreno_priv.h index 168661f0e49..48f633c71fb 100644 --- a/src/freedreno/drm/freedreno_priv.h +++ b/src/freedreno/drm/freedreno_priv.h @@ -184,15 +184,6 @@ struct fd_device { simple_mtx_t suballoc_lock; struct util_queue submit_queue; - - /** - * GEM handles can be queued/batched for freeing in cases where many - * buffers are freed together under table_lock. This enables the - * virtio backend to batch messages to the host to avoid quickly - * depleting the virtqueue ringbuffer slots. - */ - uint32_t deferred_handles[64]; - uint32_t num_deferred_handles; }; #define foreach_submit(name, list) \