vk: Set I915_CACHING_NONE for userptr BOs when !llc

Regular objects are created I915_CACHING_CACHED on LLC platforms and
I915_CACHING_NONE on non-LLC platforms. However, userptr objects are
always created as I915_CACHING_CACHED, which on non-LLC means
snooped. That can be useful but comes with a bit of overheard.  Since
we're eplicitly clflushing and don't want the overhead we need to turn
it off.

src/vulkan/anv_allocator.c

@@ -441,6 +441,18 @@ anv_block_pool_grow(struct anv_block_pool *pool, struct anv_block_state *state)
       goto fail;
    cleanup->gem_handle = gem_handle;
 
+   /* Regular objects are created I915_CACHING_CACHED on LLC platforms and
+    * I915_CACHING_NONE on non-LLC platforms. However, userptr objects are
+    * always created as I915_CACHING_CACHED, which on non-LLC means
+    * snooped. That can be useful but comes with a bit of overheard.  Since
+    * we're eplicitly clflushing and don't want the overhead we need to turn
+    * it off. */
+   if (!pool->device->info.has_llc) {
+      anv_gem_set_caching(pool->device, gem_handle, I915_CACHING_NONE);
+      anv_gem_set_domain(pool->device, gem_handle,
+                         I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
+   }
+
    /* Now that we successfull allocated everything, we can write the new
     * values back into pool. */
    pool->map = map + center_bo_offset;

src/vulkan/anv_gem.c

@@ -137,6 +137,32 @@ anv_gem_userptr(struct anv_device *device, void *mem, size_t size)
    return userptr.handle;
 }
 
+int
+anv_gem_set_caching(struct anv_device *device, int gem_handle, uint32_t caching)
+{
+   struct drm_i915_gem_caching gem_caching;
+
+   VG_CLEAR(gem_caching);
+   gem_caching.handle = gem_handle;
+   gem_caching.caching = caching;
+
+   return anv_ioctl(device->fd, DRM_IOCTL_I915_GEM_SET_CACHING, &gem_caching);
+}
+
+int
+anv_gem_set_domain(struct anv_device *device, int gem_handle,
+                   uint32_t read_domains, uint32_t write_domain)
+{
+   struct drm_i915_gem_set_domain gem_set_domain;
+
+   VG_CLEAR(gem_set_domain);
+   gem_set_domain.handle = gem_handle;
+   gem_set_domain.read_domains = read_domains;
+   gem_set_domain.write_domain = write_domain;
+
+   return anv_ioctl(device->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &gem_set_domain);
+}
+
 /**
  * On error, \a timeout_ns holds the remaining time.
  */

src/vulkan/anv_private.h

@@ -598,6 +598,9 @@ int anv_gem_get_aperture(int fd, uint64_t *size);
 int anv_gem_handle_to_fd(struct anv_device *device, int gem_handle);
 int anv_gem_fd_to_handle(struct anv_device *device, int fd);
 int anv_gem_userptr(struct anv_device *device, void *mem, size_t size);
+int anv_gem_set_caching(struct anv_device *device, int gem_handle, uint32_t caching);
+int anv_gem_set_domain(struct anv_device *device, int gem_handle,
+                       uint32_t read_domains, uint32_t write_domain);
 
 VkResult anv_bo_init_new(struct anv_bo *bo, struct anv_device *device, uint64_t size);