AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
6ece4f3fa0dd6d7c21243ed9710deaaee23e771a
mesa/src/intel/vulkan/anv_sparse.c
T
Paulo Zanoni a1628aba1f anv/sparse: we can support R64 and other atomics emulated formats 
We set sparseImageInt64Atomics to false on these formats, so there's
no need for the software detiling. Thus, we can not set the flag,
which will make ISL pick Tile64 for these formats, and things will
work.

Thanks to Lionel for pointing the fix here.

Testcase: dEQP-VK.api.info.image_format_properties.*d.optimal.r64_*int
Reviewed-by: Lionel Landwerlin <lionel.g.landwerlin@intel.com>
Reviewed-by: Iván Briano <ivan.briano@intel.com>
Signed-off-by: Paulo Zanoni <paulo.r.zanoni@intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/35524>
2025-08-01 14:51:10 -07:00

1677 lines
61 KiB
C
Raw Blame History

/*
* Copyright © 2022 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include <anv_private.h>
/* Sparse binding handling.
*
* There is one main structure passed around all over this file:
*
* - struct anv_sparse_binding_data: every resource (VkBuffer or VkImage) has
* a pointer to an instance of this structure. It contains the virtual
* memory address (VMA) used by the binding operations (which is different
* from the VMA used by the anv_bo it's bound to) and the VMA range size. We
* do not keep record of our our list of bindings (which ranges were bound
* to which buffers).
*/
__attribute__((format(printf, 1, 2)))
static void
sparse_debug(const char *format, ...)
{
if (!INTEL_DEBUG(DEBUG_SPARSE))
return;
va_list args;
va_start(args, format);
vfprintf(stderr, format, args);
va_end(args);
}
static void
dump_anv_vm_bind(struct anv_device *device,
const struct anv_vm_bind *bind)
{
sparse_debug("[%s] ", bind->op == ANV_VM_BIND ? " bind " : "unbind");
if (bind->bo)
sparse_debug("bo:%04u ", bind->bo->gem_handle);
else
sparse_debug("bo:---- ");
sparse_debug("address:%016"PRIx64" size:%08"PRIx64" "
"mem_offset:%08"PRIx64"\n",
bind->address, bind->size, bind->bo_offset);
}
static void
dump_anv_image(struct anv_image *i)
{
if (!INTEL_DEBUG(DEBUG_SPARSE))
return;
sparse_debug("anv_image:\n");
sparse_debug("- format: %d\n", i->vk.format);
sparse_debug("- extent: [%d, %d, %d]\n",
i->vk.extent.width, i->vk.extent.height, i->vk.extent.depth);
sparse_debug("- mip_levels: %d array_layers: %d samples: %d\n",
i->vk.mip_levels, i->vk.array_layers, i->vk.samples);
sparse_debug("- n_planes: %d\n", i->n_planes);
sparse_debug("- disjoint: %d\n", i->disjoint);
}
static void
dump_isl_surf(struct isl_surf *s)
{
if (!INTEL_DEBUG(DEBUG_SPARSE))
return;
sparse_debug("isl_surf:\n");
const char *dim_s = s->dim == ISL_SURF_DIM_1D ? "1D" :
s->dim == ISL_SURF_DIM_2D ? "2D" :
s->dim == ISL_SURF_DIM_3D ? "3D" :
"(ERROR)";
sparse_debug("- dim: %s\n", dim_s);
sparse_debug("- tiling: %d (%s)\n", s->tiling,
isl_tiling_to_name(s->tiling));
sparse_debug("- format: %s\n", isl_format_get_short_name(s->format));
sparse_debug("- image_alignment_el: [%d, %d, %d]\n",
s->image_alignment_el.w, s->image_alignment_el.h,
s->image_alignment_el.d);
sparse_debug("- logical_level0_px: [%d, %d, %d, %d]\n",
s->logical_level0_px.w,
s->logical_level0_px.h,
s->logical_level0_px.d,
s->logical_level0_px.a);
sparse_debug("- phys_level0_sa: [%d, %d, %d, %d]\n",
s->phys_level0_sa.w,
s->phys_level0_sa.h,
s->phys_level0_sa.d,
s->phys_level0_sa.a);
sparse_debug("- levels: %d samples: %d\n", s->levels, s->samples);
sparse_debug("- size_B: %"PRIu64" alignment_B: %u\n",
s->size_B, s->alignment_B);
sparse_debug("- row_pitch_B: %u\n", s->row_pitch_B);
sparse_debug("- array_pitch_el_rows: %u\n", s->array_pitch_el_rows);
const struct isl_format_layout *layout = isl_format_get_layout(s->format);
sparse_debug("- format layout:\n");
sparse_debug(" - format:%d bpb:%d bw:%d bh:%d bd:%d\n",
layout->format, layout->bpb, layout->bw, layout->bh,
layout->bd);
struct isl_tile_info tile_info;
isl_surf_get_tile_info(s, &tile_info);
sparse_debug("- tile info:\n");
sparse_debug(" - format_bpb: %d\n", tile_info.format_bpb);
sparse_debug(" - logical_extent_el: [%d, %d, %d, %d]\n",
tile_info.logical_extent_el.w,
tile_info.logical_extent_el.h,
tile_info.logical_extent_el.d,
tile_info.logical_extent_el.a);
sparse_debug(" - phys_extent_B: [%d, %d]\n",
tile_info.phys_extent_B.w,
tile_info.phys_extent_B.h);
}
static VkOffset3D
vk_offset3d_px_to_el(const VkOffset3D offset_px,
const struct isl_format_layout *layout)
{
return (VkOffset3D) {
.x = offset_px.x / layout->bw,
.y = offset_px.y / layout->bh,
.z = offset_px.z / layout->bd,
};
}
static VkOffset3D
vk_offset3d_el_to_px(const VkOffset3D offset_el,
const struct isl_format_layout *layout)
{
return (VkOffset3D) {
.x = offset_el.x * layout->bw,
.y = offset_el.y * layout->bh,
.z = offset_el.z * layout->bd,
};
}
static VkExtent3D
vk_extent3d_px_to_el(const VkExtent3D extent_px,
const struct isl_format_layout *layout)
{
return (VkExtent3D) {
.width = extent_px.width / layout->bw,
.height = extent_px.height / layout->bh,
.depth = extent_px.depth / layout->bd,
};
}
static VkExtent3D
vk_extent3d_el_to_px(const VkExtent3D extent_el,
const struct isl_format_layout *layout)
{
return (VkExtent3D) {
.width = extent_el.width * layout->bw,
.height = extent_el.height * layout->bh,
.depth = extent_el.depth * layout->bd,
};
}
static bool
isl_tiling_supports_standard_block_shapes(enum isl_tiling tiling)
{
return isl_tiling_is_64(tiling) ||
tiling == ISL_TILING_ICL_Ys ||
tiling == ISL_TILING_SKL_Ys;
}
static uint32_t
isl_calc_tile_size(struct isl_tile_info *tile_info)
{
uint32_t tile_size = tile_info->phys_extent_B.w *
tile_info->phys_extent_B.h;
assert(tile_size == 64 * 1024 || tile_size == 4096 || tile_size == 1);
return tile_size;
}
static const VkExtent3D block_shapes_2d_1sample[] = {
/* 8 bits: */ { .width = 256, .height = 256, .depth = 1 },
/* 16 bits: */ { .width = 256, .height = 128, .depth = 1 },
/* 32 bits: */ { .width = 128, .height = 128, .depth = 1 },
/* 64 bits: */ { .width = 128, .height = 64, .depth = 1 },
/* 128 bits: */ { .width = 64, .height = 64, .depth = 1 },
};
static const VkExtent3D block_shapes_3d_1sample[] = {
/* 8 bits: */ { .width = 64, .height = 32, .depth = 32 },
/* 16 bits: */ { .width = 32, .height = 32, .depth = 32 },
/* 32 bits: */ { .width = 32, .height = 32, .depth = 16 },
/* 64 bits: */ { .width = 32, .height = 16, .depth = 16 },
/* 128 bits: */ { .width = 16, .height = 16, .depth = 16 },
};
static const VkExtent3D block_shapes_2d_2samples[] = {
/* 8 bits: */ { .width = 128, .height = 256, .depth = 1 },
/* 16 bits: */ { .width = 128, .height = 128, .depth = 1 },
/* 32 bits: */ { .width = 64, .height = 128, .depth = 1 },
/* 64 bits: */ { .width = 64, .height = 64, .depth = 1 },
/* 128 bits: */ { .width = 32, .height = 64, .depth = 1 },
};
static const VkExtent3D block_shapes_2d_4samples[] = {
/* 8 bits: */ { .width = 128, .height = 128, .depth = 1 },
/* 16 bits: */ { .width = 128, .height = 64, .depth = 1 },
/* 32 bits: */ { .width = 64, .height = 64, .depth = 1 },
/* 64 bits: */ { .width = 64, .height = 32, .depth = 1 },
/* 128 bits: */ { .width = 32, .height = 32, .depth = 1 },
};
static const VkExtent3D block_shapes_2d_8samples[] = {
/* 8 bits: */ { .width = 64, .height = 128, .depth = 1 },
/* 16 bits: */ { .width = 64, .height = 64, .depth = 1 },
/* 32 bits: */ { .width = 32, .height = 64, .depth = 1 },
/* 64 bits: */ { .width = 32, .height = 32, .depth = 1 },
/* 128 bits: */ { .width = 16, .height = 32, .depth = 1 },
};
static const VkExtent3D block_shapes_2d_16samples[] = {
/* 8 bits: */ { .width = 64, .height = 64, .depth = 1 },
/* 16 bits: */ { .width = 64, .height = 32, .depth = 1 },
/* 32 bits: */ { .width = 32, .height = 32, .depth = 1 },
/* 64 bits: */ { .width = 32, .height = 16, .depth = 1 },
/* 128 bits: */ { .width = 16, .height = 16, .depth = 1 },
};
static VkExtent3D
anv_sparse_get_standard_image_block_shape(enum isl_format format,
VkImageType image_type,
VkSampleCountFlagBits samples,
uint16_t texel_size)
{
const struct isl_format_layout *layout = isl_format_get_layout(format);
VkExtent3D block_shape = { .width = 0, .height = 0, .depth = 0 };
int table_idx = ffs(texel_size) - 4;
switch (samples) {
case VK_SAMPLE_COUNT_1_BIT:
switch (image_type) {
case VK_IMAGE_TYPE_1D:
/* 1D images don't have a standard block format. */
assert(false);
break;
case VK_IMAGE_TYPE_2D:
block_shape = block_shapes_2d_1sample[table_idx];
break;
case VK_IMAGE_TYPE_3D:
block_shape = block_shapes_3d_1sample[table_idx];
break;
default:
fprintf(stderr, "unexpected image_type %d\n", image_type);
assert(false);
}
break;
case VK_SAMPLE_COUNT_2_BIT:
block_shape = block_shapes_2d_2samples[table_idx];
break;
case VK_SAMPLE_COUNT_4_BIT:
block_shape = block_shapes_2d_4samples[table_idx];
break;
case VK_SAMPLE_COUNT_8_BIT:
block_shape = block_shapes_2d_8samples[table_idx];
break;
case VK_SAMPLE_COUNT_16_BIT:
block_shape = block_shapes_2d_16samples[table_idx];
break;
default:
fprintf(stderr, "unexpected sample count: %d\n", samples);
assert(false);
}
return vk_extent3d_el_to_px(block_shape, layout);
}
/* Adds "bind_op" to the list in "submit", while also trying to check if we
* can just extend the last operation instead.
*/
static VkResult
anv_sparse_submission_add(struct anv_device *device,
struct anv_sparse_submission *submit,
struct anv_vm_bind *bind_op)
{
struct anv_vm_bind *prev_bind = submit->binds_len == 0 ? NULL :
&submit->binds[submit->binds_len - 1];
if (prev_bind &&
bind_op->op == prev_bind->op &&
bind_op->bo == prev_bind->bo &&
bind_op->address == prev_bind->address + prev_bind->size &&
(bind_op->bo_offset == prev_bind->bo_offset + prev_bind->size ||
prev_bind->bo == NULL)) {
prev_bind->size += bind_op->size;
return VK_SUCCESS;
}
if (submit->binds_len < submit->binds_capacity) {
submit->binds[submit->binds_len++] = *bind_op;
return VK_SUCCESS;
}
int new_capacity = MAX2(32, submit->binds_capacity * 2);
struct anv_vm_bind *new_binds =
vk_realloc(&device->vk.alloc, submit->binds,
new_capacity * sizeof(*new_binds), 8,
VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
if (!new_binds)
return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
new_binds[submit->binds_len] = *bind_op;
submit->binds = new_binds;
submit->binds_len++;
submit->binds_capacity = new_capacity;
return VK_SUCCESS;
}
/* We really want to try to have all the page tables on as few BOs as possible
* to benefit from cache locality and to keep the i915.ko relocation lists
* small. On the other hand, we don't want to waste memory on unused space.
*/
#define ANV_TRTT_PAGE_TABLE_BO_SIZE (2 * 1024 * 1024)
static VkResult
trtt_make_page_table_bo(struct anv_device *device, struct anv_bo **bo)
{
VkResult result;
struct anv_trtt *trtt = &device->trtt;
result = anv_device_alloc_bo(device, "trtt-page-table",
ANV_TRTT_PAGE_TABLE_BO_SIZE,
ANV_BO_ALLOC_INTERNAL,
0 /* explicit_address */, bo);
ANV_DMR_BO_ALLOC(&device->vk.base, *bo, result);
if (result != VK_SUCCESS)
return result;
if (trtt->num_page_table_bos < trtt->page_table_bos_capacity) {
trtt->page_table_bos[trtt->num_page_table_bos++] = *bo;
} else {
int new_capacity = MAX2(8, trtt->page_table_bos_capacity * 2);
struct anv_bo **new_page_table_bos =
vk_realloc(&device->vk.alloc, trtt->page_table_bos,
new_capacity * sizeof(*trtt->page_table_bos), 8,
VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
if (!new_page_table_bos) {
ANV_DMR_BO_FREE(&device->vk.base, *bo);
anv_device_release_bo(device, *bo);
return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
}
new_page_table_bos[trtt->num_page_table_bos] = *bo;
trtt->page_table_bos = new_page_table_bos;
trtt->page_table_bos_capacity = new_capacity;
trtt->num_page_table_bos++;
}
trtt->cur_page_table_bo = *bo;
trtt->next_page_table_bo_offset = 0;
sparse_debug("new number of page table BOs: %d\n",
trtt->num_page_table_bos);
return VK_SUCCESS;
}
static VkResult
trtt_get_page_table_bo(struct anv_device *device, struct anv_bo **bo,
uint64_t *bo_addr)
{
struct anv_trtt *trtt = &device->trtt;
VkResult result;
if (!trtt->cur_page_table_bo) {
result = trtt_make_page_table_bo(device, bo);
if (result != VK_SUCCESS)
return result;
}
*bo = trtt->cur_page_table_bo;
*bo_addr = trtt->cur_page_table_bo->offset +
trtt->next_page_table_bo_offset;
trtt->next_page_table_bo_offset += 4096;
if (trtt->next_page_table_bo_offset >= ANV_TRTT_PAGE_TABLE_BO_SIZE)
trtt->cur_page_table_bo = NULL;
return VK_SUCCESS;
}
/* For L3 and L2 pages, null and invalid entries are indicated by bits 1 and 0
* respectively. For L1 entries, the hardware compares the addresses against
* what we program to the GFX_TRTT_NULL and GFX_TRTT_INVAL registers.
*/
#define ANV_TRTT_L3L2_NULL_ENTRY (1 << 1)
#define ANV_TRTT_L3L2_INVALID_ENTRY (1 << 0)
static void
anv_trtt_bind_list_add_entry(struct util_dynarray *binds, uint64_t pte_addr,
uint64_t entry_addr)
{
struct anv_trtt_bind b = {
.pte_addr = pte_addr,
.entry_addr = entry_addr,
};
util_dynarray_append(binds, struct anv_trtt_bind, b);
}
/* Adds elements to the anv_trtt_bind structs passed. This doesn't write the
* entries to the HW yet.
*/
static VkResult
anv_trtt_bind_add(struct anv_device *device,
uint64_t trtt_addr, uint64_t dest_addr,
struct util_dynarray *l3l2_binds,
struct util_dynarray *l1_binds)
{
VkResult result = VK_SUCCESS;
struct anv_trtt *trtt = &device->trtt;
bool is_null_bind = dest_addr == ANV_TRTT_L1_NULL_TILE_VAL;
int l3_index = (trtt_addr >> 35) & 0x1FF;
int l2_index = (trtt_addr >> 26) & 0x1FF;
int l1_index = (trtt_addr >> 16) & 0x3FF;
uint64_t l2_addr = trtt->l3_mirror[l3_index];
if (l2_addr == ANV_TRTT_L3L2_NULL_ENTRY && is_null_bind) {
return VK_SUCCESS;
} else if (l2_addr == 0 || l2_addr == ANV_TRTT_L3L2_NULL_ENTRY) {
if (is_null_bind) {
trtt->l3_mirror[l3_index] = ANV_TRTT_L3L2_NULL_ENTRY;
anv_trtt_bind_list_add_entry(l3l2_binds, trtt->l3_addr +
l3_index * sizeof(uint64_t),
ANV_TRTT_L3L2_NULL_ENTRY);
return VK_SUCCESS;
}
struct anv_bo *l2_bo;
result = trtt_get_page_table_bo(device, &l2_bo, &l2_addr);
if (result != VK_SUCCESS)
return result;
trtt->l3_mirror[l3_index] = l2_addr;
anv_trtt_bind_list_add_entry(l3l2_binds, trtt->l3_addr +
l3_index * sizeof(uint64_t), l2_addr);
/* We have just created a new L2 table. Other resources may already have
* been pointing to this L2 table relying on the fact that it was marked
* as NULL, so now we need to mark every one of its entries as NULL in
* order to preserve behavior for those entries.
*/
if (!util_dynarray_ensure_cap(l3l2_binds,
l3l2_binds->capacity + 512 * sizeof(struct anv_trtt_bind)))
return VK_ERROR_OUT_OF_HOST_MEMORY;
for (int i = 0; i < 512; i++) {
if (i != l2_index) {
trtt->l2_mirror[l3_index * 512 + i] = ANV_TRTT_L3L2_NULL_ENTRY;
anv_trtt_bind_list_add_entry(l3l2_binds,
l2_addr + i * sizeof(uint64_t),
ANV_TRTT_L3L2_NULL_ENTRY);
}
}
}
assert(l2_addr != 0 && l2_addr != ANV_TRTT_L3L2_NULL_ENTRY);
/* The first page in the l2_mirror corresponds to l3_index=0 and so on. */
uint64_t l1_addr = trtt->l2_mirror[l3_index * 512 + l2_index];
if (l1_addr == ANV_TRTT_L3L2_NULL_ENTRY && is_null_bind) {
return VK_SUCCESS;
} else if (l1_addr == 0 || l1_addr == ANV_TRTT_L3L2_NULL_ENTRY) {
if (is_null_bind) {
trtt->l2_mirror[l3_index * 512 + l2_index] =
ANV_TRTT_L3L2_NULL_ENTRY;
anv_trtt_bind_list_add_entry(l3l2_binds,
l2_addr + l2_index * sizeof(uint64_t),
ANV_TRTT_L3L2_NULL_ENTRY);
return VK_SUCCESS;
}
struct anv_bo *l1_bo;
result = trtt_get_page_table_bo(device, &l1_bo, &l1_addr);
if (result != VK_SUCCESS)
return result;
trtt->l2_mirror[l3_index * 512 + l2_index] = l1_addr;
anv_trtt_bind_list_add_entry(l3l2_binds,
l2_addr + l2_index * sizeof(uint64_t),
l1_addr);
}
assert(l1_addr != 0 && l1_addr != ANV_TRTT_L3L2_NULL_ENTRY);
anv_trtt_bind_list_add_entry(l1_binds,
l1_addr + l1_index * sizeof(uint32_t),
dest_addr);
return VK_SUCCESS;
}
VkResult
anv_sparse_trtt_garbage_collect_batches(struct anv_device *device,
bool wait_completion)
{
struct anv_trtt *trtt = &device->trtt;
uint64_t last_value;
if (!wait_completion) {
VkResult result =
vk_sync_get_value(&device->vk, trtt->timeline, &last_value);
if (result != VK_SUCCESS)
return result;
/* Valgrind doesn't know that drmSyncobjQuery writes to 'last_value' on
* success.
*/
VG(VALGRIND_MAKE_MEM_DEFINED(&last_value, sizeof(last_value)));
} else {
last_value = trtt->timeline_val;
}
list_for_each_entry_safe(struct anv_trtt_submission, submit,
&trtt->in_flight_batches, link) {
if (submit->base.signal.signal_value <= last_value) {
list_del(&submit->link);
anv_async_submit_fini(&submit->base);
vk_free(&device->vk.alloc, submit);
continue;
}
if (!wait_completion)
break;
VkResult result = vk_sync_wait(
&device->vk,
submit->base.signal.sync,
submit->base.signal.signal_value,
VK_SYNC_WAIT_COMPLETE,
os_time_get_absolute_timeout(OS_TIMEOUT_INFINITE));
if (result == VK_SUCCESS) {
list_del(&submit->link);
anv_async_submit_fini(&submit->base);
vk_free(&device->vk.alloc, submit);
continue;
}
/* If the wait failed but the caller wanted completion, return the
* error.
*/
return result;
}
return VK_SUCCESS;
}
/* On success, this function initializes 'submit' and submits it, but doesn't
* wait or free it. This allows the caller to submit multiple queues at the
* same time before starting to wait for anything to complete.
* If the function fails, the caller doesn't need to wait or fini anything,
* just whatever other submissions may have succeeded in the past.
*/
static VkResult
anv_trtt_first_bind_init_queue(struct anv_queue *queue,
struct anv_async_submit *submit,
bool init_l3_table, struct anv_bo *l3_bo)
{
struct anv_device *device = queue->device;
struct anv_trtt *trtt = &device->trtt;
VkResult result;
result = anv_async_submit_init(submit, queue, &device->batch_bo_pool,
false, true);
if (result != VK_SUCCESS)
return result;
result = anv_genX(device->info, init_trtt_context_state)(submit);
if (result != VK_SUCCESS)
goto out_submit_fini;
/* We only need to do this once, so pick the first queue. */
if (init_l3_table) {
struct anv_trtt_bind l3l2_binds_data[512];
struct util_dynarray l3l2_binds;
util_dynarray_init_from_stack(&l3l2_binds, l3l2_binds_data,
sizeof(l3l2_binds_data));
for (int entry = 0; entry < 512; entry++) {
trtt->l3_mirror[entry] = ANV_TRTT_L3L2_NULL_ENTRY;
anv_trtt_bind_list_add_entry(&l3l2_binds,
trtt->l3_addr +
entry * sizeof(uint64_t),
ANV_TRTT_L3L2_NULL_ENTRY);
}
anv_genX(device->info, write_trtt_entries)(
submit, l3l2_binds.data,
util_dynarray_num_elements(&l3l2_binds, struct anv_trtt_bind),
NULL, 0);
result = anv_reloc_list_add_bo(&submit->relocs, l3_bo);
if (result != VK_SUCCESS)
goto out_submit_fini;
}
anv_genX(device->info, async_submit_end)(submit);
result = device->kmd_backend->queue_exec_async(submit, 0, NULL, 1,
&submit->signal);
if (result != VK_SUCCESS)
goto out_submit_fini;
/* If we succeed, it's our caller that's going to call
* anv_async_submit_fini(). We do this so we can start waiting for the
* submissions only after all the submissions are submitted.
*/
return VK_SUCCESS;
out_submit_fini:
/* If we fail, undo everything this function has done so the caller has
* nothing to free.
*/
anv_async_submit_fini(submit);
return result;
}
/* There are lots of applications that request for sparse binding to be
* enabled but never use it, so we choose to delay the initialization of TR-TT
* until the moment we know we're going to need it.
*/
static VkResult
anv_trtt_first_bind_init(struct anv_device *device)
{
struct anv_trtt *trtt = &device->trtt;
VkResult result = VK_SUCCESS;
/* TR-TT submission needs a queue even when the API entry point doesn't
* provide one, such as resource creation. We pick this queue from the user
* created queues at init_device_state() under anv_CreateDevice.
*
* It is technically possible for the user to create sparse resources even
* when they don't have a sparse queue: they won't be able to bind the
* resource but they should still be able to use the resource and rely on
* its unbound behavior. We haven't spotted any real world application or
* even test suite that exercises this behavior.
*
* For now let's just print an error message and return, which means that
* resource creation will succeed but the behavior will be undefined if the
* resource is used, which goes against our claim that we support the
* sparseResidencyNonResidentStrict property.
*
* TODO: be fully spec-compliant here. Maybe have a device-internal queue
* independent of the application's queues for the TR-TT operations.
*/
if (unlikely(!trtt->queue)) {
static bool warned = false;
if (unlikely(!warned)) {
fprintf(stderr, "FIXME: application has created a sparse resource "
"but no queues capable of binding sparse resources were "
"created. Using these resources will result in undefined "
"behavior.\n");
warned = true;
}
return VK_SUCCESS;
}
simple_mtx_lock(&trtt->mutex);
/* This means we have already initialized the first bind. */
if (likely(trtt->l3_addr)) {
simple_mtx_unlock(&trtt->mutex);
return VK_SUCCESS;
}
struct anv_async_submit submits[device->queue_count];
struct anv_bo *l3_bo;
result = trtt_get_page_table_bo(device, &l3_bo, &trtt->l3_addr);
if (result != VK_SUCCESS)
goto out;
trtt->l3_mirror = vk_zalloc(&device->vk.alloc, 4096, 8,
VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
if (!trtt->l3_mirror) {
result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
goto out;
}
/* L3 has 512 entries, so we can have up to 512 L2 tables. */
trtt->l2_mirror = vk_zalloc(&device->vk.alloc, 512 * 4096, 8,
VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
if (!trtt->l2_mirror) {
vk_free(&device->vk.alloc, trtt->l3_mirror);
result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
goto out;
}
int n_submits;
for (n_submits = 0; n_submits < device->queue_count; n_submits++) {
result = anv_trtt_first_bind_init_queue(&device->queues[n_submits],
&submits[n_submits],
n_submits == 0, l3_bo);
if (result != VK_SUCCESS)
break;
}
for (uint32_t i = 0; i < n_submits; i++) {
anv_async_submit_wait(&submits[i]);
anv_async_submit_fini(&submits[i]);
}
out:
if (result != VK_SUCCESS)
trtt->l3_addr = 0;
simple_mtx_unlock(&trtt->mutex);
return result;
}
static VkResult
anv_sparse_bind_trtt(struct anv_device *device,
struct anv_sparse_submission *sparse_submit)
{
struct anv_trtt *trtt = &device->trtt;
VkResult result;
/* See the same check at anv_trtt_first_bind_init(). */
if (unlikely(!trtt->queue))
return VK_SUCCESS;
if (!sparse_submit->queue)
sparse_submit->queue = trtt->queue;
struct anv_trtt_submission *submit =
vk_zalloc(&device->vk.alloc, sizeof(*submit), 8,
VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
if (submit == NULL)
return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
result = anv_async_submit_init(&submit->base, sparse_submit->queue,
&device->batch_bo_pool,
false, false);
if (result != VK_SUCCESS)
goto out_async;
simple_mtx_lock(&trtt->mutex);
/* Do this so we can avoid reallocs later. */
int l1_binds_capacity = 0;
for (int b = 0; b < sparse_submit->binds_len; b++) {
assert(sparse_submit->binds[b].size % (64 * 1024) == 0);
int pages = sparse_submit->binds[b].size / (64 * 1024);
l1_binds_capacity += pages;
}
/* Turn a series of virtual address maps, into a list of L3/L2/L1 TRTT page
* table updates.
*/
/* These are arrays of struct anv_trtt_bind. */
struct util_dynarray l3l2_binds = {};
struct util_dynarray l1_binds;
if (l1_binds_capacity <= 32) {
size_t alloc_size = l1_binds_capacity * sizeof(struct anv_trtt_bind);
struct anv_trtt_bind *ptr = alloca(alloc_size);
util_dynarray_init_from_stack(&l1_binds, ptr, alloc_size);
} else {
util_dynarray_init(&l1_binds, NULL);
if (!util_dynarray_ensure_cap(&l1_binds,
l1_binds_capacity * sizeof(struct anv_trtt_bind)))
goto out_dynarrays;
}
for (int b = 0; b < sparse_submit->binds_len; b++) {
struct anv_vm_bind *vm_bind = &sparse_submit->binds[b];
for (uint64_t i = 0; i < vm_bind->size; i += 64 * 1024) {
uint64_t trtt_addr = vm_bind->address + i;
uint64_t dest_addr =
(vm_bind->op == ANV_VM_BIND && vm_bind->bo) ?
vm_bind->bo->offset + vm_bind->bo_offset + i :
ANV_TRTT_L1_NULL_TILE_VAL;
result = anv_trtt_bind_add(device, trtt_addr, dest_addr,
&l3l2_binds, &l1_binds);
if (result != VK_SUCCESS)
goto out_dynarrays;
}
}
/* Convert the L3/L2/L1 TRTT page table updates in anv_trtt_bind elements
* into MI commands.
*/
uint32_t n_l3l2_binds =
util_dynarray_num_elements(&l3l2_binds, struct anv_trtt_bind);
uint32_t n_l1_binds =
util_dynarray_num_elements(&l1_binds, struct anv_trtt_bind);
sparse_debug("trtt_binds: num_vm_binds:%02d l3l2:%04d l1:%04d\n",
sparse_submit->binds_len, n_l3l2_binds, n_l1_binds);
/* This is not an error, the application is simply trying to reset state
* that was already there. */
if (n_l3l2_binds == 0 && n_l1_binds == 0 &&
sparse_submit->wait_count == 0 && sparse_submit->signal_count == 0)
goto out_dynarrays;
anv_genX(device->info, write_trtt_entries)(&submit->base,
l3l2_binds.data, n_l3l2_binds,
l1_binds.data, n_l1_binds);
util_dynarray_fini(&l1_binds);
util_dynarray_fini(&l3l2_binds);
anv_genX(device->info, async_submit_end)(&submit->base);
if (submit->base.batch.status != VK_SUCCESS) {
result = submit->base.batch.status;
goto out_add_bind;
}
/* Add all the BOs backing TRTT page tables to the reloc list. */
if (device->physical->uses_relocs) {
for (int i = 0; i < trtt->num_page_table_bos; i++) {
result = anv_reloc_list_add_bo(&submit->base.relocs,
trtt->page_table_bos[i]);
if (result != VK_SUCCESS)
goto out_add_bind;
}
}
anv_sparse_trtt_garbage_collect_batches(device, false);
submit->base.signal = (struct vk_sync_signal) {
.sync = trtt->timeline,
.signal_value = ++trtt->timeline_val,
};
result =
device->kmd_backend->queue_exec_async(&submit->base,
sparse_submit->wait_count,
sparse_submit->waits,
sparse_submit->signal_count,
sparse_submit->signals);
if (result != VK_SUCCESS) {
trtt->timeline_val--;
goto out_add_bind;
}
list_addtail(&submit->link, &trtt->in_flight_batches);
simple_mtx_unlock(&trtt->mutex);
ANV_RMV(vm_binds, device, sparse_submit->binds, sparse_submit->binds_len);
return VK_SUCCESS;
out_dynarrays:
util_dynarray_fini(&l1_binds);
util_dynarray_fini(&l3l2_binds);
out_add_bind:
simple_mtx_unlock(&trtt->mutex);
anv_async_submit_fini(&submit->base);
out_async:
vk_free(&device->vk.alloc, submit);
return result;
}
static VkResult
anv_sparse_bind_vm_bind(struct anv_device *device,
struct anv_sparse_submission *submit)
{
struct anv_queue *queue = submit->queue;
VkResult result = device->kmd_backend->vm_bind(device, submit,
ANV_VM_BIND_FLAG_NONE);
if (!queue) {
assert(submit->wait_count == 0 && submit->signal_count == 0 &&
submit->binds_len == 1);
return result;
}
if (result == VK_ERROR_OUT_OF_HOST_MEMORY) {
/* If we get this, the system is under memory pressure. First we
* manually wait for all our dependency syncobjs hoping that some memory
* will be released while we wait, then we try to issue each bind
* operation in a single ioctl as it requires less Kernel memory and so
* we may be able to move things forward, although slowly, while also
* waiting for each operation to complete before issuing the next.
* Performance isn't a concern at this point: we're just trying to move
* progress forward without crashing until whatever is eating too much
* memory goes away.
*/
result = vk_sync_wait_many(&device->vk, submit->wait_count,
submit->waits, VK_SYNC_WAIT_COMPLETE,
INT64_MAX);
if (result != VK_SUCCESS)
return vk_queue_set_lost(&queue->vk, "vk_sync_wait_many failed");
struct vk_sync *sync;
result = vk_sync_create(&device->vk,
&device->physical->sync_syncobj_type,
VK_SYNC_IS_TIMELINE, 0 /* initial_value */,
&sync);
if (result != VK_SUCCESS)
return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
for (int b = 0; b < submit->binds_len; b++) {
struct vk_sync_signal sync_signal = {
.sync = sync,
.signal_value = b + 1,
};
struct anv_sparse_submission s = {
.queue = submit->queue,
.binds = &submit->binds[b],
.binds_len = 1,
.binds_capacity = 1,
.wait_count = 0,
.signal_count = 1,
.waits = NULL,
.signals = &sync_signal,
};
result = device->kmd_backend->vm_bind(device, &s,
ANV_VM_BIND_FLAG_NONE);
if (result != VK_SUCCESS) {
vk_sync_destroy(&device->vk, sync);
return vk_error(device, result); /* Well, at least we tried... */
}
result = vk_sync_wait(&device->vk, sync, sync_signal.signal_value,
VK_SYNC_WAIT_COMPLETE, UINT64_MAX);
if (result != VK_SUCCESS) {
vk_sync_destroy(&device->vk, sync);
return vk_queue_set_lost(&queue->vk, "vk_sync_wait failed");
}
}
vk_sync_destroy(&device->vk, sync);
for (uint32_t i = 0; i < submit->signal_count; i++) {
struct vk_sync_signal *s = &submit->signals[i];
result = vk_sync_signal(&device->vk, s->sync, s->signal_value);
if (result != VK_SUCCESS)
return vk_queue_set_lost(&queue->vk, "vk_sync_signal failed");
}
}
return VK_SUCCESS;
}
VkResult
anv_sparse_bind(struct anv_device *device,
struct anv_sparse_submission *submit)
{
if (INTEL_DEBUG(DEBUG_SPARSE)) {
for (int b = 0; b < submit->binds_len; b++)
dump_anv_vm_bind(device, &submit->binds[b]);
}
return device->physical->sparse_type == ANV_SPARSE_TYPE_TRTT ?
anv_sparse_bind_trtt(device, submit) :
anv_sparse_bind_vm_bind(device, submit);
}
VkResult
anv_init_sparse_bindings(struct anv_device *device,
uint64_t size_,
struct anv_sparse_binding_data *sparse,
enum anv_bo_alloc_flags alloc_flags,
uint64_t client_address,
struct anv_address *out_address)
{
VkResult result;
uint64_t size = align64(size_, ANV_SPARSE_BLOCK_SIZE);
if (device->physical->sparse_type == ANV_SPARSE_TYPE_TRTT)
alloc_flags |= ANV_BO_ALLOC_TRTT;
sparse->address = anv_vma_alloc(device, size, ANV_SPARSE_BLOCK_SIZE,
alloc_flags,
intel_48b_address(client_address),
&sparse->vma_heap);
sparse->size = size;
out_address->bo = NULL;
out_address->offset = sparse->address;
if (device->physical->sparse_type == ANV_SPARSE_TYPE_TRTT) {
result = anv_trtt_first_bind_init(device);
if (result != VK_SUCCESS)
goto out_vma_free;
} else {
struct anv_vm_bind bind = {
.bo = NULL, /* That's a NULL binding. */
.address = sparse->address,
.bo_offset = 0,
.size = size,
.op = ANV_VM_BIND,
};
struct anv_sparse_submission submit = {
.queue = NULL,
.binds = &bind,
.binds_len = 1,
.binds_capacity = 1,
.wait_count = 0,
.signal_count = 0,
};
result = anv_sparse_bind(device, &submit);
if (result != VK_SUCCESS)
goto out_vma_free;
}
p_atomic_inc(&device->num_sparse_resources);
return VK_SUCCESS;
out_vma_free:
anv_vma_free(device, sparse->vma_heap, sparse->address, sparse->size);
return result;
}
void
anv_free_sparse_bindings(struct anv_device *device,
struct anv_sparse_binding_data *sparse)
{
if (!sparse->address)
return;
sparse_debug("%s: address:0x%016"PRIx64" size:0x%08"PRIx64"\n",
__func__, sparse->address, sparse->size);
p_atomic_dec(&device->num_sparse_resources);
struct anv_vm_bind unbind = {
.bo = 0,
.address = sparse->address,
.bo_offset = 0,
.size = sparse->size,
.op = ANV_VM_UNBIND,
};
struct anv_sparse_submission submit = {
.queue = NULL,
.binds = &unbind,
.binds_len = 1,
.binds_capacity = 1,
.wait_count = 0,
.signal_count = 0,
};
VkResult res = anv_sparse_bind(device, &submit);
/* Our callers don't have a way to signal failure to the upper layers, so
* just keep the vma if we fail to unbind it. Still, let's have an
* assertion because this really shouldn't be happening.
*/
assert(res == VK_SUCCESS);
if (res != VK_SUCCESS)
return;
anv_vma_free(device, sparse->vma_heap, sparse->address, sparse->size);
}
static VkExtent3D
anv_sparse_calc_block_shape(struct anv_physical_device *pdevice,
struct isl_surf *surf,
const struct isl_tile_info *tile_info)
{
const struct isl_format_layout *layout =
isl_format_get_layout(surf->format);
VkExtent3D block_shape_el = {
.width = tile_info->logical_extent_el.width,
.height = tile_info->logical_extent_el.height,
.depth = tile_info->logical_extent_el.depth,
};
VkExtent3D block_shape_px = vk_extent3d_el_to_px(block_shape_el, layout);
assert(surf->tiling != ISL_TILING_LINEAR);
return block_shape_px;
}
static bool
is_xe2_non_standard_msaa_block_shape(struct anv_physical_device *pdevice,
const VkSampleCountFlagBits samples,
const int bpb)
{
if (pdevice->info.ver < 20)
return false;
switch (samples) {
case VK_SAMPLE_COUNT_2_BIT:
if (bpb == 128)
return true;
break;
case VK_SAMPLE_COUNT_8_BIT:
if (bpb == 8 || bpb == 32)
return true;
break;
case VK_SAMPLE_COUNT_16_BIT:
if (bpb == 64)
return true;
break;
default:
break;
}
return false;
}
VkSparseImageFormatProperties
anv_sparse_calc_image_format_properties(struct anv_physical_device *pdevice,
VkImageAspectFlags aspect,
VkImageType vk_image_type,
VkSampleCountFlagBits vk_samples,
struct isl_surf *surf)
{
const struct isl_format_layout *isl_layout =
isl_format_get_layout(surf->format);
struct isl_tile_info tile_info;
isl_surf_get_tile_info(surf, &tile_info);
const int bpb = isl_layout->bpb;
assert(bpb == 8 || bpb == 16 || bpb == 32 || bpb == 64 ||bpb == 128);
VkExtent3D granularity = anv_sparse_calc_block_shape(pdevice, surf,
&tile_info);
/* Block shape related flags: every case that passes through this function
* should be marked as being one and only one of the three cases below.
* Anything that is true for more than one case or is false for all is a
* bug in the driver and should be analyzed.
*
* - shape_is_standard: matches the Vulkan spec
* - shape_not_standard_fine: doesn't match the Vulkan spec, but is not an
* issue since it doesn't need to match it
* - shape_not_standard_issue: does not match the Vulkan spec, but we
* report as standard: see the comments for each case
*/
bool shape_is_standard = false;
bool shape_not_standard_fine = false;
bool shape_not_standard_issue = false;
/* We shouldn't be able to reach this function with a 1D image. */
assert(vk_image_type != VK_IMAGE_TYPE_1D);
VkExtent3D std_shape =
anv_sparse_get_standard_image_block_shape(surf->format,
vk_image_type, vk_samples,
bpb);
/* YUV formats don't work with Tile64, which is required if we want to
* claim standard block shapes. The spec requires us to support all
* non-compressed color formats that non-sparse supports, so we can't just
* say YUV formats are not supported by Sparse. So we end supporting this
* format and anv_sparse_calc_miptail_properties() will say that everything
* is part of the miptail.
*
* For more details on the hardware restriction, please check
* isl_gfx125_filter_tiling().
*/
if (pdevice->info.verx10 >= 125 && isl_format_is_yuv(surf->format))
shape_not_standard_issue = true;
/* The standard block shapes (and by extension, the tiling formats they
* require) are simply incompatible with getting a 2D view of a 3D image.
*/
if (surf->usage & ISL_SURF_USAGE_2D_3D_COMPATIBLE_BIT)
shape_not_standard_issue = true;
/* ISL_TILING_64_XE2_BIT's block shapes are not always Vulkan's standard
* block shapes: sometimes you get, for example, 64x32x1 instead of
* 32x64x1. This is not a problem since we properly advertise
* sparseResidencyStandard2DMultisampleBlockShape to be false.
*/
if (is_xe2_non_standard_msaa_block_shape(pdevice, vk_samples, bpb))
shape_not_standard_fine = true;
shape_is_standard = granularity.width == std_shape.width &&
granularity.height == std_shape.height &&
granularity.depth == std_shape.depth;
/* TODO: dEQP seems to care about the block shapes being standard even for
* the cases where shape_not_standard_issue is true. Luckily as of today
* all of those cases are NotSupported but sooner or later we may end up
* getting a failure.
* Notice that in practice we report these cases as having the mip tail
* starting on mip level 0, so the reported block shapes are irrelevant
* since non-opaque binds are not supported. Still, dEQP seems to care.
*/
assert(shape_is_standard + shape_not_standard_fine +
shape_not_standard_issue == 1);
bool wrong_block_size = isl_calc_tile_size(&tile_info) !=
ANV_SPARSE_BLOCK_SIZE;
return (VkSparseImageFormatProperties) {
.aspectMask = aspect,
.imageGranularity = granularity,
.flags = ((shape_is_standard || shape_not_standard_issue) ? 0 :
VK_SPARSE_IMAGE_FORMAT_NONSTANDARD_BLOCK_SIZE_BIT) |
(wrong_block_size ? VK_SPARSE_IMAGE_FORMAT_SINGLE_MIPTAIL_BIT :
0),
};
}
/* The miptail is supposed to be this region where the tiniest mip levels
* are squished together in one single page, which should save us some memory.
* It's a hardware feature which our hardware supports on certain tiling
* formats - the ones we always want to use for sparse resources.
*
* For sparse, the main feature of the miptail is that it only supports opaque
* binds, so you either bind the whole miptail or you bind nothing at all,
* there are no subresources inside it to separately bind. While the idea is
* that the miptail as reported by sparse should match what our hardware does,
* in practice we can say in our sparse functions that certain mip levels are
* part of the miptail while from the point of view of our hardwared they
* aren't.
*
* If we detect we're using the sparse-friendly tiling formats and ISL
* supports miptails for them, we can just trust the miptail level set by ISL
* and things can proceed as The Spec intended.
*
* However, if that's not the case, we have to go on a best-effort policy. We
* could simply declare that every mip level is part of the miptail and be
* done, but since that kinda defeats the purpose of Sparse we try to find
* what level we really should be reporting as the first miptail level based
* on the alignments of the surface subresources.
*/
void
anv_sparse_calc_miptail_properties(struct anv_device *device,
struct anv_image *image,
VkImageAspectFlags vk_aspect,
uint32_t *imageMipTailFirstLod,
VkDeviceSize *imageMipTailSize,
VkDeviceSize *imageMipTailOffset,
VkDeviceSize *imageMipTailStride)
{
const uint32_t plane = anv_image_aspect_to_plane(image, vk_aspect);
struct isl_surf *surf = &image->planes[plane].primary_surface.isl;
uint64_t binding_plane_offset =
image->planes[plane].primary_surface.memory_range.offset;
struct isl_tile_info tile_info;
isl_surf_get_tile_info(surf, &tile_info);
uint64_t layer1_offset;
uint32_t x_off, y_off;
/* Treat the whole thing as a single miptail. We should have already
* reported this image as VK_SPARSE_IMAGE_FORMAT_SINGLE_MIPTAIL_BIT.
*
* In theory we could try to make ISL massage the alignments so that we
* could at least claim mip level 0 to be not part of the miptail, but
* that could end up wasting a lot of memory, so it's better to do
* nothing and focus our efforts into making things use the appropriate
* tiling formats that give us the standard block shapes.
*/
if (isl_calc_tile_size(&tile_info) != ANV_SPARSE_BLOCK_SIZE)
goto out_everything_is_miptail;
assert(surf->tiling != ISL_TILING_LINEAR);
if (image->vk.array_layers == 1) {
layer1_offset = surf->size_B;
} else {
isl_surf_get_image_offset_B_tile_sa(surf, 0, 1, 0, &layer1_offset,
&x_off, &y_off);
if (x_off || y_off)
goto out_everything_is_miptail;
}
assert(layer1_offset % ANV_SPARSE_BLOCK_SIZE == 0);
/* We could try to do better here, but there's not really any point since
* we should be supporting the appropriate tiling formats everywhere.
*/
if (!isl_tiling_supports_standard_block_shapes(surf->tiling))
goto out_everything_is_miptail;
int miptail_first_level = surf->miptail_start_level;
if (miptail_first_level >= image->vk.mip_levels)
goto out_no_miptail;
uint64_t miptail_offset = 0;
isl_surf_get_image_offset_B_tile_sa(surf, miptail_first_level, 0, 0,
&miptail_offset,
&x_off, &y_off);
assert(x_off == 0 && y_off == 0);
assert(miptail_offset % ANV_SPARSE_BLOCK_SIZE == 0);
*imageMipTailFirstLod = miptail_first_level;
*imageMipTailSize = ANV_SPARSE_BLOCK_SIZE;
*imageMipTailOffset = binding_plane_offset + miptail_offset;
*imageMipTailStride = layer1_offset;
goto out_debug;
out_no_miptail:
*imageMipTailFirstLod = image->vk.mip_levels;
*imageMipTailSize = 0;
*imageMipTailOffset = 0;
*imageMipTailStride = 0;
goto out_debug;
out_everything_is_miptail:
*imageMipTailFirstLod = 0;
*imageMipTailSize = surf->size_B;
*imageMipTailOffset = binding_plane_offset;
*imageMipTailStride = 0;
out_debug:
sparse_debug("miptail first_lod:%d size:%"PRIu64" offset:%"PRIu64" "
"stride:%"PRIu64"\n",
*imageMipTailFirstLod, *imageMipTailSize,
*imageMipTailOffset, *imageMipTailStride);
}
static struct anv_vm_bind
vk_bind_to_anv_vm_bind(struct anv_sparse_binding_data *sparse,
const struct VkSparseMemoryBind *vk_bind)
{
struct anv_vm_bind anv_bind = {
.bo = NULL,
.address = sparse->address + vk_bind->resourceOffset,
.bo_offset = 0,
.size = vk_bind->size,
.op = ANV_VM_BIND,
};
assert(vk_bind->size);
assert(vk_bind->resourceOffset + vk_bind->size <= sparse->size);
if (vk_bind->memory != VK_NULL_HANDLE) {
anv_bind.bo = anv_device_memory_from_handle(vk_bind->memory)->bo;
anv_bind.bo_offset = vk_bind->memoryOffset,
assert(vk_bind->memoryOffset + vk_bind->size <= anv_bind.bo->size);
}
return anv_bind;
}
static VkResult
anv_sparse_bind_resource_memory(struct anv_device *device,
struct anv_sparse_binding_data *sparse,
uint64_t resource_size,
const VkSparseMemoryBind *vk_bind,
struct anv_sparse_submission *submit)
{
struct anv_vm_bind bind = vk_bind_to_anv_vm_bind(sparse, vk_bind);
uint64_t rem = vk_bind->size % ANV_SPARSE_BLOCK_SIZE;
if (rem != 0) {
if (vk_bind->resourceOffset + vk_bind->size == resource_size)
bind.size += ANV_SPARSE_BLOCK_SIZE - rem;
else
return vk_error(device, VK_ERROR_VALIDATION_FAILED_EXT);
}
return anv_sparse_submission_add(device, submit, &bind);
}
VkResult
anv_sparse_bind_buffer(struct anv_device *device,
struct anv_buffer *buffer,
const VkSparseMemoryBind *vk_bind,
struct anv_sparse_submission *submit)
{
return anv_sparse_bind_resource_memory(device, &buffer->sparse_data,
buffer->vk.size,
vk_bind, submit);
}
VkResult
anv_sparse_bind_image_opaque(struct anv_device *device,
struct anv_image *image,
const VkSparseMemoryBind *vk_bind,
struct anv_sparse_submission *submit)
{
struct anv_image_binding *b =
&image->bindings[ANV_IMAGE_MEMORY_BINDING_MAIN];
assert(!image->disjoint);
if (INTEL_DEBUG(DEBUG_SPARSE)) {
sparse_debug("%s:\n", __func__);
dump_anv_image(image);
u_foreach_bit(b, image->vk.aspects) {
VkImageAspectFlagBits aspect = 1 << b;
const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
struct isl_surf *surf = &image->planes[plane].primary_surface.isl;
sparse_debug("aspect 0x%x (plane %d):\n", aspect, plane);
dump_isl_surf(surf);
}
sparse_debug("\n");
}
return anv_sparse_bind_resource_memory(device, &b->sparse_data,
b->memory_range.size,
vk_bind, submit);
}
VkResult
anv_sparse_bind_image_memory(struct anv_queue *queue,
struct anv_image *image,
const VkSparseImageMemoryBind *bind,
struct anv_sparse_submission *submit)
{
struct anv_device *device = queue->device;
VkImageAspectFlags aspect = bind->subresource.aspectMask;
uint32_t mip_level = bind->subresource.mipLevel;
uint32_t array_layer = bind->subresource.arrayLayer;
assert(!(bind->flags & VK_SPARSE_MEMORY_BIND_METADATA_BIT));
struct anv_image_binding *img_binding = image->disjoint ?
&image->bindings[anv_image_aspect_to_binding(image, aspect)] :
&image->bindings[ANV_IMAGE_MEMORY_BINDING_MAIN];
struct anv_sparse_binding_data *sparse_data = &img_binding->sparse_data;
const uint32_t plane = anv_image_aspect_to_plane(image, aspect);
struct isl_surf *surf = &image->planes[plane].primary_surface.isl;
uint64_t binding_plane_offset =
image->planes[plane].primary_surface.memory_range.offset;
const struct isl_format_layout *layout =
isl_format_get_layout(surf->format);
struct isl_tile_info tile_info;
isl_surf_get_tile_info(surf, &tile_info);
if (INTEL_DEBUG(DEBUG_SPARSE)) {
sparse_debug("%s:\n", __func__);
sparse_debug("mip_level:%d array_layer:%d\n", mip_level, array_layer);
sparse_debug("aspect:0x%x plane:%d\n", aspect, plane);
sparse_debug("binding offset: [%d, %d, %d] extent: [%d, %d, %d]\n",
bind->offset.x, bind->offset.y, bind->offset.z,
bind->extent.width, bind->extent.height,
bind->extent.depth);
dump_anv_image(image);
dump_isl_surf(surf);
sparse_debug("\n");
}
VkExtent3D block_shape_px =
anv_sparse_calc_block_shape(device->physical, surf, &tile_info);
VkExtent3D block_shape_el = vk_extent3d_px_to_el(block_shape_px, layout);
/* Both bind->offset and bind->extent are in pixel units. */
VkOffset3D bind_offset_el = vk_offset3d_px_to_el(bind->offset, layout);
/* The spec says we only really need to align if for a given coordinate
* offset + extent equals the corresponding dimensions of the image
* subresource, but all the other non-aligned usage is invalid, so just
* align everything.
*/
VkExtent3D bind_extent_px = {
.width = ALIGN_NPOT(bind->extent.width, block_shape_px.width),
.height = ALIGN_NPOT(bind->extent.height, block_shape_px.height),
.depth = ALIGN_NPOT(bind->extent.depth, block_shape_px.depth),
};
VkExtent3D bind_extent_el = vk_extent3d_px_to_el(bind_extent_px, layout);
/* Nothing that has a tile_size different than ANV_SPARSE_BLOCK_SIZE should
* be reaching here, as these cases should be treated as "everything is
* part of the miptail" (see anv_sparse_calc_miptail_properties()).
*/
assert(isl_calc_tile_size(&tile_info) == ANV_SPARSE_BLOCK_SIZE);
/* How many blocks are necessary to form a whole line on this image? */
const uint32_t blocks_per_line = surf->row_pitch_B / (layout->bpb / 8) /
block_shape_el.width;
/* The loop below will try to bind a whole line of blocks at a time as
* they're guaranteed to be contiguous, so we calculate how many blocks
* that is and how big is each block to figure the bind size of a whole
* line.
*/
uint64_t line_bind_size_in_blocks = bind_extent_el.width /
block_shape_el.width;
uint64_t line_bind_size = line_bind_size_in_blocks * ANV_SPARSE_BLOCK_SIZE;
assert(line_bind_size_in_blocks != 0);
assert(line_bind_size != 0);
uint64_t memory_offset = bind->memoryOffset;
for (uint32_t z = bind_offset_el.z;
z < bind_offset_el.z + bind_extent_el.depth;
z += block_shape_el.depth) {
uint64_t subresource_offset_B;
uint32_t subresource_x_offset, subresource_y_offset;
isl_surf_get_image_offset_B_tile_sa(surf, mip_level, array_layer, z,
&subresource_offset_B,
&subresource_x_offset,
&subresource_y_offset);
assert(subresource_x_offset == 0 && subresource_y_offset == 0);
assert(subresource_offset_B % ANV_SPARSE_BLOCK_SIZE == 0);
for (uint32_t y = bind_offset_el.y;
y < bind_offset_el.y + bind_extent_el.height;
y+= block_shape_el.height) {
uint32_t line_block_offset = y / block_shape_el.height *
blocks_per_line;
uint64_t line_start_B = subresource_offset_B +
line_block_offset * ANV_SPARSE_BLOCK_SIZE;
uint64_t bind_offset_B = line_start_B +
(bind_offset_el.x / block_shape_el.width) *
ANV_SPARSE_BLOCK_SIZE;
VkSparseMemoryBind opaque_bind = {
.resourceOffset = binding_plane_offset + bind_offset_B,
.size = line_bind_size,
.memory = bind->memory,
.memoryOffset = memory_offset,
.flags = bind->flags,
};
memory_offset += line_bind_size;
assert(line_start_B % ANV_SPARSE_BLOCK_SIZE == 0);
assert(opaque_bind.resourceOffset % ANV_SPARSE_BLOCK_SIZE == 0);
assert(opaque_bind.size % ANV_SPARSE_BLOCK_SIZE == 0);
struct anv_vm_bind anv_bind = vk_bind_to_anv_vm_bind(sparse_data,
&opaque_bind);
VkResult result = anv_sparse_submission_add(device, submit,
&anv_bind);
if (result != VK_SUCCESS)
return result;
}
}
return VK_SUCCESS;
}
/* Checks if we support sparse images with the support parameters.
*
* We also return in the optional pointer 'valid_samples_out' a subset of the
* 'samples' argument containing only the sample counts supported, in case
* more than one flag is passed.
*/
VkResult
anv_sparse_image_check_support(struct anv_physical_device *pdevice,
VkImageCreateFlags flags,
VkImageTiling tiling,
VkSampleCountFlagBits samples,
VkImageType type,
VkFormat vk_format,
VkSampleCountFlagBits *valid_samples_out)
{
VkSampleCountFlagBits valid_samples = samples;
assert(flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT);
/* The spec says:
* "A sparse image created using VK_IMAGE_CREATE_SPARSE_BINDING_BIT (but
* not VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) supports all formats that
* non-sparse usage supports, and supports both VK_IMAGE_TILING_OPTIMAL
* and VK_IMAGE_TILING_LINEAR tiling."
*/
if (!(flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT))
return VK_SUCCESS;
if (type == VK_IMAGE_TYPE_1D)
return VK_ERROR_FORMAT_NOT_SUPPORTED;
/* From here on, these are the rules:
* "A sparse image created using VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT
* supports all non-compressed color formats with power-of-two element
* size that non-sparse usage supports. Additional formats may also be
* supported and can be queried via
* vkGetPhysicalDeviceSparseImageFormatProperties.
* VK_IMAGE_TILING_LINEAR tiling is not supported."
*/
/* We choose not to support sparse residency on emulated compressed
* formats due to the additional image plane. It would make the
* implementation extremely complicated.
*/
if (anv_is_compressed_format_emulated(pdevice, vk_format))
return VK_ERROR_FORMAT_NOT_SUPPORTED;
/* While the spec itself says linear is not supported (see above), deqp-vk
* tries anyway to create linear sparse images, so we have to check for it.
* This is also said in VUID-VkImageCreateInfo-tiling-04121:
* "If tiling is VK_IMAGE_TILING_LINEAR, flags must not contain
* VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT"
*/
if (tiling == VK_IMAGE_TILING_LINEAR)
return VK_ERROR_FORMAT_NOT_SUPPORTED;
if (!pdevice->vk.supported_features.sparseResidency2Samples)
valid_samples &= ~VK_SAMPLE_COUNT_2_BIT;
if (!pdevice->vk.supported_features.sparseResidency4Samples)
valid_samples &= ~VK_SAMPLE_COUNT_4_BIT;
if (!pdevice->vk.supported_features.sparseResidency8Samples)
valid_samples &= ~VK_SAMPLE_COUNT_8_BIT;
if (!pdevice->vk.supported_features.sparseResidency16Samples)
valid_samples &= ~VK_SAMPLE_COUNT_16_BIT;
valid_samples &= ~(VK_SAMPLE_COUNT_32_BIT | VK_SAMPLE_COUNT_64_BIT);
/* Here we return NOT_PRESENT since the user is asking for sample counts we
* already reported we don't support with sparse.
*/
if (!valid_samples) {
if (valid_samples_out)
*valid_samples_out = 0;
return VK_ERROR_FEATURE_NOT_PRESENT;
}
/* While our hardware allows us to support sparse with some depth/stencil
* formats (e.g., single-sampled 2D), the spec seems to be expecting that,
* if we support a format, we have to support it with all the multi-sampled
* flags we support for non-sparse. Therefore, just give up depth/stencil
* entirely since games don't seem to be requiring it.
*/
VkImageAspectFlags aspects = vk_format_aspects(vk_format);
if (aspects & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT))
return VK_ERROR_FORMAT_NOT_SUPPORTED;
const struct anv_format *anv_format = anv_get_format(pdevice, vk_format);
if (!anv_format)
return VK_ERROR_FORMAT_NOT_SUPPORTED;
for (int p = 0; p < anv_format->n_planes; p++) {
enum isl_format isl_format = anv_format->planes[p].isl_format;
if (isl_format == ISL_FORMAT_UNSUPPORTED)
return VK_ERROR_FORMAT_NOT_SUPPORTED;
const struct isl_format_layout *isl_layout =
isl_format_get_layout(isl_format);
/* As quoted above, we only need to support the power-of-two formats.
* The problem with the non-power-of-two formats is that we need an
* integer number of pixels to fit into a sparse block, so we'd need the
* sparse block sizes to be, for example, 192k for 24bpp.
*
* TODO: add support for these formats.
*/
if (isl_layout->bpb != 8 && isl_layout->bpb != 16 &&
isl_layout->bpb != 32 && isl_layout->bpb != 64 &&
isl_layout->bpb != 128)
return VK_ERROR_FORMAT_NOT_SUPPORTED;
}
/* These YUV formats are considered by Vulkan to be compressed 2x1 blocks.
* We don't need to support them since they're compressed. On Gfx12 we
* can't even have Tile64 for them. Once we do support these formats we'll
* have to report the correct block shapes because dEQP cares about them,
* and we'll have to adjust for the fact that ISL treats these as 16bpp 1x1
* blocks instead of 32bpp 2x1 compressed blocks (as block shapes are
* reported in units of compressed blocks).
*/
if (vk_format == VK_FORMAT_G8B8G8R8_422_UNORM ||
vk_format == VK_FORMAT_B8G8R8G8_422_UNORM)
return VK_ERROR_FORMAT_NOT_SUPPORTED;
if (valid_samples_out)
*valid_samples_out = valid_samples;
if (!valid_samples)
return VK_ERROR_FORMAT_NOT_SUPPORTED;
return VK_SUCCESS;
}
Reference in New Issue View Git Blame Copy Permalink