diff --git a/src/nouveau/compiler/nak_nir_lower_tex.c b/src/nouveau/compiler/nak_nir_lower_tex.c index 8a280648694..8c5aeabf358 100644 --- a/src/nouveau/compiler/nak_nir_lower_tex.c +++ b/src/nouveau/compiler/nak_nir_lower_tex.c @@ -322,7 +322,28 @@ shrink_image_load(nir_builder *b, nir_intrinsic_instr *intrin, const struct nak_compiler *nak) { enum pipe_format format = nir_intrinsic_format(intrin); - nir_component_mask_t comps_read = nir_def_components_read(&intrin->def); + nir_component_mask_t color_comps_read = + nir_def_components_read(&intrin->def); + + assert(intrin->intrinsic == nir_intrinsic_bindless_image_load || + intrin->intrinsic == nir_intrinsic_bindless_image_sparse_load); + + /* Pick off the sparse resident component (if any) before we do anything + * else. This makes later logic easier. + */ + bool is_sparse = false; + if (intrin->intrinsic == nir_intrinsic_bindless_image_sparse_load) { + unsigned resident_comp = intrin->def.num_components - 1; + if (color_comps_read & BITFIELD_BIT(resident_comp)) { + is_sparse = true; + color_comps_read &= ~BITFIELD_BIT(resident_comp); + } else { + /* If the sparse bit is never used, get rid of it */ + intrin->intrinsic = nir_intrinsic_bindless_image_load; + intrin->num_components--; + intrin->def.num_components--; + } + } if (intrin->def.bit_size == 64) { assert(format == PIPE_FORMAT_NONE || @@ -331,8 +352,8 @@ shrink_image_load(nir_builder *b, nir_intrinsic_instr *intrin, b->cursor = nir_after_instr(&intrin->instr); - nir_def *data_xy, *data_w; - if (comps_read & BITFIELD_BIT(3)) { + nir_def *data_xy, *data_w, *resident = NULL; + if (color_comps_read & BITFIELD_BIT(3)) { /* Thanks to descriptor indexing, we need to ensure that null * descriptor behavior works properly. In particular, normal zero * reads will return (0, 0, 0, 1) whereas null descriptor reads need @@ -341,25 +362,38 @@ shrink_image_load(nir_builder *b, nir_intrinsic_instr *intrin, * to extend the original RG32 with z = 0 and w = 1 and copy the w * value all the way out to 64-bit w value. */ - assert(intrin->num_components == 4); - assert(intrin->def.num_components == 4); + assert(intrin->num_components == 4 + is_sparse); + assert(intrin->def.num_components == 4 + is_sparse); intrin->def.bit_size = 32; data_xy = nir_channels(b, &intrin->def, 0x3); data_w = nir_channels(b, &intrin->def, 0x8); + if (is_sparse) + resident = nir_channel(b, &intrin->def, 4); } else { - intrin->num_components = 2; - intrin->def.num_components = 2; + intrin->num_components = 2 + is_sparse; + intrin->def.num_components = 2 + is_sparse; intrin->def.bit_size = 32; data_xy = nir_channels(b, &intrin->def, 0x3); data_w = nir_imm_int(b, 0); + if (is_sparse) + resident = nir_channel(b, &intrin->def, 2); } - nir_def *data = nir_vec4(b, nir_pack_64_2x32(b, data_xy), - nir_imm_zero(b, 1, 64), - nir_imm_zero(b, 1, 64), - nir_u2u64(b, data_w)); + nir_def *data; + if (is_sparse) { + data = nir_vec5(b, nir_pack_64_2x32(b, data_xy), + nir_imm_zero(b, 1, 64), + nir_imm_zero(b, 1, 64), + nir_u2u64(b, data_w), + nir_u2u64(b, resident)); + } else { + data = nir_vec4(b, nir_pack_64_2x32(b, data_xy), + nir_imm_zero(b, 1, 64), + nir_imm_zero(b, 1, 64), + nir_u2u64(b, data_w)); + } nir_def_rewrite_uses_after(&intrin->def, data, data->parent_instr); return true; @@ -372,28 +406,28 @@ shrink_image_load(nir_builder *b, nir_intrinsic_instr *intrin, * loads when the alpha channel is read even if we know the format has * fewer channels. */ - if (comps_read & BITFIELD_BIT(3)) + if (color_comps_read & BITFIELD_BIT(3)) return false; const unsigned old_comps = intrin->def.num_components; unsigned new_comps = util_format_get_nr_components(format); new_comps = util_next_power_of_two(new_comps); - if (comps_read <= BITFIELD_MASK(2)) + if (color_comps_read <= BITFIELD_MASK(2)) new_comps = 2; - if (comps_read <= BITFIELD_MASK(1)) + if (color_comps_read <= BITFIELD_MASK(1)) new_comps = 1; - if (new_comps >= intrin->num_components) + if (new_comps + is_sparse >= intrin->num_components) return false; b->cursor = nir_after_instr(&intrin->instr); - intrin->num_components = new_comps; - intrin->def.num_components = new_comps; + intrin->num_components = new_comps + is_sparse; + intrin->def.num_components = new_comps + is_sparse; assert(new_comps <= 4); - nir_def *comps[4]; + nir_def *comps[5]; for (unsigned c = 0; c < new_comps; c++) comps[c] = nir_channel(b, &intrin->def, c); for (unsigned c = new_comps; c < 3; c++) @@ -401,6 +435,10 @@ shrink_image_load(nir_builder *b, nir_intrinsic_instr *intrin, if (new_comps < 4) comps[3] = nir_imm_intN_t(b, 1, intrin->def.bit_size); + /* The resident bit always goes in the last channel */ + if (is_sparse) + comps[old_comps - 1] = nir_channel(b, &intrin->def, new_comps); + nir_def *data = nir_vec(b, comps, old_comps); nir_def_rewrite_uses_after(&intrin->def, data, data->parent_instr); return true; @@ -525,6 +563,7 @@ lower_tex_instr(nir_builder *b, nir_instr *instr, void *_data) nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); switch (intrin->intrinsic) { case nir_intrinsic_bindless_image_load: + case nir_intrinsic_bindless_image_sparse_load: return shrink_image_load(b, intrin, nak); case nir_intrinsic_bindless_image_store: return shrink_image_store(b, intrin, nak);