From e5d4c4934d772fc6c37cbf764fda1c075a05b923 Mon Sep 17 00:00:00 2001 From: Lorenzo Rossi Date: Fri, 23 May 2025 15:56:07 +0200 Subject: [PATCH] nak: Add an image address lowering pass On Kepler, we don't have real surface opcodes. Instead, we have a bunch of helper ops which make image calculations in the shader reasonably efficient and we have to lower to that and an address-based load/store. Part-of: --- src/nouveau/compiler/meson.build | 2 + src/nouveau/compiler/nak.h | 2 + .../compiler/nak_nir_lower_image_addrs.c | 621 ++++++++++++++++++ src/nouveau/compiler/nak_nir_lower_tex.c | 66 -- src/nouveau/meson.build | 2 +- src/nouveau/nil/meson.build | 5 + 6 files changed, 631 insertions(+), 67 deletions(-) create mode 100644 src/nouveau/compiler/nak_nir_lower_image_addrs.c diff --git a/src/nouveau/compiler/meson.build b/src/nouveau/compiler/meson.build index 518a5a99414..7a4c60b18b2 100644 --- a/src/nouveau/compiler/meson.build +++ b/src/nouveau/compiler/meson.build @@ -18,6 +18,7 @@ libnak_c_files = files( 'nak_nir_lower_cf.c', 'nak_nir_lower_fs_inputs.c', 'nak_nir_lower_gs_intrinsics.c', + 'nak_nir_lower_image_addrs.c', 'nak_nir_lower_non_uniform_ldcx.c', 'nak_nir_lower_scan_reduce.c', 'nak_nir_lower_kepler_shared_atomics.c', @@ -43,6 +44,7 @@ libnak_deps = [ idep_mesautil, idep_nir_headers, idep_nvidia_headers, + idep_nil_h, ] _nak_bindings_rs = rust.bindgen( diff --git a/src/nouveau/compiler/nak.h b/src/nouveau/compiler/nak.h index acea65d1331..aa2c8628791 100644 --- a/src/nouveau/compiler/nak.h +++ b/src/nouveau/compiler/nak.h @@ -32,6 +32,8 @@ nak_nir_options(const struct nak_compiler *nak); void nak_preprocess_nir(nir_shader *nir, const struct nak_compiler *nak); +bool nak_nir_lower_image_addrs(nir_shader *nir, const struct nak_compiler *nak); + struct nak_sample_location { uint8_t x_u4 : 4; uint8_t y_u4 : 4; diff --git a/src/nouveau/compiler/nak_nir_lower_image_addrs.c b/src/nouveau/compiler/nak_nir_lower_image_addrs.c new file mode 100644 index 00000000000..684d82891b6 --- /dev/null +++ b/src/nouveau/compiler/nak_nir_lower_image_addrs.c @@ -0,0 +1,621 @@ +/* + * Copyright © 2023 Collabora, Ltd. + * SPDX-License-Identifier: MIT + */ + +#include "nak.h" +#include "nak_private.h" +#include "nil.h" + +#include "nir_format_convert.h" + +static nir_def * +build_load_su_info(nir_builder *b, nir_deref_instr *deref, uint32_t offset) +{ + return nir_image_deref_load_info_nv(b, 1, &deref->def, .base = offset); +} + +#define load_su_info(b, d, field) \ + build_load_su_info((b), (d), offsetof(struct nil_su_info, field)) + +static enum pipe_format +format_for_bits(unsigned bits) +{ + switch (bits) { + case 8: return PIPE_FORMAT_R8_UINT; + case 16: return PIPE_FORMAT_R16_UINT; + case 32: return PIPE_FORMAT_R32_UINT; + case 64: return PIPE_FORMAT_R32G32_UINT; + case 128: return PIPE_FORMAT_R32G32B32A32_UINT; + default: unreachable("Unknown number of image format bits"); + } +} + +static unsigned +sampler_dim_len(enum glsl_sampler_dim dim) +{ + switch (dim) { + case GLSL_SAMPLER_DIM_1D: + case GLSL_SAMPLER_DIM_BUF: + return 1; + case GLSL_SAMPLER_DIM_CUBE: + case GLSL_SAMPLER_DIM_2D: + case GLSL_SAMPLER_DIM_RECT: + case GLSL_SAMPLER_DIM_MS: + return 2; + case GLSL_SAMPLER_DIM_3D: + return 3; + default: + unreachable("Unhandled sampler dim"); + return 1;// Never reached + } +} + +static nir_def * +lower_formatted_image_load(nir_builder *b, + nir_intrinsic_instr *intrin, + enum pipe_format format) +{ + if (format == PIPE_FORMAT_NONE) + return false; + + unsigned bits = util_format_get_blocksizebits(format); + + assert(intrin->intrinsic == nir_intrinsic_suldga_nv); + + nir_intrinsic_set_format(intrin, format_for_bits(bits)); + + const unsigned num_raw_components = DIV_ROUND_UP(bits, 32); + intrin->num_components = num_raw_components; + intrin->def.num_components = num_raw_components; + + b->cursor = nir_after_instr(&intrin->instr); + nir_def *rgba = NULL; + switch (format) { + case PIPE_FORMAT_R64_UINT: + case PIPE_FORMAT_R64_SINT: + rgba = nir_vec4(b, nir_pack_64_2x32(b, &intrin->def), + nir_imm_int64(b, 0), + nir_imm_int64(b, 0), + nir_imm_int64(b, 1)); + break; + default: + rgba = nir_format_unpack_rgba(b, &intrin->def, format); + break; + } + + return rgba; +} + +static nir_def * +load_su_info_clamp(nir_builder *b, nir_deref_instr *deref, + unsigned xyz) +{ + /* Array length is always stored in clamp_z */ + enum glsl_sampler_dim dim = glsl_get_sampler_dim(deref->type); + if (dim == GLSL_SAMPLER_DIM_1D && xyz == 1) + return load_su_info(b, deref, clamp_z); + + switch (xyz) { + case 0: return load_su_info(b, deref, clamp_x); + case 1: return load_su_info(b, deref, clamp_y); + case 2: return load_su_info(b, deref, clamp_z); + default: unreachable("Invalid image dimension"); + } +} + +static nir_def * +clamp_coord(nir_builder *b, nir_deref_instr *deref, + nir_def* coord, int xyz) +{ + nir_def *clamp = load_su_info_clamp(b, deref, xyz); + + const struct nak_nir_suclamp_flags flags = { + .mode = NAK_SUCLAMP_MODE_STORED_DESCRIPTOR, + .round = NAK_SUCLAMP_ROUND_R1, + .is_s32 = false, + .is_2d = true, + }; + + nir_def *dst = nir_suclamp_nv(b, coord, clamp, + .flags = NAK_AS_U32(flags)); + return nir_channel(b, dst, 0); +} + +static void +load_sample_size(nir_builder *b, nir_deref_instr *deref, + nir_def **sample_width_log2, + nir_def **sample_height_log2) +{ + /* MS width and height are stored in the lower 8 bits of pitch */ + nir_def *pitch = load_su_info(b, deref, pitch); + *sample_width_log2 = nir_ubitfield_extract_imm(b, pitch, 24, 4); + *sample_height_log2 = nir_ubitfield_extract_imm(b, pitch, 28, 4); +} + +// Kepler only supports suldga, sustga, so we need to compute +// the raw address manually, this is done through a weird dance +// and custom ops. +// - each coordinate is clamped through suclamp +// - compute block offset using gob coordinates (y * pitch + x) +// - merge bitfields (output of suclamps) +// this will both merge the block coordinates, combine +// pixel coordinates (in GoB-space) and OR together the OOB +// predicate +// - compute the effective upper address by combining block_offset, +// bitfield and base_address +// - combine effective upper address and the combined bitfield that +// contains the lower 8 bits of global address +// - pass the combined values into suldga/sustga +// +// Linear Layout support: +// Shader cannot know at compile time if an image is in linear format +// or in block format, thus it needs to support both modes using the +// same opcodes, the only difference should be in descriptors. +// The address for linear layout is computed with: +// eff_addr = base_addr + (y * pitch + x)*el_size_B +// Here we first compute off = (y * pitch + x) with imadsp +// then we split what goes in the first 8 bits using subfm +// (in 0..8 it loads off << el_size_B.log2, in 16..32 it +// loads off >> el_size_B.log2) +// For this we need to emit eau = sueau bf.x, bf.y, off +// this would totally screw up block-linear calcs, but we can use +// is_3d=false to skip the third argument only in block-linear. +static void +compute_image_address(nir_builder *b, + nir_intrinsic_instr *intrin, + enum glsl_sampler_dim sampler_dim, + nir_def **addr, + nir_def **is_oob) +{ + nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); + + const unsigned dim = sampler_dim_len(sampler_dim); + const bool is_cube = sampler_dim == GLSL_SAMPLER_DIM_CUBE; + + // A CubeMap is a 6-array of 2D images, a CubeMap array is just + // multiple CubeMap concatenated together, so the cube access is alway + // treated as an array + const bool is_array = nir_intrinsic_image_array(intrin) || is_cube; + + // Prevent read/write for null-descriptors + nir_def *base_addr = load_su_info(b, deref, addr_shifted8); + *is_oob = nir_ieq_imm(b, base_addr, 0); + + nir_def *coords[3]; + for (int i = 0; i < 3; i++) + coords[i] = nir_channel(b, intrin->src[1].ssa, i); + + // Lower multi-sample coords + if (sampler_dim == GLSL_SAMPLER_DIM_MS) { + nir_def *s = intrin->src[2].ssa; + nir_def *sw_log2, *sh_log2; + load_sample_size(b, deref, &sw_log2, &sh_log2); + nir_def *s_map = load_su_info(b, deref, extra);// multi-sample table + nir_def *num_samples = nir_ishl(b, nir_imm_int(b, 1), + nir_iadd(b, sw_log2, sh_log2)); + + nir_def *s_xy = nir_ushr(b, s_map, nir_imul_imm(b, s, 4)); + nir_def *sx = nir_ubitfield_extract_imm(b, s_xy, 0, 2); + nir_def *sy = nir_ubitfield_extract_imm(b, s_xy, 2, 2); + + nir_def *sw = nir_ishl(b, nir_imm_int(b, 1), sw_log2); + nir_def *sh = nir_ishl(b, nir_imm_int(b, 1), sh_log2); + + nir_def *x = nir_imad(b, coords[0], sw, sx); + nir_def *y = nir_imad(b, coords[1], sh, sy); + + // Check if OOB + *is_oob = nir_ior(b, *is_oob, nir_uge(b, s, num_samples)); + + coords[0] = x; + coords[1] = y; + } + + // Clamp coordinates + // This computes a bitfield with the following info inside: + // - Block coordinates + // - GoB coordinates + // - predicate for OOB access + // - Wether the clamp is using pitch_linear + // - n. of block tiles for the coordinate + nir_def *clamped_x = clamp_coord(b, deref, coords[0], 0); + nir_def *clamped_y = dim >= 2 ? clamp_coord(b, deref, coords[1], 1) : nir_imm_int(b, 0); + nir_def *clamped_z = dim >= 3 ? clamp_coord(b, deref, coords[2], 2) : nir_imm_int(b, 0); + + // For arrays the clamp is "plain", no bitfield is computed + // just the OOB predicate. + nir_def *array_idx = NULL; + if (is_array) { + nir_def *clamp = load_su_info_clamp(b, deref, dim); + nir_def *coord = coords[dim]; + + const struct nak_nir_suclamp_flags flags = { + .mode = NAK_SUCLAMP_MODE_PITCH_LINEAR, + .round = NAK_SUCLAMP_ROUND_R1, + .is_s32 = false, + .is_2d = false, + }; + + nir_def *dst = nir_suclamp_nv(b, coord, clamp, + .flags = NAK_AS_U32(flags)); + array_idx = nir_channel(b, dst, 0); + *is_oob = nir_ior(b, *is_oob, nir_ine_imm(b, nir_channel(b, dst, 1), 0)); + } + + // Compute offset + // for Block-Linear: GOB coordinates (offset that contributes *64) + // for Pitch-Linear: offset in pixels (y * pitch) + x + // for Buffer: offset = x + nir_def *off; + if (dim == 1) { + // This can be only 16 bits because it's only tile coordinates + // (actually it's 20 bits in pitch-linear mode, we don't support + // images that big) + off = nir_iand_imm(b, clamped_x, 0xffff); + } else if (dim == 2) { + // off = clamped.y * pitch + clamped.x + nir_def *pitch = load_su_info(b, deref, pitch); + + const struct nak_nir_imadsp_flags flags = { + .src0 = NAK_IMAD_TYPE_U16_LO, + .src1 = NAK_IMAD_TYPE_U24, + .src2 = NAK_IMAD_TYPE_U16_LO, + .params_from_src1 = false, + }; + off = nir_imadsp_nv(b, clamped_y, pitch, clamped_x, + .flags = NAK_AS_U32(flags)); + } else { + assert(dim == 3); + // off = (clamped.z * height + clamped.y) * pitch + clamped.x + // height is the height in blocks, we can compute this by doing a + // block linear clamp with the maximum value. + // (block shift-right is applied by suclamp) + nir_def *dim_y = load_su_info_clamp(b, deref, 1); + const struct nak_nir_suclamp_flags clamp_flags = { + .mode = NAK_SUCLAMP_MODE_BLOCK_LINEAR, + .round = NAK_SUCLAMP_ROUND_R1, + .is_s32 = false, + .is_2d = false, + }; + nir_def *max_y = nir_suclamp_nv(b, nir_imm_int(b, -1), dim_y, + .flags=NAK_AS_U32(clamp_flags)); + + // max_y is still a bitfield, we can add 1 but we must only use the + // lower 16 bits of height_b. + nir_def *height_b = nir_iadd_imm(b, nir_channel(b, max_y, 0), 1); + + const struct nak_nir_imadsp_flags flags_zy = { + .src0 = NAK_IMAD_TYPE_U16_LO, + .src1 = NAK_IMAD_TYPE_U16_LO, + .src2 = NAK_IMAD_TYPE_U16_LO, + .params_from_src1 = false, + }; + nir_def *off_2d = nir_imadsp_nv(b, clamped_z, height_b, clamped_y, + .flags = NAK_AS_U32(flags_zy)); + + nir_def *pitch = load_su_info(b, deref, pitch); + const struct nak_nir_imadsp_flags flags = { + .src0 = NAK_IMAD_TYPE_U32, + .src1 = NAK_IMAD_TYPE_U24, + .src2 = NAK_IMAD_TYPE_U16_LO, + .params_from_src1 = false, + }; + off = nir_imadsp_nv(b, off_2d, pitch, clamped_x, + .flags = NAK_AS_U32(flags)); + } + + // Compute merged bitfield + nir_def *bf; + { + // bf, pred = subfm clamped.x, clamped.y, clamped.z + nir_def *bfz; + bool is_3d = dim >= 3; + + if (dim == 2 && !is_array) { + // Special case for pitch_linear support, see comment above. + bfz = off; + } else { + bfz = clamped_z; + } + + nir_def *combined = nir_subfm_nv(b, clamped_x, clamped_y, bfz, + .flags = is_3d); + bf = nir_channel(b, combined, 0); + nir_def *bfm_oob = nir_ine_imm(b, nir_channel(b, combined, 1), 0); + *is_oob = nir_ior(b, *is_oob, bfm_oob); + } + + nir_def *eau = nir_sueau_nv(b, off, bf, base_addr); + + // Apply array layer offset + if (is_array) { + nir_def *array_stride = load_su_info(b, deref, array_stride_shifted8); + + // Note: this only works because array_idx has been plain-clamped + // so it's not a bitfield (and we can read more than u16) + const struct nak_nir_imadsp_flags flags = { + .src0 = NAK_IMAD_TYPE_U32, + .src1 = NAK_IMAD_TYPE_U24, + .src2 = NAK_IMAD_TYPE_U32, + .params_from_src1 = false, + }; + + eau = nir_imadsp_nv(b, array_idx, array_stride, eau, + .flags = NAK_AS_U32(flags)); + } + + *addr = nir_vec2(b, bf, eau); +} + +// Buffer address calculation is much simpler than images +// In the end we need the global address to reach in two registers +// The high register should have the highest 32-bit part of the address +// The lower register should have the lowest 8-bit part +// This computation can be summarized as: +// res = addr + clamp(x) * num_comps +// Unfortunately, given the weird register division, we need some weird +// ops. We abuse some special functions from image addressing to reduce +// the number of instructions. +// +// For null descriptors, we put bit 31 high in lower_addr, +// this is then passed on bfm and sets the oob predicate high. +// And that's how we get safe null descriptors for free. +// +// TODO: check if we can also use vector instructions, codegen used VSHL +// but it also didn't support sub-0x100 aligned buffers +static void +compute_buffer_address(nir_builder *b, + nir_intrinsic_instr *intrin, + enum glsl_sampler_dim sampler_dim, + nir_def **addr, + nir_def **is_oob) +{ + nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); + + nir_def *num_elems = load_su_info(b, deref, clamp_x); + nir_def *el_size_B = load_su_info(b, deref, pitch); + nir_def *lower_addr = load_su_info(b, deref, extra); + + nir_def *raw_off = nir_channel(b, intrin->src[1].ssa, 0); + + *is_oob = nir_uge(b, raw_off, num_elems); + + nir_def *offset = nir_imad(b, raw_off, el_size_B, lower_addr); + nir_def *base_addr = load_su_info(b, deref, addr_shifted8); + + *addr = nir_vec2(b, offset, base_addr); +} + +static nir_def * +compute_address_from_ga_offset(nir_builder *b, nir_def *addr, + enum pipe_format format, + enum nak_su_ga_offset_mode offset_mode) +{ + assert(offset_mode == NAK_SUGA_OFF_MODE_U32 || + offset_mode == NAK_SUGA_OFF_MODE_U8); + // mode U8: addr_hi contains bits 8..40, addr_lo contains 0..8 + // mode U32: addr_hi contains bits 8..40, addr_lo contains 0..32 + // and they should be added to addr_hi + nir_def *lo_8; + nir_def *hi_32 = nir_channel(b, addr, 1); + + // With what should we fill the lower 8 bits? + if (offset_mode == NAK_SUGA_OFF_MODE_U8) { + lo_8 = nir_channel(b, addr, 0); + } else { + lo_8 = nir_imm_int(b, 0); + } + + // Construct the 64-bit address (hi_32 << 8 | lo_8) + nir_def *low = nir_prmt_nv(b, nir_imm_int(b, 0x6540), lo_8, hi_32); + nir_def *high = nir_prmt_nv(b, nir_imm_int(b, 0x0007), + nir_imm_int(b, 0), hi_32); + nir_def *full_addr = nir_pack_64_2x32_split(b, low, high); + + if (offset_mode == NAK_SUGA_OFF_MODE_U32) { + full_addr = nir_iadd(b, full_addr, nir_u2u64(b, nir_channel(b, addr, 0))); + } + + return full_addr; +} + +static void +lower_image_access(nir_builder *b, nir_intrinsic_instr *intrin) +{ + b->cursor = nir_instr_remove(&intrin->instr); + nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); + + enum pipe_format format = nir_intrinsic_format(intrin); + + if (format == PIPE_FORMAT_NONE) + format = nir_deref_instr_get_variable(deref)->data.image.format; + + const unsigned int num_dst_components = intrin->def.num_components; + const enum glsl_sampler_dim sampler_dim = nir_intrinsic_image_dim(intrin); + enum nak_su_ga_offset_mode offset_mode; + + nir_def *addr, *is_oob; + if (sampler_dim != GLSL_SAMPLER_DIM_BUF) { + compute_image_address(b, intrin, sampler_dim, &addr, &is_oob); + offset_mode = NAK_SUGA_OFF_MODE_U8; + } else { + compute_buffer_address(b, intrin, sampler_dim, &addr, &is_oob); + offset_mode = NAK_SUGA_OFF_MODE_U32; + } + + switch (intrin->intrinsic) { + case nir_intrinsic_image_deref_load: { + // .format intrinsic index is only used to pass how many bits to store + nir_def *fmt = load_su_info(b, deref, format_info); + nir_def *new_ssa = nir_suldga_nv(b, num_dst_components, addr, fmt, + is_oob, + .format = format, + .access = nir_intrinsic_access(intrin), + .flags = offset_mode); + + nir_intrinsic_instr *parent = nir_instr_as_intrinsic(new_ssa->parent_instr); + new_ssa = lower_formatted_image_load(b, parent, format); + nir_def_rewrite_uses(&intrin->def, new_ssa); + + break; + } + case nir_intrinsic_image_deref_store: { + nir_def *fmt = load_su_info(b, deref, format_info); + nir_sustga_nv(b, addr, fmt, is_oob, intrin->src[3].ssa, + .access = nir_intrinsic_access(intrin), + .flags = offset_mode); + break; + } + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_deref_atomic_swap: { + nir_atomic_op atomic_op = nir_intrinsic_atomic_op(intrin); + + // suldga and sustga expect address as [low_8, high_32] + // while global_atomic expects a 64-bit address + nir_def *full_addr = compute_address_from_ga_offset(b, addr, + format, + offset_mode); + + const unsigned bit_size = (format == PIPE_FORMAT_R64_UINT || + format == PIPE_FORMAT_R64_SINT) ? 64 : 32; + + nir_def *res, *res_ib, *res_oob; + nir_push_if(b, nir_inot(b, is_oob)); + if (intrin->intrinsic == nir_intrinsic_image_deref_atomic) { + res_ib = nir_global_atomic(b, bit_size, full_addr, + intrin->src[3].ssa, + .atomic_op = atomic_op); + } else { + res_ib = nir_global_atomic_swap(b, bit_size, full_addr, + intrin->src[3].ssa, + intrin->src[4].ssa, + .atomic_op = atomic_op); + } + nir_push_else(b, NULL); { + res_oob = nir_imm_intN_t(b, 0, bit_size); + } + nir_pop_if(b, NULL); + res = nir_if_phi(b, res_ib, res_oob); + + nir_def_rewrite_uses(&intrin->def, res); + break; + } + default: + unreachable("Unknown image intrinsic"); + } +} + +static void +lower_image_size(nir_builder *b, nir_intrinsic_instr *intrin) +{ + b->cursor = nir_instr_remove(&intrin->instr); + nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); + + const unsigned dim = sampler_dim_len(nir_intrinsic_image_dim(intrin)); + + const bool is_array = nir_intrinsic_image_array(intrin); + const unsigned cdim = dim + is_array; + assert(cdim <= 3); + nir_def *comps[3]; + + nir_def *is_null = nir_ieq_imm(b, load_su_info(b, deref, addr_shifted8), + 0); + nir_def *one_if_present = nir_bcsel(b, is_null, nir_imm_int(b, 0), + nir_imm_int(b, 1)); + + // In descriptors we don't really have the size, but the clamp + // since it's inclusive, it is size - 1. + // Also, clamp is a bitfield, we need to extract the lower 16 bits. + // We can do both operations (extraction and addition) with an imadsp. + // To handle null descriptors, the addition accumulator of the imadsp + // is 1 only for non-null descriptors. + for (int i = 0; i < cdim; i++) { + nir_def *clamp = load_su_info_clamp(b, deref, i); + const struct nak_nir_imadsp_flags flags = { + .src0 = NAK_IMAD_TYPE_U16_LO, + .src1 = NAK_IMAD_TYPE_U24, + .src2 = NAK_IMAD_TYPE_U16_LO, + .params_from_src1 = false, + }; + comps[i] = nir_imadsp_nv(b, clamp, one_if_present, + one_if_present, + .flags = NAK_AS_U32(flags)); + } + // Clamp has multi-sampling already lowered, we need to de-lower it. + if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_MS) { + nir_def *ms_w_log2, *ms_h_log2; + load_sample_size(b, deref, &ms_w_log2, &ms_h_log2); + + comps[0] = nir_ishr(b, comps[0], ms_w_log2); + comps[1] = nir_ishr(b, comps[1], ms_h_log2); + } + + nir_def_rewrite_uses(&intrin->def, nir_vec(b, comps, cdim)); +} + +static void +lower_buffer_size(nir_builder *b, nir_intrinsic_instr *intrin) +{ + b->cursor = nir_instr_remove(&intrin->instr); + nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); + nir_def *num_elems = load_su_info(b, deref, clamp_x); + + nir_def_rewrite_uses(&intrin->def, num_elems); +} + +static void +lower_image_samples(nir_builder *b, nir_intrinsic_instr *intrin) +{ + b->cursor = nir_instr_remove(&intrin->instr); + nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); + + nir_def *sw_log2, *sh_log2; + load_sample_size(b, deref, &sw_log2, &sh_log2); + + nir_def *samples_log2 = nir_iadd(b, sw_log2, sh_log2); + nir_def *samples = nir_ishl(b, nir_imm_int(b, 1), samples_log2); + + // Handle null descriptors + nir_def *addr = load_su_info(b, deref, addr_shifted8); + nir_def *is_null = nir_ieq_imm(b, addr, 0); + samples = nir_bcsel(b, is_null, nir_imm_int(b, 0), samples); + + nir_def_rewrite_uses(&intrin->def, samples); +} + +static bool +lower_image_intrin(nir_builder *b, + nir_intrinsic_instr *intrin, + void *_data) +{ + switch (intrin->intrinsic) { + case nir_intrinsic_image_deref_load: + case nir_intrinsic_image_deref_store: + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_deref_atomic_swap: + lower_image_access(b, intrin); + return true; + case nir_intrinsic_image_deref_size: + if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_BUF) { + lower_buffer_size(b, intrin); + } else { + lower_image_size(b, intrin); + } + return true; + case nir_intrinsic_image_deref_samples: + lower_image_samples(b, intrin); + return true; + default: + return false; + } +} + +bool +nak_nir_lower_image_addrs(nir_shader *nir, + const struct nak_compiler *nak) +{ + return nir_shader_intrinsics_pass(nir, lower_image_intrin, + nir_metadata_none, + (void *)nak); +} diff --git a/src/nouveau/compiler/nak_nir_lower_tex.c b/src/nouveau/compiler/nak_nir_lower_tex.c index be770a212d0..41950acaba5 100644 --- a/src/nouveau/compiler/nak_nir_lower_tex.c +++ b/src/nouveau/compiler/nak_nir_lower_tex.c @@ -414,69 +414,6 @@ lower_txq(nir_builder *b, nir_tex_instr *tex, const struct nak_compiler *nak) return true; } -static enum pipe_format -format_for_bits(unsigned bits) -{ - switch (bits) { - case 8: return PIPE_FORMAT_R8_UINT; - case 16: return PIPE_FORMAT_R16_UINT; - case 32: return PIPE_FORMAT_R32_UINT; - case 64: return PIPE_FORMAT_R32G32_UINT; - case 128: return PIPE_FORMAT_R32G32B32A32_UINT; - default: unreachable("Unknown number of image format bits"); - } -} - -static bool -lower_formatted_image_load(nir_builder *b, nir_intrinsic_instr *intrin, - const struct nak_compiler *nak) -{ - enum pipe_format format = nir_intrinsic_format(intrin); - if (format == PIPE_FORMAT_NONE) - return false; - - unsigned bits = util_format_get_blocksizebits(format); - - switch (intrin->intrinsic) { - case nir_intrinsic_bindless_image_load: - intrin->intrinsic = nir_intrinsic_bindless_image_load_raw_nv; - break; - default: - unreachable("Unknown image intrinsic"); - } - - nir_intrinsic_set_format(intrin, format_for_bits(bits)); - - ASSERTED const unsigned rgba_bit_size = intrin->def.bit_size; - intrin->def.bit_size = 32; - - nir_intrinsic_set_dest_type(intrin, nir_type_uint32); - const unsigned num_raw_components = DIV_ROUND_UP(bits, 32); - intrin->num_components = num_raw_components; - intrin->def.num_components = num_raw_components; - - b->cursor = nir_after_instr(&intrin->instr); - nir_def *rgba = NULL; - switch (format) { - case PIPE_FORMAT_R64_UINT: - case PIPE_FORMAT_R64_SINT: - assert(rgba_bit_size == 64); - rgba = nir_vec4(b, nir_pack_64_2x32(b, &intrin->def), - nir_imm_int64(b, 0), - nir_imm_int64(b, 0), - nir_imm_int64(b, 1)); - break; - default: - assert(rgba_bit_size == 32); - rgba = nir_format_unpack_rgba(b, &intrin->def, format); - break; - } - - nir_def_rewrite_uses_after(&intrin->def, rgba, rgba->parent_instr); - - return true; -} - static bool shrink_image_load(nir_builder *b, nir_intrinsic_instr *intrin, const struct nak_compiler *nak) @@ -735,9 +672,6 @@ lower_tex_instr(nir_builder *b, nir_instr *instr, void *_data) nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); switch (intrin->intrinsic) { case nir_intrinsic_bindless_image_load: - if (nak->sm < 50 && lower_formatted_image_load(b, intrin, nak)) - return true; - FALLTHROUGH; case nir_intrinsic_bindless_image_sparse_load: return shrink_image_load(b, intrin, nak); case nir_intrinsic_bindless_image_store: diff --git a/src/nouveau/meson.build b/src/nouveau/meson.build index c2cff484e7f..5468334f299 100644 --- a/src/nouveau/meson.build +++ b/src/nouveau/meson.build @@ -6,6 +6,7 @@ subdir('headers') subdir('winsys') if with_nouveau_vk subdir('rust') + subdir('nil') subdir('compiler') endif if with_tools.contains('drm-shim') @@ -14,7 +15,6 @@ endif subdir('codegen') if with_nouveau_vk - subdir('nil') subdir('mme') subdir('vulkan') endif diff --git a/src/nouveau/nil/meson.build b/src/nouveau/nil/meson.build index 8b13bc821de..4b1350b5799 100644 --- a/src/nouveau/nil/meson.build +++ b/src/nouveau/nil/meson.build @@ -96,6 +96,11 @@ _nil_h = custom_target( depfile : 'nil.h.d', ) +idep_nil_h = declare_dependency( + include_directories : include_directories('.'), + sources : [_nil_h], +) + idep_nil = declare_dependency( include_directories : include_directories('.'), link_with : _libnil,