diff --git a/src/gallium/drivers/freedreno/ir3/ir3_descriptor.c b/src/gallium/drivers/freedreno/ir3/ir3_descriptor.c new file mode 100644 index 00000000000..b8db83fd8ab --- /dev/null +++ b/src/gallium/drivers/freedreno/ir3/ir3_descriptor.c @@ -0,0 +1,131 @@ +/* + * Copyright © 2022 Google, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "nir.h" +#include "nir_builder.h" + +#include "ir3/ir3_descriptor.h" + +static bool +lower_intrinsic(nir_builder *b, nir_intrinsic_instr *intr) +{ + unsigned desc_offset; + + switch (intr->intrinsic) { + case nir_intrinsic_load_ssbo: + case nir_intrinsic_store_ssbo: + case nir_intrinsic_ssbo_atomic_add: + case nir_intrinsic_ssbo_atomic_imin: + case nir_intrinsic_ssbo_atomic_umin: + case nir_intrinsic_ssbo_atomic_imax: + case nir_intrinsic_ssbo_atomic_umax: + case nir_intrinsic_ssbo_atomic_and: + case nir_intrinsic_ssbo_atomic_or: + case nir_intrinsic_ssbo_atomic_xor: + case nir_intrinsic_ssbo_atomic_exchange: + case nir_intrinsic_ssbo_atomic_comp_swap: + case nir_intrinsic_ssbo_atomic_fadd: + case nir_intrinsic_ssbo_atomic_fmin: + case nir_intrinsic_ssbo_atomic_fmax: + case nir_intrinsic_ssbo_atomic_fcomp_swap: + case nir_intrinsic_get_ssbo_size: + desc_offset = IR3_BINDLESS_SSBO_OFFSET; + break; + case nir_intrinsic_image_load: + case nir_intrinsic_image_store: + case nir_intrinsic_image_atomic_add: + case nir_intrinsic_image_atomic_imin: + case nir_intrinsic_image_atomic_umin: + case nir_intrinsic_image_atomic_imax: + case nir_intrinsic_image_atomic_umax: + case nir_intrinsic_image_atomic_and: + case nir_intrinsic_image_atomic_or: + case nir_intrinsic_image_atomic_xor: + case nir_intrinsic_image_atomic_exchange: + case nir_intrinsic_image_atomic_comp_swap: + case nir_intrinsic_image_size: + case nir_intrinsic_image_samples: + desc_offset = IR3_BINDLESS_IMAGE_OFFSET; + break; + default: + return false; + } + + unsigned buffer_src; + if (intr->intrinsic == nir_intrinsic_store_ssbo) { + /* store_ssbo has the value first, and ssbo src as 2nd src: */ + buffer_src = 1; + } else { + /* the rest have ssbo src as 1st src: */ + buffer_src = 0; + } + + unsigned set = ir3_shader_descriptor_set(b->shader->info.stage); + nir_ssa_def *src = nir_ssa_for_src(b, intr->src[buffer_src], 1); + src = nir_iadd(b, src, nir_imm_int(b, desc_offset)); + /* An out-of-bounds index into an SSBO/image array can cause a GPU fault + * on access to the descriptor (I don't see any hw mechanism to bound the + * access). We could just allow the resulting iova fault (it is a read + * fault, so shouldn't corrupt anything), but at the cost of one extra + * instruction (as long as IR3_BINDLESS_DESC_COUNT is a power-of-two) we + * can avoid the dmesg spam and users thinking this is a driver bug: + */ + src = nir_umod(b, src, nir_imm_int(b, IR3_BINDLESS_DESC_COUNT)); + nir_ssa_def *bindless = nir_bindless_resource_ir3(b, 32, src, set); + nir_instr_rewrite_src_ssa(&intr->instr, &intr->src[buffer_src], bindless); + + return true; +} + +static bool +lower_instr(nir_builder *b, nir_instr *instr, void *cb_data) +{ + b->cursor = nir_before_instr(instr); + switch (instr->type) { + case nir_instr_type_intrinsic: + return lower_intrinsic(b, nir_instr_as_intrinsic(instr)); + default: + return false; + } +} + +/** + * Lower bindful image/SSBO to bindless + */ +bool +ir3_nir_lower_io_to_bindless(nir_shader *shader) +{ + /* Note: We don't currently support API level bindless, as we assume we + * can remap bindful images/SSBOs to bindless while controlling the entire + * descriptor set space. + * + * If we needed to support API level bindless, we could probably just remap + * bindful ops to a range of the descriptor set space that does not conflict + * with what we advertise for bindless descriptors? But I'm not sure that + * ARB_bindless_texture is of too much value to care about, especially for + * GLES + */ + assert(!shader->info.uses_bindless); + + return nir_shader_instructions_pass(shader, lower_instr, nir_metadata_none, NULL); +} diff --git a/src/gallium/drivers/freedreno/ir3/ir3_descriptor.h b/src/gallium/drivers/freedreno/ir3/ir3_descriptor.h new file mode 100644 index 00000000000..fa14f855134 --- /dev/null +++ b/src/gallium/drivers/freedreno/ir3/ir3_descriptor.h @@ -0,0 +1,70 @@ +/* + * Copyright © 2022 Google, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef IR3_DESCRIPTOR_H_ +#define IR3_DESCRIPTOR_H_ + +#include "ir3/ir3_shader.h" + +/* + * When using bindless descriptor sets for image/SSBO (and fb-read) state, + * since the descriptor sets are large, layout the descriptor set with the + * first IR3_BINDLESS_SSBO_COUNT slots for SSBOs followed by + * IR3_BINDLESS_IMAGE_COUNT slots for images. (For fragment shaders, the + * last image slot is reserved for fb-read tex descriptor.) + * + * Note that these limits are more or less arbitrary. But the enable_mask + * in fd_shaderbuf_stateobj / fd_shaderimg_stateobj would need to be more + * than uint32_t to support more than 32. + */ + +#define IR3_BINDLESS_SSBO_OFFSET 0 +#define IR3_BINDLESS_SSBO_COUNT 32 +#define IR3_BINDLESS_IMAGE_OFFSET IR3_BINDLESS_SSBO_COUNT +#define IR3_BINDLESS_IMAGE_COUNT 32 +#define IR3_BINDLESS_DESC_COUNT (IR3_BINDLESS_IMAGE_OFFSET + IR3_BINDLESS_IMAGE_COUNT) + +/** + * When using bindless descriptor sets for IBO/etc, each shader stage gets + * it's own descriptor set, avoiding the need to merge image/ssbo state + * across shader stages. + */ +static inline unsigned +ir3_shader_descriptor_set(enum pipe_shader_type shader) +{ + switch (shader) { + case PIPE_SHADER_VERTEX: return 0; + case PIPE_SHADER_TESS_CTRL: return 1; + case PIPE_SHADER_TESS_EVAL: return 2; + case PIPE_SHADER_GEOMETRY: return 3; + case PIPE_SHADER_FRAGMENT: return 4; + case PIPE_SHADER_COMPUTE: return 0; + default: + unreachable("bad shader stage"); + return ~0; + } +} + +bool ir3_nir_lower_io_to_bindless(nir_shader *shader); + +#endif /* IR3_DESCRIPTOR_H_ */ diff --git a/src/gallium/drivers/freedreno/meson.build b/src/gallium/drivers/freedreno/meson.build index a5bb13ca692..d2c33d6649f 100644 --- a/src/gallium/drivers/freedreno/meson.build +++ b/src/gallium/drivers/freedreno/meson.build @@ -217,6 +217,8 @@ files_libfreedreno = files( 'ir3/ir3_cache.c', 'ir3/ir3_cache.h', 'ir3/ir3_const.h', + 'ir3/ir3_descriptor.c', + 'ir3/ir3_descriptor.h', 'ir3/ir3_gallium.c', 'ir3/ir3_gallium.h', )