diff --git a/src/gallium/drivers/panfrost/meson.build b/src/gallium/drivers/panfrost/meson.build index 1debe17f49b..dd8ff95f982 100644 --- a/src/gallium/drivers/panfrost/meson.build +++ b/src/gallium/drivers/panfrost/meson.build @@ -16,6 +16,7 @@ files_panfrost = files( 'pan_resource.c', 'pan_resource.h', 'pan_context.c', + 'pan_blend_cso.c', 'pan_blit.c', 'pan_job.c', 'pan_shader.c', @@ -47,6 +48,7 @@ libpanfrost_versions = [] foreach ver : panfrost_versions files_panfrost_vx = [ 'pan_cmdstream.c', + 'pan_blend_cso.c', 'pan_fb_preload.c', 'pan_precomp.c', pan_packers, diff --git a/src/gallium/drivers/panfrost/pan_blend_cso.c b/src/gallium/drivers/panfrost/pan_blend_cso.c new file mode 100644 index 00000000000..e55d64c87dc --- /dev/null +++ b/src/gallium/drivers/panfrost/pan_blend_cso.c @@ -0,0 +1,161 @@ +/* + * Copyright (C) 2018 Alyssa Rosenzweig + * Copyright (C) 2019-2025 Collabora, Ltd. + * SPDX-License-Identifier: MIT + */ + +#include "pan_blend_cso.h" + +#include "compiler/nir/nir.h" +#include "compiler/nir/nir_builder.h" +#include "pan_shader.h" +#include "panfrost/util/pan_lower_framebuffer.h" + +#ifndef PAN_ARCH + +DERIVE_HASH_TABLE(pan_blend_shader_key); + +void +pan_blend_shader_cache_init(struct pan_blend_shader_cache *cache, + unsigned gpu_id) +{ + cache->gpu_id = gpu_id; + cache->shaders = pan_blend_shader_key_table_create(NULL); + pthread_mutex_init(&cache->lock, NULL); +} + +void +pan_blend_shader_cache_cleanup(struct pan_blend_shader_cache *cache) +{ + _mesa_hash_table_destroy(cache->shaders, NULL); + pthread_mutex_destroy(&cache->lock); +} + +#else /* PAN_ARCH */ + +static bool +pan_inline_blend_constants(nir_builder *b, nir_intrinsic_instr *intr, + void *data) +{ + if (intr->intrinsic != nir_intrinsic_load_blend_const_color_rgba) + return false; + + float *floats = data; + const nir_const_value constants[4] = { + nir_const_value_for_float(floats[0], 32), + nir_const_value_for_float(floats[1], 32), + nir_const_value_for_float(floats[2], 32), + nir_const_value_for_float(floats[3], 32)}; + + b->cursor = nir_after_instr(&intr->instr); + nir_def *constant = nir_build_imm(b, 4, 32, constants); + nir_def_replace(&intr->def, constant); + return true; +} + +struct pan_blend_shader_variant * +GENX(pan_blend_get_shader_locked)(struct pan_blend_shader_cache *cache, + const struct pan_blend_state *state, + nir_alu_type src0_type, + nir_alu_type src1_type, unsigned rt) +{ + struct pan_blend_shader_key key = { + .format = state->rts[rt].format, + .src0_type = src0_type, + .src1_type = src1_type, + .rt = rt, + .has_constants = pan_blend_constant_mask(state->rts[rt].equation) != 0, + .logicop_enable = state->logicop_enable, + .logicop_func = state->logicop_func, + .nr_samples = state->rts[rt].nr_samples, + .equation = state->rts[rt].equation, + .alpha_to_one = state->alpha_to_one, + }; + /* Blend shaders should only be used for blending on Bifrost onwards */ + assert(PAN_ARCH <= 5 || state->logicop_enable || state->alpha_to_one || + !pan_blend_is_opaque(state->rts[rt].equation)); + assert(state->rts[rt].equation.color_mask != 0); + + struct hash_entry *he = + _mesa_hash_table_search(cache->shaders, &key); + struct pan_blend_shader *shader = he ? he->data : NULL; + + if (!shader) { + shader = rzalloc(cache->shaders, struct pan_blend_shader); + shader->key = key; + list_inithead(&shader->variants); + _mesa_hash_table_insert(cache->shaders, &shader->key, shader); + } + + list_for_each_entry(struct pan_blend_shader_variant, iter, &shader->variants, + node) { + if (!key.has_constants || + !memcmp(iter->constants, state->constants, sizeof(iter->constants))) { + return iter; + } + } + + struct pan_blend_shader_variant *variant = NULL; + + if (shader->nvariants < PAN_BLEND_SHADER_MAX_VARIANTS) { + variant = rzalloc(shader, struct pan_blend_shader_variant); + util_dynarray_init(&variant->binary, variant); + list_add(&variant->node, &shader->variants); + shader->nvariants++; + } else { + variant = list_last_entry(&shader->variants, + struct pan_blend_shader_variant, node); + list_del(&variant->node); + list_add(&variant->node, &shader->variants); + util_dynarray_clear(&variant->binary); + } + + memcpy(variant->constants, state->constants, sizeof(variant->constants)); + + nir_shader *nir = + GENX(pan_blend_create_shader)(state, src0_type, src1_type, rt); + + nir_shader_intrinsics_pass(nir, pan_inline_blend_constants, + nir_metadata_control_flow, + (void *)state->constants); + + /* Compile the NIR shader */ + struct panfrost_compile_inputs inputs = { + .gpu_id = cache->gpu_id, + .is_blend = true, + .blend.nr_samples = key.nr_samples, + }; + + enum pipe_format rt_formats[8] = {0}; + rt_formats[rt] = key.format; + +#if PAN_ARCH >= 6 + inputs.blend.bifrost_blend_desc = + GENX(pan_blend_get_internal_desc)(key.format, key.rt, 0, false); +#endif + + struct pan_shader_info info; + pan_shader_preprocess(nir, inputs.gpu_id); + +#if PAN_ARCH >= 6 + NIR_PASS(_, nir, GENX(pan_inline_rt_conversion), rt_formats); +#else + NIR_PASS(_, nir, pan_lower_framebuffer, rt_formats, + pan_raw_format_mask_midgard(rt_formats), MAX2(key.nr_samples, 1), + cache->gpu_id < 0x700); +#endif + + GENX(pan_shader_compile)(nir, &inputs, &variant->binary, &info); + + variant->work_reg_count = info.work_reg_count; + +#if PAN_ARCH <= 5 + variant->first_tag = info.midgard.first_tag; +#endif + + ralloc_free(nir); + + return variant; +} + +#endif /* PAN_ARCH */ diff --git a/src/gallium/drivers/panfrost/pan_blend_cso.h b/src/gallium/drivers/panfrost/pan_blend_cso.h index 83ba3f80683..f093b1d377f 100644 --- a/src/gallium/drivers/panfrost/pan_blend_cso.h +++ b/src/gallium/drivers/panfrost/pan_blend_cso.h @@ -30,10 +30,11 @@ #include "util/hash_table.h" #include "nir.h" +#include "pipe/p_state.h" #include "pan_blend.h" -#include "pan_job.h" struct panfrost_bo; +struct panfrost_batch; struct pan_blend_info { unsigned constant_mask : 4; @@ -58,7 +59,45 @@ struct panfrost_blend_state { unsigned enabled_mask : PIPE_MAX_COLOR_BUFS; }; +struct pan_blend_shader_cache { + unsigned gpu_id; + struct hash_table *shaders; + pthread_mutex_t lock; +}; + +struct pan_blend_shader_variant { + struct list_head node; + float constants[4]; + struct util_dynarray binary; + unsigned first_tag; + unsigned work_reg_count; +}; + +#define PAN_BLEND_SHADER_MAX_VARIANTS 32 + +struct pan_blend_shader { + struct pan_blend_shader_key key; + unsigned nvariants; + struct list_head variants; +}; + uint64_t panfrost_get_blend(struct panfrost_batch *batch, unsigned rt, struct panfrost_bo **bo, unsigned *shader_offset); +void pan_blend_shader_cache_init(struct pan_blend_shader_cache *cache, + unsigned gpu_id); + +void pan_blend_shader_cache_cleanup(struct pan_blend_shader_cache *cache); + +#ifdef PAN_ARCH + +/* Take blend_shaders.lock before calling this function and release it when + * you're done with the shader variant object. + */ +struct pan_blend_shader_variant *GENX(pan_blend_get_shader_locked)( + struct pan_blend_shader_cache *cache, const struct pan_blend_state *state, + nir_alu_type src0_type, nir_alu_type src1_type, unsigned rt); + +#endif + #endif diff --git a/src/gallium/drivers/panfrost/pan_device.h b/src/gallium/drivers/panfrost/pan_device.h index c08afb01ffc..c1ff2cb1a86 100644 --- a/src/gallium/drivers/panfrost/pan_device.h +++ b/src/gallium/drivers/panfrost/pan_device.h @@ -39,7 +39,7 @@ #include "util/u_dynarray.h" #include "panfrost/util/pan_ir.h" -#include "pan_blend.h" +#include "pan_blend_cso.h" #include "pan_fb_preload.h" #include "pan_indirect_dispatch.h" #include "pan_pool.h" diff --git a/src/gallium/drivers/panfrost/pan_fb_preload.c b/src/gallium/drivers/panfrost/pan_fb_preload.c index 874bf1210a5..1f9a95616c3 100644 --- a/src/gallium/drivers/panfrost/pan_fb_preload.c +++ b/src/gallium/drivers/panfrost/pan_fb_preload.c @@ -29,7 +29,7 @@ #include #include "compiler/nir/nir_builder.h" #include "util/u_math.h" -#include "pan_blend.h" +#include "pan_blend_cso.h" #include "pan_desc.h" #include "pan_encoder.h" #include "pan_fb_preload.h" diff --git a/src/panfrost/lib/pan_blend.c b/src/panfrost/lib/pan_blend.c index e3f92f4a96c..b4fc947e646 100644 --- a/src/panfrost/lib/pan_blend.c +++ b/src/panfrost/lib/pan_blend.c @@ -33,7 +33,6 @@ #include "compiler/nir/nir_builder.h" #include "compiler/nir/nir_conversion_builder.h" #include "compiler/nir/nir_lower_blend.h" -#include "panfrost/util/pan_lower_framebuffer.h" #include "util/format/u_format.h" #include "pan_texture.h" @@ -467,24 +466,6 @@ pan_pack_blend(const struct pan_blend_equation equation) return out.opaque[0]; } -DERIVE_HASH_TABLE(pan_blend_shader_key); - -void -pan_blend_shader_cache_init(struct pan_blend_shader_cache *cache, - unsigned gpu_id) -{ - cache->gpu_id = gpu_id; - cache->shaders = pan_blend_shader_key_table_create(NULL); - pthread_mutex_init(&cache->lock, NULL); -} - -void -pan_blend_shader_cache_cleanup(struct pan_blend_shader_cache *cache) -{ - _mesa_hash_table_destroy(cache->shaders, NULL); - pthread_mutex_destroy(&cache->lock); -} - #else /* ifndef PAN_ARCH */ static const char * @@ -593,25 +574,6 @@ get_equation_str(const struct pan_blend_rt_state *rt_state, char *str, } } -static bool -pan_inline_blend_constants(nir_builder *b, nir_intrinsic_instr *intr, - void *data) -{ - if (intr->intrinsic != nir_intrinsic_load_blend_const_color_rgba) - return false; - - float *floats = data; - const nir_const_value constants[4] = { - nir_const_value_for_float(floats[0], 32), - nir_const_value_for_float(floats[1], 32), - nir_const_value_for_float(floats[2], 32), - nir_const_value_for_float(floats[3], 32)}; - - b->cursor = nir_after_instr(&intr->instr); - nir_def *constant = nir_build_imm(b, 4, 32, constants); - nir_def_replace(&intr->def, constant); - return true; -} nir_shader * GENX(pan_blend_create_shader)(const struct pan_blend_state *state, @@ -797,108 +759,4 @@ GENX(pan_inline_rt_conversion)(nir_shader *s, enum pipe_format *formats) } #endif -struct pan_blend_shader_variant * -GENX(pan_blend_get_shader_locked)(struct pan_blend_shader_cache *cache, - const struct pan_blend_state *state, - nir_alu_type src0_type, - nir_alu_type src1_type, unsigned rt) -{ - struct pan_blend_shader_key key = { - .format = state->rts[rt].format, - .src0_type = src0_type, - .src1_type = src1_type, - .rt = rt, - .has_constants = pan_blend_constant_mask(state->rts[rt].equation) != 0, - .logicop_enable = state->logicop_enable, - .logicop_func = state->logicop_func, - .nr_samples = state->rts[rt].nr_samples, - .equation = state->rts[rt].equation, - .alpha_to_one = state->alpha_to_one, - }; - /* Blend shaders should only be used for blending on Bifrost onwards */ - assert(PAN_ARCH <= 5 || state->logicop_enable || state->alpha_to_one || - !pan_blend_is_opaque(state->rts[rt].equation)); - assert(state->rts[rt].equation.color_mask != 0); - - struct hash_entry *he = - _mesa_hash_table_search(cache->shaders, &key); - struct pan_blend_shader *shader = he ? he->data : NULL; - - if (!shader) { - shader = rzalloc(cache->shaders, struct pan_blend_shader); - shader->key = key; - list_inithead(&shader->variants); - _mesa_hash_table_insert(cache->shaders, &shader->key, shader); - } - - list_for_each_entry(struct pan_blend_shader_variant, iter, &shader->variants, - node) { - if (!key.has_constants || - !memcmp(iter->constants, state->constants, sizeof(iter->constants))) { - return iter; - } - } - - struct pan_blend_shader_variant *variant = NULL; - - if (shader->nvariants < PAN_BLEND_SHADER_MAX_VARIANTS) { - variant = rzalloc(shader, struct pan_blend_shader_variant); - util_dynarray_init(&variant->binary, variant); - list_add(&variant->node, &shader->variants); - shader->nvariants++; - } else { - variant = list_last_entry(&shader->variants, - struct pan_blend_shader_variant, node); - list_del(&variant->node); - list_add(&variant->node, &shader->variants); - util_dynarray_clear(&variant->binary); - } - - memcpy(variant->constants, state->constants, sizeof(variant->constants)); - - nir_shader *nir = - GENX(pan_blend_create_shader)(state, src0_type, src1_type, rt); - - nir_shader_intrinsics_pass(nir, pan_inline_blend_constants, - nir_metadata_control_flow, - (void *)state->constants); - - /* Compile the NIR shader */ - struct panfrost_compile_inputs inputs = { - .gpu_id = cache->gpu_id, - .is_blend = true, - .blend.nr_samples = key.nr_samples, - }; - - enum pipe_format rt_formats[8] = {0}; - rt_formats[rt] = key.format; - -#if PAN_ARCH >= 6 - inputs.blend.bifrost_blend_desc = - GENX(pan_blend_get_internal_desc)(key.format, key.rt, 0, false); -#endif - - struct pan_shader_info info; - pan_shader_preprocess(nir, inputs.gpu_id); - -#if PAN_ARCH >= 6 - NIR_PASS(_, nir, GENX(pan_inline_rt_conversion), rt_formats); -#else - NIR_PASS(_, nir, pan_lower_framebuffer, rt_formats, - pan_raw_format_mask_midgard(rt_formats), MAX2(key.nr_samples, 1), - cache->gpu_id < 0x700); -#endif - - GENX(pan_shader_compile)(nir, &inputs, &variant->binary, &info); - - variant->work_reg_count = info.work_reg_count; - -#if PAN_ARCH <= 5 - variant->first_tag = info.midgard.first_tag; -#endif - - ralloc_free(nir); - - return variant; -} #endif /* ifndef PAN_ARCH */ diff --git a/src/panfrost/lib/pan_blend.h b/src/panfrost/lib/pan_blend.h index 2882ad888d7..5aa1c78002f 100644 --- a/src/panfrost/lib/pan_blend.h +++ b/src/panfrost/lib/pan_blend.h @@ -30,18 +30,11 @@ #include "compiler/nir/nir.h" #include "util/blend.h" #include "util/format/u_format.h" -#include "util/u_dynarray.h" #include "panfrost/util/pan_ir.h" struct MALI_BLEND_EQUATION; -struct pan_blend_shader_cache { - unsigned gpu_id; - struct hash_table *shaders; - pthread_mutex_t lock; -}; - struct pan_blend_equation { unsigned blend_enable : 1; enum pipe_blend_func rgb_func : 3; @@ -86,22 +79,6 @@ struct pan_blend_shader_key { struct pan_blend_equation equation; }; -struct pan_blend_shader_variant { - struct list_head node; - float constants[4]; - struct util_dynarray binary; - unsigned first_tag; - unsigned work_reg_count; -}; - -#define PAN_BLEND_SHADER_MAX_VARIANTS 32 - -struct pan_blend_shader { - struct pan_blend_shader_key key; - unsigned nvariants; - struct list_head variants; -}; - bool pan_blend_reads_dest(const struct pan_blend_equation eq); bool pan_blend_can_fixed_function(const struct pan_blend_equation equation, @@ -150,11 +127,6 @@ void pan_blend_to_fixed_function_equation(const struct pan_blend_equation eq, uint32_t pan_pack_blend(const struct pan_blend_equation equation); -void pan_blend_shader_cache_init(struct pan_blend_shader_cache *cache, - unsigned gpu_id); - -void pan_blend_shader_cache_cleanup(struct pan_blend_shader_cache *cache); - #ifdef PAN_ARCH nir_shader *GENX(pan_blend_create_shader)(const struct pan_blend_state *state, @@ -168,12 +140,6 @@ uint64_t GENX(pan_blend_get_internal_desc)(enum pipe_format fmt, unsigned rt, bool GENX(pan_inline_rt_conversion)(nir_shader *s, enum pipe_format *formats); #endif -/* Take blend_shaders.lock before calling this function and release it when - * you're done with the shader variant object. - */ -struct pan_blend_shader_variant *GENX(pan_blend_get_shader_locked)( - struct pan_blend_shader_cache *cache, const struct pan_blend_state *state, - nir_alu_type src0_type, nir_alu_type src1_type, unsigned rt); #endif #endif