diff --git a/src/amd/common/nir/ac_nir.h b/src/amd/common/nir/ac_nir.h index 70bd5c583c6..5f75f84e948 100644 --- a/src/amd/common/nir/ac_nir.h +++ b/src/amd/common/nir/ac_nir.h @@ -281,6 +281,12 @@ typedef struct { int force_front_face; /* 0 -> keep, 1 -> set to true, -1 -> set to false */ unsigned ps_iter_samples; + /* fbfetch_output */ + bool fbfetch_is_1D; + bool fbfetch_layered; + bool fbfetch_msaa; + bool fbfetch_apply_fmask; + /* Outputs. */ bool clamp_color; /* GL only */ bool alpha_test_alpha_to_one; /* GL only, this only affects alpha test */ diff --git a/src/amd/common/nir/ac_nir_lower_ps_early.c b/src/amd/common/nir/ac_nir_lower_ps_early.c index ebb79932edb..ceea3749a45 100644 --- a/src/amd/common/nir/ac_nir_lower_ps_early.c +++ b/src/amd/common/nir/ac_nir_lower_ps_early.c @@ -314,6 +314,59 @@ lower_load_barycentric_at_offset(nir_builder *b, nir_def *offset, enum glsl_inte return nir_vec2(b, offset_i, offset_j); } +static nir_def * +fbfetch_color_buffer0(nir_builder *b, lower_ps_early_state *s) +{ + nir_def *zero = nir_imm_zero(b, 1, 32); + nir_def *undef = nir_undef(b, 1, 32); + + unsigned chan = 0; + nir_def *coord_vec[4] = {undef, undef, undef, undef}; + nir_def *pixel_coord = nir_u2u32(b, nir_load_pixel_coord(b)); + + coord_vec[chan++] = nir_channel(b, pixel_coord, 0); + + if (!s->options->fbfetch_is_1D) + coord_vec[chan++] = nir_channel(b, pixel_coord, 1); + + /* Get the current render target layer index. */ + if (s->options->fbfetch_layered) + coord_vec[chan++] = nir_load_layer_id(b); + + nir_def *coords = nir_vec(b, coord_vec, 4); + + enum glsl_sampler_dim dim; + if (s->options->fbfetch_msaa) + dim = GLSL_SAMPLER_DIM_MS; + else if (s->options->fbfetch_is_1D) + dim = GLSL_SAMPLER_DIM_1D; + else + dim = GLSL_SAMPLER_DIM_2D; + + nir_def *sample_id; + if (s->options->fbfetch_msaa) { + sample_id = nir_load_sample_id(b); + + if (s->options->fbfetch_apply_fmask) { + nir_def *fmask = + nir_bindless_image_fragment_mask_load_amd( + b, nir_load_fbfetch_image_fmask_desc_amd(b), coords, + .image_dim = dim, + .image_array = s->options->fbfetch_layered, + .access = ACCESS_CAN_REORDER); + sample_id = nir_ubfe(b, fmask, nir_ishl_imm(b, sample_id, 2), nir_imm_int(b, 3)); + } + } else { + sample_id = zero; + } + + return nir_bindless_image_load(b, 4, 32, nir_load_fbfetch_image_desc_amd(b), coords, sample_id, + zero, + .image_dim = dim, + .image_array = s->options->fbfetch_layered, + .access = ACCESS_CAN_REORDER); +} + static bool lower_ps_intrinsic(nir_builder *b, nir_instr *instr, void *state) { @@ -401,6 +454,12 @@ lower_ps_intrinsic(nir_builder *b, nir_instr *instr, void *state) } return true; } + case nir_intrinsic_load_output: + if (nir_intrinsic_io_semantics(intrin).fb_fetch_output) { + nir_def_replace(&intrin->def, fbfetch_color_buffer0(b, s)); + return true; + } + break; default: break; } diff --git a/src/compiler/nir/nir_divergence_analysis.c b/src/compiler/nir/nir_divergence_analysis.c index 2d9853148fa..c34f5722998 100644 --- a/src/compiler/nir/nir_divergence_analysis.c +++ b/src/compiler/nir/nir_divergence_analysis.c @@ -341,6 +341,8 @@ visit_intrinsic(nir_intrinsic_instr *instr, struct divergence_state *state) case nir_intrinsic_load_primitive_location_ir3: case nir_intrinsic_preamble_start_ir3: case nir_intrinsic_optimization_barrier_sgpr_amd: + case nir_intrinsic_load_fbfetch_image_fmask_desc_amd: + case nir_intrinsic_load_fbfetch_image_desc_amd: case nir_intrinsic_load_printf_buffer_address: case nir_intrinsic_load_printf_buffer_size: case nir_intrinsic_load_printf_base_identifier: diff --git a/src/compiler/nir/nir_intrinsics.py b/src/compiler/nir/nir_intrinsics.py index 51ec07f404b..1f9808ff5b5 100644 --- a/src/compiler/nir/nir_intrinsics.py +++ b/src/compiler/nir/nir_intrinsics.py @@ -1915,6 +1915,11 @@ intrinsic("sleep_amd", indices=[BASE]) # s_nop BASE (sleep for BASE+1 cycles, BASE must be in [0, 15]). intrinsic("nop_amd", indices=[BASE]) +# Return the FMASK descriptor of color buffer 0. +system_value("fbfetch_image_fmask_desc_amd", 8) +# Return the image descriptor of color buffer 0. +system_value("fbfetch_image_desc_amd", 8) + system_value("ray_tracing_stack_base_lvp", 1) system_value("shader_call_data_offset_lvp", 1) diff --git a/src/gallium/drivers/radeonsi/si_nir_lower_abi.c b/src/gallium/drivers/radeonsi/si_nir_lower_abi.c index 0ce7cdca73c..bc180e4f40b 100644 --- a/src/gallium/drivers/radeonsi/si_nir_lower_abi.c +++ b/src/gallium/drivers/radeonsi/si_nir_lower_abi.c @@ -60,72 +60,6 @@ static nir_def *build_attr_ring_desc(nir_builder *b, struct si_shader *shader, return nir_vec(b, comp, 4); } -static nir_def * -fetch_framebuffer(nir_builder *b, struct si_shader_args *args, - struct si_shader_selector *sel, union si_shader_key *key) -{ - /* Load the image descriptor. */ - STATIC_ASSERT(SI_PS_IMAGE_COLORBUF0 % 2 == 0); - STATIC_ASSERT(SI_PS_IMAGE_COLORBUF0_FMASK % 2 == 0); - - nir_def *zero = nir_imm_zero(b, 1, 32); - nir_def *undef = nir_undef(b, 1, 32); - - unsigned chan = 0; - nir_def *vec[4] = {undef, undef, undef, undef}; - - vec[chan++] = ac_nir_unpack_arg(b, &args->ac, args->ac.pos_fixed_pt, 0, 16); - - if (!key->ps.mono.fbfetch_is_1D) - vec[chan++] = ac_nir_unpack_arg(b, &args->ac, args->ac.pos_fixed_pt, 16, 16); - - /* Get the current render target layer index. */ - if (key->ps.mono.fbfetch_layered) - vec[chan++] = ac_nir_unpack_arg(b, &args->ac, args->ac.ancillary, 16, 11); - - nir_def *coords = nir_vec(b, vec, 4); - - enum glsl_sampler_dim dim; - if (key->ps.mono.fbfetch_msaa) - dim = GLSL_SAMPLER_DIM_MS; - else if (key->ps.mono.fbfetch_is_1D) - dim = GLSL_SAMPLER_DIM_1D; - else - dim = GLSL_SAMPLER_DIM_2D; - - nir_def *sample_id; - if (key->ps.mono.fbfetch_msaa) { - sample_id = ac_nir_unpack_arg(b, &args->ac, args->ac.ancillary, 8, 4); - - if (sel->screen->info.gfx_level < GFX11 && - !(sel->screen->debug_flags & DBG(NO_FMASK))) { - nir_def *desc = - si_nir_load_internal_binding(b, args, SI_PS_IMAGE_COLORBUF0_FMASK, 8); - - nir_def *fmask = - nir_bindless_image_fragment_mask_load_amd( - b, desc, coords, - .image_dim = dim, - .image_array = key->ps.mono.fbfetch_layered, - .access = ACCESS_CAN_REORDER); - - nir_def *offset = nir_ishl_imm(b, sample_id, 2); - /* 3 for EQAA handling, see lower_image_to_fragment_mask_load() */ - nir_def *width = nir_imm_int(b, 3); - sample_id = nir_ubfe(b, fmask, offset, width); - } - } else { - sample_id = zero; - } - - nir_def *desc = si_nir_load_internal_binding(b, args, SI_PS_IMAGE_COLORBUF0, 8); - - return nir_bindless_image_load(b, 4, 32, desc, coords, sample_id, zero, - .image_dim = dim, - .image_array = key->ps.mono.fbfetch_layered, - .access = ACCESS_CAN_REORDER); -} - static nir_def *build_tess_ring_desc(nir_builder *b, struct si_screen *screen, struct si_shader_args *args) { @@ -539,18 +473,6 @@ static bool lower_intrinsic(nir_builder *b, nir_instr *instr, struct lower_abi_s replacement = nir_load_ubo(b, 2, 32, buf, offset, .range = ~0); break; } - case nir_intrinsic_load_output: { - nir_io_semantics sem = nir_intrinsic_io_semantics(intrin); - - /* not fbfetch */ - if (!(stage == MESA_SHADER_FRAGMENT && sem.fb_fetch_output)) - return false; - - /* Ignore src0, because KHR_blend_func_extended disallows multiple render targets. */ - - replacement = fetch_framebuffer(b, args, sel, key); - break; - } case nir_intrinsic_load_ring_tess_factors_amd: { assert(s->tess_offchip_ring); nir_def *addr = nir_channel(b, s->tess_offchip_ring, 0); @@ -677,6 +599,14 @@ static bool lower_intrinsic(nir_builder *b, nir_instr *instr, struct lower_abi_s } break; } + case nir_intrinsic_load_fbfetch_image_fmask_desc_amd: + STATIC_ASSERT(SI_PS_IMAGE_COLORBUF0_FMASK % 2 == 0); + replacement = si_nir_load_internal_binding(b, args, SI_PS_IMAGE_COLORBUF0_FMASK, 8); + break; + case nir_intrinsic_load_fbfetch_image_desc_amd: + STATIC_ASSERT(SI_PS_IMAGE_COLORBUF0 % 2 == 0); + replacement = si_nir_load_internal_binding(b, args, SI_PS_IMAGE_COLORBUF0, 8); + break; default: return false; } diff --git a/src/gallium/drivers/radeonsi/si_shader.c b/src/gallium/drivers/radeonsi/si_shader.c index 5d2ba04ed03..bb444f60be6 100644 --- a/src/gallium/drivers/radeonsi/si_shader.c +++ b/src/gallium/drivers/radeonsi/si_shader.c @@ -2533,6 +2533,12 @@ static struct nir_shader *si_get_nir_shader(struct si_shader *shader, struct si_ .force_front_face = key->ps.opt.force_front_face_input, .ps_iter_samples = 1 << key->ps.part.prolog.samplemask_log_ps_iter, + .fbfetch_is_1D = key->ps.mono.fbfetch_is_1D, + .fbfetch_layered = key->ps.mono.fbfetch_layered, + .fbfetch_msaa = key->ps.mono.fbfetch_msaa, + .fbfetch_apply_fmask = sel->screen->info.gfx_level < GFX11 && + !(sel->screen->debug_flags & DBG(NO_FMASK)), + .clamp_color = key->ps.part.epilog.clamp_color, .alpha_test_alpha_to_one = key->ps.part.epilog.alpha_to_one, .alpha_func = key->ps.part.epilog.alpha_func,