From ae78e8d198af48d0c159788b1d6512c0e9acd4de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Ol=C5=A1=C3=A1k?= Date: Mon, 2 Jun 2025 13:07:09 -0400 Subject: [PATCH] ac/nir: handle VARYING_SLOT_VARn_16BIT the same as other slots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit They are the same as regular VARn. Reviewed-by: Samuel Pitoiset Reviewed-by: Timur Kristóf Part-of: --- .../common/nir/ac_nir_create_gs_copy_shader.c | 34 +-- src/amd/common/nir/ac_nir_helpers.h | 19 +- src/amd/common/nir/ac_nir_lower_legacy_gs.c | 41 +--- src/amd/common/nir/ac_nir_lower_ngg.c | 40 +--- src/amd/common/nir/ac_nir_lower_ngg_gs.c | 51 +--- src/amd/common/nir/ac_nir_prerast_utils.c | 219 ++---------------- src/util/bitscan.h | 10 + 7 files changed, 47 insertions(+), 367 deletions(-) diff --git a/src/amd/common/nir/ac_nir_create_gs_copy_shader.c b/src/amd/common/nir/ac_nir_create_gs_copy_shader.c index 12d5d78d5c9..a7b86d00056 100644 --- a/src/amd/common/nir/ac_nir_create_gs_copy_shader.c +++ b/src/amd/common/nir/ac_nir_create_gs_copy_shader.c @@ -39,7 +39,8 @@ ac_nir_create_gs_copy_shader(const nir_shader *gs_nir, ac_nir_lower_legacy_gs_op unsigned offset = 0; - u_foreach_bit64 (i, gs_nir->info.outputs_written) { + u_foreach_bit64_two_masks(i, gs_nir->info.outputs_written, + VARYING_SLOT_VAR0_16BIT, gs_nir->info.outputs_written_16bit) { u_foreach_bit (j, out->infos[i].components_mask) { if (((out->infos[i].stream >> (j * 2)) & 0x3) != stream) continue; @@ -58,37 +59,6 @@ ac_nir_create_gs_copy_shader(const nir_shader *gs_nir, ac_nir_lower_legacy_gs_op } } - u_foreach_bit (i, gs_nir->info.outputs_written_16bit) { - unsigned mask = out->infos_16bit_lo[i].components_mask | - out->infos_16bit_hi[i].components_mask; - - u_foreach_bit (j, mask) { - bool has_lo_16bit = ((out->infos_16bit_lo[i].stream >> (j * 2)) & 0x3) == stream; - bool has_hi_16bit = ((out->infos_16bit_hi[i].stream >> (j * 2)) & 0x3) == stream; - - if (!has_lo_16bit && !has_hi_16bit) - continue; - - nir_def *load_val; - - if (ac_nir_is_const_output(out, VARYING_SLOT_VAR0_16BIT + i, j)) { - load_val = ac_nir_get_const_output(&b, out, i, j); - } else { - unsigned base = offset * gs_nir->info.gs.vertices_out * 16; - load_val = nir_load_buffer_amd(&b, 1, 32, gsvs_ring, vtx_offset, zero, zero, - .base = base, - .access = ACCESS_COHERENT | ACCESS_NON_TEMPORAL); - offset += 4; - } - - if (has_lo_16bit) - out->outputs_16bit_lo[i][j] = nir_unpack_32_2x16_split_x(&b, load_val); - - if (has_hi_16bit) - out->outputs_16bit_hi[i][j] = nir_unpack_32_2x16_split_y(&b, load_val); - } - } - if (stream_id) ac_nir_emit_legacy_streamout(&b, stream, info, out); diff --git a/src/amd/common/nir/ac_nir_helpers.h b/src/amd/common/nir/ac_nir_helpers.h index aae689ce62b..692af4477b5 100644 --- a/src/amd/common/nir/ac_nir_helpers.h +++ b/src/amd/common/nir/ac_nir_helpers.h @@ -78,21 +78,12 @@ typedef struct typedef struct { - nir_def *outputs[VARYING_SLOT_MAX][4]; - nir_def *outputs_16bit_lo[16][4]; - nir_def *outputs_16bit_hi[16][4]; + /* Low and high 16 bits are packed into 32 bits. */ + nir_def *outputs[NUM_TOTAL_VARYING_SLOTS][4]; + uint32_t const_values[NUM_TOTAL_VARYING_SLOTS][4]; - /* Low and high 16 bits are packed. */ - uint32_t const_values[VARYING_SLOT_MAX][4]; - uint32_t const_values_16bit[16][4]; - - nir_alu_type types[VARYING_SLOT_MAX][4]; - nir_alu_type types_16bit_lo[16][4]; - nir_alu_type types_16bit_hi[16][4]; - - ac_nir_prerast_per_output_info infos[VARYING_SLOT_MAX]; - ac_nir_prerast_per_output_info infos_16bit_lo[16]; - ac_nir_prerast_per_output_info infos_16bit_hi[16]; + nir_alu_type types[NUM_TOTAL_VARYING_SLOTS][4]; + ac_nir_prerast_per_output_info infos[NUM_TOTAL_VARYING_SLOTS]; /* The size of all components, packed. */ uint16_t total_packed_gs_out_size; diff --git a/src/amd/common/nir/ac_nir_lower_legacy_gs.c b/src/amd/common/nir/ac_nir_lower_legacy_gs.c index 8379fbdfec8..990d0b1c6be 100644 --- a/src/amd/common/nir/ac_nir_lower_legacy_gs.c +++ b/src/amd/common/nir/ac_nir_lower_legacy_gs.c @@ -40,7 +40,8 @@ lower_legacy_gs_emit_vertex_with_counter(nir_builder *b, nir_intrinsic_instr *in unsigned offset = 0; - u_foreach_bit64 (slot, b->shader->info.outputs_written) { + u_foreach_bit64_two_masks(slot, b->shader->info.outputs_written, + VARYING_SLOT_VAR0_16BIT, b->shader->info.outputs_written_16bit) { unsigned mask = ac_nir_gs_output_component_mask_with_stream(&s->out.infos[slot], stream); nir_def **output = s->out.outputs[slot]; @@ -77,44 +78,6 @@ lower_legacy_gs_emit_vertex_with_counter(nir_builder *b, nir_intrinsic_instr *in memset(s->out.outputs[slot], 0, sizeof(s->out.outputs[slot])); } - u_foreach_bit (slot, b->shader->info.outputs_written_16bit) { - const unsigned mask_lo = ac_nir_gs_output_component_mask_with_stream(s->out.infos_16bit_lo + slot, stream); - const unsigned mask_hi = ac_nir_gs_output_component_mask_with_stream(s->out.infos_16bit_hi + slot, stream); - unsigned mask = mask_lo | mask_hi; - - nir_def **output_lo = s->out.outputs_16bit_lo[slot]; - nir_def **output_hi = s->out.outputs_16bit_hi[slot]; - nir_def *undef = nir_undef(b, 1, 16); - - u_foreach_bit(c, mask) { - /* The shader hasn't written this output yet. */ - if ((!output_lo[c] && !output_hi[c]) || - ac_nir_is_const_output(&s->out, VARYING_SLOT_VAR0_16BIT + slot, c)) - continue; - - nir_def *lo = output_lo[c] ? output_lo[c] : undef; - nir_def *hi = output_hi[c] ? output_hi[c] : undef; - nir_def *store_val = nir_pack_32_2x16_split(b, lo, hi); - - unsigned base = offset * b->shader->info.gs.vertices_out; - - nir_def *voffset = nir_iadd_imm(b, vtxidx, base); - voffset = nir_ishl_imm(b, voffset, 2); - - nir_store_buffer_amd(b, store_val, - gsvs_ring, voffset, soffset, nir_imm_int(b, 0), - .access = ACCESS_COHERENT | ACCESS_NON_TEMPORAL | - ACCESS_IS_SWIZZLED_AMD, - /* For ACO to not reorder this store around EmitVertex/EndPrimitve */ - .memory_modes = nir_var_shader_out); - offset += 4; - } - - /* Clear all outputs (they are undefined after emit_vertex) */ - memset(s->out.outputs_16bit_lo[slot], 0, sizeof(s->out.outputs_16bit_lo[slot])); - memset(s->out.outputs_16bit_hi[slot], 0, sizeof(s->out.outputs_16bit_hi[slot])); - } - assert(offset / 4 < 256); s->out_info->num_components_per_stream[stream] = offset / 4; diff --git a/src/amd/common/nir/ac_nir_lower_ngg.c b/src/amd/common/nir/ac_nir_lower_ngg.c index 39aadecda51..44889b7e8d6 100644 --- a/src/amd/common/nir/ac_nir_lower_ngg.c +++ b/src/amd/common/nir/ac_nir_lower_ngg.c @@ -1311,32 +1311,23 @@ ngg_nogs_store_xfb_outputs_to_lds(nir_builder *b, lower_ngg_nogs_state *s) uint64_t xfb_outputs = 0; unsigned xfb_outputs_16bit = 0; - uint8_t xfb_mask[VARYING_SLOT_MAX] = {0}; - uint8_t xfb_mask_16bit_lo[16] = {0}; - uint8_t xfb_mask_16bit_hi[16] = {0}; + uint8_t xfb_mask[NUM_TOTAL_VARYING_SLOTS] = {0}; /* Get XFB output mask for each slot. */ for (int i = 0; i < info->output_count; i++) { nir_xfb_output_info *out = info->outputs + i; + xfb_mask[out->location] |= out->component_mask; - if (out->location < VARYING_SLOT_VAR0_16BIT) { + if (out->location < VARYING_SLOT_VAR0_16BIT) xfb_outputs |= BITFIELD64_BIT(out->location); - xfb_mask[out->location] |= out->component_mask; - } else { - unsigned index = out->location - VARYING_SLOT_VAR0_16BIT; - xfb_outputs_16bit |= BITFIELD_BIT(index); - - if (out->high_16bits) - xfb_mask_16bit_hi[index] |= out->component_mask; - else - xfb_mask_16bit_lo[index] |= out->component_mask; - } + else + xfb_outputs_16bit |= BITFIELD_BIT(out->location - VARYING_SLOT_VAR0_16BIT); } nir_def *tid = nir_load_local_invocation_index(b); nir_def *addr = pervertex_lds_addr(b, s, tid, s->pervertex_lds_bytes); - u_foreach_bit64(slot, xfb_outputs) { + u_foreach_bit64_two_masks(slot, xfb_outputs, VARYING_SLOT_VAR0_16BIT, xfb_outputs_16bit) { u_foreach_bit(c, xfb_mask[slot]) { if (!s->out.outputs[slot][c]) continue; @@ -1350,25 +1341,6 @@ ngg_nogs_store_xfb_outputs_to_lds(nir_builder *b, lower_ngg_nogs_state *s) ac_nir_store_shared_xfb(b, s->out.outputs[slot][c], addr, &s->out, slot, c); } } - - u_foreach_bit64(slot, xfb_outputs_16bit) { - unsigned mask_lo = xfb_mask_16bit_lo[slot]; - unsigned mask_hi = xfb_mask_16bit_hi[slot]; - nir_def **outputs_lo = s->out.outputs_16bit_lo[slot]; - nir_def **outputs_hi = s->out.outputs_16bit_hi[slot]; - nir_def *undef = nir_undef(b, 1, 16); - - u_foreach_bit(c, mask_lo | mask_hi) { - if (!outputs_lo[c] && !outputs_hi[c]) - continue; - - nir_def *lo = mask_lo & BITFIELD_BIT(c) ? outputs_lo[c] : undef; - nir_def *hi = mask_hi & BITFIELD_BIT(c) ? outputs_hi[c] : undef; - nir_def *store_val = nir_pack_32_2x16_split(b, lo, hi); - - ac_nir_store_shared_xfb(b, store_val, addr, &s->out, VARYING_SLOT_VAR0_16BIT + slot, c); - } - } } static void diff --git a/src/amd/common/nir/ac_nir_lower_ngg_gs.c b/src/amd/common/nir/ac_nir_lower_ngg_gs.c index 37b347b0e5f..d8b287bd483 100644 --- a/src/amd/common/nir/ac_nir_lower_ngg_gs.c +++ b/src/amd/common/nir/ac_nir_lower_ngg_gs.c @@ -146,7 +146,8 @@ lower_ngg_gs_emit_vertex_with_counter(nir_builder *b, nir_intrinsic_instr *intri /* Store generic 32-bit outputs to LDS. * In case of packed 16-bit, we assume that has been already packed into 32 bit slots by now. */ - u_foreach_bit64(slot, b->shader->info.outputs_written) { + u_foreach_bit64_two_masks(slot, b->shader->info.outputs_written, + VARYING_SLOT_VAR0_16BIT, b->shader->info.outputs_written_16bit) { unsigned mask = ac_nir_gs_output_component_mask_with_stream(&s->out.infos[slot], stream); nir_def **output = s->out.outputs[slot]; @@ -162,33 +163,6 @@ lower_ngg_gs_emit_vertex_with_counter(nir_builder *b, nir_intrinsic_instr *intri memset(s->out.outputs[slot], 0, sizeof(s->out.outputs[slot])); } - /* Store dedicated 16-bit outputs to LDS. */ - u_foreach_bit(slot, b->shader->info.outputs_written_16bit) { - const unsigned mask_lo = ac_nir_gs_output_component_mask_with_stream(s->out.infos_16bit_lo + slot, stream); - const unsigned mask_hi = ac_nir_gs_output_component_mask_with_stream(s->out.infos_16bit_hi + slot, stream); - unsigned mask = mask_lo | mask_hi; - - nir_def **output_lo = s->out.outputs_16bit_lo[slot]; - nir_def **output_hi = s->out.outputs_16bit_hi[slot]; - nir_def *undef = nir_undef(b, 1, 16); - - u_foreach_bit(c, mask) { - /* The shader hasn't written this output yet. */ - if (!output_lo[c] && !output_hi[c]) - continue; - - nir_def *lo = output_lo[c] ? output_lo[c] : undef; - nir_def *hi = output_hi[c] ? output_hi[c] : undef; - nir_def *store_val = nir_pack_32_2x16_split(b, lo, hi); - - ac_nir_store_shared_gs_out(b, store_val, gs_emit_vtx_addr, &s->out, VARYING_SLOT_VAR0_16BIT + slot, c); - } - - /* Clear all outputs (they are undefined after emit_vertex) */ - memset(s->out.outputs_16bit_lo[slot], 0, sizeof(s->out.outputs_16bit_lo[slot])); - memset(s->out.outputs_16bit_hi[slot], 0, sizeof(s->out.outputs_16bit_hi[slot])); - } - /* Calculate and store per-vertex primitive flags based on vertex counts: * - bit 0: whether this vertex finishes a primitive (a real primitive, not the strip) * - bit 1: whether the primitive index is odd (if we are emitting triangle strips, otherwise always 0) @@ -345,7 +319,8 @@ ngg_gs_process_out_vertex(nir_builder *b, nir_def *out_vtx_lds_addr, lower_ngg_g exported_out_vtx_lds_addr = ngg_gs_out_vertex_addr(b, nir_u2u32(b, exported_vtx_idx), s); } - u_foreach_bit64(slot, b->shader->info.outputs_written) { + u_foreach_bit64_two_masks(slot, b->shader->info.outputs_written, + VARYING_SLOT_VAR0_16BIT, b->shader->info.outputs_written_16bit) { unsigned mask = ac_nir_gs_output_component_mask_with_stream(&s->out.infos[slot], 0); u_foreach_bit(c, mask) { @@ -354,24 +329,6 @@ ngg_gs_process_out_vertex(nir_builder *b, nir_def *out_vtx_lds_addr, lower_ngg_g } } - /* Dedicated 16-bit outputs. */ - u_foreach_bit(i, b->shader->info.outputs_written_16bit) { - const unsigned mask_lo = ac_nir_gs_output_component_mask_with_stream(&s->out.infos_16bit_lo[i], 0); - const unsigned mask_hi = ac_nir_gs_output_component_mask_with_stream(&s->out.infos_16bit_hi[i], 0); - unsigned mask = mask_lo | mask_hi; - - u_foreach_bit(c, mask) { - nir_def *load_val = ac_nir_load_shared_gs_out(b, exported_out_vtx_lds_addr, - &s->out, VARYING_SLOT_VAR0_16BIT + i, c); - - if (mask_lo & BITFIELD_BIT(c)) - s->out.outputs_16bit_lo[i][c] = nir_unpack_32_2x16_split_x(b, load_val); - - if (mask_hi & BITFIELD_BIT(c)) - s->out.outputs_16bit_hi[i][c] = nir_unpack_32_2x16_split_y(b, load_val); - } - } - /* This should be after streamout and before exports. */ ac_nir_clamp_vertex_color_outputs(b, &s->out); } diff --git a/src/amd/common/nir/ac_nir_prerast_utils.c b/src/amd/common/nir/ac_nir_prerast_utils.c index 1b7efee9b7c..73ea9cb5bed 100644 --- a/src/amd/common/nir/ac_nir_prerast_utils.c +++ b/src/amd/common/nir/ac_nir_prerast_utils.c @@ -98,27 +98,9 @@ void ac_nir_gather_prerast_store_output_info(nir_builder *b, nir_intrinsic_instr nir_def *store_val = intrin->src[0].ssa; assert(store_val->bit_size == 16 || store_val->bit_size == 32); - nir_def **output; - nir_alu_type *type; - ac_nir_prerast_per_output_info *info; - - if (slot >= VARYING_SLOT_VAR0_16BIT) { - const unsigned index = slot - VARYING_SLOT_VAR0_16BIT; - - if (io_sem.high_16bits) { - output = out->outputs_16bit_hi[index]; - type = out->types_16bit_hi[index]; - info = &out->infos_16bit_hi[index]; - } else { - output = out->outputs_16bit_lo[index]; - type = out->types_16bit_lo[index]; - info = &out->infos_16bit_lo[index]; - } - } else { - output = out->outputs[slot]; - type = out->types[slot]; - info = &out->infos[slot]; - } + nir_def **output = out->outputs[slot]; + nir_alu_type *type = out->types[slot]; + ac_nir_prerast_per_output_info *info = &out->infos[slot]; unsigned component_offset = nir_intrinsic_component(intrin); unsigned write_mask = nir_intrinsic_write_mask(intrin); @@ -127,9 +109,6 @@ void ac_nir_gather_prerast_store_output_info(nir_builder *b, nir_intrinsic_instr b->cursor = nir_before_instr(&intrin->instr); - /* 16-bit output stored in a normal varying slot that isn't a dedicated 16-bit slot. */ - const bool non_dedicated_16bit = slot < VARYING_SLOT_VAR0_16BIT && store_val->bit_size == 16; - u_foreach_bit (i, write_mask) { const unsigned stream = (io_sem.gs_streams >> (i * 2)) & 0x3; @@ -168,10 +147,7 @@ void ac_nir_gather_prerast_store_output_info(nir_builder *b, nir_intrinsic_instr /* Get the old value pointer and the new value. */ if (store_component->bit_size == 16) { - if (slot < VARYING_SLOT_VAR0_16BIT) - saved_value = &out->const_values[slot][c]; - else - saved_value = &out->const_values_16bit[slot - VARYING_SLOT_VAR0_16BIT][c]; + saved_value = &out->const_values[slot][c]; if (io_sem.high_16bits) new_value = (*saved_value & 0xffff) | ((uint32_t)value.u16 << 16); @@ -197,7 +173,7 @@ void ac_nir_gather_prerast_store_output_info(nir_builder *b, nir_intrinsic_instr } } - if (non_dedicated_16bit) { + if (store_val->bit_size == 16) { if (gather_values) { if (io_sem.high_16bits) { nir_def *lo = output[c] ? nir_unpack_32_2x16_split_x(b, output[c]) : nir_imm_intN_t(b, 0, 16); @@ -466,7 +442,8 @@ ac_nir_export_parameters(nir_builder *b, { uint32_t exported_params = 0; - u_foreach_bit64 (slot, outputs_written) { + u_foreach_bit64_two_masks(slot, outputs_written, + VARYING_SLOT_VAR0_16BIT, outputs_written_16bit) { unsigned offset = param_offsets[slot]; if (offset > AC_EXP_PARAM_OFFSET_31) continue; @@ -494,43 +471,6 @@ ac_nir_export_parameters(nir_builder *b, .write_mask = write_mask); exported_params |= BITFIELD_BIT(offset); } - - u_foreach_bit (slot, outputs_written_16bit) { - unsigned offset = param_offsets[VARYING_SLOT_VAR0_16BIT + slot]; - if (offset > AC_EXP_PARAM_OFFSET_31) - continue; - - uint32_t write_mask = 0; - for (int i = 0; i < 4; i++) { - if (out->outputs_16bit_lo[slot][i] || out->outputs_16bit_hi[slot][i]) - write_mask |= BITFIELD_BIT(i); - } - - /* no one set this output slot, we can skip the param export */ - if (!write_mask) - continue; - - /* Since param_offsets[] can map multiple varying slots to the same - * param export index (that's radeonsi-specific behavior), we need to - * do this so as not to emit duplicated exports. - */ - if (exported_params & BITFIELD_BIT(offset)) - continue; - - nir_def *vec[4]; - nir_def *undef = nir_undef(b, 1, 16); - for (int i = 0; i < 4; i++) { - nir_def *lo = out->outputs_16bit_lo[slot][i] ? out->outputs_16bit_lo[slot][i] : undef; - nir_def *hi = out->outputs_16bit_hi[slot][i] ? out->outputs_16bit_hi[slot][i] : undef; - vec[i] = nir_pack_32_2x16_split(b, lo, hi); - } - - nir_export_amd( - b, nir_vec(b, vec, 4), - .base = V_008DFC_SQ_EXP_PARAM + offset, - .write_mask = write_mask); - exported_params |= BITFIELD_BIT(offset); - } } void @@ -558,7 +498,8 @@ ac_nir_store_parameters_to_attr_ring(nir_builder *b, uint32_t exported_params = 0; - u_foreach_bit64 (slot, outputs_written) { + u_foreach_bit64_two_masks(slot, outputs_written, + VARYING_SLOT_VAR0_16BIT, outputs_written_16bit) { const unsigned offset = param_offsets[slot]; if (offset > AC_EXP_PARAM_OFFSET_31) @@ -584,35 +525,6 @@ ac_nir_store_parameters_to_attr_ring(nir_builder *b, exported_params |= BITFIELD_BIT(offset); } - u_foreach_bit (i, outputs_written_16bit) { - const unsigned offset = param_offsets[VARYING_SLOT_VAR0_16BIT + i]; - - if (offset > AC_EXP_PARAM_OFFSET_31) - continue; - - if (!out->infos_16bit_lo[i].as_varying_mask && - !out->infos_16bit_hi[i].as_varying_mask) - continue; - - if (exported_params & BITFIELD_BIT(offset)) - continue; - - nir_def *comp[4]; - for (unsigned j = 0; j < 4; j++) { - nir_def *lo = out->outputs_16bit_lo[i][j] ? out->outputs_16bit_lo[i][j] : undef; - nir_def *hi = out->outputs_16bit_hi[i][j] ? out->outputs_16bit_hi[i][j] : undef; - comp[j] = nir_pack_32_2x16_split(b, lo, hi); - } - - nir_store_buffer_amd(b, nir_vec(b, comp, 4), attr_rsrc, voffset, attr_offset, vindex, - .base = offset * 16, - .memory_modes = nir_var_shader_out, - .access = ACCESS_COHERENT | ACCESS_IS_SWIZZLED_AMD, - .align_mul = 16, .align_offset = 0); - - exported_params |= BITFIELD_BIT(offset); - } - nir_pop_if(b, if_attr_ring_store); } @@ -646,35 +558,6 @@ ac_nir_get_sorted_xfb_info(const nir_shader *nir) return info; } -static nir_def ** -get_output_and_type(ac_nir_prerast_out *out, unsigned slot, bool high_16bits, - nir_alu_type **types) -{ - nir_def **data; - nir_alu_type *type; - - /* Only VARYING_SLOT_VARn_16BIT slots need output type to convert 16bit output - * to 32bit. Vulkan is not allowed to streamout output less than 32bit. - */ - if (slot < VARYING_SLOT_VAR0_16BIT) { - data = out->outputs[slot]; - type = NULL; - } else { - unsigned index = slot - VARYING_SLOT_VAR0_16BIT; - - if (high_16bits) { - data = out->outputs_16bit_hi[index]; - type = out->types_16bit_hi[index]; - } else { - data = out->outputs[index]; - type = out->types_16bit_lo[index]; - } - } - - *types = type; - return data; -} - void ac_nir_emit_legacy_streamout(nir_builder *b, unsigned stream, nir_xfb_info *info, ac_nir_prerast_out *out) { @@ -705,9 +588,7 @@ ac_nir_emit_legacy_streamout(nir_builder *b, unsigned stream, nir_xfb_info *info if (stream != info->buffer_to_stream[output->buffer]) continue; - nir_alu_type *output_type; - nir_def **output_data = - get_output_and_type(out, output->location, output->high_16bits, &output_type); + nir_def **output_data = out->outputs[output->location]; u_foreach_bit(out_comp, output->component_mask) { if (!output_data[out_comp]) @@ -1000,22 +881,13 @@ ac_nir_create_output_phis(nir_builder *b, { nir_def *undef = nir_undef(b, 1, 32); /* inserted at the start of the shader */ - u_foreach_bit64(slot, outputs_written) { + u_foreach_bit64_two_masks(slot, outputs_written, + VARYING_SLOT_VAR0_16BIT, outputs_written_16bit) { for (unsigned j = 0; j < 4; j++) { if (out->outputs[slot][j]) out->outputs[slot][j] = nir_if_phi(b, out->outputs[slot][j], undef); } } - - u_foreach_bit64(i, outputs_written_16bit) { - for (unsigned j = 0; j < 4; j++) { - if (out->outputs_16bit_hi[i][j]) - out->outputs_16bit_hi[i][j] = nir_if_phi(b, out->outputs_16bit_hi[i][j], undef); - - if (out->outputs_16bit_lo[i][j]) - out->outputs_16bit_lo[i][j] = nir_if_phi(b, out->outputs_16bit_lo[i][j], undef); - } - } } static nir_def * @@ -1351,22 +1223,8 @@ static unsigned ac_nir_get_lds_gs_out_slot_offset(ac_nir_prerast_out *pr_out, gl_varying_slot slot, unsigned component) { assert(component < 4); - unsigned lds_slot_offset, lds_component_mask; - - if (slot >= VARYING_SLOT_VAR0_16BIT) { - unsigned i = slot - VARYING_SLOT_VAR0_16BIT; - assert(pr_out->infos_16bit_lo[i].packed_slot_gs_out_offset == - pr_out->infos_16bit_hi[i].packed_slot_gs_out_offset); - - lds_slot_offset = pr_out->infos_16bit_lo[i].packed_slot_gs_out_offset; - lds_component_mask = (pr_out->infos_16bit_lo[i].components_mask | - pr_out->infos_16bit_hi[i].components_mask) & - ~(pr_out->infos_16bit_lo[i].const_mask & - pr_out->infos_16bit_hi[i].const_mask); - } else { - lds_slot_offset = pr_out->infos[slot].packed_slot_gs_out_offset; - lds_component_mask = pr_out->infos[slot].components_mask & ~pr_out->infos[slot].const_mask; - } + unsigned lds_slot_offset = pr_out->infos[slot].packed_slot_gs_out_offset; + unsigned lds_component_mask = pr_out->infos[slot].components_mask & ~pr_out->infos[slot].const_mask; return lds_slot_offset + util_bitcount(lds_component_mask & BITFIELD_MASK(component)) * 4; } @@ -1375,20 +1233,8 @@ static unsigned ac_nir_ngg_get_xfb_lds_offset(ac_nir_prerast_out *pr_out, gl_varying_slot slot, unsigned component) { assert(component < 4); - unsigned lds_slot_offset = 0, lds_component_mask = 0; - - if (slot >= VARYING_SLOT_VAR0_16BIT) { - unsigned i = slot - VARYING_SLOT_VAR0_16BIT; - assert(pr_out->infos_16bit_lo[i].packed_slot_xfb_lds_offset == - pr_out->infos_16bit_hi[i].packed_slot_xfb_lds_offset); - - lds_slot_offset = pr_out->infos_16bit_lo[i].packed_slot_xfb_lds_offset; - lds_component_mask = pr_out->infos_16bit_lo[i].xfb_lds_components_mask | - pr_out->infos_16bit_hi[i].xfb_lds_components_mask; - } else { - lds_slot_offset = pr_out->infos[slot].packed_slot_xfb_lds_offset; - lds_component_mask = pr_out->infos[slot].xfb_lds_components_mask & ~pr_out->infos[slot].const_mask; - } + unsigned lds_slot_offset = pr_out->infos[slot].packed_slot_xfb_lds_offset; + unsigned lds_component_mask = pr_out->infos[slot].xfb_lds_components_mask & ~pr_out->infos[slot].const_mask; return lds_slot_offset + util_bitcount(lds_component_mask & BITFIELD_MASK(component)) * 4; } @@ -1396,13 +1242,7 @@ ac_nir_ngg_get_xfb_lds_offset(ac_nir_prerast_out *pr_out, gl_varying_slot slot, bool ac_nir_is_const_output(ac_nir_prerast_out *pr_out, gl_varying_slot slot, unsigned component) { - if (slot >= VARYING_SLOT_VAR0_16BIT) { - slot -= VARYING_SLOT_VAR0_16BIT; - return pr_out->infos_16bit_lo[slot].const_mask & - pr_out->infos_16bit_hi[slot].const_mask & BITFIELD_BIT(component); - } else { - return pr_out->infos[slot].const_mask & BITFIELD_BIT(component); - } + return pr_out->infos[slot].const_mask & BITFIELD_BIT(component); } nir_def * @@ -1412,10 +1252,7 @@ ac_nir_get_const_output(nir_builder *b, ac_nir_prerast_out *pr_out, gl_varying_s if (!ac_nir_is_const_output(pr_out, slot, component)) return NULL; - if (slot >= VARYING_SLOT_VAR0_16BIT) - return nir_imm_intN_t(b, pr_out->const_values_16bit[slot - VARYING_SLOT_VAR0_16BIT][component], 32); - else - return nir_imm_intN_t(b, pr_out->const_values[slot][component], 32); + return nir_imm_intN_t(b, pr_out->const_values[slot][component], 32); } void @@ -1567,26 +1404,6 @@ ac_nir_compute_prerast_packed_output_info(ac_nir_prerast_out *pr_out) } } - for (unsigned i = 0; i < ARRAY_SIZE(pr_out->infos_16bit_lo); i++) { - unsigned component_mask = pr_out->infos_16bit_lo[i].components_mask | - pr_out->infos_16bit_hi[i].components_mask; - unsigned xfb_component_mask = pr_out->infos_16bit_lo[i].xfb_lds_components_mask | - pr_out->infos_16bit_hi[i].xfb_lds_components_mask; - unsigned const_mask = pr_out->infos_16bit_lo[i].const_mask & - pr_out->infos_16bit_hi[i].const_mask; - assert(gs_out_offset < BITFIELD_BIT(12)); - assert(xfb_lds_offset < BITFIELD_BIT(12)); - pr_out->infos_16bit_lo[i].packed_slot_gs_out_offset = gs_out_offset; - pr_out->infos_16bit_hi[i].packed_slot_gs_out_offset = gs_out_offset; - pr_out->infos_16bit_lo[i].packed_slot_xfb_lds_offset = xfb_lds_offset; - pr_out->infos_16bit_hi[i].packed_slot_xfb_lds_offset = xfb_lds_offset; - - if (component_mask & ~const_mask) - gs_out_offset += util_bitcount(component_mask & ~const_mask) * 4; - if (xfb_component_mask & ~const_mask) - xfb_lds_offset += util_bitcount(xfb_component_mask & ~const_mask) * 4; - } - assert(gs_out_offset < BITFIELD_BIT(16)); assert(xfb_lds_offset < BITFIELD_BIT(16)); pr_out->total_packed_gs_out_size = gs_out_offset; diff --git a/src/util/bitscan.h b/src/util/bitscan.h index 21bdafd1e49..67f84cc74ba 100644 --- a/src/util/bitscan.h +++ b/src/util/bitscan.h @@ -122,6 +122,16 @@ u_bit_scan64(uint64_t *mask) ((b) = ffsll(__dword) - 1, __dword); \ __dword &= ~(1ull << (b))) +/* Given two bitmasks, loop over all bits of both of them. + * Bits of mask1 are: b = scan_bit(mask1); + * Bits of mask2 are: b = offset + scan_bit(mask2); + */ +#define u_foreach_bit64_two_masks(b, mask1, offset, mask2) \ + for (uint64_t __mask1 = (mask1), __mask2 = (mask2), b; \ + (__mask1 ? ((b) = ffsll(__mask1) - 1) \ + : ((b) = ffsll(__mask2) - 1 + offset), __mask1 || __mask2); \ + __mask1 ? (__mask1 &= ~(1ull << (b))) : (__mask2 &= ~(1ull << (b - offset)))) + /* Determine if an uint32_t value is a power of two. * * \note