diff --git a/src/amd/compiler/aco_instruction_selection.cpp b/src/amd/compiler/aco_instruction_selection.cpp index eb8f9dc8ef7..5948aa0640d 100644 --- a/src/amd/compiler/aco_instruction_selection.cpp +++ b/src/amd/compiler/aco_instruction_selection.cpp @@ -8271,6 +8271,24 @@ get_interp_param(isel_context* ctx, nir_intrinsic_op intrin, } } +void +ds_ordered_count_offsets(isel_context *ctx, unsigned index_operand, + unsigned wave_release, unsigned wave_done, + unsigned *offset0, unsigned *offset1) +{ + unsigned ordered_count_index = index_operand & 0x3f; + unsigned count_dword = (index_operand >> 24) & 0xf; + + assert(ctx->options->gfx_level >= GFX10); + assert(count_dword >= 1 && count_dword <= 4); + + *offset0 = ordered_count_index << 2; + *offset1 = wave_release | (wave_done << 1) | ((count_dword - 1) << 6); + + if (ctx->options->gfx_level < GFX11) + *offset1 |= 3 /* GS shader type */ << 2; +} + void visit_intrinsic(isel_context* ctx, nir_intrinsic_instr* instr) { @@ -9196,6 +9214,64 @@ visit_intrinsic(isel_context* ctx, nir_intrinsic_instr* instr) emit_split_vector(ctx, dst, dst.size()); break; } + case nir_intrinsic_ordered_xfb_counter_add_amd: { + Temp dst = get_ssa_temp(ctx, &instr->dest.ssa); + Temp ordered_id = get_ssa_temp(ctx, instr->src[0].ssa); + Temp counter = get_ssa_temp(ctx, instr->src[1].ssa); + + Temp gds_base = bld.copy(bld.def(v1), Operand::c32(0u)); + unsigned offset0, offset1; + Instruction *ds_instr; + Operand m; + + /* Lock a GDS mutex. */ + ds_ordered_count_offsets(ctx, 1 << 24u, false, false, &offset0, &offset1); + m = bld.m0(bld.as_uniform(ordered_id)); + ds_instr = bld.ds(aco_opcode::ds_ordered_count, bld.def(v1), gds_base, m, + offset0, offset1, true); + ds_instr->ds().sync = memory_sync_info(storage_gds, semantic_volatile); + + aco_ptr vec{create_instruction( + aco_opcode::p_create_vector, Format::PSEUDO, instr->num_components, 1)}; + unsigned write_mask = nir_intrinsic_write_mask(instr); + + for (unsigned i = 0; i < instr->num_components; i++) { + if (write_mask & (1 << i)) { + Temp chan_counter = emit_extract_vector(ctx, counter, i, v1); + + m = bld.m0((Temp)bld.copy(bld.def(s1, m0), Operand::c32(0x100u))); + + ds_instr = bld.ds(aco_opcode::ds_add_rtn_u32, bld.def(v1), + gds_base, chan_counter, m, i * 4, 0u, true); + ds_instr->ds().sync = memory_sync_info(storage_gds, semantic_atomicrmw); + + vec->operands[i] = Operand(ds_instr->definitions[0].getTemp()); + } else { + vec->operands[i] = Operand::zero(); + } + } + + vec->definitions[0] = Definition(dst); + ctx->block->instructions.emplace_back(std::move(vec)); + + /* Unlock a GDS mutex. */ + ds_ordered_count_offsets(ctx, 1 << 24u, true, true, &offset0, &offset1); + m = bld.m0(bld.as_uniform(ordered_id)); + ds_instr = bld.ds(aco_opcode::ds_ordered_count, bld.def(v1), gds_base, m, + offset0, offset1, true); + ds_instr->ds().sync = memory_sync_info(storage_gds, semantic_volatile); + + emit_split_vector(ctx, dst, instr->num_components); + break; + } + case nir_intrinsic_memory_barrier_buffer: { + wait_imm wait; + wait.lgkm = 0; + wait.vm = 0; + bld.sopp(aco_opcode::s_waitcnt, -1, wait.pack(bld.program->gfx_level)); + bld.sopk(aco_opcode::s_waitcnt_vscnt, Definition(sgpr_null, s1), 0); + break; + } default: isel_err(&instr->instr, "Unimplemented intrinsic instr"); abort(); diff --git a/src/amd/compiler/aco_instruction_selection_setup.cpp b/src/amd/compiler/aco_instruction_selection_setup.cpp index e655726755a..b813cb2d0ba 100644 --- a/src/amd/compiler/aco_instruction_selection_setup.cpp +++ b/src/amd/compiler/aco_instruction_selection_setup.cpp @@ -693,7 +693,8 @@ init_context(isel_context* ctx, nir_shader* shader) case nir_intrinsic_load_initial_edgeflags_amd: case nir_intrinsic_gds_atomic_add_amd: case nir_intrinsic_bvh64_intersect_ray_amd: - case nir_intrinsic_load_vector_arg_amd: type = RegType::vgpr; break; + case nir_intrinsic_load_vector_arg_amd: + case nir_intrinsic_ordered_xfb_counter_add_amd: type = RegType::vgpr; break; case nir_intrinsic_load_shared: case nir_intrinsic_load_shared2_amd: /* When the result of these loads is only used by cross-lane instructions,