From 86c63b29bca740d52bbe003c1b31f6864a444a43 Mon Sep 17 00:00:00 2001 From: Rhys Perry Date: Tue, 26 Nov 2024 12:00:35 +0000 Subject: [PATCH] aco/gfx12: insert wait between VMEM WaW https://github.com/llvm/llvm-project/pull/105549 fossil-db (gfx1200): Totals from 1783 (2.25% of 79395) affected shaders: Instrs: 7398391 -> 7404566 (+0.08%); split: -0.00%, +0.08% CodeSize: 38862456 -> 38886364 (+0.06%); split: -0.00%, +0.06% Latency: 83191513 -> 84211504 (+1.23%); split: -0.00%, +1.23% InvThroughput: 15185936 -> 15345744 (+1.05%); split: -0.01%, +1.06% Signed-off-by: Rhys Perry Part-of: --- src/amd/compiler/aco_insert_waitcnt.cpp | 8 ++++++-- src/amd/compiler/tests/test_insert_waitcnt.cpp | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/amd/compiler/aco_insert_waitcnt.cpp b/src/amd/compiler/aco_insert_waitcnt.cpp index 06a1e117c49..523da86fb3a 100644 --- a/src/amd/compiler/aco_insert_waitcnt.cpp +++ b/src/amd/compiler/aco_insert_waitcnt.cpp @@ -278,9 +278,13 @@ check_instr(wait_ctx& ctx, wait_imm& wait, Instruction* instr) wait_imm reg_imm = it->second.imm; - /* Vector Memory reads and writes return in the order they were issued */ + /* Vector Memory reads and writes decrease the counter in the order they were issued. + * Before GFX12, they also write VGPRs in order if they're of the same type. + * TODO: We can do this for GFX12 and different types for GFX11 if we know that the two + * VMEM loads do not write the same lanes. Since GFX11, we track VMEM operations on the + * linear CFG, so this is difficult */ uint8_t vmem_type = get_vmem_type(ctx.gfx_level, instr); - if (vmem_type) { + if (vmem_type && ctx.gfx_level < GFX12) { wait_event event = get_vmem_event(ctx, instr, vmem_type); wait_type type = (wait_type)(ffs(ctx.info->get_counters_for_event(event)) - 1); if ((it->second.events & ctx.info->events[type]) == event && diff --git a/src/amd/compiler/tests/test_insert_waitcnt.cpp b/src/amd/compiler/tests/test_insert_waitcnt.cpp index edc34bfc7f8..c69772c5d05 100644 --- a/src/amd/compiler/tests/test_insert_waitcnt.cpp +++ b/src/amd/compiler/tests/test_insert_waitcnt.cpp @@ -192,6 +192,7 @@ BEGIN_TEST(insert_waitcnt.waw.vmem_types) //>> p_unit_test 0 //! v1: %0:v[4] = buffer_load_dword %0:s[0-3], %0:v[0], 0 + //~gfx12! s_wait_loadcnt imm:0 //! v1: %0:v[4] = buffer_load_dword %0:s[0-3], %0:v[0], 0 bld.pseudo(aco_opcode::p_unit_test, Operand::c32(0)); bld.mubuf(aco_opcode::buffer_load_dword, def_v4, desc_s4, op_v0, Operand::zero(), 0, false); @@ -220,6 +221,7 @@ BEGIN_TEST(insert_waitcnt.waw.vmem_types) //>> p_unit_test 3 //! v1: %0:v[4] = image_sample %0:s[8-15], %0:s[0-3], v1: undef, %0:v[0] 1d + //~gfx12! s_wait_samplecnt imm:0 //! v1: %0:v[4] = image_sample %0:s[8-15], %0:s[0-3], v1: undef, %0:v[0] 1d bld.reset(program->create_and_insert_block()); bld.pseudo(aco_opcode::p_unit_test, Operand::c32(3)); @@ -249,6 +251,7 @@ BEGIN_TEST(insert_waitcnt.waw.vmem_types) //>> p_unit_test 6 //! v1: %0:v[4] = image_bvh64_intersect_ray %0:s[0-3], s4: undef, v1: undef, %0:v[16-26] 1d + //~gfx12! s_wait_bvhcnt imm:0 //! v1: %0:v[4] = image_bvh64_intersect_ray %0:s[0-3], s4: undef, v1: undef, %0:v[16-26] 1d bld.reset(program->create_and_insert_block()); bld.pseudo(aco_opcode::p_unit_test, Operand::c32(6)); @@ -300,6 +303,7 @@ BEGIN_TEST(insert_waitcnt.waw.vmem_types) //>> BB11 //! /* logical preds: BB9, BB10, / linear preds: BB9, BB10, / kind: uniform, */ //! p_unit_test 9 + //~gfx12! s_wait_loadcnt imm:0 //! v1: %0:v[4] = buffer_load_dword %0:s[0-3], %0:v[0], 0 bld.pseudo(aco_opcode::p_unit_test, Operand::c32(9)); bld.mubuf(aco_opcode::buffer_load_dword, def_v4, desc_s4, op_v0, Operand::zero(), 0, false); @@ -326,6 +330,7 @@ BEGIN_TEST(insert_waitcnt.waw.vmem_types) //! /* logical preds: BB12, BB13, / linear preds: BB12, BB13, / kind: uniform, */ //! p_unit_test 10 //~gfx11! s_waitcnt vmcnt(0) + //~gfx12! s_wait_loadcnt imm:0 //~gfx12! s_wait_samplecnt imm:0 //! v1: %0:v[4] = buffer_load_dword %0:s[0-3], %0:v[0], 0 bld.pseudo(aco_opcode::p_unit_test, Operand::c32(10));