diff --git a/src/amd/compiler/tests/helpers.cpp b/src/amd/compiler/tests/helpers.cpp index 730f9ddb23d..88dc95b73c9 100644 --- a/src/amd/compiler/tests/helpers.cpp +++ b/src/amd/compiler/tests/helpers.cpp @@ -350,6 +350,71 @@ Temp ext_ubyte(Temp src, unsigned idx, Builder b) Operand::c32(8u), Operand::c32(false)); } +void emit_divergent_if_else(Program* prog, aco::Builder& b, Operand cond, std::function then, + std::function els) +{ + prog->blocks.reserve(prog->blocks.size() + 6); + + Block* if_block = &prog->blocks.back(); + Block* then_logical = prog->create_and_insert_block(); + Block* then_linear = prog->create_and_insert_block(); + Block* invert = prog->create_and_insert_block(); + Block* else_logical = prog->create_and_insert_block(); + Block* else_linear = prog->create_and_insert_block(); + Block* endif_block = prog->create_and_insert_block(); + + if_block->kind |= block_kind_branch; + invert->kind |= block_kind_invert; + endif_block->kind |= block_kind_merge; + + /* Set up logical CF */ + then_logical->logical_preds.push_back(if_block->index); + else_logical->logical_preds.push_back(if_block->index); + endif_block->logical_preds.push_back(then_logical->index); + endif_block->logical_preds.push_back(else_logical->index); + + /* Set up linear CF */ + then_logical->linear_preds.push_back(if_block->index); + then_linear->linear_preds.push_back(if_block->index); + invert->linear_preds.push_back(then_logical->index); + invert->linear_preds.push_back(then_linear->index); + else_logical->linear_preds.push_back(invert->index); + else_linear->linear_preds.push_back(invert->index); + endif_block->linear_preds.push_back(else_logical->index); + endif_block->linear_preds.push_back(else_linear->index); + + PhysReg saved_exec_reg(84); + + b.reset(if_block); + Temp saved_exec = b.sop1(Builder::s_and_saveexec, b.def(b.lm, saved_exec_reg), Definition(scc, s1), Definition(exec, b.lm), cond, Operand(exec, b.lm)); + b.branch(aco_opcode::p_cbranch_nz, Definition(vcc, bld.lm), then_logical->index, then_linear->index); + + b.reset(then_logical); + b.pseudo(aco_opcode::p_logical_start); + then(); + b.pseudo(aco_opcode::p_logical_end); + b.branch(aco_opcode::p_branch, Definition(vcc, bld.lm), invert->index); + + b.reset(then_linear); + b.branch(aco_opcode::p_branch, Definition(vcc, bld.lm), invert->index); + + b.reset(invert); + b.sop2(Builder::s_andn2, Definition(exec, bld.lm), Definition(scc, s1), Operand(saved_exec, saved_exec_reg), Operand(exec, bld.lm)); + b.branch(aco_opcode::p_cbranch_nz, Definition(vcc, bld.lm), else_logical->index, else_linear->index); + + b.reset(else_logical); + b.pseudo(aco_opcode::p_logical_start); + els(); + b.pseudo(aco_opcode::p_logical_end); + b.branch(aco_opcode::p_branch, Definition(vcc, bld.lm), endif_block->index); + + b.reset(else_linear); + b.branch(aco_opcode::p_branch, Definition(vcc, bld.lm), endif_block->index); + + b.reset(endif_block); + b.pseudo(aco_opcode::p_parallelcopy, Definition(exec, bld.lm), Operand(saved_exec, saved_exec_reg)); +} + VkDevice get_vk_device(enum amd_gfx_level gfx_level) { enum radeon_family family; diff --git a/src/amd/compiler/tests/helpers.h b/src/amd/compiler/tests/helpers.h index 2151e99a80c..e2282b7f18e 100644 --- a/src/amd/compiler/tests/helpers.h +++ b/src/amd/compiler/tests/helpers.h @@ -26,6 +26,7 @@ #include "framework.h" #include "vulkan/vulkan.h" +#include enum QoShaderDeclType { QoShaderDeclType_ubo, @@ -102,6 +103,8 @@ aco::Temp fma(aco::Temp src0, aco::Temp src1, aco::Temp src2, aco::Builder b=bld aco::Temp fsat(aco::Temp src, aco::Builder b=bld); aco::Temp ext_ushort(aco::Temp src, unsigned idx, aco::Builder b=bld); aco::Temp ext_ubyte(aco::Temp src, unsigned idx, aco::Builder b=bld); +void emit_divergent_if_else(aco::Program* prog, aco::Builder& b, aco::Operand cond, std::function then, + std::function els); /* vulkan helpers */ VkDevice get_vk_device(enum amd_gfx_level gfx_level); diff --git a/src/amd/compiler/tests/test_optimizer_postRA.cpp b/src/amd/compiler/tests/test_optimizer_postRA.cpp index 5e77de22cee..6507a361feb 100644 --- a/src/amd/compiler/tests/test_optimizer_postRA.cpp +++ b/src/amd/compiler/tests/test_optimizer_postRA.cpp @@ -430,3 +430,153 @@ BEGIN_TEST(optimizer_postRA.dpp) finish_optimizer_postRA_test(); END_TEST +BEGIN_TEST(optimizer_postRA.dpp_across_cf) + //>> v1: %a:v[0], v1: %b:v[1], v1: %c:v[2], v1: %d:v[3], s2: %e:s[0-1] = p_startpgm + if (!setup_cs("v1 v1 v1 v1 s2", GFX10_3)) + return; + + aco_ptr& startpgm = bld.instructions->at(0); + startpgm->definitions[0].setFixed(PhysReg(256)); + startpgm->definitions[1].setFixed(PhysReg(257)); + startpgm->definitions[2].setFixed(PhysReg(258)); + startpgm->definitions[3].setFixed(PhysReg(259)); + startpgm->definitions[4].setFixed(PhysReg(0)); + + Operand a(inputs[0], PhysReg(256)); /* source for DPP */ + Operand b(inputs[1], PhysReg(257)); /* source for fadd */ + Operand c(inputs[2], PhysReg(258)); /* buffer store address */ + Operand d(inputs[3], PhysReg(259)); /* buffer store value */ + Operand e(inputs[4], PhysReg(0)); /* condition */ + PhysReg reg_v12(268); /* temporary register */ + + Temp dpp_tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1, reg_v12), a, dpp_row_mirror); + + //! s2: %saved_exec:s[84-85], s1: %0:scc, s2: %0:exec = s_and_saveexec_b64 %e:s[0-1], %0:exec + //! s2: %0:vcc = p_cbranch_nz BB1, BB2 + + emit_divergent_if_else(program.get(), bld, e, [&]() -> void { + /* --- logical then --- */ + //! BB1 + //! /* logical preds: BB0, / linear preds: BB0, / kind: */ + //! p_logical_start + + //! buffer_store_dword %c:v[2], 0, %d:v[3], 0 offen storage: semantics: scope:invocation + bld.mubuf(aco_opcode::buffer_store_dword, c, Operand::zero(), d, Operand::zero(), 0, true); + + //! p_logical_end + //! s2: %0:vcc = p_branch BB3 + + /* --- linear then --- */ + //! BB2 + //! /* logical preds: / linear preds: BB0, / kind: */ + //! s2: %0:vcc = p_branch BB3 + + /* --- invert --- */ + //! BB3 + //! /* logical preds: / linear preds: BB1, BB2, / kind: invert, */ + //! s2: %0:exec, s1: %0:scc = s_andn2_b64 %saved_exec:s[84-85], %0:exec + //! s2: %0:vcc = p_cbranch_nz BB4, BB5 + }, [&]() -> void { + /* --- logical else --- */ + //! BB4 + //! /* logical preds: BB0, / linear preds: BB3, / kind: */ + //! p_logical_start + //! p_logical_end + //! s2: %0:vcc = p_branch BB6 + + /* --- linear else --- */ + //! BB5 + //! /* logical preds: / linear preds: BB3, / kind: */ + //! s2: %0:vcc = p_branch BB6 + }); + + /* --- merge block --- */ + //! BB6 + //! /* logical preds: BB1, BB4, / linear preds: BB4, BB5, / kind: uniform, merge, */ + //! s2: %0:exec = p_parallelcopy %saved_exec:s[84-85] + + //! v1: %res10:v[12] = v_add_f32 %a:v[0], %b:v[1] row_mirror bound_ctrl:1 + //! p_unit_test 10, %res10:v[12] + Temp result = bld.vop2(aco_opcode::v_add_f32, bld.def(v1, reg_v12), Operand(dpp_tmp, reg_v12), b); + writeout(10, Operand(result, reg_v12)); + + finish_optimizer_postRA_test(); +END_TEST + +BEGIN_TEST(optimizer_postRA.dpp_across_cf_overwritten) + //>> v1: %a:v[0], v1: %b:v[1], v1: %c:v[2], v1: %d:v[3], s2: %e:s[0-1], s1: %f:s[2] = p_startpgm + if (!setup_cs("v1 v1 v1 v1 s2 s1", GFX10_3)) + return; + + aco_ptr& startpgm = bld.instructions->at(0); + startpgm->definitions[0].setFixed(PhysReg(256)); + startpgm->definitions[1].setFixed(PhysReg(257)); + startpgm->definitions[2].setFixed(PhysReg(258)); + startpgm->definitions[3].setFixed(PhysReg(259)); + startpgm->definitions[4].setFixed(PhysReg(0)); + startpgm->definitions[5].setFixed(PhysReg(2)); + + Operand a(inputs[0], PhysReg(256)); /* source for DPP */ + Operand b(inputs[1], PhysReg(257)); /* source for fadd */ + Operand d(inputs[3], PhysReg(259)); /* buffer store value */ + Operand e(inputs[4], PhysReg(0)); /* condition */ + Operand f(inputs[5], PhysReg(2)); /* buffer store address (scalar) */ + PhysReg reg_v12(268); /* temporary register */ + + //! v1: %dpp_tmp:v[12] = v_mov_b32 %a:v[0] row_mirror bound_ctrl:1 + Temp dpp_tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1, reg_v12), a, dpp_row_mirror); + + //! s2: %saved_exec:s[84-85], s1: %0:scc, s2: %0:exec = s_and_saveexec_b64 %e:s[0-1], %0:exec + //! s2: %0:vcc = p_cbranch_nz BB1, BB2 + + emit_divergent_if_else(program.get(), bld, e, [&]() -> void { + /* --- logical then --- */ + //! BB1 + //! /* logical preds: BB0, / linear preds: BB0, / kind: */ + //! p_logical_start + + //! v1: %addr:v[0] = p_parallelcopy %f:s[2] + Temp addr = bld.pseudo(aco_opcode::p_parallelcopy, bld.def(v1, a.physReg()), f); + + //! buffer_store_dword %addr:v[0], 0, %d:v[3], 0 offen storage: semantics: scope:invocation + bld.mubuf(aco_opcode::buffer_store_dword, Operand(addr, a.physReg()), Operand::zero(), d, Operand::zero(), 0, true); + + //! p_logical_end + //! s2: %0:vcc = p_branch BB3 + + /* --- linear then --- */ + //! BB2 + //! /* logical preds: / linear preds: BB0, / kind: */ + //! s2: %0:vcc = p_branch BB3 + + /* --- invert --- */ + //! BB3 + //! /* logical preds: / linear preds: BB1, BB2, / kind: invert, */ + //! s2: %0:exec, s1: %0:scc = s_andn2_b64 %saved_exec:s[84-85], %0:exec + //! s2: %0:vcc = p_cbranch_nz BB4, BB5 + }, [&]() -> void { + /* --- logical else --- */ + //! BB4 + //! /* logical preds: BB0, / linear preds: BB3, / kind: */ + //! p_logical_start + //! p_logical_end + //! s2: %0:vcc = p_branch BB6 + + /* --- linear else --- */ + //! BB5 + //! /* logical preds: / linear preds: BB3, / kind: */ + //! s2: %0:vcc = p_branch BB6 + }); + + /* --- merge block --- */ + //! BB6 + //! /* logical preds: BB1, BB4, / linear preds: BB4, BB5, / kind: uniform, merge, */ + //! s2: %0:exec = p_parallelcopy %saved_exec:s[84-85] + + //! v1: %result:v[12] = v_add_f32 %dpp_mov_tmp:v[12], %b:v[1] + Temp result = bld.vop2(aco_opcode::v_add_f32, bld.def(v1, reg_v12), Operand(dpp_tmp, reg_v12), b); + //! p_unit_test 10, %result:v[12] + writeout(10, Operand(result, reg_v12)); + + finish_optimizer_postRA_test(); +END_TEST