From 27c8cddec1ab456fc69ba22d686071207a07204a Mon Sep 17 00:00:00 2001 From: Gert Wollny Date: Wed, 26 Oct 2022 22:13:31 +0200 Subject: [PATCH] r600/sfn: Add .clang-format file and apply style Signed-off-by: Gert Wollny Part-of: --- src/gallium/drivers/r600/sfn/.clang-format | 37 + .../drivers/r600/sfn/sfn_alu_defines.cpp | 614 +++--- .../drivers/r600/sfn/sfn_alu_defines.h | 86 +- .../r600/sfn/sfn_alu_readport_validation.cpp | 99 +- .../r600/sfn/sfn_alu_readport_validation.h | 11 +- .../drivers/r600/sfn/sfn_assembler.cpp | 846 ++++---- src/gallium/drivers/r600/sfn/sfn_assembler.h | 9 +- .../drivers/r600/sfn/sfn_callstack.cpp | 82 +- src/gallium/drivers/r600/sfn/sfn_callstack.h | 3 +- .../r600/sfn/sfn_conditionaljumptracker.cpp | 57 +- .../r600/sfn/sfn_conditionaljumptracker.h | 7 +- src/gallium/drivers/r600/sfn/sfn_debug.cpp | 85 +- src/gallium/drivers/r600/sfn/sfn_debug.h | 37 +- src/gallium/drivers/r600/sfn/sfn_defines.h | 286 ++- src/gallium/drivers/r600/sfn/sfn_docu.txt | 5 +- src/gallium/drivers/r600/sfn/sfn_instr.cpp | 303 +-- src/gallium/drivers/r600/sfn/sfn_instr.h | 167 +- .../drivers/r600/sfn/sfn_instr_alu.cpp | 1730 ++++++++++------- src/gallium/drivers/r600/sfn/sfn_instr_alu.h | 136 +- .../drivers/r600/sfn/sfn_instr_alugroup.cpp | 87 +- .../drivers/r600/sfn/sfn_instr_alugroup.h | 37 +- .../r600/sfn/sfn_instr_controlflow.cpp | 105 +- .../drivers/r600/sfn/sfn_instr_controlflow.h | 16 +- .../drivers/r600/sfn/sfn_instr_export.cpp | 370 ++-- .../drivers/r600/sfn/sfn_instr_export.h | 124 +- .../drivers/r600/sfn/sfn_instr_fetch.cpp | 462 +++-- .../drivers/r600/sfn/sfn_instr_fetch.h | 81 +- .../drivers/r600/sfn/sfn_instr_lds.cpp | 114 +- src/gallium/drivers/r600/sfn/sfn_instr_lds.h | 31 +- .../drivers/r600/sfn/sfn_instr_mem.cpp | 522 +++-- src/gallium/drivers/r600/sfn/sfn_instr_mem.h | 71 +- .../drivers/r600/sfn/sfn_instr_tex.cpp | 659 ++++--- src/gallium/drivers/r600/sfn/sfn_instr_tex.h | 90 +- .../drivers/r600/sfn/sfn_instrfactory.cpp | 70 +- .../drivers/r600/sfn/sfn_instrfactory.h | 11 +- .../r600/sfn/sfn_liverangeevaluator.cpp | 207 +- .../drivers/r600/sfn/sfn_liverangeevaluator.h | 9 +- .../sfn/sfn_liverangeevaluator_helpers.cpp | 276 +-- .../r600/sfn/sfn_liverangeevaluator_helpers.h | 24 +- .../drivers/r600/sfn/sfn_memorypool.cpp | 65 +- src/gallium/drivers/r600/sfn/sfn_memorypool.h | 60 +- src/gallium/drivers/r600/sfn/sfn_nir.cpp | 341 ++-- src/gallium/drivers/r600/sfn/sfn_nir.h | 75 +- .../sfn/sfn_nir_legalize_image_load_store.cpp | 52 +- .../drivers/r600/sfn/sfn_nir_lower_64bit.cpp | 343 ++-- .../drivers/r600/sfn/sfn_nir_lower_alu.cpp | 47 +- .../drivers/r600/sfn/sfn_nir_lower_alu.h | 10 +- .../sfn/sfn_nir_lower_fs_out_to_vector.cpp | 189 +- .../r600/sfn/sfn_nir_lower_fs_out_to_vector.h | 3 +- .../r600/sfn/sfn_nir_lower_tess_io.cpp | 287 +-- .../drivers/r600/sfn/sfn_nir_lower_tex.cpp | 112 +- .../drivers/r600/sfn/sfn_nir_lower_tex.h | 9 +- .../r600/sfn/sfn_nir_vectorize_vs_inputs.c | 102 +- .../drivers/r600/sfn/sfn_optimizer.cpp | 332 ++-- src/gallium/drivers/r600/sfn/sfn_optimizer.h | 17 +- src/gallium/drivers/r600/sfn/sfn_peephole.cpp | 169 +- src/gallium/drivers/r600/sfn/sfn_peephole.h | 4 +- src/gallium/drivers/r600/sfn/sfn_ra.cpp | 84 +- src/gallium/drivers/r600/sfn/sfn_ra.h | 22 +- .../drivers/r600/sfn/sfn_scheduler.cpp | 319 ++- src/gallium/drivers/r600/sfn/sfn_scheduler.h | 4 +- src/gallium/drivers/r600/sfn/sfn_shader.cpp | 589 +++--- src/gallium/drivers/r600/sfn/sfn_shader.h | 160 +- .../drivers/r600/sfn/sfn_shader_cs.cpp | 49 +- src/gallium/drivers/r600/sfn/sfn_shader_cs.h | 19 +- .../drivers/r600/sfn/sfn_shader_fs.cpp | 433 +++-- src/gallium/drivers/r600/sfn/sfn_shader_fs.h | 44 +- .../drivers/r600/sfn/sfn_shader_gs.cpp | 189 +- src/gallium/drivers/r600/sfn/sfn_shader_gs.h | 16 +- .../drivers/r600/sfn/sfn_shader_tess.cpp | 91 +- .../drivers/r600/sfn/sfn_shader_tess.h | 24 +- .../drivers/r600/sfn/sfn_shader_vs.cpp | 266 +-- src/gallium/drivers/r600/sfn/sfn_shader_vs.h | 50 +- .../drivers/r600/sfn/sfn_valuefactory.cpp | 453 +++-- .../drivers/r600/sfn/sfn_valuefactory.h | 301 +-- .../drivers/r600/sfn/sfn_virtualvalues.cpp | 720 ++++--- .../drivers/r600/sfn/sfn_virtualvalues.h | 358 ++-- .../drivers/r600/sfn/tests/meson.build | 1 - .../drivers/r600/sfn/tests/sfn_instr_test.cpp | 536 +++-- .../sfn/tests/sfn_instrfromstring_test.cpp | 296 +-- .../r600/sfn/tests/sfn_liverange_test.cpp | 15 +- .../r600/sfn/tests/sfn_optimizer_test.cpp | 36 +- .../sfn/tests/sfn_shaderfromstring_test.cpp | 62 +- .../r600/sfn/tests/sfn_test_shaders.cpp | 205 +- .../drivers/r600/sfn/tests/sfn_test_shaders.h | 5 +- .../drivers/r600/sfn/tests/sfn_value_test.cpp | 30 +- .../r600/sfn/tests/sfn_valuefactory_test.cpp | 57 +- 87 files changed, 8782 insertions(+), 6835 deletions(-) create mode 100644 src/gallium/drivers/r600/sfn/.clang-format diff --git a/src/gallium/drivers/r600/sfn/.clang-format b/src/gallium/drivers/r600/sfn/.clang-format new file mode 100644 index 00000000000..f5b31ba8d23 --- /dev/null +++ b/src/gallium/drivers/r600/sfn/.clang-format @@ -0,0 +1,37 @@ +BasedOnStyle: LLVM +AlwaysBreakAfterReturnType: TopLevel +AllowShortEnumsOnASingleLine: false +BinPackParameters: false +BinPackArguments: false +BraceWrapping: + AfterControlStatement: false + AfterEnum: false + AfterFunction: true + AfterStruct: false + BeforeElse: false + SplitEmptyFunction: true +BreakBeforeBraces: Custom +ColumnLimit: 90 +ContinuationIndentWidth: 3 +Cpp11BracedListStyle: true +ForEachMacros: + - LIST_FOR_EACH_ENTRY + - LIST_FOR_EACH_ENTRY_SAFE +IncludeBlocks: Regroup +IndentWidth: 3 +TabWidth: 3 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyExcessCharacter: 100 +SpaceAfterCStyleCast: false +SpaceBeforeCpp11BracedList: false +AlignArrayOfStructures: Left +ReferenceAlignment: Left +PenaltyBreakOpenParenthesis: 90 +SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpacesInContainerLiterals: false +PackConstructorInitializers: Never +IndentAccessModifiers: false +AccessModifierOffset: -3 +BreakConstructorInitializers: AfterColon +SpaceBeforeCtorInitializerColon: False diff --git a/src/gallium/drivers/r600/sfn/sfn_alu_defines.cpp b/src/gallium/drivers/r600/sfn/sfn_alu_defines.cpp index e3ae0101186..17818b07c32 100644 --- a/src/gallium/drivers/r600/sfn/sfn_alu_defines.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_alu_defines.cpp @@ -28,318 +28,328 @@ namespace r600 { +#define A AluOp::a +#define V AluOp::v +#define T AluOp::t +#define X AluOp::x + const std::map alu_ops = { - {op0_nop ,AluOp(0, 0, AluOp::a, AluOp::a, AluOp::a, "NOP")}, - {op0_group_barrier ,AluOp(0, 0, AluOp::a, AluOp::a, AluOp::a, "GROUP_BARRIER")}, - {op0_group_seq_begin ,AluOp(0, 0, AluOp::a, AluOp::a, AluOp::a, "GROUP_SEQ_BEGIN")}, - {op0_group_seq_end ,AluOp(0, 0, AluOp::a, AluOp::a, AluOp::a, "GROUP_SEQ_END")}, - {op0_pred_set_clr ,AluOp(0, 1, AluOp::a, AluOp::a, AluOp::a, "PRED_SET_CLR")}, - {op0_store_flags ,AluOp(0, 0, AluOp::v, AluOp::v, AluOp::v, "STORE_FLAGS")}, - {op0_lds_1a ,AluOp(0, 0, AluOp::v, AluOp::v, AluOp::v, "LDS_1A")}, - {op0_lds_1a1d ,AluOp(0, 0, AluOp::v, AluOp::v, AluOp::v, "LDS_1A1D")}, - {op0_lds_2a ,AluOp(0, 0, AluOp::v, AluOp::v, AluOp::v, "LDS_2A")}, + {op0_nop, AluOp(0, 0, A, A, A, "NOP") }, + {op0_group_barrier, AluOp(0, 0, X, X, X, "GROUP_BARRIER") }, + {op0_group_seq_begin, AluOp(0, 0, A, A, A, "GROUP_SEQ_BEGIN") }, + {op0_group_seq_end, AluOp(0, 0, A, A, A, "GROUP_SEQ_END") }, + {op0_pred_set_clr, AluOp(0, 1, A, A, A, "PRED_SET_CLR") }, + {op0_store_flags, AluOp(0, 0, V, V, V, "STORE_FLAGS") }, + {op0_lds_1a, AluOp(0, 0, V, V, V, "LDS_1A") }, + {op0_lds_1a1d, AluOp(0, 0, V, V, V, "LDS_1A1D") }, + {op0_lds_2a, AluOp(0, 0, V, V, V, "LDS_2A") }, - {op1_bcnt_int ,AluOp(1, 0, AluOp::v, AluOp::v, AluOp::v, "BCNT_INT")}, - {op1_bcnt_accum_prev_int ,AluOp(1, 0, AluOp::v, AluOp::v, AluOp::v, "BCNT_ACCUM_PREV_INT")}, - {op1_bfrev_int ,AluOp(1, 0, AluOp::a, AluOp::a, AluOp::a, "BFREV_INT")}, - {op1_ceil ,AluOp(1, 1, AluOp::a, AluOp::a, AluOp::a, "CEIL")}, - {op1_cos ,AluOp(1, 1, AluOp::t, AluOp::t, AluOp::t, "COS")}, - {op1_exp_ieee ,AluOp(1, 1, AluOp::t, AluOp::t, AluOp::t, "EXP_IEEE")}, - {op1_floor ,AluOp(1, 1, AluOp::a, AluOp::a, AluOp::a, "FLOOR")}, - {op1_flt_to_int ,AluOp(1, 0, AluOp::t, AluOp::t, AluOp::v, "FLT_TO_INT")}, - {op1_flt_to_uint ,AluOp(1, 1, AluOp::t, AluOp::t, AluOp::t, "FLT_TO_UINT")}, - {op1_flt_to_int_rpi ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "FLT_TO_INT_RPI")}, - {op1_flt_to_int_floor ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "FLT_TO_INT_FLOOR")}, - {op1_flt16_to_flt32 ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "FLT16_TO_FLT32")}, - {op1_flt32_to_flt16 ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "FLT32_TO_FLT16")}, - {op1_flt32_to_flt64 ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "FLT32_TO_FLT64")}, - {op1_flt64_to_flt32 ,AluOp(1, 1, AluOp::a, AluOp::a, AluOp::a, "FLT64_TO_FLT32")}, - {op1_fract ,AluOp(1, 1, AluOp::a, AluOp::a, AluOp::a, "FRACT")}, - {op1_fract_64 ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "FRACT_64")}, - {op1_frexp_64 ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "FREXP_64")}, - {op1_int_to_flt ,AluOp(1, 0, AluOp::t, AluOp::t, AluOp::t, "INT_TO_FLT")}, - {op1_ldexp_64 ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "LDEXP_64")}, - {op1_interp_load_p0 ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "INTERP_LOAD_P0")}, - {op1_interp_load_p10 ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "INTERP_LOAD_P10")}, - {op1_interp_load_p20 ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "INTERP_LOAD_P20")}, - {op1_load_store_flags ,AluOp(1, 0, AluOp::v, AluOp::v, AluOp::v, "LOAD_STORE_FLAGS")}, - {op1_log_clamped ,AluOp(1, 1, AluOp::t, AluOp::t, AluOp::t, "LOG_CLAMPED")}, - {op1_log_ieee ,AluOp(1, 1, AluOp::t, AluOp::t, AluOp::t, "LOG_IEEE")}, - {op1_max4 ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "MAX4")}, - {op1_mbcnt_32hi_int ,AluOp(1, 0, AluOp::v, AluOp::v, AluOp::v, "MBCNT_32HI_INT")}, - {op1_mbcnt_32lo_accum_prev_int ,AluOp(1, 0, AluOp::v, AluOp::v, AluOp::v, "MBCNT_32LO_ACCUM_PREV_INT")}, - {op1_mov ,AluOp(1, 0, AluOp::a, AluOp::a, AluOp::a, "MOV")}, - {op1_mova_int ,AluOp(1, 0, AluOp::v, AluOp::v, AluOp::v, "MOVA_INT")}, - {op1_not_int ,AluOp(1, 0, AluOp::a, AluOp::a, AluOp::a, "NOT_INT")}, - {op1_offset_to_flt ,AluOp(1, 0, AluOp::v, AluOp::v, AluOp::v, "OFFSET_TO_FLT")}, - {op1_pred_set_inv ,AluOp(1, 1, AluOp::a, AluOp::a, AluOp::a, "PRED_SET_INV")}, - {op1_pred_set_restore ,AluOp(1, 1, AluOp::a, AluOp::a, AluOp::a, "PRED_SET_RESTORE")}, - {op1_set_cf_idx0 ,AluOp(1, 0, AluOp::a, AluOp::a, AluOp::a, "SET_CF_IDX0")}, /* Reads from AR register? */ - {op1_set_cf_idx1 ,AluOp(1, 0, AluOp::a, AluOp::a, AluOp::a, "SET_CF_IDX1")}, /* Reads from AR register? */ - {op1_recip_clamped ,AluOp(1, 1, AluOp::t, AluOp::t, AluOp::t, "RECIP_CLAMPED")}, - {op1_recip_ff ,AluOp(1, 1, AluOp::t, AluOp::t, AluOp::t, "RECIP_FF")}, - {op1_recip_ieee ,AluOp(1, 1, AluOp::t, AluOp::t, AluOp::t, "RECIP_IEEE")}, - {op1_recipsqrt_clamped ,AluOp(1, 1, AluOp::t, AluOp::t, AluOp::t, "RECIPSQRT_CLAMPED")}, - {op1_recipsqrt_ff ,AluOp(1, 1, AluOp::t, AluOp::t, AluOp::t, "RECIPSQRT_FF")}, - {op1_recipsqrt_ieee1 ,AluOp(1, 1, AluOp::t, AluOp::t, AluOp::t, "RECIPSQRT_IEEE")}, - {op1_recip_int ,AluOp(1, 0, AluOp::t, AluOp::t, AluOp::t, "RECIP_INT")}, - {op1_recip_uint ,AluOp(1, 0, AluOp::t, AluOp::t, AluOp::t, "RECIP_UINT")}, - {op1_recip_64 ,AluOp(2, 1, AluOp::t, AluOp::t, AluOp::t, "RECIP_64")}, - {op1_recip_clamped_64 ,AluOp(2, 1, AluOp::t, AluOp::t, AluOp::t, "RECIP_CLAMPED_64")}, - {op1_recipsqrt_64 ,AluOp(2, 1, AluOp::t, AluOp::t, AluOp::t, "RECIPSQRT_64")}, - {op1_recipsqrt_clamped_64,AluOp(2, 1, AluOp::t, AluOp::t, AluOp::t, "RECIPSQRT_CLAMPED_64")}, - {op1_rndne ,AluOp(1, 1, AluOp::a, AluOp::a, AluOp::a, "RNDNE")}, - {op1_sqrt_ieee ,AluOp(1, 1, AluOp::t, AluOp::t, AluOp::t, "SQRT_IEEE")}, - {op1_sin ,AluOp(1, 1, AluOp::t, AluOp::t, AluOp::t, "SIN")}, - {op1_trunc ,AluOp(1, 1, AluOp::a, AluOp::a, AluOp::a, "TRUNC")}, - {op1_sqrt_64 ,AluOp(2, 1, AluOp::t, AluOp::t, AluOp::t, "SQRT_64")}, - {op1_ubyte0_flt ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "UBYTE0_FLT")}, - {op1_ubyte1_flt ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "UBYTE1_FLT")}, - {op1_ubyte2_flt ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "UBYTE2_FLT")}, - {op1_ubyte3_flt ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "UBYTE3_FLT")}, - {op1_uint_to_flt ,AluOp(1, 0, AluOp::t, AluOp::t, AluOp::t, "UINT_TO_FLT")}, - {op1_ffbh_uint ,AluOp(1, 0, AluOp::v, AluOp::v, AluOp::v, "FFBH_UINT")}, - {op1_ffbl_int ,AluOp(1, 0, AluOp::v, AluOp::v, AluOp::v, "FFBL_INT")}, - {op1_ffbh_int ,AluOp(1, 0, AluOp::v, AluOp::v, AluOp::v, "FFBH_INT")}, - {op1_flt_to_uint4 ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "FLT_TO_UINT4")}, - {op1v_flt32_to_flt64 ,AluOp(1, 1, AluOp::a, AluOp::a, AluOp::a, "FLT32_TO_FLT64")}, - {op1v_flt64_to_flt32 ,AluOp(1, 1, AluOp::v, AluOp::v, AluOp::v, "FLT64_TO_FLT32")}, + {op1_bcnt_int, AluOp(1, 0, V, V, V, "BCNT_INT") }, + {op1_bcnt_accum_prev_int, AluOp(1, 0, V, V, V, "BCNT_ACCUM_PREV_INT") }, + {op1_bfrev_int, AluOp(1, 0, A, A, A, "BFREV_INT") }, + {op1_ceil, AluOp(1, 1, A, A, A, "CEIL") }, + {op1_cos, AluOp(1, 1, T, T, T, "COS") }, + {op1_exp_ieee, AluOp(1, 1, T, T, T, "EXP_IEEE") }, + {op1_floor, AluOp(1, 1, A, A, A, "FLOOR") }, + {op1_flt_to_int, AluOp(1, 0, T, T, V, "FLT_TO_INT") }, + {op1_flt_to_uint, AluOp(1, 1, T, T, T, "FLT_TO_UINT") }, + {op1_flt_to_int_rpi, AluOp(1, 1, V, V, V, "FLT_TO_INT_RPI") }, + {op1_flt_to_int_floor, AluOp(1, 1, V, V, V, "FLT_TO_INT_FLOOR") }, + {op1_flt16_to_flt32, AluOp(1, 1, V, V, V, "FLT16_TO_FLT32") }, + {op1_flt32_to_flt16, AluOp(1, 1, V, V, V, "FLT32_TO_FLT16") }, + {op1_flt32_to_flt64, AluOp(1, 1, V, V, V, "FLT32_TO_FLT64") }, + {op1_flt64_to_flt32, AluOp(1, 1, A, A, A, "FLT64_TO_FLT32") }, + {op1_fract, AluOp(1, 1, A, A, A, "FRACT") }, + {op1_fract_64, AluOp(1, 1, V, V, V, "FRACT_64") }, + {op1_frexp_64, AluOp(1, 1, V, V, V, "FREXP_64") }, + {op1_int_to_flt, AluOp(1, 0, T, T, T, "INT_TO_FLT") }, + {op1_ldexp_64, AluOp(1, 1, V, V, V, "LDEXP_64") }, + {op1_interp_load_p0, AluOp(1, 1, V, V, V, "INTERP_LOAD_P0") }, + {op1_interp_load_p10, AluOp(1, 1, V, V, V, "INTERP_LOAD_P10") }, + {op1_interp_load_p20, AluOp(1, 1, V, V, V, "INTERP_LOAD_P20") }, + {op1_load_store_flags, AluOp(1, 0, V, V, V, "LOAD_STORE_FLAGS") }, + {op1_log_clamped, AluOp(1, 1, T, T, T, "LOG_CLAMPED") }, + {op1_log_ieee, AluOp(1, 1, T, T, T, "LOG_IEEE") }, + {op1_max4, AluOp(1, 1, V, V, V, "MAX4") }, + {op1_mbcnt_32hi_int, AluOp(1, 0, V, V, V, "MBCNT_32HI_INT") }, + {op1_mbcnt_32lo_accum_prev_int, AluOp(1, 0, V, V, V, "MBCNT_32LO_ACCUM_PREV_INT")}, + {op1_mov, AluOp(1, 0, A, A, A, "MOV") }, + {op1_mova_int, AluOp(1, 0, X, X, X, "MOVA_INT") }, + {op1_not_int, AluOp(1, 0, A, A, A, "NOT_INT") }, + {op1_offset_to_flt, AluOp(1, 0, V, V, V, "OFFSET_TO_FLT") }, + {op1_pred_set_inv, AluOp(1, 1, A, A, A, "PRED_SET_INV") }, + {op1_pred_set_restore, AluOp(1, 1, A, A, A, "PRED_SET_RESTORE") }, + {op1_set_cf_idx0, AluOp(1, 0, A, A, A, "SET_CF_IDX0") }, + {op1_set_cf_idx1, AluOp(1, 0, A, A, A, "SET_CF_IDX1") }, + {op1_recip_clamped, AluOp(1, 1, T, T, T, "RECIP_CLAMPED") }, + {op1_recip_ff, AluOp(1, 1, T, T, T, "RECIP_FF") }, + {op1_recip_ieee, AluOp(1, 1, T, T, T, "RECIP_IEEE") }, + {op1_recipsqrt_clamped, AluOp(1, 1, T, T, T, "RECIPSQRT_CLAMPED") }, + {op1_recipsqrt_ff, AluOp(1, 1, T, T, T, "RECIPSQRT_FF") }, + {op1_recipsqrt_ieee1, AluOp(1, 1, T, T, T, "RECIPSQRT_IEEE") }, + {op1_recip_int, AluOp(1, 0, T, T, T, "RECIP_INT") }, + {op1_recip_uint, AluOp(1, 0, T, T, T, "RECIP_UINT") }, + {op1_recip_64, AluOp(2, 1, T, T, T, "RECIP_64") }, + {op1_recip_clamped_64, AluOp(2, 1, T, T, T, "RECIP_CLAMPED_64") }, + {op1_recipsqrt_64, AluOp(2, 1, T, T, T, "RECIPSQRT_64") }, + {op1_recipsqrt_clamped_64, AluOp(2, 1, T, T, T, "RECIPSQRT_CLAMPED_64") }, + {op1_rndne, AluOp(1, 1, A, A, A, "RNDNE") }, + {op1_sqrt_ieee, AluOp(1, 1, T, T, T, "SQRT_IEEE") }, + {op1_sin, AluOp(1, 1, T, T, T, "SIN") }, + {op1_trunc, AluOp(1, 1, A, A, A, "TRUNC") }, + {op1_sqrt_64, AluOp(2, 1, T, T, T, "SQRT_64") }, + {op1_ubyte0_flt, AluOp(1, 1, V, V, V, "UBYTE0_FLT") }, + {op1_ubyte1_flt, AluOp(1, 1, V, V, V, "UBYTE1_FLT") }, + {op1_ubyte2_flt, AluOp(1, 1, V, V, V, "UBYTE2_FLT") }, + {op1_ubyte3_flt, AluOp(1, 1, V, V, V, "UBYTE3_FLT") }, + {op1_uint_to_flt, AluOp(1, 0, T, T, T, "UINT_TO_FLT") }, + {op1_ffbh_uint, AluOp(1, 0, V, V, V, "FFBH_UINT") }, + {op1_ffbl_int, AluOp(1, 0, V, V, V, "FFBL_INT") }, + {op1_ffbh_int, AluOp(1, 0, V, V, V, "FFBH_INT") }, + {op1_flt_to_uint4, AluOp(1, 1, V, V, V, "FLT_TO_UINT4") }, + {op1v_flt32_to_flt64, AluOp(1, 1, A, A, A, "FLT32_TO_FLT64") }, + {op1v_flt64_to_flt32, AluOp(1, 1, V, V, V, "FLT64_TO_FLT32") }, - {op2_add ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "ADD")}, - {op2_bfm_int ,AluOp(2, 0, AluOp::v, AluOp::v, AluOp::v, "BFM_INT")}, - {op2_mul ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "MUL")}, - {op2_mul_ieee ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "MUL_IEEE")}, - {op2_max ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "MAX")}, - {op2_min ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "MIN")}, - {op2_max_dx10 ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "MAX_DX10")}, - {op2_min_dx10 ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "MIN_DX10")}, - {op2_sete ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "SETE")}, - {op2_setgt ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "SETGT")}, - {op2_setge ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "SETGE")}, - {op2_setne ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "SETNE")}, - {op2_sete_dx10 ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "SETE_DX10")}, - {op2_setgt_dx10 ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "SETGT_DX10")}, - {op2_setge_dx10 ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "SETGE_DX10")}, - {op2_setne_dx10 ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "SETNE_DX10")}, - {op2_ashr_int ,AluOp(2, 0, AluOp::t, AluOp::a, AluOp::a, "ASHR_INT")}, - {op2_lshr_int ,AluOp(2, 0, AluOp::t, AluOp::a, AluOp::a, "LSHR_INT")}, - {op2_lshl_int ,AluOp(2, 0, AluOp::t, AluOp::a, AluOp::a, "LSHL_INT")}, - {op2_mul_64 ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "MUL_64")}, - {op2_pred_setgt_uint ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "PRED_SETGT_UINT")}, - {op2_pred_setge_uint ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "PRED_SETGE_UINT")}, - {op2_pred_sete ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "PRED_SETE")}, - {op2_pred_setgt ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "PRED_SETGT")}, - {op2_pred_setge ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "PRED_SETGE")}, - {op2_pred_setne ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "PRED_SETNE")}, - {op2_pred_set_pop ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "PRED_SET_POP")}, - {op2_pred_sete_push ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "PRED_SETE_PUSH")}, - {op2_pred_setgt_push ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "PRED_SETGT_PUSH")}, - {op2_pred_setge_push ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "PRED_SETGE_PUSH")}, - {op2_pred_setne_push ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "PRED_SETNE_PUSH")}, - {op2_kille ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "KILLE")}, - {op2_killgt ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "KILLGT")}, - {op2_killge ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "KILLGE")}, - {op2_killne ,AluOp(2, 1, AluOp::a, AluOp::a, AluOp::a, "KILLNE")}, - {op2_and_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "AND_INT")}, - {op2_or_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "OR_INT")}, - {op2_xor_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "XOR_INT")}, - {op2_add_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "ADD_INT")}, - {op2_sub_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "SUB_INT")}, - {op2_max_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "MAX_INT")}, - {op2_min_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "MIN_INT")}, - {op2_max_uint ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "MAX_UINT")}, - {op2_min_uint ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "MIN_UINT")}, - {op2_sete_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "SETE_INT")}, - {op2_setgt_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "SETGT_INT")}, - {op2_setge_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "SETGE_INT")}, - {op2_setne_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "SETNE_INT")}, - {op2_setgt_uint ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "SETGT_UINT")}, - {op2_setge_uint ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "SETGE_UINT")}, - {op2_killgt_uint ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "KILLGT_UINT")}, - {op2_killge_uint ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "KILLGE_UINT")}, - {op2_prede_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "PREDE_INT")}, - {op2_pred_setgt_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "PRED_SETGT_INT")}, - {op2_pred_setge_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "PRED_SETGE_INT")}, - {op2_pred_setne_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "PRED_SETNE_INT")}, - {op2_kille_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "KILLE_INT")}, - {op2_killgt_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "KILLGT_INT")}, - {op2_killge_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "KILLGE_INT")}, - {op2_killne_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "KILLNE_INT")}, - {op2_pred_sete_push_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "PRED_SETE_PUSH_INT")}, - {op2_pred_setgt_push_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "PRED_SETGT_PUSH_INT")}, - {op2_pred_setge_push_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "PRED_SETGE_PUSH_INT")}, - {op2_pred_setne_push_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "PRED_SETNE_PUSH_INT")}, - {op2_pred_setlt_push_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "PRED_SETLT_PUSH_INT")}, - {op2_pred_setle_push_int ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "PRED_SETLE_PUSH_INT")}, - {op2_addc_uint ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "ADDC_UINT")}, - {op2_subb_uint ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "SUBB_UINT")}, - {op2_set_mode ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "SET_MODE")}, - {op2_set_lds_size ,AluOp(2, 0, AluOp::a, AluOp::a, AluOp::a, "SET_LDS_SIZE")}, - {op2_mullo_int ,AluOp(2, 0, AluOp::t, AluOp::t, AluOp::t, "MULLO_INT")}, - {op2_mulhi_int ,AluOp(2, 0, AluOp::t, AluOp::t, AluOp::t, "MULHI_INT")}, - {op2_mullo_uint ,AluOp(2, 0, AluOp::t, AluOp::t, AluOp::t, "MULLO_UINT")}, - {op2_mulhi_uint ,AluOp(2, 0, AluOp::t, AluOp::t, AluOp::t, "MULHI_UINT")}, - {op2_dot_ieee ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "DOT_IEEE")}, - {op2_mulhi_uint24 ,AluOp(2, 0, AluOp::v, AluOp::v, AluOp::v, "MULHI_UINT24")}, - {op2_mul_uint24 ,AluOp(2, 0, AluOp::v, AluOp::v, AluOp::v, "MUL_UINT24")}, - {op2_sete_64 ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "SETE_64")}, - {op2_setne_64 ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "SETNE_64")}, - {op2_setgt_64 ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "SETGT_64")}, - {op2_setge_64 ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "SETGE_64")}, - {op2_min_64 ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "MIN_64")}, - {op2_max_64 ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "MAX_64")}, - {op2_dot4 ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "DOT4")}, - {op2_dot4_ieee ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "DOT4_IEEE")}, - {op2_cube ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "CUBE")}, - {op2_pred_setgt_64 ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "PRED_SETGT_64")}, - {op2_pred_sete_64 ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "PRED_SETE_64")}, - {op2_pred_setge_64 ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "PRED_SETGE_64")}, - {OP2V_MUL_64 ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "MUL_64")}, - {op2_add_64 ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "ADD_64")}, - {op2_sad_accum_prev_uint ,AluOp(2, 0, AluOp::v, AluOp::v, AluOp::v, "SAD_ACCUM_PREV_UINT")}, - {op2_dot ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "DOT")}, - {op1_mul_prev ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "MUL_PREV")}, - {op1_mul_ieee_prev ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "MUL_IEEE_PREV")}, - {op1_add_prev ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "ADD_PREV")}, - {op2_muladd_prev ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "MULADD_PREV")}, - {op2_muladd_ieee_prev ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "MULADD_IEEE_PREV")}, - {op2_interp_xy ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "INTERP_XY")}, - {op2_interp_zw ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "INTERP_ZW")}, - {op2_interp_x ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "INTERP_X")}, - {op2_interp_z ,AluOp(2, 1, AluOp::v, AluOp::v, AluOp::v, "INTERP_Z")}, + {op2_add, AluOp(2, 1, A, A, A, "ADD") }, + {op2_bfm_int, AluOp(2, 0, V, V, V, "BFM_INT") }, + {op2_mul, AluOp(2, 1, A, A, A, "MUL") }, + {op2_mul_ieee, AluOp(2, 1, A, A, A, "MUL_IEEE") }, + {op2_max, AluOp(2, 1, A, A, A, "MAX") }, + {op2_min, AluOp(2, 1, A, A, A, "MIN") }, + {op2_max_dx10, AluOp(2, 1, A, A, A, "MAX_DX10") }, + {op2_min_dx10, AluOp(2, 1, A, A, A, "MIN_DX10") }, + {op2_sete, AluOp(2, 1, A, A, A, "SETE") }, + {op2_setgt, AluOp(2, 1, A, A, A, "SETGT") }, + {op2_setge, AluOp(2, 1, A, A, A, "SETGE") }, + {op2_setne, AluOp(2, 1, A, A, A, "SETNE") }, + {op2_sete_dx10, AluOp(2, 1, A, A, A, "SETE_DX10") }, + {op2_setgt_dx10, AluOp(2, 1, A, A, A, "SETGT_DX10") }, + {op2_setge_dx10, AluOp(2, 1, A, A, A, "SETGE_DX10") }, + {op2_setne_dx10, AluOp(2, 1, A, A, A, "SETNE_DX10") }, + {op2_ashr_int, AluOp(2, 0, T, A, A, "ASHR_INT") }, + {op2_lshr_int, AluOp(2, 0, T, A, A, "LSHR_INT") }, + {op2_lshl_int, AluOp(2, 0, T, A, A, "LSHL_INT") }, + {op2_mul_64, AluOp(2, 1, A, A, A, "MUL_64") }, + {op2_pred_setgt_uint, AluOp(2, 0, A, A, A, "PRED_SETGT_UINT") }, + {op2_pred_setge_uint, AluOp(2, 0, A, A, A, "PRED_SETGE_UINT") }, + {op2_pred_sete, AluOp(2, 1, A, A, A, "PRED_SETE") }, + {op2_pred_setgt, AluOp(2, 1, A, A, A, "PRED_SETGT") }, + {op2_pred_setge, AluOp(2, 1, A, A, A, "PRED_SETGE") }, + {op2_pred_setne, AluOp(2, 1, A, A, A, "PRED_SETNE") }, + {op2_pred_set_pop, AluOp(2, 1, A, A, A, "PRED_SET_POP") }, + {op2_pred_sete_push, AluOp(2, 1, A, A, A, "PRED_SETE_PUSH") }, + {op2_pred_setgt_push, AluOp(2, 1, A, A, A, "PRED_SETGT_PUSH") }, + {op2_pred_setge_push, AluOp(2, 1, A, A, A, "PRED_SETGE_PUSH") }, + {op2_pred_setne_push, AluOp(2, 1, A, A, A, "PRED_SETNE_PUSH") }, + {op2_kille, AluOp(2, 1, A, A, A, "KILLE") }, + {op2_killgt, AluOp(2, 1, A, A, A, "KILLGT") }, + {op2_killge, AluOp(2, 1, A, A, A, "KILLGE") }, + {op2_killne, AluOp(2, 1, A, A, A, "KILLNE") }, + {op2_and_int, AluOp(2, 0, A, A, A, "AND_INT") }, + {op2_or_int, AluOp(2, 0, A, A, A, "OR_INT") }, + {op2_xor_int, AluOp(2, 0, A, A, A, "XOR_INT") }, + {op2_add_int, AluOp(2, 0, A, A, A, "ADD_INT") }, + {op2_sub_int, AluOp(2, 0, A, A, A, "SUB_INT") }, + {op2_max_int, AluOp(2, 0, A, A, A, "MAX_INT") }, + {op2_min_int, AluOp(2, 0, A, A, A, "MIN_INT") }, + {op2_max_uint, AluOp(2, 0, A, A, A, "MAX_UINT") }, + {op2_min_uint, AluOp(2, 0, A, A, A, "MIN_UINT") }, + {op2_sete_int, AluOp(2, 0, A, A, A, "SETE_INT") }, + {op2_setgt_int, AluOp(2, 0, A, A, A, "SETGT_INT") }, + {op2_setge_int, AluOp(2, 0, A, A, A, "SETGE_INT") }, + {op2_setne_int, AluOp(2, 0, A, A, A, "SETNE_INT") }, + {op2_setgt_uint, AluOp(2, 0, A, A, A, "SETGT_UINT") }, + {op2_setge_uint, AluOp(2, 0, A, A, A, "SETGE_UINT") }, + {op2_killgt_uint, AluOp(2, 0, A, A, A, "KILLGT_UINT") }, + {op2_killge_uint, AluOp(2, 0, A, A, A, "KILLGE_UINT") }, + {op2_prede_int, AluOp(2, 0, A, A, A, "PREDE_INT") }, + {op2_pred_setgt_int, AluOp(2, 0, A, A, A, "PRED_SETGT_INT") }, + {op2_pred_setge_int, AluOp(2, 0, A, A, A, "PRED_SETGE_INT") }, + {op2_pred_setne_int, AluOp(2, 0, A, A, A, "PRED_SETNE_INT") }, + {op2_kille_int, AluOp(2, 0, A, A, A, "KILLE_INT") }, + {op2_killgt_int, AluOp(2, 0, A, A, A, "KILLGT_INT") }, + {op2_killge_int, AluOp(2, 0, A, A, A, "KILLGE_INT") }, + {op2_killne_int, AluOp(2, 0, A, A, A, "KILLNE_INT") }, + {op2_pred_sete_push_int, AluOp(2, 0, A, A, A, "PRED_SETE_PUSH_INT") }, + {op2_pred_setgt_push_int, AluOp(2, 0, A, A, A, "PRED_SETGT_PUSH_INT") }, + {op2_pred_setge_push_int, AluOp(2, 0, A, A, A, "PRED_SETGE_PUSH_INT") }, + {op2_pred_setne_push_int, AluOp(2, 0, A, A, A, "PRED_SETNE_PUSH_INT") }, + {op2_pred_setlt_push_int, AluOp(2, 0, A, A, A, "PRED_SETLT_PUSH_INT") }, + {op2_pred_setle_push_int, AluOp(2, 0, A, A, A, "PRED_SETLE_PUSH_INT") }, + {op2_addc_uint, AluOp(2, 0, A, A, A, "ADDC_UINT") }, + {op2_subb_uint, AluOp(2, 0, A, A, A, "SUBB_UINT") }, + {op2_set_mode, AluOp(2, 0, A, A, A, "SET_MODE") }, + {op2_set_lds_size, AluOp(2, 0, A, A, A, "SET_LDS_SIZE") }, + {op2_mullo_int, AluOp(2, 0, T, T, T, "MULLO_INT") }, + {op2_mulhi_int, AluOp(2, 0, T, T, T, "MULHI_INT") }, + {op2_mullo_uint, AluOp(2, 0, T, T, T, "MULLO_UINT") }, + {op2_mulhi_uint, AluOp(2, 0, T, T, T, "MULHI_UINT") }, + {op2_dot_ieee, AluOp(2, 1, V, V, V, "DOT_IEEE") }, + {op2_mulhi_uint24, AluOp(2, 0, V, V, V, "MULHI_UINT24") }, + {op2_mul_uint24, AluOp(2, 0, V, V, V, "MUL_UINT24") }, + {op2_sete_64, AluOp(2, 1, V, V, V, "SETE_64") }, + {op2_setne_64, AluOp(2, 1, V, V, V, "SETNE_64") }, + {op2_setgt_64, AluOp(2, 1, V, V, V, "SETGT_64") }, + {op2_setge_64, AluOp(2, 1, V, V, V, "SETGE_64") }, + {op2_min_64, AluOp(2, 1, V, V, V, "MIN_64") }, + {op2_max_64, AluOp(2, 1, V, V, V, "MAX_64") }, + {op2_dot4, AluOp(2, 1, V, V, V, "DOT4") }, + {op2_dot4_ieee, AluOp(2, 1, V, V, V, "DOT4_IEEE") }, + {op2_cube, AluOp(2, 1, V, V, V, "CUBE") }, + {op2_pred_setgt_64, AluOp(2, 1, V, V, V, "PRED_SETGT_64") }, + {op2_pred_sete_64, AluOp(2, 1, V, V, V, "PRED_SETE_64") }, + {op2_pred_setge_64, AluOp(2, 1, V, V, V, "PRED_SETGE_64") }, + {OP2V_MUL_64, AluOp(2, 1, V, V, V, "MUL_64") }, + {op2_add_64, AluOp(2, 1, V, V, V, "ADD_64") }, + {op2_sad_accum_prev_uint, AluOp(2, 0, V, V, V, "SAD_ACCUM_PREV_UINT") }, + {op2_dot, AluOp(2, 1, V, V, V, "DOT") }, + {op1_mul_prev, AluOp(2, 1, V, V, V, "MUL_PREV") }, + {op1_mul_ieee_prev, AluOp(2, 1, V, V, V, "MUL_IEEE_PREV") }, + {op1_add_prev, AluOp(2, 1, V, V, V, "ADD_PREV") }, + {op2_muladd_prev, AluOp(2, 1, V, V, V, "MULADD_PREV") }, + {op2_muladd_ieee_prev, AluOp(2, 1, V, V, V, "MULADD_IEEE_PREV") }, + {op2_interp_xy, AluOp(2, 1, V, V, V, "INTERP_XY") }, + {op2_interp_zw, AluOp(2, 1, V, V, V, "INTERP_ZW") }, + {op2_interp_x, AluOp(2, 1, V, V, V, "INTERP_X") }, + {op2_interp_z, AluOp(2, 1, V, V, V, "INTERP_Z") }, - {op3_bfe_uint ,AluOp(3, 0, AluOp::v, AluOp::v, AluOp::v, "BFE_UINT")}, - {op3_bfe_int ,AluOp(3, 0, AluOp::v, AluOp::v, AluOp::v, "BFE_INT")}, - {op3_bfi_int ,AluOp(3, 0, AluOp::v, AluOp::v, AluOp::v, "BFI_INT")}, - {op3_fma ,AluOp(3, 1, AluOp::v, AluOp::v, AluOp::v, "FMA")}, - {op3_cndne_64 ,AluOp(3, 1, AluOp::v, AluOp::v, AluOp::v, "CNDNE_64")}, - {op3_fma_64 ,AluOp(3, 1, AluOp::v, AluOp::v, AluOp::v, "FMA_64")}, - {op3_lerp_uint ,AluOp(3, 0, AluOp::v, AluOp::v, AluOp::v, "LERP_UINT")}, - {op3_bit_align_int ,AluOp(3, 0, AluOp::v, AluOp::v, AluOp::v, "BIT_ALIGN_INT")}, - {op3_byte_align_int ,AluOp(3, 0, AluOp::v, AluOp::v, AluOp::v, "BYTE_ALIGN_INT")}, - {op3_sad_accum_uint ,AluOp(3, 0, AluOp::v, AluOp::v, AluOp::v, "SAD_ACCUM_UINT")}, - {op3_sad_accum_hi_uint ,AluOp(3, 0, AluOp::v, AluOp::v, AluOp::v, "SAD_ACCUM_HI_UINT")}, - {op3_muladd_uint24 ,AluOp(3, 0, AluOp::v, AluOp::v, AluOp::v, "MULADD_UINT24")}, - {op3_lds_idx_op ,AluOp(3, 0, AluOp::x, AluOp::x, AluOp::x,"LDS_IDX_OP")}, - {op3_muladd ,AluOp(3, 1, AluOp::a, AluOp::a, AluOp::a, "MULADD")}, - {op3_muladd_m2 ,AluOp(3, 1, AluOp::a, AluOp::a, AluOp::a, "MULADD_M2")}, - {op3_muladd_m4 ,AluOp(3, 1, AluOp::a, AluOp::a, AluOp::a, "MULADD_M4")}, - {op3_muladd_d2 ,AluOp(3, 1, AluOp::a, AluOp::a, AluOp::a, "MULADD_D2")}, - {op3_muladd_ieee ,AluOp(3, 1, AluOp::a, AluOp::a, AluOp::a, "MULADD_IEEE")}, - {op3_cnde ,AluOp(3, 1, AluOp::a, AluOp::a, AluOp::a, "CNDE")}, - {op3_cndgt ,AluOp(3, 1, AluOp::a, AluOp::a, AluOp::a, "CNDGT")}, - {op3_cndge ,AluOp(3, 1, AluOp::a, AluOp::a, AluOp::a, "CNDGE")}, - {op3_cnde_int ,AluOp(3, 0, AluOp::a, AluOp::a, AluOp::a, "CNDE_INT")}, - {op3_cndgt_int ,AluOp(3, 0, AluOp::a, AluOp::a, AluOp::a, "CNDGT_INT")}, - {op3_cndge_int ,AluOp(3, 0, AluOp::a, AluOp::a, AluOp::a, "CNDGE_INT")}, - {op3_mul_lit ,AluOp(3, 1, AluOp::t, AluOp::t, AluOp::t, "MUL_LIT")} + {op3_bfe_uint, AluOp(3, 0, V, V, V, "BFE_UINT") }, + {op3_bfe_int, AluOp(3, 0, V, V, V, "BFE_INT") }, + {op3_bfi_int, AluOp(3, 0, V, V, V, "BFI_INT") }, + {op3_fma, AluOp(3, 1, V, V, V, "FMA") }, + {op3_cndne_64, AluOp(3, 1, V, V, V, "CNDNE_64") }, + {op3_fma_64, AluOp(3, 1, V, V, V, "FMA_64") }, + {op3_lerp_uint, AluOp(3, 0, V, V, V, "LERP_UINT") }, + {op3_bit_align_int, AluOp(3, 0, V, V, V, "BIT_ALIGN_INT") }, + {op3_byte_align_int, AluOp(3, 0, V, V, V, "BYTE_ALIGN_INT") }, + {op3_sad_accum_uint, AluOp(3, 0, V, V, V, "SAD_ACCUM_UINT") }, + {op3_sad_accum_hi_uint, AluOp(3, 0, V, V, V, "SAD_ACCUM_HI_UINT") }, + {op3_muladd_uint24, AluOp(3, 0, V, V, V, "MULADD_UINT24") }, + {op3_lds_idx_op, AluOp(3, 0, X, X, X, "LDS_IDX_OP") }, + {op3_muladd, AluOp(3, 1, A, A, A, "MULADD") }, + {op3_muladd_m2, AluOp(3, 1, A, A, A, "MULADD_M2") }, + {op3_muladd_m4, AluOp(3, 1, A, A, A, "MULADD_M4") }, + {op3_muladd_d2, AluOp(3, 1, A, A, A, "MULADD_D2") }, + {op3_muladd_ieee, AluOp(3, 1, A, A, A, "MULADD_IEEE") }, + {op3_cnde, AluOp(3, 1, A, A, A, "CNDE") }, + {op3_cndgt, AluOp(3, 1, A, A, A, "CNDGT") }, + {op3_cndge, AluOp(3, 1, A, A, A, "CNDGE") }, + {op3_cnde_int, AluOp(3, 0, A, A, A, "CNDE_INT") }, + {op3_cndgt_int, AluOp(3, 0, A, A, A, "CNDGT_INT") }, + {op3_cndge_int, AluOp(3, 0, A, A, A, "CNDGE_INT") }, + {op3_mul_lit, AluOp(3, 1, T, T, T, "MUL_LIT") } }; +#undef A +#undef V +#undef T +#undef X + const std::map alu_src_const = { - {ALU_SRC_LDS_OQ_A, {false, "LDS_OQ_A"}}, - {ALU_SRC_LDS_OQ_B, {false, "LDS_OQ_B"}}, - {ALU_SRC_LDS_OQ_A_POP, {false, "LDS_OQ_A_POP"}}, - {ALU_SRC_LDS_OQ_B_POP, {false, "LDS_OQ_B_POP"}}, - {ALU_SRC_LDS_DIRECT_A, {false, "LDS_DIRECT_A"}}, - {ALU_SRC_LDS_DIRECT_B, {false, "LDS_DIRECT_B"}}, - {ALU_SRC_TIME_HI, {false, "TIME_HI"}}, - {ALU_SRC_TIME_LO, {false, "TIME_LO"}}, - {ALU_SRC_MASK_HI, {false, "MASK_HI"}}, - {ALU_SRC_MASK_LO, {false, "MASK_LO"}}, - {ALU_SRC_HW_WAVE_ID, {false, "HW_WAVE_ID"}}, - {ALU_SRC_SIMD_ID, {false, "SIMD_ID"}}, - {ALU_SRC_SE_ID, {false, "SE_ID"}}, - {ALU_SRC_HW_THREADGRP_ID, {false, "HW_THREADGRP_ID"}}, - {ALU_SRC_WAVE_ID_IN_GRP, {false, "WAVE_ID_IN_GRP"}}, + {ALU_SRC_LDS_OQ_A, {false, "LDS_OQ_A"} }, + {ALU_SRC_LDS_OQ_B, {false, "LDS_OQ_B"} }, + {ALU_SRC_LDS_OQ_A_POP, {false, "LDS_OQ_A_POP"} }, + {ALU_SRC_LDS_OQ_B_POP, {false, "LDS_OQ_B_POP"} }, + {ALU_SRC_LDS_DIRECT_A, {false, "LDS_DIRECT_A"} }, + {ALU_SRC_LDS_DIRECT_B, {false, "LDS_DIRECT_B"} }, + {ALU_SRC_TIME_HI, {false, "TIME_HI"} }, + {ALU_SRC_TIME_LO, {false, "TIME_LO"} }, + {ALU_SRC_MASK_HI, {false, "MASK_HI"} }, + {ALU_SRC_MASK_LO, {false, "MASK_LO"} }, + {ALU_SRC_HW_WAVE_ID, {false, "HW_WAVE_ID"} }, + {ALU_SRC_SIMD_ID, {false, "SIMD_ID"} }, + {ALU_SRC_SE_ID, {false, "SE_ID"} }, + {ALU_SRC_HW_THREADGRP_ID, {false, "HW_THREADGRP_ID"} }, + {ALU_SRC_WAVE_ID_IN_GRP, {false, "WAVE_ID_IN_GRP"} }, {ALU_SRC_NUM_THREADGRP_WAVES, {false, "NUM_THREADGRP_WAVES"}}, - {ALU_SRC_HW_ALU_ODD, {false, "HW_ALU_ODD"}}, - {ALU_SRC_LOOP_IDX, {false, "LOOP_IDX"}}, - {ALU_SRC_PARAM_BASE_ADDR, {false, "PARAM_BASE_ADDR"}}, - {ALU_SRC_NEW_PRIM_MASK, {false, "NEW_PRIM_MASK"}}, - {ALU_SRC_PRIM_MASK_HI, {false, "PRIM_MASK_HI"}}, - {ALU_SRC_PRIM_MASK_LO, {false, "PRIM_MASK_LO"}}, - {ALU_SRC_1_DBL_L, {false, "1.0L"}}, - {ALU_SRC_1_DBL_M, {false, "1.0H"}}, - {ALU_SRC_0_5_DBL_L, {false, "0.5L"}}, - {ALU_SRC_0_5_DBL_M, {false, "0.5H"}}, - {ALU_SRC_0, {false, "0"}}, - {ALU_SRC_1, {false, "1.0"}}, - {ALU_SRC_1_INT, {false, "1"}}, - {ALU_SRC_M_1_INT, {false, "-1"}}, - {ALU_SRC_0_5, {false, "0.5"}}, - {ALU_SRC_LITERAL, {true, "ALU_SRC_LITERAL"}}, - {ALU_SRC_PV, {true, "PV"}}, - {ALU_SRC_PS, {false, "PS"}} + {ALU_SRC_HW_ALU_ODD, {false, "HW_ALU_ODD"} }, + {ALU_SRC_LOOP_IDX, {false, "LOOP_IDX"} }, + {ALU_SRC_PARAM_BASE_ADDR, {false, "PARAM_BASE_ADDR"} }, + {ALU_SRC_NEW_PRIM_MASK, {false, "NEW_PRIM_MASK"} }, + {ALU_SRC_PRIM_MASK_HI, {false, "PRIM_MASK_HI"} }, + {ALU_SRC_PRIM_MASK_LO, {false, "PRIM_MASK_LO"} }, + {ALU_SRC_1_DBL_L, {false, "1.0L"} }, + {ALU_SRC_1_DBL_M, {false, "1.0H"} }, + {ALU_SRC_0_5_DBL_L, {false, "0.5L"} }, + {ALU_SRC_0_5_DBL_M, {false, "0.5H"} }, + {ALU_SRC_0, {false, "0"} }, + {ALU_SRC_1, {false, "1.0"} }, + {ALU_SRC_1_INT, {false, "1"} }, + {ALU_SRC_M_1_INT, {false, "-1"} }, + {ALU_SRC_0_5, {false, "0.5"} }, + {ALU_SRC_LITERAL, {true, "ALU_SRC_LITERAL"} }, + {ALU_SRC_PV, {true, "PV"} }, + {ALU_SRC_PS, {false, "PS"} } }; const std::map lds_ops = { - {DS_OP_ADD , {2, "ADD"}}, - {DS_OP_SUB , {2, "SUB"}}, - {DS_OP_RSUB , {2, "RSUB"}}, - {DS_OP_INC , {2, "INC"}}, - {DS_OP_DEC , {2, "DEC"}}, - {DS_OP_MIN_INT , {2, "MIN_INT"}}, - {DS_OP_MAX_INT , {2, "MAX_INT"}}, - {DS_OP_MIN_UINT , {2, "MIN_UINT"}}, - {DS_OP_MAX_UINT , {2, "MAX_UINT"}}, - {DS_OP_AND , {2, "AND"}}, - {DS_OP_OR , {2, "OR"}}, - {DS_OP_XOR , {2, "XOR"}}, - {DS_OP_MSKOR , {3, "MSKOR"}}, - {DS_OP_WRITE , {2, "WRITE"}}, - {DS_OP_WRITE_REL , {3, "WRITE_REL"}}, - {DS_OP_WRITE2 , {3, "WRITE2"}}, - {DS_OP_CMP_STORE , {3, "CMP_STORE"}}, - {DS_OP_CMP_STORE_SPF , {3, "CMP_STORE_SPF"}}, - {DS_OP_BYTE_WRITE , {2, "BYTE_WRITE"}}, - {DS_OP_SHORT_WRITE , {2, "SHORT_WRITE"}}, - {DS_OP_ADD_RET , {2, "ADD_RET"}}, - {DS_OP_SUB_RET , {2, "SUB_RET"}}, - {DS_OP_RSUB_RET , {2, "RSUB_RET"}}, - {DS_OP_INC_RET , {2, "INC_RET"}}, - {DS_OP_DEC_RET , {2, "DEC_RET"}}, - {DS_OP_MIN_INT_RET , {2, "MIN_INT_RET"}}, - {DS_OP_MAX_INT_RET , {2, "MAX_INT_RET"}}, - {DS_OP_MIN_UINT_RET , {2, "MIN_UINT_RET"}}, - {DS_OP_MAX_UINT_RET , {2, "MAX_UINT_RET"}}, - {DS_OP_AND_RET , {2, "AND_RET"}}, - {DS_OP_OR_RET , {2, "OR_RET"}}, - {DS_OP_XOR_RET , {2, "XOR_RET"}}, - {DS_OP_MSKOR_RET , {3, "MSKOR_RET"}}, - {DS_OP_XCHG_RET , {2, "XCHG_RET"}}, - {DS_OP_XCHG_REL_RET , {3, "XCHG_REL_RET"}}, - {DS_OP_XCHG2_RET , {3, "XCHG2_RET"}}, - {DS_OP_CMP_XCHG_RET , {3, "CMP_XCHG_RET"}}, - {DS_OP_CMP_XCHG_SPF_RET, {3, "CMP_XCHG_SPF_RET"}}, - {DS_OP_READ_RET , {1, "READ_RET"}}, - {DS_OP_READ_REL_RET , {1, "READ_REL_RET"}}, - {DS_OP_READ2_RET , {2, "READ2_RET"}}, - {DS_OP_READWRITE_RET , {3, "READWRITE_RET"}}, - {DS_OP_BYTE_READ_RET , {1, "BYTE_READ_RET"}}, - {DS_OP_UBYTE_READ_RET, {1, "UBYTE_READ_RET"}}, - {DS_OP_SHORT_READ_RET, {1, "SHORT_READ_RET"}}, - {DS_OP_USHORT_READ_RET, {1, "USHORT_READ_RET"}}, - {DS_OP_ATOMIC_ORDERED_ALLOC_RET , {3, "ATOMIC_ORDERED_ALLOC_RET"}}, - {LDS_ADD_RET, {2, "LDS_ADD_RET"}}, - {LDS_ADD, {2, "LDS_ADD"}}, - {LDS_AND_RET, {2, "LDS_AND_RET"}}, - {LDS_AND, {2, "LDS_AND"}}, - {LDS_WRITE, {2, "LDS_WRITE"}}, - {LDS_OR_RET, {2, "LDS_OR_RET"}}, - {LDS_OR, {2, "LDS_OR"}}, - {LDS_MAX_INT_RET, {2, "LDS_MAX_INT_RET"}}, - {LDS_MAX_INT, {2, "LDS_MAX_INT"}}, - {LDS_MAX_UINT_RET, {2, "LDS_MAX_UINT_RET"}}, - {LDS_MAX_UINT, {2, "LDS_MAX_UINT"}}, - {LDS_MIN_INT_RET, {2, "LDS_MIN_INT_RET"}}, - {LDS_MIN_INT, {2, "LDS_MIN_INT"}}, - {LDS_MIN_UINT_RET, {2, "LDS_MIN_UINT_RET"}}, - {LDS_MIN_UINT, {2, "LDS_MIN_UINT"}}, - {LDS_XOR_RET, {2, "LDS_XOR"}}, - {LDS_XOR, {2, "LDS_XOR"}}, - {LDS_XCHG_RET, {2, "LDS_XCHG_RET"}}, - {LDS_CMP_XCHG_RET, {3, "LDS_CMP_XCHG_RET"}}, - {LDS_WRITE_REL, {3, "LDS_WRITE_REL"}}, + {DS_OP_ADD, {2, "ADD"} }, + {DS_OP_SUB, {2, "SUB"} }, + {DS_OP_RSUB, {2, "RSUB"} }, + {DS_OP_INC, {2, "INC"} }, + {DS_OP_DEC, {2, "DEC"} }, + {DS_OP_MIN_INT, {2, "MIN_INT"} }, + {DS_OP_MAX_INT, {2, "MAX_INT"} }, + {DS_OP_MIN_UINT, {2, "MIN_UINT"} }, + {DS_OP_MAX_UINT, {2, "MAX_UINT"} }, + {DS_OP_AND, {2, "AND"} }, + {DS_OP_OR, {2, "OR"} }, + {DS_OP_XOR, {2, "XOR"} }, + {DS_OP_MSKOR, {3, "MSKOR"} }, + {DS_OP_WRITE, {2, "WRITE"} }, + {DS_OP_WRITE_REL, {3, "WRITE_REL"} }, + {DS_OP_WRITE2, {3, "WRITE2"} }, + {DS_OP_CMP_STORE, {3, "CMP_STORE"} }, + {DS_OP_CMP_STORE_SPF, {3, "CMP_STORE_SPF"} }, + {DS_OP_BYTE_WRITE, {2, "BYTE_WRITE"} }, + {DS_OP_SHORT_WRITE, {2, "SHORT_WRITE"} }, + {DS_OP_ADD_RET, {2, "ADD_RET"} }, + {DS_OP_SUB_RET, {2, "SUB_RET"} }, + {DS_OP_RSUB_RET, {2, "RSUB_RET"} }, + {DS_OP_INC_RET, {2, "INC_RET"} }, + {DS_OP_DEC_RET, {2, "DEC_RET"} }, + {DS_OP_MIN_INT_RET, {2, "MIN_INT_RET"} }, + {DS_OP_MAX_INT_RET, {2, "MAX_INT_RET"} }, + {DS_OP_MIN_UINT_RET, {2, "MIN_UINT_RET"} }, + {DS_OP_MAX_UINT_RET, {2, "MAX_UINT_RET"} }, + {DS_OP_AND_RET, {2, "AND_RET"} }, + {DS_OP_OR_RET, {2, "OR_RET"} }, + {DS_OP_XOR_RET, {2, "XOR_RET"} }, + {DS_OP_MSKOR_RET, {3, "MSKOR_RET"} }, + {DS_OP_XCHG_RET, {2, "XCHG_RET"} }, + {DS_OP_XCHG_REL_RET, {3, "XCHG_REL_RET"} }, + {DS_OP_XCHG2_RET, {3, "XCHG2_RET"} }, + {DS_OP_CMP_XCHG_RET, {3, "CMP_XCHG_RET"} }, + {DS_OP_CMP_XCHG_SPF_RET, {3, "CMP_XCHG_SPF_RET"} }, + {DS_OP_READ_RET, {1, "READ_RET"} }, + {DS_OP_READ_REL_RET, {1, "READ_REL_RET"} }, + {DS_OP_READ2_RET, {2, "READ2_RET"} }, + {DS_OP_READWRITE_RET, {3, "READWRITE_RET"} }, + {DS_OP_BYTE_READ_RET, {1, "BYTE_READ_RET"} }, + {DS_OP_UBYTE_READ_RET, {1, "UBYTE_READ_RET"} }, + {DS_OP_SHORT_READ_RET, {1, "SHORT_READ_RET"} }, + {DS_OP_USHORT_READ_RET, {1, "USHORT_READ_RET"} }, + {DS_OP_ATOMIC_ORDERED_ALLOC_RET, {3, "ATOMIC_ORDERED_ALLOC_RET"}}, + {LDS_ADD_RET, {2, "LDS_ADD_RET"} }, + {LDS_ADD, {2, "LDS_ADD"} }, + {LDS_AND_RET, {2, "LDS_AND_RET"} }, + {LDS_AND, {2, "LDS_AND"} }, + {LDS_WRITE, {2, "LDS_WRITE"} }, + {LDS_OR_RET, {2, "LDS_OR_RET"} }, + {LDS_OR, {2, "LDS_OR"} }, + {LDS_MAX_INT_RET, {2, "LDS_MAX_INT_RET"} }, + {LDS_MAX_INT, {2, "LDS_MAX_INT"} }, + {LDS_MAX_UINT_RET, {2, "LDS_MAX_UINT_RET"} }, + {LDS_MAX_UINT, {2, "LDS_MAX_UINT"} }, + {LDS_MIN_INT_RET, {2, "LDS_MIN_INT_RET"} }, + {LDS_MIN_INT, {2, "LDS_MIN_INT"} }, + {LDS_MIN_UINT_RET, {2, "LDS_MIN_UINT_RET"} }, + {LDS_MIN_UINT, {2, "LDS_MIN_UINT"} }, + {LDS_XOR_RET, {2, "LDS_XOR"} }, + {LDS_XOR, {2, "LDS_XOR"} }, + {LDS_XCHG_RET, {2, "LDS_XCHG_RET"} }, + {LDS_CMP_XCHG_RET, {3, "LDS_CMP_XCHG_RET"} }, + {LDS_WRITE_REL, {3, "LDS_WRITE_REL"} }, }; -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_alu_defines.h b/src/gallium/drivers/r600/sfn/sfn_alu_defines.h index dbe58809333..38d1bf4620d 100644 --- a/src/gallium/drivers/r600/sfn/sfn_alu_defines.h +++ b/src/gallium/drivers/r600/sfn/sfn_alu_defines.h @@ -29,8 +29,8 @@ #include "../r600_isa.h" -#include #include +#include namespace r600 { @@ -213,31 +213,31 @@ enum EAluOp { op1_interp_load_p10 = 125, op1_interp_load_p20 = 126, // op 3 all left shift 6 - op3_bfe_uint = 4<< 6, - op3_bfe_int = 5<< 6, - op3_bfi_int = 6<< 6, - op3_fma = 7<< 6, - op3_cndne_64 = 9<< 6, - op3_fma_64 = 10<< 6, - op3_lerp_uint = 11<< 6, - op3_bit_align_int = 12<< 6, - op3_byte_align_int = 13<< 6, - op3_sad_accum_uint = 14<< 6, - op3_sad_accum_hi_uint = 15<< 6, - op3_muladd_uint24 = 16<< 6, - op3_lds_idx_op = 17<< 6, - op3_muladd = 20<< 6, - op3_muladd_m2 = 21<< 6, - op3_muladd_m4 = 22<< 6, - op3_muladd_d2 = 23<< 6, - op3_muladd_ieee = 24<< 6, - op3_cnde = 25<< 6, - op3_cndgt = 26<< 6, - op3_cndge = 27<< 6, - op3_cnde_int = 28<< 6, - op3_cndgt_int = 29<< 6, - op3_cndge_int = 30<< 6, - op3_mul_lit = 31<< 6, + op3_bfe_uint = 4 << 6, + op3_bfe_int = 5 << 6, + op3_bfi_int = 6 << 6, + op3_fma = 7 << 6, + op3_cndne_64 = 9 << 6, + op3_fma_64 = 10 << 6, + op3_lerp_uint = 11 << 6, + op3_bit_align_int = 12 << 6, + op3_byte_align_int = 13 << 6, + op3_sad_accum_uint = 14 << 6, + op3_sad_accum_hi_uint = 15 << 6, + op3_muladd_uint24 = 16 << 6, + op3_lds_idx_op = 17 << 6, + op3_muladd = 20 << 6, + op3_muladd_m2 = 21 << 6, + op3_muladd_m4 = 22 << 6, + op3_muladd_d2 = 23 << 6, + op3_muladd_ieee = 24 << 6, + op3_cnde = 25 << 6, + op3_cndgt = 26 << 6, + op3_cndge = 27 << 6, + op3_cnde_int = 28 << 6, + op3_cndgt_int = 29 << 6, + op3_cndge_int = 30 << 6, + op3_mul_lit = 31 << 6, op_invalid = 0xffff }; @@ -291,17 +291,19 @@ enum AluBankSwizzle { alu_vec_102 = 3, sq_alu_scl_221 = 3, alu_vec_201 = 4, - sq_alu_scl_unknown = 4, + sq_alu_scl_unknown = 4, alu_vec_210 = 5, alu_vec_unknown = 6 }; -inline AluBankSwizzle operator ++(AluBankSwizzle& x) { +inline AluBankSwizzle +operator++(AluBankSwizzle& x) +{ x = static_cast(x + 1); return x; } -using AluOpFlags=std::bitset; +using AluOpFlags = std::bitset; struct AluOp { static constexpr int x = 1; @@ -313,27 +315,30 @@ struct AluOp { static constexpr int a = 31; AluOp(int ns, int f, uint8_t um_r600, uint8_t um_r700, uint8_t um_eg, const char *n): - nsrc(ns), is_float(f), name(n) + nsrc(ns), + is_float(f), + name(n) { - unit_mask[0] = um_r600; - unit_mask[1] = um_r700; - unit_mask[2] = um_eg; + unit_mask[0] = um_r600; + unit_mask[1] = um_r700; + unit_mask[2] = um_eg; } - bool can_channel(int flags, r600_chip_class unit_type) const { - assert(unit_type < 3); + bool can_channel(int flags, r600_chip_class unit_type) const + { + assert(unit_type < 3); return flags & unit_mask[unit_type]; } - int nsrc: 4; - int is_float:1; + int nsrc : 4; + int is_float : 1; uint8_t unit_mask[3]; const char *name; }; extern const std::map alu_ops; -enum AluInlineConstants { +enum AluInlineConstants { ALU_SRC_LDS_OQ_A = 219, ALU_SRC_LDS_OQ_B = 220, ALU_SRC_LDS_OQ_A_POP = 221, @@ -379,7 +384,7 @@ struct AluInlineConstantDescr { extern const std::map alu_src_const; -#define LDSOP2(X) LDS_ ## X = LDS_OP2_LDS_ ## X +#define LDSOP2(X) LDS_##X = LDS_OP2_LDS_##X enum ESDOp { DS_OP_ADD = 0, @@ -472,7 +477,6 @@ struct KCacheLine { } mode{free}; }; - -} +} // namespace r600 #endif // ALU_DEFINES_H diff --git a/src/gallium/drivers/r600/sfn/sfn_alu_readport_validation.cpp b/src/gallium/drivers/r600/sfn/sfn_alu_readport_validation.cpp index 77171eb692b..d468122809a 100644 --- a/src/gallium/drivers/r600/sfn/sfn_alu_readport_validation.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_alu_readport_validation.cpp @@ -50,7 +50,6 @@ public: static const int max_const_readports = 2; }; - class ReserveReadportVec : public ReserveReadport { public: using ReserveReadport::ReserveReadport; @@ -60,8 +59,7 @@ public: void visit(const UniformValue& value) override; }; -class ReserveReadportTrans : public ReserveReadport -{ +class ReserveReadportTrans : public ReserveReadport { public: ReserveReadportTrans(AluReadportReservation& reserv); @@ -79,7 +77,6 @@ public: void visit(const LiteralConstant& value) override; }; - class ReserveReadportTransPass2 : public ReserveReadportTrans { public: using ReserveReadportTrans::ReserveReadportTrans; @@ -89,7 +86,10 @@ public: void visit(const UniformValue& value) override; }; -bool AluReadportReservation::schedule_vec_src(PVirtualValue src[3], int nsrc, AluBankSwizzle swz) +bool +AluReadportReservation::schedule_vec_src(PVirtualValue src[3], + int nsrc, + AluBankSwizzle swz) { ReserveReadportVec visitor(*this); @@ -110,7 +110,8 @@ bool AluReadportReservation::schedule_vec_src(PVirtualValue src[3], int nsrc, A return visitor.success; } -bool AluReadportReservation::schedule_vec_instruction(const AluInstr& alu, AluBankSwizzle swz) +bool +AluReadportReservation::schedule_vec_instruction(const AluInstr& alu, AluBankSwizzle swz) { ReserveReadportVec visitor(*this); @@ -124,7 +125,9 @@ bool AluReadportReservation::schedule_vec_instruction(const AluInstr& alu, AluBa return visitor.success; } -bool AluReadportReservation::schedule_trans_instruction(const AluInstr& alu, AluBankSwizzle swz) +bool +AluReadportReservation::schedule_trans_instruction(const AluInstr& alu, + AluBankSwizzle swz) { ReserveReadportTransPass1 visitor1(*this); @@ -136,11 +139,9 @@ bool AluReadportReservation::schedule_trans_instruction(const AluInstr& alu, Alu if (!visitor1.success) return false; - ReserveReadportTransPass2 visitor2(*this); visitor2.n_consts = visitor1.n_consts; - for (unsigned i = 0; i < alu.n_sources(); ++i) { visitor2.cycle = cycle_trans(swz, i); @@ -149,7 +150,6 @@ bool AluReadportReservation::schedule_trans_instruction(const AluInstr& alu, Alu return visitor2.success; } - AluReadportReservation::AluReadportReservation() { for (int i = 0; i < max_chan_channels; ++i) { @@ -161,19 +161,19 @@ AluReadportReservation::AluReadportReservation() } } - -bool AluReadportReservation::reserve_gpr(int sel, int chan, int cycle) +bool +AluReadportReservation::reserve_gpr(int sel, int chan, int cycle) { if (m_hw_gpr[cycle][chan] == -1) { m_hw_gpr[cycle][chan] = sel; - } - else if (m_hw_gpr[cycle][chan] != sel) { + } else if (m_hw_gpr[cycle][chan] != sel) { return false; } return true; } -bool AluReadportReservation::reserve_const(const UniformValue& value) +bool +AluReadportReservation::reserve_const(const UniformValue& value) { int match = -1; int empty = -1; @@ -199,8 +199,9 @@ bool AluReadportReservation::reserve_const(const UniformValue& value) return true; } -bool AluReadportReservation::add_literal(uint32_t value) -{ +bool +AluReadportReservation::add_literal(uint32_t value) +{ for (unsigned i = 0; i < m_nliterals; ++i) { if (m_literals[i] == value) return true; @@ -212,7 +213,8 @@ bool AluReadportReservation::add_literal(uint32_t value) return false; } -int AluReadportReservation::cycle_vec(AluBankSwizzle swz, int src) +int +AluReadportReservation::cycle_vec(AluBankSwizzle swz, int src) { static const int mapping[AluBankSwizzle::alu_vec_unknown][max_gpr_readports] = { {0, 1, 2}, @@ -225,7 +227,8 @@ int AluReadportReservation::cycle_vec(AluBankSwizzle swz, int src) return mapping[swz][src]; } -int AluReadportReservation::cycle_trans(AluBankSwizzle swz, int src) +int +AluReadportReservation::cycle_trans(AluBankSwizzle swz, int src) { static const int mapping[AluBankSwizzle::sq_alu_scl_unknown][max_gpr_readports] = { {2, 1, 0}, @@ -236,69 +239,79 @@ int AluReadportReservation::cycle_trans(AluBankSwizzle swz, int src) return mapping[swz][src]; } - ReserveReadport::ReserveReadport(AluReadportReservation& reserv): - reserver(reserv) + reserver(reserv) { } -void ReserveReadport::visit(const LocalArray& value) +void +ReserveReadport::visit(const LocalArray& value) { (void)value; unreachable("a full array is not available here"); } -void ReserveReadport::visit(const LiteralConstant& value) +void +ReserveReadport::visit(const LiteralConstant& value) { success &= reserver.add_literal(value.value()); } -void ReserveReadport::visit(const InlineConstant& value) +void +ReserveReadport::visit(const InlineConstant& value) { (void)value; } -void ReserveReadportVec::visit(const Register& value) +void +ReserveReadportVec::visit(const Register& value) { reserve_gpr(value.sel(), value.chan()); } -void ReserveReadportVec::visit(const LocalArrayValue& value) +void +ReserveReadportVec::visit(const LocalArrayValue& value) { // Set the hightest non-sign bit to indicated that we use the // AR register reserve_gpr(0x4000000 | value.sel(), value.chan()); } -void ReserveReadport::reserve_gpr(int sel, int chan) +void +ReserveReadport::reserve_gpr(int sel, int chan) { if (isrc == 1 && src0_sel == sel && src0_chan == chan) return; success &= reserver.reserve_gpr(sel, chan, cycle); } -void ReserveReadportVec::visit(const UniformValue& value) +void +ReserveReadportVec::visit(const UniformValue& value) { // kcache bank? success &= reserver.reserve_const(value); } ReserveReadportTrans::ReserveReadportTrans(AluReadportReservation& reserv): - ReserveReadport(reserv), - n_consts(0) -{} + ReserveReadport(reserv), + n_consts(0) +{ +} -void ReserveReadportTransPass1::visit(const Register& value) +void +ReserveReadportTransPass1::visit(const Register& value) { (void)value; } -void ReserveReadportTransPass1::visit(const LocalArrayValue& value) +void +ReserveReadportTransPass1::visit(const LocalArrayValue& value) { (void)value; } -void ReserveReadportTransPass1::visit(const UniformValue& value) +void +ReserveReadportTransPass1::visit(const UniformValue& value) { if (n_consts >= max_const_readports) { success = false; @@ -308,7 +321,8 @@ void ReserveReadportTransPass1::visit(const UniformValue& value) success &= reserver.reserve_const(value); } -void ReserveReadportTransPass1::visit(const InlineConstant& value) +void +ReserveReadportTransPass1::visit(const InlineConstant& value) { (void)value; if (n_consts >= max_const_readports) { @@ -318,7 +332,8 @@ void ReserveReadportTransPass1::visit(const InlineConstant& value) n_consts++; } -void ReserveReadportTransPass1::visit(const LiteralConstant& value) +void +ReserveReadportTransPass1::visit(const LiteralConstant& value) { if (n_consts >= max_const_readports) { success = false; @@ -328,7 +343,8 @@ void ReserveReadportTransPass1::visit(const LiteralConstant& value) success &= reserver.add_literal(value.value()); } -void ReserveReadportTransPass2::visit(const Register& value) +void +ReserveReadportTransPass2::visit(const Register& value) { if (cycle < n_consts) { success = false; @@ -337,7 +353,8 @@ void ReserveReadportTransPass2::visit(const Register& value) reserve_gpr(value.sel(), value.chan()); } -void ReserveReadportTransPass2::visit(const LocalArrayValue& value) +void +ReserveReadportTransPass2::visit(const LocalArrayValue& value) { if (cycle < n_consts) { success = false; @@ -346,10 +363,10 @@ void ReserveReadportTransPass2::visit(const LocalArrayValue& value) reserve_gpr(0x4000000 | value.sel(), value.chan()); } -void ReserveReadportTransPass2::visit(const UniformValue& value) +void +ReserveReadportTransPass2::visit(const UniformValue& value) { (void)value; } - -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_alu_readport_validation.h b/src/gallium/drivers/r600/sfn/sfn_alu_readport_validation.h index b27aa8b47b9..ce924aa8a98 100644 --- a/src/gallium/drivers/r600/sfn/sfn_alu_readport_validation.h +++ b/src/gallium/drivers/r600/sfn/sfn_alu_readport_validation.h @@ -35,9 +35,9 @@ class AluReadportReservation { public: AluReadportReservation(); AluReadportReservation(const AluReadportReservation& orig) = default; - AluReadportReservation& operator = (const AluReadportReservation& orig) = default; + AluReadportReservation& operator=(const AluReadportReservation& orig) = default; - bool schedule_vec_src(PVirtualValue src[3], int nsrc, AluBankSwizzle swz); + bool schedule_vec_src(PVirtualValue src[3], int nsrc, AluBankSwizzle swz); bool schedule_vec_instruction(const AluInstr& alu, AluBankSwizzle swz); bool schedule_trans_instruction(const AluInstr& alu, AluBankSwizzle swz); @@ -55,13 +55,12 @@ public: std::array, max_gpr_readports> m_hw_gpr; std::array m_hw_const_addr; - std::array m_hw_const_chan; - std::array m_hw_const_bank; + std::array m_hw_const_chan; + std::array m_hw_const_bank; std::array m_literals; uint32_t m_nliterals{0}; }; - -} +} // namespace r600 #endif // ALUREADPORTVALIDATION_H diff --git a/src/gallium/drivers/r600/sfn/sfn_assembler.cpp b/src/gallium/drivers/r600/sfn/sfn_assembler.cpp index fc06727e47c..6fa49287c63 100644 --- a/src/gallium/drivers/r600/sfn/sfn_assembler.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_assembler.cpp @@ -25,22 +25,22 @@ */ #include "sfn_assembler.h" + +#include "../eg_sq.h" +#include "sfn_callstack.h" +#include "sfn_conditionaljumptracker.h" #include "sfn_debug.h" #include "sfn_instr_alugroup.h" #include "sfn_instr_controlflow.h" -#include "sfn_instr_fetch.h" #include "sfn_instr_export.h" +#include "sfn_instr_fetch.h" #include "sfn_instr_mem.h" #include "sfn_instr_tex.h" -#include "sfn_conditionaljumptracker.h" -#include "sfn_callstack.h" - -#include "../eg_sq.h" - namespace r600 { Assembler::Assembler(r600_shader *sh, const r600_shader_key& key): - m_sh(sh), m_key(key) + m_sh(sh), + m_key(key) { } @@ -80,8 +80,7 @@ public: bool copy_dst(r600_bytecode_alu_dst& dst, const Register& d, bool write); PVirtualValue copy_src(r600_bytecode_alu_src& src, const VirtualValue& s); - EBufferIndexMode - emit_index_reg(const VirtualValue& addr, unsigned idx); + EBufferIndexMode emit_index_reg(const VirtualValue& addr, unsigned idx); void emit_endif(); void emit_else(); @@ -121,7 +120,8 @@ public: bool m_result{true}; }; -bool Assembler::lower(Shader *shader) +bool +Assembler::lower(Shader *shader) { AssamblerVisitor ass(m_sh, m_key); @@ -135,26 +135,25 @@ bool Assembler::lower(Shader *shader) ass.finalize(); return ass.m_result; - } AssamblerVisitor::AssamblerVisitor(r600_shader *sh, const r600_shader_key& key): - m_key(key), - m_shader(sh), + m_key(key), + m_shader(sh), - m_bc(&sh->bc), - m_callstack(sh->bc), - ps_alpha_to_one(key.ps.alpha_to_one) + m_bc(&sh->bc), + m_callstack(sh->bc), + ps_alpha_to_one(key.ps.alpha_to_one) { if (m_shader->processor_type == PIPE_SHADER_FRAGMENT) m_max_color_exports = MAX2(m_key.ps.nr_cbufs, 1); - if (m_shader->processor_type == PIPE_SHADER_VERTEX && - m_shader->ninput > 0) - r600_bytecode_add_cfinst(m_bc, CF_OP_CALL_FS); + if (m_shader->processor_type == PIPE_SHADER_VERTEX && m_shader->ninput > 0) + r600_bytecode_add_cfinst(m_bc, CF_OP_CALL_FS); } -void AssamblerVisitor::finalize() +void +AssamblerVisitor::finalize() { const struct cf_op_info *last = nullptr; @@ -163,12 +162,12 @@ void AssamblerVisitor::finalize() /* alu clause instructions don't have EOP bit, so add NOP */ if (m_shader->bc.gfx_level < CAYMAN && - (!last || last->flags & CF_ALU || m_bc->cf_last->op == CF_OP_LOOP_END - || m_bc->cf_last->op == CF_OP_POP)) + (!last || last->flags & CF_ALU || m_bc->cf_last->op == CF_OP_LOOP_END || + m_bc->cf_last->op == CF_OP_POP)) r600_bytecode_add_cfinst(m_bc, CF_OP_NOP); - /* A fetch shader only can't be EOP (results in hang), but we can replace it - * by a NOP */ + /* A fetch shader only can't be EOP (results in hang), but we can replace + * it by a NOP */ else if (last && m_bc->cf_last->op == CF_OP_CALL_FS) m_bc->cf_last->op = CF_OP_NOP; @@ -180,7 +179,8 @@ void AssamblerVisitor::finalize() extern const std::map opcode_map; -void AssamblerVisitor::visit(const AluInstr& ai) +void +AssamblerVisitor::visit(const AluInstr& ai) { assert(vtx_fetch_results.empty()); assert(tex_fetch_results.empty()); @@ -191,7 +191,8 @@ void AssamblerVisitor::visit(const AluInstr& ai) emit_alu_op(ai); } -void AssamblerVisitor::emit_lds_op(const AluInstr& lds) +void +AssamblerVisitor::emit_lds_op(const AluInstr& lds) { struct r600_bytecode_alu alu; memset(&alu, 0, sizeof(alu)); @@ -202,7 +203,7 @@ void AssamblerVisitor::emit_lds_op(const AluInstr& lds) bool has_lds_fetch = false; switch (alu.op) { case LDS_WRITE: - alu.op =LDS_OP2_LDS_WRITE; + alu.op = LDS_OP2_LDS_WRITE; break; case LDS_WRITE_REL: alu.op = LDS_OP3_LDS_WRITE_REL; @@ -259,13 +260,14 @@ void AssamblerVisitor::emit_lds_op(const AluInstr& lds) m_result = false; } -void AssamblerVisitor::emit_alu_op(const AluInstr& ai) +void +AssamblerVisitor::emit_alu_op(const AluInstr& ai) { struct r600_bytecode_alu alu; memset(&alu, 0, sizeof(alu)); if (opcode_map.find(ai.opcode()) == opcode_map.end()) { - std::cerr << "Opcode not handled for " << ai <<"\n"; + std::cerr << "Opcode not handled for " << ai << "\n"; m_result = false; return; } @@ -322,15 +324,18 @@ void AssamblerVisitor::emit_alu_op(const AluInstr& ai) alu.execute_mask = ai.has_alu_flag(alu_update_exec); /* If the destination register is equal to the last loaded address register - * then clear the latter one, because the values will no longer be identical */ + * then clear the latter one, because the values will no longer be + * identical */ if (m_last_addr) - sfn_log << SfnLog::assembly << " Current address register is " << *m_last_addr << "\n"; + sfn_log << SfnLog::assembly << " Current address register is " << *m_last_addr + << "\n"; if (dst) sfn_log << SfnLog::assembly << " Current dst register is " << *dst << "\n"; if (dst && m_last_addr && *dst == *m_last_addr) { - sfn_log << SfnLog::assembly << " Clear address register (was " << *m_last_addr << "\n"; + sfn_log << SfnLog::assembly << " Clear address register (was " << *m_last_addr + << "\n"; m_last_addr = nullptr; } @@ -338,14 +343,30 @@ void AssamblerVisitor::emit_alu_op(const AluInstr& ai) unsigned type = 0; switch (cf_op) { - case cf_alu: type = CF_OP_ALU; break; - case cf_alu_push_before: type = CF_OP_ALU_PUSH_BEFORE; break; - case cf_alu_pop_after: type = CF_OP_ALU_POP_AFTER; break; - case cf_alu_pop2_after: type = CF_OP_ALU_POP2_AFTER; break; - case cf_alu_break: type = CF_OP_ALU_BREAK; break; - case cf_alu_else_after: type = CF_OP_ALU_ELSE_AFTER; break; - case cf_alu_continue: type = CF_OP_ALU_CONTINUE; break; - case cf_alu_extended: type = CF_OP_ALU_EXT; break; + case cf_alu: + type = CF_OP_ALU; + break; + case cf_alu_push_before: + type = CF_OP_ALU_PUSH_BEFORE; + break; + case cf_alu_pop_after: + type = CF_OP_ALU_POP_AFTER; + break; + case cf_alu_pop2_after: + type = CF_OP_ALU_POP2_AFTER; + break; + case cf_alu_break: + type = CF_OP_ALU_BREAK; + break; + case cf_alu_else_after: + type = CF_OP_ALU_ELSE_AFTER; + break; + case cf_alu_continue: + type = CF_OP_ALU_CONTINUE; + break; + case cf_alu_extended: + type = CF_OP_ALU_EXT; + break; default: assert(0 && "cf_alu_undefined should have been replaced"); } @@ -353,7 +374,6 @@ void AssamblerVisitor::emit_alu_op(const AluInstr& ai) if (alu.last) m_nliterals_in_group.clear(); - m_result = !r600_bytecode_add_alu_type(m_bc, &alu, type); if (ai.opcode() == op1_mova_int) @@ -365,13 +385,13 @@ void AssamblerVisitor::emit_alu_op(const AluInstr& ai) if (ai.opcode() == op1_set_cf_idx1) m_bc->index_loaded[1] = 1; - m_bc->force_add_cf |= (ai.opcode() == op2_kille || - ai.opcode() == op2_killne_int || - ai.opcode() == op1_set_cf_idx0 || - ai.opcode() == op1_set_cf_idx1); + m_bc->force_add_cf |= + (ai.opcode() == op2_kille || ai.opcode() == op2_killne_int || + ai.opcode() == op1_set_cf_idx0 || ai.opcode() == op1_set_cf_idx1); } -void AssamblerVisitor::visit(const AluGroup& group) +void +AssamblerVisitor::visit(const AluGroup& group) { clear_states(sf_vtx | sf_tex); @@ -391,10 +411,8 @@ void AssamblerVisitor::visit(const AluGroup& group) m_last_addr = nullptr; } else { auto instr = *group.begin(); - if (instr && - !instr->has_alu_flag(alu_is_lds) && - instr->opcode() == op0_group_barrier && - m_bc->cf_last->ndw + 14 > 240) { + if (instr && !instr->has_alu_flag(alu_is_lds) && + instr->opcode() == op0_group_barrier && m_bc->cf_last->ndw + 14 > 240) { assert(m_bc->cf_last->nlds_read == 0); m_bc->force_add_cf = 1; m_last_addr = nullptr; @@ -406,8 +424,7 @@ void AssamblerVisitor::visit(const AluGroup& group) if (addr.first) { if (!addr.second) { - if (!m_last_addr || !m_bc->ar_loaded || - !m_last_addr->equal_to(*addr.first)) { + if (!m_last_addr || !m_bc->ar_loaded || !m_last_addr->equal_to(*addr.first)) { m_bc->ar_reg = addr.first->sel(); m_bc->ar_chan = addr.first->chan(); m_last_addr = addr.first; @@ -426,7 +443,8 @@ void AssamblerVisitor::visit(const AluGroup& group) } } -void AssamblerVisitor::visit(const TexInstr& tex_instr) +void +AssamblerVisitor::visit(const TexInstr& tex_instr) { clear_states(sf_vtx | sf_alu); @@ -436,8 +454,7 @@ void AssamblerVisitor::visit(const TexInstr& tex_instr) if (addr) index_mode = emit_index_reg(*addr, 1); - if (tex_fetch_results.find(tex_instr.src().sel()) != - tex_fetch_results.end()) { + if (tex_fetch_results.find(tex_instr.src().sel()) != tex_fetch_results.end()) { m_bc->force_add_cf = 1; tex_fetch_results.clear(); } @@ -467,10 +484,7 @@ void AssamblerVisitor::visit(const TexInstr& tex_instr) tex.resource_index_mode = index_mode; tex.sampler_index_mode = index_mode; - if (tex.dst_sel_x < 4 && - tex.dst_sel_y < 4 && - tex.dst_sel_z < 4 && - tex.dst_sel_w < 4) + if (tex.dst_sel_x < 4 && tex.dst_sel_y < 4 && tex.dst_sel_z < 4 && tex.dst_sel_w < 4) tex_fetch_results.insert(tex.dst_gpr); if (tex_instr.opcode() == TexInstr::get_gradient_h || @@ -484,7 +498,8 @@ void AssamblerVisitor::visit(const TexInstr& tex_instr) } } -void AssamblerVisitor::visit(const ExportInstr& exi) +void +AssamblerVisitor::visit(const ExportInstr& exi) { const auto& value = exi.value(); @@ -497,24 +512,23 @@ void AssamblerVisitor::visit(const ExportInstr& exi) output.swizzle_y = value[1]->chan(); output.swizzle_z = value[2]->chan(); output.burst_count = 1; - output.op = exi.is_last_export() ? CF_OP_EXPORT_DONE: CF_OP_EXPORT; + output.op = exi.is_last_export() ? CF_OP_EXPORT_DONE : CF_OP_EXPORT; output.type = exi.export_type(); - clear_states(sf_all); switch (exi.export_type()) { case ExportInstr::pixel: output.swizzle_w = ps_alpha_to_one ? 5 : exi.value()[3]->chan(); output.array_base = exi.location(); - break; + break; case ExportInstr::pos: output.swizzle_w = exi.value()[3]->chan(); - output.array_base = 60 + exi.location(); - break; + output.array_base = 60 + exi.location(); + break; case ExportInstr::param: output.swizzle_w = exi.value()[3]->chan(); output.array_base = exi.location(); - break; + break; default: R600_ERR("shader_from_nir: export %d type not yet supported\n", exi.export_type()); m_result = false; @@ -523,18 +537,19 @@ void AssamblerVisitor::visit(const ExportInstr& exi) /* If all register elements pinned to fixed values * we can override the gpr (the register allocator doesn't see * this because it doesn't take these channels into account. */ - if (output.swizzle_x > 3 && output.swizzle_y > 3 && - output.swizzle_z > 3 && output.swizzle_w > 3) - output.gpr = 0; + if (output.swizzle_x > 3 && output.swizzle_y > 3 && output.swizzle_z > 3 && + output.swizzle_w > 3) + output.gpr = 0; int r = 0; - if ((r =r600_bytecode_add_output(m_bc, &output))) { + if ((r = r600_bytecode_add_output(m_bc, &output))) { R600_ERR("Error adding export at location %d : err: %d\n", exi.location(), r); m_result = false; } } -void AssamblerVisitor::visit(const ScratchIOInstr& instr) +void +AssamblerVisitor::visit(const ScratchIOInstr& instr) { clear_states(sf_all); @@ -567,13 +582,14 @@ void AssamblerVisitor::visit(const ScratchIOInstr& instr) cf.array_base = instr.location(); } - if (r600_bytecode_add_output(m_bc, &cf)){ + if (r600_bytecode_add_output(m_bc, &cf)) { R600_ERR("shader_from_nir: Error creating SCRATCH_WR assembly instruction\n"); m_result = false; } } -void AssamblerVisitor::visit(const StreamOutInstr& instr) +void +AssamblerVisitor::visit(const StreamOutInstr& instr) { struct r600_bytecode_output output; memset(&output, 0, sizeof(struct r600_bytecode_output)); @@ -587,14 +603,14 @@ void AssamblerVisitor::visit(const StreamOutInstr& instr) output.comp_mask = instr.comp_mask(); output.op = instr.op(m_shader->bc.gfx_level); - - if (r600_bytecode_add_output(m_bc, &output)) { + if (r600_bytecode_add_output(m_bc, &output)) { R600_ERR("shader_from_nir: Error creating stream output instruction\n"); m_result = false; } } -void AssamblerVisitor::visit(const MemRingOutInstr& instr) +void +AssamblerVisitor::visit(const MemRingOutInstr& instr) { struct r600_bytecode_output output; memset(&output, 0, sizeof(struct r600_bytecode_output)); @@ -618,7 +634,8 @@ void AssamblerVisitor::visit(const MemRingOutInstr& instr) } } -void AssamblerVisitor::visit(const EmitVertexInstr& instr) +void +AssamblerVisitor::visit(const EmitVertexInstr& instr) { int r = r600_bytecode_add_cfinst(m_bc, instr.op()); if (!r) @@ -628,7 +645,8 @@ void AssamblerVisitor::visit(const EmitVertexInstr& instr) assert(m_bc->cf_last->count < 4); } -void AssamblerVisitor::visit(const FetchInstr& fetch_instr) +void +AssamblerVisitor::visit(const FetchInstr& fetch_instr) { clear_states(sf_tex | sf_alu); @@ -641,18 +659,16 @@ void AssamblerVisitor::visit(const FetchInstr& fetch_instr) if (fetch_instr.has_fetch_flag(FetchInstr::wait_ack)) emit_wait_ack(); - bool use_tc = fetch_instr.has_fetch_flag(FetchInstr::use_tc) || - (m_bc->gfx_level == CAYMAN); + bool use_tc = + fetch_instr.has_fetch_flag(FetchInstr::use_tc) || (m_bc->gfx_level == CAYMAN); if (!use_tc && - vtx_fetch_results.find(fetch_instr.src().sel()) != - vtx_fetch_results.end()) { + vtx_fetch_results.find(fetch_instr.src().sel()) != vtx_fetch_results.end()) { m_bc->force_add_cf = 1; vtx_fetch_results.clear(); } if (fetch_instr.has_fetch_flag(FetchInstr::use_tc) && - tex_fetch_results.find(fetch_instr.src().sel()) != - tex_fetch_results.end()) { + tex_fetch_results.find(fetch_instr.src().sel()) != tex_fetch_results.end()) { m_bc->force_add_cf = 1; tex_fetch_results.clear(); } @@ -671,13 +687,13 @@ void AssamblerVisitor::visit(const FetchInstr& fetch_instr) vtx.src_sel_x = fetch_instr.src().chan(); vtx.mega_fetch_count = fetch_instr.mega_fetch_count(); vtx.dst_gpr = fetch_instr.dst().sel(); - vtx.dst_sel_x = fetch_instr.dest_swizzle(0); /* SEL_X */ - vtx.dst_sel_y = fetch_instr.dest_swizzle(1); /* SEL_Y */ - vtx.dst_sel_z = fetch_instr.dest_swizzle(2); /* SEL_Z */ - vtx.dst_sel_w = fetch_instr.dest_swizzle(3); /* SEL_W */ + vtx.dst_sel_x = fetch_instr.dest_swizzle(0); /* SEL_X */ + vtx.dst_sel_y = fetch_instr.dest_swizzle(1); /* SEL_Y */ + vtx.dst_sel_z = fetch_instr.dest_swizzle(2); /* SEL_Z */ + vtx.dst_sel_w = fetch_instr.dest_swizzle(3); /* SEL_W */ vtx.use_const_fields = fetch_instr.has_fetch_flag(FetchInstr::use_const_field); vtx.data_format = fetch_instr.data_format(); - vtx.num_format_all = fetch_instr.num_format(); /* NUM_FORMAT_SCALED */ + vtx.num_format_all = fetch_instr.num_format(); /* NUM_FORMAT_SCALED */ vtx.format_comp_all = fetch_instr.has_fetch_flag(FetchInstr::format_comp_signed); vtx.endian = fetch_instr.endian_swap(); vtx.buffer_index_mode = rat_index_mode; @@ -702,12 +718,13 @@ void AssamblerVisitor::visit(const FetchInstr& fetch_instr) } } - m_bc->cf_last->vpm = (m_bc->type == PIPE_SHADER_FRAGMENT) && - fetch_instr.has_fetch_flag(FetchInstr::vpm); + m_bc->cf_last->vpm = + (m_bc->type == PIPE_SHADER_FRAGMENT) && fetch_instr.has_fetch_flag(FetchInstr::vpm); m_bc->cf_last->barrier = 1; } -void AssamblerVisitor::visit(const WriteTFInstr& instr) +void +AssamblerVisitor::visit(const WriteTFInstr& instr) { struct r600_bytecode_gds gds; @@ -748,7 +765,8 @@ void AssamblerVisitor::visit(const WriteTFInstr& instr) } } -void AssamblerVisitor::visit(const RatInstr& instr) +void +AssamblerVisitor::visit(const RatInstr& instr) { struct r600_bytecode_gds gds; @@ -780,9 +798,9 @@ void AssamblerVisitor::visit(const RatInstr& instr) assert(instr.data_swz(0) == PIPE_SWIZZLE_X); if (cf->rat.inst != RatInstr::STORE_TYPED) { assert(instr.data_swz(1) == PIPE_SWIZZLE_Y || - instr.data_swz(1) == PIPE_SWIZZLE_MAX) ; + instr.data_swz(1) == PIPE_SWIZZLE_MAX); assert(instr.data_swz(2) == PIPE_SWIZZLE_Z || - instr.data_swz(2) == PIPE_SWIZZLE_MAX) ; + instr.data_swz(2) == PIPE_SWIZZLE_MAX); } cf->vpm = m_bc->type == PIPE_SHADER_FRAGMENT; @@ -793,8 +811,8 @@ void AssamblerVisitor::visit(const RatInstr& instr) m_ack_suggested |= instr.need_ack(); } - -void AssamblerVisitor::clear_states(const uint32_t& states) +void +AssamblerVisitor::clear_states(const uint32_t& states) { if (states & sf_vtx) vtx_fetch_results.clear(); @@ -806,17 +824,17 @@ void AssamblerVisitor::clear_states(const uint32_t& states) m_last_op_was_barrier = false; m_last_addr = nullptr; } - } - -void AssamblerVisitor::visit(const Block& block) +void +AssamblerVisitor::visit(const Block& block) { if (block.empty()) return; m_bc->force_add_cf = block.has_instr_flag(Instr::force_cf); - sfn_log << SfnLog::assembly << "Translate block size: " << block.size() << " new_cf:" << m_bc->force_add_cf << "\n"; + sfn_log << SfnLog::assembly << "Translate block size: " << block.size() + << " new_cf:" << m_bc->force_add_cf << "\n"; for (const auto& i : block) { sfn_log << SfnLog::assembly << "Translate " << *i << " "; @@ -828,7 +846,8 @@ void AssamblerVisitor::visit(const Block& block) } } -void AssamblerVisitor::visit(const IfInstr& instr) +void +AssamblerVisitor::visit(const IfInstr& instr) { int elems = m_callstack.push(FC_PUSH_VPM); bool needs_workaround = false; @@ -836,10 +855,8 @@ void AssamblerVisitor::visit(const IfInstr& instr) if (m_bc->gfx_level == CAYMAN && m_bc->stack.loop > 1) needs_workaround = true; - if (m_bc->gfx_level == EVERGREEN && - m_bc->family != CHIP_HEMLOCK && - m_bc->family != CHIP_CYPRESS && - m_bc->family != CHIP_JUNIPER) { + if (m_bc->gfx_level == EVERGREEN && m_bc->family != CHIP_HEMLOCK && + m_bc->family != CHIP_CYPRESS && m_bc->family != CHIP_JUNIPER) { unsigned dmod1 = (elems - 1) % m_bc->stack.entry_size; unsigned dmod2 = (elems) % m_bc->stack.entry_size; @@ -848,16 +865,17 @@ void AssamblerVisitor::visit(const IfInstr& instr) } auto pred = instr.predicate(); - auto [addr, dummy0, dummy1 ] = pred->indirect_addr(); {} + auto [addr, dummy0, dummy1] = pred->indirect_addr(); + { + } if (addr) { - if (!m_last_addr || !m_bc->ar_loaded || - !m_last_addr->equal_to(*addr)) { + if (!m_last_addr || !m_bc->ar_loaded || !m_last_addr->equal_to(*addr)) { m_bc->ar_reg = addr->sel(); - m_bc->ar_chan = addr->chan(); - m_last_addr = addr; - m_bc->ar_loaded = 0; + m_bc->ar_chan = addr->chan(); + m_last_addr = addr; + m_bc->ar_loaded = 0; - r600_load_ar(m_bc, true); + r600_load_ar(m_bc, true); } } @@ -867,7 +885,7 @@ void AssamblerVisitor::visit(const IfInstr& instr) pred->set_cf_type(cf_alu); } - clear_states(sf_tex|sf_vtx); + clear_states(sf_tex | sf_vtx); pred->accept(*this); r600_bytecode_add_cfinst(m_bc, CF_OP_JUMP); @@ -876,7 +894,8 @@ void AssamblerVisitor::visit(const IfInstr& instr) m_jump_tracker.push(m_bc->cf_last, jt_if); } -void AssamblerVisitor::visit(const ControlFlowInstr& instr) +void +AssamblerVisitor::visit(const ControlFlowInstr& instr) { clear_states(sf_all); switch (instr.cf_type()) { @@ -888,8 +907,8 @@ void AssamblerVisitor::visit(const ControlFlowInstr& instr) break; case ControlFlowInstr::cf_loop_begin: { bool use_vpm = m_shader->processor_type == PIPE_SHADER_FRAGMENT && - instr.has_instr_flag(Instr::vpm) && - !instr.has_instr_flag(Instr::helper); + instr.has_instr_flag(Instr::vpm) && + !instr.has_instr_flag(Instr::helper); emit_loop_begin(use_vpm); break; } @@ -902,8 +921,7 @@ void AssamblerVisitor::visit(const ControlFlowInstr& instr) case ControlFlowInstr::cf_loop_continue: emit_loop_cont(); break; - case ControlFlowInstr::cf_wait_ack: - { + case ControlFlowInstr::cf_wait_ack: { int r = r600_bytecode_add_cfinst(m_bc, CF_OP_WAIT_ACK); if (!r) { m_bc->cf_last->cf_addr = 0; @@ -912,14 +930,14 @@ void AssamblerVisitor::visit(const ControlFlowInstr& instr) } else { m_result = false; } - } - break; + } break; default: unreachable("Unknown CF instruction type"); } } -void AssamblerVisitor::visit(const GDSInstr& instr) +void +AssamblerVisitor::visit(const GDSInstr& instr) { struct r600_bytecode_gds gds; @@ -949,10 +967,17 @@ void AssamblerVisitor::visit(const GDSInstr& instr) gds.dst_sel_w = 7; switch (instr.dest()->chan()) { - case 0: gds.dst_sel_x = 0;break; - case 1: gds.dst_sel_y = 0;break; - case 2: gds.dst_sel_z = 0;break; - case 3: gds.dst_sel_w = 0; + case 0: + gds.dst_sel_x = 0; + break; + case 1: + gds.dst_sel_y = 0; + break; + case 2: + gds.dst_sel_z = 0; + break; + case 3: + gds.dst_sel_w = 0; } gds.src_gpr2 = 0; @@ -967,13 +992,15 @@ void AssamblerVisitor::visit(const GDSInstr& instr) m_bc->cf_last->barrier = 1; } -void AssamblerVisitor::visit(const LDSAtomicInstr& instr) +void +AssamblerVisitor::visit(const LDSAtomicInstr& instr) { (void)instr; unreachable("LDSAtomicInstr must be lowered to ALUInstr"); } -void AssamblerVisitor::visit(const LDSReadInstr& instr) +void +AssamblerVisitor::visit(const LDSReadInstr& instr) { (void)instr; unreachable("LDSReadInstr must be lowered to ALUInstr"); @@ -985,13 +1012,13 @@ AssamblerVisitor::emit_index_reg(const VirtualValue& addr, unsigned idx) assert(idx < 2); if (!m_bc->index_loaded[idx] || m_loop_nesting || - m_bc->index_reg[idx] != (unsigned)addr.sel() - || m_bc->index_reg_chan[idx] != (unsigned)addr.chan()) { + m_bc->index_reg[idx] != (unsigned)addr.sel() || + m_bc->index_reg_chan[idx] != (unsigned)addr.chan()) { struct r600_bytecode_alu alu; // Make sure MOVA is not last instr in clause - if (!m_bc->cf_last || (m_bc->cf_last->ndw>>1) >= 110) + if (!m_bc->cf_last || (m_bc->cf_last->ndw >> 1) >= 110) m_bc->force_add_cf = 1; if (m_bc->gfx_level != CAYMAN) { @@ -1042,14 +1069,16 @@ AssamblerVisitor::emit_index_reg(const VirtualValue& addr, unsigned idx) return idx == 0 ? bim_zero : bim_one; } -void AssamblerVisitor::emit_else() +void +AssamblerVisitor::emit_else() { r600_bytecode_add_cfinst(m_bc, CF_OP_ELSE); m_bc->cf_last->pop_count = 1; m_result &= m_jump_tracker.add_mid(m_bc->cf_last, jt_if); } -void AssamblerVisitor::emit_endif() +void +AssamblerVisitor::emit_endif() { m_callstack.pop(FC_PUSH_VPM); @@ -1080,7 +1109,8 @@ void AssamblerVisitor::emit_endif() m_result &= m_jump_tracker.pop(m_bc->cf_last, jt_if); } -void AssamblerVisitor::emit_loop_begin(bool vpm) +void +AssamblerVisitor::emit_loop_begin(bool vpm) { r600_bytecode_add_cfinst(m_bc, CF_OP_LOOP_START_DX10); m_bc->cf_last->vpm = vpm && m_bc->type == PIPE_SHADER_FRAGMENT; @@ -1089,7 +1119,8 @@ void AssamblerVisitor::emit_loop_begin(bool vpm) ++m_loop_nesting; } -void AssamblerVisitor::emit_loop_end() +void +AssamblerVisitor::emit_loop_end() { if (m_ack_suggested) { emit_wait_ack(); @@ -1103,23 +1134,26 @@ void AssamblerVisitor::emit_loop_end() m_result |= m_jump_tracker.pop(m_bc->cf_last, jt_loop); } -void AssamblerVisitor::emit_loop_break() +void +AssamblerVisitor::emit_loop_break() { r600_bytecode_add_cfinst(m_bc, CF_OP_LOOP_BREAK); m_result |= m_jump_tracker.add_mid(m_bc->cf_last, jt_loop); } -void AssamblerVisitor::emit_loop_cont() +void +AssamblerVisitor::emit_loop_cont() { r600_bytecode_add_cfinst(m_bc, CF_OP_LOOP_CONTINUE); m_result |= m_jump_tracker.add_mid(m_bc->cf_last, jt_loop); } -bool AssamblerVisitor::copy_dst(r600_bytecode_alu_dst& dst, - const Register& d, bool write) +bool +AssamblerVisitor::copy_dst(r600_bytecode_alu_dst& dst, const Register& d, bool write) { if (write && d.sel() > 124) { - R600_ERR("shader_from_nir: Don't support more then 124 GPRs, but try using %d\n", + R600_ERR("shader_from_nir: Don't support more then 124 GPRs, but try " + "using %d\n", d.sel()); m_result = false; return false; @@ -1128,18 +1162,17 @@ bool AssamblerVisitor::copy_dst(r600_bytecode_alu_dst& dst, dst.sel = d.sel(); dst.chan = d.chan(); - if (m_bc->index_reg[1] == dst.sel && - m_bc->index_reg_chan[1] == dst.chan) + if (m_bc->index_reg[1] == dst.sel && m_bc->index_reg_chan[1] == dst.chan) m_bc->index_loaded[1] = false; - if (m_bc->index_reg[0] == dst.sel && - m_bc->index_reg_chan[0] == dst.chan) + if (m_bc->index_reg[0] == dst.sel && m_bc->index_reg_chan[0] == dst.chan) m_bc->index_loaded[0] = false; return true; } -void AssamblerVisitor::emit_wait_ack() +void +AssamblerVisitor::emit_wait_ack() { int r = r600_bytecode_add_cfinst(m_bc, CF_OP_WAIT_ACK); if (!r) { @@ -1152,7 +1185,6 @@ void AssamblerVisitor::emit_wait_ack() class EncodeSourceVisitor : public ConstRegisterVisitor { public: - EncodeSourceVisitor(r600_bytecode_alu_src& s, r600_bytecode *bc); void visit(const Register& value) override; void visit(const LocalArray& value) override; @@ -1166,7 +1198,8 @@ public: PVirtualValue m_buffer_offset{nullptr}; }; -PVirtualValue AssamblerVisitor::copy_src(r600_bytecode_alu_src& src, const VirtualValue& s) +PVirtualValue +AssamblerVisitor::copy_src(r600_bytecode_alu_src& src, const VirtualValue& s) { EncodeSourceVisitor visitor(src, m_bc); @@ -1178,299 +1211,304 @@ PVirtualValue AssamblerVisitor::copy_src(r600_bytecode_alu_src& src, const Virtu } EncodeSourceVisitor::EncodeSourceVisitor(r600_bytecode_alu_src& s, r600_bytecode *bc): - src(s), m_bc(bc) + src(s), + m_bc(bc) { } -void EncodeSourceVisitor::visit(const Register& value) +void +EncodeSourceVisitor::visit(const Register& value) { assert(value.sel() <= 124 && "Only have 124 registers"); } -void EncodeSourceVisitor::visit(const LocalArray& value) +void +EncodeSourceVisitor::visit(const LocalArray& value) { (void)value; unreachable("An array can't be a source register"); } -void EncodeSourceVisitor::visit(const LocalArrayValue& value) +void +EncodeSourceVisitor::visit(const LocalArrayValue& value) { src.rel = value.addr() ? 1 : 0; } -void EncodeSourceVisitor::visit(const UniformValue& value) +void +EncodeSourceVisitor::visit(const UniformValue& value) { assert(value.sel() >= 512 && "Uniform values must have a sel >= 512"); m_buffer_offset = value.buf_addr(); src.kc_bank = value.kcache_bank(); } -void EncodeSourceVisitor::visit(const LiteralConstant& value) +void +EncodeSourceVisitor::visit(const LiteralConstant& value) { src.value = value.value(); } -void EncodeSourceVisitor::visit(const InlineConstant& value) +void +EncodeSourceVisitor::visit(const InlineConstant& value) { (void)value; } - - const std::map opcode_map = { - {op2_add, ALU_OP2_ADD}, - {op2_mul, ALU_OP2_MUL}, - {op2_mul_ieee, ALU_OP2_MUL_IEEE}, - {op2_max, ALU_OP2_MAX}, - {op2_min, ALU_OP2_MIN}, - {op2_max_dx10, ALU_OP2_MAX_DX10}, - {op2_min_dx10, ALU_OP2_MIN_DX10}, - {op2_sete, ALU_OP2_SETE}, - {op2_setgt, ALU_OP2_SETGT}, - {op2_setge, ALU_OP2_SETGE}, - {op2_setne, ALU_OP2_SETNE}, - {op2_sete_dx10, ALU_OP2_SETE_DX10}, - {op2_setgt_dx10, ALU_OP2_SETGT_DX10}, - {op2_setge_dx10, ALU_OP2_SETGE_DX10}, - {op2_setne_dx10, ALU_OP2_SETNE_DX10}, - {op1_fract, ALU_OP1_FRACT}, - {op1_trunc, ALU_OP1_TRUNC}, - {op1_ceil, ALU_OP1_CEIL}, - {op1_rndne, ALU_OP1_RNDNE}, - {op1_floor, ALU_OP1_FLOOR}, - {op2_ashr_int, ALU_OP2_ASHR_INT}, - {op2_lshr_int, ALU_OP2_LSHR_INT}, - {op2_lshl_int, ALU_OP2_LSHL_INT}, - {op1_mov, ALU_OP1_MOV}, - {op0_nop, ALU_OP0_NOP}, - {op2_mul_64, ALU_OP2_MUL_64}, - {op1v_flt64_to_flt32, ALU_OP1_FLT64_TO_FLT32}, - {op1v_flt32_to_flt64, ALU_OP1_FLT32_TO_FLT64}, - {op2_prede_int, ALU_OP2_PRED_SETE_INT}, - {op2_pred_setne_int, ALU_OP2_PRED_SETNE_INT}, - {op2_pred_setge_int, ALU_OP2_PRED_SETGE_INT}, - {op2_pred_setgt_int, ALU_OP2_PRED_SETGT_INT}, - {op2_pred_setgt_uint, ALU_OP2_PRED_SETGT_UINT}, - {op2_pred_setge_uint, ALU_OP2_PRED_SETGE_UINT}, - {op2_pred_sete, ALU_OP2_PRED_SETE}, - {op2_pred_setgt, ALU_OP2_PRED_SETGT}, - {op2_pred_setge, ALU_OP2_PRED_SETGE}, - {op2_pred_setne, ALU_OP2_PRED_SETNE}, - {op0_pred_set_clr, ALU_OP0_PRED_SET_CLR}, - {op1_pred_set_restore, ALU_OP1_PRED_SET_RESTORE}, - {op2_pred_sete_push, ALU_OP2_PRED_SETE_PUSH}, - {op2_pred_setgt_push, ALU_OP2_PRED_SETGT_PUSH}, - {op2_pred_setge_push, ALU_OP2_PRED_SETGE_PUSH}, - {op2_pred_setne_push, ALU_OP2_PRED_SETNE_PUSH}, - {op2_kille, ALU_OP2_KILLE}, - {op2_killgt, ALU_OP2_KILLGT}, - {op2_killge, ALU_OP2_KILLGE}, - {op2_killne, ALU_OP2_KILLNE}, - {op2_and_int, ALU_OP2_AND_INT}, - {op2_or_int, ALU_OP2_OR_INT}, - {op2_xor_int, ALU_OP2_XOR_INT}, - {op1_not_int, ALU_OP1_NOT_INT}, - {op2_add_int, ALU_OP2_ADD_INT}, - {op2_sub_int, ALU_OP2_SUB_INT}, - {op2_max_int, ALU_OP2_MAX_INT}, - {op2_min_int, ALU_OP2_MIN_INT}, - {op2_max_uint, ALU_OP2_MAX_UINT}, - {op2_min_uint, ALU_OP2_MIN_UINT}, - {op2_sete_int, ALU_OP2_SETE_INT}, - {op2_setgt_int, ALU_OP2_SETGT_INT}, - {op2_setge_int, ALU_OP2_SETGE_INT}, - {op2_setne_int, ALU_OP2_SETNE_INT}, - {op2_setgt_uint, ALU_OP2_SETGT_UINT}, - {op2_setge_uint, ALU_OP2_SETGE_UINT}, - {op2_killgt_uint, ALU_OP2_KILLGT_UINT}, - {op2_killge_uint, ALU_OP2_KILLGE_UINT}, - {op2_pred_setgt_int, ALU_OP2_PRED_SETGT_INT}, - {op2_pred_setge_int, ALU_OP2_PRED_SETGE_INT}, - {op2_pred_setne_int, ALU_OP2_PRED_SETNE_INT}, - {op2_kille_int, ALU_OP2_KILLE_INT}, - {op2_killgt_int, ALU_OP2_KILLGT_INT}, - {op2_killge_int, ALU_OP2_KILLGE_INT}, - {op2_killne_int, ALU_OP2_KILLNE_INT}, - {op2_pred_sete_push_int, ALU_OP2_PRED_SETE_PUSH_INT}, - {op2_pred_setgt_push_int, ALU_OP2_PRED_SETGT_PUSH_INT}, - {op2_pred_setge_push_int, ALU_OP2_PRED_SETGE_PUSH_INT}, - {op2_pred_setne_push_int, ALU_OP2_PRED_SETNE_PUSH_INT}, - {op2_pred_setlt_push_int, ALU_OP2_PRED_SETLT_PUSH_INT}, - {op2_pred_setle_push_int, ALU_OP2_PRED_SETLE_PUSH_INT}, - {op1_flt_to_int, ALU_OP1_FLT_TO_INT}, - {op1_bfrev_int, ALU_OP1_BFREV_INT}, - {op2_addc_uint, ALU_OP2_ADDC_UINT}, - {op2_subb_uint, ALU_OP2_SUBB_UINT}, - {op0_group_barrier, ALU_OP0_GROUP_BARRIER}, - {op0_group_seq_begin, ALU_OP0_GROUP_SEQ_BEGIN}, - {op0_group_seq_end, ALU_OP0_GROUP_SEQ_END}, - {op2_set_mode, ALU_OP2_SET_MODE}, - {op1_set_cf_idx0, ALU_OP0_SET_CF_IDX0}, - {op1_set_cf_idx1, ALU_OP0_SET_CF_IDX1}, - {op2_set_lds_size, ALU_OP2_SET_LDS_SIZE}, - {op1_exp_ieee, ALU_OP1_EXP_IEEE}, - {op1_log_clamped, ALU_OP1_LOG_CLAMPED}, - {op1_log_ieee, ALU_OP1_LOG_IEEE}, - {op1_recip_clamped, ALU_OP1_RECIP_CLAMPED}, - {op1_recip_ff, ALU_OP1_RECIP_FF}, - {op1_recip_ieee, ALU_OP1_RECIP_IEEE}, - {op1_recipsqrt_clamped, ALU_OP1_RECIPSQRT_CLAMPED}, - {op1_recipsqrt_ff, ALU_OP1_RECIPSQRT_FF}, - {op1_recipsqrt_ieee1, ALU_OP1_RECIPSQRT_IEEE}, - {op1_sqrt_ieee, ALU_OP1_SQRT_IEEE}, - {op1_sin, ALU_OP1_SIN}, - {op1_cos, ALU_OP1_COS}, - {op2_mullo_int, ALU_OP2_MULLO_INT}, - {op2_mulhi_int, ALU_OP2_MULHI_INT}, - {op2_mullo_uint, ALU_OP2_MULLO_UINT}, - {op2_mulhi_uint, ALU_OP2_MULHI_UINT}, - {op1_recip_int, ALU_OP1_RECIP_INT}, - {op1_recip_uint, ALU_OP1_RECIP_UINT}, - {op1_recip_64, ALU_OP2_RECIP_64}, - {op1_recip_clamped_64, ALU_OP2_RECIP_CLAMPED_64}, - {op1_recipsqrt_64, ALU_OP2_RECIPSQRT_64}, - {op1_recipsqrt_clamped_64, ALU_OP2_RECIPSQRT_CLAMPED_64}, - {op1_sqrt_64, ALU_OP2_SQRT_64}, - {op1_flt_to_uint, ALU_OP1_FLT_TO_UINT}, - {op1_int_to_flt, ALU_OP1_INT_TO_FLT}, - {op1_uint_to_flt, ALU_OP1_UINT_TO_FLT}, - {op2_bfm_int, ALU_OP2_BFM_INT}, - {op1_flt32_to_flt16, ALU_OP1_FLT32_TO_FLT16}, - {op1_flt16_to_flt32, ALU_OP1_FLT16_TO_FLT32}, - {op1_ubyte0_flt, ALU_OP1_UBYTE0_FLT}, - {op1_ubyte1_flt, ALU_OP1_UBYTE1_FLT}, - {op1_ubyte2_flt, ALU_OP1_UBYTE2_FLT}, - {op1_ubyte3_flt, ALU_OP1_UBYTE3_FLT}, - {op1_bcnt_int, ALU_OP1_BCNT_INT}, - {op1_ffbh_uint, ALU_OP1_FFBH_UINT}, - {op1_ffbl_int, ALU_OP1_FFBL_INT}, - {op1_ffbh_int, ALU_OP1_FFBH_INT}, - {op1_flt_to_uint4, ALU_OP1_FLT_TO_UINT4}, - {op2_dot_ieee, ALU_OP2_DOT_IEEE}, - {op1_flt_to_int_rpi, ALU_OP1_FLT_TO_INT_RPI}, - {op1_flt_to_int_floor, ALU_OP1_FLT_TO_INT_FLOOR}, - {op2_mulhi_uint24, ALU_OP2_MULHI_UINT24}, - {op1_mbcnt_32hi_int, ALU_OP1_MBCNT_32HI_INT}, - {op1_offset_to_flt, ALU_OP1_OFFSET_TO_FLT}, - {op2_mul_uint24, ALU_OP2_MUL_UINT24}, - {op1_bcnt_accum_prev_int, ALU_OP1_BCNT_ACCUM_PREV_INT}, + {op2_add, ALU_OP2_ADD }, + {op2_mul, ALU_OP2_MUL }, + {op2_mul_ieee, ALU_OP2_MUL_IEEE }, + {op2_max, ALU_OP2_MAX }, + {op2_min, ALU_OP2_MIN }, + {op2_max_dx10, ALU_OP2_MAX_DX10 }, + {op2_min_dx10, ALU_OP2_MIN_DX10 }, + {op2_sete, ALU_OP2_SETE }, + {op2_setgt, ALU_OP2_SETGT }, + {op2_setge, ALU_OP2_SETGE }, + {op2_setne, ALU_OP2_SETNE }, + {op2_sete_dx10, ALU_OP2_SETE_DX10 }, + {op2_setgt_dx10, ALU_OP2_SETGT_DX10 }, + {op2_setge_dx10, ALU_OP2_SETGE_DX10 }, + {op2_setne_dx10, ALU_OP2_SETNE_DX10 }, + {op1_fract, ALU_OP1_FRACT }, + {op1_trunc, ALU_OP1_TRUNC }, + {op1_ceil, ALU_OP1_CEIL }, + {op1_rndne, ALU_OP1_RNDNE }, + {op1_floor, ALU_OP1_FLOOR }, + {op2_ashr_int, ALU_OP2_ASHR_INT }, + {op2_lshr_int, ALU_OP2_LSHR_INT }, + {op2_lshl_int, ALU_OP2_LSHL_INT }, + {op1_mov, ALU_OP1_MOV }, + {op0_nop, ALU_OP0_NOP }, + {op2_mul_64, ALU_OP2_MUL_64 }, + {op1v_flt64_to_flt32, ALU_OP1_FLT64_TO_FLT32 }, + {op1v_flt32_to_flt64, ALU_OP1_FLT32_TO_FLT64 }, + {op2_prede_int, ALU_OP2_PRED_SETE_INT }, + {op2_pred_setne_int, ALU_OP2_PRED_SETNE_INT }, + {op2_pred_setge_int, ALU_OP2_PRED_SETGE_INT }, + {op2_pred_setgt_int, ALU_OP2_PRED_SETGT_INT }, + {op2_pred_setgt_uint, ALU_OP2_PRED_SETGT_UINT }, + {op2_pred_setge_uint, ALU_OP2_PRED_SETGE_UINT }, + {op2_pred_sete, ALU_OP2_PRED_SETE }, + {op2_pred_setgt, ALU_OP2_PRED_SETGT }, + {op2_pred_setge, ALU_OP2_PRED_SETGE }, + {op2_pred_setne, ALU_OP2_PRED_SETNE }, + {op0_pred_set_clr, ALU_OP0_PRED_SET_CLR }, + {op1_pred_set_restore, ALU_OP1_PRED_SET_RESTORE }, + {op2_pred_sete_push, ALU_OP2_PRED_SETE_PUSH }, + {op2_pred_setgt_push, ALU_OP2_PRED_SETGT_PUSH }, + {op2_pred_setge_push, ALU_OP2_PRED_SETGE_PUSH }, + {op2_pred_setne_push, ALU_OP2_PRED_SETNE_PUSH }, + {op2_kille, ALU_OP2_KILLE }, + {op2_killgt, ALU_OP2_KILLGT }, + {op2_killge, ALU_OP2_KILLGE }, + {op2_killne, ALU_OP2_KILLNE }, + {op2_and_int, ALU_OP2_AND_INT }, + {op2_or_int, ALU_OP2_OR_INT }, + {op2_xor_int, ALU_OP2_XOR_INT }, + {op1_not_int, ALU_OP1_NOT_INT }, + {op2_add_int, ALU_OP2_ADD_INT }, + {op2_sub_int, ALU_OP2_SUB_INT }, + {op2_max_int, ALU_OP2_MAX_INT }, + {op2_min_int, ALU_OP2_MIN_INT }, + {op2_max_uint, ALU_OP2_MAX_UINT }, + {op2_min_uint, ALU_OP2_MIN_UINT }, + {op2_sete_int, ALU_OP2_SETE_INT }, + {op2_setgt_int, ALU_OP2_SETGT_INT }, + {op2_setge_int, ALU_OP2_SETGE_INT }, + {op2_setne_int, ALU_OP2_SETNE_INT }, + {op2_setgt_uint, ALU_OP2_SETGT_UINT }, + {op2_setge_uint, ALU_OP2_SETGE_UINT }, + {op2_killgt_uint, ALU_OP2_KILLGT_UINT }, + {op2_killge_uint, ALU_OP2_KILLGE_UINT }, + {op2_pred_setgt_int, ALU_OP2_PRED_SETGT_INT }, + {op2_pred_setge_int, ALU_OP2_PRED_SETGE_INT }, + {op2_pred_setne_int, ALU_OP2_PRED_SETNE_INT }, + {op2_kille_int, ALU_OP2_KILLE_INT }, + {op2_killgt_int, ALU_OP2_KILLGT_INT }, + {op2_killge_int, ALU_OP2_KILLGE_INT }, + {op2_killne_int, ALU_OP2_KILLNE_INT }, + {op2_pred_sete_push_int, ALU_OP2_PRED_SETE_PUSH_INT }, + {op2_pred_setgt_push_int, ALU_OP2_PRED_SETGT_PUSH_INT }, + {op2_pred_setge_push_int, ALU_OP2_PRED_SETGE_PUSH_INT }, + {op2_pred_setne_push_int, ALU_OP2_PRED_SETNE_PUSH_INT }, + {op2_pred_setlt_push_int, ALU_OP2_PRED_SETLT_PUSH_INT }, + {op2_pred_setle_push_int, ALU_OP2_PRED_SETLE_PUSH_INT }, + {op1_flt_to_int, ALU_OP1_FLT_TO_INT }, + {op1_bfrev_int, ALU_OP1_BFREV_INT }, + {op2_addc_uint, ALU_OP2_ADDC_UINT }, + {op2_subb_uint, ALU_OP2_SUBB_UINT }, + {op0_group_barrier, ALU_OP0_GROUP_BARRIER }, + {op0_group_seq_begin, ALU_OP0_GROUP_SEQ_BEGIN }, + {op0_group_seq_end, ALU_OP0_GROUP_SEQ_END }, + {op2_set_mode, ALU_OP2_SET_MODE }, + {op1_set_cf_idx0, ALU_OP0_SET_CF_IDX0 }, + {op1_set_cf_idx1, ALU_OP0_SET_CF_IDX1 }, + {op2_set_lds_size, ALU_OP2_SET_LDS_SIZE }, + {op1_exp_ieee, ALU_OP1_EXP_IEEE }, + {op1_log_clamped, ALU_OP1_LOG_CLAMPED }, + {op1_log_ieee, ALU_OP1_LOG_IEEE }, + {op1_recip_clamped, ALU_OP1_RECIP_CLAMPED }, + {op1_recip_ff, ALU_OP1_RECIP_FF }, + {op1_recip_ieee, ALU_OP1_RECIP_IEEE }, + {op1_recipsqrt_clamped, ALU_OP1_RECIPSQRT_CLAMPED }, + {op1_recipsqrt_ff, ALU_OP1_RECIPSQRT_FF }, + {op1_recipsqrt_ieee1, ALU_OP1_RECIPSQRT_IEEE }, + {op1_sqrt_ieee, ALU_OP1_SQRT_IEEE }, + {op1_sin, ALU_OP1_SIN }, + {op1_cos, ALU_OP1_COS }, + {op2_mullo_int, ALU_OP2_MULLO_INT }, + {op2_mulhi_int, ALU_OP2_MULHI_INT }, + {op2_mullo_uint, ALU_OP2_MULLO_UINT }, + {op2_mulhi_uint, ALU_OP2_MULHI_UINT }, + {op1_recip_int, ALU_OP1_RECIP_INT }, + {op1_recip_uint, ALU_OP1_RECIP_UINT }, + {op1_recip_64, ALU_OP2_RECIP_64 }, + {op1_recip_clamped_64, ALU_OP2_RECIP_CLAMPED_64 }, + {op1_recipsqrt_64, ALU_OP2_RECIPSQRT_64 }, + {op1_recipsqrt_clamped_64, ALU_OP2_RECIPSQRT_CLAMPED_64 }, + {op1_sqrt_64, ALU_OP2_SQRT_64 }, + {op1_flt_to_uint, ALU_OP1_FLT_TO_UINT }, + {op1_int_to_flt, ALU_OP1_INT_TO_FLT }, + {op1_uint_to_flt, ALU_OP1_UINT_TO_FLT }, + {op2_bfm_int, ALU_OP2_BFM_INT }, + {op1_flt32_to_flt16, ALU_OP1_FLT32_TO_FLT16 }, + {op1_flt16_to_flt32, ALU_OP1_FLT16_TO_FLT32 }, + {op1_ubyte0_flt, ALU_OP1_UBYTE0_FLT }, + {op1_ubyte1_flt, ALU_OP1_UBYTE1_FLT }, + {op1_ubyte2_flt, ALU_OP1_UBYTE2_FLT }, + {op1_ubyte3_flt, ALU_OP1_UBYTE3_FLT }, + {op1_bcnt_int, ALU_OP1_BCNT_INT }, + {op1_ffbh_uint, ALU_OP1_FFBH_UINT }, + {op1_ffbl_int, ALU_OP1_FFBL_INT }, + {op1_ffbh_int, ALU_OP1_FFBH_INT }, + {op1_flt_to_uint4, ALU_OP1_FLT_TO_UINT4 }, + {op2_dot_ieee, ALU_OP2_DOT_IEEE }, + {op1_flt_to_int_rpi, ALU_OP1_FLT_TO_INT_RPI }, + {op1_flt_to_int_floor, ALU_OP1_FLT_TO_INT_FLOOR }, + {op2_mulhi_uint24, ALU_OP2_MULHI_UINT24 }, + {op1_mbcnt_32hi_int, ALU_OP1_MBCNT_32HI_INT }, + {op1_offset_to_flt, ALU_OP1_OFFSET_TO_FLT }, + {op2_mul_uint24, ALU_OP2_MUL_UINT24 }, + {op1_bcnt_accum_prev_int, ALU_OP1_BCNT_ACCUM_PREV_INT }, {op1_mbcnt_32lo_accum_prev_int, ALU_OP1_MBCNT_32LO_ACCUM_PREV_INT}, - {op2_sete_64, ALU_OP2_SETE_64}, - {op2_setne_64, ALU_OP2_SETNE_64}, - {op2_setgt_64, ALU_OP2_SETGT_64}, - {op2_setge_64, ALU_OP2_SETGE_64}, - {op2_min_64, ALU_OP2_MIN_64}, - {op2_max_64, ALU_OP2_MAX_64}, - {op2_dot4, ALU_OP2_DOT4}, - {op2_dot4_ieee, ALU_OP2_DOT4_IEEE}, - {op2_cube, ALU_OP2_CUBE}, - {op1_max4, ALU_OP1_MAX4}, - {op1_frexp_64, ALU_OP1_FREXP_64}, - {op1_ldexp_64, ALU_OP2_LDEXP_64}, - {op1_fract_64, ALU_OP1_FRACT_64}, - {op2_pred_setgt_64, ALU_OP2_PRED_SETGT_64}, - {op2_pred_sete_64, ALU_OP2_PRED_SETE_64}, - {op2_pred_setge_64, ALU_OP2_PRED_SETGE_64}, - {op2_add_64, ALU_OP2_ADD_64}, - {op1_mova_int, ALU_OP1_MOVA_INT}, - {op1v_flt64_to_flt32, ALU_OP1_FLT64_TO_FLT32}, - {op1_flt32_to_flt64, ALU_OP1_FLT32_TO_FLT64}, - {op2_sad_accum_prev_uint, ALU_OP2_SAD_ACCUM_PREV_UINT}, - {op2_dot, ALU_OP2_DOT}, - {op1_mul_prev, ALU_OP1_MUL_PREV}, - {op1_mul_ieee_prev, ALU_OP1_MUL_IEEE_PREV}, - {op1_add_prev, ALU_OP1_ADD_PREV}, - {op2_muladd_prev, ALU_OP2_MULADD_PREV}, - {op2_muladd_ieee_prev, ALU_OP2_MULADD_IEEE_PREV}, - {op2_interp_xy, ALU_OP2_INTERP_XY}, - {op2_interp_zw, ALU_OP2_INTERP_ZW}, - {op2_interp_x, ALU_OP2_INTERP_X}, - {op2_interp_z, ALU_OP2_INTERP_Z}, - {op0_store_flags, ALU_OP1_STORE_FLAGS}, - {op1_load_store_flags, ALU_OP1_LOAD_STORE_FLAGS}, - {op0_lds_1a, ALU_OP2_LDS_1A}, - {op0_lds_1a1d, ALU_OP2_LDS_1A1D}, - {op0_lds_2a, ALU_OP2_LDS_2A}, - {op1_interp_load_p0, ALU_OP1_INTERP_LOAD_P0}, - {op1_interp_load_p10, ALU_OP1_INTERP_LOAD_P10}, - {op1_interp_load_p20, ALU_OP1_INTERP_LOAD_P20}, - {op3_bfe_uint, ALU_OP3_BFE_UINT}, - {op3_bfe_int, ALU_OP3_BFE_INT}, - {op3_bfi_int, ALU_OP3_BFI_INT}, - {op3_fma, ALU_OP3_FMA}, - {op3_cndne_64, ALU_OP3_CNDNE_64}, - {op3_fma_64, ALU_OP3_FMA_64}, - {op3_lerp_uint, ALU_OP3_LERP_UINT}, - {op3_bit_align_int, ALU_OP3_BIT_ALIGN_INT}, - {op3_byte_align_int, ALU_OP3_BYTE_ALIGN_INT}, - {op3_sad_accum_uint, ALU_OP3_SAD_ACCUM_UINT}, - {op3_sad_accum_hi_uint, ALU_OP3_SAD_ACCUM_HI_UINT}, - {op3_muladd_uint24, ALU_OP3_MULADD_UINT24}, - {op3_lds_idx_op, ALU_OP3_LDS_IDX_OP}, - {op3_muladd, ALU_OP3_MULADD}, - {op3_muladd_m2, ALU_OP3_MULADD_M2}, - {op3_muladd_m4, ALU_OP3_MULADD_M4}, - {op3_muladd_d2, ALU_OP3_MULADD_D2}, - {op3_muladd_ieee, ALU_OP3_MULADD_IEEE}, - {op3_cnde, ALU_OP3_CNDE}, - {op3_cndgt, ALU_OP3_CNDGT}, - {op3_cndge, ALU_OP3_CNDGE}, - {op3_cnde_int, ALU_OP3_CNDE_INT}, - {op3_cndgt_int, ALU_OP3_CNDGT_INT}, - {op3_cndge_int, ALU_OP3_CNDGE_INT}, - {op3_mul_lit, ALU_OP3_MUL_LIT}, + {op2_sete_64, ALU_OP2_SETE_64 }, + {op2_setne_64, ALU_OP2_SETNE_64 }, + {op2_setgt_64, ALU_OP2_SETGT_64 }, + {op2_setge_64, ALU_OP2_SETGE_64 }, + {op2_min_64, ALU_OP2_MIN_64 }, + {op2_max_64, ALU_OP2_MAX_64 }, + {op2_dot4, ALU_OP2_DOT4 }, + {op2_dot4_ieee, ALU_OP2_DOT4_IEEE }, + {op2_cube, ALU_OP2_CUBE }, + {op1_max4, ALU_OP1_MAX4 }, + {op1_frexp_64, ALU_OP1_FREXP_64 }, + {op1_ldexp_64, ALU_OP2_LDEXP_64 }, + {op1_fract_64, ALU_OP1_FRACT_64 }, + {op2_pred_setgt_64, ALU_OP2_PRED_SETGT_64 }, + {op2_pred_sete_64, ALU_OP2_PRED_SETE_64 }, + {op2_pred_setge_64, ALU_OP2_PRED_SETGE_64 }, + {op2_add_64, ALU_OP2_ADD_64 }, + {op1_mova_int, ALU_OP1_MOVA_INT }, + {op1v_flt64_to_flt32, ALU_OP1_FLT64_TO_FLT32 }, + {op1_flt32_to_flt64, ALU_OP1_FLT32_TO_FLT64 }, + {op2_sad_accum_prev_uint, ALU_OP2_SAD_ACCUM_PREV_UINT }, + {op2_dot, ALU_OP2_DOT }, + {op1_mul_prev, ALU_OP1_MUL_PREV }, + {op1_mul_ieee_prev, ALU_OP1_MUL_IEEE_PREV }, + {op1_add_prev, ALU_OP1_ADD_PREV }, + {op2_muladd_prev, ALU_OP2_MULADD_PREV }, + {op2_muladd_ieee_prev, ALU_OP2_MULADD_IEEE_PREV }, + {op2_interp_xy, ALU_OP2_INTERP_XY }, + {op2_interp_zw, ALU_OP2_INTERP_ZW }, + {op2_interp_x, ALU_OP2_INTERP_X }, + {op2_interp_z, ALU_OP2_INTERP_Z }, + {op0_store_flags, ALU_OP1_STORE_FLAGS }, + {op1_load_store_flags, ALU_OP1_LOAD_STORE_FLAGS }, + {op0_lds_1a, ALU_OP2_LDS_1A }, + {op0_lds_1a1d, ALU_OP2_LDS_1A1D }, + {op0_lds_2a, ALU_OP2_LDS_2A }, + {op1_interp_load_p0, ALU_OP1_INTERP_LOAD_P0 }, + {op1_interp_load_p10, ALU_OP1_INTERP_LOAD_P10 }, + {op1_interp_load_p20, ALU_OP1_INTERP_LOAD_P20 }, + {op3_bfe_uint, ALU_OP3_BFE_UINT }, + {op3_bfe_int, ALU_OP3_BFE_INT }, + {op3_bfi_int, ALU_OP3_BFI_INT }, + {op3_fma, ALU_OP3_FMA }, + {op3_cndne_64, ALU_OP3_CNDNE_64 }, + {op3_fma_64, ALU_OP3_FMA_64 }, + {op3_lerp_uint, ALU_OP3_LERP_UINT }, + {op3_bit_align_int, ALU_OP3_BIT_ALIGN_INT }, + {op3_byte_align_int, ALU_OP3_BYTE_ALIGN_INT }, + {op3_sad_accum_uint, ALU_OP3_SAD_ACCUM_UINT }, + {op3_sad_accum_hi_uint, ALU_OP3_SAD_ACCUM_HI_UINT }, + {op3_muladd_uint24, ALU_OP3_MULADD_UINT24 }, + {op3_lds_idx_op, ALU_OP3_LDS_IDX_OP }, + {op3_muladd, ALU_OP3_MULADD }, + {op3_muladd_m2, ALU_OP3_MULADD_M2 }, + {op3_muladd_m4, ALU_OP3_MULADD_M4 }, + {op3_muladd_d2, ALU_OP3_MULADD_D2 }, + {op3_muladd_ieee, ALU_OP3_MULADD_IEEE }, + {op3_cnde, ALU_OP3_CNDE }, + {op3_cndgt, ALU_OP3_CNDGT }, + {op3_cndge, ALU_OP3_CNDGE }, + {op3_cnde_int, ALU_OP3_CNDE_INT }, + {op3_cndgt_int, ALU_OP3_CNDGT_INT }, + {op3_cndge_int, ALU_OP3_CNDGE_INT }, + {op3_mul_lit, ALU_OP3_MUL_LIT }, }; const std::map ds_opcode_map = { - {DS_OP_ADD, FETCH_OP_GDS_ADD}, - {DS_OP_SUB, FETCH_OP_GDS_SUB}, - {DS_OP_RSUB, FETCH_OP_GDS_RSUB}, - {DS_OP_INC, FETCH_OP_GDS_INC}, - {DS_OP_DEC, FETCH_OP_GDS_DEC}, - {DS_OP_MIN_INT, FETCH_OP_GDS_MIN_INT}, - {DS_OP_MAX_INT, FETCH_OP_GDS_MAX_INT}, - {DS_OP_MIN_UINT, FETCH_OP_GDS_MIN_UINT}, - {DS_OP_MAX_UINT, FETCH_OP_GDS_MAX_UINT}, - {DS_OP_AND, FETCH_OP_GDS_AND}, - {DS_OP_OR, FETCH_OP_GDS_OR}, - {DS_OP_XOR, FETCH_OP_GDS_XOR}, - {DS_OP_MSKOR, FETCH_OP_GDS_MSKOR}, - {DS_OP_WRITE, FETCH_OP_GDS_WRITE}, - {DS_OP_WRITE_REL, FETCH_OP_GDS_WRITE_REL}, - {DS_OP_WRITE2, FETCH_OP_GDS_WRITE2}, - {DS_OP_CMP_STORE, FETCH_OP_GDS_CMP_STORE}, - {DS_OP_CMP_STORE_SPF, FETCH_OP_GDS_CMP_STORE_SPF}, - {DS_OP_BYTE_WRITE, FETCH_OP_GDS_BYTE_WRITE}, - {DS_OP_SHORT_WRITE, FETCH_OP_GDS_SHORT_WRITE}, - {DS_OP_ADD_RET, FETCH_OP_GDS_ADD_RET}, - {DS_OP_SUB_RET, FETCH_OP_GDS_SUB_RET}, - {DS_OP_RSUB_RET, FETCH_OP_GDS_RSUB_RET}, - {DS_OP_INC_RET, FETCH_OP_GDS_INC_RET}, - {DS_OP_DEC_RET, FETCH_OP_GDS_DEC_RET}, - {DS_OP_MIN_INT_RET, FETCH_OP_GDS_MIN_INT_RET}, - {DS_OP_MAX_INT_RET, FETCH_OP_GDS_MAX_INT_RET}, - {DS_OP_MIN_UINT_RET, FETCH_OP_GDS_MIN_UINT_RET}, - {DS_OP_MAX_UINT_RET, FETCH_OP_GDS_MAX_UINT_RET}, - {DS_OP_AND_RET, FETCH_OP_GDS_AND_RET}, - {DS_OP_OR_RET, FETCH_OP_GDS_OR_RET}, - {DS_OP_XOR_RET, FETCH_OP_GDS_XOR_RET}, - {DS_OP_MSKOR_RET, FETCH_OP_GDS_MSKOR_RET}, - {DS_OP_XCHG_RET, FETCH_OP_GDS_XCHG_RET}, - {DS_OP_XCHG_REL_RET, FETCH_OP_GDS_XCHG_REL_RET}, - {DS_OP_XCHG2_RET, FETCH_OP_GDS_XCHG2_RET}, - {DS_OP_CMP_XCHG_RET, FETCH_OP_GDS_CMP_XCHG_RET}, - {DS_OP_CMP_XCHG_SPF_RET, FETCH_OP_GDS_CMP_XCHG_SPF_RET}, - {DS_OP_READ_RET, FETCH_OP_GDS_READ_RET}, - {DS_OP_READ_REL_RET, FETCH_OP_GDS_READ_REL_RET}, - {DS_OP_READ2_RET, FETCH_OP_GDS_READ2_RET}, - {DS_OP_READWRITE_RET, FETCH_OP_GDS_READWRITE_RET}, - {DS_OP_BYTE_READ_RET, FETCH_OP_GDS_BYTE_READ_RET}, - {DS_OP_UBYTE_READ_RET, FETCH_OP_GDS_UBYTE_READ_RET}, - {DS_OP_SHORT_READ_RET, FETCH_OP_GDS_SHORT_READ_RET}, - {DS_OP_USHORT_READ_RET, FETCH_OP_GDS_USHORT_READ_RET}, + {DS_OP_ADD, FETCH_OP_GDS_ADD }, + {DS_OP_SUB, FETCH_OP_GDS_SUB }, + {DS_OP_RSUB, FETCH_OP_GDS_RSUB }, + {DS_OP_INC, FETCH_OP_GDS_INC }, + {DS_OP_DEC, FETCH_OP_GDS_DEC }, + {DS_OP_MIN_INT, FETCH_OP_GDS_MIN_INT }, + {DS_OP_MAX_INT, FETCH_OP_GDS_MAX_INT }, + {DS_OP_MIN_UINT, FETCH_OP_GDS_MIN_UINT }, + {DS_OP_MAX_UINT, FETCH_OP_GDS_MAX_UINT }, + {DS_OP_AND, FETCH_OP_GDS_AND }, + {DS_OP_OR, FETCH_OP_GDS_OR }, + {DS_OP_XOR, FETCH_OP_GDS_XOR }, + {DS_OP_MSKOR, FETCH_OP_GDS_MSKOR }, + {DS_OP_WRITE, FETCH_OP_GDS_WRITE }, + {DS_OP_WRITE_REL, FETCH_OP_GDS_WRITE_REL }, + {DS_OP_WRITE2, FETCH_OP_GDS_WRITE2 }, + {DS_OP_CMP_STORE, FETCH_OP_GDS_CMP_STORE }, + {DS_OP_CMP_STORE_SPF, FETCH_OP_GDS_CMP_STORE_SPF }, + {DS_OP_BYTE_WRITE, FETCH_OP_GDS_BYTE_WRITE }, + {DS_OP_SHORT_WRITE, FETCH_OP_GDS_SHORT_WRITE }, + {DS_OP_ADD_RET, FETCH_OP_GDS_ADD_RET }, + {DS_OP_SUB_RET, FETCH_OP_GDS_SUB_RET }, + {DS_OP_RSUB_RET, FETCH_OP_GDS_RSUB_RET }, + {DS_OP_INC_RET, FETCH_OP_GDS_INC_RET }, + {DS_OP_DEC_RET, FETCH_OP_GDS_DEC_RET }, + {DS_OP_MIN_INT_RET, FETCH_OP_GDS_MIN_INT_RET }, + {DS_OP_MAX_INT_RET, FETCH_OP_GDS_MAX_INT_RET }, + {DS_OP_MIN_UINT_RET, FETCH_OP_GDS_MIN_UINT_RET }, + {DS_OP_MAX_UINT_RET, FETCH_OP_GDS_MAX_UINT_RET }, + {DS_OP_AND_RET, FETCH_OP_GDS_AND_RET }, + {DS_OP_OR_RET, FETCH_OP_GDS_OR_RET }, + {DS_OP_XOR_RET, FETCH_OP_GDS_XOR_RET }, + {DS_OP_MSKOR_RET, FETCH_OP_GDS_MSKOR_RET }, + {DS_OP_XCHG_RET, FETCH_OP_GDS_XCHG_RET }, + {DS_OP_XCHG_REL_RET, FETCH_OP_GDS_XCHG_REL_RET }, + {DS_OP_XCHG2_RET, FETCH_OP_GDS_XCHG2_RET }, + {DS_OP_CMP_XCHG_RET, FETCH_OP_GDS_CMP_XCHG_RET }, + {DS_OP_CMP_XCHG_SPF_RET, FETCH_OP_GDS_CMP_XCHG_SPF_RET }, + {DS_OP_READ_RET, FETCH_OP_GDS_READ_RET }, + {DS_OP_READ_REL_RET, FETCH_OP_GDS_READ_REL_RET }, + {DS_OP_READ2_RET, FETCH_OP_GDS_READ2_RET }, + {DS_OP_READWRITE_RET, FETCH_OP_GDS_READWRITE_RET }, + {DS_OP_BYTE_READ_RET, FETCH_OP_GDS_BYTE_READ_RET }, + {DS_OP_UBYTE_READ_RET, FETCH_OP_GDS_UBYTE_READ_RET }, + {DS_OP_SHORT_READ_RET, FETCH_OP_GDS_SHORT_READ_RET }, + {DS_OP_USHORT_READ_RET, FETCH_OP_GDS_USHORT_READ_RET }, {DS_OP_ATOMIC_ORDERED_ALLOC_RET, FETCH_OP_GDS_ATOMIC_ORDERED_ALLOC}, - {DS_OP_INVALID, 0}, + {DS_OP_INVALID, 0 }, }; -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_assembler.h b/src/gallium/drivers/r600/sfn/sfn_assembler.h index e4a741b8bc7..dd598d4da2a 100644 --- a/src/gallium/drivers/r600/sfn/sfn_assembler.h +++ b/src/gallium/drivers/r600/sfn/sfn_assembler.h @@ -29,24 +29,21 @@ #include "../r600_pipe.h" #include "../r600_shader.h" - #include "sfn_shader.h" namespace r600 { -class Assembler -{ +class Assembler { public: Assembler(r600_shader *sh, const r600_shader_key& key); bool lower(Shader *shader); + private: r600_shader *m_sh; const r600_shader_key& m_key; }; - - -} +} // namespace r600 #endif // ASSAMBLY_H diff --git a/src/gallium/drivers/r600/sfn/sfn_callstack.cpp b/src/gallium/drivers/r600/sfn/sfn_callstack.cpp index ce0a00aed2f..92423f6da68 100644 --- a/src/gallium/drivers/r600/sfn/sfn_callstack.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_callstack.cpp @@ -29,16 +29,14 @@ namespace r600 { CallStack::CallStack(r600_bytecode& bc): - m_bc(bc) -{ - -} - -CallStack::~CallStack() + m_bc(bc) { } -int CallStack::push(unsigned type) +CallStack::~CallStack() {} + +int +CallStack::push(unsigned type) { switch (type) { case FC_PUSH_VPM: @@ -52,14 +50,15 @@ int CallStack::push(unsigned type) break; default: assert(0); - } + } return update_max_depth(type); } -void CallStack::pop(unsigned type) +void +CallStack::pop(unsigned type) { - switch(type) { + switch (type) { case FC_PUSH_VPM: --m_bc.stack.push; assert(m_bc.stack.push >= 0); @@ -78,7 +77,8 @@ void CallStack::pop(unsigned type) } } -int CallStack::update_max_depth(unsigned type) +int +CallStack::update_max_depth(unsigned type) { r600_stack_info& stack = m_bc.stack; @@ -87,43 +87,43 @@ int CallStack::update_max_depth(unsigned type) int entry_size = stack.entry_size; - elements = (stack.loop + stack.push_wqm ) * entry_size; + elements = (stack.loop + stack.push_wqm) * entry_size; elements += stack.push; switch (m_bc.gfx_level) { case R600: case R700: - /* pre-r8xx: if any non-WQM PUSH instruction is invoked, 2 elements on - * the stack must be reserved to hold the current active/continue - * masks */ - if (type == FC_PUSH_VPM || stack.push > 0) { - elements += 2; - } - break; + /* pre-r8xx: if any non-WQM PUSH instruction is invoked, 2 elements on + * the stack must be reserved to hold the current active/continue + * masks */ + if (type == FC_PUSH_VPM || stack.push > 0) { + elements += 2; + } + break; case CAYMAN: - /* r9xx: any stack operation on empty stack consumes 2 additional - * elements */ - elements += 2; - break; + /* r9xx: any stack operation on empty stack consumes 2 additional + * elements */ + elements += 2; + break; case EVERGREEN: - /* r8xx+: 2 extra elements are not always required, but one extra - * element must be added for each of the following cases: - * 1. There is an ALU_ELSE_AFTER instruction at the point of greatest - * stack usage. - * (Currently we don't use ALU_ELSE_AFTER.) - * 2. There are LOOP/WQM frames on the stack when any flavor of non-WQM - * PUSH instruction executed. - * - * NOTE: it seems we also need to reserve additional element in some - * other cases, e.g. when we have 4 levels of PUSH_VPM in the shader, - * then STACK_SIZE should be 2 instead of 1 */ - if (type == FC_PUSH_VPM || stack.push > 0) { - elements += 1; - } - break; + /* r8xx+: 2 extra elements are not always required, but one extra + * element must be added for each of the following cases: + * 1. There is an ALU_ELSE_AFTER instruction at the point of greatest + * stack usage. + * (Currently we don't use ALU_ELSE_AFTER.) + * 2. There are LOOP/WQM frames on the stack when any flavor of non-WQM + * PUSH instruction executed. + * + * NOTE: it seems we also need to reserve additional element in some + * other cases, e.g. when we have 4 levels of PUSH_VPM in the shader, + * then STACK_SIZE should be 2 instead of 1 */ + if (type == FC_PUSH_VPM || stack.push > 0) { + elements += 1; + } + break; default: - assert(0); - break; + assert(0); + break; } entry_size = 4; @@ -136,4 +136,4 @@ int CallStack::update_max_depth(unsigned type) return elements; } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_callstack.h b/src/gallium/drivers/r600/sfn/sfn_callstack.h index e1babb7c16e..66e072d3a32 100644 --- a/src/gallium/drivers/r600/sfn/sfn_callstack.h +++ b/src/gallium/drivers/r600/sfn/sfn_callstack.h @@ -38,10 +38,11 @@ public: int push(unsigned type); void pop(unsigned type); int update_max_depth(unsigned type); + private: r600_bytecode& m_bc; }; -} +} // namespace r600 #endif // SFN_CALLSTACK_HH diff --git a/src/gallium/drivers/r600/sfn/sfn_conditionaljumptracker.cpp b/src/gallium/drivers/r600/sfn/sfn_conditionaljumptracker.cpp index ad9a03f8fdf..8b30382648d 100644 --- a/src/gallium/drivers/r600/sfn/sfn_conditionaljumptracker.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_conditionaljumptracker.cpp @@ -25,25 +25,27 @@ */ #include "sfn_conditionaljumptracker.h" + #include "sfn_debug.h" +#include +#include #include #include -#include -#include namespace r600 { +using std::shared_ptr; using std::stack; using std::vector; -using std::shared_ptr; struct StackFrame { StackFrame(r600_bytecode_cf *s, JumpType t): - type(t), - start(s) - {} + type(t), + start(s) + { + } virtual ~StackFrame(); @@ -77,29 +79,26 @@ struct ConditionalJumpTrackerImpl { }; ConditionalJumpTrackerImpl::ConditionalJumpTrackerImpl(): - m_current_loop_stack_pos(0) + m_current_loop_stack_pos(0) { - } -ConditionalJumpTracker::~ConditionalJumpTracker() -{ - delete impl; -} +ConditionalJumpTracker::~ConditionalJumpTracker() { delete impl; } ConditionalJumpTracker::ConditionalJumpTracker() { impl = new ConditionalJumpTrackerImpl(); } -void ConditionalJumpTracker::push(r600_bytecode_cf *start, JumpType type) +void +ConditionalJumpTracker::push(r600_bytecode_cf *start, JumpType type) { PStackFrame f; switch (type) { - case jt_if: + case jt_if: f.reset(new IfFrame(start)); break; - case jt_loop: + case jt_loop: f.reset(new LoopFrame(start)); impl->m_loop_stack.push(f); break; @@ -107,7 +106,8 @@ void ConditionalJumpTracker::push(r600_bytecode_cf *start, JumpType type) impl->m_jump_stack.push(f); } -bool ConditionalJumpTracker::pop(r600_bytecode_cf *final, JumpType type) +bool +ConditionalJumpTracker::pop(r600_bytecode_cf *final, JumpType type) { if (impl->m_jump_stack.empty()) return false; @@ -123,7 +123,8 @@ bool ConditionalJumpTracker::pop(r600_bytecode_cf *final, JumpType type) return true; } -bool ConditionalJumpTracker::add_mid(r600_bytecode_cf *source, JumpType type) +bool +ConditionalJumpTracker::add_mid(r600_bytecode_cf *source, JumpType type) { if (impl->m_jump_stack.empty()) { sfn_log << "Jump stack empty\n"; @@ -147,21 +148,21 @@ bool ConditionalJumpTracker::add_mid(r600_bytecode_cf *source, JumpType type) } IfFrame::IfFrame(r600_bytecode_cf *s): - StackFrame (s, jt_if) + StackFrame(s, jt_if) { } -StackFrame::~StackFrame() -{ -} +StackFrame::~StackFrame() {} -void IfFrame::fixup_mid(r600_bytecode_cf *source) +void +IfFrame::fixup_mid(r600_bytecode_cf *source) { /* JUMP target is ELSE */ start->cf_addr = source->id; } -void IfFrame::fixup_pop(r600_bytecode_cf *final) +void +IfFrame::fixup_pop(r600_bytecode_cf *final) { /* JUMP or ELSE target is one past last CF instruction */ unsigned offset = final->eg_alu_extended ? 4 : 2; @@ -171,15 +172,17 @@ void IfFrame::fixup_pop(r600_bytecode_cf *final) } LoopFrame::LoopFrame(r600_bytecode_cf *s): - StackFrame(s, jt_loop) + StackFrame(s, jt_loop) { } -void LoopFrame::fixup_mid(UNUSED r600_bytecode_cf *mid) +void +LoopFrame::fixup_mid(UNUSED r600_bytecode_cf *mid) { } -void LoopFrame::fixup_pop(r600_bytecode_cf *final) +void +LoopFrame::fixup_pop(r600_bytecode_cf *final) { /* LOOP END address is past LOOP START */ final->cf_addr = start->id + 2; @@ -192,4 +195,4 @@ void LoopFrame::fixup_pop(r600_bytecode_cf *final) m->cf_addr = final->id; } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_conditionaljumptracker.h b/src/gallium/drivers/r600/sfn/sfn_conditionaljumptracker.h index cd2b975ba26..db37187edcd 100644 --- a/src/gallium/drivers/r600/sfn/sfn_conditionaljumptracker.h +++ b/src/gallium/drivers/r600/sfn/sfn_conditionaljumptracker.h @@ -39,8 +39,7 @@ enum JumpType { /** Class to link the jump locations */ -class ConditionalJumpTracker -{ +class ConditionalJumpTracker { public: ConditionalJumpTracker(); ~ConditionalJumpTracker(); @@ -57,9 +56,9 @@ public: bool add_mid(r600_bytecode_cf *source, JumpType type); private: - struct ConditionalJumpTrackerImpl * impl; + struct ConditionalJumpTrackerImpl *impl; }; -} +} // namespace r600 #endif // SFN_CONDITIONALJUMPTRACKER_H diff --git a/src/gallium/drivers/r600/sfn/sfn_debug.cpp b/src/gallium/drivers/r600/sfn/sfn_debug.cpp index fce891e9939..d23123c5fdf 100644 --- a/src/gallium/drivers/r600/sfn/sfn_debug.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_debug.cpp @@ -24,53 +24,54 @@ * USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include "util/u_debug.h" #include "sfn_debug.h" +#include "util/u_debug.h" + namespace r600 { -stderr_streambuf::stderr_streambuf() -{ +stderr_streambuf::stderr_streambuf() {} -} - -int stderr_streambuf::sync() +int +stderr_streambuf::sync() { fflush(stderr); return 0; } -int stderr_streambuf::overflow(int c) +int +stderr_streambuf::overflow(int c) { fputc(c, stderr); return 0; } static const struct debug_named_value sfn_debug_options[] = { - {"instr", SfnLog::instr, "Log all consumed nir instructions"}, - {"ir", SfnLog::r600ir, "Log created R600 IR"}, - {"cc", SfnLog::cc, "Log R600 IR to assembly code creation"}, - {"noerr", SfnLog::err, "Don't log shader conversion errors"}, - {"si", SfnLog::shader_info, "Log shader info (non-zero values)"}, - {"ts", SfnLog::test_shader, "Log shaders in tests"}, - {"reg", SfnLog::reg, "Log register allocation and lookup"}, - {"io", SfnLog::io, "Log shader in and output"}, - {"ass", SfnLog::assembly, "Log IR to assembly conversion"}, - {"flow", SfnLog::flow, "Log Flow instructions"}, - {"merge", SfnLog::merge, "Log register merge operations"}, - {"nomerge", SfnLog::nomerge, "Skip register merge step"}, - {"tex", SfnLog::tex, "Log texture ops"}, - {"trans", SfnLog::trans, "Log generic translation messages"}, - {"schedule", SfnLog::schedule, "Log scheduling"}, - {"opt", SfnLog::opt, "Log optimization"}, - {"steps", SfnLog::steps, "Log shaders at transformation steps"}, - {"noopt", SfnLog::noopt, "Don't run backend optimizations"}, + {"instr", SfnLog::instr, "Log all consumed nir instructions" }, + {"ir", SfnLog::r600ir, "Log created R600 IR" }, + {"cc", SfnLog::cc, "Log R600 IR to assembly code creation"}, + {"noerr", SfnLog::err, "Don't log shader conversion errors" }, + {"si", SfnLog::shader_info, "Log shader info (non-zero values)" }, + {"ts", SfnLog::test_shader, "Log shaders in tests" }, + {"reg", SfnLog::reg, "Log register allocation and lookup" }, + {"io", SfnLog::io, "Log shader in and output" }, + {"ass", SfnLog::assembly, "Log IR to assembly conversion" }, + {"flow", SfnLog::flow, "Log Flow instructions" }, + {"merge", SfnLog::merge, "Log register merge operations" }, + {"nomerge", SfnLog::nomerge, "Skip register merge step" }, + {"tex", SfnLog::tex, "Log texture ops" }, + {"trans", SfnLog::trans, "Log generic translation messages" }, + {"schedule", SfnLog::schedule, "Log scheduling" }, + {"opt", SfnLog::opt, "Log optimization" }, + {"steps", SfnLog::steps, "Log shaders at transformation steps" }, + {"noopt", SfnLog::noopt, "Don't run backend optimizations" }, DEBUG_NAMED_VALUE_END }; SfnLog sfn_log; -std::streamsize stderr_streambuf::xsputn ( const char *s, std::streamsize n ) +std::streamsize +stderr_streambuf::xsputn(const char *s, std::streamsize n) { std::streamsize i = n; while (i--) @@ -79,36 +80,40 @@ std::streamsize stderr_streambuf::xsputn ( const char *s, std::streamsize n ) } SfnLog::SfnLog(): - m_active_log_flags(0), - m_log_mask(0), - m_buf(), - m_output(&m_buf) + m_active_log_flags(0), + m_log_mask(0), + m_buf(), + m_output(&m_buf) { m_log_mask = debug_get_flags_option("R600_NIR_DEBUG", sfn_debug_options, 0); m_log_mask ^= err; } -SfnLog& SfnLog::operator << (SfnLog::LogFlag const l) +SfnLog& +SfnLog::operator<<(SfnLog::LogFlag const l) { m_active_log_flags = l; return *this; } -SfnLog& SfnLog::operator << (UNUSED std::ostream & (*f)(std::ostream&)) +SfnLog& +SfnLog::operator<<(UNUSED std::ostream& (*f)(std::ostream&)) { if (m_active_log_flags & m_log_mask) m_output << f; return *this; } -SfnLog& SfnLog::operator << (nir_shader& sh) +SfnLog& +SfnLog::operator<<(nir_shader& sh) { if (m_active_log_flags & m_log_mask) nir_print_shader(&sh, stderr); return *this; } -SfnLog& SfnLog::operator << (nir_instr &instr) +SfnLog& +SfnLog::operator<<(nir_instr& instr) { if (m_active_log_flags & m_log_mask) nir_print_instr(&instr, stderr); @@ -116,19 +121,17 @@ SfnLog& SfnLog::operator << (nir_instr &instr) } SfnTrace::SfnTrace(SfnLog::LogFlag flag, const char *msg): - m_flag(flag), - m_msg(msg) + m_flag(flag), + m_msg(msg) { - sfn_log << m_flag << std::string(" ", 2 * m_indention++) - << "BEGIN: " << m_msg << "\n"; + sfn_log << m_flag << std::string(" ", 2 * m_indention++) << "BEGIN: " << m_msg << "\n"; } SfnTrace::~SfnTrace() { - sfn_log << m_flag << std::string(" ", 2 * m_indention--) - << "END: " << m_msg << "\n"; + sfn_log << m_flag << std::string(" ", 2 * m_indention--) << "END: " << m_msg << "\n"; } int SfnTrace::m_indention = 0; -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_debug.h b/src/gallium/drivers/r600/sfn/sfn_debug.h index fd3c0fb9193..be7c3bb00a6 100644 --- a/src/gallium/drivers/r600/sfn/sfn_debug.h +++ b/src/gallium/drivers/r600/sfn/sfn_debug.h @@ -27,25 +27,25 @@ #ifndef SFN_STDERR_STREAMLOG_H #define SFN_STDERR_STREAMLOG_H - -#include -#include -#include #include "compiler/nir/nir.h" +#include +#include +#include + namespace r600 { /* Implement some logging for shader-from-nir */ -class stderr_streambuf : public std::streambuf -{ +class stderr_streambuf : public std::streambuf { public: stderr_streambuf(); + protected: int sync(); int overflow(int c); - std::streamsize xsputn ( const char *s, std::streamsize n ); + std::streamsize xsputn(const char *s, std::streamsize n); }; class SfnLog { @@ -53,8 +53,8 @@ public: enum LogFlag { instr = 1 << 0, r600ir = 1 << 1, - cc = 1 << 2, - err = 1 << 3, + cc = 1 << 2, + err = 1 << 3, shader_info = 1 << 4, test_shader = 1 << 5, reg = 1 << 6, @@ -77,14 +77,13 @@ public: /** a special handling to set the output level "inline" \param l the level of the following messages */ - SfnLog& operator << (LogFlag const l); + SfnLog& operator<<(LogFlag const l); /* general output routine; output is only given, if the log flags and the * currently active log mask overlap \returns a reference to this object */ - template - SfnLog& operator << (const T& text) + template SfnLog& operator<<(const T& text) { if (m_active_log_flags & m_log_mask) m_output << text; @@ -96,15 +95,13 @@ public: idea of Dave Brondsema: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=8567 */ - SfnLog& operator << (std::ostream & (*f)(std::ostream&)); + SfnLog& operator<<(std::ostream& (*f)(std::ostream&)); - SfnLog& operator << (nir_shader &sh); + SfnLog& operator<<(nir_shader& sh); - SfnLog& operator << (nir_instr& instr); + SfnLog& operator<<(nir_instr& instr); - int has_debug_flag(uint64_t flag) { - return (m_log_mask & flag) == flag; - } + int has_debug_flag(uint64_t flag) { return (m_log_mask & flag) == flag; } private: uint64_t m_active_log_flags; @@ -117,13 +114,13 @@ class SfnTrace { public: SfnTrace(SfnLog::LogFlag flag, const char *msg); ~SfnTrace(); + private: SfnLog::LogFlag m_flag; const char *m_msg; static int m_indention; }; - #ifndef NDEBUG #define SFN_TRACE_FUNC(LEVEL, MSG) SfnTrace __trace(LEVEL, MSG) #else @@ -132,5 +129,5 @@ private: extern SfnLog sfn_log; -} +} // namespace r600 #endif // SFN_STDERR_STREAMBUF_H diff --git a/src/gallium/drivers/r600/sfn/sfn_defines.h b/src/gallium/drivers/r600/sfn/sfn_defines.h index c5a18b0d5fe..79aac253ab8 100644 --- a/src/gallium/drivers/r600/sfn/sfn_defines.h +++ b/src/gallium/drivers/r600/sfn/sfn_defines.h @@ -31,176 +31,172 @@ #include "amd_family.h" namespace r600 { - enum EGWSOpCode { - cf_sema_v = 0, - cf_sema_p = 1, - cf_gws_barrier = 2, - cf_gws_init = 3, + cf_sema_v = 0, + cf_sema_p = 1, + cf_gws_barrier = 2, + cf_gws_init = 3, }; /* CF ALU instructions [29:26], highest bit always set. */ enum ECFAluOpCode { - cf_alu_undefined = 0, - cf_alu = CF_OP_ALU, - cf_alu_push_before = CF_OP_ALU_PUSH_BEFORE, - cf_alu_pop_after = CF_OP_ALU_POP_AFTER, - cf_alu_pop2_after = CF_OP_ALU_POP2_AFTER, - cf_alu_extended = CF_OP_ALU_EXT, - cf_alu_continue = CF_OP_ALU_CONTINUE, - cf_alu_break = CF_OP_ALU_BREAK, - cf_alu_else_after = CF_OP_ALU_ELSE_AFTER, + cf_alu_undefined = 0, + cf_alu = CF_OP_ALU, + cf_alu_push_before = CF_OP_ALU_PUSH_BEFORE, + cf_alu_pop_after = CF_OP_ALU_POP_AFTER, + cf_alu_pop2_after = CF_OP_ALU_POP2_AFTER, + cf_alu_extended = CF_OP_ALU_EXT, + cf_alu_continue = CF_OP_ALU_CONTINUE, + cf_alu_break = CF_OP_ALU_BREAK, + cf_alu_else_after = CF_OP_ALU_ELSE_AFTER, }; enum ECFAluOpCodeEG { - eg_cf_alu_undefined = 0, - eg_cf_alu = 8, - eg_cf_alu_push_before = 9, - eg_cf_alu_pop_after = 10, - eg_cf_alu_pop2_after = 11, - eg_cf_alu_extended = 12, - eg_cf_alu_continue = 13, - eg_cf_alu_break = 14, - eg_cf_alu_else_after = 15, + eg_cf_alu_undefined = 0, + eg_cf_alu = 8, + eg_cf_alu_push_before = 9, + eg_cf_alu_pop_after = 10, + eg_cf_alu_pop2_after = 11, + eg_cf_alu_extended = 12, + eg_cf_alu_continue = 13, + eg_cf_alu_break = 14, + eg_cf_alu_else_after = 15, }; - enum ECFOpCode { - cf_nop = CF_OP_NOP, - cf_tc = CF_OP_TEX, - cf_vc = CF_OP_VTX, - cf_gds = CF_OP_GDS, - cf_loop_start = CF_OP_LOOP_START, - cf_loop_end = CF_OP_LOOP_END, - cf_loop_start_dx10 = CF_OP_LOOP_START_DX10, - cf_loop_start_no_al = CF_OP_LOOP_START_NO_AL, - cf_loop_continue = CF_OP_LOOP_CONTINUE, - cf_loop_break = CF_OP_LOOP_BREAK, - cf_jump = CF_OP_JUMP, - cf_push = CF_OP_PUSH, - cf_else = CF_OP_ELSE, - cf_pop = CF_OP_POP, - /* 15 - 17 reserved */ - cf_call = CF_OP_CALL, - cf_call_fs = CF_OP_CALL_FS, - cf_return = CF_OP_RET, - cf_emit_vertex = CF_OP_EMIT_VERTEX, - cf_emit_cut_vertex = CF_OP_EMIT_CUT_VERTEX, - cf_cut_vertex = CF_OP_CUT_VERTEX, - cf_kill = CF_OP_KILL, - /* 25 reserved */ - cf_wait_ack = CF_OP_WAIT_ACK, - cf_tc_ack = CF_OP_TEX_ACK, - cf_vc_ack = CF_OP_VTX_ACK, - cf_jump_table = CF_OP_JUMPTABLE, - cf_global_wave_sync = CF_OP_WAVE_SYNC, - cf_halt = CF_OP_HALT, - /* gap 32-63*/ - cf_mem_stream0_buf0 = CF_OP_MEM_STREAM0_BUF0, - cf_mem_stream0_buf1 = CF_OP_MEM_STREAM0_BUF1, - cf_mem_stream0_buf2 = CF_OP_MEM_STREAM0_BUF2, - cf_mem_stream0_buf3 = CF_OP_MEM_STREAM0_BUF3, + cf_nop = CF_OP_NOP, + cf_tc = CF_OP_TEX, + cf_vc = CF_OP_VTX, + cf_gds = CF_OP_GDS, + cf_loop_start = CF_OP_LOOP_START, + cf_loop_end = CF_OP_LOOP_END, + cf_loop_start_dx10 = CF_OP_LOOP_START_DX10, + cf_loop_start_no_al = CF_OP_LOOP_START_NO_AL, + cf_loop_continue = CF_OP_LOOP_CONTINUE, + cf_loop_break = CF_OP_LOOP_BREAK, + cf_jump = CF_OP_JUMP, + cf_push = CF_OP_PUSH, + cf_else = CF_OP_ELSE, + cf_pop = CF_OP_POP, + /* 15 - 17 reserved */ + cf_call = CF_OP_CALL, + cf_call_fs = CF_OP_CALL_FS, + cf_return = CF_OP_RET, + cf_emit_vertex = CF_OP_EMIT_VERTEX, + cf_emit_cut_vertex = CF_OP_EMIT_CUT_VERTEX, + cf_cut_vertex = CF_OP_CUT_VERTEX, + cf_kill = CF_OP_KILL, + /* 25 reserved */ + cf_wait_ack = CF_OP_WAIT_ACK, + cf_tc_ack = CF_OP_TEX_ACK, + cf_vc_ack = CF_OP_VTX_ACK, + cf_jump_table = CF_OP_JUMPTABLE, + cf_global_wave_sync = CF_OP_WAVE_SYNC, + cf_halt = CF_OP_HALT, + /* gap 32-63*/ + cf_mem_stream0_buf0 = CF_OP_MEM_STREAM0_BUF0, + cf_mem_stream0_buf1 = CF_OP_MEM_STREAM0_BUF1, + cf_mem_stream0_buf2 = CF_OP_MEM_STREAM0_BUF2, + cf_mem_stream0_buf3 = CF_OP_MEM_STREAM0_BUF3, - cf_mem_stream1_buf0 = CF_OP_MEM_STREAM1_BUF0, - cf_mem_stream1_buf1 = CF_OP_MEM_STREAM1_BUF1, - cf_mem_stream1_buf2 = CF_OP_MEM_STREAM1_BUF2, - cf_mem_stream1_buf3 = CF_OP_MEM_STREAM1_BUF3, + cf_mem_stream1_buf0 = CF_OP_MEM_STREAM1_BUF0, + cf_mem_stream1_buf1 = CF_OP_MEM_STREAM1_BUF1, + cf_mem_stream1_buf2 = CF_OP_MEM_STREAM1_BUF2, + cf_mem_stream1_buf3 = CF_OP_MEM_STREAM1_BUF3, - cf_mem_stream2_buf0 = CF_OP_MEM_STREAM2_BUF0, - cf_mem_stream2_buf1 = CF_OP_MEM_STREAM2_BUF1, - cf_mem_stream2_buf2 = CF_OP_MEM_STREAM2_BUF2, - cf_mem_stream2_buf3 = CF_OP_MEM_STREAM2_BUF3, + cf_mem_stream2_buf0 = CF_OP_MEM_STREAM2_BUF0, + cf_mem_stream2_buf1 = CF_OP_MEM_STREAM2_BUF1, + cf_mem_stream2_buf2 = CF_OP_MEM_STREAM2_BUF2, + cf_mem_stream2_buf3 = CF_OP_MEM_STREAM2_BUF3, - cf_mem_stream3_buf0 = CF_OP_MEM_STREAM3_BUF0, - cf_mem_stream3_buf1 = CF_OP_MEM_STREAM3_BUF1, - cf_mem_stream3_buf2 = CF_OP_MEM_STREAM3_BUF2, - cf_mem_stream3_buf3 = CF_OP_MEM_STREAM3_BUF3, + cf_mem_stream3_buf0 = CF_OP_MEM_STREAM3_BUF0, + cf_mem_stream3_buf1 = CF_OP_MEM_STREAM3_BUF1, + cf_mem_stream3_buf2 = CF_OP_MEM_STREAM3_BUF2, + cf_mem_stream3_buf3 = CF_OP_MEM_STREAM3_BUF3, - cf_mem_write_scratch = CF_OP_MEM_SCRATCH , - /* reserved 81 */ - cf_mem_ring = CF_OP_MEM_RING, - cf_export = CF_OP_EXPORT, - cf_export_done = CF_OP_EXPORT_DONE, - cf_mem_export = CF_OP_MEM_EXPORT, - cf_mem_rat = CF_OP_MEM_RAT, - cf_mem_rat_cacheless = CF_OP_MEM_RAT_NOCACHE, - - cf_mem_ring1 = CF_OP_MEM_RING1, - cf_mem_ring2 = CF_OP_MEM_RING2, - cf_mem_ring3 = CF_OP_MEM_RING3, - cf_mem_export_combined = CF_OP_MEM_MEM_COMBINED, - cf_mem_rat_combined_cacheless = CF_OP_MEM_RAT_COMBINED_NOCACHE + cf_mem_write_scratch = CF_OP_MEM_SCRATCH, + /* reserved 81 */ + cf_mem_ring = CF_OP_MEM_RING, + cf_export = CF_OP_EXPORT, + cf_export_done = CF_OP_EXPORT_DONE, + cf_mem_export = CF_OP_MEM_EXPORT, + cf_mem_rat = CF_OP_MEM_RAT, + cf_mem_rat_cacheless = CF_OP_MEM_RAT_NOCACHE, + cf_mem_ring1 = CF_OP_MEM_RING1, + cf_mem_ring2 = CF_OP_MEM_RING2, + cf_mem_ring3 = CF_OP_MEM_RING3, + cf_mem_export_combined = CF_OP_MEM_MEM_COMBINED, + cf_mem_rat_combined_cacheless = CF_OP_MEM_RAT_COMBINED_NOCACHE }; enum ECFOpCodeEG { - eg_cf_nop = 0, - eg_cf_tc = 1, - eg_cf_vc = 2, - eg_cf_gds = 3, - eg_cf_loop_start = 4, - eg_cf_loop_end = 5, - eg_cf_loop_start_dx10 = 6, - eg_cf_loop_start_no_al = 7, - eg_cf_loop_continue = 8, - eg_cf_loop_break = 9, - eg_cf_jump = 10, - eg_cf_push = 11, - eg_cf_else = 13, - eg_cf_pop = 14, - /* 15 - 17 reserved */ - eg_cf_call = 18, - eg_cf_call_fs, - eg_cf_return, - eg_cf_emit_vertex, - eg_cf_emit_cut_vertex, - eg_cf_cut_vertex, - eg_cf_kill, - /* 25 reserved */ - eg_cf_wait_ack = 26, - eg_cf_tc_ack, - eg_cf_vc_ack, - eg_cf_jump_table, - eg_cf_global_wave_sync, - eg_cf_halt, - /* gap 32-63*/ - eg_cf_mem_stream0_buf0 = 64, - eg_cf_mem_stream0_buf1, - eg_cf_mem_stream0_buf2, - eg_cf_mem_stream0_buf3, + eg_cf_nop = 0, + eg_cf_tc = 1, + eg_cf_vc = 2, + eg_cf_gds = 3, + eg_cf_loop_start = 4, + eg_cf_loop_end = 5, + eg_cf_loop_start_dx10 = 6, + eg_cf_loop_start_no_al = 7, + eg_cf_loop_continue = 8, + eg_cf_loop_break = 9, + eg_cf_jump = 10, + eg_cf_push = 11, + eg_cf_else = 13, + eg_cf_pop = 14, + /* 15 - 17 reserved */ + eg_cf_call = 18, + eg_cf_call_fs, + eg_cf_return, + eg_cf_emit_vertex, + eg_cf_emit_cut_vertex, + eg_cf_cut_vertex, + eg_cf_kill, + /* 25 reserved */ + eg_cf_wait_ack = 26, + eg_cf_tc_ack, + eg_cf_vc_ack, + eg_cf_jump_table, + eg_cf_global_wave_sync, + eg_cf_halt, + /* gap 32-63*/ + eg_cf_mem_stream0_buf0 = 64, + eg_cf_mem_stream0_buf1, + eg_cf_mem_stream0_buf2, + eg_cf_mem_stream0_buf3, - eg_cf_mem_stream1_buf0, - eg_cf_mem_stream1_buf1, - eg_cf_mem_stream1_buf2, - eg_cf_mem_stream1_buf3, + eg_cf_mem_stream1_buf0, + eg_cf_mem_stream1_buf1, + eg_cf_mem_stream1_buf2, + eg_cf_mem_stream1_buf3, - eg_cf_mem_stream2_buf0, - eg_cf_mem_stream2_buf1, - eg_cf_mem_stream2_buf2, - eg_cf_mem_stream2_buf3, + eg_cf_mem_stream2_buf0, + eg_cf_mem_stream2_buf1, + eg_cf_mem_stream2_buf2, + eg_cf_mem_stream2_buf3, - eg_cf_mem_stream3_buf0, - eg_cf_mem_stream3_buf1, - eg_cf_mem_stream3_buf2, - eg_cf_mem_stream3_buf3, + eg_cf_mem_stream3_buf0, + eg_cf_mem_stream3_buf1, + eg_cf_mem_stream3_buf2, + eg_cf_mem_stream3_buf3, - eg_cf_mem_write_scratch, - /* reserved 81 */ - eg_cf_mem_ring = 82, - eg_cf_export, - eg_cf_export_done, - eg_cf_mem_export, - eg_cf_mem_rat, - eg_cf_mem_rat_cacheless, + eg_cf_mem_write_scratch, + /* reserved 81 */ + eg_cf_mem_ring = 82, + eg_cf_export, + eg_cf_export_done, + eg_cf_mem_export, + eg_cf_mem_rat, + eg_cf_mem_rat_cacheless, - eg_cf_mem_ring1, - eg_cf_mem_ring2, - eg_cf_mem_ring3, - eg_cf_mem_export_combined, - eg_cf_mem_rat_combined_cacheless + eg_cf_mem_ring1, + eg_cf_mem_ring2, + eg_cf_mem_ring3, + eg_cf_mem_export_combined, + eg_cf_mem_rat_combined_cacheless }; - enum EVFetchInstr { vc_fetch = FETCH_OP_VFETCH, vc_semantic = FETCH_OP_SEMFETCH, @@ -316,6 +312,6 @@ enum EBufferIndexMode { bim_invalid }; -} +} // namespace r600 #endif // DEFINES_H diff --git a/src/gallium/drivers/r600/sfn/sfn_docu.txt b/src/gallium/drivers/r600/sfn/sfn_docu.txt index 4784599377a..31d38af2cc1 100644 --- a/src/gallium/drivers/r600/sfn/sfn_docu.txt +++ b/src/gallium/drivers/r600/sfn/sfn_docu.txt @@ -1,4 +1,4 @@ -# R600 shader from NIR +#R600 shader from NIR This code is an attempt to implement a NIR backend for r600. @@ -29,6 +29,3 @@ piglit: - don't rely on the backend to merge some alu groups ## There are still some hangs - - - diff --git a/src/gallium/drivers/r600/sfn/sfn_instr.cpp b/src/gallium/drivers/r600/sfn/sfn_instr.cpp index aa860c3b9c1..f95e032eca8 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_instr.cpp @@ -25,17 +25,17 @@ */ #include "sfn_instr_alugroup.h" +#include "sfn_instr_controlflow.h" #include "sfn_instr_export.h" #include "sfn_instr_fetch.h" -#include "sfn_instr_mem.h" #include "sfn_instr_lds.h" +#include "sfn_instr_mem.h" #include "sfn_instr_tex.h" -#include "sfn_instr_controlflow.h" #include #include -#include #include +#include namespace r600 { @@ -43,23 +43,22 @@ using std::string; using std::vector; Instr::Instr(): - m_use_count(0), - m_block_id(std::numeric_limits::max()), - m_index(std::numeric_limits::max()) + m_use_count(0), + m_block_id(std::numeric_limits::max()), + m_index(std::numeric_limits::max()) { } -Instr::~Instr() -{ +Instr::~Instr() {} -} - -void Instr::print(std::ostream& os) const +void +Instr::print(std::ostream& os) const { do_print(os); } -bool Instr::ready() const +bool +Instr::ready() const { for (auto& i : m_required_instr) if (!i->ready()) @@ -67,7 +66,8 @@ bool Instr::ready() const return do_ready(); } -int int_from_string_with_prefix(const std::string& str, const std::string& prefix) +int +int_from_string_with_prefix(const std::string& str, const std::string& prefix) { if (str.substr(0, prefix.length()) != prefix) { std::cerr << "Expect '" << prefix << "' as start of '" << str << "'\n"; @@ -80,7 +80,8 @@ int int_from_string_with_prefix(const std::string& str, const std::string& prefi return retval; } -int sel_and_szw_from_string(const std::string& str, RegisterVec4::Swizzle &swz, bool& is_ssa) +int +sel_and_szw_from_string(const std::string& str, RegisterVec4::Swizzle& swz, bool& is_ssa) { assert(str[0] == 'R' || str[0] == '_' || str[0] == 'S'); int sel = 0; @@ -105,13 +106,27 @@ int sel_and_szw_from_string(const std::string& str, RegisterVec4::Swizzle &swz, int i = 0; while (istr != str.end()) { switch (*istr) { - case 'x': swz[i] = 0; break; - case 'y': swz[i] = 1; break; - case 'z': swz[i] = 2; break; - case 'w': swz[i] = 3; break; - case '0': swz[i] = 4; break; - case '1': swz[i] = 5; break; - case '_': swz[i] = 7; break; + case 'x': + swz[i] = 0; + break; + case 'y': + swz[i] = 1; + break; + case 'z': + swz[i] = 2; + break; + case 'w': + swz[i] = 3; + break; + case '0': + swz[i] = 4; + break; + case '1': + swz[i] = 5; + break; + case '_': + swz[i] = 7; + break; default: unreachable("Unknown swizzle character"); } @@ -124,12 +139,14 @@ int sel_and_szw_from_string(const std::string& str, RegisterVec4::Swizzle &swz, return sel; } -bool Instr::is_last() const +bool +Instr::is_last() const { return true; } -bool Instr::set_dead() +bool +Instr::set_dead() { if (m_instr_flags.test(always_keep)) return false; @@ -138,26 +155,30 @@ bool Instr::set_dead() return is_dead; } -bool Instr::propagate_death() +bool +Instr::propagate_death() { return true; } -bool Instr::replace_source(PRegister old_src, PVirtualValue new_src) +bool +Instr::replace_source(PRegister old_src, PVirtualValue new_src) { (void)old_src; (void)new_src; return false; } -void Instr::add_required_instr(Instr *instr) +void +Instr::add_required_instr(Instr *instr) { assert(instr); m_required_instr.push_back(instr); instr->m_dependend_instr.push_back(this); } -void Instr::replace_required_instr(Instr *old_instr, Instr *new_instr) +void +Instr::replace_required_instr(Instr *old_instr, Instr *new_instr) { for (auto i = m_required_instr.begin(); i != m_required_instr.end(); ++i) { @@ -166,22 +187,24 @@ void Instr::replace_required_instr(Instr *old_instr, Instr *new_instr) } } -bool Instr::replace_dest(PRegister new_dest, r600::AluInstr *move_instr) +bool +Instr::replace_dest(PRegister new_dest, r600::AluInstr *move_instr) { (void)new_dest; (void)move_instr; return false; } -void Instr::set_blockid(int id, int index) +void +Instr::set_blockid(int id, int index) { m_block_id = id; m_index = index; forward_set_blockid(id, index); } - -void Instr::forward_set_blockid(int id, int index) +void +Instr::forward_set_blockid(int id, int index) { (void)id; (void)index; @@ -191,9 +214,9 @@ InstrWithVectorResult::InstrWithVectorResult(const RegisterVec4& dest, const RegisterVec4::Swizzle& dest_swizzle, int resource_base, PRegister resource_offset): - InstrWithResource(resource_base, resource_offset), - m_dest(dest), - m_dest_swizzle(dest_swizzle) + InstrWithResource(resource_base, resource_offset), + m_dest(dest), + m_dest_swizzle(dest_swizzle) { for (int i = 0; i < 4; ++i) { if (m_dest_swizzle[i] < 6) @@ -201,18 +224,20 @@ InstrWithVectorResult::InstrWithVectorResult(const RegisterVec4& dest, } } -void InstrWithVectorResult::print_dest(std::ostream& os) const +void +InstrWithVectorResult::print_dest(std::ostream& os) const { - os << (m_dest[0]->is_ssa() ? 'S' : 'R' ) << m_dest.sel(); + os << (m_dest[0]->is_ssa() ? 'S' : 'R') << m_dest.sel(); os << "."; for (int i = 0; i < 4; ++i) os << VirtualValue::chanchar[m_dest_swizzle[i]]; } -bool InstrWithVectorResult::comp_dest(const RegisterVec4& dest, - const RegisterVec4::Swizzle& dest_swizzle) const +bool +InstrWithVectorResult::comp_dest(const RegisterVec4& dest, + const RegisterVec4::Swizzle& dest_swizzle) const { - for(int i = 0; i < 4; ++i) { + for (int i = 0; i < 4; ++i) { if (!m_dest[i]->equal_to(*dest[i])) { return false; } @@ -222,7 +247,8 @@ bool InstrWithVectorResult::comp_dest(const RegisterVec4& dest, return true; } -void Block::do_print(std::ostream& os) const +void +Block::do_print(std::ostream& os) const { for (int j = 0; j < 2 * m_nesting_depth; ++j) os << ' '; @@ -237,7 +263,8 @@ void Block::do_print(std::ostream& os) const os << "BLOCK END\n"; } -bool Block::is_equal_to(const Block& lhs) const +bool +Block::is_equal_to(const Block& lhs) const { if (m_id != lhs.m_id || m_nesting_depth != lhs.m_nesting_depth) return false; @@ -245,53 +272,64 @@ bool Block::is_equal_to(const Block& lhs) const if (m_instructions.size() != lhs.m_instructions.size()) return false; - return std::inner_product(m_instructions.begin(), m_instructions.end(), lhs.m_instructions.begin(), - true, - [] (bool l, bool r) { return l && r;}, - [](PInst l, PInst r) { return l->equal_to(*r);}); + return std::inner_product( + m_instructions.begin(), + m_instructions.end(), + lhs.m_instructions.begin(), + true, + [](bool l, bool r) { return l && r; }, + [](PInst l, PInst r) { return l->equal_to(*r); }); } -inline bool operator != (const Block& lhs, const Block& rhs) +inline bool +operator!=(const Block& lhs, const Block& rhs) { return !lhs.is_equal_to(rhs); } -void Block::erase(iterator node) +void +Block::erase(iterator node) { m_instructions.erase(node); } -void Block::set_type(Type t) +void +Block::set_type(Type t) { m_blocK_type = t; switch (t) { case vtx: case gds: - case tex: m_remaining_slots = 8; break; /* TODO: 16 for >= EVERGREEN */ + case tex: + m_remaining_slots = 8; + break; /* TODO: 16 for >= EVERGREEN */ default: m_remaining_slots = 0xffff; } } Block::Block(int nesting_depth, int id): - m_nesting_depth(nesting_depth), - m_id(id), - m_next_index(0) + m_nesting_depth(nesting_depth), + m_id(id), + m_next_index(0) { assert(!has_instr_flag(force_cf)); } -void Block::accept(ConstInstrVisitor& visitor) const +void +Block::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void Block::accept(InstrVisitor& visitor) +void +Block::accept(InstrVisitor& visitor) { visitor.visit(this); } -void Block::push_back(PInst instr) +void +Block::push_back(PInst instr) { instr->set_blockid(m_id, m_next_index++); if (m_remaining_slots != 0xffff) { @@ -304,12 +342,13 @@ void Block::push_back(PInst instr) m_instructions.push_back(instr); } -bool Block::try_reserve_kcache(const AluGroup& group) +bool +Block::try_reserve_kcache(const AluGroup& group) { auto kcache = m_kcache; auto kcache_constants = group.get_kconsts(); - for (auto& kc : kcache_constants) { + for (auto& kc : kcache_constants) { auto u = kc->as_uniform(); assert(u); if (!try_reserve_kcache(*u, kcache)) { @@ -323,7 +362,8 @@ bool Block::try_reserve_kcache(const AluGroup& group) return true; } -bool Block::try_reserve_kcache(const AluInstr& instr) +bool +Block::try_reserve_kcache(const AluInstr& instr) { auto kcache = m_kcache; @@ -341,7 +381,8 @@ bool Block::try_reserve_kcache(const AluInstr& instr) return true; } -void Block::set_chipclass(r600_chip_class chip_class) +void +Block::set_chipclass(r600_chip_class chip_class) { if (chip_class < ISA_CC_EVERGREEN) s_max_kcache_banks = 2; @@ -351,13 +392,13 @@ void Block::set_chipclass(r600_chip_class chip_class) unsigned Block::s_max_kcache_banks = 4; -bool Block::try_reserve_kcache(const UniformValue& u, - std::array& kcache) const +bool +Block::try_reserve_kcache(const UniformValue& u, std::array& kcache) const { const int kcache_banks = s_max_kcache_banks; // TODO: handle pre-evergreen int bank = u.kcache_bank(); - int sel = (u.sel() - 512); + int sel = (u.sel() - 512); int line = sel >> 4; bool found = false; @@ -367,13 +408,14 @@ bool Block::try_reserve_kcache(const UniformValue& u, if (kcache[i].bank < bank) continue; - if ((kcache[i].bank == bank && - kcache[i].addr > line + 1) || + if ((kcache[i].bank == bank && kcache[i].addr > line + 1) || kcache[i].bank > bank) { if (kcache[kcache_banks - 1].mode) return false; - memmove(&kcache[i+1],&kcache[i], (kcache_banks-i-1)*sizeof(KCacheLine)); + memmove(&kcache[i + 1], + &kcache[i], + (kcache_banks - i - 1) * sizeof(KCacheLine)); kcache[i].mode = KCacheLine::lock_1; kcache[i].bank = bank; kcache[i].addr = line; @@ -413,14 +455,16 @@ bool Block::try_reserve_kcache(const UniformValue& u, return false; } -void Block::lds_group_start(AluInstr *alu) +void +Block::lds_group_start(AluInstr *alu) { assert(!m_lds_group_start); m_lds_group_start = alu; m_lds_group_requirement = 0; } -void Block::lds_group_end() +void +Block::lds_group_end() { assert(m_lds_group_start); m_lds_group_start->set_required_slots(m_lds_group_requirement); @@ -428,32 +472,29 @@ void Block::lds_group_end() } InstrWithVectorResult::InstrWithVectorResult(const InstrWithVectorResult& orig): - InstrWithResource(orig), - m_dest(orig.m_dest), - m_dest_swizzle(orig.m_dest_swizzle) + InstrWithResource(orig), + m_dest(orig.m_dest), + m_dest_swizzle(orig.m_dest_swizzle) { } class InstrComparer : public ConstInstrVisitor { public: InstrComparer() = default; - bool result {false}; + bool result{false}; -#define DECLARE_MEMBER(TYPE) \ - InstrComparer(const TYPE *instr) \ - { \ - this_ ## TYPE = instr; \ - } \ - \ - void visit(const TYPE& instr) \ - { \ - result = false; \ - if (!this_ ## TYPE) \ - return; \ - result = this_ ## TYPE->is_equal_to(instr); \ - } \ - \ - const TYPE *this_ ## TYPE{nullptr}; +#define DECLARE_MEMBER(TYPE) \ + InstrComparer(const TYPE *instr) { this_##TYPE = instr; } \ + \ + void visit(const TYPE& instr) \ + { \ + result = false; \ + if (!this_##TYPE) \ + return; \ + result = this_##TYPE->is_equal_to(instr); \ + } \ + \ + const TYPE *this_##TYPE{nullptr}; DECLARE_MEMBER(AluInstr); DECLARE_MEMBER(AluGroup); @@ -474,82 +515,65 @@ public: DECLARE_MEMBER(RatInstr); }; -class InstrCompareForward: public ConstInstrVisitor { +class InstrCompareForward : public ConstInstrVisitor { public: + void visit(const AluInstr& instr) override { m_comparer = InstrComparer(&instr); } - void visit(const AluInstr& instr) override { + void visit(const AluGroup& instr) override { m_comparer = InstrComparer(&instr); } + + void visit(const TexInstr& instr) override { m_comparer = InstrComparer(&instr); } + + void visit(const ExportInstr& instr) override { m_comparer = InstrComparer(&instr); } + + void visit(const FetchInstr& instr) override { m_comparer = InstrComparer(&instr); } + + void visit(const Block& instr) override { m_comparer = InstrComparer(&instr); } + + void visit(const ControlFlowInstr& instr) override + { m_comparer = InstrComparer(&instr); } - void visit(const AluGroup& instr) override { + void visit(const IfInstr& instr) override { m_comparer = InstrComparer(&instr); } + + void visit(const ScratchIOInstr& instr) override + { m_comparer = InstrComparer(&instr); } - void visit(const TexInstr& instr) override { + void visit(const StreamOutInstr& instr) override + { m_comparer = InstrComparer(&instr); } - void visit(const ExportInstr& instr) override { + void visit(const MemRingOutInstr& instr) override + { m_comparer = InstrComparer(&instr); } - void visit(const FetchInstr& instr) override { + void visit(const EmitVertexInstr& instr) override + { m_comparer = InstrComparer(&instr); } - void visit(const Block& instr) override { + void visit(const GDSInstr& instr) override { m_comparer = InstrComparer(&instr); } + + void visit(const WriteTFInstr& instr) override { m_comparer = InstrComparer(&instr); } + + void visit(const LDSAtomicInstr& instr) override + { m_comparer = InstrComparer(&instr); } - void visit(const ControlFlowInstr& instr) override { - m_comparer = InstrComparer(&instr); - } + void visit(const LDSReadInstr& instr) override { m_comparer = InstrComparer(&instr); } - void visit(const IfInstr& instr) override { - m_comparer = InstrComparer(&instr); - } - - void visit(const ScratchIOInstr& instr) override { - m_comparer = InstrComparer(&instr); - } - - void visit(const StreamOutInstr& instr) override { - m_comparer = InstrComparer(&instr); - } - - void visit(const MemRingOutInstr& instr) override { - m_comparer = InstrComparer(&instr); - } - - void visit(const EmitVertexInstr& instr) override { - m_comparer = InstrComparer(&instr); - } - - void visit(const GDSInstr& instr) override { - m_comparer = InstrComparer(&instr); - } - - void visit(const WriteTFInstr& instr) override { - m_comparer = InstrComparer(&instr); - } - - void visit(const LDSAtomicInstr& instr) override { - m_comparer = InstrComparer(&instr); - } - - void visit(const LDSReadInstr& instr) override { - m_comparer = InstrComparer(&instr); - } - - void visit(const RatInstr& instr) override { - m_comparer = InstrComparer(&instr); - } + void visit(const RatInstr& instr) override { m_comparer = InstrComparer(&instr); } InstrComparer m_comparer; }; - -bool Instr::equal_to(const Instr& lhs) const +bool +Instr::equal_to(const Instr& lhs) const { InstrCompareForward cmp; accept(cmp); @@ -558,7 +582,4 @@ bool Instr::equal_to(const Instr& lhs) const return cmp.m_comparer.result; } - - - -} // ns r600 +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_instr.h b/src/gallium/drivers/r600/sfn/sfn_instr.h index dee1744d51b..6314030bfa7 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr.h +++ b/src/gallium/drivers/r600/sfn/sfn_instr.h @@ -26,12 +26,13 @@ #pragma once -#include "sfn_virtualvalues.h" #include "sfn_alu_defines.h" #include "sfn_defines.h" -#include -#include +#include "sfn_virtualvalues.h" + #include +#include +#include namespace r600 { @@ -55,13 +56,13 @@ class LDSAtomicInstr; class LDSReadInstr; class RatInstr; - -int int_from_string_with_prefix(const std::string& str, const std::string& prefix); -int sel_and_szw_from_string(const std::string& str, RegisterVec4::Swizzle& swz, bool& is_ssa); +int +int_from_string_with_prefix(const std::string& str, const std::string& prefix); +int +sel_and_szw_from_string(const std::string& str, RegisterVec4::Swizzle& swz, bool& is_ssa); class Instr : public Allocate { public: - enum Flags { always_keep, dead, @@ -71,7 +72,7 @@ public: ack_rat_return_write, helper, nflags - }; + }; Instr(); @@ -86,52 +87,59 @@ public: virtual void accept(ConstInstrVisitor& visitor) const = 0; virtual void accept(InstrVisitor& visitor) = 0; - virtual bool end_group() const { return true;} + virtual bool end_group() const { return true; } virtual bool is_last() const; - void set_always_keep() {m_instr_flags.set(always_keep);} + void set_always_keep() { m_instr_flags.set(always_keep); } bool set_dead(); - virtual void set_scheduled() { m_instr_flags.set(scheduled); forward_set_scheduled();} - void add_use() {++m_use_count;} - void dec_use() {assert(m_use_count > 0); --m_use_count;} - bool is_dead() const {return m_instr_flags.test(dead);} - bool is_scheduled() const {return m_instr_flags.test(scheduled);} - bool keep() const {return m_instr_flags.test(always_keep);} - bool has_uses() const {return m_use_count > 0;} + virtual void set_scheduled() + { + m_instr_flags.set(scheduled); + forward_set_scheduled(); + } + void add_use() { ++m_use_count; } + void dec_use() + { + assert(m_use_count > 0); + --m_use_count; + } + bool is_dead() const { return m_instr_flags.test(dead); } + bool is_scheduled() const { return m_instr_flags.test(scheduled); } + bool keep() const { return m_instr_flags.test(always_keep); } + bool has_uses() const { return m_use_count > 0; } - bool has_instr_flag(Flags f) const {return m_instr_flags.test(f);} - void set_instr_flag(Flags f) { m_instr_flags.set(f);} + bool has_instr_flag(Flags f) const { return m_instr_flags.test(f); } + void set_instr_flag(Flags f) { m_instr_flags.set(f); } virtual bool replace_source(PRegister old_src, PVirtualValue new_src); virtual bool replace_dest(PRegister new_dest, AluInstr *move_instr); - virtual int nesting_corr() const { return 0;} + virtual int nesting_corr() const { return 0; } - virtual bool end_block() const { return false;} - virtual int nesting_offset() const { return 0;} + virtual bool end_block() const { return false; } + virtual int nesting_offset() const { return 0; } void set_blockid(int id, int index); - int block_id() const {return m_block_id;} - int index() const { return m_index;} + int block_id() const { return m_block_id; } + int index() const { return m_index; } void add_required_instr(Instr *instr); void replace_required_instr(Instr *old_instr, Instr *new_instr); bool ready() const; - virtual uint32_t slots() const {return 0;}; + virtual uint32_t slots() const { return 0; }; using InstrList = std::list>; - const InstrList& dependend_instr() { return m_dependend_instr;} + const InstrList& dependend_instr() { return m_dependend_instr; } - virtual AluInstr *as_alu() { return nullptr;} + virtual AluInstr *as_alu() { return nullptr; } virtual uint8_t allowed_dest_chan_mask() const { return 0; } protected: - - const InstrList& required_instr() const {return m_required_instr; } + const InstrList& required_instr() const { return m_required_instr; } private: virtual void forward_set_blockid(int id, int index); @@ -149,13 +157,11 @@ private: int m_block_id; int m_index; std::bitset m_instr_flags{0}; - }; using PInst = Instr::Pointer; class Block : public Instr { public: - enum Type { cf, alu, @@ -180,10 +186,10 @@ public: reverse_iterator rbegin() { return m_instructions.rbegin(); } reverse_iterator rend() { return m_instructions.rend(); } - const_iterator begin() const { return m_instructions.begin();} - const_iterator end() const { return m_instructions.end();} + const_iterator begin() const { return m_instructions.begin(); } + const_iterator end() const { return m_instructions.end(); } - bool empty() const { return m_instructions.empty();} + bool empty() const { return m_instructions.empty(); } void erase(iterator node); @@ -192,29 +198,29 @@ public: void accept(ConstInstrVisitor& visitor) const override; void accept(InstrVisitor& visitor) override; - int nesting_depth() const { return m_nesting_depth;} + int nesting_depth() const { return m_nesting_depth; } - int id() const {return m_id;} + int id() const { return m_id; } - auto type() const {return m_blocK_type; } + auto type() const { return m_blocK_type; } void set_type(Type t); - uint32_t remaining_slots() const { return m_remaining_slots;} + uint32_t remaining_slots() const { return m_remaining_slots; } bool try_reserve_kcache(const AluGroup& instr); bool try_reserve_kcache(const AluInstr& group); - auto last_lds_instr() {return m_last_lds_instr;} - void set_last_lds_instr(Instr *instr) {m_last_lds_instr = instr;} + auto last_lds_instr() { return m_last_lds_instr; } + void set_last_lds_instr(Instr *instr) { m_last_lds_instr = instr; } void lds_group_start(AluInstr *alu); void lds_group_end(); - bool lds_group_active() { return m_lds_group_start != nullptr;} + bool lds_group_active() { return m_lds_group_start != nullptr; } - size_t size() const { return m_instructions.size();} + size_t size() const { return m_instructions.size(); } - bool kcache_reservation_failed() const { return m_kcache_alloc_failed;} + bool kcache_reservation_failed() const { return m_kcache_alloc_failed; } - int inc_rat_emitted() { return ++m_emitted_rat_instr;} + int inc_rat_emitted() { return ++m_emitted_rat_instr; } static void set_chipclass(r600_chip_class chip_class); @@ -222,7 +228,7 @@ private: bool try_reserve_kcache(const UniformValue& u, std::array& kcache) const; - bool do_ready() const override {return true;}; + bool do_ready() const override { return true; }; void do_print(std::ostream& os) const override; Instructions m_instructions; int m_nesting_depth; @@ -245,14 +251,16 @@ private: class InstrWithResource : public Instr { public: - InstrWithResource(int base, PRegister offset) : - m_base(base), m_offset(offset) + InstrWithResource(int base, PRegister offset): + m_base(base), + m_offset(offset) { if (m_offset) { m_offset->add_use(this); } } - bool replace_resource_offset(PRegister old_offset, PRegister new_offset) { + bool replace_resource_offset(PRegister old_offset, PRegister new_offset) + { if (m_offset && old_offset->equal_to(*m_offset)) { m_offset->del_use(this); m_offset = new_offset; @@ -261,7 +269,8 @@ public: } return false; } - void set_resource_offset(PRegister offset) { + void set_resource_offset(PRegister offset) + { if (m_offset) m_offset->del_use(this); m_offset = offset; @@ -270,7 +279,8 @@ public: } } - bool resource_is_equal(const InstrWithResource& other) const { + bool resource_is_equal(const InstrWithResource& other) const + { if (m_base != other.m_base) return false; if (m_offset && other.m_offset) @@ -278,35 +288,41 @@ public: return !m_offset && !other.m_offset; } - auto resource_base() const {return m_base;} + auto resource_base() const { return m_base; } - auto resource_offset() const {return m_offset;} + auto resource_offset() const { return m_offset; } - auto buffer_index_mode() const -> EBufferIndexMode { + auto buffer_index_mode() const -> EBufferIndexMode + { if (!m_offset) return bim_none; switch (m_offset->sel()) { - case 1: return bim_zero; - case 2: return bim_one; + case 1: + return bim_zero; + case 2: + return bim_one; default: unreachable("Invalid resource offset, scheduler must substitute registers"); } } - bool resource_ready(int block_id, int index) const { + bool resource_ready(int block_id, int index) const + { return !m_offset || m_offset->ready(block_id, index); } + protected: - void print_resource_offset(std::ostream& os) const { + void print_resource_offset(std::ostream& os) const + { if (m_offset) os << " + " << *m_offset; } + private: int m_base{0}; PRegister m_offset{nullptr}; }; - class InstrWithVectorResult : public InstrWithResource { public: InstrWithVectorResult(const RegisterVec4& dest, @@ -314,40 +330,48 @@ public: int resource_base, PRegister resource_offset); - void set_dest_swizzle(const RegisterVec4::Swizzle& swz) {m_dest_swizzle = swz;} - int dest_swizzle(int i) const { return m_dest_swizzle[i];} - const RegisterVec4::Swizzle& all_dest_swizzle() const { return m_dest_swizzle;} - const RegisterVec4& dst() const {return m_dest;} + void set_dest_swizzle(const RegisterVec4::Swizzle& swz) { m_dest_swizzle = swz; } + int dest_swizzle(int i) const { return m_dest_swizzle[i]; } + const RegisterVec4::Swizzle& all_dest_swizzle() const { return m_dest_swizzle; } + const RegisterVec4& dst() const { return m_dest; } protected: InstrWithVectorResult(const InstrWithVectorResult& orig); void print_dest(std::ostream& os) const; - bool comp_dest(const RegisterVec4& dest, const RegisterVec4::Swizzle& dest_swizzle) const; + bool comp_dest(const RegisterVec4& dest, + const RegisterVec4::Swizzle& dest_swizzle) const; private: RegisterVec4 m_dest; RegisterVec4::Swizzle m_dest_swizzle; }; -inline bool operator == (const Instr& lhs, const Instr& rhs) { +inline bool +operator==(const Instr& lhs, const Instr& rhs) +{ return lhs.equal_to(rhs); } -inline bool operator != (const Instr& lhs, const Instr& rhs) { +inline bool +operator!=(const Instr& lhs, const Instr& rhs) +{ return !(lhs == rhs); } -inline std::ostream& operator << (std::ostream& os, const Instr& instr) +inline std::ostream& +operator<<(std::ostream& os, const Instr& instr) { instr.print(os); return os; } template >> -std::ostream& operator<<(std::ostream& os, const T& instr) { - instr.print(os); - return os; +std::ostream& +operator<<(std::ostream& os, const T& instr) +{ + instr.print(os); + return os; } class ConstInstrVisitor { @@ -373,7 +397,7 @@ public: class InstrVisitor { public: - virtual void visit(AluInstr *instr) = 0; + virtual void visit(AluInstr *instr) = 0; virtual void visit(AluGroup *instr) = 0; virtual void visit(TexInstr *instr) = 0; virtual void visit(ExportInstr *instr) = 0; @@ -392,5 +416,4 @@ public: virtual void visit(RatInstr *instr) = 0; }; - -} // ns r600 +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_alu.cpp b/src/gallium/drivers/r600/sfn/sfn_instr_alu.cpp index 74d0a3e9961..997348f24ec 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_alu.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_instr_alu.cpp @@ -25,84 +25,93 @@ */ #include "sfn_instr_alu.h" + +#include "sfn_debug.h" #include "sfn_instr_alugroup.h" #include "sfn_instr_tex.h" #include "sfn_shader.h" -#include "sfn_debug.h" #include #include - namespace r600 { -using std::string; using std::istream; +using std::string; using std::vector; -AluInstr::AluInstr(EAluOp opcode, PRegister dest, +AluInstr::AluInstr(EAluOp opcode, + PRegister dest, SrcValues src, - const std::set& flags, int slots): - m_opcode(opcode), - m_dest(dest), - m_bank_swizzle(alu_vec_unknown), - m_cf_type(cf_alu), - m_alu_slots(slots) + const std::set& flags, + int slots): + m_opcode(opcode), + m_dest(dest), + m_bank_swizzle(alu_vec_unknown), + m_cf_type(cf_alu), + m_alu_slots(slots) { m_src.swap(src); if (m_src.size() == 3) m_alu_flags.set(alu_op3); - for(auto f : flags) + for (auto f : flags) m_alu_flags.set(f); - ASSERT_OR_THROW(m_src.size() == static_cast(alu_ops.at(opcode).nsrc * m_alu_slots), + ASSERT_OR_THROW(m_src.size() == + static_cast(alu_ops.at(opcode).nsrc * m_alu_slots), "Unexpected number of source values"); if (m_alu_flags.test(alu_write)) ASSERT_OR_THROW(dest, "Write flag is set, but no destination register is given"); update_uses(); - } AluInstr::AluInstr(EAluOp opcode): - AluInstr(opcode, nullptr, SrcValues(alu_ops.at(opcode).nsrc), {}, 1) + AluInstr(opcode, nullptr, SrcValues(alu_ops.at(opcode).nsrc), {}, 1) { } AluInstr::AluInstr(EAluOp opcode, int chan): - AluInstr(opcode, nullptr, SrcValues(), {}, 1) + AluInstr(opcode, nullptr, SrcValues(), {}, 1) { m_fallback_chan = chan; } -AluInstr::AluInstr(EAluOp opcode, PRegister dest, PVirtualValue src0, +AluInstr::AluInstr(EAluOp opcode, + PRegister dest, + PVirtualValue src0, const std::set& m_flags): - AluInstr(opcode, dest, SrcValues{src0}, m_flags, 1) + AluInstr(opcode, dest, SrcValues{src0}, m_flags, 1) { - } -AluInstr::AluInstr(EAluOp opcode, PRegister dest, - PVirtualValue src0, PVirtualValue src1, +AluInstr::AluInstr(EAluOp opcode, + PRegister dest, + PVirtualValue src0, + PVirtualValue src1, const std::set& m_flags): - AluInstr(opcode, dest, SrcValues{src0, src1}, m_flags, 1) + AluInstr(opcode, dest, SrcValues{src0, src1}, m_flags, 1) { - } -AluInstr::AluInstr(EAluOp opcode, PRegister dest, PVirtualValue src0, PVirtualValue src1, +AluInstr::AluInstr(EAluOp opcode, + PRegister dest, + PVirtualValue src0, + PVirtualValue src1, PVirtualValue src2, const std::set& m_flags): - AluInstr(opcode, dest, SrcValues{src0, src1, src2}, m_flags, 1) + AluInstr(opcode, dest, SrcValues{src0, src1, src2}, m_flags, 1) { - } -AluInstr::AluInstr(ESDOp op, PVirtualValue src0, PVirtualValue src1, PVirtualValue address): - m_lds_opcode(op) +AluInstr::AluInstr(ESDOp op, + PVirtualValue src0, + PVirtualValue src1, + PVirtualValue address): + m_lds_opcode(op) { set_alu_flag(alu_is_lds); @@ -116,17 +125,18 @@ AluInstr::AluInstr(ESDOp op, PVirtualValue src0, PVirtualValue src1, PVirtualVal } AluInstr::AluInstr(ESDOp op, const SrcValues& src, const std::set& flags): - m_lds_opcode(op), - m_src(src) + m_lds_opcode(op), + m_src(src) { - for(auto f : flags) + for (auto f : flags) set_alu_flag(f); set_alu_flag(alu_is_lds); update_uses(); } -void AluInstr::update_uses() +void +AluInstr::update_uses() { for (auto& s : m_src) { auto r = s->as_register(); @@ -158,23 +168,25 @@ void AluInstr::update_uses() } } -void AluInstr::accept(ConstInstrVisitor& visitor) const +void +AluInstr::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void AluInstr::accept(InstrVisitor& visitor) +void +AluInstr::accept(InstrVisitor& visitor) { visitor.visit(this); } const std::map AluInstr::cf_map = { - {cf_alu_break, "BREAK"}, - {cf_alu_continue, "CONT"}, - {cf_alu_else_after, "ELSE_AFTER"}, - {cf_alu_extended, "EXTENDED"}, - {cf_alu_pop_after, "POP_AFTER"}, - {cf_alu_pop2_after, "POP2_AFTER"}, + {cf_alu_break, "BREAK" }, + {cf_alu_continue, "CONT" }, + {cf_alu_else_after, "ELSE_AFTER" }, + {cf_alu_extended, "EXTENDED" }, + {cf_alu_pop_after, "POP_AFTER" }, + {cf_alu_pop2_after, "POP2_AFTER" }, {cf_alu_push_before, "PUSH_BEFORE"} }; @@ -187,16 +199,16 @@ const std::map AluInstr::bank_swizzle_map = { {alu_vec_210, "VEC_210"} }; -const AluModifiers AluInstr::src_abs_flags[2] = -{alu_src0_abs, alu_src1_abs}; -const AluModifiers AluInstr::src_neg_flags[3] = -{alu_src0_neg, alu_src1_neg, alu_src2_neg}; -const AluModifiers AluInstr::src_rel_flags[3] = -{alu_src0_rel, alu_src1_rel, alu_src2_rel}; +const AluModifiers AluInstr::src_abs_flags[2] = {alu_src0_abs, alu_src1_abs}; +const AluModifiers AluInstr::src_neg_flags[3] = { + alu_src0_neg, alu_src1_neg, alu_src2_neg}; +const AluModifiers AluInstr::src_rel_flags[3] = { + alu_src0_rel, alu_src1_rel, alu_src2_rel}; struct ValuePrintFlags { - ValuePrintFlags(int im, int f):index_mode(im), - flags(f) + ValuePrintFlags(int im, int f): + index_mode(im), + flags(f) { } int index_mode = 0; @@ -209,7 +221,8 @@ struct ValuePrintFlags { static const int index_loopidx = 32; }; -void AluInstr::do_print(std::ostream& os) const +void +AluInstr::do_print(std::ostream& os) const { const char swzchar[] = "xyzw01?_"; @@ -230,7 +243,8 @@ void AluInstr::do_print(std::ostream& os) const if (has_alu_flag(alu_write)) os << " " << *m_dest; else - os << " __" << "." << swzchar[m_dest->chan()]; + os << " __" + << "." << swzchar[m_dest->chan()]; if (!has_alu_flag(alu_write) && m_dest->pin() != pin_none) os << "@" << m_dest->pin(); os << " : "; @@ -239,8 +253,8 @@ void AluInstr::do_print(std::ostream& os) const } } - const int n_source_per_slot = has_alu_flag(alu_is_lds) ? - m_src.size() : alu_ops.at(m_opcode).nsrc; + const int n_source_per_slot = + has_alu_flag(alu_is_lds) ? m_src.size() : alu_ops.at(m_opcode).nsrc; for (int s = 0; s < m_alu_slots; ++s) { @@ -251,36 +265,47 @@ void AluInstr::do_print(std::ostream& os) const int pflags = 0; if (i) os << ' '; - if (has_alu_flag(src_neg_flags[k])) pflags |= ValuePrintFlags::has_neg; - if (has_alu_flag(src_rel_flags[k])) pflags |= ValuePrintFlags::is_rel; + if (has_alu_flag(src_neg_flags[k])) + pflags |= ValuePrintFlags::has_neg; + if (has_alu_flag(src_rel_flags[k])) + pflags |= ValuePrintFlags::is_rel; if (i < 2) - if (has_alu_flag(src_abs_flags[k])) pflags |= ValuePrintFlags::has_abs; + if (has_alu_flag(src_abs_flags[k])) + pflags |= ValuePrintFlags::has_abs; - if (pflags & ValuePrintFlags::has_neg) os << '-'; - if (pflags & ValuePrintFlags::has_abs) os << '|'; + if (pflags & ValuePrintFlags::has_neg) + os << '-'; + if (pflags & ValuePrintFlags::has_abs) + os << '|'; os << *m_src[i]; - if (pflags & ValuePrintFlags::has_abs) os << '|'; + if (pflags & ValuePrintFlags::has_abs) + os << '|'; ++i; } } os << " {"; - if (has_alu_flag(alu_write)) os << 'W'; - if (has_alu_flag(alu_last_instr)) os << 'L'; - if (has_alu_flag(alu_update_exec)) os << 'E'; - if (has_alu_flag(alu_update_pred)) os << 'P'; + if (has_alu_flag(alu_write)) + os << 'W'; + if (has_alu_flag(alu_last_instr)) + os << 'L'; + if (has_alu_flag(alu_update_exec)) + os << 'E'; + if (has_alu_flag(alu_update_pred)) + os << 'P'; os << "}"; auto bs_name = bank_swizzle_map.find(m_bank_swizzle); if (bs_name != bank_swizzle_map.end()) - os << ' ' << bs_name->second; + os << ' ' << bs_name->second; auto cf_name = cf_map.find(m_cf_type); if (cf_name != cf_map.end()) - os << ' ' << cf_name->second; + os << ' ' << cf_name->second; } -bool AluInstr::can_propagate_src() const +bool +AluInstr::can_propagate_src() const { /* We can use the source in the next instruction */ if (!can_copy_propagate()) @@ -292,8 +317,6 @@ bool AluInstr::can_propagate_src() const assert(m_dest); - - if (!m_dest->is_ssa()) { return false; } @@ -303,15 +326,15 @@ bool AluInstr::can_propagate_src() const if (m_dest->pin() == pin_chan) return src_reg->pin() == pin_none || - (src_reg->pin() == pin_chan && - src_reg->chan() == m_dest->chan()); + (src_reg->pin() == pin_chan && src_reg->chan() == m_dest->chan()); return m_dest->pin() == pin_none || m_dest->pin() == pin_free; } -bool AluInstr::can_propagate_dest() const +bool +AluInstr::can_propagate_dest() const { - if (!can_copy_propagate()){ + if (!can_copy_propagate()) { return false; } @@ -330,30 +353,28 @@ bool AluInstr::can_propagate_dest() const return false; if (src_reg->pin() == pin_chan) - return m_dest->pin() == pin_none || - m_dest->pin() == pin_free || - ((m_dest->pin() == pin_chan || - m_dest->pin() == pin_group) && - src_reg->chan() == m_dest->chan()); + return m_dest->pin() == pin_none || m_dest->pin() == pin_free || + ((m_dest->pin() == pin_chan || m_dest->pin() == pin_group) && + src_reg->chan() == m_dest->chan()); - return (src_reg->pin() == pin_none || - src_reg->pin() == pin_free); + return (src_reg->pin() == pin_none || src_reg->pin() == pin_free); } -bool AluInstr::can_copy_propagate() const +bool +AluInstr::can_copy_propagate() const { if (m_opcode != op1_mov) return false; - if (has_alu_flag(alu_src0_abs) || - has_alu_flag(alu_src0_neg) || + if (has_alu_flag(alu_src0_abs) || has_alu_flag(alu_src0_neg) || has_alu_flag(alu_dst_clamp)) return false; return has_alu_flag(alu_write); } -bool AluInstr::replace_source(PRegister old_src, PVirtualValue new_src) +bool +AluInstr::replace_source(PRegister old_src, PVirtualValue new_src) { bool process = false; @@ -376,11 +397,12 @@ bool AluInstr::replace_source(PRegister old_src, PVirtualValue new_src) } if (m_dest) { - /* We don't allow src and dst with rel and different indirect register addresses */ + /* We don't allow src and dst with rel and different indirect register + * addresses */ if (m_dest->pin() == pin_array && new_src->pin() == pin_array) { auto dav = static_cast(m_dest)->addr(); auto sav = static_cast(new_src)->addr(); - if (dav && sav && dav->as_register() && !dav->equal_to(*sav)) + if (dav && sav && dav->as_register() && !dav->equal_to(*sav)) return false; } } @@ -388,9 +410,7 @@ bool AluInstr::replace_source(PRegister old_src, PVirtualValue new_src) /* Check the readports */ if (m_alu_slots * alu_ops.at(m_opcode).nsrc > 2 || m_parent_group) { AluReadportReservation read_port_check = - !m_parent_group ? - AluReadportReservation() : - m_parent_group->readport_reserer(); + !m_parent_group ? AluReadportReservation() : m_parent_group->readport_reserer(); int nsrc = alu_ops.at(m_opcode).nsrc; PVirtualValue src[3]; @@ -402,7 +422,7 @@ bool AluInstr::replace_source(PRegister old_src, PVirtualValue new_src) } AluBankSwizzle bs = alu_vec_012; while (bs != alu_vec_unknown) { - if (read_port_check.schedule_vec_src(src, nsrc, bs)) { + if (read_port_check.schedule_vec_src(src, nsrc, bs)) { break; } ++bs; @@ -429,7 +449,8 @@ bool AluInstr::replace_source(PRegister old_src, PVirtualValue new_src) return process; } -void AluInstr::set_sources(SrcValues src) +void +AluInstr::set_sources(SrcValues src) { for (auto& s : m_src) { auto r = s->as_register(); @@ -444,7 +465,8 @@ void AluInstr::set_sources(SrcValues src) } } -uint8_t AluInstr::allowed_dest_chan_mask() const +uint8_t +AluInstr::allowed_dest_chan_mask() const { if (alu_slots() != 1) { if (has_alu_flag(alu_is_cayman_trans)) { @@ -456,7 +478,8 @@ uint8_t AluInstr::allowed_dest_chan_mask() const return 0xf; } -bool AluInstr::replace_dest(PRegister new_dest, AluInstr *move_instr) +bool +AluInstr::replace_dest(PRegister new_dest, AluInstr *move_instr) { if (m_dest->equal_to(*new_dest)) return false; @@ -484,11 +507,9 @@ bool AluInstr::replace_dest(PRegister new_dest, AluInstr *move_instr) } */ - if (m_dest->pin() == pin_chan && - new_dest->chan() != m_dest->chan()) + if (m_dest->pin() == pin_chan && new_dest->chan() != m_dest->chan()) return false; - if (m_dest->pin() == pin_chan) { if (new_dest->pin() == pin_group) new_dest->set_pin(pin_chgr); @@ -513,20 +534,22 @@ bool AluInstr::replace_dest(PRegister new_dest, AluInstr *move_instr) return true; } -void AluInstr::pin_sources_to_chan() +void +AluInstr::pin_sources_to_chan() { - for (auto s: m_src) { + for (auto s : m_src) { auto r = s->as_register(); if (r) { if (r->pin() == pin_free) r->set_pin(pin_chan); else if (r->pin() == pin_group) - r->set_pin(pin_chgr); + r->set_pin(pin_chgr); } } } -bool AluInstr::check_readport_validation(PRegister old_src, PVirtualValue new_src) const +bool +AluInstr::check_readport_validation(PRegister old_src, PVirtualValue new_src) const { bool success = true; AluReadportReservation rpr_sum; @@ -537,7 +560,6 @@ bool AluInstr::check_readport_validation(PRegister old_src, PVirtualValue new_sr unsigned nsrc = alu_ops.at(m_opcode).nsrc; assert(nsrc * m_alu_slots == m_src.size()); - for (int s = 0; s < m_alu_slots && success; ++s) { for (AluBankSwizzle i = alu_vec_012; i != alu_vec_unknown; ++i) { auto ireg = m_src.begin() + s * nsrc; @@ -559,20 +581,19 @@ bool AluInstr::check_readport_validation(PRegister old_src, PVirtualValue new_sr return success; } -void AluInstr::add_extra_dependency(PVirtualValue value) +void +AluInstr::add_extra_dependency(PVirtualValue value) { auto reg = value->as_register(); if (reg) m_extra_dependencies.insert(reg); } - -bool AluInstr::is_equal_to(const AluInstr& lhs) const +bool +AluInstr::is_equal_to(const AluInstr& lhs) const { - if (lhs.m_opcode != m_opcode || - lhs.m_bank_swizzle != m_bank_swizzle || - lhs.m_cf_type != m_cf_type || - lhs.m_alu_flags != m_alu_flags) { + if (lhs.m_opcode != m_opcode || lhs.m_bank_swizzle != m_bank_swizzle || + lhs.m_cf_type != m_cf_type || lhs.m_alu_flags != m_alu_flags) { return false; } @@ -604,27 +625,33 @@ bool AluInstr::is_equal_to(const AluInstr& lhs) const return true; } -class ResolveIndirectArrayAddr: public ConstRegisterVisitor { +class ResolveIndirectArrayAddr : public ConstRegisterVisitor { public: - void visit(const Register& value){(void) value;} - void visit(const LocalArray& value) {(void)value; unreachable("An array can't be used as address");} + void visit(const Register& value) { (void)value; } + void visit(const LocalArray& value) + { + (void)value; + unreachable("An array can't be used as address"); + } void visit(const LocalArrayValue& value); void visit(const UniformValue& value); - void visit(const LiteralConstant& value) {(void)value;} - void visit(const InlineConstant& value) {(void)value;} + void visit(const LiteralConstant& value) { (void)value; } + void visit(const InlineConstant& value) { (void)value; } PRegister addr{nullptr}; bool is_index{false}; }; -void ResolveIndirectArrayAddr::visit(const LocalArrayValue& value) +void +ResolveIndirectArrayAddr::visit(const LocalArrayValue& value) { auto a = value.addr(); if (a) addr = a->as_register(); } -void ResolveIndirectArrayAddr::visit(const UniformValue& value) +void +ResolveIndirectArrayAddr::visit(const UniformValue& value) { auto a = value.buf_addr(); if (a) { @@ -633,7 +660,8 @@ void ResolveIndirectArrayAddr::visit(const UniformValue& value) } } -std::tuple AluInstr::indirect_addr() const +std::tuple +AluInstr::indirect_addr() const { ResolveIndirectArrayAddr visitor; @@ -643,7 +671,7 @@ std::tuple AluInstr::indirect_addr() const return {visitor.addr, false, false}; } - for (auto s: m_src) { + for (auto s : m_src) { s->accept(visitor); if (visitor.addr) { return {visitor.addr, !visitor.is_index, visitor.is_index}; @@ -652,7 +680,8 @@ std::tuple AluInstr::indirect_addr() const return {nullptr, false, false}; } -AluGroup *AluInstr::split(ValueFactory& vf) +AluGroup * +AluInstr::split(ValueFactory& vf) { if (m_alu_slots == 1) return nullptr; @@ -714,8 +743,7 @@ AluGroup *AluInstr::split(ValueFactory& vf) sfn_log << SfnLog::instr << " " << *instr << "\n"; if (!group->add_instruction(instr)) { - std::cerr << "Unable to schedule '" << *instr << "' into\n" - << *group << "\n"; + std::cerr << "Unable to schedule '" << *instr << "' into\n" << *group << "\n"; unreachable("Invalid group instruction"); } @@ -732,18 +760,19 @@ AluGroup *AluInstr::split(ValueFactory& vf) return group; } -/* Alu instructions that have SSA dest registers increase the regietsr pressure - * Alu instructions that read from SSA registers may decresase the register pressure - * hency evaluate a priorityx values based on register pressure change */ -int AluInstr::register_priority() const +/* Alu instructions that have SSA dest registers increase the regietsr + * pressure Alu instructions that read from SSA registers may decresase the + * register pressure hency evaluate a priorityx values based on register + * pressure change */ +int +AluInstr::register_priority() const { int priority = 0; if (!has_alu_flag(alu_no_schedule_bias)) { if (m_dest) { if (m_dest->is_ssa() && has_alu_flag(alu_write)) { - if (m_dest->pin() != pin_group && - m_dest->pin() != pin_chgr) + if (m_dest->pin() != pin_group && m_dest->pin() != pin_chgr) priority--; } else { // Arrays and registers are pre-allocated, hence scheduling @@ -770,13 +799,13 @@ int AluInstr::register_priority() const return priority; } -bool AluInstr::propagate_death() +bool +AluInstr::propagate_death() { if (!m_dest) return true; - if (m_dest->pin() == pin_group || - m_dest->pin() == pin_chan) { + if (m_dest->pin() == pin_group || m_dest->pin() == pin_chan) { switch (m_opcode) { case op2_interp_x: case op2_interp_xy: @@ -784,12 +813,11 @@ bool AluInstr::propagate_death() case op2_interp_zw: reset_alu_flag(alu_write); return false; - default: - ; + default:; } } - if (m_dest->pin() == pin_array) + if (m_dest->pin() == pin_array) return false; /* We assume that nir does a good job in eliminating all ALU results that @@ -798,7 +826,7 @@ bool AluInstr::propagate_death() if (has_alu_flag(alu_is_cayman_trans)) return false; - for (auto& src: m_src) { + for (auto& src : m_src) { auto reg = src->as_register(); if (reg) reg->del_use(this); @@ -806,20 +834,21 @@ bool AluInstr::propagate_death() return true; } -bool AluInstr::has_lds_access() const +bool +AluInstr::has_lds_access() const { return has_alu_flag(alu_is_lds) || has_lds_queue_read(); } -bool AluInstr::has_lds_queue_read() const +bool +AluInstr::has_lds_queue_read() const { for (auto& s : m_src) { auto ic = s->as_inline_const(); if (!ic) continue; - if (ic->sel() == ALU_SRC_LDS_OQ_A_POP || - ic->sel() == ALU_SRC_LDS_OQ_B_POP) + if (ic->sel() == ALU_SRC_LDS_OQ_A_POP || ic->sel() == ALU_SRC_LDS_OQ_B_POP) return true; } return false; @@ -836,7 +865,8 @@ struct OpDescr { static std::map s_alu_map_by_name; static std::map s_lds_map_by_name; -Instr::Pointer AluInstr::from_string(istream& is, ValueFactory& value_factory, AluGroup *group) +Instr::Pointer +AluInstr::from_string(istream& is, ValueFactory& value_factory, AluGroup *group) { vector tokens; @@ -872,7 +902,7 @@ Instr::Pointer AluInstr::from_string(istream& is, ValueFactory& value_factory, A if (is_lds) { auto op = s_lds_map_by_name.find(opstr); if (op == s_lds_map_by_name.end()) { - for(auto [opcode, opdescr] : lds_ops ) { + for (auto [opcode, opdescr] : lds_ops) { if (opstr == opdescr.name) { op_descr.lds_opcode = opcode; op_descr.nsrc = opdescr.nsrc; @@ -892,7 +922,7 @@ Instr::Pointer AluInstr::from_string(istream& is, ValueFactory& value_factory, A } else { auto op = s_alu_map_by_name.find(opstr); if (op == s_alu_map_by_name.end()) { - for(auto [opcode, opdescr] : alu_ops ) { + for (auto [opcode, opdescr] : alu_ops) { if (opstr == opdescr.name) { op_descr = {{opcode}, opdescr.nsrc}; s_alu_map_by_name[opstr] = op_descr; @@ -958,15 +988,22 @@ Instr::Pointer AluInstr::from_string(istream& is, ValueFactory& value_factory, A break; switch (*iflag) { - case 'L': flags.insert(alu_last_instr); break; - case 'W': flags.insert(alu_write); break; - case 'E': flags.insert(alu_update_exec); break; - case 'P': flags.insert(alu_update_pred); break; + case 'L': + flags.insert(alu_last_instr); + break; + case 'W': + flags.insert(alu_write); + break; + case 'E': + flags.insert(alu_update_exec); + break; + case 'P': + flags.insert(alu_update_pred); + break; } ++iflag; } - } - break; + } break; case 'V': { string bs = *t; @@ -986,8 +1023,7 @@ Instr::Pointer AluInstr::from_string(istream& is, ValueFactory& value_factory, A std::cerr << "'" << bs << "': "; unreachable("Unknowe bankswizzle given"); } - } - break; + } break; default: { string cf_str = *t; @@ -1029,17 +1065,18 @@ Instr::Pointer AluInstr::from_string(istream& is, ValueFactory& value_factory, A retval->set_cf_type(cf); if (group) { group->add_instruction(retval); - retval= nullptr; + retval = nullptr; } return retval; } -bool AluInstr::do_ready() const +bool +AluInstr::do_ready() const { /* Alu instructions are shuffled by the scheduler, so * we have to make sure that required ops are already * scheduled before marking this one ready */ - for (auto i: required_instr()) { + for (auto i : required_instr()) { if (!i->is_scheduled()) return false; } @@ -1062,7 +1099,7 @@ bool AluInstr::do_ready() const auto av = static_cast(m_dest); auto addr = av->addr(); /* For true indiect dest access we have to make sure that all - * instructions that write the value before are schedukled */ + * instructions that write the value before are schedukled */ if (addr && (!addr->ready(block_id(), index()) || !m_dest->ready(block_id(), index() - 1))) return false; @@ -1072,8 +1109,7 @@ bool AluInstr::do_ready() const * update are scheduled, otherwise we may use the updated value when we * shouldn't */ for (auto u : m_dest->uses()) { - if (u->block_id() <= block_id() && u->index() < index() && - !u->is_scheduled()) { + if (u->block_id() <= block_id() && u->index() < index() && !u->is_scheduled()) { return false; } } @@ -1087,7 +1123,8 @@ bool AluInstr::do_ready() const return true; } -void AluInstrVisitor::visit(AluGroup *instr) +void +AluInstrVisitor::visit(AluGroup *instr) { for (auto& i : *instr) { if (i) @@ -1095,100 +1132,163 @@ void AluInstrVisitor::visit(AluGroup *instr) } } -void AluInstrVisitor::visit(Block *instr) +void +AluInstrVisitor::visit(Block *instr) { for (auto& i : *instr) i->accept(*this); } -void AluInstrVisitor::visit(IfInstr *instr) +void +AluInstrVisitor::visit(IfInstr *instr) { instr->predicate()->accept(*this); } -static bool emit_alu_b2x(const nir_alu_instr& alu, AluInlineConstants mask, Shader& shader); +static bool +emit_alu_b2x(const nir_alu_instr& alu, AluInlineConstants mask, Shader& shader); -static bool emit_alu_op1(const nir_alu_instr& alu, EAluOp opcode, Shader& shader, - const AluOpFlags& flags = 0); -static bool emit_alu_op1_64bit(const nir_alu_instr& alu, EAluOp opcode, Shader& shader, bool switch_chan); -static bool emit_alu_mov_64bit(const nir_alu_instr& alu, Shader& shader); -static bool emit_alu_neg(const nir_alu_instr& alu, Shader& shader); -static bool emit_alu_op1_64bit_trans(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); -static bool emit_alu_op2_64bit(const nir_alu_instr& alu, EAluOp opcode, Shader& shader, bool switch_order); -static bool emit_alu_op2_64bit_one_dst(const nir_alu_instr& alu, EAluOp opcode, Shader& shader, bool switch_order); -static bool emit_alu_fma_64bit(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); -static bool emit_alu_b2f64(const nir_alu_instr& alu, Shader& shader); -static bool emit_alu_f2f64(const nir_alu_instr& alu, Shader& shader); -static bool emit_alu_i2f64(const nir_alu_instr& alu, EAluOp op, Shader& shader); -static bool emit_alu_f2f32(const nir_alu_instr& alu, Shader& shader); -static bool emit_alu_abs64(const nir_alu_instr& alu, Shader& shader); +static bool +emit_alu_op1(const nir_alu_instr& alu, + EAluOp opcode, + Shader& shader, + const AluOpFlags& flags = 0); +static bool +emit_alu_op1_64bit(const nir_alu_instr& alu, + EAluOp opcode, + Shader& shader, + bool switch_chan); +static bool +emit_alu_mov_64bit(const nir_alu_instr& alu, Shader& shader); +static bool +emit_alu_neg(const nir_alu_instr& alu, Shader& shader); +static bool +emit_alu_op1_64bit_trans(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); +static bool +emit_alu_op2_64bit(const nir_alu_instr& alu, + EAluOp opcode, + Shader& shader, + bool switch_order); +static bool +emit_alu_op2_64bit_one_dst(const nir_alu_instr& alu, + EAluOp opcode, + Shader& shader, + bool switch_order); +static bool +emit_alu_fma_64bit(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); +static bool +emit_alu_b2f64(const nir_alu_instr& alu, Shader& shader); +static bool +emit_alu_f2f64(const nir_alu_instr& alu, Shader& shader); +static bool +emit_alu_i2f64(const nir_alu_instr& alu, EAluOp op, Shader& shader); +static bool +emit_alu_f2f32(const nir_alu_instr& alu, Shader& shader); +static bool +emit_alu_abs64(const nir_alu_instr& alu, Shader& shader); -static bool emit_alu_op2(const nir_alu_instr& alu, EAluOp opcode, Shader& shader, - AluInstr::Op2Options opts = AluInstr::op2_opt_none); -static bool emit_alu_op2_int(const nir_alu_instr& alu, EAluOp opcode, Shader& shader, - AluInstr::Op2Options opts = AluInstr::op2_opt_none); -static bool emit_alu_op3(const nir_alu_instr& alu, EAluOp opcode, Shader& shader, - const std::array& src_shuffle = {0,1,2}); -static bool emit_any_all_fcomp2(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); -static bool emit_any_all_fcomp(const nir_alu_instr& alu, EAluOp opcode, int nc, bool all, Shader& shader); -static bool emit_any_all_icomp(const nir_alu_instr& alu, EAluOp opcode, int nc, bool all, Shader& shader); +static bool +emit_alu_op2(const nir_alu_instr& alu, + EAluOp opcode, + Shader& shader, + AluInstr::Op2Options opts = AluInstr::op2_opt_none); +static bool +emit_alu_op2_int(const nir_alu_instr& alu, + EAluOp opcode, + Shader& shader, + AluInstr::Op2Options opts = AluInstr::op2_opt_none); +static bool +emit_alu_op3(const nir_alu_instr& alu, + EAluOp opcode, + Shader& shader, + const std::array& src_shuffle = {0, 1, 2}); +static bool +emit_any_all_fcomp2(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); +static bool +emit_any_all_fcomp( + const nir_alu_instr& alu, EAluOp opcode, int nc, bool all, Shader& shader); +static bool +emit_any_all_icomp( + const nir_alu_instr& alu, EAluOp opcode, int nc, bool all, Shader& shader); -static bool emit_alu_i2orf2_b1(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); +static bool +emit_alu_i2orf2_b1(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); -static bool emit_alu_comb_with_zero(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); -static bool emit_unpack_64_2x32_split(const nir_alu_instr& alu, int comp, Shader& shader); -static bool emit_pack_64_2x32(const nir_alu_instr& alu, Shader& shader); -static bool emit_unpack_64_2x32(const nir_alu_instr& alu, Shader& shader); -static bool emit_pack_64_2x32_split(const nir_alu_instr& alu, Shader& shader); -static bool emit_pack_32_2x16_split(const nir_alu_instr& alu, Shader& shader); -static bool emit_alu_vec2_64(const nir_alu_instr& alu, Shader& shader); +static bool +emit_alu_comb_with_zero(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); +static bool +emit_unpack_64_2x32_split(const nir_alu_instr& alu, int comp, Shader& shader); +static bool +emit_pack_64_2x32(const nir_alu_instr& alu, Shader& shader); +static bool +emit_unpack_64_2x32(const nir_alu_instr& alu, Shader& shader); +static bool +emit_pack_64_2x32_split(const nir_alu_instr& alu, Shader& shader); +static bool +emit_pack_32_2x16_split(const nir_alu_instr& alu, Shader& shader); +static bool +emit_alu_vec2_64(const nir_alu_instr& alu, Shader& shader); -static bool emit_unpack_32_2x16_split_x(const nir_alu_instr& alu, Shader& shader); -static bool emit_unpack_32_2x16_split_y(const nir_alu_instr& alu, Shader& shader); +static bool +emit_unpack_32_2x16_split_x(const nir_alu_instr& alu, Shader& shader); +static bool +emit_unpack_32_2x16_split_y(const nir_alu_instr& alu, Shader& shader); -static bool emit_dot(const nir_alu_instr& alu, int nelm, Shader& shader); -static bool emit_create_vec(const nir_alu_instr& instr, unsigned nc, Shader& shader); +static bool +emit_dot(const nir_alu_instr& alu, int nelm, Shader& shader); +static bool +emit_create_vec(const nir_alu_instr& instr, unsigned nc, Shader& shader); -static bool emit_alu_trans_op1_eg(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); -static bool emit_alu_trans_op1_cayman(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); +static bool +emit_alu_trans_op1_eg(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); +static bool +emit_alu_trans_op1_cayman(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); -static bool emit_alu_trans_op2_eg(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); -static bool emit_alu_trans_op2_cayman(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); +static bool +emit_alu_trans_op2_eg(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); +static bool +emit_alu_trans_op2_cayman(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); -static bool emit_alu_f2i32_or_u32_eg(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); +static bool +emit_alu_f2i32_or_u32_eg(const nir_alu_instr& alu, EAluOp opcode, Shader& shader); -static bool emit_tex_fdd(const nir_alu_instr& alu, TexInstr::Opcode opcode, bool fine, Shader& shader); +static bool +emit_tex_fdd(const nir_alu_instr& alu, TexInstr::Opcode opcode, bool fine, Shader& shader); -static bool emit_alu_cube(const nir_alu_instr& alu, Shader& shader); +static bool +emit_alu_cube(const nir_alu_instr& alu, Shader& shader); -static bool emit_fdph(const nir_alu_instr& alu, Shader& shader); +static bool +emit_fdph(const nir_alu_instr& alu, Shader& shader); -static bool check_64_bit_op_src(nir_src *src, void *state) +static bool +check_64_bit_op_src(nir_src *src, void *state) { if (nir_src_bit_size(*src) == 64) { - *(bool*)state = true; + *(bool *)state = true; return false; } return true; } -static bool check_64_bit_op_dest(nir_dest *dest, void *state) +static bool +check_64_bit_op_dest(nir_dest *dest, void *state) { if (nir_dest_bit_size(*dest) == 64) { - *(bool*)state = true; + *(bool *)state = true; return false; } return true; } -bool AluInstr::from_nir(nir_alu_instr *alu, Shader& shader) +bool +AluInstr::from_nir(nir_alu_instr *alu, Shader& shader) { bool is_64bit_op = false; nir_foreach_src(&alu->instr, check_64_bit_op_src, &is_64bit_op); if (!is_64bit_op) nir_foreach_dest(&alu->instr, check_64_bit_op_dest, &is_64bit_op); - if (is_64bit_op) { switch (alu->op) { case nir_op_pack_64_2x32: @@ -1196,255 +1296,421 @@ bool AluInstr::from_nir(nir_alu_instr *alu, Shader& shader) case nir_op_pack_64_2x32_split: case nir_op_pack_half_2x16_split: case nir_op_unpack_64_2x32_split_x: - case nir_op_unpack_64_2x32_split_y: break; - case nir_op_mov: return emit_alu_mov_64bit(*alu, shader); - case nir_op_fneg: return emit_alu_neg(*alu, shader); - case nir_op_ffract: return emit_alu_op1_64bit(*alu, op1_fract_64, shader, true); - case nir_op_feq32: return emit_alu_op2_64bit_one_dst(*alu, op2_sete_64, shader, false); - case nir_op_fge32: return emit_alu_op2_64bit_one_dst(*alu, op2_setge_64, shader, false); - case nir_op_flt32: return emit_alu_op2_64bit_one_dst(*alu, op2_setgt_64, shader, true); - case nir_op_fneu32: return emit_alu_op2_64bit_one_dst(*alu, op2_setne_64, shader, false); - case nir_op_ffma: return emit_alu_fma_64bit(*alu, op3_fma_64, shader); + case nir_op_unpack_64_2x32_split_y: + break; + case nir_op_mov: + return emit_alu_mov_64bit(*alu, shader); + case nir_op_fneg: + return emit_alu_neg(*alu, shader); + case nir_op_ffract: + return emit_alu_op1_64bit(*alu, op1_fract_64, shader, true); + case nir_op_feq32: + return emit_alu_op2_64bit_one_dst(*alu, op2_sete_64, shader, false); + case nir_op_fge32: + return emit_alu_op2_64bit_one_dst(*alu, op2_setge_64, shader, false); + case nir_op_flt32: + return emit_alu_op2_64bit_one_dst(*alu, op2_setgt_64, shader, true); + case nir_op_fneu32: + return emit_alu_op2_64bit_one_dst(*alu, op2_setne_64, shader, false); + case nir_op_ffma: + return emit_alu_fma_64bit(*alu, op3_fma_64, shader); - case nir_op_fadd: return emit_alu_op2_64bit(*alu, op2_add_64, shader, false); - case nir_op_fmul: return emit_alu_op2_64bit(*alu, op2_mul_64, shader, false); - case nir_op_fmax: return emit_alu_op2_64bit(*alu, op2_max_64, shader, false); - case nir_op_fmin: return emit_alu_op2_64bit(*alu, op2_min_64, shader, false); - case nir_op_b2f64: return emit_alu_b2f64(*alu, shader); - case nir_op_f2f64: return emit_alu_f2f64(*alu, shader); - case nir_op_i2f64: return emit_alu_i2f64(*alu, op1_int_to_flt, shader); - case nir_op_u2f64: return emit_alu_i2f64(*alu, op1_uint_to_flt, shader); - case nir_op_f2f32: return emit_alu_f2f32(*alu, shader); - case nir_op_fabs: return emit_alu_abs64(*alu, shader); - case nir_op_fsqrt: return emit_alu_op1_64bit_trans(*alu, op1_sqrt_64, shader); - case nir_op_frcp: return emit_alu_op1_64bit_trans(*alu, op1_recip_64, shader); - case nir_op_frsq: return emit_alu_op1_64bit_trans(*alu, op1_recipsqrt_64, shader); - case nir_op_vec2: return emit_alu_vec2_64(*alu, shader); + case nir_op_fadd: + return emit_alu_op2_64bit(*alu, op2_add_64, shader, false); + case nir_op_fmul: + return emit_alu_op2_64bit(*alu, op2_mul_64, shader, false); + case nir_op_fmax: + return emit_alu_op2_64bit(*alu, op2_max_64, shader, false); + case nir_op_fmin: + return emit_alu_op2_64bit(*alu, op2_min_64, shader, false); + case nir_op_b2f64: + return emit_alu_b2f64(*alu, shader); + case nir_op_f2f64: + return emit_alu_f2f64(*alu, shader); + case nir_op_i2f64: + return emit_alu_i2f64(*alu, op1_int_to_flt, shader); + case nir_op_u2f64: + return emit_alu_i2f64(*alu, op1_uint_to_flt, shader); + case nir_op_f2f32: + return emit_alu_f2f32(*alu, shader); + case nir_op_fabs: + return emit_alu_abs64(*alu, shader); + case nir_op_fsqrt: + return emit_alu_op1_64bit_trans(*alu, op1_sqrt_64, shader); + case nir_op_frcp: + return emit_alu_op1_64bit_trans(*alu, op1_recip_64, shader); + case nir_op_frsq: + return emit_alu_op1_64bit_trans(*alu, op1_recipsqrt_64, shader); + case nir_op_vec2: + return emit_alu_vec2_64(*alu, shader); default: return false; ; } } - if (shader.chip_class() == ISA_CC_CAYMAN) { switch (alu->op) { - case nir_op_fcos_amd: return emit_alu_trans_op1_cayman(*alu, op1_cos, shader); - case nir_op_fexp2: return emit_alu_trans_op1_cayman(*alu, op1_exp_ieee, shader); - case nir_op_flog2: return emit_alu_trans_op1_cayman(*alu, op1_log_clamped, shader); - case nir_op_frcp: return emit_alu_trans_op1_cayman(*alu, op1_recip_ieee, shader); - case nir_op_frsq: return emit_alu_trans_op1_cayman(*alu, op1_recipsqrt_ieee1, shader); - case nir_op_fsqrt: return emit_alu_trans_op1_cayman(*alu, op1_sqrt_ieee, shader); - case nir_op_fsin_amd: return emit_alu_trans_op1_cayman(*alu, op1_sin, shader); - case nir_op_i2f32: return emit_alu_op1(*alu, op1_int_to_flt, shader); - case nir_op_u2f32: return emit_alu_op1(*alu, op1_uint_to_flt, shader); - case nir_op_imul: return emit_alu_trans_op2_cayman(*alu, op2_mullo_int, shader); - case nir_op_imul_high: return emit_alu_trans_op2_cayman(*alu, op2_mulhi_int, shader); - case nir_op_umul_high: return emit_alu_trans_op2_cayman(*alu, op2_mulhi_uint, shader); - case nir_op_f2u32: return emit_alu_op1(*alu, op1_flt_to_uint, shader); - case nir_op_f2i32: return emit_alu_op1(*alu, op1_flt_to_int, shader); - case nir_op_ishl: return emit_alu_op2_int(*alu, op2_lshl_int, shader); - case nir_op_ishr: return emit_alu_op2_int(*alu, op2_ashr_int, shader); - case nir_op_ushr: return emit_alu_op2_int(*alu, op2_lshr_int, shader); - default: - ; + case nir_op_fcos_amd: + return emit_alu_trans_op1_cayman(*alu, op1_cos, shader); + case nir_op_fexp2: + return emit_alu_trans_op1_cayman(*alu, op1_exp_ieee, shader); + case nir_op_flog2: + return emit_alu_trans_op1_cayman(*alu, op1_log_clamped, shader); + case nir_op_frcp: + return emit_alu_trans_op1_cayman(*alu, op1_recip_ieee, shader); + case nir_op_frsq: + return emit_alu_trans_op1_cayman(*alu, op1_recipsqrt_ieee1, shader); + case nir_op_fsqrt: + return emit_alu_trans_op1_cayman(*alu, op1_sqrt_ieee, shader); + case nir_op_fsin_amd: + return emit_alu_trans_op1_cayman(*alu, op1_sin, shader); + case nir_op_i2f32: + return emit_alu_op1(*alu, op1_int_to_flt, shader); + case nir_op_u2f32: + return emit_alu_op1(*alu, op1_uint_to_flt, shader); + case nir_op_imul: + return emit_alu_trans_op2_cayman(*alu, op2_mullo_int, shader); + case nir_op_imul_high: + return emit_alu_trans_op2_cayman(*alu, op2_mulhi_int, shader); + case nir_op_umul_high: + return emit_alu_trans_op2_cayman(*alu, op2_mulhi_uint, shader); + case nir_op_f2u32: + return emit_alu_op1(*alu, op1_flt_to_uint, shader); + case nir_op_f2i32: + return emit_alu_op1(*alu, op1_flt_to_int, shader); + case nir_op_ishl: + return emit_alu_op2_int(*alu, op2_lshl_int, shader); + case nir_op_ishr: + return emit_alu_op2_int(*alu, op2_ashr_int, shader); + case nir_op_ushr: + return emit_alu_op2_int(*alu, op2_lshr_int, shader); + default:; } } else { if (shader.chip_class() == ISA_CC_EVERGREEN) { switch (alu->op) { - case nir_op_f2i32: return emit_alu_f2i32_or_u32_eg(*alu, op1_flt_to_int, shader); - case nir_op_f2u32: return emit_alu_f2i32_or_u32_eg(*alu, op1_flt_to_uint, shader); - default: - ; + case nir_op_f2i32: + return emit_alu_f2i32_or_u32_eg(*alu, op1_flt_to_int, shader); + case nir_op_f2u32: + return emit_alu_f2i32_or_u32_eg(*alu, op1_flt_to_uint, shader); + default:; } - } - + } + if (shader.chip_class() >= ISA_CC_R700) { switch (alu->op) { - case nir_op_ishl: return emit_alu_op2_int(*alu, op2_lshl_int, shader); - case nir_op_ishr: return emit_alu_op2_int(*alu, op2_ashr_int, shader); - case nir_op_ushr: return emit_alu_op2_int(*alu, op2_lshr_int, shader); - default: - ; - } - } else { + case nir_op_ishl: + return emit_alu_op2_int(*alu, op2_lshl_int, shader); + case nir_op_ishr: + return emit_alu_op2_int(*alu, op2_ashr_int, shader); + case nir_op_ushr: + return emit_alu_op2_int(*alu, op2_lshr_int, shader); + default:; + } + } else { switch (alu->op) { - case nir_op_ishl: return emit_alu_trans_op2_eg(*alu, op2_lshl_int, shader); - case nir_op_ishr: return emit_alu_trans_op2_eg(*alu, op2_ashr_int, shader); - case nir_op_ushr: return emit_alu_trans_op2_eg(*alu, op2_lshr_int, shader); - default: - ; - } + case nir_op_ishl: + return emit_alu_trans_op2_eg(*alu, op2_lshl_int, shader); + case nir_op_ishr: + return emit_alu_trans_op2_eg(*alu, op2_ashr_int, shader); + case nir_op_ushr: + return emit_alu_trans_op2_eg(*alu, op2_lshr_int, shader); + default:; + } } - + switch (alu->op) { - case nir_op_f2i32: return emit_alu_trans_op1_eg(*alu, op1_flt_to_int, shader); - case nir_op_f2u32: return emit_alu_trans_op1_eg(*alu, op1_flt_to_uint, shader); - case nir_op_fcos_amd: return emit_alu_trans_op1_eg(*alu, op1_cos, shader); - case nir_op_fexp2: return emit_alu_trans_op1_eg(*alu, op1_exp_ieee, shader); - case nir_op_flog2: return emit_alu_trans_op1_eg(*alu, op1_log_clamped, shader); - case nir_op_frcp: return emit_alu_trans_op1_eg(*alu, op1_recip_ieee, shader); - case nir_op_frsq: return emit_alu_trans_op1_eg(*alu, op1_recipsqrt_ieee1, shader); - case nir_op_fsin_amd: return emit_alu_trans_op1_eg(*alu, op1_sin, shader); - case nir_op_fsqrt: return emit_alu_trans_op1_eg(*alu, op1_sqrt_ieee, shader); - case nir_op_i2f32: return emit_alu_trans_op1_eg(*alu, op1_int_to_flt, shader); - case nir_op_u2f32: return emit_alu_trans_op1_eg(*alu, op1_uint_to_flt, shader); - case nir_op_imul: return emit_alu_trans_op2_eg(*alu, op2_mullo_int, shader); - case nir_op_imul_high: return emit_alu_trans_op2_eg(*alu, op2_mulhi_int, shader); - case nir_op_umul_high: return emit_alu_trans_op2_eg(*alu, op2_mulhi_uint, shader); - default: - ; + case nir_op_f2i32: + return emit_alu_trans_op1_eg(*alu, op1_flt_to_int, shader); + case nir_op_f2u32: + return emit_alu_trans_op1_eg(*alu, op1_flt_to_uint, shader); + case nir_op_fcos_amd: + return emit_alu_trans_op1_eg(*alu, op1_cos, shader); + case nir_op_fexp2: + return emit_alu_trans_op1_eg(*alu, op1_exp_ieee, shader); + case nir_op_flog2: + return emit_alu_trans_op1_eg(*alu, op1_log_clamped, shader); + case nir_op_frcp: + return emit_alu_trans_op1_eg(*alu, op1_recip_ieee, shader); + case nir_op_frsq: + return emit_alu_trans_op1_eg(*alu, op1_recipsqrt_ieee1, shader); + case nir_op_fsin_amd: + return emit_alu_trans_op1_eg(*alu, op1_sin, shader); + case nir_op_fsqrt: + return emit_alu_trans_op1_eg(*alu, op1_sqrt_ieee, shader); + case nir_op_i2f32: + return emit_alu_trans_op1_eg(*alu, op1_int_to_flt, shader); + case nir_op_u2f32: + return emit_alu_trans_op1_eg(*alu, op1_uint_to_flt, shader); + case nir_op_imul: + return emit_alu_trans_op2_eg(*alu, op2_mullo_int, shader); + case nir_op_imul_high: + return emit_alu_trans_op2_eg(*alu, op2_mulhi_int, shader); + case nir_op_umul_high: + return emit_alu_trans_op2_eg(*alu, op2_mulhi_uint, shader); + default:; } } switch (alu->op) { - case nir_op_b2b1: return emit_alu_op1(*alu, op1_mov, shader); - case nir_op_b2b32: return emit_alu_op1(*alu, op1_mov, shader); - case nir_op_b2f32: return emit_alu_b2x(*alu, ALU_SRC_1, shader); - case nir_op_b2i32: return emit_alu_b2x(*alu, ALU_SRC_1_INT, shader); + case nir_op_b2b1: + return emit_alu_op1(*alu, op1_mov, shader); + case nir_op_b2b32: + return emit_alu_op1(*alu, op1_mov, shader); + case nir_op_b2f32: + return emit_alu_b2x(*alu, ALU_SRC_1, shader); + case nir_op_b2i32: + return emit_alu_b2x(*alu, ALU_SRC_1_INT, shader); - case nir_op_bfm: return emit_alu_op2_int(*alu, op2_bfm_int, shader, op2_opt_none); - case nir_op_bit_count: return emit_alu_op1(*alu, op1_bcnt_int, shader); + case nir_op_bfm: + return emit_alu_op2_int(*alu, op2_bfm_int, shader, op2_opt_none); + case nir_op_bit_count: + return emit_alu_op1(*alu, op1_bcnt_int, shader); - case nir_op_bitfield_reverse: return emit_alu_op1(*alu, op1_bfrev_int, shader); - case nir_op_bitfield_select: return emit_alu_op3(*alu, op3_bfi_int, shader); + case nir_op_bitfield_reverse: + return emit_alu_op1(*alu, op1_bfrev_int, shader); + case nir_op_bitfield_select: + return emit_alu_op3(*alu, op3_bfi_int, shader); - case nir_op_b32all_fequal2: return emit_any_all_fcomp2(*alu, op2_sete_dx10, shader); - case nir_op_b32all_fequal3: return emit_any_all_fcomp(*alu, op2_sete, 3, true, shader); - case nir_op_b32all_fequal4: return emit_any_all_fcomp(*alu, op2_sete, 4, true, shader); - case nir_op_b32all_iequal2: return emit_any_all_icomp(*alu, op2_sete_int, 2, true, shader); - case nir_op_b32all_iequal3: return emit_any_all_icomp(*alu, op2_sete_int, 3, true, shader); - case nir_op_b32all_iequal4: return emit_any_all_icomp(*alu, op2_sete_int, 4, true, shader); - case nir_op_b32any_fnequal2: return emit_any_all_fcomp2(*alu, op2_setne_dx10, shader); - case nir_op_b32any_fnequal3: return emit_any_all_fcomp(*alu, op2_setne, 3, false, shader); - case nir_op_b32any_fnequal4: return emit_any_all_fcomp(*alu, op2_setne, 4, false, shader); - case nir_op_b32any_inequal2: return emit_any_all_icomp(*alu, op2_setne_int, 2, false, shader); - case nir_op_b32any_inequal3: return emit_any_all_icomp(*alu, op2_setne_int, 3, false, shader); - case nir_op_b32any_inequal4: return emit_any_all_icomp(*alu, op2_setne_int, 4, false, shader); - case nir_op_b32csel: return emit_alu_op3(*alu, op3_cnde_int, shader, {0, 2, 1}); + case nir_op_b32all_fequal2: + return emit_any_all_fcomp2(*alu, op2_sete_dx10, shader); + case nir_op_b32all_fequal3: + return emit_any_all_fcomp(*alu, op2_sete, 3, true, shader); + case nir_op_b32all_fequal4: + return emit_any_all_fcomp(*alu, op2_sete, 4, true, shader); + case nir_op_b32all_iequal2: + return emit_any_all_icomp(*alu, op2_sete_int, 2, true, shader); + case nir_op_b32all_iequal3: + return emit_any_all_icomp(*alu, op2_sete_int, 3, true, shader); + case nir_op_b32all_iequal4: + return emit_any_all_icomp(*alu, op2_sete_int, 4, true, shader); + case nir_op_b32any_fnequal2: + return emit_any_all_fcomp2(*alu, op2_setne_dx10, shader); + case nir_op_b32any_fnequal3: + return emit_any_all_fcomp(*alu, op2_setne, 3, false, shader); + case nir_op_b32any_fnequal4: + return emit_any_all_fcomp(*alu, op2_setne, 4, false, shader); + case nir_op_b32any_inequal2: + return emit_any_all_icomp(*alu, op2_setne_int, 2, false, shader); + case nir_op_b32any_inequal3: + return emit_any_all_icomp(*alu, op2_setne_int, 3, false, shader); + case nir_op_b32any_inequal4: + return emit_any_all_icomp(*alu, op2_setne_int, 4, false, shader); + case nir_op_b32csel: + return emit_alu_op3(*alu, op3_cnde_int, shader, {0, 2, 1}); - case nir_op_f2b32: return emit_alu_comb_with_zero(*alu, op2_setne_dx10, shader); - case nir_op_fabs: return emit_alu_op1(*alu, op1_mov, shader, {1 << alu_src0_abs}); - case nir_op_fadd: return emit_alu_op2(*alu, op2_add, shader); - case nir_op_fceil: return emit_alu_op1(*alu, op1_ceil, shader); - case nir_op_fcsel: return emit_alu_op3(*alu, op3_cnde, shader, {0, 2, 1}); - case nir_op_fcsel_ge: return emit_alu_op3(*alu, op3_cndge, shader, {0, 1, 2}); - case nir_op_fcsel_gt: return emit_alu_op3(*alu, op3_cndgt, shader, {0, 1, 2}); + case nir_op_f2b32: + return emit_alu_comb_with_zero(*alu, op2_setne_dx10, shader); + case nir_op_fabs: + return emit_alu_op1(*alu, op1_mov, shader, {1 << alu_src0_abs}); + case nir_op_fadd: + return emit_alu_op2(*alu, op2_add, shader); + case nir_op_fceil: + return emit_alu_op1(*alu, op1_ceil, shader); + case nir_op_fcsel: + return emit_alu_op3(*alu, op3_cnde, shader, {0, 2, 1}); + case nir_op_fcsel_ge: + return emit_alu_op3(*alu, op3_cndge, shader, {0, 1, 2}); + case nir_op_fcsel_gt: + return emit_alu_op3(*alu, op3_cndgt, shader, {0, 1, 2}); - case nir_op_fdph: return emit_fdph(*alu, shader); - case nir_op_fdot2: return emit_dot(*alu, 2, shader); - case nir_op_fdot3: return emit_dot(*alu, 3, shader); - case nir_op_fdot4: return emit_dot(*alu, 4, shader); + case nir_op_fdph: + return emit_fdph(*alu, shader); + case nir_op_fdot2: + return emit_dot(*alu, 2, shader); + case nir_op_fdot3: + return emit_dot(*alu, 3, shader); + case nir_op_fdot4: + return emit_dot(*alu, 4, shader); case nir_op_feq32: - case nir_op_feq: return emit_alu_op2(*alu, op2_sete_dx10, shader); - case nir_op_ffloor: return emit_alu_op1(*alu, op1_floor, shader); - case nir_op_ffract: return emit_alu_op1(*alu, op1_fract, shader); - case nir_op_fge32: return emit_alu_op2(*alu, op2_setge_dx10, shader); - case nir_op_fge: return emit_alu_op2(*alu, op2_setge_dx10, shader); - case nir_op_find_lsb: return emit_alu_op1(*alu, op1_ffbl_int, shader); + case nir_op_feq: + return emit_alu_op2(*alu, op2_sete_dx10, shader); + case nir_op_ffloor: + return emit_alu_op1(*alu, op1_floor, shader); + case nir_op_ffract: + return emit_alu_op1(*alu, op1_fract, shader); + case nir_op_fge32: + return emit_alu_op2(*alu, op2_setge_dx10, shader); + case nir_op_fge: + return emit_alu_op2(*alu, op2_setge_dx10, shader); + case nir_op_find_lsb: + return emit_alu_op1(*alu, op1_ffbl_int, shader); - case nir_op_flt32: return emit_alu_op2(*alu, op2_setgt_dx10, shader, op2_opt_reverse); - case nir_op_flt: return emit_alu_op2(*alu, op2_setgt_dx10, shader, op2_opt_reverse); - case nir_op_fmax: return emit_alu_op2(*alu, op2_max_dx10, shader); - case nir_op_fmin: return emit_alu_op2(*alu, op2_min_dx10, shader); + case nir_op_flt32: + return emit_alu_op2(*alu, op2_setgt_dx10, shader, op2_opt_reverse); + case nir_op_flt: + return emit_alu_op2(*alu, op2_setgt_dx10, shader, op2_opt_reverse); + case nir_op_fmax: + return emit_alu_op2(*alu, op2_max_dx10, shader); + case nir_op_fmin: + return emit_alu_op2(*alu, op2_min_dx10, shader); case nir_op_fmul: if (!shader.has_flag(Shader::sh_legacy_math_rules)) return emit_alu_op2(*alu, op2_mul_ieee, shader); FALLTHROUGH; - case nir_op_fmulz: return emit_alu_op2(*alu, op2_mul, shader); + case nir_op_fmulz: + return emit_alu_op2(*alu, op2_mul, shader); - case nir_op_fneg: return emit_alu_op1(*alu, op1_mov, shader, {1 << alu_src0_neg}); - case nir_op_fneu32: return emit_alu_op2(*alu, op2_setne_dx10, shader); - case nir_op_fneu: return emit_alu_op2(*alu, op2_setne_dx10, shader); + case nir_op_fneg: + return emit_alu_op1(*alu, op1_mov, shader, {1 << alu_src0_neg}); + case nir_op_fneu32: + return emit_alu_op2(*alu, op2_setne_dx10, shader); + case nir_op_fneu: + return emit_alu_op2(*alu, op2_setne_dx10, shader); - case nir_op_fround_even: return emit_alu_op1(*alu, op1_rndne, shader); - case nir_op_fsat: return emit_alu_op1(*alu, op1_mov, shader, {1 << alu_dst_clamp}); - case nir_op_fsub: return emit_alu_op2(*alu, op2_add, shader, op2_opt_neg_src1); - case nir_op_ftrunc: return emit_alu_op1(*alu, op1_trunc, shader); + case nir_op_fround_even: + return emit_alu_op1(*alu, op1_rndne, shader); + case nir_op_fsat: + return emit_alu_op1(*alu, op1_mov, shader, {1 << alu_dst_clamp}); + case nir_op_fsub: + return emit_alu_op2(*alu, op2_add, shader, op2_opt_neg_src1); + case nir_op_ftrunc: + return emit_alu_op1(*alu, op1_trunc, shader); case nir_op_i2b1: - case nir_op_i2b32: return emit_alu_i2orf2_b1(*alu, op2_setne_int, shader); - case nir_op_iadd: return emit_alu_op2_int(*alu, op2_add_int, shader); - case nir_op_iand: return emit_alu_op2_int(*alu, op2_and_int, shader); - case nir_op_ibfe: return emit_alu_op3(*alu, op3_bfe_int, shader); - case nir_op_i32csel_ge: return emit_alu_op3(*alu, op3_cndge_int, shader, {0, 1, 2}); - case nir_op_i32csel_gt: return emit_alu_op3(*alu, op3_cndgt_int, shader, {0, 1, 2}); - case nir_op_ieq32: return emit_alu_op2_int(*alu, op2_sete_int, shader); - case nir_op_ieq: return emit_alu_op2_int(*alu, op2_sete_int, shader); - case nir_op_ifind_msb_rev: return emit_alu_op1(*alu, op1_ffbh_int, shader); - case nir_op_ige32: return emit_alu_op2_int(*alu, op2_setge_int, shader); - case nir_op_ige: return emit_alu_op2_int(*alu, op2_setge_int, shader); - case nir_op_ilt32: return emit_alu_op2_int(*alu, op2_setgt_int, shader, op2_opt_reverse); - case nir_op_ilt: return emit_alu_op2_int(*alu, op2_setgt_int, shader, op2_opt_reverse); - case nir_op_imax: return emit_alu_op2_int(*alu, op2_max_int, shader); - case nir_op_imin: return emit_alu_op2_int(*alu, op2_min_int, shader); - case nir_op_ine32: return emit_alu_op2_int(*alu, op2_setne_int, shader); - case nir_op_ine: return emit_alu_op2_int(*alu, op2_setne_int, shader); - case nir_op_ineg: return emit_alu_comb_with_zero(*alu, op2_sub_int, shader); - case nir_op_inot: return emit_alu_op1(*alu, op1_not_int, shader); - case nir_op_ior: return emit_alu_op2_int(*alu, op2_or_int, shader); - case nir_op_isub: return emit_alu_op2_int(*alu, op2_sub_int, shader); - case nir_op_ixor: return emit_alu_op2_int(*alu, op2_xor_int, shader); - case nir_op_pack_64_2x32: return emit_pack_64_2x32(*alu, shader); - case nir_op_unpack_64_2x32: return emit_unpack_64_2x32(*alu, shader); - case nir_op_pack_64_2x32_split: return emit_pack_64_2x32_split(*alu, shader); - case nir_op_pack_half_2x16_split: return emit_pack_32_2x16_split(*alu, shader); - case nir_op_slt: return emit_alu_op2(*alu, op2_setgt, shader, op2_opt_reverse); - case nir_op_sge: return emit_alu_op2(*alu, op2_setge, shader); - case nir_op_seq: return emit_alu_op2(*alu, op2_sete, shader); - case nir_op_sne: return emit_alu_op2(*alu, op2_setne, shader); - case nir_op_ubfe: return emit_alu_op3(*alu, op3_bfe_uint, shader); - case nir_op_ufind_msb_rev: return emit_alu_op1(*alu, op1_ffbh_uint, shader); - case nir_op_uge32: return emit_alu_op2_int(*alu, op2_setge_uint, shader); - case nir_op_uge: return emit_alu_op2_int(*alu, op2_setge_uint, shader); - case nir_op_ult32: return emit_alu_op2_int(*alu, op2_setgt_uint, shader, op2_opt_reverse); - case nir_op_ult: return emit_alu_op2_int(*alu, op2_setgt_uint, shader, op2_opt_reverse); - case nir_op_umad24: return emit_alu_op3(*alu, op3_muladd_uint24, shader, {0, 1, 2}); - case nir_op_umax: return emit_alu_op2_int(*alu, op2_max_uint, shader); - case nir_op_umin: return emit_alu_op2_int(*alu, op2_min_uint, shader); - case nir_op_umul24: return emit_alu_op2(*alu, op2_mul_uint24, shader); - case nir_op_unpack_64_2x32_split_x: return emit_unpack_64_2x32_split(*alu, 0, shader); - case nir_op_unpack_64_2x32_split_y: return emit_unpack_64_2x32_split(*alu, 1, shader); - case nir_op_unpack_half_2x16_split_x: return emit_unpack_32_2x16_split_x(*alu, shader); - case nir_op_unpack_half_2x16_split_y: return emit_unpack_32_2x16_split_y(*alu, shader); + case nir_op_i2b32: + return emit_alu_i2orf2_b1(*alu, op2_setne_int, shader); + case nir_op_iadd: + return emit_alu_op2_int(*alu, op2_add_int, shader); + case nir_op_iand: + return emit_alu_op2_int(*alu, op2_and_int, shader); + case nir_op_ibfe: + return emit_alu_op3(*alu, op3_bfe_int, shader); + case nir_op_i32csel_ge: + return emit_alu_op3(*alu, op3_cndge_int, shader, {0, 1, 2}); + case nir_op_i32csel_gt: + return emit_alu_op3(*alu, op3_cndgt_int, shader, {0, 1, 2}); + case nir_op_ieq32: + return emit_alu_op2_int(*alu, op2_sete_int, shader); + case nir_op_ieq: + return emit_alu_op2_int(*alu, op2_sete_int, shader); + case nir_op_ifind_msb_rev: + return emit_alu_op1(*alu, op1_ffbh_int, shader); + case nir_op_ige32: + return emit_alu_op2_int(*alu, op2_setge_int, shader); + case nir_op_ige: + return emit_alu_op2_int(*alu, op2_setge_int, shader); + case nir_op_ilt32: + return emit_alu_op2_int(*alu, op2_setgt_int, shader, op2_opt_reverse); + case nir_op_ilt: + return emit_alu_op2_int(*alu, op2_setgt_int, shader, op2_opt_reverse); + case nir_op_imax: + return emit_alu_op2_int(*alu, op2_max_int, shader); + case nir_op_imin: + return emit_alu_op2_int(*alu, op2_min_int, shader); + case nir_op_ine32: + return emit_alu_op2_int(*alu, op2_setne_int, shader); + case nir_op_ine: + return emit_alu_op2_int(*alu, op2_setne_int, shader); + case nir_op_ineg: + return emit_alu_comb_with_zero(*alu, op2_sub_int, shader); + case nir_op_inot: + return emit_alu_op1(*alu, op1_not_int, shader); + case nir_op_ior: + return emit_alu_op2_int(*alu, op2_or_int, shader); + case nir_op_isub: + return emit_alu_op2_int(*alu, op2_sub_int, shader); + case nir_op_ixor: + return emit_alu_op2_int(*alu, op2_xor_int, shader); + case nir_op_pack_64_2x32: + return emit_pack_64_2x32(*alu, shader); + case nir_op_unpack_64_2x32: + return emit_unpack_64_2x32(*alu, shader); + case nir_op_pack_64_2x32_split: + return emit_pack_64_2x32_split(*alu, shader); + case nir_op_pack_half_2x16_split: + return emit_pack_32_2x16_split(*alu, shader); + case nir_op_slt: + return emit_alu_op2(*alu, op2_setgt, shader, op2_opt_reverse); + case nir_op_sge: + return emit_alu_op2(*alu, op2_setge, shader); + case nir_op_seq: + return emit_alu_op2(*alu, op2_sete, shader); + case nir_op_sne: + return emit_alu_op2(*alu, op2_setne, shader); + case nir_op_ubfe: + return emit_alu_op3(*alu, op3_bfe_uint, shader); + case nir_op_ufind_msb_rev: + return emit_alu_op1(*alu, op1_ffbh_uint, shader); + case nir_op_uge32: + return emit_alu_op2_int(*alu, op2_setge_uint, shader); + case nir_op_uge: + return emit_alu_op2_int(*alu, op2_setge_uint, shader); + case nir_op_ult32: + return emit_alu_op2_int(*alu, op2_setgt_uint, shader, op2_opt_reverse); + case nir_op_ult: + return emit_alu_op2_int(*alu, op2_setgt_uint, shader, op2_opt_reverse); + case nir_op_umad24: + return emit_alu_op3(*alu, op3_muladd_uint24, shader, {0, 1, 2}); + case nir_op_umax: + return emit_alu_op2_int(*alu, op2_max_uint, shader); + case nir_op_umin: + return emit_alu_op2_int(*alu, op2_min_uint, shader); + case nir_op_umul24: + return emit_alu_op2(*alu, op2_mul_uint24, shader); + case nir_op_unpack_64_2x32_split_x: + return emit_unpack_64_2x32_split(*alu, 0, shader); + case nir_op_unpack_64_2x32_split_y: + return emit_unpack_64_2x32_split(*alu, 1, shader); + case nir_op_unpack_half_2x16_split_x: + return emit_unpack_32_2x16_split_x(*alu, shader); + case nir_op_unpack_half_2x16_split_y: + return emit_unpack_32_2x16_split_y(*alu, shader); case nir_op_ffma: if (!shader.has_flag(Shader::sh_legacy_math_rules)) return emit_alu_op3(*alu, op3_muladd_ieee, shader); FALLTHROUGH; - case nir_op_ffmaz: return emit_alu_op3(*alu, op3_muladd, shader); + case nir_op_ffmaz: + return emit_alu_op3(*alu, op3_muladd, shader); - case nir_op_mov: return emit_alu_op1(*alu, op1_mov, shader); - case nir_op_f2i32: return emit_alu_op1(*alu, op1_flt_to_int, shader); - case nir_op_vec2: return emit_create_vec(*alu, 2, shader); - case nir_op_vec3: return emit_create_vec(*alu, 3, shader); - case nir_op_vec4: return emit_create_vec(*alu, 4, shader); + case nir_op_mov: + return emit_alu_op1(*alu, op1_mov, shader); + case nir_op_f2i32: + return emit_alu_op1(*alu, op1_flt_to_int, shader); + case nir_op_vec2: + return emit_create_vec(*alu, 2, shader); + case nir_op_vec3: + return emit_create_vec(*alu, 3, shader); + case nir_op_vec4: + return emit_create_vec(*alu, 4, shader); case nir_op_fddx: - case nir_op_fddx_coarse: return emit_tex_fdd(*alu, TexInstr::get_gradient_h, false, shader); - case nir_op_fddx_fine: return emit_tex_fdd(*alu, TexInstr::get_gradient_h, true, shader); + case nir_op_fddx_coarse: + return emit_tex_fdd(*alu, TexInstr::get_gradient_h, false, shader); + case nir_op_fddx_fine: + return emit_tex_fdd(*alu, TexInstr::get_gradient_h, true, shader); case nir_op_fddy: - case nir_op_fddy_coarse: return emit_tex_fdd(*alu,TexInstr::get_gradient_v, false, shader); - case nir_op_fddy_fine: return emit_tex_fdd(*alu, TexInstr::get_gradient_v, true, shader); - case nir_op_cube_r600: return emit_alu_cube(*alu, shader); + case nir_op_fddy_coarse: + return emit_tex_fdd(*alu, TexInstr::get_gradient_v, false, shader); + case nir_op_fddy_fine: + return emit_tex_fdd(*alu, TexInstr::get_gradient_v, true, shader); + case nir_op_cube_r600: + return emit_alu_cube(*alu, shader); default: fprintf(stderr, "Unknown instruction '"); nir_print_instr(&alu->instr, stderr); fprintf(stderr, "'\n"); assert(0); - return false; + return false; } } -static Pin pin_for_components(const nir_alu_instr& alu) +static Pin +pin_for_components(const nir_alu_instr& alu) { - return (alu.dest.dest.is_ssa && - (nir_dest_num_components(alu.dest.dest) == 1)) ? pin_free : pin_none; - + return (alu.dest.dest.is_ssa && (nir_dest_num_components(alu.dest.dest) == 1)) + ? pin_free + : pin_none; } -static bool emit_alu_op1_64bit(const nir_alu_instr& alu, EAluOp opcode, Shader& shader, bool switch_chan) +static bool +emit_alu_op1_64bit(const nir_alu_instr& alu, + EAluOp opcode, + Shader& shader, + bool switch_chan) { auto& value_factory = shader.value_factory(); @@ -1452,22 +1718,24 @@ static bool emit_alu_op1_64bit(const nir_alu_instr& alu, EAluOp opcode, Shader& AluInstr *ir = nullptr; - int swz[2] = {0,1}; + int swz[2] = {0, 1}; if (switch_chan) { swz[0] = 1; swz[1] = 0; } - for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest) ; ++i) { - for (unsigned c = 0; c < 2 ; ++c) { + for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest); ++i) { + for (unsigned c = 0; c < 2; ++c) { ir = new AluInstr(opcode, value_factory.dest(alu.dest, 2 * i + c, pin_chan), value_factory.src64(alu.src[0], i, swz[c]), {alu_write}); group->add_instruction(ir); } - if (alu.src[0].abs) ir->set_alu_flag(alu_src0_abs); - if (alu.src[0].negate) ir->set_alu_flag(alu_src0_neg); + if (alu.src[0].abs) + ir->set_alu_flag(alu_src0_abs); + if (alu.src[0].negate) + ir->set_alu_flag(alu_src0_neg); } if (ir) ir->set_alu_flag(alu_last_instr); @@ -1475,29 +1743,33 @@ static bool emit_alu_op1_64bit(const nir_alu_instr& alu, EAluOp opcode, Shader& return true; } -static bool emit_alu_mov_64bit(const nir_alu_instr& alu, Shader& shader) +static bool +emit_alu_mov_64bit(const nir_alu_instr& alu, Shader& shader) { auto& value_factory = shader.value_factory(); AluInstr *ir = nullptr; - for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest) ; ++i) { - for (unsigned c = 0; c < 2 ; ++c) { + for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest); ++i) { + for (unsigned c = 0; c < 2; ++c) { ir = new AluInstr(op1_mov, value_factory.dest(alu.dest, 2 * i + c, pin_free), value_factory.src64(alu.src[0], i, c), {alu_write}); shader.emit_instruction(ir); } - if (alu.src[0].abs) ir->set_alu_flag(alu_src0_abs); - if (alu.src[0].negate) ir->set_alu_flag(alu_src0_neg); + if (alu.src[0].abs) + ir->set_alu_flag(alu_src0_abs); + if (alu.src[0].negate) + ir->set_alu_flag(alu_src0_neg); } if (ir) ir->set_alu_flag(alu_last_instr); return true; } -static bool emit_alu_neg(const nir_alu_instr& alu, Shader& shader) +static bool +emit_alu_neg(const nir_alu_instr& alu, Shader& shader) { auto& value_factory = shader.value_factory(); @@ -1505,8 +1777,8 @@ static bool emit_alu_neg(const nir_alu_instr& alu, Shader& shader) AluInstr *ir = nullptr; - for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest) ; ++i) { - for (unsigned c = 0; c < 2 ; ++c) { + for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest); ++i) { + for (unsigned c = 0; c < 2; ++c) { ir = new AluInstr(op1_mov, value_factory.dest(alu.dest, 2 * i + c, pin_chan), value_factory.src64(alu.src[0], i, c), @@ -1521,17 +1793,20 @@ static bool emit_alu_neg(const nir_alu_instr& alu, Shader& shader) return true; } -static bool emit_alu_abs64(const nir_alu_instr& alu, Shader& shader) +static bool +emit_alu_abs64(const nir_alu_instr& alu, Shader& shader) { auto& value_factory = shader.value_factory(); assert(nir_dest_num_components(alu.dest.dest) == 1); - shader.emit_instruction(new AluInstr(op1_mov, value_factory.dest(alu.dest, 0, pin_chan), + shader.emit_instruction(new AluInstr(op1_mov, + value_factory.dest(alu.dest, 0, pin_chan), value_factory.src64(alu.src[0], 0, 0), AluInstr::write)); - auto ir = new AluInstr(op1_mov, value_factory.dest(alu.dest, 1, pin_chan), + auto ir = new AluInstr(op1_mov, + value_factory.dest(alu.dest, 1, pin_chan), value_factory.src64(alu.src[0], 0, 1), AluInstr::last_write); ir->set_alu_flag(alu_src0_abs); @@ -1539,27 +1814,30 @@ static bool emit_alu_abs64(const nir_alu_instr& alu, Shader& shader) return true; } -static bool emit_alu_op2_64bit(const nir_alu_instr& alu, EAluOp opcode, Shader& shader, bool switch_src) +static bool +emit_alu_op2_64bit(const nir_alu_instr& alu, + EAluOp opcode, + Shader& shader, + bool switch_src) { auto& value_factory = shader.value_factory(); auto group = new AluGroup(); AluInstr *ir = nullptr; - int order[2] = {0, 1}; + int order[2] = {0, 1}; if (switch_src) { order[0] = 1; order[1] = 0; } - int num_emit0 = opcode == op2_mul_64 ? 3 : 1; + int num_emit0 = opcode == op2_mul_64 ? 3 : 1; assert(num_emit0 == 1 || nir_dest_num_components(alu.dest.dest) == 1); - for (unsigned k = 0; k < nir_dest_num_components(alu.dest.dest); ++k) { int i = 0; for (; i < num_emit0; ++i) { - auto dest = i < 2 ? value_factory.dest(alu.dest, i, pin_chan) : - value_factory.dummy_dest(i); + auto dest = i < 2 ? value_factory.dest(alu.dest, i, pin_chan) + : value_factory.dummy_dest(i); ir = new AluInstr(opcode, dest, @@ -1567,10 +1845,14 @@ static bool emit_alu_op2_64bit(const nir_alu_instr& alu, EAluOp opcode, Shader& value_factory.src64(alu.src[order[1]], k, 1), i < 2 ? AluInstr::write : AluInstr::empty); - if (alu.src[0].abs) ir->set_alu_flag(switch_src ? alu_src1_abs : alu_src0_abs); - if (alu.src[1].abs) ir->set_alu_flag(switch_src ? alu_src0_abs : alu_src1_abs); - if (alu.src[0].negate) ir->set_alu_flag(switch_src ? alu_src1_neg : alu_src0_neg); - if (alu.src[1].negate) ir->set_alu_flag(switch_src ? alu_src0_neg : alu_src1_neg); + if (alu.src[0].abs) + ir->set_alu_flag(switch_src ? alu_src1_abs : alu_src0_abs); + if (alu.src[1].abs) + ir->set_alu_flag(switch_src ? alu_src0_abs : alu_src1_abs); + if (alu.src[0].negate) + ir->set_alu_flag(switch_src ? alu_src1_neg : alu_src0_neg); + if (alu.src[1].negate) + ir->set_alu_flag(switch_src ? alu_src0_neg : alu_src1_neg); if (alu.dest.saturate && i == 0) { ir->set_alu_flag(alu_dst_clamp); } @@ -1578,8 +1860,8 @@ static bool emit_alu_op2_64bit(const nir_alu_instr& alu, EAluOp opcode, Shader& group->add_instruction(ir); } - auto dest = i == 1 ? value_factory.dest(alu.dest, i, pin_chan) : - value_factory.dummy_dest(i); + auto dest = + i == 1 ? value_factory.dest(alu.dest, i, pin_chan) : value_factory.dummy_dest(i); ir = new AluInstr(opcode, dest, @@ -1595,12 +1877,15 @@ static bool emit_alu_op2_64bit(const nir_alu_instr& alu, EAluOp opcode, Shader& return true; } -static bool emit_alu_op2_64bit_one_dst(const nir_alu_instr& alu, EAluOp opcode, - Shader& shader, bool switch_order) +static bool +emit_alu_op2_64bit_one_dst(const nir_alu_instr& alu, + EAluOp opcode, + Shader& shader, + bool switch_order) { auto& value_factory = shader.value_factory(); AluInstr *ir = nullptr; - int order[2] = {0, 1}; + int order[2] = {0, 1}; if (switch_order) { order[0] = 1; order[1] = 0; @@ -1617,10 +1902,14 @@ static bool emit_alu_op2_64bit_one_dst(const nir_alu_instr& alu, EAluOp opcode, ir = new AluInstr(opcode, dest, src, AluInstr::write, 2); - if (alu.src[0].abs) ir->set_alu_flag(switch_order ? alu_src1_abs : alu_src0_abs); - if (alu.src[1].abs) ir->set_alu_flag(switch_order ? alu_src0_abs : alu_src1_abs); - if (alu.src[0].negate) ir->set_alu_flag(switch_order ? alu_src1_neg : alu_src0_neg); - if (alu.src[1].negate) ir->set_alu_flag(switch_order ? alu_src0_neg : alu_src1_neg); + if (alu.src[0].abs) + ir->set_alu_flag(switch_order ? alu_src1_abs : alu_src0_abs); + if (alu.src[1].abs) + ir->set_alu_flag(switch_order ? alu_src0_abs : alu_src1_abs); + if (alu.src[0].negate) + ir->set_alu_flag(switch_order ? alu_src1_neg : alu_src0_neg); + if (alu.src[1].negate) + ir->set_alu_flag(switch_order ? alu_src0_neg : alu_src1_neg); ir->set_alu_flag(alu_64bit_op); shader.emit_instruction(ir); @@ -1631,21 +1920,24 @@ static bool emit_alu_op2_64bit_one_dst(const nir_alu_instr& alu, EAluOp opcode, return true; } -static bool emit_alu_op1_64bit_trans(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) +static bool +emit_alu_op1_64bit_trans(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) { auto& value_factory = shader.value_factory(); auto group = new AluGroup(); AluInstr *ir = nullptr; for (unsigned i = 0; i < 3; ++i) { ir = new AluInstr(opcode, - i < 2 ? value_factory.dest(alu.dest, i, pin_chan) : - value_factory.dummy_dest(i), + i < 2 ? value_factory.dest(alu.dest, i, pin_chan) + : value_factory.dummy_dest(i), value_factory.src64(alu.src[0], 0, 1), value_factory.src64(alu.src[0], 0, 0), i < 2 ? AluInstr::write : AluInstr::empty); - if (alu.src[0].abs || opcode == op1_sqrt_64) ir->set_alu_flag(alu_src1_abs); - if (alu.src[0].negate) ir->set_alu_flag(alu_src1_neg); + if (alu.src[0].abs || opcode == op1_sqrt_64) + ir->set_alu_flag(alu_src1_abs); + if (alu.src[0].negate) + ir->set_alu_flag(alu_src1_neg); group->add_instruction(ir); } @@ -1653,19 +1945,19 @@ static bool emit_alu_op1_64bit_trans(const nir_alu_instr& alu, EAluOp opcode, Sh ir->set_alu_flag(alu_last_instr); shader.emit_instruction(group); return true; - } -static bool emit_alu_fma_64bit(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) +static bool +emit_alu_fma_64bit(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) { auto& value_factory = shader.value_factory(); auto group = new AluGroup(); AluInstr *ir = nullptr; - for (unsigned i = 0; i < 4 ; ++i) { + for (unsigned i = 0; i < 4; ++i) { int chan = i < 3 ? 1 : 0; - auto dest = i < 2 ? value_factory.dest(alu.dest, i, pin_chan) : - value_factory.dummy_dest(i); + auto dest = + i < 2 ? value_factory.dest(alu.dest, i, pin_chan) : value_factory.dummy_dest(i); ir = new AluInstr(opcode, dest, @@ -1675,13 +1967,15 @@ static bool emit_alu_fma_64bit(const nir_alu_instr& alu, EAluOp opcode, Shader& i < 2 ? AluInstr::write : AluInstr::empty); if (i < 3) { - if (alu.src[0].negate) ir->set_alu_flag(alu_src0_neg); - if (alu.src[1].negate) ir->set_alu_flag(alu_src1_neg); - if (alu.src[2].negate) ir->set_alu_flag(alu_src2_neg); + if (alu.src[0].negate) + ir->set_alu_flag(alu_src0_neg); + if (alu.src[1].negate) + ir->set_alu_flag(alu_src1_neg); + if (alu.src[2].negate) + ir->set_alu_flag(alu_src2_neg); } group->add_instruction(ir); - } if (ir) ir->set_alu_flag(alu_last_instr); @@ -1689,26 +1983,27 @@ static bool emit_alu_fma_64bit(const nir_alu_instr& alu, EAluOp opcode, Shader& return true; } -static bool emit_alu_b2f64(const nir_alu_instr& alu, Shader& shader) +static bool +emit_alu_b2f64(const nir_alu_instr& alu, Shader& shader) { auto& value_factory = shader.value_factory(); auto group = new AluGroup(); AluInstr *ir = nullptr; - for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest) ; ++i) { + for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest); ++i) { ir = new AluInstr(op2_and_int, - value_factory.dest(alu.dest, 2 * i, pin_group), - value_factory.src(alu.src[0], i), - value_factory.zero(), - {alu_write}); - group->add_instruction(ir); + value_factory.dest(alu.dest, 2 * i, pin_group), + value_factory.src(alu.src[0], i), + value_factory.zero(), + {alu_write}); + group->add_instruction(ir); - ir = new AluInstr(op2_and_int, - value_factory.dest(alu.dest, 2 * i + 1, pin_group), - value_factory.src(alu.src[0], i), - value_factory.literal(0x3ff00000), - {alu_write}); - group->add_instruction(ir); + ir = new AluInstr(op2_and_int, + value_factory.dest(alu.dest, 2 * i + 1, pin_group), + value_factory.src(alu.src[0], i), + value_factory.literal(0x3ff00000), + {alu_write}); + group->add_instruction(ir); } if (ir) ir->set_alu_flag(alu_last_instr); @@ -1716,7 +2011,8 @@ static bool emit_alu_b2f64(const nir_alu_instr& alu, Shader& shader) return true; } -static bool emit_alu_i2f64(const nir_alu_instr& alu, EAluOp op, Shader& shader) +static bool +emit_alu_i2f64(const nir_alu_instr& alu, EAluOp op, Shader& shader) { /* int 64 to f64 should have been lowered, so we only handle i32 to f64 */ auto& value_factory = shader.value_factory(); @@ -1726,45 +2022,36 @@ static bool emit_alu_i2f64(const nir_alu_instr& alu, EAluOp op, Shader& shader) assert(nir_dest_num_components(alu.dest.dest) == 1); auto tmpx = value_factory.temp_register(); - shader.emit_instruction(new AluInstr(op2_and_int, tmpx, + shader.emit_instruction(new AluInstr(op2_and_int, + tmpx, value_factory.src(alu.src[0], 0), value_factory.literal(0xffffff00), AluInstr::write)); auto tmpy = value_factory.temp_register(); - shader.emit_instruction(new AluInstr(op2_and_int, tmpy, + shader.emit_instruction(new AluInstr(op2_and_int, + tmpy, value_factory.src(alu.src[0], 0), value_factory.literal(0xff), AluInstr::last_write)); - auto tmpx2 = value_factory.temp_register(); auto tmpy2 = value_factory.temp_register(); - shader.emit_instruction(new AluInstr(op, tmpx2, tmpx, - AluInstr::last_write)); - shader.emit_instruction(new AluInstr(op, tmpy2, tmpy, - AluInstr::last_write)); + shader.emit_instruction(new AluInstr(op, tmpx2, tmpx, AluInstr::last_write)); + shader.emit_instruction(new AluInstr(op, tmpy2, tmpy, AluInstr::last_write)); auto tmpx3 = value_factory.temp_register(0); auto tmpy3 = value_factory.temp_register(1); auto tmpz3 = value_factory.temp_register(2); auto tmpw3 = value_factory.temp_register(3); - - ir = new AluInstr(op1_flt32_to_flt64, - tmpx3, - tmpx2, AluInstr::write); + ir = new AluInstr(op1_flt32_to_flt64, tmpx3, tmpx2, AluInstr::write); group->add_instruction(ir); - ir = new AluInstr(op1_flt32_to_flt64, - tmpy3, - value_factory.zero(), AluInstr::write); + ir = new AluInstr(op1_flt32_to_flt64, tmpy3, value_factory.zero(), AluInstr::write); group->add_instruction(ir); - ir = new AluInstr(op1_flt32_to_flt64, - tmpz3, - tmpy2, AluInstr::write); + ir = new AluInstr(op1_flt32_to_flt64, tmpz3, tmpy2, AluInstr::write); group->add_instruction(ir); - ir = new AluInstr(op1_flt32_to_flt64, - tmpw3, - value_factory.zero(), AluInstr::last_write); + ir = + new AluInstr(op1_flt32_to_flt64, tmpw3, value_factory.zero(), AluInstr::last_write); group->add_instruction(ir); shader.emit_instruction(group); @@ -1772,18 +2059,23 @@ static bool emit_alu_i2f64(const nir_alu_instr& alu, EAluOp op, Shader& shader) ir = new AluInstr(op2_add_64, value_factory.dest(alu.dest, 0, pin_chan), - tmpy3, tmpw3, AluInstr::write); + tmpy3, + tmpw3, + AluInstr::write); group->add_instruction(ir); ir = new AluInstr(op2_add_64, value_factory.dest(alu.dest, 1, pin_chan), - tmpx3, tmpz3, AluInstr::write); + tmpx3, + tmpz3, + AluInstr::write); group->add_instruction(ir); shader.emit_instruction(group); return true; } -static bool emit_alu_f2f64(const nir_alu_instr& alu, Shader& shader) +static bool +emit_alu_f2f64(const nir_alu_instr& alu, Shader& shader) { auto& value_factory = shader.value_factory(); auto group = new AluGroup(); @@ -1793,17 +2085,20 @@ static bool emit_alu_f2f64(const nir_alu_instr& alu, Shader& shader) ir = new AluInstr(op1_flt32_to_flt64, value_factory.dest(alu.dest, 0, pin_chan), - value_factory.src(alu.src[0], 0), AluInstr::write); + value_factory.src(alu.src[0], 0), + AluInstr::write); group->add_instruction(ir); ir = new AluInstr(op1_flt32_to_flt64, value_factory.dest(alu.dest, 1, pin_chan), - value_factory.zero(), AluInstr::last_write); + value_factory.zero(), + AluInstr::last_write); group->add_instruction(ir); shader.emit_instruction(group); return true; } -static bool emit_alu_f2f32(const nir_alu_instr& alu, Shader& shader) +static bool +emit_alu_f2f32(const nir_alu_instr& alu, Shader& shader) { auto& value_factory = shader.value_factory(); auto group = new AluGroup(); @@ -1811,33 +2106,37 @@ static bool emit_alu_f2f32(const nir_alu_instr& alu, Shader& shader) ir = new AluInstr(op1v_flt64_to_flt32, value_factory.dest(alu.dest, 0, pin_chan), - value_factory.src64(alu.src[0], 0, 1), {alu_write}); + value_factory.src64(alu.src[0], 0, 1), + {alu_write}); group->add_instruction(ir); ir = new AluInstr(op1v_flt64_to_flt32, value_factory.dummy_dest(1), - value_factory.src64(alu.src[0], 0, 0), AluInstr::last); + value_factory.src64(alu.src[0], 0, 0), + AluInstr::last); group->add_instruction(ir); shader.emit_instruction(group); return true; - } -static bool emit_alu_b2x(const nir_alu_instr& alu, AluInlineConstants mask, Shader& shader) +static bool +emit_alu_b2x(const nir_alu_instr& alu, AluInlineConstants mask, Shader& shader) { auto& value_factory = shader.value_factory(); AluInstr *ir = nullptr; auto pin = pin_for_components(alu); - for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest) ; ++i) { - if (alu.dest.write_mask & (1 << i)){ + for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest); ++i) { + if (alu.dest.write_mask & (1 << i)) { auto src = value_factory.src(alu.src[0], i); ir = new AluInstr(op2_and_int, value_factory.dest(alu.dest, i, pin), src, value_factory.inline_const(mask, 0), {alu_write}); - if (alu.src[0].negate) ir->set_alu_flag(alu_src0_neg); - if (alu.src[0].abs) ir->set_alu_flag(alu_src0_abs); + if (alu.src[0].negate) + ir->set_alu_flag(alu_src0_neg); + if (alu.src[0].abs) + ir->set_alu_flag(alu_src0_abs); shader.emit_instruction(ir); } } @@ -1846,17 +2145,23 @@ static bool emit_alu_b2x(const nir_alu_instr& alu, AluInlineConstants mask, Shad return true; } -static bool emit_alu_op1(const nir_alu_instr& alu, EAluOp opcode, Shader& shader, const AluOpFlags& flags) +static bool +emit_alu_op1(const nir_alu_instr& alu, + EAluOp opcode, + Shader& shader, + const AluOpFlags& flags) { auto& value_factory = shader.value_factory(); AluInstr *ir = nullptr; auto pin = pin_for_components(alu); - for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest) ; ++i) { - if (alu.dest.write_mask & (1 << i)){ - ir = new AluInstr(opcode, value_factory.dest(alu.dest, i, pin), - value_factory.src(alu.src[0], i), {alu_write}); + for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest); ++i) { + if (alu.dest.write_mask & (1 << i)) { + ir = new AluInstr(opcode, + value_factory.dest(alu.dest, i, pin), + value_factory.src(alu.src[0], i), + {alu_write}); if (flags.test(alu_src0_abs) || alu.src[0].abs) ir->set_alu_flag(alu_src0_abs); @@ -1875,8 +2180,11 @@ static bool emit_alu_op1(const nir_alu_instr& alu, EAluOp opcode, Shader& shader return true; } -static bool emit_alu_op2(const nir_alu_instr& alu, EAluOp opcode, Shader& shader, - AluInstr::Op2Options opts) +static bool +emit_alu_op2(const nir_alu_instr& alu, + EAluOp opcode, + Shader& shader, + AluInstr::Op2Options opts) { auto& value_factory = shader.value_factory(); const nir_alu_src *src0 = &alu.src[0]; @@ -1893,18 +2201,24 @@ static bool emit_alu_op2(const nir_alu_instr& alu, EAluOp opcode, Shader& shader auto pin = pin_for_components(alu); AluInstr *ir = nullptr; - for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest) ; ++i) { - if (alu.dest.write_mask & (1 << i)){ + for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest); ++i) { + if (alu.dest.write_mask & (1 << i)) { ir = new AluInstr(opcode, value_factory.dest(alu.dest.dest, i, pin), value_factory.src(*src0, i), - value_factory.src(*src1, i), {alu_write}); + value_factory.src(*src1, i), + {alu_write}); - if (src0->negate) ir->set_alu_flag(alu_src0_neg); - if (src0->abs) ir->set_alu_flag(alu_src0_abs); - if (src1_negate) ir->set_alu_flag(alu_src1_neg); - if (src1->abs) ir->set_alu_flag(alu_src1_abs); - if (alu.dest.saturate) ir->set_alu_flag(alu_dst_clamp); + if (src0->negate) + ir->set_alu_flag(alu_src0_neg); + if (src0->abs) + ir->set_alu_flag(alu_src0_abs); + if (src1_negate) + ir->set_alu_flag(alu_src1_neg); + if (src1->abs) + ir->set_alu_flag(alu_src1_abs); + if (alu.dest.saturate) + ir->set_alu_flag(alu_dst_clamp); shader.emit_instruction(ir); } } @@ -1913,8 +2227,11 @@ static bool emit_alu_op2(const nir_alu_instr& alu, EAluOp opcode, Shader& shader return true; } -static bool emit_alu_op2_int(const nir_alu_instr& alu, EAluOp opcode, Shader& shader, - AluInstr::Op2Options opts) +static bool +emit_alu_op2_int(const nir_alu_instr& alu, + EAluOp opcode, + Shader& shader, + AluInstr::Op2Options opts) { assert(!alu.src[0].abs); assert(!alu.src[0].negate); @@ -1924,8 +2241,11 @@ static bool emit_alu_op2_int(const nir_alu_instr& alu, EAluOp opcode, Shader& sh return emit_alu_op2(alu, opcode, shader, opts); } -static bool emit_alu_op3(const nir_alu_instr& alu, EAluOp opcode, Shader& shader, - const std::array& src_shuffle) +static bool +emit_alu_op3(const nir_alu_instr& alu, + EAluOp opcode, + Shader& shader, + const std::array& src_shuffle) { auto& value_factory = shader.value_factory(); const nir_alu_src *src[3]; @@ -1935,23 +2255,28 @@ static bool emit_alu_op3(const nir_alu_instr& alu, EAluOp opcode, Shader& shader auto pin = pin_for_components(alu); AluInstr *ir = nullptr; - for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest) ; ++i) { - if (alu.dest.write_mask & (1 << i)){ - ir = new AluInstr(opcode, value_factory.dest(alu.dest.dest, i, pin), + for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest); ++i) { + if (alu.dest.write_mask & (1 << i)) { + ir = new AluInstr(opcode, + value_factory.dest(alu.dest.dest, i, pin), value_factory.src(*src[0], i), - value_factory.src(*src[1], i), - value_factory.src(*src[2], i), - {alu_write}); + value_factory.src(*src[1], i), + value_factory.src(*src[2], i), + {alu_write}); - if (src[0]->negate) ir->set_alu_flag(alu_src0_neg); - if (src[1]->negate) ir->set_alu_flag(alu_src1_neg); - if (src[2]->negate) ir->set_alu_flag(alu_src2_neg); + if (src[0]->negate) + ir->set_alu_flag(alu_src0_neg); + if (src[1]->negate) + ir->set_alu_flag(alu_src1_neg); + if (src[2]->negate) + ir->set_alu_flag(alu_src2_neg); assert(!src[0]->abs); assert(!src[1]->abs); assert(!src[2]->abs); - if (alu.dest.saturate) ir->set_alu_flag(alu_dst_clamp); + if (alu.dest.saturate) + ir->set_alu_flag(alu_dst_clamp); ir->set_alu_flag(alu_write); shader.emit_instruction(ir); } @@ -1961,7 +2286,8 @@ static bool emit_alu_op3(const nir_alu_instr& alu, EAluOp opcode, Shader& shader return true; } -static bool emit_any_all_fcomp2(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) +static bool +emit_any_all_fcomp2(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) { AluInstr *ir = nullptr; auto& value_factory = shader.value_factory(); @@ -1970,10 +2296,12 @@ static bool emit_any_all_fcomp2(const nir_alu_instr& alu, EAluOp opcode, Shader& tmp[0] = value_factory.temp_register(); tmp[1] = value_factory.temp_register(); - for (unsigned i = 0; i < 2 ; ++i) { - ir = new AluInstr(opcode, tmp[i], + for (unsigned i = 0; i < 2; ++i) { + ir = new AluInstr(opcode, + tmp[i], value_factory.src(alu.src[0], i), - value_factory.src(alu.src[1], i), {alu_write}); + value_factory.src(alu.src[1], i), + {alu_write}); if (alu.src[0].abs) ir->set_alu_flag(alu_src0_abs); if (alu.src[0].negate) @@ -1988,15 +2316,18 @@ static bool emit_any_all_fcomp2(const nir_alu_instr& alu, EAluOp opcode, Shader& } ir->set_alu_flag(alu_last_instr); - opcode = (opcode == op2_setne_dx10) ? op2_or_int: op2_and_int; + opcode = (opcode == op2_setne_dx10) ? op2_or_int : op2_and_int; ir = new AluInstr(opcode, value_factory.dest(alu.dest, 0, pin_free), - tmp[0], tmp[1], AluInstr::last_write); + tmp[0], + tmp[1], + AluInstr::last_write); shader.emit_instruction(ir); return true; } -static bool emit_any_all_fcomp(const nir_alu_instr& alu, EAluOp op, int nc, bool all, Shader& shader) +static bool +emit_any_all_fcomp(const nir_alu_instr& alu, EAluOp op, int nc, bool all, Shader& shader) { /* This should probabyl be lowered in nir */ auto& value_factory = shader.value_factory(); @@ -2005,17 +2336,19 @@ static bool emit_any_all_fcomp(const nir_alu_instr& alu, EAluOp op, int nc, bool RegisterVec4 v = value_factory.temp_vec4(pin_group); AluInstr::SrcValues s; - for (int i = 0; i < nc ; ++i) { + for (int i = 0; i < nc; ++i) { s.push_back(v[i]); } - for (int i = nc; i < 4 ; ++i) + for (int i = nc; i < 4; ++i) s.push_back(value_factory.inline_const(all ? ALU_SRC_1 : ALU_SRC_0, 0)); - for (int i = 0; i < nc ; ++i) { - ir = new AluInstr(op, v[i], + for (int i = 0; i < nc; ++i) { + ir = new AluInstr(op, + v[i], value_factory.src(alu.src[0], i), - value_factory.src(alu.src[1], i), {alu_write}); + value_factory.src(alu.src[1], i), + {alu_write}); if (alu.src[0].abs) ir->set_alu_flag(alu_src0_abs); @@ -2042,9 +2375,9 @@ static bool emit_any_all_fcomp(const nir_alu_instr& alu, EAluOp op, int nc, bool shader.emit_instruction(ir); if (all) - op = (op == op2_sete) ? op2_sete_dx10: op2_setne_dx10; + op = (op == op2_sete) ? op2_sete_dx10 : op2_setne_dx10; else - op = (op == op2_sete) ? op2_setne_dx10: op2_sete_dx10; + op = (op == op2_sete) ? op2_setne_dx10 : op2_sete_dx10; ir = new AluInstr(op, value_factory.dest(alu.dest, 0, pin_free), @@ -2058,7 +2391,8 @@ static bool emit_any_all_fcomp(const nir_alu_instr& alu, EAluOp op, int nc, bool return true; } -static bool emit_any_all_icomp(const nir_alu_instr& alu, EAluOp op, int nc, bool all, Shader& shader) +static bool +emit_any_all_icomp(const nir_alu_instr& alu, EAluOp op, int nc, bool all, Shader& shader) { assert(!alu.src[0].abs); assert(!alu.src[0].negate); @@ -2073,20 +2407,23 @@ static bool emit_any_all_icomp(const nir_alu_instr& alu, EAluOp op, int nc, bool auto dest = value_factory.dest(alu.dest.dest, 0, pin_free); - for (int i = 0; i < nc + nc/2; ++i) + for (int i = 0; i < nc + nc / 2; ++i) v[i] = value_factory.temp_register(); EAluOp combine = all ? op2_and_int : op2_or_int; - for (int i = 0; i < nc ; ++i) { - ir = new AluInstr(op, v[i], value_factory.src(alu.src[0], i), - value_factory.src(alu.src[1], i), AluInstr::write); + for (int i = 0; i < nc; ++i) { + ir = new AluInstr(op, + v[i], + value_factory.src(alu.src[0], i), + value_factory.src(alu.src[1], i), + AluInstr::write); shader.emit_instruction(ir); } if (ir) - ir->set_alu_flag(alu_last_instr); + ir->set_alu_flag(alu_last_instr); - if (nc ==2) { + if (nc == 2) { ir = new AluInstr(combine, dest, v[0], v[1], AluInstr::last_write); shader.emit_instruction(ir); return true; @@ -2113,7 +2450,8 @@ static bool emit_any_all_icomp(const nir_alu_instr& alu, EAluOp op, int nc, bool return false; } -static bool emit_dot(const nir_alu_instr& alu, int n, Shader& shader) +static bool +emit_dot(const nir_alu_instr& alu, int n, Shader& shader) { auto& value_factory = shader.value_factory(); const nir_alu_src& src0 = alu.src[0]; @@ -2123,32 +2461,38 @@ static bool emit_dot(const nir_alu_instr& alu, int n, Shader& shader) AluInstr::SrcValues srcs(8); - for (int i = 0; i < n ; ++i) { - srcs[2 * i ] = value_factory.src(src0, i); + for (int i = 0; i < n; ++i) { + srcs[2 * i] = value_factory.src(src0, i); srcs[2 * i + 1] = value_factory.src(src1, i); } - for (int i = n; i < 4 ; ++i) { - srcs[2 * i ] = value_factory.zero(); + for (int i = n; i < 4; ++i) { + srcs[2 * i] = value_factory.zero(); srcs[2 * i + 1] = value_factory.zero(); } - auto op = unlikely(shader.has_flag(Shader::sh_legacy_math_rules)) ? - op2_dot4 : op2_dot4_ieee; - AluInstr *ir = new AluInstr(op, dest, srcs, AluInstr::last_write, 4); + auto op = + unlikely(shader.has_flag(Shader::sh_legacy_math_rules)) ? op2_dot4 : op2_dot4_ieee; + AluInstr *ir = new AluInstr(op, dest, srcs, AluInstr::last_write, 4); - if (src0.negate) ir->set_alu_flag(alu_src0_neg); - if (src0.abs) ir->set_alu_flag(alu_src0_abs); - if (src1.negate) ir->set_alu_flag(alu_src1_neg); - if (src1.abs) ir->set_alu_flag(alu_src1_abs); + if (src0.negate) + ir->set_alu_flag(alu_src0_neg); + if (src0.abs) + ir->set_alu_flag(alu_src0_abs); + if (src1.negate) + ir->set_alu_flag(alu_src1_neg); + if (src1.abs) + ir->set_alu_flag(alu_src1_abs); - if (alu.dest.saturate) ir->set_alu_flag(alu_dst_clamp); + if (alu.dest.saturate) + ir->set_alu_flag(alu_dst_clamp); shader.emit_instruction(ir); return true; } -static bool emit_fdph(const nir_alu_instr& alu, Shader& shader) +static bool +emit_fdph(const nir_alu_instr& alu, Shader& shader) { auto& value_factory = shader.value_factory(); const nir_alu_src& src0 = alu.src[0]; @@ -2158,43 +2502,52 @@ static bool emit_fdph(const nir_alu_instr& alu, Shader& shader) AluInstr::SrcValues srcs(8); - for (int i = 0; i < 3 ; ++i) { - srcs[2 * i ] = value_factory.src(src0, i); + for (int i = 0; i < 3; ++i) { + srcs[2 * i] = value_factory.src(src0, i); srcs[2 * i + 1] = value_factory.src(src1, i); } srcs[6] = value_factory.one(); srcs[7] = value_factory.src(src1, 3); - auto op = unlikely(shader.has_flag(Shader::sh_legacy_math_rules)) ? - op2_dot4 : op2_dot4_ieee; - AluInstr *ir = new AluInstr(op, dest, srcs, AluInstr::last_write, 4); + auto op = + unlikely(shader.has_flag(Shader::sh_legacy_math_rules)) ? op2_dot4 : op2_dot4_ieee; + AluInstr *ir = new AluInstr(op, dest, srcs, AluInstr::last_write, 4); - if (src0.negate) ir->set_alu_flag(alu_src0_neg); - if (src0.abs) ir->set_alu_flag(alu_src0_abs); - if (src1.negate) ir->set_alu_flag(alu_src1_neg); - if (src1.abs) ir->set_alu_flag(alu_src1_abs); + if (src0.negate) + ir->set_alu_flag(alu_src0_neg); + if (src0.abs) + ir->set_alu_flag(alu_src0_abs); + if (src1.negate) + ir->set_alu_flag(alu_src1_neg); + if (src1.abs) + ir->set_alu_flag(alu_src1_abs); - if (alu.dest.saturate) ir->set_alu_flag(alu_dst_clamp); + if (alu.dest.saturate) + ir->set_alu_flag(alu_dst_clamp); shader.emit_instruction(ir); return true; } -static bool emit_create_vec(const nir_alu_instr& instr, unsigned nc, Shader& shader) +static bool +emit_create_vec(const nir_alu_instr& instr, unsigned nc, Shader& shader) { auto& value_factory = shader.value_factory(); AluInstr *ir = nullptr; - for(unsigned i = 0; i < nc; ++i) { - if (instr.dest.write_mask & (1 << i)){ + for (unsigned i = 0; i < nc; ++i) { + if (instr.dest.write_mask & (1 << i)) { auto src = value_factory.src(instr.src[i].src, instr.src[i].swizzle[0]); auto dst = value_factory.dest(instr.dest.dest, i, pin_chan); ir = new AluInstr(op1_mov, dst, src, {alu_write}); - if (instr.dest.saturate) ir->set_alu_flag(alu_dst_clamp); - if (instr.src[i].negate) ir->set_alu_flag(alu_src0_neg); - if (instr.src[i].abs) ir->set_alu_flag(alu_src0_abs); + if (instr.dest.saturate) + ir->set_alu_flag(alu_dst_clamp); + if (instr.src[i].negate) + ir->set_alu_flag(alu_src0_neg); + if (instr.src[i].abs) + ir->set_alu_flag(alu_src0_abs); shader.emit_instruction(ir); } @@ -2205,13 +2558,14 @@ static bool emit_create_vec(const nir_alu_instr& instr, unsigned nc, Shader& sha return true; } -static bool emit_alu_i2orf2_b1(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) +static bool +emit_alu_i2orf2_b1(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) { auto& value_factory = shader.value_factory(); AluInstr *ir = nullptr; Pin pin = nir_dest_num_components(alu.dest.dest) == 1 ? pin_free : pin_none; - for (int i = 0; i < 4 ; ++i) { + for (int i = 0; i < 4; ++i) { if (alu.dest.write_mask & (1 << i)) { ir = new AluInstr(opcode, value_factory.dest(alu.dest, i, pin), @@ -2226,13 +2580,14 @@ static bool emit_alu_i2orf2_b1(const nir_alu_instr& alu, EAluOp opcode, Shader& return true; } -static bool emit_alu_comb_with_zero(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) +static bool +emit_alu_comb_with_zero(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) { auto& value_factory = shader.value_factory(); AluInstr *ir = nullptr; auto pin = pin_for_components(alu); - for (int i = 0; i < 4 ; ++i) { - if (alu.dest.write_mask & (1 << i)){ + for (int i = 0; i < 4; ++i) { + if (alu.dest.write_mask & (1 << i)) { ir = new AluInstr(opcode, value_factory.dest(alu.dest, i, pin), value_factory.zero(), @@ -2247,65 +2602,79 @@ static bool emit_alu_comb_with_zero(const nir_alu_instr& alu, EAluOp opcode, Sha return true; } -static bool emit_pack_64_2x32_split(const nir_alu_instr& alu, Shader& shader) +static bool +emit_pack_64_2x32_split(const nir_alu_instr& alu, Shader& shader) { auto& value_factory = shader.value_factory(); AluInstr *ir = nullptr; for (unsigned i = 0; i < 2; ++i) { - ir = new AluInstr(op1_mov, value_factory.dest(alu.dest, i, pin_none), - value_factory.src(alu.src[i], 0), AluInstr::write); - shader.emit_instruction(ir); + ir = new AluInstr(op1_mov, + value_factory.dest(alu.dest, i, pin_none), + value_factory.src(alu.src[i], 0), + AluInstr::write); + shader.emit_instruction(ir); } ir->set_alu_flag(alu_last_instr); return true; } -static bool emit_pack_64_2x32(const nir_alu_instr& alu, Shader& shader) +static bool +emit_pack_64_2x32(const nir_alu_instr& alu, Shader& shader) { auto& value_factory = shader.value_factory(); AluInstr *ir = nullptr; for (unsigned i = 0; i < 2; ++i) { - ir = new AluInstr(op1_mov, value_factory.dest(alu.dest, i, pin_none), - value_factory.src(alu.src[0], i), AluInstr::write); - shader.emit_instruction(ir); + ir = new AluInstr(op1_mov, + value_factory.dest(alu.dest, i, pin_none), + value_factory.src(alu.src[0], i), + AluInstr::write); + shader.emit_instruction(ir); } ir->set_alu_flag(alu_last_instr); return true; } - -static bool emit_unpack_64_2x32(const nir_alu_instr& alu, Shader& shader) +static bool +emit_unpack_64_2x32(const nir_alu_instr& alu, Shader& shader) { auto& value_factory = shader.value_factory(); AluInstr *ir = nullptr; for (unsigned i = 0; i < 2; ++i) { - ir = new AluInstr(op1_mov, value_factory.dest(alu.dest, i, pin_none), - value_factory.src64(alu.src[0], 0, i), AluInstr::write); - shader.emit_instruction(ir); + ir = new AluInstr(op1_mov, + value_factory.dest(alu.dest, i, pin_none), + value_factory.src64(alu.src[0], 0, i), + AluInstr::write); + shader.emit_instruction(ir); } ir->set_alu_flag(alu_last_instr); return true; } -bool emit_alu_vec2_64(const nir_alu_instr& alu, Shader& shader) +bool +emit_alu_vec2_64(const nir_alu_instr& alu, Shader& shader) { auto& value_factory = shader.value_factory(); AluInstr *ir = nullptr; for (unsigned i = 0; i < 2; ++i) { - ir = new AluInstr(op1_mov, value_factory.dest(alu.dest, i, pin_chan), - value_factory.src64(alu.src[0], 0, i), AluInstr::write); - shader.emit_instruction(ir); + ir = new AluInstr(op1_mov, + value_factory.dest(alu.dest, i, pin_chan), + value_factory.src64(alu.src[0], 0, i), + AluInstr::write); + shader.emit_instruction(ir); } for (unsigned i = 0; i < 2; ++i) { - ir = new AluInstr(op1_mov, value_factory.dest(alu.dest, i + 2, pin_chan), - value_factory.src64(alu.src[1], 1, i), AluInstr::write); - shader.emit_instruction(ir); + ir = new AluInstr(op1_mov, + value_factory.dest(alu.dest, i + 2, pin_chan), + value_factory.src64(alu.src[1], 1, i), + AluInstr::write); + shader.emit_instruction(ir); } ir->set_alu_flag(alu_last_instr); return true; } -static bool emit_pack_32_2x16_split(const nir_alu_instr& alu, Shader& shader) +static bool +emit_pack_32_2x16_split(const nir_alu_instr& alu, Shader& shader) { auto& value_factory = shader.value_factory(); @@ -2313,53 +2682,64 @@ static bool emit_pack_32_2x16_split(const nir_alu_instr& alu, Shader& shader) auto y = value_factory.temp_register(); auto yy = value_factory.temp_register(); - shader.emit_instruction(new AluInstr(op1_flt32_to_flt16, x, - value_factory.src(alu.src[0], 0), AluInstr::last_write)); + shader.emit_instruction(new AluInstr( + op1_flt32_to_flt16, x, value_factory.src(alu.src[0], 0), AluInstr::last_write)); - shader.emit_instruction(new AluInstr(op1_flt32_to_flt16, y, - value_factory.src(alu.src[1], 0), AluInstr::last_write)); + shader.emit_instruction(new AluInstr( + op1_flt32_to_flt16, y, value_factory.src(alu.src[1], 0), AluInstr::last_write)); - shader.emit_instruction(new AluInstr(op2_lshl_int, yy, y, value_factory.literal(16), AluInstr::last_write)); + shader.emit_instruction( + new AluInstr(op2_lshl_int, yy, y, value_factory.literal(16), AluInstr::last_write)); shader.emit_instruction(new AluInstr(op2_or_int, value_factory.dest(alu.dest, 0, pin_free), - x, yy, AluInstr::last_write)); + x, + yy, + AluInstr::last_write)); return true; } -static bool emit_unpack_64_2x32_split(const nir_alu_instr& alu, int comp, Shader& shader) +static bool +emit_unpack_64_2x32_split(const nir_alu_instr& alu, int comp, Shader& shader) { auto& value_factory = shader.value_factory(); - shader.emit_instruction(new AluInstr(op1_mov, value_factory.dest(alu.dest, 0, pin_free), - value_factory.src64(alu.src[0], 0, comp), AluInstr::last_write)); + shader.emit_instruction(new AluInstr(op1_mov, + value_factory.dest(alu.dest, 0, pin_free), + value_factory.src64(alu.src[0], 0, comp), + AluInstr::last_write)); return true; } -static bool emit_unpack_32_2x16_split_x(const nir_alu_instr& alu, Shader& shader) +static bool +emit_unpack_32_2x16_split_x(const nir_alu_instr& alu, Shader& shader) { auto& value_factory = shader.value_factory(); - shader.emit_instruction(new AluInstr(op1_flt16_to_flt32, value_factory.dest(alu.dest, 0, pin_free), - value_factory.src(alu.src[0], 0), AluInstr::last_write)); + shader.emit_instruction(new AluInstr(op1_flt16_to_flt32, + value_factory.dest(alu.dest, 0, pin_free), + value_factory.src(alu.src[0], 0), + AluInstr::last_write)); return true; } -static bool emit_unpack_32_2x16_split_y(const nir_alu_instr& alu, Shader& shader) +static bool +emit_unpack_32_2x16_split_y(const nir_alu_instr& alu, Shader& shader) { auto& value_factory = shader.value_factory(); auto tmp = value_factory.temp_register(); - shader.emit_instruction(new AluInstr(op2_lshr_int, tmp, + shader.emit_instruction(new AluInstr(op2_lshr_int, + tmp, value_factory.src(alu.src[0], 0), value_factory.literal(16), AluInstr::last_write)); shader.emit_instruction(new AluInstr(op1_flt16_to_flt32, value_factory.dest(alu.dest, 0, pin_free), - tmp, AluInstr::last_write)); + tmp, + AluInstr::last_write)); return true; } - - -static bool emit_alu_trans_op1_eg(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) +static bool +emit_alu_trans_op1_eg(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) { auto& value_factory = shader.value_factory(); const nir_alu_src& src0 = alu.src[0]; @@ -2368,14 +2748,17 @@ static bool emit_alu_trans_op1_eg(const nir_alu_instr& alu, EAluOp opcode, Shade auto pin = pin_for_components(alu); for (unsigned i = 0; i < nir_dest_num_components(alu.dest.dest); ++i) { - if (alu.dest.write_mask & (1 << i)){ + if (alu.dest.write_mask & (1 << i)) { ir = new AluInstr(opcode, value_factory.dest(alu.dest.dest, i, pin), value_factory.src(src0, i), AluInstr::last_write); - if (src0.negate) ir->set_alu_flag(alu_src0_neg); - if (src0.abs) ir->set_alu_flag(alu_src0_abs); - if (alu.dest.saturate) ir->set_alu_flag(alu_dst_clamp); + if (src0.negate) + ir->set_alu_flag(alu_src0_neg); + if (src0.abs) + ir->set_alu_flag(alu_src0_abs); + if (alu.dest.saturate) + ir->set_alu_flag(alu_dst_clamp); ir->set_alu_flag(alu_is_trans); shader.emit_instruction(ir); } @@ -2384,7 +2767,8 @@ static bool emit_alu_trans_op1_eg(const nir_alu_instr& alu, EAluOp opcode, Shade return true; } -static bool emit_alu_f2i32_or_u32_eg(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) +static bool +emit_alu_f2i32_or_u32_eg(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) { auto& value_factory = shader.value_factory(); AluInstr *ir = nullptr; @@ -2395,9 +2779,14 @@ static bool emit_alu_f2i32_or_u32_eg(const nir_alu_instr& alu, EAluOp opcode, Sh for (int i = 0; i < num_comp; ++i) { reg[i] = value_factory.temp_register(); - ir = new AluInstr(op1_trunc, reg[i], value_factory.src(alu.src[0], i), AluInstr::last_write); - if (alu.src[0].abs) ir->set_alu_flag(alu_src0_abs); - if (alu.src[0].negate) ir->set_alu_flag(alu_src0_neg); + ir = new AluInstr(op1_trunc, + reg[i], + value_factory.src(alu.src[0], i), + AluInstr::last_write); + if (alu.src[0].abs) + ir->set_alu_flag(alu_src0_abs); + if (alu.src[0].negate) + ir->set_alu_flag(alu_src0_neg); shader.emit_instruction(ir); } @@ -2405,7 +2794,8 @@ static bool emit_alu_f2i32_or_u32_eg(const nir_alu_instr& alu, EAluOp opcode, Sh for (int i = 0; i < num_comp; ++i) { ir = new AluInstr(opcode, value_factory.dest(alu.dest, i, pin), - reg[i], AluInstr::write); + reg[i], + AluInstr::write); if (opcode == op1_flt_to_uint) { ir->set_alu_flag(alu_is_trans); ir->set_alu_flag(alu_last_instr); @@ -2416,7 +2806,8 @@ static bool emit_alu_f2i32_or_u32_eg(const nir_alu_instr& alu, EAluOp opcode, Sh return true; } -static bool emit_alu_trans_op1_cayman(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) +static bool +emit_alu_trans_op1_cayman(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) { auto& value_factory = shader.value_factory(); const nir_alu_src& src0 = alu.src[0]; @@ -2430,28 +2821,30 @@ static bool emit_alu_trans_op1_cayman(const nir_alu_instr& alu, EAluOp opcode, S for (unsigned j = 0; j < ncomp; ++j) { if (alu.dest.write_mask & (1 << j)) { AluInstr::SrcValues srcs(ncomp); - PRegister dest = value_factory.dest(alu.dest.dest, j, pin, - (1 << ncomp) - 1); + PRegister dest = value_factory.dest(alu.dest.dest, j, pin, (1 << ncomp) - 1); for (unsigned i = 0; i < ncomp; ++i) srcs[i] = value_factory.src(src0, j); - auto ir = new AluInstr(opcode, dest, srcs, AluInstr::last_write, ncomp); + auto ir = new AluInstr(opcode, dest, srcs, AluInstr::last_write, ncomp); - if (alu.src[0].abs) ir->set_alu_flag(alu_src0_abs); - if (alu.src[0].negate) ir->set_alu_flag(alu_src0_neg); - if (alu.dest.saturate) ir->set_alu_flag(alu_dst_clamp); + if (alu.src[0].abs) + ir->set_alu_flag(alu_src0_abs); + if (alu.src[0].negate) + ir->set_alu_flag(alu_src0_neg); + if (alu.dest.saturate) + ir->set_alu_flag(alu_dst_clamp); ir->set_alu_flag(alu_is_cayman_trans); - shader.emit_instruction(ir); } } return true; } -static bool emit_alu_trans_op2_eg(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) +static bool +emit_alu_trans_op2_eg(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) { auto& value_factory = shader.value_factory(); @@ -2461,16 +2854,23 @@ static bool emit_alu_trans_op2_eg(const nir_alu_instr& alu, EAluOp opcode, Shade AluInstr *ir = nullptr; auto pin = pin_for_components(alu); - for (int i = 0; i < 4 ; ++i) { - if (alu.dest.write_mask & (1 << i)){ - ir = new AluInstr(opcode, value_factory.dest(alu.dest.dest, i, pin), + for (int i = 0; i < 4; ++i) { + if (alu.dest.write_mask & (1 << i)) { + ir = new AluInstr(opcode, + value_factory.dest(alu.dest.dest, i, pin), value_factory.src(src0, i), - value_factory.src(src1, i), AluInstr::last_write); - if (src0.negate) ir->set_alu_flag(alu_src0_neg); - if (src0.abs) ir->set_alu_flag(alu_src0_abs); - if (src1.negate) ir->set_alu_flag(alu_src1_neg); - if (src1.abs) ir->set_alu_flag(alu_src1_abs); - if (alu.dest.saturate) ir->set_alu_flag(alu_dst_clamp); + value_factory.src(src1, i), + AluInstr::last_write); + if (src0.negate) + ir->set_alu_flag(alu_src0_neg); + if (src0.abs) + ir->set_alu_flag(alu_src0_abs); + if (src1.negate) + ir->set_alu_flag(alu_src1_neg); + if (src1.abs) + ir->set_alu_flag(alu_src1_abs); + if (alu.dest.saturate) + ir->set_alu_flag(alu_dst_clamp); ir->set_alu_flag(alu_is_trans); shader.emit_instruction(ir); } @@ -2478,7 +2878,8 @@ static bool emit_alu_trans_op2_eg(const nir_alu_instr& alu, EAluOp opcode, Shade return true; } -static bool emit_alu_trans_op2_cayman(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) +static bool +emit_alu_trans_op2_cayman(const nir_alu_instr& alu, EAluOp opcode, Shader& shader) { auto& value_factory = shader.value_factory(); @@ -2492,19 +2893,23 @@ static bool emit_alu_trans_op2_cayman(const nir_alu_instr& alu, EAluOp opcode, S AluInstr::SrcValues srcs(2 * last_slot); PRegister dest = value_factory.dest(alu.dest.dest, k, pin_free); - for (unsigned i = 0; i < last_slot ; ++i) { - srcs[2 * i ] = value_factory.src(src0, k); + for (unsigned i = 0; i < last_slot; ++i) { + srcs[2 * i] = value_factory.src(src0, k); srcs[2 * i + 1] = value_factory.src(src1, k); } - auto ir = new AluInstr(opcode, - dest, srcs, AluInstr::last_write, last_slot); + auto ir = new AluInstr(opcode, dest, srcs, AluInstr::last_write, last_slot); - if (src0.negate) ir->set_alu_flag(alu_src0_neg); - if (src0.abs) ir->set_alu_flag(alu_src0_abs); - if (src1.negate) ir->set_alu_flag(alu_src1_neg); - if (src1.abs) ir->set_alu_flag(alu_src1_abs); - if (alu.dest.saturate) ir->set_alu_flag(alu_dst_clamp); + if (src0.negate) + ir->set_alu_flag(alu_src0_neg); + if (src0.abs) + ir->set_alu_flag(alu_src0_abs); + if (src1.negate) + ir->set_alu_flag(alu_src1_neg); + if (src1.abs) + ir->set_alu_flag(alu_src1_abs); + if (alu.dest.saturate) + ir->set_alu_flag(alu_dst_clamp); ir->set_alu_flag(alu_is_cayman_trans); shader.emit_instruction(ir); } @@ -2512,14 +2917,14 @@ static bool emit_alu_trans_op2_cayman(const nir_alu_instr& alu, EAluOp opcode, S return true; } - -static bool emit_tex_fdd(const nir_alu_instr& alu, TexInstr::Opcode opcode, bool fine, Shader& shader) +static bool +emit_tex_fdd(const nir_alu_instr& alu, TexInstr::Opcode opcode, bool fine, Shader& shader) { auto& value_factory = shader.value_factory(); int ncomp = nir_dest_num_components(alu.dest.dest); - RegisterVec4::Swizzle src_swz = {7,7,7,7}; - RegisterVec4::Swizzle tmp_swz = {7,7,7,7}; + RegisterVec4::Swizzle src_swz = {7, 7, 7, 7}; + RegisterVec4::Swizzle tmp_swz = {7, 7, 7, 7}; for (auto i = 0; i < ncomp; ++i) { src_swz[i] = alu.src[0].swizzle[i]; tmp_swz[i] = i; @@ -2541,7 +2946,7 @@ static bool emit_tex_fdd(const nir_alu_instr& alu, TexInstr::Opcode opcode, bool mv->set_alu_flag(alu_last_instr); auto dst = value_factory.dest_vec4(alu.dest.dest, pin_group); - RegisterVec4::Swizzle dst_swz = {7,7,7,7}; + RegisterVec4::Swizzle dst_swz = {7, 7, 7, 7}; for (auto i = 0; i < ncomp; ++i) { if (alu.dest.write_mask & (1 << i)) dst_swz[i] = i; @@ -2557,7 +2962,8 @@ static bool emit_tex_fdd(const nir_alu_instr& alu, TexInstr::Opcode opcode, bool return true; } -static bool emit_alu_cube(const nir_alu_instr& alu, Shader& shader) +static bool +emit_alu_cube(const nir_alu_instr& alu, Shader& shader) { auto& value_factory = shader.value_factory(); AluInstr *ir = nullptr; @@ -2567,10 +2973,10 @@ static bool emit_alu_cube(const nir_alu_instr& alu, Shader& shader) auto group = new AluGroup(); - for (int i = 0; i < 4; ++i) { + for (int i = 0; i < 4; ++i) { - - ir = new AluInstr(op2_cube, value_factory.dest(alu.dest.dest, i, pin_chan), + ir = new AluInstr(op2_cube, + value_factory.dest(alu.dest.dest, i, pin_chan), value_factory.src(alu.src[0], src0_chan[i]), value_factory.src(alu.src[0], src1_chan[i]), AluInstr::write); @@ -2586,4 +2992,4 @@ const std::set AluInstr::write({alu_write}); const std::set AluInstr::last({alu_last_instr}); const std::set AluInstr::last_write({alu_write, alu_last_instr}); -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_alu.h b/src/gallium/drivers/r600/sfn/sfn_instr_alu.h index 1c282702c64..f2c1928f758 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_alu.h +++ b/src/gallium/drivers/r600/sfn/sfn_instr_alu.h @@ -40,7 +40,6 @@ class ValueFactory; class AluInstr : public Instr { public: - using SrcValues = std::vector>; enum Op2Options { @@ -51,13 +50,7 @@ public: }; static constexpr const AluBankSwizzle bs[6] = { - alu_vec_012, - alu_vec_021, - alu_vec_120, - alu_vec_102, - alu_vec_201, - alu_vec_210 - }; + alu_vec_012, alu_vec_021, alu_vec_120, alu_vec_102, alu_vec_201, alu_vec_210}; static const AluModifiers src_abs_flags[2]; static const AluModifiers src_neg_flags[3]; @@ -65,18 +58,27 @@ public: AluInstr(EAluOp opcode); AluInstr(EAluOp opcode, int chan); - AluInstr(EAluOp opcode, PRegister dest, + AluInstr(EAluOp opcode, + PRegister dest, SrcValues src0, - const std::set& flags, int alu_slot); + const std::set& flags, + int alu_slot); - AluInstr(EAluOp opcode, PRegister dest, PVirtualValue src0, + AluInstr(EAluOp opcode, + PRegister dest, + PVirtualValue src0, const std::set& flags); - AluInstr(EAluOp opcode, PRegister dest, - PVirtualValue src0, PVirtualValue src1, + AluInstr(EAluOp opcode, + PRegister dest, + PVirtualValue src0, + PVirtualValue src1, const std::set& flags); - AluInstr(EAluOp opcode, PRegister dest, PVirtualValue src0, PVirtualValue src1, + AluInstr(EAluOp opcode, + PRegister dest, + PVirtualValue src0, + PVirtualValue src1, PVirtualValue src2, const std::set& flags); @@ -86,8 +88,16 @@ public: void accept(ConstInstrVisitor& visitor) const override; void accept(InstrVisitor& visitor) override; - auto opcode() const {assert(!has_alu_flag(alu_is_lds)); return m_opcode;} - auto lds_opcode() const {assert(has_alu_flag(alu_is_lds)); return m_lds_opcode;} + auto opcode() const + { + assert(!has_alu_flag(alu_is_lds)); + return m_opcode; + } + auto lds_opcode() const + { + assert(has_alu_flag(alu_is_lds)); + return m_lds_opcode; + } bool can_propagate_src() const; bool can_propagate_dest() const; @@ -95,34 +105,42 @@ public: bool replace_source(PRegister old_src, PVirtualValue new_src) override; bool replace_dest(PRegister new_dest, AluInstr *move_instr) override; - void set_op(EAluOp op) {m_opcode = op;} + void set_op(EAluOp op) { m_opcode = op; } - PRegister dest() const {return m_dest;} - unsigned n_sources() const {return m_src.size();} + PRegister dest() const { return m_dest; } + unsigned n_sources() const { return m_src.size(); } - int dest_chan() const {return m_dest ? m_dest->chan() : m_fallback_chan;} + int dest_chan() const { return m_dest ? m_dest->chan() : m_fallback_chan; } - PVirtualValue psrc(unsigned i) {return i < m_src.size() ? m_src[i] : nullptr;} - VirtualValue& src(unsigned i) {assert(i < m_src.size() && m_src[i]); return *m_src[i];} - const VirtualValue& src(unsigned i) const {assert(i < m_src.size() && m_src[i]); return *m_src[i];} + PVirtualValue psrc(unsigned i) { return i < m_src.size() ? m_src[i] : nullptr; } + VirtualValue& src(unsigned i) + { + assert(i < m_src.size() && m_src[i]); + return *m_src[i]; + } + const VirtualValue& src(unsigned i) const + { + assert(i < m_src.size() && m_src[i]); + return *m_src[i]; + } void set_sources(SrcValues src); - const SrcValues& sources() const {return m_src;} + const SrcValues& sources() const { return m_src; } void pin_sources_to_chan(); int register_priority() const; - void reset_alu_flag(AluModifiers flag) {m_alu_flags.reset(flag);} - void set_alu_flag(AluModifiers flag) {m_alu_flags.set(flag);} - bool has_alu_flag(AluModifiers f) const {return m_alu_flags.test(f);} + void reset_alu_flag(AluModifiers flag) { m_alu_flags.reset(flag); } + void set_alu_flag(AluModifiers flag) { m_alu_flags.set(flag); } + bool has_alu_flag(AluModifiers f) const { return m_alu_flags.test(f); } - ECFAluOpCode cf_type() const {return m_cf_type;} - void set_cf_type(ECFAluOpCode cf_type){ m_cf_type = cf_type; } - void set_bank_swizzle(AluBankSwizzle swz) {m_bank_swizzle = swz;} - AluBankSwizzle bank_swizzle() const {return m_bank_swizzle;} + ECFAluOpCode cf_type() const { return m_cf_type; } + void set_cf_type(ECFAluOpCode cf_type) { m_cf_type = cf_type; } + void set_bank_swizzle(AluBankSwizzle swz) { m_bank_swizzle = swz; } + AluBankSwizzle bank_swizzle() const { return m_bank_swizzle; } - void set_index_offset(unsigned offs) {m_idx_offset = offs;} - auto index_offset() const {return m_idx_offset;} + void set_index_offset(unsigned offs) { m_idx_offset = offs; } + auto index_offset() const { return m_idx_offset; } bool is_equal_to(const AluInstr& lhs) const; @@ -131,14 +149,15 @@ public: static const std::map cf_map; static const std::map bank_swizzle_map; - static Instr::Pointer from_string(std::istream &is, ValueFactory& value_factory, AluGroup *); + static Instr::Pointer + from_string(std::istream& is, ValueFactory& value_factory, AluGroup *); static bool from_nir(nir_alu_instr *alu, Shader& shader); - int alu_slots() const {return m_alu_slots;} + int alu_slots() const { return m_alu_slots; } - AluGroup *split(ValueFactory &vf); + AluGroup *split(ValueFactory& vf); - bool end_group() const override { return m_alu_flags.test(alu_last_instr);} + bool end_group() const override { return m_alu_flags.test(alu_last_instr); } static const std::set empty; static const std::set write; @@ -149,16 +168,16 @@ public: void add_extra_dependency(PVirtualValue reg); - void set_required_slots(int nslots) { m_required_slots = nslots;} - unsigned required_slots() const { return m_required_slots;} + void set_required_slots(int nslots) { m_required_slots = nslots; } + unsigned required_slots() const { return m_required_slots; } - void add_priority(int priority) { m_priority += priority;} - int priority() const { return m_priority;} - void inc_priority() { ++m_priority;} + void add_priority(int priority) { m_priority += priority; } + int priority() const { return m_priority; } + void inc_priority() { ++m_priority; } - void set_parent_group(AluGroup *group) { m_parent_group = group;} + void set_parent_group(AluGroup *group) { m_parent_group = group; } - AluInstr *as_alu() override { return this;} + AluInstr *as_alu() override { return this; } uint8_t allowed_dest_chan_mask() const override; @@ -204,21 +223,20 @@ public: void visit(Block *instr) override; void visit(IfInstr *instr) override; - void visit(TexInstr *instr) override {(void)instr;} - void visit(ExportInstr *instr) override {(void)instr;} - void visit(FetchInstr *instr) override {(void)instr;} - void visit(ControlFlowInstr *instr) override {(void)instr;} - void visit(ScratchIOInstr *instr) override {(void)instr;} - void visit(StreamOutInstr *instr) override {(void)instr;} - void visit(MemRingOutInstr *instr) override {(void)instr;} - void visit(EmitVertexInstr *instr) override {(void)instr;} - void visit(GDSInstr *instr) override {(void)instr;}; - void visit(WriteTFInstr *instr) override {(void)instr;}; - void visit(LDSAtomicInstr *instr) override {(void)instr;}; - void visit(LDSReadInstr *instr) override {(void)instr;}; - void visit(RatInstr *instr) override {(void)instr;}; + void visit(TexInstr *instr) override { (void)instr; } + void visit(ExportInstr *instr) override { (void)instr; } + void visit(FetchInstr *instr) override { (void)instr; } + void visit(ControlFlowInstr *instr) override { (void)instr; } + void visit(ScratchIOInstr *instr) override { (void)instr; } + void visit(StreamOutInstr *instr) override { (void)instr; } + void visit(MemRingOutInstr *instr) override { (void)instr; } + void visit(EmitVertexInstr *instr) override { (void)instr; } + void visit(GDSInstr *instr) override { (void)instr; }; + void visit(WriteTFInstr *instr) override { (void)instr; }; + void visit(LDSAtomicInstr *instr) override { (void)instr; }; + void visit(LDSReadInstr *instr) override { (void)instr; }; + void visit(RatInstr *instr) override { (void)instr; }; }; - -} +} // namespace r600 #endif // INSTRALU_H diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_alugroup.cpp b/src/gallium/drivers/r600/sfn/sfn_instr_alugroup.cpp index 9f2cc872dcf..69fb7193c17 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_alugroup.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_instr_alugroup.cpp @@ -25,20 +25,20 @@ */ #include "sfn_instr_alugroup.h" + +#include "sfn_debug.h" #include "sfn_instr_export.h" #include "sfn_instr_mem.h" #include "sfn_instr_tex.h" -#include "sfn_debug.h" + #include namespace r600 { -AluGroup::AluGroup() -{ - std::fill(m_slots.begin(), m_slots.end(), nullptr); -} +AluGroup::AluGroup() { std::fill(m_slots.begin(), m_slots.end(), nullptr); } -static bool is_kill(EAluOp op) +static bool +is_kill(EAluOp op) { switch (op) { case op2_kille: @@ -57,7 +57,8 @@ static bool is_kill(EAluOp op) } } -bool AluGroup::add_instruction(AluInstr *instr) +bool +AluGroup::add_instruction(AluInstr *instr) { /* we can only schedule one op that accesses LDS or the LDS read queue */ @@ -84,8 +85,7 @@ bool AluGroup::add_instruction(AluInstr *instr) auto opinfo = alu_ops.find(instr->opcode()); assert(opinfo != alu_ops.end()); - if (s_max_slots > 4 && - opinfo->second.can_channel(AluOp::t, s_chip_class) && + if (s_max_slots > 4 && opinfo->second.can_channel(AluOp::t, s_chip_class) && add_trans_instructions(instr)) { instr->set_parent_group(this); if (is_kill(instr->opcode())) @@ -96,7 +96,8 @@ bool AluGroup::add_instruction(AluInstr *instr) return false; } -bool AluGroup::add_trans_instructions(AluInstr *instr) +bool +AluGroup::add_trans_instructions(AluInstr *instr) { if (m_slots[4] || s_max_slots < 5) return false; @@ -132,7 +133,7 @@ bool AluGroup::add_trans_instructions(AluInstr *instr) } } - for (AluBankSwizzle i = sq_alu_scl_201; i != sq_alu_scl_unknown ; ++i) { + for (AluBankSwizzle i = sq_alu_scl_201; i != sq_alu_scl_unknown; ++i) { AluReadportReservation readports_evaluator = m_readports_evaluator; if (readports_evaluator.schedule_trans_instruction(*instr, i)) { m_readports_evaluator = readports_evaluator; @@ -143,25 +144,26 @@ bool AluGroup::add_trans_instructions(AluInstr *instr) /* We added a vector op in the trans channel, so we have to * make sure the corresponding vector channel is used */ if (!instr->has_alu_flag(alu_is_trans) && !m_slots[instr->dest_chan()]) - m_slots[instr->dest_chan()] = - new AluInstr(op0_nop, instr->dest_chan()); + m_slots[instr->dest_chan()] = new AluInstr(op0_nop, instr->dest_chan()); return true; } } return false; } -int AluGroup::free_slots() const +int +AluGroup::free_slots() const { int free_mask = 0; - for(int i = 0; i < s_max_slots; ++i) { + for (int i = 0; i < s_max_slots; ++i) { if (!m_slots[i]) free_mask |= 1 << i; } return free_mask; } -bool AluGroup::add_vec_instructions(AluInstr *instr) +bool +AluGroup::add_vec_instructions(AluInstr *instr) { if (!update_indirect_access(instr)) return false; @@ -187,7 +189,7 @@ bool AluGroup::add_vec_instructions(AluInstr *instr) if (!m_slots[preferred_chan]) { if (instr->bank_swizzle() != alu_vec_unknown) { if (try_readport(instr, instr->bank_swizzle())) - return true; + return true; } else { for (AluBankSwizzle i = alu_vec_012; i != alu_vec_unknown; ++i) { if (try_readport(instr, i)) @@ -207,8 +209,7 @@ bool AluGroup::add_vec_instructions(AluInstr *instr) } int free_chan = 0; - while (free_chan < 4 && (m_slots[free_chan] || - !(free_mask & (1 << free_chan)))) + while (free_chan < 4 && (m_slots[free_chan] || !(free_mask & (1 << free_chan)))) free_chan++; if (free_chan < 4) { @@ -229,7 +230,8 @@ bool AluGroup::add_vec_instructions(AluInstr *instr) return false; } -bool AluGroup::try_readport(AluInstr *instr, AluBankSwizzle cycle) +bool +AluGroup::try_readport(AluInstr *instr, AluBankSwizzle cycle) { int preferred_chan = instr->dest_chan(); AluReadportReservation readports_evaluator = m_readports_evaluator; @@ -251,9 +253,10 @@ bool AluGroup::try_readport(AluInstr *instr, AluBankSwizzle cycle) return false; } -bool AluGroup::update_indirect_access(AluInstr *instr) +bool +AluGroup::update_indirect_access(AluInstr *instr) { - auto [indirect_addr, for_src, is_index ] = instr->indirect_addr(); + auto [indirect_addr, for_src, is_index] = instr->indirect_addr(); if (indirect_addr) { if (!m_addr_used) { @@ -268,17 +271,20 @@ bool AluGroup::update_indirect_access(AluInstr *instr) return true; } -void AluGroup::accept(ConstInstrVisitor& visitor) const +void +AluGroup::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void AluGroup::accept(InstrVisitor& visitor) +void +AluGroup::accept(InstrVisitor& visitor) { visitor.visit(this); } -void AluGroup::set_scheduled() +void +AluGroup::set_scheduled() { for (int i = 0; i < s_max_slots; ++i) { if (m_slots[i]) @@ -286,7 +292,8 @@ void AluGroup::set_scheduled() } } -void AluGroup::fix_last_flag() +void +AluGroup::fix_last_flag() { bool last_seen = false; for (int i = s_max_slots - 1; i >= 0; --i) { @@ -301,7 +308,8 @@ void AluGroup::fix_last_flag() } } -bool AluGroup::is_equal_to(const AluGroup& other) const +bool +AluGroup::is_equal_to(const AluGroup& other) const { for (int i = 0; i < s_max_slots; ++i) { if (!other.m_slots[i]) { @@ -321,7 +329,8 @@ bool AluGroup::is_equal_to(const AluGroup& other) const return true; } -bool AluGroup::has_lds_group_end() const +bool +AluGroup::has_lds_group_end() const { for (int i = 0; i < s_max_slots; ++i) { if (m_slots[i] && m_slots[i]->has_alu_flag(alu_lds_group_end)) @@ -330,7 +339,8 @@ bool AluGroup::has_lds_group_end() const return false; } -bool AluGroup::do_ready() const +bool +AluGroup::do_ready() const { for (int i = 0; i < s_max_slots; ++i) { if (m_slots[i] && !m_slots[i]->ready()) @@ -339,7 +349,8 @@ bool AluGroup::do_ready() const return true; } -void AluGroup::forward_set_blockid(int id, int index) +void +AluGroup::forward_set_blockid(int id, int index) { for (int i = 0; i < s_max_slots; ++i) { if (m_slots[i]) { @@ -348,7 +359,8 @@ void AluGroup::forward_set_blockid(int id, int index) } } -uint32_t AluGroup::slots() const +uint32_t +AluGroup::slots() const { uint32_t result = (m_readports_evaluator.m_nliterals + 1) >> 1; for (int i = 0; i < s_max_slots; ++i) { @@ -364,7 +376,8 @@ uint32_t AluGroup::slots() const return result; } -void AluGroup::do_print(std::ostream& os) const +void +AluGroup::do_print(std::ostream& os) const { const char slotname[] = "xyzwt"; @@ -383,7 +396,8 @@ void AluGroup::do_print(std::ostream& os) const os << "ALU_GROUP_END"; } -AluInstr::SrcValues AluGroup::get_kconsts() const +AluInstr::SrcValues +AluGroup::get_kconsts() const { AluInstr::SrcValues result; @@ -397,12 +411,13 @@ AluInstr::SrcValues AluGroup::get_kconsts() const return result; } -void AluGroup::set_chipclass(r600_chip_class chip_class) +void +AluGroup::set_chipclass(r600_chip_class chip_class) { s_chip_class = chip_class; - s_max_slots = chip_class == ISA_CC_CAYMAN ? 4 : 5; + s_max_slots = chip_class == ISA_CC_CAYMAN ? 4 : 5; } int AluGroup::s_max_slots = 5; r600_chip_class AluGroup::s_chip_class = ISA_CC_EVERGREEN; -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_alugroup.h b/src/gallium/drivers/r600/sfn/sfn_instr_alugroup.h index 2ff31628c2c..4a2abc154e1 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_alugroup.h +++ b/src/gallium/drivers/r600/sfn/sfn_instr_alugroup.h @@ -27,13 +27,12 @@ #ifndef ALUGROUP_H #define ALUGROUP_H -#include "sfn_instr_alu.h" #include "sfn_alu_readport_validation.h" +#include "sfn_instr_alu.h" namespace r600 { -class AluGroup : public Instr -{ +class AluGroup : public Instr { public: using Slots = std::array; @@ -51,16 +50,16 @@ public: void accept(ConstInstrVisitor& visitor) const override; void accept(InstrVisitor& visitor) override; - auto begin() {return m_slots.begin(); } - auto end() {return m_slots.begin() + s_max_slots; } - auto begin() const {return m_slots.begin(); } - auto end() const {return m_slots.begin() + s_max_slots; } + auto begin() { return m_slots.begin(); } + auto end() { return m_slots.begin() + s_max_slots; } + auto begin() const { return m_slots.begin(); } + auto end() const { return m_slots.begin() + s_max_slots; } bool end_group() const override { return true; } void set_scheduled() override; - void set_nesting_depth(int depth) {m_nesting_depth = depth;} + void set_nesting_depth(int depth) { m_nesting_depth = depth; } void fix_last_flag(); @@ -68,26 +67,29 @@ public: int free_slots() const; - auto addr() const {return std::make_pair(m_addr_used, m_addr_is_index);} + auto addr() const { return std::make_pair(m_addr_used, m_addr_is_index); } uint32_t slots() const override; AluInstr::SrcValues get_kconsts() const; - bool has_lds_group_start() const { return m_slots[0] ? - m_slots[0]->has_alu_flag(alu_lds_group_start) : false;} + bool has_lds_group_start() const + { + return m_slots[0] ? m_slots[0]->has_alu_flag(alu_lds_group_start) : false; + } bool has_lds_group_end() const; const auto& readport_reserer() const { return m_readports_evaluator; } - void set_readport_reserer(const AluReadportReservation& rr) { - m_readports_evaluator = rr; + void set_readport_reserer(const AluReadportReservation& rr) + { + m_readports_evaluator = rr; }; - static bool has_t() { return s_max_slots == 5;} + static bool has_t() { return s_max_slots == 5; } - bool addr_for_src() const { return m_addr_for_src;} - bool has_kill_op() const {return m_has_kill_op;} + bool addr_for_src() const { return m_addr_for_src; } + bool has_kill_op() const { return m_has_kill_op; } private: void forward_set_blockid(int id, int index) override; @@ -115,7 +117,6 @@ private: bool m_has_kill_op{false}; }; - -} +} // namespace r600 #endif // ALUGROUP_H diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_controlflow.cpp b/src/gallium/drivers/r600/sfn/sfn_instr_controlflow.cpp index 1a4ee143704..ca837aa28cc 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_controlflow.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_instr_controlflow.cpp @@ -31,48 +31,67 @@ namespace r600 { ControlFlowInstr::ControlFlowInstr(CFType type): - m_type(type) + m_type(type) { } -bool ControlFlowInstr::do_ready() const +bool +ControlFlowInstr::do_ready() const { /* Have to rework this, but the CF should always */ return true; } - -bool ControlFlowInstr::is_equal_to(const ControlFlowInstr& rhs) const +bool +ControlFlowInstr::is_equal_to(const ControlFlowInstr& rhs) const { return m_type == rhs.m_type; } -void ControlFlowInstr::accept(ConstInstrVisitor& visitor) const +void +ControlFlowInstr::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void ControlFlowInstr::accept(InstrVisitor& visitor) +void +ControlFlowInstr::accept(InstrVisitor& visitor) { visitor.visit(this); } -void ControlFlowInstr::do_print(std::ostream& os) const +void +ControlFlowInstr::do_print(std::ostream& os) const { switch (m_type) { - case cf_else: os << "ELSE"; break; - case cf_endif: os << "ENDIF";break; - case cf_loop_begin: os << "LOOP_BEGIN"; break; - case cf_loop_end: os << "LOOP_END"; break; - case cf_loop_break: os << "BREAK"; break; - case cf_loop_continue: os << "CONTINUE"; break; - case cf_wait_ack: os << "WAIT_ACK"; break; + case cf_else: + os << "ELSE"; + break; + case cf_endif: + os << "ENDIF"; + break; + case cf_loop_begin: + os << "LOOP_BEGIN"; + break; + case cf_loop_end: + os << "LOOP_END"; + break; + case cf_loop_break: + os << "BREAK"; + break; + case cf_loop_continue: + os << "CONTINUE"; + break; + case cf_wait_ack: + os << "WAIT_ACK"; + break; default: unreachable("Unknown CF type"); } } -Instr::Pointer ControlFlowInstr::from_string(std::string type_str) +Instr::Pointer +ControlFlowInstr::from_string(std::string type_str) { if (type_str == "ELSE") return new ControlFlowInstr(cf_else); @@ -92,86 +111,98 @@ Instr::Pointer ControlFlowInstr::from_string(std::string type_str) return nullptr; } -int ControlFlowInstr::nesting_corr() const +int +ControlFlowInstr::nesting_corr() const { switch (m_type) { case cf_else: case cf_endif: - case cf_loop_end: return -1; + case cf_loop_end: + return -1; default: return 0; } } -int ControlFlowInstr::nesting_offset() const +int +ControlFlowInstr::nesting_offset() const { switch (m_type) { case cf_endif: - case cf_loop_end: return -1; - case cf_loop_begin: return 1; + case cf_loop_end: + return -1; + case cf_loop_begin: + return 1; default: return 0; } } IfInstr::IfInstr(AluInstr *pred): - m_predicate(pred) + m_predicate(pred) { assert(pred); } -IfInstr::IfInstr(const IfInstr& orig) -{ - m_predicate = new AluInstr(*orig.m_predicate); -} +IfInstr::IfInstr(const IfInstr& orig) { m_predicate = new AluInstr(*orig.m_predicate); } -bool IfInstr::is_equal_to(const IfInstr& rhs) const +bool +IfInstr::is_equal_to(const IfInstr& rhs) const { return m_predicate->equal_to(*rhs.m_predicate); } -void IfInstr::accept(ConstInstrVisitor& visitor) const +void +IfInstr::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void IfInstr::accept(InstrVisitor& visitor) +void +IfInstr::accept(InstrVisitor& visitor) { visitor.visit(this); } -bool IfInstr::replace_source(PRegister old_src, PVirtualValue new_src) +bool +IfInstr::replace_source(PRegister old_src, PVirtualValue new_src) { return m_predicate->replace_source(old_src, new_src); } -bool IfInstr::do_ready() const +bool +IfInstr::do_ready() const { return m_predicate->ready(); } -void IfInstr::forward_set_scheduled() +void +IfInstr::forward_set_scheduled() { m_predicate->set_scheduled(); } -void IfInstr::forward_set_blockid(int id, int index) +void +IfInstr::forward_set_blockid(int id, int index) { m_predicate->set_blockid(id, index); } -void IfInstr::do_print(std::ostream& os) const +void +IfInstr::do_print(std::ostream& os) const { os << "IF (( " << *m_predicate << " ))"; } -void IfInstr::set_predicate(AluInstr *new_predicate) +void +IfInstr::set_predicate(AluInstr *new_predicate) { m_predicate = new_predicate; m_predicate->set_blockid(block_id(), index()); } -Instr::Pointer IfInstr::from_string(std::istream &is, ValueFactory& value_factory) +Instr::Pointer +IfInstr::from_string(std::istream& is, ValueFactory& value_factory) { std::string pred_start; is >> pred_start; @@ -196,7 +227,7 @@ Instr::Pointer IfInstr::from_string(std::istream &is, ValueFactory& value_factor return nullptr; auto pred = AluInstr::from_string(bufstr, value_factory, nullptr); - return new IfInstr(static_cast(pred)); + return new IfInstr(static_cast(pred)); } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_controlflow.h b/src/gallium/drivers/r600/sfn/sfn_instr_controlflow.h index 356d913b676..11d73039639 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_controlflow.h +++ b/src/gallium/drivers/r600/sfn/sfn_instr_controlflow.h @@ -31,8 +31,7 @@ namespace r600 { -class ControlFlowInstr : public Instr -{ +class ControlFlowInstr : public Instr { public: enum CFType { cf_else, @@ -54,13 +53,13 @@ public: void accept(ConstInstrVisitor& visitor) const override; void accept(InstrVisitor& visitor) override; - CFType cf_type() const { return m_type;} + CFType cf_type() const { return m_type; } int nesting_corr() const override; static Instr::Pointer from_string(std::string type_str); - bool end_block() const override { return true;} + bool end_block() const override { return true; } int nesting_offset() const override; @@ -73,7 +72,6 @@ private: class IfInstr : public Instr { public: - IfInstr(AluInstr *pred); IfInstr(const IfInstr& orig); @@ -88,10 +86,10 @@ public: bool replace_source(PRegister old_src, PVirtualValue new_src) override; - static Instr::Pointer from_string(std::istream &is, ValueFactory& value_factory); + static Instr::Pointer from_string(std::istream& is, ValueFactory& value_factory); - bool end_block() const override { return true;} - int nesting_offset() const override { return 1;} + bool end_block() const override { return true; } + int nesting_offset() const override { return 1; } private: bool do_ready() const override; @@ -102,6 +100,6 @@ private: AluInstr *m_predicate; }; -} +} // namespace r600 #endif // CONTROLFLOWINSTR_H diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_export.cpp b/src/gallium/drivers/r600/sfn/sfn_instr_export.cpp index dac0e101855..a85fa3bed72 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_export.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_instr_export.cpp @@ -25,6 +25,7 @@ */ #include "sfn_instr_export.h" + #include "sfn_valuefactory.h" #include @@ -33,7 +34,8 @@ namespace r600 { using std::string; -static char *writemask_to_swizzle(int writemask, char *buf) +static char * +writemask_to_swizzle(int writemask, char *buf) { const char *swz = "xyzw"; for (int i = 0; i < 4; ++i) { @@ -43,87 +45,98 @@ static char *writemask_to_swizzle(int writemask, char *buf) } WriteOutInstr::WriteOutInstr(const RegisterVec4& value): - m_value(value) + m_value(value) { m_value.add_use(this); set_always_keep(); } -void WriteOutInstr::override_chan(int i, int chan) +void +WriteOutInstr::override_chan(int i, int chan) { - m_value.set_value(i, - new Register(m_value[i]->sel(), chan, - m_value[i]->pin())); + m_value.set_value(i, new Register(m_value[i]->sel(), chan, m_value[i]->pin())); } ExportInstr::ExportInstr(ExportType type, unsigned loc, const RegisterVec4& value): - WriteOutInstr(value), - m_type(type), - m_loc(loc), - m_is_last(false) + WriteOutInstr(value), + m_type(type), + m_loc(loc), + m_is_last(false) { } -void ExportInstr::accept(ConstInstrVisitor& visitor) const +void +ExportInstr::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void ExportInstr::accept(InstrVisitor& visitor) +void +ExportInstr::accept(InstrVisitor& visitor) { visitor.visit(this); } - -bool ExportInstr::is_equal_to(const ExportInstr& lhs) const +bool +ExportInstr::is_equal_to(const ExportInstr& lhs) const { return - (m_type == lhs.m_type && - m_loc == lhs.m_loc && - value() == lhs.value() && - m_is_last == lhs.m_is_last); + (m_type == lhs.m_type && m_loc == lhs.m_loc && value() == lhs.value() && + m_is_last == lhs.m_is_last); } -ExportInstr::ExportType ExportInstr::type_from_string(const std::string& s) +ExportInstr::ExportType +ExportInstr::type_from_string(const std::string& s) { (void)s; return param; } -void ExportInstr::do_print(std::ostream& os) const +void +ExportInstr::do_print(std::ostream& os) const { os << "EXPORT"; if (m_is_last) os << "_DONE"; switch (m_type) { - case param: os << " PARAM "; break; - case pos: os << " POS "; break; - case pixel: os << " PIXEL "; break; + case param: + os << " PARAM "; + break; + case pos: + os << " POS "; + break; + case pixel: + os << " PIXEL "; + break; } os << m_loc << " "; value().print(os); } -bool ExportInstr::do_ready() const +bool +ExportInstr::do_ready() const { return value().ready(block_id(), index()); } -Instr::Pointer ExportInstr::from_string(std::istream& is, ValueFactory& vf) +Instr::Pointer +ExportInstr::from_string(std::istream& is, ValueFactory& vf) { return from_string_impl(is, vf); } -Instr::Pointer ExportInstr::last_from_string(std::istream& is, ValueFactory &vf) +Instr::Pointer +ExportInstr::last_from_string(std::istream& is, ValueFactory& vf) { auto result = from_string_impl(is, vf); result->set_is_last_export(true); return result; } -ExportInstr::Pointer ExportInstr::from_string_impl(std::istream& is, ValueFactory &vf) +ExportInstr::Pointer +ExportInstr::from_string_impl(std::istream& is, ValueFactory& vf) { string typestr; int pos; @@ -144,24 +157,29 @@ ExportInstr::Pointer ExportInstr::from_string_impl(std::istream& is, ValueFactor RegisterVec4 value = vf.src_vec4_from_string(value_str); - return new ExportInstr( type, pos, value); + return new ExportInstr(type, pos, value); } -uint8_t ExportInstr::allowed_dest_chan_mask() const +uint8_t +ExportInstr::allowed_dest_chan_mask() const { return value().free_chan_mask(); } -ScratchIOInstr::ScratchIOInstr(const RegisterVec4& value, PRegister addr, - int align, int align_offset, int writemask, - int array_size, bool is_read): - WriteOutInstr(value), - m_address(addr), - m_align(align), - m_align_offset(align_offset), - m_writemask(writemask), - m_array_size(array_size - 1), - m_read(is_read) +ScratchIOInstr::ScratchIOInstr(const RegisterVec4& value, + PRegister addr, + int align, + int align_offset, + int writemask, + int array_size, + bool is_read): + WriteOutInstr(value), + m_address(addr), + m_align(align), + m_align_offset(align_offset), + m_writemask(writemask), + m_array_size(array_size - 1), + m_read(is_read) { addr->add_use(this); if (m_read) { @@ -170,15 +188,18 @@ ScratchIOInstr::ScratchIOInstr(const RegisterVec4& value, PRegister addr, } } -ScratchIOInstr::ScratchIOInstr(const RegisterVec4& value, int loc, - int align, int align_offset,int writemask, +ScratchIOInstr::ScratchIOInstr(const RegisterVec4& value, + int loc, + int align, + int align_offset, + int writemask, bool is_read): - WriteOutInstr(value), - m_loc(loc), - m_align(align), - m_align_offset(align_offset), - m_writemask(writemask), - m_read(is_read) + WriteOutInstr(value), + m_loc(loc), + m_align(align), + m_align_offset(align_offset), + m_writemask(writemask), + m_read(is_read) { if (m_read) { @@ -187,35 +208,36 @@ ScratchIOInstr::ScratchIOInstr(const RegisterVec4& value, int loc, } } -void ScratchIOInstr::accept(ConstInstrVisitor& visitor) const +void +ScratchIOInstr::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void ScratchIOInstr::accept(InstrVisitor& visitor) +void +ScratchIOInstr::accept(InstrVisitor& visitor) { visitor.visit(this); } -bool ScratchIOInstr::is_equal_to(const ScratchIOInstr& lhs) const +bool +ScratchIOInstr::is_equal_to(const ScratchIOInstr& lhs) const { if (m_address) { if (!lhs.m_address) return false; - if (! m_address->equal_to(*lhs.m_address)) + if (!m_address->equal_to(*lhs.m_address)) return false; } else if (lhs.m_address) return false; - return m_loc == lhs.m_loc && - m_align == lhs.m_align && - m_align_offset == lhs.m_align_offset && - m_writemask == lhs.m_writemask && - m_array_size == lhs.m_array_size && - value().sel() == lhs.value().sel(); + return m_loc == lhs.m_loc && m_align == lhs.m_align && + m_align_offset == lhs.m_align_offset && m_writemask == lhs.m_writemask && + m_array_size == lhs.m_array_size && value().sel() == lhs.value().sel(); } -bool ScratchIOInstr::do_ready() const +bool +ScratchIOInstr::do_ready() const { bool address_ready = !m_address || m_address->ready(block_id(), index()); if (is_read()) @@ -224,31 +246,33 @@ bool ScratchIOInstr::do_ready() const return address_ready && value().ready(block_id(), index()); } -void ScratchIOInstr::do_print(std::ostream& os) const +void +ScratchIOInstr::do_print(std::ostream& os) const { char buf[6] = {0}; os << (is_read() ? "READ_SCRATCH " : "WRITE_SCRATCH "); if (is_read()) { - os << (value()[0]->is_ssa() ? " S" : " R") - << value().sel() << "." << writemask_to_swizzle(m_writemask, buf) - << " "; + os << (value()[0]->is_ssa() ? " S" : " R") << value().sel() << "." + << writemask_to_swizzle(m_writemask, buf) << " "; } if (m_address) - os << "@" << *m_address << "[" << m_array_size + 1<<"]"; + os << "@" << *m_address << "[" << m_array_size + 1 << "]"; else os << m_loc; if (!is_read()) - os << (value()[0]->is_ssa() ? " S" : " R") - << value().sel() << "." << writemask_to_swizzle(m_writemask, buf); + os << (value()[0]->is_ssa() ? " S" : " R") << value().sel() << "." + << writemask_to_swizzle(m_writemask, buf); - os << " " << "AL:" << m_align << " ALO:" << m_align_offset; + os << " " + << "AL:" << m_align << " ALO:" << m_align_offset; } -auto ScratchIOInstr::from_string(std::istream& is, ValueFactory &vf) -> Pointer +auto +ScratchIOInstr::from_string(std::istream& is, ValueFactory& vf) -> Pointer { string loc_str; string value_str; @@ -259,7 +283,7 @@ auto ScratchIOInstr::from_string(std::istream& is, ValueFactory &vf) -> Pointer int array_size = 0; PVirtualValue addr_reg = nullptr; - is >> loc_str >> value_str >> align_str >> align_offset_str; + is >> loc_str >> value_str >> align_str >> align_offset_str; std::istringstream loc_ss(loc_str); @@ -290,34 +314,47 @@ auto ScratchIOInstr::from_string(std::istream& is, ValueFactory &vf) -> Pointer loc_ss >> array_size; loc_ss >> c; assert(c == ']'); - return new ScratchIOInstr(value, addr_reg->as_register(), align, align_offset, writemask, array_size); + return new ScratchIOInstr( + value, addr_reg->as_register(), align, align_offset, writemask, array_size); } else { loc_ss >> offset; return new ScratchIOInstr(value, offset, align, align_offset, writemask); } } -StreamOutInstr::StreamOutInstr(const RegisterVec4& value, int num_components, - int array_base, int comp_mask, int out_buffer, - int stream): - WriteOutInstr(value), - m_element_size(num_components == 3 ? 3 : num_components - 1), - m_array_base(array_base), - m_writemask(comp_mask), - m_output_buffer(out_buffer), - m_stream(stream) +StreamOutInstr::StreamOutInstr(const RegisterVec4& value, + int num_components, + int array_base, + int comp_mask, + int out_buffer, + int stream): + WriteOutInstr(value), + m_element_size(num_components == 3 ? 3 : num_components - 1), + m_array_base(array_base), + m_writemask(comp_mask), + m_output_buffer(out_buffer), + m_stream(stream) { } -unsigned StreamOutInstr::op(amd_gfx_level gfx_level) const +unsigned +StreamOutInstr::op(amd_gfx_level gfx_level) const { int op = 0; if (gfx_level >= EVERGREEN) { switch (m_output_buffer) { - case 0: op = CF_OP_MEM_STREAM0_BUF0; break; - case 1: op = CF_OP_MEM_STREAM0_BUF1; break; - case 2: op = CF_OP_MEM_STREAM0_BUF2; break; - case 3: op = CF_OP_MEM_STREAM0_BUF3; break; + case 0: + op = CF_OP_MEM_STREAM0_BUF0; + break; + case 1: + op = CF_OP_MEM_STREAM0_BUF1; + break; + case 2: + op = CF_OP_MEM_STREAM0_BUF2; + break; + case 3: + op = CF_OP_MEM_STREAM0_BUF3; + break; } return 4 * m_stream + op; } else { @@ -326,95 +363,99 @@ unsigned StreamOutInstr::op(amd_gfx_level gfx_level) const } } -bool StreamOutInstr::is_equal_to(const StreamOutInstr& oth) const +bool +StreamOutInstr::is_equal_to(const StreamOutInstr& oth) const { - return value() == oth.value() && - m_element_size == oth.m_element_size && - m_burst_count == oth.m_burst_count && - m_array_base == oth.m_array_base && - m_array_size == oth.m_array_size && - m_writemask == oth.m_writemask && - m_output_buffer == oth.m_output_buffer && - m_stream == oth.m_stream; + return value() == oth.value() && m_element_size == oth.m_element_size && + m_burst_count == oth.m_burst_count && m_array_base == oth.m_array_base && + m_array_size == oth.m_array_size && m_writemask == oth.m_writemask && + m_output_buffer == oth.m_output_buffer && m_stream == oth.m_stream; } -void StreamOutInstr::do_print(std::ostream& os) const +void +StreamOutInstr::do_print(std::ostream& os) const { - os << "WRITE STREAM(" << m_stream << ") " << value() - << " ES:" << m_element_size - << " BC:" << m_burst_count - << " BUF:" << m_output_buffer - << " ARRAY:" << m_array_base; + os << "WRITE STREAM(" << m_stream << ") " << value() << " ES:" << m_element_size + << " BC:" << m_burst_count << " BUF:" << m_output_buffer + << " ARRAY:" << m_array_base; if (m_array_size != 0xfff) os << "+" << m_array_size; } -bool StreamOutInstr::do_ready() const +bool +StreamOutInstr::do_ready() const { return value().ready(block_id(), index()); } -void StreamOutInstr::accept(ConstInstrVisitor& visitor) const +void +StreamOutInstr::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void StreamOutInstr::accept(InstrVisitor& visitor) +void +StreamOutInstr::accept(InstrVisitor& visitor) { visitor.visit(this); } - -MemRingOutInstr::MemRingOutInstr(ECFOpCode ring, EMemWriteType type, - const RegisterVec4& value, - unsigned base_addr, unsigned ncomp, - PRegister index): - WriteOutInstr(value), - m_ring_op(ring), - m_type(type), - m_base_address(base_addr), - m_num_comp(ncomp), - m_export_index(index) +MemRingOutInstr::MemRingOutInstr(ECFOpCode ring, + EMemWriteType type, + const RegisterVec4& value, + unsigned base_addr, + unsigned ncomp, + PRegister index): + WriteOutInstr(value), + m_ring_op(ring), + m_type(type), + m_base_address(base_addr), + m_num_comp(ncomp), + m_export_index(index) { - assert(m_ring_op == cf_mem_ring || m_ring_op == cf_mem_ring1|| - m_ring_op == cf_mem_ring2 || m_ring_op == cf_mem_ring3); + assert(m_ring_op == cf_mem_ring || m_ring_op == cf_mem_ring1 || + m_ring_op == cf_mem_ring2 || m_ring_op == cf_mem_ring3); assert(m_num_comp <= 4); if (m_export_index) m_export_index->add_use(this); } -unsigned MemRingOutInstr::ncomp() const +unsigned +MemRingOutInstr::ncomp() const { switch (m_num_comp) { - case 1: return 0; - case 2: return 1; + case 1: + return 0; + case 2: + return 1; case 3: - case 4: return 3; + case 4: + return 3; default: assert(0); } return 3; } -bool MemRingOutInstr::is_equal_to(const MemRingOutInstr& oth) const +bool +MemRingOutInstr::is_equal_to(const MemRingOutInstr& oth) const { - bool equal = value() == oth.value() && - m_ring_op == oth.m_ring_op && - m_type == oth.m_type && - m_num_comp == oth.m_num_comp && + bool equal = value() == oth.value() && m_ring_op == oth.m_ring_op && + m_type == oth.m_type && m_num_comp == oth.m_num_comp && m_base_address == oth.m_base_address; if (m_type == mem_write_ind || m_type == mem_write_ind_ack) equal &= (*m_export_index == *oth.m_export_index); return equal; - } -static const char *write_type_str[4] = {"WRITE", "WRITE_IDX", "WRITE_ACK", "WRITE_IDX_ACK" }; -void MemRingOutInstr::do_print(std::ostream& os) const +static const char *write_type_str[4] = { + "WRITE", "WRITE_IDX", "WRITE_ACK", "WRITE_IDX_ACK"}; +void +MemRingOutInstr::do_print(std::ostream& os) const { os << "MEM_RING " << (m_ring_op == cf_mem_ring ? 0 : m_ring_op - cf_mem_ring1 + 1); @@ -425,7 +466,8 @@ void MemRingOutInstr::do_print(std::ostream& os) const os << " ES:" << m_num_comp; } -void MemRingOutInstr::patch_ring(int stream, PRegister index) +void +MemRingOutInstr::patch_ring(int stream, PRegister index) { const ECFOpCode ring_op[4] = {cf_mem_ring, cf_mem_ring1, cf_mem_ring2, cf_mem_ring3}; @@ -434,7 +476,8 @@ void MemRingOutInstr::patch_ring(int stream, PRegister index) m_export_index = index; } -bool MemRingOutInstr::do_ready() const +bool +MemRingOutInstr::do_ready() const { if (m_export_index && !m_export_index->ready(block_id(), index())) return false; @@ -442,25 +485,27 @@ bool MemRingOutInstr::do_ready() const return value().ready(block_id(), index()); } -void MemRingOutInstr::accept(ConstInstrVisitor& visitor) const +void +MemRingOutInstr::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void MemRingOutInstr::accept(InstrVisitor& visitor) +void +MemRingOutInstr::accept(InstrVisitor& visitor) { visitor.visit(this); } -static const std::map type_lookop = -{ - {"WRITE", MemRingOutInstr::mem_write}, - {"WRITE_IDX", MemRingOutInstr::mem_write_ind}, - {"WRITE_ACK", MemRingOutInstr::mem_write_ack}, +static const std::map type_lookop = { + {"WRITE", MemRingOutInstr::mem_write }, + {"WRITE_IDX", MemRingOutInstr::mem_write_ind }, + {"WRITE_ACK", MemRingOutInstr::mem_write_ack }, {"WRITE_IDX_ACK", MemRingOutInstr::mem_write_ind_ack} }; -auto MemRingOutInstr::from_string(std::istream& is, ValueFactory &vf) -> Pointer +auto +MemRingOutInstr::from_string(std::istream& is, ValueFactory& vf) -> Pointer { string type_str; @@ -482,7 +527,7 @@ auto MemRingOutInstr::from_string(std::istream& is, ValueFactory &vf) -> Pointer char c; string index_str; is >> c >> index_str; - assert('@' == c ); + assert('@' == c); index = vf.src_from_string(index_str); } @@ -496,44 +541,48 @@ auto MemRingOutInstr::from_string(std::istream& is, ValueFactory &vf) -> Pointer ECFOpCode opcodes[4] = {cf_mem_ring, cf_mem_ring1, cf_mem_ring2, cf_mem_ring3}; assert(ring < 4); - return new MemRingOutInstr(opcodes[ring], type, value, base_address, num_comp, index->as_register()); + return new MemRingOutInstr( + opcodes[ring], type, value, base_address, num_comp, index->as_register()); } EmitVertexInstr::EmitVertexInstr(int stream, bool cut): - m_stream(stream), - m_cut(cut) + m_stream(stream), + m_cut(cut) { - } - -bool EmitVertexInstr::is_equal_to(const EmitVertexInstr& oth) const +bool +EmitVertexInstr::is_equal_to(const EmitVertexInstr& oth) const { - return oth.m_stream == m_stream && - oth.m_cut == m_cut; + return oth.m_stream == m_stream && oth.m_cut == m_cut; } -void EmitVertexInstr::accept(ConstInstrVisitor& visitor) const +void +EmitVertexInstr::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void EmitVertexInstr::accept(InstrVisitor& visitor) +void +EmitVertexInstr::accept(InstrVisitor& visitor) { visitor.visit(this); } -bool EmitVertexInstr::do_ready() const +bool +EmitVertexInstr::do_ready() const { return true; } -void EmitVertexInstr::do_print(std::ostream& os) const +void +EmitVertexInstr::do_print(std::ostream& os) const { os << (m_cut ? "EMIT_CUT_VERTEX @" : "EMIT_VERTEX @") << m_stream; } -auto EmitVertexInstr::from_string(std::istream& is, bool cut) -> Pointer +auto +EmitVertexInstr::from_string(std::istream& is, bool cut) -> Pointer { char c; is >> c; @@ -545,22 +594,26 @@ auto EmitVertexInstr::from_string(std::istream& is, bool cut) -> Pointer return new EmitVertexInstr(stream, cut); } -void WriteTFInstr::accept(ConstInstrVisitor& visitor) const +void +WriteTFInstr::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void WriteTFInstr::accept(InstrVisitor& visitor) +void +WriteTFInstr::accept(InstrVisitor& visitor) { visitor.visit(this); } -bool WriteTFInstr::is_equal_to(const WriteTFInstr& rhs) const +bool +WriteTFInstr::is_equal_to(const WriteTFInstr& rhs) const { return value() == rhs.value(); } -auto WriteTFInstr::from_string(std::istream& is, ValueFactory &vf) -> Pointer +auto +WriteTFInstr::from_string(std::istream& is, ValueFactory& vf) -> Pointer { string value_str; is >> value_str; @@ -570,15 +623,16 @@ auto WriteTFInstr::from_string(std::istream& is, ValueFactory &vf) -> Pointer return new WriteTFInstr(value); } -bool WriteTFInstr::do_ready() const +bool +WriteTFInstr::do_ready() const { return value().ready(block_id(), index()); } -void WriteTFInstr::do_print(std::ostream& os) const +void +WriteTFInstr::do_print(std::ostream& os) const { os << "WRITE_TF " << value(); } -} - +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_export.h b/src/gallium/drivers/r600/sfn/sfn_instr_export.h index d8ab4f0cd89..12edaece4fb 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_export.h +++ b/src/gallium/drivers/r600/sfn/sfn_instr_export.h @@ -33,24 +33,21 @@ namespace r600 { class ValueFactory; - -class WriteOutInstr: public Instr { +class WriteOutInstr : public Instr { public: WriteOutInstr(const RegisterVec4& value); WriteOutInstr(const WriteOutInstr& orig) = delete; void override_chan(int i, int chan); - const RegisterVec4& value() const {return m_value;}; - RegisterVec4& value() {return m_value;}; - + const RegisterVec4& value() const { return m_value; }; + RegisterVec4& value() { return m_value; }; private: - RegisterVec4 m_value; }; -class ExportInstr: public WriteOutInstr { +class ExportInstr : public WriteOutInstr { public: enum ExportType { pixel, @@ -68,23 +65,22 @@ public: bool is_equal_to(const ExportInstr& lhs) const; - static ExportType type_from_string(const std::string& s); - ExportType export_type() const {return m_type;} + ExportType export_type() const { return m_type; } - unsigned location() const {return m_loc;} + unsigned location() const { return m_loc; } - void set_is_last_export(bool value) {m_is_last = value;} - bool is_last_export() const {return m_is_last;} + void set_is_last_export(bool value) { m_is_last = value; } + bool is_last_export() const { return m_is_last; } - static Instr::Pointer from_string(std::istream& is, ValueFactory &vf); - static Instr::Pointer last_from_string(std::istream& is, ValueFactory &vf); + static Instr::Pointer from_string(std::istream& is, ValueFactory& vf); + static Instr::Pointer last_from_string(std::istream& is, ValueFactory& vf); uint8_t allowed_dest_chan_mask() const override; private: - static ExportInstr::Pointer from_string_impl(std::istream& is, ValueFactory &vf); + static ExportInstr::Pointer from_string_impl(std::istream& is, ValueFactory& vf); bool do_ready() const override; void do_print(std::ostream& os) const override; @@ -96,32 +92,40 @@ private: class ScratchIOInstr : public WriteOutInstr { public: - ScratchIOInstr(const RegisterVec4& value, PRegister addr, - int align, int align_offset, int writemask, int array_size, + ScratchIOInstr(const RegisterVec4& value, + PRegister addr, + int align, + int align_offset, + int writemask, + int array_size, + bool is_read = false); + ScratchIOInstr(const RegisterVec4& value, + int addr, + int align, + int align_offset, + int writemask, bool is_read = false); - ScratchIOInstr(const RegisterVec4& value, int addr, int align, int align_offset, - int writemask, bool is_read = false); void accept(ConstInstrVisitor& visitor) const override; void accept(InstrVisitor& visitor) override; bool is_equal_to(const ScratchIOInstr& lhs) const; - unsigned location() const { return m_loc;}; - int write_mask() const { return m_writemask;} - auto address() const { return m_address;} - bool indirect() const { return !!m_address;} - int array_size() const { return m_array_size;} - bool is_read() const {return m_read; } + unsigned location() const { return m_loc; }; + int write_mask() const { return m_writemask; } + auto address() const { return m_address; } + bool indirect() const { return !!m_address; } + int array_size() const { return m_array_size; } + bool is_read() const { return m_read; } + + static auto from_string(std::istream& is, ValueFactory& vf) -> Pointer; - static auto from_string(std::istream& is, ValueFactory &vf) -> Pointer; private: - bool do_ready() const override; void do_print(std::ostream& os) const override; unsigned m_loc{0}; - PRegister m_address {nullptr}; + PRegister m_address{nullptr}; unsigned m_align; unsigned m_align_offset; unsigned m_writemask; @@ -129,24 +133,27 @@ private: bool m_read{false}; }; -class StreamOutInstr: public WriteOutInstr { +class StreamOutInstr : public WriteOutInstr { public: - StreamOutInstr(const RegisterVec4& value, int num_components, - int array_base, int comp_mask, int out_buffer, - int stream); - int element_size() const { return m_element_size;} - int burst_count() const { return m_burst_count;} - int array_base() const { return m_array_base;} - int array_size() const { return m_array_size;} - int comp_mask() const { return m_writemask;} + StreamOutInstr(const RegisterVec4& value, + int num_components, + int array_base, + int comp_mask, + int out_buffer, + int stream); + int element_size() const { return m_element_size; } + int burst_count() const { return m_burst_count; } + int array_base() const { return m_array_base; } + int array_size() const { return m_array_size; } + int comp_mask() const { return m_writemask; } unsigned op(amd_gfx_level gfx_level) const; bool is_equal_to(const StreamOutInstr& lhs) const; void accept(ConstInstrVisitor& visitor) const override; void accept(InstrVisitor& visitor) override; -private: +private: bool do_ready() const override; void do_print(std::ostream& os) const override; @@ -159,9 +166,8 @@ private: int m_stream{0}; }; -class MemRingOutInstr: public WriteOutInstr { +class MemRingOutInstr : public WriteOutInstr { public: - enum EMemWriteType { mem_write = 0, mem_write_ind = 1, @@ -169,17 +175,24 @@ public: mem_write_ind_ack = 3, }; - MemRingOutInstr(ECFOpCode ring, EMemWriteType type, - const RegisterVec4& value, unsigned base_addr, - unsigned ncomp, PRegister m_index); + MemRingOutInstr(ECFOpCode ring, + EMemWriteType type, + const RegisterVec4& value, + unsigned base_addr, + unsigned ncomp, + PRegister m_index); - unsigned op() const{return m_ring_op;} + unsigned op() const { return m_ring_op; } unsigned ncomp() const; - unsigned addr() const {return m_base_address;} - EMemWriteType type() const {return m_type;} - unsigned index_reg() const {assert(m_export_index->sel() >= 0); return m_export_index->sel();} - unsigned array_base() const {return m_base_address; } - PVirtualValue export_index() const {return m_export_index;} + unsigned addr() const { return m_base_address; } + EMemWriteType type() const { return m_type; } + unsigned index_reg() const + { + assert(m_export_index->sel() >= 0); + return m_export_index->sel(); + } + unsigned array_base() const { return m_base_address; } + PVirtualValue export_index() const { return m_export_index; } void patch_ring(int stream, PRegister index); @@ -188,10 +201,9 @@ public: bool is_equal_to(const MemRingOutInstr& lhs) const; - static auto from_string(std::istream& is, ValueFactory &vf) -> Pointer; + static auto from_string(std::istream& is, ValueFactory& vf) -> Pointer; private: - bool do_ready() const override; void do_print(std::ostream& os) const override; @@ -205,8 +217,8 @@ private: class EmitVertexInstr : public Instr { public: EmitVertexInstr(int stream, bool cut); - ECFOpCode op() const {return m_cut ? cf_cut_vertex: cf_emit_vertex;} - int stream() const { return m_stream;} + ECFOpCode op() const { return m_cut ? cf_cut_vertex : cf_emit_vertex; } + int stream() const { return m_stream; } void accept(ConstInstrVisitor& visitor) const override; void accept(InstrVisitor& visitor) override; @@ -232,15 +244,13 @@ public: bool is_equal_to(const WriteTFInstr& rhs) const; - static auto from_string(std::istream& is, ValueFactory &vf) -> Pointer; + static auto from_string(std::istream& is, ValueFactory& vf) -> Pointer; private: bool do_ready() const override; void do_print(std::ostream& os) const override; - }; - -} +} // namespace r600 #endif // INSTR_EXPORT_H diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_fetch.cpp b/src/gallium/drivers/r600/sfn/sfn_instr_fetch.cpp index 22906c0a133..9f0a1012d17 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_fetch.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_instr_fetch.cpp @@ -25,15 +25,16 @@ */ #include "sfn_instr_fetch.h" -#include "sfn_valuefactory.h" + #include "sfn_defines.h" +#include "sfn_valuefactory.h" #include namespace r600 { -using std::string; using std::istringstream; +using std::string; FetchInstr::FetchInstr(EVFetchInstr opcode, const RegisterVec4& dst, @@ -46,35 +47,35 @@ FetchInstr::FetchInstr(EVFetchInstr opcode, EVFetchEndianSwap endian_swap, uint32_t resource_id, PRegister resource_offset): - InstrWithVectorResult(dst, dest_swizzle, resource_id, resource_offset), - m_opcode(opcode), - m_src(src), - m_src_offset(src_offset), - m_fetch_type(fetch_type), - m_data_format(data_format), - m_num_format(num_format), - m_endian_swap(endian_swap), - m_mega_fetch_count(0), - m_array_base(0), - m_array_size(0), - m_elm_size(0) + InstrWithVectorResult(dst, dest_swizzle, resource_id, resource_offset), + m_opcode(opcode), + m_src(src), + m_src_offset(src_offset), + m_fetch_type(fetch_type), + m_data_format(data_format), + m_num_format(num_format), + m_endian_swap(endian_swap), + m_mega_fetch_count(0), + m_array_base(0), + m_array_size(0), + m_elm_size(0) { switch (m_opcode) { - case vc_fetch : - m_opname ="VFETCH"; - break; - case vc_semantic : + case vc_fetch: + m_opname = "VFETCH"; + break; + case vc_semantic: m_opname = "FETCH_SEMANTIC"; - break; - case vc_get_buf_resinfo : + break; + case vc_get_buf_resinfo: set_print_skip(mfc); set_print_skip(fmt); set_print_skip(ftype); m_opname = "GET_BUF_RESINFO"; - break; - case vc_read_scratch : + break; + case vc_read_scratch: m_opname = "READ_SCRATCH"; - break; + break; default: unreachable("Unknwon fetch instruction"); } @@ -83,17 +84,20 @@ FetchInstr::FetchInstr(EVFetchInstr opcode, m_src->add_use(this); } -void FetchInstr::accept(ConstInstrVisitor& visitor) const +void +FetchInstr::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void FetchInstr::accept(InstrVisitor& visitor) +void +FetchInstr::accept(InstrVisitor& visitor) { visitor.visit(this); } -bool FetchInstr::is_equal_to(const FetchInstr& rhs) const +bool +FetchInstr::is_equal_to(const FetchInstr& rhs) const { if (m_src) { if (rhs.m_src) { @@ -116,20 +120,16 @@ bool FetchInstr::is_equal_to(const FetchInstr& rhs) const } else if (!(!!resource_offset() == !!rhs.resource_offset())) return false; - return m_opcode == rhs.m_opcode && - m_src_offset == rhs.m_src_offset && - m_fetch_type == rhs.m_fetch_type && - m_data_format == rhs.m_data_format && - m_num_format == rhs.m_num_format && - m_endian_swap == rhs.m_endian_swap && - m_mega_fetch_count == rhs.m_mega_fetch_count && - m_array_base == rhs.m_array_base && - m_array_size == rhs.m_array_size && - m_elm_size == rhs.m_elm_size && - resource_base() == rhs.resource_base(); + return m_opcode == rhs.m_opcode && m_src_offset == rhs.m_src_offset && + m_fetch_type == rhs.m_fetch_type && m_data_format == rhs.m_data_format && + m_num_format == rhs.m_num_format && m_endian_swap == rhs.m_endian_swap && + m_mega_fetch_count == rhs.m_mega_fetch_count && + m_array_base == rhs.m_array_base && m_array_size == rhs.m_array_size && + m_elm_size == rhs.m_elm_size && resource_base() == rhs.resource_base(); } -bool FetchInstr::propagate_death() +bool +FetchInstr::propagate_death() { auto reg = m_src->as_register(); if (reg) @@ -137,7 +137,8 @@ bool FetchInstr::propagate_death() return true; } -bool FetchInstr::replace_source(PRegister old_src, PVirtualValue new_src) +bool +FetchInstr::replace_source(PRegister old_src, PVirtualValue new_src) { bool success = false; auto new_reg = new_src->as_register(); @@ -153,20 +154,22 @@ bool FetchInstr::replace_source(PRegister old_src, PVirtualValue new_src) return success; } -bool FetchInstr::do_ready() const +bool +FetchInstr::do_ready() const { - for (auto i: required_instr()) { + for (auto i : required_instr()) { if (!i->is_scheduled()) return false; } bool result = m_src && m_src->ready(block_id(), index()); if (resource_offset()) - result &= resource_offset()->ready(block_id(), index()); + result &= resource_offset()->ready(block_id(), index()); return result; } -void FetchInstr::do_print(std::ostream& os) const +void +FetchInstr::do_print(std::ostream& os) const { os << m_opname << ' '; @@ -190,9 +193,15 @@ void FetchInstr::do_print(std::ostream& os) const if (!m_skip_print.test(ftype)) { switch (m_fetch_type) { - case vertex_data : os << " VERTEX"; break; - case instance_data : os << " INSTANCE_DATA"; break; - case no_index_offset : os << " NO_IDX_OFFSET"; break; + case vertex_data: + os << " VERTEX"; + break; + case instance_data: + os << " INSTANCE_DATA"; + break; + case no_index_offset: + os << " NO_IDX_OFFSET"; + break; default: unreachable("Unknwon fetch instruction type"); } @@ -212,9 +221,15 @@ void FetchInstr::do_print(std::ostream& os) const os << "U"; switch (m_num_format) { - case vtx_nf_norm : os << "NORM"; break; - case vtx_nf_int : os << "INT"; break; - case vtx_nf_scaled: os << "SCALED"; break; + case vtx_nf_norm: + os << "NORM"; + break; + case vtx_nf_int: + os << "INT"; + break; + case vtx_nf_scaled: + os << "SCALED"; + break; default: unreachable("Unknwon number format"); } @@ -238,23 +253,34 @@ void FetchInstr::do_print(std::ostream& os) const if (m_elm_size) os << " ES:" << m_elm_size; - if (m_tex_flags.test(fetch_whole_quad)) os << " WQ"; - if (m_tex_flags.test(use_const_field)) os << " UCF"; - if (m_tex_flags.test(srf_mode)) os << " SRF"; - if (m_tex_flags.test(buf_no_stride)) os << " BNS"; - if (m_tex_flags.test(alt_const)) os << " AC"; - if (m_tex_flags.test(use_tc)) os << " TC"; - if (m_tex_flags.test(vpm)) os << " VPM"; - if (m_tex_flags.test(uncached) && m_opcode != vc_read_scratch) os << " UNCACHED"; - if (m_tex_flags.test(indexed) && m_opcode != vc_read_scratch) os << " INDEXED"; + if (m_tex_flags.test(fetch_whole_quad)) + os << " WQ"; + if (m_tex_flags.test(use_const_field)) + os << " UCF"; + if (m_tex_flags.test(srf_mode)) + os << " SRF"; + if (m_tex_flags.test(buf_no_stride)) + os << " BNS"; + if (m_tex_flags.test(alt_const)) + os << " AC"; + if (m_tex_flags.test(use_tc)) + os << " TC"; + if (m_tex_flags.test(vpm)) + os << " VPM"; + if (m_tex_flags.test(uncached) && m_opcode != vc_read_scratch) + os << " UNCACHED"; + if (m_tex_flags.test(indexed) && m_opcode != vc_read_scratch) + os << " INDEXED"; } -Instr::Pointer FetchInstr::from_string(std::istream& is, ValueFactory& vf) +Instr::Pointer +FetchInstr::from_string(std::istream& is, ValueFactory& vf) { return from_string_impl(is, vc_fetch, vf); } -Instr::Pointer FetchInstr::from_string_impl(std::istream& is, EVFetchInstr opcode, ValueFactory& vf) +Instr::Pointer +FetchInstr::from_string_impl(std::istream& is, EVFetchInstr opcode, ValueFactory& vf) { std::string deststr; is >> deststr; @@ -269,7 +295,7 @@ Instr::Pointer FetchInstr::from_string_impl(std::istream& is, EVFetchInstr opcod string srcstr; is >> srcstr; - std::cerr << "Get source " << srcstr << "\n"; + std::cerr << "Get source " << srcstr << "\n"; auto src_reg = vf.src_from_string(srcstr)->as_register(); assert(src_reg); @@ -316,7 +342,7 @@ Instr::Pointer FetchInstr::from_string_impl(std::istream& is, EVFetchInstr opcod char c; fmt_stream >> c; - if (c == ',') { + if (c == ',') { is_num_fmr = true; continue; } @@ -329,7 +355,7 @@ Instr::Pointer FetchInstr::from_string_impl(std::istream& is, EVFetchInstr opcod EVTXDataFormat fmt = fmt_invalid; - for (auto& [f, name] : s_data_format_map) { + for (auto& [f, name] : s_data_format_map) { if (data_format == name) { fmt = f; break; @@ -342,7 +368,7 @@ Instr::Pointer FetchInstr::from_string_impl(std::istream& is, EVFetchInstr opcod assert(fmt_signed || num_format_str[0] == 'U'); size_t num_format_end = num_format_str.find(')'); - num_format_str = num_format_str.substr(1, num_format_end - 1) ; + num_format_str = num_format_str.substr(1, num_format_end - 1); EVFetchNumFormat num_fmt; if (num_format_str == "NORM") @@ -356,9 +382,17 @@ Instr::Pointer FetchInstr::from_string_impl(std::istream& is, EVFetchInstr opcod unreachable("Unknown number format"); } - auto fetch = new FetchInstr(opcode, dest_reg, dst_swz, - src_reg, src_offset_val, fetch_type, fmt, num_fmt, - vtx_es_none, res_id, nullptr); + auto fetch = new FetchInstr(opcode, + dest_reg, + dst_swz, + src_reg, + src_offset_val, + fetch_type, + fmt, + num_fmt, + vtx_es_none, + res_id, + nullptr); if (fmt_signed) fetch->set_fetch_flag(format_comp_signed); @@ -379,15 +413,16 @@ Instr::Pointer FetchInstr::from_string_impl(std::istream& is, EVFetchInstr opcod return fetch; } -void FetchInstr::set_param_from_string(const std::string& token) +void +FetchInstr::set_param_from_string(const std::string& token) { - if (token.substr(0,4) == "MFC:") + if (token.substr(0, 4) == "MFC:") set_mfc(int_from_string_with_prefix(token, "MFC:")); - else if (token.substr(0,5) == "ARRB:") + else if (token.substr(0, 5) == "ARRB:") set_array_base(int_from_string_with_prefix(token, "ARRB:")); - else if (token.substr(0,5) == "ARRS:") + else if (token.substr(0, 5) == "ARRS:") set_array_size(int_from_string_with_prefix(token, "ARRS:")); - else if (token.substr(0,3) == "ES:") + else if (token.substr(0, 3) == "ES:") set_element_size(int_from_string_with_prefix(token, "ES:")); else { std::cerr << "Token '" << token << "': "; @@ -395,7 +430,8 @@ void FetchInstr::set_param_from_string(const std::string& token) } } -void FetchInstr::set_flag_from_string(const std::string& token) +void +FetchInstr::set_flag_from_string(const std::string& token) { auto flag = s_flag_map.find(token.c_str()); if (flag != s_flag_map.end()) @@ -406,100 +442,99 @@ void FetchInstr::set_flag_from_string(const std::string& token) } } - const std::map FetchInstr::s_flag_map = { - {"WQ", fetch_whole_quad}, - {"UCF", use_const_field}, - {"SRF", srf_mode}, - {"BNS", buf_no_stride}, - {"AC", alt_const}, - {"TC", use_tc}, - {"VPM", vpm}, - {"UNCACHED", uncached}, - {"INDEXED", indexed} + {"WQ", fetch_whole_quad}, + {"UCF", use_const_field }, + {"SRF", srf_mode }, + {"BNS", buf_no_stride }, + {"AC", alt_const }, + {"TC", use_tc }, + {"VPM", vpm }, + {"UNCACHED", uncached }, + {"INDEXED", indexed } }; const std::map FetchInstr::s_data_format_map = { - {fmt_invalid, "INVALID"}, - {fmt_8, "8"}, - {fmt_4_4, "4_4"}, - {fmt_3_3_2, "3_3_2"}, - {fmt_reserved_4, "RESERVED_4"}, - {fmt_16, "16"}, - {fmt_16_float, "16F"}, - {fmt_8_8, "8_8"}, - {fmt_5_6_5, "5_6_5"}, - {fmt_6_5_5, "6_5_5"}, - {fmt_1_5_5_5, "1_5_5_5"}, - {fmt_4_4_4_4, "4_4_4_4"}, - {fmt_5_5_5_1, "5_5_5_1"}, - {fmt_32, "32"}, - {fmt_32_float, "32F"}, - {fmt_16_16, "16_16"}, - {fmt_16_16_float, "16_16F"}, - {fmt_8_24, "8_24"}, - {fmt_8_24_float, "8_24F"}, - {fmt_24_8, "24_8"}, - {fmt_24_8_float, "24_8F"}, - {fmt_10_11_11, "10_11_11"}, - {fmt_10_11_11_float, "10_11_11F"}, - {fmt_11_11_10, "11_11_10"}, - {fmt_10_11_11_float, "11_11_10F"}, - {fmt_2_10_10_10, "2_10_10_10"}, - {fmt_8_8_8_8, "8_8_8_8"}, - {fmt_10_10_10_2, "10_10_10_2"}, - {fmt_x24_8_32_float, "X24_8_32F"}, - {fmt_32_32, "32_32"}, - {fmt_32_32_float, "32_32F"}, - {fmt_16_16_16_16, "16_16_16_16"}, - {fmt_16_16_16_16_float, "16_16_16_16F"}, - {fmt_reserved_33, "RESERVED_33"}, - {fmt_32_32_32_32, "32_32_32_32"}, - {fmt_32_32_32_32_float, "32_32_32_32F"}, - {fmt_reserved_36, "RESERVED_36"}, - {fmt_1, "1"}, - {fmt_1_reversed, "1_REVERSED"}, - {fmt_gb_gr, "GB_GR"}, - {fmt_bg_rg, "BG_RG"}, - {fmt_32_as_8, "32_AS_8"}, - {fmt_32_as_8_8, "32_AS_8_8"}, + {fmt_invalid, "INVALID" }, + {fmt_8, "8" }, + {fmt_4_4, "4_4" }, + {fmt_3_3_2, "3_3_2" }, + {fmt_reserved_4, "RESERVED_4" }, + {fmt_16, "16" }, + {fmt_16_float, "16F" }, + {fmt_8_8, "8_8" }, + {fmt_5_6_5, "5_6_5" }, + {fmt_6_5_5, "6_5_5" }, + {fmt_1_5_5_5, "1_5_5_5" }, + {fmt_4_4_4_4, "4_4_4_4" }, + {fmt_5_5_5_1, "5_5_5_1" }, + {fmt_32, "32" }, + {fmt_32_float, "32F" }, + {fmt_16_16, "16_16" }, + {fmt_16_16_float, "16_16F" }, + {fmt_8_24, "8_24" }, + {fmt_8_24_float, "8_24F" }, + {fmt_24_8, "24_8" }, + {fmt_24_8_float, "24_8F" }, + {fmt_10_11_11, "10_11_11" }, + {fmt_10_11_11_float, "10_11_11F" }, + {fmt_11_11_10, "11_11_10" }, + {fmt_10_11_11_float, "11_11_10F" }, + {fmt_2_10_10_10, "2_10_10_10" }, + {fmt_8_8_8_8, "8_8_8_8" }, + {fmt_10_10_10_2, "10_10_10_2" }, + {fmt_x24_8_32_float, "X24_8_32F" }, + {fmt_32_32, "32_32" }, + {fmt_32_32_float, "32_32F" }, + {fmt_16_16_16_16, "16_16_16_16" }, + {fmt_16_16_16_16_float, "16_16_16_16F" }, + {fmt_reserved_33, "RESERVED_33" }, + {fmt_32_32_32_32, "32_32_32_32" }, + {fmt_32_32_32_32_float, "32_32_32_32F" }, + {fmt_reserved_36, "RESERVED_36" }, + {fmt_1, "1" }, + {fmt_1_reversed, "1_REVERSED" }, + {fmt_gb_gr, "GB_GR" }, + {fmt_bg_rg, "BG_RG" }, + {fmt_32_as_8, "32_AS_8" }, + {fmt_32_as_8_8, "32_AS_8_8" }, {fmt_5_9_9_9_sharedexp, "5_9_9_9_SHAREDEXP"}, - {fmt_8_8_8, "8_8_8"}, - {fmt_16_16_16, "16_16_16"}, - {fmt_16_16_16_float, "16_16_16F"}, - {fmt_32_32_32, "32_32_32"}, - {fmt_32_32_32_float, "32_32_32F"}, - {fmt_bc1, "BC1"}, - {fmt_bc2, "BC2"}, - {fmt_bc3, "BC3"}, - {fmt_bc4, "BC4"}, - {fmt_bc5, "BC5"}, - {fmt_apc0, "APC0"}, - {fmt_apc1, "APC1"}, - {fmt_apc2, "APC2"}, - {fmt_apc3, "APC3"}, - {fmt_apc4, "APC4"}, - {fmt_apc5, "APC5"}, - {fmt_apc6, "APC6"}, - {fmt_apc7, "APC7"}, - {fmt_ctx1, "CTX1"}, - {fmt_reserved_63, "RESERVED_63"} + {fmt_8_8_8, "8_8_8" }, + {fmt_16_16_16, "16_16_16" }, + {fmt_16_16_16_float, "16_16_16F" }, + {fmt_32_32_32, "32_32_32" }, + {fmt_32_32_32_float, "32_32_32F" }, + {fmt_bc1, "BC1" }, + {fmt_bc2, "BC2" }, + {fmt_bc3, "BC3" }, + {fmt_bc4, "BC4" }, + {fmt_bc5, "BC5" }, + {fmt_apc0, "APC0" }, + {fmt_apc1, "APC1" }, + {fmt_apc2, "APC2" }, + {fmt_apc3, "APC3" }, + {fmt_apc4, "APC4" }, + {fmt_apc5, "APC5" }, + {fmt_apc6, "APC6" }, + {fmt_apc7, "APC7" }, + {fmt_ctx1, "CTX1" }, + {fmt_reserved_63, "RESERVED_63" } }; - QueryBufferSizeInstr::QueryBufferSizeInstr(const RegisterVec4& dst, const RegisterVec4::Swizzle& dst_swz, uint32_t resid): - FetchInstr(vc_get_buf_resinfo, - dst, dst_swz, - new Register( 0, 7, pin_fully), - 0, - no_index_offset, - fmt_32_32_32_32, - vtx_nf_norm, - vtx_es_none, - resid, - nullptr) + FetchInstr(vc_get_buf_resinfo, + dst, + dst_swz, + new Register(0, 7, pin_fully), + 0, + no_index_offset, + fmt_32_32_32_32, + vtx_nf_norm, + vtx_es_none, + resid, + nullptr) { set_fetch_flag(format_comp_signed); set_print_skip(mfc); @@ -507,7 +542,8 @@ QueryBufferSizeInstr::QueryBufferSizeInstr(const RegisterVec4& dst, set_print_skip(ftype); } -Instr::Pointer QueryBufferSizeInstr::from_string(std::istream& is, ValueFactory& vf) +Instr::Pointer +QueryBufferSizeInstr::from_string(std::istream& is, ValueFactory& vf) { std::string deststr, res_id_str; is >> deststr; @@ -522,14 +558,27 @@ Instr::Pointer QueryBufferSizeInstr::from_string(std::istream& is, ValueFactory& auto dst = vf.dest_vec4_from_string(deststr, dst_swz, pin_group); int res_id = int_from_string_with_prefix(res_id_str, "RID:"); - return new QueryBufferSizeInstr( dst, dst_swz, res_id); + return new QueryBufferSizeInstr(dst, dst_swz, res_id); } -LoadFromBuffer::LoadFromBuffer(const RegisterVec4& dst, const RegisterVec4::Swizzle& dst_swizzle, - PRegister addr, uint32_t addr_offset, - uint32_t resid, PRegister res_offset, EVTXDataFormat data_format): - FetchInstr(vc_fetch, dst, dst_swizzle, addr, addr_offset, no_index_offset, - data_format, vtx_nf_scaled, vtx_es_none, resid, res_offset) +LoadFromBuffer::LoadFromBuffer(const RegisterVec4& dst, + const RegisterVec4::Swizzle& dst_swizzle, + PRegister addr, + uint32_t addr_offset, + uint32_t resid, + PRegister res_offset, + EVTXDataFormat data_format): + FetchInstr(vc_fetch, + dst, + dst_swizzle, + addr, + addr_offset, + no_index_offset, + data_format, + vtx_nf_scaled, + vtx_es_none, + resid, + res_offset) { set_fetch_flag(format_comp_signed); set_mfc(16); @@ -539,7 +588,8 @@ LoadFromBuffer::LoadFromBuffer(const RegisterVec4& dst, const RegisterVec4::Swiz set_print_skip(ftype); } -Instr::Pointer LoadFromBuffer::from_string(std::istream& is, ValueFactory& vf) +Instr::Pointer +LoadFromBuffer::from_string(std::istream& is, ValueFactory& vf) { std::string deststr; is >> deststr; @@ -581,9 +631,8 @@ Instr::Pointer LoadFromBuffer::from_string(std::istream& is, ValueFactory& vf) res_offset = vf.src_from_string(res_offset_str)->as_register(); } - auto fetch = new LoadFromBuffer( dst, dst_swz, - addr_reg, addr_offset_val, - res_id, res_offset, fmt_32_32_32_32_float); + auto fetch = new LoadFromBuffer( + dst, dst_swz, addr_reg, addr_offset_val, res_id, res_offset, fmt_32_32_32_32_float); is >> next; if (next == "SRF") fetch->set_fetch_flag(srf_mode); @@ -591,40 +640,63 @@ Instr::Pointer LoadFromBuffer::from_string(std::istream& is, ValueFactory& vf) return fetch; } -class AddrResolver: public RegisterVisitor { +class AddrResolver : public RegisterVisitor { public: - AddrResolver(LoadFromScratch *lfs) : m_lfs(lfs) {} + AddrResolver(LoadFromScratch *lfs): + m_lfs(lfs) + { + } - void visit(Register& value) { + void visit(Register& value) + { m_lfs->set_fetch_flag(FetchInstr::indexed); m_lfs->set_src(&value); value.add_use(m_lfs); } - void visit(LocalArray& value) {assert(0);(void)value;} - void visit(LocalArrayValue& value) {assert(0);(void)value;} - void visit(UniformValue& value) {assert(0);(void)value;} - void visit(LiteralConstant& value) { - m_lfs->set_array_base(value.value()); - m_lfs->set_src(new Register( 0, 7, pin_none)); + void visit(LocalArray& value) + { + assert(0); + (void)value; + } + void visit(LocalArrayValue& value) + { + assert(0); + (void)value; + } + void visit(UniformValue& value) + { + assert(0); + (void)value; + } + void visit(LiteralConstant& value) + { + m_lfs->set_array_base(value.value()); + m_lfs->set_src(new Register(0, 7, pin_none)); + } + void visit(InlineConstant& value) + { + assert(0); + (void)value; } - void visit(InlineConstant& value) {assert(0);(void)value;} LoadFromScratch *m_lfs; }; - - -LoadFromScratch::LoadFromScratch(const RegisterVec4& dst, const RegisterVec4::Swizzle& dst_swz, PVirtualValue addr, uint32_t scratch_size): - FetchInstr(vc_read_scratch, - dst, dst_swz, - nullptr, - 0, - no_index_offset, - fmt_32_32_32_32, - vtx_nf_int, - vtx_es_none, - 0, - nullptr) +LoadFromScratch::LoadFromScratch(const RegisterVec4& dst, + const RegisterVec4::Swizzle& dst_swz, + PVirtualValue addr, + uint32_t scratch_size): + FetchInstr(vc_read_scratch, + dst, + dst_swz, + nullptr, + 0, + no_index_offset, + fmt_32_32_32_32, + vtx_nf_int, + vtx_es_none, + 0, + nullptr) { set_fetch_flag(uncached); set_fetch_flag(wait_ack); @@ -641,7 +713,8 @@ LoadFromScratch::LoadFromScratch(const RegisterVec4& dst, const RegisterVec4::Sw set_element_size(3); } -Instr::Pointer LoadFromScratch::from_string(std::istream& is, ValueFactory &vf) +Instr::Pointer +LoadFromScratch::from_string(std::istream& is, ValueFactory& vf) { std::string deststr; is >> deststr; @@ -662,8 +735,7 @@ Instr::Pointer LoadFromScratch::from_string(std::istream& is, ValueFactory &vf) int size = int_from_string_with_prefix(offsetstr, "SIZE:"); assert(size >= 1); - return new LoadFromScratch( dest, dst_swz, addr_reg, size); -} - + return new LoadFromScratch(dest, dst_swz, addr_reg, size); } +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_fetch.h b/src/gallium/drivers/r600/sfn/sfn_instr_fetch.h index 695fa1b357f..be3dd744a48 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_fetch.h +++ b/src/gallium/drivers/r600/sfn/sfn_instr_fetch.h @@ -35,7 +35,6 @@ class ValueFactory; class FetchInstr : public InstrWithVectorResult { public: - enum EFlags { fetch_whole_quad, use_const_field, @@ -75,46 +74,55 @@ public: void accept(InstrVisitor& visitor) override; void set_src(PRegister src) { m_src = src; } - const auto& src() const {assert(m_src); return *m_src;} - uint32_t src_offset() const {return m_src_offset;} + const auto& src() const + { + assert(m_src); + return *m_src; + } + uint32_t src_offset() const { return m_src_offset; } - uint32_t resource_id() const __attribute__((deprecated)) {return resource_base();} + uint32_t resource_id() const __attribute__((deprecated)) { return resource_base(); } - EVFetchType fetch_type() const {return m_fetch_type;} - EVTXDataFormat data_format() const {return m_data_format;} - void set_num_format(EVFetchNumFormat nf) {m_num_format = nf;} - EVFetchNumFormat num_format() const {return m_num_format;} - EVFetchEndianSwap endian_swap() const {return m_endian_swap;} + EVFetchType fetch_type() const { return m_fetch_type; } + EVTXDataFormat data_format() const { return m_data_format; } + void set_num_format(EVFetchNumFormat nf) { m_num_format = nf; } + EVFetchNumFormat num_format() const { return m_num_format; } + EVFetchEndianSwap endian_swap() const { return m_endian_swap; } - uint32_t mega_fetch_count() const {return m_mega_fetch_count;} - uint32_t array_base() const {return m_array_base;} - uint32_t array_size() const {return m_array_size;} - uint32_t elm_size() const {return m_elm_size;} + uint32_t mega_fetch_count() const { return m_mega_fetch_count; } + uint32_t array_base() const { return m_array_base; } + uint32_t array_size() const { return m_array_size; } + uint32_t elm_size() const { return m_elm_size; } - void reset_fetch_flag(EFlags flag) {m_tex_flags.reset(flag);} - void set_fetch_flag(EFlags flag) {m_tex_flags.set(flag);} - bool has_fetch_flag(EFlags flag) const { return m_tex_flags.test(flag);} + void reset_fetch_flag(EFlags flag) { m_tex_flags.reset(flag); } + void set_fetch_flag(EFlags flag) { m_tex_flags.set(flag); } + bool has_fetch_flag(EFlags flag) const { return m_tex_flags.test(flag); } - EVFetchInstr opcode() const {return m_opcode;} + EVFetchInstr opcode() const { return m_opcode; } bool is_equal_to(const FetchInstr& rhs) const; - static Instr::Pointer from_string(std::istream& is, ValueFactory &vf); + static Instr::Pointer from_string(std::istream& is, ValueFactory& vf); - void set_mfc(int mfc) {m_tex_flags.set(is_mega_fetch); m_mega_fetch_count = mfc;} - void set_array_base(int arrb) {m_array_base = arrb;} - void set_array_size(int arrs) {m_array_size = arrs;} + void set_mfc(int mfc) + { + m_tex_flags.set(is_mega_fetch); + m_mega_fetch_count = mfc; + } + void set_array_base(int arrb) { m_array_base = arrb; } + void set_array_size(int arrs) { m_array_size = arrs; } - void set_element_size(int size) { m_elm_size = size;} - void set_print_skip(EPrintSkip skip) {m_skip_print.set(skip);} - uint32_t slots() const override {return 1;}; + void set_element_size(int size) { m_elm_size = size; } + void set_print_skip(EPrintSkip skip) { m_skip_print.set(skip); } + uint32_t slots() const override { return 1; }; bool replace_source(PRegister old_src, PVirtualValue new_src) override; protected: - static Instr::Pointer from_string_impl(std::istream& is, EVFetchInstr opcode, ValueFactory &vf); + static Instr::Pointer + from_string_impl(std::istream& is, EVFetchInstr opcode, ValueFactory& vf); - void override_opname(const char *opname) { m_opname = opname;} + void override_opname(const char *opname) { m_opname = opname; } private: bool do_ready() const override; @@ -152,23 +160,32 @@ private: class QueryBufferSizeInstr : public FetchInstr { public: - QueryBufferSizeInstr(const RegisterVec4& dst, const RegisterVec4::Swizzle& swizzle, uint32_t resid); + QueryBufferSizeInstr(const RegisterVec4& dst, + const RegisterVec4::Swizzle& swizzle, + uint32_t resid); static Instr::Pointer from_string(std::istream& is, ValueFactory& vf); }; class LoadFromBuffer : public FetchInstr { public: - LoadFromBuffer(const RegisterVec4& dst, const RegisterVec4::Swizzle& swizzle, - PRegister addr, uint32_t addr_offset, - uint32_t resid, PRegister res_offset, EVTXDataFormat data_format); + LoadFromBuffer(const RegisterVec4& dst, + const RegisterVec4::Swizzle& swizzle, + PRegister addr, + uint32_t addr_offset, + uint32_t resid, + PRegister res_offset, + EVTXDataFormat data_format); static Instr::Pointer from_string(std::istream& is, ValueFactory& vf); }; class LoadFromScratch : public FetchInstr { public: - LoadFromScratch(const RegisterVec4& dst, const RegisterVec4::Swizzle& swizzle, PVirtualValue addr, uint32_t offset); + LoadFromScratch(const RegisterVec4& dst, + const RegisterVec4::Swizzle& swizzle, + PVirtualValue addr, + uint32_t offset); static Instr::Pointer from_string(std::istream& is, ValueFactory& vf); }; -} +} // namespace r600 #endif // INSTR_FETCH_H diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_lds.cpp b/src/gallium/drivers/r600/sfn/sfn_instr_lds.cpp index 8b42bc9371e..be9bda29212 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_lds.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_instr_lds.cpp @@ -25,8 +25,9 @@ */ #include "sfn_instr_lds.h" -#include "sfn_instr_alu.h" + #include "sfn_debug.h" +#include "sfn_instr_alu.h" namespace r600 { @@ -34,30 +35,33 @@ using std::istream; LDSReadInstr::LDSReadInstr(std::vector>& value, AluInstr::SrcValues& address): - m_address(address), - m_dest_value(value) + m_address(address), + m_dest_value(value) { assert(m_address.size() == m_dest_value.size()); - for (auto& v: value) + for (auto& v : value) v->add_parent(this); - for (auto& s: m_address) + for (auto& s : m_address) if (s->as_register()) s->as_register()->add_use(this); } -void LDSReadInstr::accept(ConstInstrVisitor& visitor) const +void +LDSReadInstr::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void LDSReadInstr::accept(InstrVisitor& visitor) +void +LDSReadInstr::accept(InstrVisitor& visitor) { visitor.visit(this); } -bool LDSReadInstr::remove_unused_components() +bool +LDSReadInstr::remove_unused_components() { uint8_t inactive_mask = 0; for (size_t i = 0; i < m_dest_value.size(); ++i) { @@ -90,21 +94,20 @@ bool LDSReadInstr::remove_unused_components() class SetLDSAddrProperty : public AluInstrVisitor { using AluInstrVisitor::visit; - void visit(AluInstr *instr) override { - instr->set_alu_flag(alu_lds_address); - } + void visit(AluInstr *instr) override { instr->set_alu_flag(alu_lds_address); } }; -AluInstr *LDSReadInstr::split(std::vector& out_block, AluInstr *last_lds_instr) +AluInstr * +LDSReadInstr::split(std::vector& out_block, AluInstr *last_lds_instr) { - AluInstr* first_instr = nullptr; + AluInstr *first_instr = nullptr; SetLDSAddrProperty prop; - for (auto& addr: m_address) { + for (auto& addr : m_address) { auto reg = addr->as_register(); if (reg) { reg->del_use(this); if (reg->parents().size() == 1) { - for (auto& p: reg->parents()) { + for (auto& p : reg->parents()) { p->accept(prop); } } @@ -131,9 +134,10 @@ AluInstr *LDSReadInstr::split(std::vector& out_block, AluInstr *last_ } } - for (auto& dest: m_dest_value) { + for (auto& dest : m_dest_value) { dest->del_parent(this); - auto instr = new AluInstr(op1_mov, dest, + auto instr = new AluInstr(op1_mov, + dest, new InlineConstant(ALU_SRC_LDS_OQ_A_POP), AluInstr::last_write); instr->add_required_instr(last_lds_instr); @@ -147,28 +151,31 @@ AluInstr *LDSReadInstr::split(std::vector& out_block, AluInstr *last_ return last_lds_instr; } -bool LDSReadInstr::do_ready() const +bool +LDSReadInstr::do_ready() const { unreachable("This instruction is not handled by the schduler"); return false; } -void LDSReadInstr::do_print(std::ostream& os) const +void +LDSReadInstr::do_print(std::ostream& os) const { os << "LDS_READ "; os << "[ "; - for (auto d: m_dest_value) { + for (auto d : m_dest_value) { os << *d << " "; } os << "] : [ "; - for (auto a: m_address) { + for (auto a : m_address) { os << *a << " "; } os << "]"; } -bool LDSReadInstr::is_equal_to(const LDSReadInstr& rhs) const +bool +LDSReadInstr::is_equal_to(const LDSReadInstr& rhs) const { if (m_address.size() != rhs.m_address.size()) return false; @@ -182,7 +189,8 @@ bool LDSReadInstr::is_equal_to(const LDSReadInstr& rhs) const return true; } -auto LDSReadInstr::from_string(istream& is, ValueFactory& value_factory) -> Pointer +auto +LDSReadInstr::from_string(istream& is, ValueFactory& value_factory) -> Pointer { /* LDS_READ [ d1, d2, d3 ... ] : a1 a2 a3 ... */ @@ -191,7 +199,7 @@ auto LDSReadInstr::from_string(istream& is, ValueFactory& value_factory) -> Poin is >> temp_str; assert(temp_str == "["); - std::vector > dests; + std::vector> dests; AluInstr::SrcValues srcs; is >> temp_str; @@ -219,12 +227,14 @@ auto LDSReadInstr::from_string(istream& is, ValueFactory& value_factory) -> Poin return new LDSReadInstr(dests, srcs); } -LDSAtomicInstr::LDSAtomicInstr(ESDOp op, PRegister dest, PVirtualValue address, +LDSAtomicInstr::LDSAtomicInstr(ESDOp op, + PRegister dest, + PVirtualValue address, const SrcValues& srcs): - m_opcode(op), - m_address(address), - m_dest(dest), - m_srcs(srcs) + m_opcode(op), + m_address(address), + m_dest(dest), + m_srcs(srcs) { if (m_dest) m_dest->add_parent(this); @@ -232,31 +242,33 @@ LDSAtomicInstr::LDSAtomicInstr(ESDOp op, PRegister dest, PVirtualValue address, if (m_address->as_register()) m_address->as_register()->add_use(this); - for (auto& s: m_srcs) { + for (auto& s : m_srcs) { if (s->as_register()) s->as_register()->add_use(this); } } - -void LDSAtomicInstr::accept(ConstInstrVisitor& visitor) const +void +LDSAtomicInstr::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void LDSAtomicInstr::accept(InstrVisitor& visitor) +void +LDSAtomicInstr::accept(InstrVisitor& visitor) { visitor.visit(this); } -AluInstr *LDSAtomicInstr::split(std::vector& out_block, AluInstr *last_lds_instr) +AluInstr * +LDSAtomicInstr::split(std::vector& out_block, AluInstr *last_lds_instr) { AluInstr::SrcValues srcs = {m_address}; - for(auto& s : m_srcs) + for (auto& s : m_srcs) srcs.push_back(s); - for(auto& s :srcs) { + for (auto& s : srcs) { if (s->as_register()) s->as_register()->del_use(this); } @@ -266,7 +278,7 @@ AluInstr *LDSAtomicInstr::split(std::vector& out_block, AluInstr *la if (reg) { reg->del_use(this); if (reg->parents().size() == 1) { - for (auto& p: reg->parents()) { + for (auto& p : reg->parents()) { p->accept(prop); } } @@ -284,7 +296,8 @@ AluInstr *LDSAtomicInstr::split(std::vector& out_block, AluInstr *la if (m_dest) { op_instr->set_alu_flag(alu_lds_group_start); m_dest->del_parent(this); - auto read_instr = new AluInstr(op1_mov, m_dest, + auto read_instr = new AluInstr(op1_mov, + m_dest, new InlineConstant(ALU_SRC_LDS_OQ_A_POP), AluInstr::last_write); read_instr->add_required_instr(op_instr); @@ -296,11 +309,11 @@ AluInstr *LDSAtomicInstr::split(std::vector& out_block, AluInstr *la return last_lds_instr; } -bool LDSAtomicInstr::replace_source(PRegister old_src, PVirtualValue new_src) +bool +LDSAtomicInstr::replace_source(PRegister old_src, PVirtualValue new_src) { bool process = false; - if (new_src->as_uniform() && m_srcs.size() > 2) { int nconst = 0; for (auto& s : m_srcs) { @@ -345,13 +358,15 @@ bool LDSAtomicInstr::replace_source(PRegister old_src, PVirtualValue new_src) return process; } -bool LDSAtomicInstr::do_ready() const +bool +LDSAtomicInstr::do_ready() const { unreachable("This instruction is not handled by the schduler"); return false; } -void LDSAtomicInstr::do_print(std::ostream& os) const +void +LDSAtomicInstr::do_print(std::ostream& os) const { auto ii = lds_ops.find(m_opcode); assert(ii != lds_ops.end()); @@ -367,7 +382,8 @@ void LDSAtomicInstr::do_print(std::ostream& os) const os << " " << *m_srcs[1]; } -bool LDSAtomicInstr::is_equal_to(const LDSAtomicInstr& rhs) const +bool +LDSAtomicInstr::is_equal_to(const LDSAtomicInstr& rhs) const { if (m_srcs.size() != rhs.m_srcs.size()) return false; @@ -377,13 +393,12 @@ bool LDSAtomicInstr::is_equal_to(const LDSAtomicInstr& rhs) const return false; } - return m_opcode == rhs.m_opcode && - sfn_value_equal(m_address, rhs.m_address) && - sfn_value_equal(m_dest, rhs.m_dest); + return m_opcode == rhs.m_opcode && sfn_value_equal(m_address, rhs.m_address) && + sfn_value_equal(m_dest, rhs.m_dest); } - -auto LDSAtomicInstr::from_string(istream& is, ValueFactory& value_factory) -> Pointer +auto +LDSAtomicInstr::from_string(istream& is, ValueFactory& value_factory) -> Pointer { /* LDS WRITE2 __.x [ R1.x ] : R2.y R3.z */ /* LDS WRITE __.x [ R1.x ] : R2.y */ @@ -434,5 +449,4 @@ auto LDSAtomicInstr::from_string(istream& is, ValueFactory& value_factory) -> Po return new LDSAtomicInstr(opcode, dest, addr, srcs); } - -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_lds.h b/src/gallium/drivers/r600/sfn/sfn_instr_lds.h index da4a587e80c..f354cabe5a7 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_lds.h +++ b/src/gallium/drivers/r600/sfn/sfn_instr_lds.h @@ -37,12 +37,12 @@ public: LDSReadInstr(std::vector>& value, AluInstr::SrcValues& address); - unsigned num_values() const { return m_dest_value.size();} - auto address(unsigned i) const { return m_address[i];} - auto dest(unsigned i) const { return m_dest_value[i];} + unsigned num_values() const { return m_dest_value.size(); } + auto address(unsigned i) const { return m_address[i]; } + auto dest(unsigned i) const { return m_dest_value[i]; } - auto address(unsigned i){ return m_address[i];} - auto dest(unsigned i) { return m_dest_value[i];} + auto address(unsigned i) { return m_address[i]; } + auto dest(unsigned i) { return m_dest_value[i]; } void accept(ConstInstrVisitor& visitor) const override; void accept(InstrVisitor& visitor) override; @@ -55,7 +55,6 @@ public: bool remove_unused_components(); private: - bool do_ready() const override; void do_print(std::ostream& os) const override; @@ -70,17 +69,17 @@ public: LDSAtomicInstr(ESDOp op, PRegister dest, PVirtualValue address, const SrcValues& src); - auto address() const { return m_address;} - auto dest() const { return m_dest;} - auto src0() const { return m_srcs[0];} - auto src1() const { return m_srcs.size() > 1 ? m_srcs[1] : nullptr;} + auto address() const { return m_address; } + auto dest() const { return m_dest; } + auto src0() const { return m_srcs[0]; } + auto src1() const { return m_srcs.size() > 1 ? m_srcs[1] : nullptr; } - PVirtualValue address() { return m_address;} - PRegister dest() { return m_dest;} - PVirtualValue src0() { return m_srcs[0];} - PVirtualValue src1() { return m_srcs.size() > 1 ? m_srcs[1] : nullptr;} + PVirtualValue address() { return m_address; } + PRegister dest() { return m_dest; } + PVirtualValue src0() { return m_srcs[0]; } + PVirtualValue src1() { return m_srcs.size() > 1 ? m_srcs[1] : nullptr; } - unsigned op() const {return m_opcode;} + unsigned op() const { return m_opcode; } void accept(ConstInstrVisitor& visitor) const override; void accept(InstrVisitor& visitor) override; @@ -101,6 +100,6 @@ private: SrcValues m_srcs; }; -} +} // namespace r600 #endif // LDSINSTR_H diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_mem.cpp b/src/gallium/drivers/r600/sfn/sfn_instr_mem.cpp index 77b9ee05ab3..99f00ab6272 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_mem.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_instr_mem.cpp @@ -25,20 +25,19 @@ */ #include "sfn_instr_mem.h" + #include "sfn_instr_fetch.h" #include "sfn_instr_tex.h" #include "sfn_shader.h" namespace r600 { - -GDSInstr::GDSInstr(ESDOp op, Register *dest, - const RegisterVec4& src, int uav_base, - PRegister uav_id): - InstrWithResource(uav_base, uav_id), - m_op(op), - m_dest(dest), - m_src(src) +GDSInstr::GDSInstr( + ESDOp op, Register *dest, const RegisterVec4& src, int uav_base, PRegister uav_id): + InstrWithResource(uav_base, uav_id), + m_op(op), + m_dest(dest), + m_src(src) { set_always_keep(); @@ -46,12 +45,12 @@ GDSInstr::GDSInstr(ESDOp op, Register *dest, m_dest->add_parent(this); } -bool GDSInstr::is_equal_to(const GDSInstr& rhs) const +bool +GDSInstr::is_equal_to(const GDSInstr& rhs) const { -#define NE(X) (X != rhs. X) +#define NE(X) (X != rhs.X) - if (NE(m_op) || - NE(m_src)) + if (NE(m_op) || NE(m_src)) return false; sfn_value_equal(m_dest, rhs.m_dest); @@ -59,33 +58,36 @@ bool GDSInstr::is_equal_to(const GDSInstr& rhs) const return resource_is_equal(rhs); } -void GDSInstr::accept(ConstInstrVisitor& visitor) const +void +GDSInstr::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void GDSInstr::accept(InstrVisitor& visitor) +void +GDSInstr::accept(InstrVisitor& visitor) { visitor.visit(this); } -bool GDSInstr::do_ready() const +bool +GDSInstr::do_ready() const { - return m_src.ready(block_id(), index()) && - resource_ready(block_id(), index()); + return m_src.ready(block_id(), index()) && resource_ready(block_id(), index()); } -void GDSInstr::do_print(std::ostream& os) const +void +GDSInstr::do_print(std::ostream& os) const { - os << "GDS " << lds_ops.at(m_op).name - << *m_dest; + os << "GDS " << lds_ops.at(m_op).name << *m_dest; os << " " << m_src; os << " BASE:" << resource_base(); print_resource_offset(os); } -bool GDSInstr::emit_atomic_counter(nir_intrinsic_instr *intr, Shader& shader) +bool +GDSInstr::emit_atomic_counter(nir_intrinsic_instr *intr, Shader& shader) { switch (intr->intrinsic) { case nir_intrinsic_atomic_counter_add: @@ -109,7 +111,8 @@ bool GDSInstr::emit_atomic_counter(nir_intrinsic_instr *intr, Shader& shader) } } -static ESDOp get_opcode(const nir_intrinsic_op opcode) +static ESDOp +get_opcode(const nir_intrinsic_op opcode) { switch (opcode) { case nir_intrinsic_atomic_counter_add: @@ -140,7 +143,8 @@ static ESDOp get_opcode(const nir_intrinsic_op opcode) } } -static ESDOp get_opcode_wo(const nir_intrinsic_op opcode) +static ESDOp +get_opcode_wo(const nir_intrinsic_op opcode) { switch (opcode) { case nir_intrinsic_atomic_counter_add: @@ -169,19 +173,21 @@ static ESDOp get_opcode_wo(const nir_intrinsic_op opcode) } } - -bool GDSInstr::emit_atomic_op2(nir_intrinsic_instr *instr, Shader& shader) +bool +GDSInstr::emit_atomic_op2(nir_intrinsic_instr *instr, Shader& shader) { auto& vf = shader.value_factory(); bool read_result = !instr->dest.is_ssa || !list_is_empty(&instr->dest.ssa.uses); - ESDOp op = read_result ? get_opcode(instr->intrinsic) : - get_opcode_wo(instr->intrinsic); + ESDOp op = + read_result ? get_opcode(instr->intrinsic) : get_opcode_wo(instr->intrinsic); if (DS_OP_INVALID == op) return false; - auto [offset, uav_id] = shader.evaluate_resource_offset(instr, 0); {} + auto [offset, uav_id] = shader.evaluate_resource_offset(instr, 0); + { + } offset += nir_intrinsic_base(instr); auto dest = vf.dest(instr->dest, 0, pin_free); @@ -190,7 +196,8 @@ bool GDSInstr::emit_atomic_op2(nir_intrinsic_instr *instr, Shader& shader) auto src_val = vf.src(instr->src[1], 0); if (!src_val->as_register()) { auto temp_src_val = vf.temp_register(); - shader.emit_instruction(new AluInstr(op1_mov, temp_src_val, src_val, AluInstr::last_write)); + shader.emit_instruction( + new AluInstr(op1_mov, temp_src_val, src_val, AluInstr::last_write)); src_as_register = temp_src_val; } else src_as_register = src_val->as_register(); @@ -207,22 +214,31 @@ bool GDSInstr::emit_atomic_op2(nir_intrinsic_instr *instr, Shader& shader) auto dest = vf.dest(instr->dest, 0, pin_free); auto tmp = vf.temp_vec4(pin_group, {0, 1, 7, 7}); if (uav_id) - shader.emit_instruction(new AluInstr(op3_muladd_uint24, tmp[0], uav_id, vf.literal(4), vf.literal(4 * offset), - AluInstr::write)); + shader.emit_instruction(new AluInstr(op3_muladd_uint24, + tmp[0], + uav_id, + vf.literal(4), + vf.literal(4 * offset), + AluInstr::write)); else - shader.emit_instruction(new AluInstr(op1_mov, tmp[0], vf.literal(4 * offset), AluInstr::write)); - shader.emit_instruction(new AluInstr(op1_mov, tmp[1], src_val, AluInstr::last_write)); + shader.emit_instruction( + new AluInstr(op1_mov, tmp[0], vf.literal(4 * offset), AluInstr::write)); + shader.emit_instruction( + new AluInstr(op1_mov, tmp[1], src_val, AluInstr::last_write)); ir = new GDSInstr(op, dest, tmp, 0, nullptr); } shader.emit_instruction(ir); return true; } -bool GDSInstr::emit_atomic_read(nir_intrinsic_instr *instr, Shader& shader) +bool +GDSInstr::emit_atomic_read(nir_intrinsic_instr *instr, Shader& shader) { auto& vf = shader.value_factory(); - auto [offset, uav_id] = shader.evaluate_resource_offset(instr, 0); {} + auto [offset, uav_id] = shader.evaluate_resource_offset(instr, 0); + { + } offset += shader.remap_atomic_base(nir_intrinsic_base(instr)); auto dest = vf.dest(instr->dest, 0, pin_free); @@ -230,15 +246,20 @@ bool GDSInstr::emit_atomic_read(nir_intrinsic_instr *instr, Shader& shader) GDSInstr *ir = nullptr; if (shader.chip_class() < ISA_CC_CAYMAN) { - RegisterVec4 src = RegisterVec4(0, true, {7,7,7,7}); + RegisterVec4 src = RegisterVec4(0, true, {7, 7, 7, 7}); ir = new GDSInstr(DS_OP_READ_RET, dest, src, offset, uav_id); } else { auto tmp = vf.temp_vec4(pin_group, {0, 7, 7, 7}); if (uav_id) - shader.emit_instruction(new AluInstr(op3_muladd_uint24, tmp[0], uav_id, vf.literal(4), vf.literal(4 * offset), - AluInstr::write)); + shader.emit_instruction(new AluInstr(op3_muladd_uint24, + tmp[0], + uav_id, + vf.literal(4), + vf.literal(4 * offset), + AluInstr::write)); else - shader.emit_instruction(new AluInstr(op1_mov, tmp[0], vf.literal(4 * offset), AluInstr::write)); + shader.emit_instruction( + new AluInstr(op1_mov, tmp[0], vf.literal(4 * offset), AluInstr::write)); ir = new GDSInstr(DS_OP_READ_RET, dest, tmp, 0, nullptr); } @@ -247,12 +268,15 @@ bool GDSInstr::emit_atomic_read(nir_intrinsic_instr *instr, Shader& shader) return true; } -bool GDSInstr::emit_atomic_inc(nir_intrinsic_instr *instr, Shader& shader) +bool +GDSInstr::emit_atomic_inc(nir_intrinsic_instr *instr, Shader& shader) { auto& vf = shader.value_factory(); bool read_result = !instr->dest.is_ssa || !list_is_empty(&instr->dest.ssa.uses); - auto [offset, uav_id] = shader.evaluate_resource_offset(instr, 0); {} + auto [offset, uav_id] = shader.evaluate_resource_offset(instr, 0); + { + } offset += shader.remap_atomic_base(nir_intrinsic_base(instr)); GDSInstr *ir = nullptr; @@ -260,31 +284,39 @@ bool GDSInstr::emit_atomic_inc(nir_intrinsic_instr *instr, Shader& shader) if (shader.chip_class() < ISA_CC_CAYMAN) { auto dest = vf.dest(instr->dest, 0, pin_free); RegisterVec4 src(nullptr, shader.atomic_update(), nullptr, nullptr, pin_chan); - ir = new GDSInstr(read_result ? DS_OP_ADD_RET : DS_OP_ADD, - dest, src, offset, uav_id); + ir = + new GDSInstr(read_result ? DS_OP_ADD_RET : DS_OP_ADD, dest, src, offset, uav_id); } else { auto dest = vf.dest(instr->dest, 0, pin_free); auto tmp = vf.temp_vec4(pin_group, {0, 1, 7, 7}); if (uav_id) - shader.emit_instruction(new AluInstr(op3_muladd_uint24, tmp[0], uav_id, vf.literal(4), vf.literal(4 * offset), - AluInstr::write)); + shader.emit_instruction(new AluInstr(op3_muladd_uint24, + tmp[0], + uav_id, + vf.literal(4), + vf.literal(4 * offset), + AluInstr::write)); else - shader.emit_instruction(new AluInstr(op1_mov, tmp[0], vf.literal(4 * offset), AluInstr::write)); + shader.emit_instruction( + new AluInstr(op1_mov, tmp[0], vf.literal(4 * offset), AluInstr::write)); - shader.emit_instruction(new AluInstr(op1_mov, tmp[1], shader.atomic_update(), AluInstr::last_write)); - ir = new GDSInstr(read_result ? DS_OP_ADD_RET : DS_OP_ADD, - dest, tmp, 0, nullptr); + shader.emit_instruction( + new AluInstr(op1_mov, tmp[1], shader.atomic_update(), AluInstr::last_write)); + ir = new GDSInstr(read_result ? DS_OP_ADD_RET : DS_OP_ADD, dest, tmp, 0, nullptr); } shader.emit_instruction(ir); return true; } -bool GDSInstr::emit_atomic_pre_dec(nir_intrinsic_instr *instr, Shader& shader) +bool +GDSInstr::emit_atomic_pre_dec(nir_intrinsic_instr *instr, Shader& shader) { auto& vf = shader.value_factory(); - auto [offset, uav_id] = shader.evaluate_resource_offset(instr, 0); {} + auto [offset, uav_id] = shader.evaluate_resource_offset(instr, 0); + { + } offset += shader.remap_atomic_base(nir_intrinsic_base(instr)); auto *tmp_dest = vf.temp_register(); @@ -297,81 +329,95 @@ bool GDSInstr::emit_atomic_pre_dec(nir_intrinsic_instr *instr, Shader& shader) } else { auto tmp = vf.temp_vec4(pin_group, {0, 1, 7, 7}); if (uav_id) - shader.emit_instruction(new AluInstr(op3_muladd_uint24, tmp[0], uav_id, vf.literal(4), vf.literal(4 * offset), - AluInstr::write)); + shader.emit_instruction(new AluInstr(op3_muladd_uint24, + tmp[0], + uav_id, + vf.literal(4), + vf.literal(4 * offset), + AluInstr::write)); else - shader.emit_instruction(new AluInstr(op1_mov, tmp[0], vf.literal(4 * offset), AluInstr::write)); + shader.emit_instruction( + new AluInstr(op1_mov, tmp[0], vf.literal(4 * offset), AluInstr::write)); - shader.emit_instruction(new AluInstr(op1_mov, tmp[1], shader.atomic_update(), AluInstr::last_write)); + shader.emit_instruction( + new AluInstr(op1_mov, tmp[1], shader.atomic_update(), AluInstr::last_write)); ir = new GDSInstr(DS_OP_SUB_RET, tmp_dest, tmp, 0, nullptr); } shader.emit_instruction(ir); - shader.emit_instruction(new AluInstr(op2_sub_int, vf.dest(instr->dest, 0, pin_free), - tmp_dest, vf.one_i(), AluInstr::last_write)); + shader.emit_instruction(new AluInstr(op2_sub_int, + vf.dest(instr->dest, 0, pin_free), + tmp_dest, + vf.one_i(), + AluInstr::last_write)); return true; } - -RatInstr::RatInstr(ECFOpCode cf_opcode, ERatOp rat_op, - const RegisterVec4& data, const RegisterVec4& index, - int rat_id, PRegister rat_id_offset, - int burst_count, int comp_mask, int element_size): - InstrWithResource(rat_id, rat_id_offset), - m_cf_opcode(cf_opcode), - m_rat_op(rat_op), - m_data(data), - m_index(index), - m_burst_count(burst_count), - m_comp_mask(comp_mask), - m_element_size(element_size) +RatInstr::RatInstr(ECFOpCode cf_opcode, + ERatOp rat_op, + const RegisterVec4& data, + const RegisterVec4& index, + int rat_id, + PRegister rat_id_offset, + int burst_count, + int comp_mask, + int element_size): + InstrWithResource(rat_id, rat_id_offset), + m_cf_opcode(cf_opcode), + m_rat_op(rat_op), + m_data(data), + m_index(index), + m_burst_count(burst_count), + m_comp_mask(comp_mask), + m_element_size(element_size) { set_always_keep(); m_data.add_use(this); m_index.add_use(this); } - -void RatInstr::accept(ConstInstrVisitor& visitor) const +void +RatInstr::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void RatInstr::accept(InstrVisitor& visitor) +void +RatInstr::accept(InstrVisitor& visitor) { visitor.visit(this); } -bool RatInstr::is_equal_to(const RatInstr& lhs) const +bool +RatInstr::is_equal_to(const RatInstr& lhs) const { (void)lhs; assert(0); return false; } -bool RatInstr::do_ready() const -{ +bool +RatInstr::do_ready() const +{ if (m_rat_op != STORE_TYPED) { - for (auto i: required_instr()) { + for (auto i : required_instr()) { if (!i->is_scheduled()) { return false; } } } - return m_data.ready(block_id(), index()) && - m_index.ready(block_id(), index()); + return m_data.ready(block_id(), index()) && m_index.ready(block_id(), index()); } -void RatInstr::do_print(std::ostream& os) const +void +RatInstr::do_print(std::ostream& os) const { os << "MEM_RAT RAT " << resource_base(); print_resource_offset(os); os << " @" << m_index; os << " OP:" << m_rat_op << " " << m_data; - os << " BC:" << m_burst_count - << " MASK:" << m_comp_mask - << " ES:" << m_element_size; + os << " BC:" << m_burst_count << " MASK:" << m_comp_mask << " ES:" << m_element_size; if (m_need_ack) os << " ACK"; } @@ -379,7 +425,7 @@ void RatInstr::do_print(std::ostream& os) const static RatInstr::ERatOp get_rat_opcode(const nir_intrinsic_op opcode, pipe_format format) { - switch (opcode) { + switch (opcode) { case nir_intrinsic_image_load: return RatInstr::NOP_RTN; case nir_intrinsic_ssbo_atomic_add: @@ -423,7 +469,7 @@ get_rat_opcode(const nir_intrinsic_op opcode, pipe_format format) static RatInstr::ERatOp get_rat_opcode_wo(const nir_intrinsic_op opcode, pipe_format format) { - switch (opcode) { + switch (opcode) { case nir_intrinsic_ssbo_atomic_add: case nir_intrinsic_image_atomic_add: return RatInstr::ADD; @@ -462,7 +508,8 @@ get_rat_opcode_wo(const nir_intrinsic_op opcode, pipe_format format) } } -bool RatInstr::emit(nir_intrinsic_instr *intr, Shader& shader) +bool +RatInstr::emit(nir_intrinsic_instr *intr, Shader& shader) { switch (intr->intrinsic) { case nir_intrinsic_load_ssbo: @@ -503,9 +550,10 @@ bool RatInstr::emit(nir_intrinsic_instr *intr, Shader& shader) } } -bool RatInstr::emit_ssbo_load(nir_intrinsic_instr *intr, Shader& shader) +bool +RatInstr::emit_ssbo_load(nir_intrinsic_instr *intr, Shader& shader) { - auto &vf = shader.value_factory(); + auto& vf = shader.value_factory(); auto dest = vf.dest_vec4(intr->dest, pin_group); /** src0 not used, should be some offset */ @@ -513,32 +561,28 @@ bool RatInstr::emit_ssbo_load(nir_intrinsic_instr *intr, Shader& shader) auto addr_temp = vf.temp_register(); /** Should be lowered in nir */ - shader.emit_instruction(new AluInstr(op2_lshr_int, addr_temp, addr, vf.literal(2), - {alu_write, alu_last_instr})); + shader.emit_instruction(new AluInstr( + op2_lshr_int, addr_temp, addr, vf.literal(2), {alu_write, alu_last_instr})); - const EVTXDataFormat formats[4] = { - fmt_32, - fmt_32_32, - fmt_32_32_32, - fmt_32_32_32_32 - }; + const EVTXDataFormat formats[4] = {fmt_32, fmt_32_32, fmt_32_32_32, fmt_32_32_32_32}; RegisterVec4::Swizzle dest_swz[4] = { - {0,7,7,7}, - {0,1,7,7}, - {0,1,2,7}, - {0,1,2,3} + {0, 7, 7, 7}, + {0, 1, 7, 7}, + {0, 1, 2, 7}, + {0, 1, 2, 3} }; int comp_idx = nir_dest_num_components(intr->dest) - 1; - auto [offset, res_offset] = shader.evaluate_resource_offset(intr, 0); {} + auto [offset, res_offset] = shader.evaluate_resource_offset(intr, 0); + { + } - auto res_id = R600_IMAGE_REAL_RESOURCE_OFFSET + offset + - shader.ssbo_image_offset(); + auto res_id = R600_IMAGE_REAL_RESOURCE_OFFSET + offset + shader.ssbo_image_offset(); - auto ir = new LoadFromBuffer(dest, dest_swz[comp_idx], addr_temp, 0, - res_id, res_offset, formats[comp_idx]); + auto ir = new LoadFromBuffer( + dest, dest_swz[comp_idx], addr_temp, 0, res_id, res_offset, formats[comp_idx]); ir->set_fetch_flag(FetchInstr::use_tc); ir->set_num_format(vtx_nf_int); @@ -546,81 +590,104 @@ bool RatInstr::emit_ssbo_load(nir_intrinsic_instr *intr, Shader& shader) return true; } -bool RatInstr::emit_ssbo_store(nir_intrinsic_instr *instr, Shader& shader) +bool +RatInstr::emit_ssbo_store(nir_intrinsic_instr *instr, Shader& shader) { - auto &vf = shader.value_factory(); + auto& vf = shader.value_factory(); auto orig_addr = vf.src(instr->src[2], 0); auto addr_base = vf.temp_register(); auto [offset, rat_id] = shader.evaluate_resource_offset(instr, 1); - shader.emit_instruction(new AluInstr(op2_lshr_int, addr_base, orig_addr, - vf.literal(2), AluInstr::write)); + shader.emit_instruction( + new AluInstr(op2_lshr_int, addr_base, orig_addr, vf.literal(2), AluInstr::write)); for (unsigned i = 0; i < nir_src_num_components(instr->src[0]); ++i) { - auto addr_vec = vf.temp_vec4(pin_group, {0,1,2,7}); + auto addr_vec = vf.temp_vec4(pin_group, {0, 1, 2, 7}); if (i == 0) { - shader.emit_instruction(new AluInstr(op1_mov, addr_vec[0], addr_base, AluInstr::last_write)); + shader.emit_instruction( + new AluInstr(op1_mov, addr_vec[0], addr_base, AluInstr::last_write)); } else { - shader.emit_instruction(new AluInstr(op2_add_int, addr_vec[0], addr_base, - vf.literal(i), - AluInstr::last_write)); + shader.emit_instruction(new AluInstr( + op2_add_int, addr_vec[0], addr_base, vf.literal(i), AluInstr::last_write)); } auto value = vf.src(instr->src[0], i); PRegister v = vf.temp_register(0); shader.emit_instruction(new AluInstr(op1_mov, v, value, AluInstr::last_write)); auto value_vec = RegisterVec4(v, nullptr, nullptr, nullptr, pin_chan); - auto store = new RatInstr(cf_mem_rat, RatInstr::STORE_TYPED, - value_vec, addr_vec, offset + shader.ssbo_image_offset(), - rat_id, 1, 1, 0); + auto store = new RatInstr(cf_mem_rat, + RatInstr::STORE_TYPED, + value_vec, + addr_vec, + offset + shader.ssbo_image_offset(), + rat_id, + 1, + 1, + 0); shader.emit_instruction(store); } return true; } -bool RatInstr::emit_ssbo_atomic_op(nir_intrinsic_instr *intr, Shader& shader) +bool +RatInstr::emit_ssbo_atomic_op(nir_intrinsic_instr *intr, Shader& shader) { auto& vf = shader.value_factory(); - auto [imageid, image_offset] = shader.evaluate_resource_offset(intr, 0); {} - - bool read_result = !intr->dest.is_ssa || !list_is_empty(&intr->dest.ssa.uses); - auto opcode = read_result ? get_rat_opcode(intr->intrinsic, PIPE_FORMAT_R32_UINT) : - get_rat_opcode_wo(intr->intrinsic, PIPE_FORMAT_R32_UINT); - - auto coord_orig = vf.src(intr->src[1], 0); - auto coord = vf.temp_register(0); - - auto data_vec4 = vf.temp_vec4(pin_chgr, {0,1,2,3}); - - shader.emit_instruction(new AluInstr(op2_lshr_int, coord, coord_orig, vf.literal(2), AluInstr::last_write)); - - shader.emit_instruction(new AluInstr(op1_mov, data_vec4[1], shader.rat_return_address(), AluInstr::write)); - - - if (intr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap) { - shader.emit_instruction(new AluInstr(op1_mov, data_vec4[0], vf.src(intr->src[3], 0), AluInstr::write)); - shader.emit_instruction(new AluInstr(op1_mov, data_vec4[shader.chip_class() == ISA_CC_CAYMAN ? 2 : 3], - vf.src(intr->src[2], 0), {alu_last_instr, alu_write})); - } else { - shader.emit_instruction(new AluInstr(op1_mov, data_vec4[0], vf.src(intr->src[2], 0), AluInstr::last_write)); + auto [imageid, image_offset] = shader.evaluate_resource_offset(intr, 0); + { } + bool read_result = !intr->dest.is_ssa || !list_is_empty(&intr->dest.ssa.uses); + auto opcode = read_result ? get_rat_opcode(intr->intrinsic, PIPE_FORMAT_R32_UINT) + : get_rat_opcode_wo(intr->intrinsic, PIPE_FORMAT_R32_UINT); + + auto coord_orig = vf.src(intr->src[1], 0); + auto coord = vf.temp_register(0); + + auto data_vec4 = vf.temp_vec4(pin_chgr, {0, 1, 2, 3}); + + shader.emit_instruction( + new AluInstr(op2_lshr_int, coord, coord_orig, vf.literal(2), AluInstr::last_write)); + + shader.emit_instruction( + new AluInstr(op1_mov, data_vec4[1], shader.rat_return_address(), AluInstr::write)); + + if (intr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap) { + shader.emit_instruction( + new AluInstr(op1_mov, data_vec4[0], vf.src(intr->src[3], 0), AluInstr::write)); + shader.emit_instruction( + new AluInstr(op1_mov, + data_vec4[shader.chip_class() == ISA_CC_CAYMAN ? 2 : 3], + vf.src(intr->src[2], 0), + {alu_last_instr, alu_write})); + } else { + shader.emit_instruction(new AluInstr( + op1_mov, data_vec4[0], vf.src(intr->src[2], 0), AluInstr::last_write)); + } RegisterVec4 out_vec(coord, coord, coord, coord, pin_chgr); - auto atomic = new RatInstr(cf_mem_rat, opcode, data_vec4, out_vec, imageid + shader.ssbo_image_offset(), - image_offset, 1, 0xf, 0); + auto atomic = new RatInstr(cf_mem_rat, + opcode, + data_vec4, + out_vec, + imageid + shader.ssbo_image_offset(), + image_offset, + 1, + 0xf, + 0); shader.emit_instruction(atomic); - atomic->set_ack(); + atomic->set_ack(); if (read_result) { atomic->set_instr_flag(ack_rat_return_write); auto dest = vf.dest_vec4(intr->dest, pin_group); auto fetch = new FetchInstr(vc_fetch, - dest, {0, 1, 2, 3}, + dest, + {0, 1, 2, 3}, shader.rat_return_address(), 0, no_index_offset, @@ -640,10 +707,10 @@ bool RatInstr::emit_ssbo_atomic_op(nir_intrinsic_instr *intr, Shader& shader) } return true; - } -bool RatInstr::emit_ssbo_size(nir_intrinsic_instr *intr, Shader& shader) +bool +RatInstr::emit_ssbo_size(nir_intrinsic_instr *intr, Shader& shader) { auto& vf = shader.value_factory(); auto dest = vf.dest_vec4(intr->dest, pin_group); @@ -655,39 +722,43 @@ bool RatInstr::emit_ssbo_size(nir_intrinsic_instr *intr, Shader& shader) else assert(0 && "dynamic buffer offset not supported in buffer_size"); - shader.emit_instruction(new QueryBufferSizeInstr(dest, {0,1,2,3},res_id)); + shader.emit_instruction(new QueryBufferSizeInstr(dest, {0, 1, 2, 3}, res_id)); return true; } -bool RatInstr::emit_image_store(nir_intrinsic_instr *intrin, Shader& shader) +bool +RatInstr::emit_image_store(nir_intrinsic_instr *intrin, Shader& shader) { auto& vf = shader.value_factory(); - auto [imageid, image_offset] = shader.evaluate_resource_offset(intrin, 0); {} - + auto [imageid, image_offset] = shader.evaluate_resource_offset(intrin, 0); + { + } auto coord_load = vf.src_vec4(intrin->src[1], pin_chan); - auto coord = vf.temp_vec4(pin_chgr); + auto coord = vf.temp_vec4(pin_chgr); auto value_load = vf.src_vec4(intrin->src[3], pin_chan); - auto value = vf.temp_vec4(pin_chgr); + auto value = vf.temp_vec4(pin_chgr); - RegisterVec4::Swizzle swizzle = {0,1,2,3}; + RegisterVec4::Swizzle swizzle = {0, 1, 2, 3}; if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_1D && nir_intrinsic_image_array(intrin)) - swizzle = {0,2,1,3}; + swizzle = {0, 2, 1, 3}; for (int i = 0; i < 4; ++i) { auto flags = i != 3 ? AluInstr::write : AluInstr::last_write; - shader.emit_instruction(new AluInstr(op1_mov, coord[swizzle[i]], coord_load[i], flags)); + shader.emit_instruction( + new AluInstr(op1_mov, coord[swizzle[i]], coord_load[i], flags)); } for (int i = 0; i < 4; ++i) { auto flags = i != 3 ? AluInstr::write : AluInstr::last_write; shader.emit_instruction(new AluInstr(op1_mov, value[i], value_load[i], flags)); } - auto op = cf_mem_rat; //nir_intrinsic_access(intrin) & ACCESS_COHERENT ? cf_mem_rat_cacheless : cf_mem_rat; - auto store = new RatInstr(op, RatInstr::STORE_TYPED, value, coord, imageid, - image_offset, 1, 0xf, 0); + auto op = cf_mem_rat; // nir_intrinsic_access(intrin) & ACCESS_COHERENT ? + // cf_mem_rat_cacheless : cf_mem_rat; + auto store = new RatInstr( + op, RatInstr::STORE_TYPED, value, coord, imageid, image_offset, 1, 0xf, 0); store->set_ack(); if (nir_intrinsic_access(intrin) & ACCESS_INCLUDE_HELPERS) @@ -697,44 +768,54 @@ bool RatInstr::emit_image_store(nir_intrinsic_instr *intrin, Shader& shader) return true; } -bool RatInstr::emit_image_load_or_atomic(nir_intrinsic_instr *intrin, Shader& shader) +bool +RatInstr::emit_image_load_or_atomic(nir_intrinsic_instr *intrin, Shader& shader) { auto& vf = shader.value_factory(); - auto [imageid, image_offset] = shader.evaluate_resource_offset(intrin, 0); {} + auto [imageid, image_offset] = shader.evaluate_resource_offset(intrin, 0); + { + } bool read_result = !intrin->dest.is_ssa || !list_is_empty(&intrin->dest.ssa.uses); - auto opcode = read_result ? get_rat_opcode(intrin->intrinsic, PIPE_FORMAT_R32_UINT) : - get_rat_opcode_wo(intrin->intrinsic, PIPE_FORMAT_R32_UINT); + auto opcode = read_result ? get_rat_opcode(intrin->intrinsic, PIPE_FORMAT_R32_UINT) + : get_rat_opcode_wo(intrin->intrinsic, PIPE_FORMAT_R32_UINT); - auto coord_orig = vf.src_vec4(intrin->src[1], pin_chan); + auto coord_orig = vf.src_vec4(intrin->src[1], pin_chan); auto coord = vf.temp_vec4(pin_chgr); - auto data_vec4 = vf.temp_vec4(pin_chgr, {0,1,2,3}); + auto data_vec4 = vf.temp_vec4(pin_chgr, {0, 1, 2, 3}); - RegisterVec4::Swizzle swizzle = {0,1,2,3}; + RegisterVec4::Swizzle swizzle = {0, 1, 2, 3}; if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_1D && nir_intrinsic_image_array(intrin)) - swizzle = {0,2,1,3}; + swizzle = {0, 2, 1, 3}; for (int i = 0; i < 4; ++i) { auto flags = i != 3 ? AluInstr::write : AluInstr::last_write; - shader.emit_instruction(new AluInstr(op1_mov, coord[swizzle[i]], coord_orig[i], flags)); + shader.emit_instruction( + new AluInstr(op1_mov, coord[swizzle[i]], coord_orig[i], flags)); } - shader.emit_instruction(new AluInstr(op1_mov, data_vec4[1], shader.rat_return_address(), AluInstr::write)); + shader.emit_instruction( + new AluInstr(op1_mov, data_vec4[1], shader.rat_return_address(), AluInstr::write)); if (intrin->intrinsic == nir_intrinsic_image_atomic_comp_swap) { - shader.emit_instruction(new AluInstr(op1_mov, data_vec4[0], vf.src(intrin->src[4], 0), AluInstr::write)); - shader.emit_instruction(new AluInstr(op1_mov, data_vec4[shader.chip_class() == ISA_CC_CAYMAN ? 2 : 3], - vf.src(intrin->src[3], 0), AluInstr::last_write)); + shader.emit_instruction( + new AluInstr(op1_mov, data_vec4[0], vf.src(intrin->src[4], 0), AluInstr::write)); + shader.emit_instruction( + new AluInstr(op1_mov, + data_vec4[shader.chip_class() == ISA_CC_CAYMAN ? 2 : 3], + vf.src(intrin->src[3], 0), + AluInstr::last_write)); } else { - shader.emit_instruction(new AluInstr(op1_mov, data_vec4[0], - vf.src(intrin->src[3], 0), AluInstr::write)); - shader.emit_instruction(new AluInstr(op1_mov, data_vec4[2], vf.zero(), AluInstr::last_write)); + shader.emit_instruction( + new AluInstr(op1_mov, data_vec4[0], vf.src(intrin->src[3], 0), AluInstr::write)); + shader.emit_instruction( + new AluInstr(op1_mov, data_vec4[2], vf.zero(), AluInstr::last_write)); } - auto atomic = new RatInstr(cf_mem_rat, opcode, data_vec4, coord, imageid, - image_offset, 1, 0xf, 0); + auto atomic = + new RatInstr(cf_mem_rat, opcode, data_vec4, coord, imageid, image_offset, 1, 0xf, 0); shader.emit_instruction(atomic); atomic->set_ack(); @@ -750,7 +831,8 @@ bool RatInstr::emit_image_load_or_atomic(nir_intrinsic_instr *intrin, Shader& sh r600_vertex_data_type(format, &fmt, &num_format, &format_comp, &endian); auto fetch = new FetchInstr(vc_fetch, - dest, {0, 1, 2, 3}, + dest, + {0, 1, 2, 3}, shader.rat_return_address(), 0, no_index_offset, @@ -776,11 +858,12 @@ bool RatInstr::emit_image_load_or_atomic(nir_intrinsic_instr *intrin, Shader& sh #define R600_SHADER_BUFFER_INFO_SEL (512 + R600_BUFFER_INFO_OFFSET / 16) -bool RatInstr::emit_image_size(nir_intrinsic_instr *intrin, Shader& shader) +bool +RatInstr::emit_image_size(nir_intrinsic_instr *intrin, Shader& shader) { auto& vf = shader.value_factory(); - auto src = RegisterVec4(0, true, {4,4,4,4}); + auto src = RegisterVec4(0, true, {4, 4, 4, 4}); assert(nir_src_as_uint(intrin->src[1]) == 0); @@ -795,28 +878,38 @@ bool RatInstr::emit_image_size(nir_intrinsic_instr *intrin, Shader& shader) if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_BUF) { auto dest = vf.dest_vec4(intrin->dest, pin_group); - shader.emit_instruction(new QueryBufferSizeInstr(dest, {0,1,2,3}, res_id)); + shader.emit_instruction(new QueryBufferSizeInstr(dest, {0, 1, 2, 3}, res_id)); return true; } else { if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_CUBE && - nir_intrinsic_image_array(intrin) && nir_dest_num_components(intrin->dest) > 2) { + nir_intrinsic_image_array(intrin) && + nir_dest_num_components(intrin->dest) > 2) { /* Need to load the layers from a const buffer */ auto dest = vf.dest_vec4(intrin->dest, pin_group); - shader.emit_instruction(new TexInstr(TexInstr::get_resinfo, dest, {0,1,7,3}, - src, 0/* ?? */, res_id, dyn_offset)); + shader.emit_instruction(new TexInstr(TexInstr::get_resinfo, + dest, + {0, 1, 7, 3}, + src, + 0 /* ?? */, + res_id, + dyn_offset)); shader.set_flag(Shader::sh_txs_cube_array_comp); if (const_offset) { unsigned lookup_resid = const_offset[0].u32; - shader.emit_instruction(new AluInstr(op1_mov, dest[2], - vf.uniform(lookup_resid/4 + R600_SHADER_BUFFER_INFO_SEL, lookup_resid % 4, - R600_BUFFER_INFO_CONST_BUFFER), - AluInstr::last_write)); + shader.emit_instruction( + new AluInstr(op1_mov, + dest[2], + vf.uniform(lookup_resid / 4 + R600_SHADER_BUFFER_INFO_SEL, + lookup_resid % 4, + R600_BUFFER_INFO_CONST_BUFFER), + AluInstr::last_write)); } else { - /* If the adressing is indirect we have to get the z-value by using a binary search */ + /* If the adressing is indirect we have to get the z-value by + * using a binary search */ auto addr = vf.temp_register(); auto comp1 = vf.temp_register(); auto comp2 = vf.temp_register(); @@ -825,31 +918,50 @@ bool RatInstr::emit_image_size(nir_intrinsic_instr *intrin, Shader& shader) auto trgt = vf.temp_vec4(pin_group); - shader.emit_instruction(new AluInstr(op2_lshr_int, addr, vf.src(intrin->src[0], 0), - vf.literal(2), AluInstr::write)); - shader.emit_instruction(new AluInstr(op2_and_int, low_bit, vf.src(intrin->src[0], 0), - vf.one_i(), AluInstr::write)); - shader.emit_instruction(new AluInstr(op2_and_int, high_bit, vf.src(intrin->src[0], 0), - vf.literal(2), AluInstr::last_write)); + shader.emit_instruction(new AluInstr(op2_lshr_int, + addr, + vf.src(intrin->src[0], 0), + vf.literal(2), + AluInstr::write)); + shader.emit_instruction(new AluInstr(op2_and_int, + low_bit, + vf.src(intrin->src[0], 0), + vf.one_i(), + AluInstr::write)); + shader.emit_instruction(new AluInstr(op2_and_int, + high_bit, + vf.src(intrin->src[0], 0), + vf.literal(2), + AluInstr::last_write)); - shader.emit_instruction(new LoadFromBuffer(trgt, {0, 1, 2, 3}, addr, R600_SHADER_BUFFER_INFO_SEL, - R600_BUFFER_INFO_CONST_BUFFER, nullptr, fmt_32_32_32_32_float)); + shader.emit_instruction(new LoadFromBuffer(trgt, + {0, 1, 2, 3}, + addr, + R600_SHADER_BUFFER_INFO_SEL, + R600_BUFFER_INFO_CONST_BUFFER, + nullptr, + fmt_32_32_32_32_float)); // this may be wrong - shader.emit_instruction(new AluInstr(op3_cnde_int, comp1, high_bit, trgt[0], trgt[2], - AluInstr::write)); - shader.emit_instruction(new AluInstr(op3_cnde_int, comp2, high_bit, trgt[1], trgt[3], - AluInstr::last_write)); - shader.emit_instruction(new AluInstr(op3_cnde_int, dest[2], low_bit, comp1, comp2, AluInstr::last_write)); + shader.emit_instruction(new AluInstr( + op3_cnde_int, comp1, high_bit, trgt[0], trgt[2], AluInstr::write)); + shader.emit_instruction(new AluInstr( + op3_cnde_int, comp2, high_bit, trgt[1], trgt[3], AluInstr::last_write)); + shader.emit_instruction(new AluInstr( + op3_cnde_int, dest[2], low_bit, comp1, comp2, AluInstr::last_write)); } } else { auto dest = vf.dest_vec4(intrin->dest, pin_group); - shader.emit_instruction(new TexInstr(TexInstr::get_resinfo, dest, {0,1,2,3}, - src, 0/* ?? */, res_id, dyn_offset)); - + shader.emit_instruction(new TexInstr(TexInstr::get_resinfo, + dest, + {0, 1, 2, 3}, + src, + 0 /* ?? */, + res_id, + dyn_offset)); } } return true; } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_mem.h b/src/gallium/drivers/r600/sfn/sfn_instr_mem.h index 72eb250d872..69d59cc508a 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_mem.h +++ b/src/gallium/drivers/r600/sfn/sfn_instr_mem.h @@ -36,10 +36,8 @@ class Shader; class GDSInstr : public InstrWithResource { public: - - GDSInstr(ESDOp op, Register *dest, - const RegisterVec4& src, int uav_base, - PRegister uav_id); + GDSInstr( + ESDOp op, Register *dest, const RegisterVec4& src, int uav_base, PRegister uav_id); bool is_equal_to(const GDSInstr& lhs) const; @@ -48,19 +46,18 @@ public: bool do_ready() const override; - auto opcode() const {return m_op;} - auto src() const { return m_src;} + auto opcode() const { return m_op; } + auto src() const { return m_src; } - const auto& dest() const { return m_dest;} - auto& dest() { return m_dest;} + const auto& dest() const { return m_dest; } + auto& dest() { return m_dest; } static auto from_string(std::istream& is, ValueFactory& value_factory) -> Pointer; static bool emit_atomic_counter(nir_intrinsic_instr *intr, Shader& shader); - uint32_t slots() const override {return 1;}; + uint32_t slots() const override { return 1; }; private: - static bool emit_atomic_read(nir_intrinsic_instr *intr, Shader& shader); static bool emit_atomic_op2(nir_intrinsic_instr *intr, Shader& shader); static bool emit_atomic_inc(nir_intrinsic_instr *intr, Shader& shader); @@ -76,7 +73,6 @@ private: std::bitset<8> m_tex_flags; }; - class RatInstr : public InstrWithResource { public: @@ -122,35 +118,44 @@ public: UNSUPPORTED }; - RatInstr(ECFOpCode cf_opcode, ERatOp rat_op, - const RegisterVec4& data, const RegisterVec4& index, - int rat_id, PRegister rat_id_offset, - int burst_count, int comp_mask, int element_size); + RatInstr(ECFOpCode cf_opcode, + ERatOp rat_op, + const RegisterVec4& data, + const RegisterVec4& index, + int rat_id, + PRegister rat_id_offset, + int burst_count, + int comp_mask, + int element_size); - ERatOp rat_op() const {return m_rat_op;} + ERatOp rat_op() const { return m_rat_op; } - const auto& value() const { return m_data;} - auto& value() { return m_data;} + const auto& value() const { return m_data; } + auto& value() { return m_data; } - const auto& addr() const { return m_index;} - auto& addr() { return m_index;} + const auto& addr() const { return m_index; } + auto& addr() { return m_index; } - int data_gpr() const {return m_data.sel();} - int index_gpr() const {return m_index.sel();} - int elm_size() const {return m_element_size;} + int data_gpr() const { return m_data.sel(); } + int index_gpr() const { return m_index.sel(); } + int elm_size() const { return m_element_size; } - int comp_mask() const {return m_comp_mask;} + int comp_mask() const { return m_comp_mask; } - bool need_ack() const {return m_need_ack;} - int burst_count() const {return m_burst_count;} + bool need_ack() const { return m_need_ack; } + int burst_count() const { return m_burst_count; } - int data_swz(int chan) const {return m_data[chan]->chan();} + int data_swz(int chan) const { return m_data[chan]->chan(); } - ECFOpCode cf_opcode() const { return m_cf_opcode;} + ECFOpCode cf_opcode() const { return m_cf_opcode; } - void set_ack() {m_need_ack = true; set_mark(); } - void set_mark() {m_need_mark = true; } - bool mark() {return m_need_mark;} + void set_ack() + { + m_need_ack = true; + set_mark(); + } + void set_mark() { m_need_mark = true; } + bool mark() { return m_need_mark; } void accept(ConstInstrVisitor& visitor) const override; void accept(InstrVisitor& visitor) override; @@ -160,7 +165,6 @@ public: static bool emit(nir_intrinsic_instr *intr, Shader& shader); private: - static bool emit_ssbo_load(nir_intrinsic_instr *intr, Shader& shader); static bool emit_ssbo_store(nir_intrinsic_instr *intr, Shader& shader); static bool emit_ssbo_atomic_op(nir_intrinsic_instr *intr, Shader& shader); @@ -186,7 +190,6 @@ private: bool m_need_mark{false}; }; - -} +} // namespace r600 #endif // GDSINSTR_H diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_tex.cpp b/src/gallium/drivers/r600/sfn/sfn_instr_tex.cpp index f687392c394..5ddd9539205 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_tex.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_instr_tex.cpp @@ -25,59 +25,68 @@ */ #include "sfn_instr_tex.h" + +#include "nir_builder.h" +#include "sfn_debug.h" #include "sfn_instr_alu.h" #include "sfn_instr_fetch.h" #include "sfn_nir.h" -#include "sfn_debug.h" - -#include "nir_builder.h" namespace r600 { using std::string; -TexInstr::TexInstr(Opcode op, const RegisterVec4& dest, +TexInstr::TexInstr(Opcode op, + const RegisterVec4& dest, const RegisterVec4::Swizzle& dest_swizzle, - const RegisterVec4& src, unsigned sid, unsigned rid, + const RegisterVec4& src, + unsigned sid, + unsigned rid, PRegister sampler_offs): - InstrWithVectorResult(dest, dest_swizzle, sid, sampler_offs), - m_opcode(op), - m_src(src), - m_inst_mode(0), - m_resource_id(rid) + InstrWithVectorResult(dest, dest_swizzle, sid, sampler_offs), + m_opcode(op), + m_src(src), + m_inst_mode(0), + m_resource_id(rid) { memset(m_offset, 0, sizeof(m_offset)); m_src.add_use(this); } -void TexInstr::accept(ConstInstrVisitor& visitor) const +void +TexInstr::accept(ConstInstrVisitor& visitor) const { visitor.visit(*this); } -void TexInstr::accept(InstrVisitor& visitor) +void +TexInstr::accept(InstrVisitor& visitor) { visitor.visit(this); } -void TexInstr::set_offset(unsigned index, int32_t val) +void +TexInstr::set_offset(unsigned index, int32_t val) { assert(index < 3); m_offset[index] = val; } -int TexInstr::get_offset(unsigned index) const +int +TexInstr::get_offset(unsigned index) const { assert(index < 3); return m_offset[index] << 1; } -void TexInstr::set_gather_comp(int cmp) +void +TexInstr::set_gather_comp(int cmp) { m_inst_mode = cmp; } -bool TexInstr::is_equal_to(const TexInstr& lhs) const +bool +TexInstr::is_equal_to(const TexInstr& lhs) const { if (m_opcode != lhs.m_opcode) return false; @@ -98,38 +107,40 @@ bool TexInstr::is_equal_to(const TexInstr& lhs) const if (m_tex_flags != lhs.m_tex_flags) return false; - for(int i = 0; i < 3; ++i) { + for (int i = 0; i < 3; ++i) { if (m_offset[i] != lhs.m_offset[i]) return false; } - return m_inst_mode == lhs.m_inst_mode && - resource_base() == lhs.resource_base() && - m_resource_id == lhs.m_resource_id; + return m_inst_mode == lhs.m_inst_mode && resource_base() == lhs.resource_base() && + m_resource_id == lhs.m_resource_id; } -bool TexInstr::propagate_death() +bool +TexInstr::propagate_death() { m_src.del_use(this); return true; } -bool TexInstr::do_ready() const +bool +TexInstr::do_ready() const { for (auto p : m_prepare_instr) if (!p->ready()) return false; - for (auto p :required_instr()) + for (auto p : required_instr()) if (!p->is_scheduled() && !p->is_dead()) { return false; } if (resource_offset() && !resource_offset()->ready(block_id(), index())) - return false; + return false; return m_src.ready(block_id(), index()); } -void TexInstr::do_print(std::ostream& os) const +void +TexInstr::do_print(std::ostream& os) const { for (auto& p : prepare_instr()) { @@ -142,8 +153,7 @@ void TexInstr::do_print(std::ostream& os) const os << " : "; m_src.print(os); - os << " RID:" << m_resource_id - << " SID:" << resource_base(); + os << " RID:" << m_resource_id << " SID:" << resource_base(); if (resource_offset()) os << " SO:" << *resource_offset(); @@ -165,77 +175,105 @@ void TexInstr::do_print(std::ostream& os) const os << (m_tex_flags.test(w_unnormalized) ? "U" : "N"); } -const char *TexInstr::opname(Opcode op) +const char * +TexInstr::opname(Opcode op) { switch (op) { - case ld: return "LD"; - case get_resinfo: return "GET_TEXTURE_RESINFO"; - case get_nsamples: return "GET_NUMBER_OF_SAMPLES"; - case get_tex_lod: return "GET_LOD"; - case get_gradient_h: return "GET_GRADIENTS_H"; - case get_gradient_v: return "GET_GRADIENTS_V"; - case set_offsets: return "SET_TEXTURE_OFFSETS"; - case keep_gradients: return "KEEP_GRADIENTS"; - case set_gradient_h: return "SET_GRADIENTS_H"; - case set_gradient_v: return "SET_GRADIENTS_V"; - case sample: return "SAMPLE"; - case sample_l: return "SAMPLE_L"; - case sample_lb: return "SAMPLE_LB"; - case sample_lz: return "SAMPLE_LZ"; - case sample_g: return "SAMPLE_G"; - case sample_g_lb: return "SAMPLE_G_L"; - case gather4: return "GATHER4"; - case gather4_o: return "GATHER4_O"; - case sample_c: return "SAMPLE_C"; - case sample_c_l: return "SAMPLE_C_L"; - case sample_c_lb: return "SAMPLE_C_LB"; - case sample_c_lz: return "SAMPLE_C_LZ"; - case sample_c_g: return "SAMPLE_C_G"; - case sample_c_g_lb: return "SAMPLE_C_G_L"; - case gather4_c: return "GATHER4_C"; - case gather4_c_o: return "OP_GATHER4_C_O"; - default: return "ERROR"; + case ld: + return "LD"; + case get_resinfo: + return "GET_TEXTURE_RESINFO"; + case get_nsamples: + return "GET_NUMBER_OF_SAMPLES"; + case get_tex_lod: + return "GET_LOD"; + case get_gradient_h: + return "GET_GRADIENTS_H"; + case get_gradient_v: + return "GET_GRADIENTS_V"; + case set_offsets: + return "SET_TEXTURE_OFFSETS"; + case keep_gradients: + return "KEEP_GRADIENTS"; + case set_gradient_h: + return "SET_GRADIENTS_H"; + case set_gradient_v: + return "SET_GRADIENTS_V"; + case sample: + return "SAMPLE"; + case sample_l: + return "SAMPLE_L"; + case sample_lb: + return "SAMPLE_LB"; + case sample_lz: + return "SAMPLE_LZ"; + case sample_g: + return "SAMPLE_G"; + case sample_g_lb: + return "SAMPLE_G_L"; + case gather4: + return "GATHER4"; + case gather4_o: + return "GATHER4_O"; + case sample_c: + return "SAMPLE_C"; + case sample_c_l: + return "SAMPLE_C_L"; + case sample_c_lb: + return "SAMPLE_C_LB"; + case sample_c_lz: + return "SAMPLE_C_LZ"; + case sample_c_g: + return "SAMPLE_C_G"; + case sample_c_g_lb: + return "SAMPLE_C_G_L"; + case gather4_c: + return "GATHER4_C"; + case gather4_c_o: + return "OP_GATHER4_C_O"; + default: + return "ERROR"; } - } const std::map TexInstr::s_opcode_map = { - {ld, "LD"}, - {get_resinfo,"GET_TEXTURE_RESINFO"}, - {get_nsamples,"GET_NUMBER_OF_SAMPLES"}, - {get_tex_lod,"GET_LOD"}, - {get_gradient_h,"GET_GRADIENTS_H"}, - {get_gradient_v,"GET_GRADIENTS_V"}, - {set_offsets,"SET_TEXTURE_OFFSETS"}, - {keep_gradients,"KEEP_GRADIENTS"}, - {set_gradient_h,"SET_GRADIENTS_H"}, - {set_gradient_v,"SET_GRADIENTS_V"}, - {sample,"SAMPLE"}, - {sample_l,"SAMPLE_L"}, - {sample_lb,"SAMPLE_LB"}, - {sample_lz,"SAMPLE_LZ"}, - {sample_g,"SAMPLE_G"}, - {sample_g_lb,"SAMPLE_G_L"}, - {gather4,"GATHER4"}, - {gather4_o,"GATHER4_O"}, - {sample_c,"SAMPLE_C"}, - {sample_c_l,"SAMPLE_C_L"}, - {sample_c_lb,"SAMPLE_C_LB"}, - {sample_c_lz,"SAMPLE_C_LZ"}, - {sample_c_g,"SAMPLE_C_G"}, - {sample_c_g_lb,"SAMPLE_C_G_L"}, - {gather4_c,"GATHER4_C"}, - {gather4_c_o,"OP_GATHER4_C_O"}, - {unknown, "ERROR"} + {ld, "LD" }, + {get_resinfo, "GET_TEXTURE_RESINFO" }, + {get_nsamples, "GET_NUMBER_OF_SAMPLES"}, + {get_tex_lod, "GET_LOD" }, + {get_gradient_h, "GET_GRADIENTS_H" }, + {get_gradient_v, "GET_GRADIENTS_V" }, + {set_offsets, "SET_TEXTURE_OFFSETS" }, + {keep_gradients, "KEEP_GRADIENTS" }, + {set_gradient_h, "SET_GRADIENTS_H" }, + {set_gradient_v, "SET_GRADIENTS_V" }, + {sample, "SAMPLE" }, + {sample_l, "SAMPLE_L" }, + {sample_lb, "SAMPLE_LB" }, + {sample_lz, "SAMPLE_LZ" }, + {sample_g, "SAMPLE_G" }, + {sample_g_lb, "SAMPLE_G_L" }, + {gather4, "GATHER4" }, + {gather4_o, "GATHER4_O" }, + {sample_c, "SAMPLE_C" }, + {sample_c_l, "SAMPLE_C_L" }, + {sample_c_lb, "SAMPLE_C_LB" }, + {sample_c_lz, "SAMPLE_C_LZ" }, + {sample_c_g, "SAMPLE_C_G" }, + {sample_c_g_lb, "SAMPLE_C_G_L" }, + {gather4_c, "GATHER4_C" }, + {gather4_c_o, "OP_GATHER4_C_O" }, + {unknown, "ERROR" } }; -bool TexInstr::is_gather(Opcode op) +bool +TexInstr::is_gather(Opcode op) { - return op == gather4 || op == gather4_c || - op == gather4_o || op == gather4_c_o; + return op == gather4 || op == gather4_c || op == gather4_o || op == gather4_c_o; } -TexInstr::Opcode TexInstr::op_from_string(const std::string& s) +TexInstr::Opcode +TexInstr::op_from_string(const std::string& s) { for (auto& [op, str] : s_opcode_map) { if (s == str) @@ -244,7 +282,8 @@ TexInstr::Opcode TexInstr::op_from_string(const std::string& s) return unknown; } -Instr::Pointer TexInstr::from_string(std::istream& is, ValueFactory& value_fctory) +Instr::Pointer +TexInstr::from_string(std::istream& is, ValueFactory& value_fctory) { string opstr; string deststr; @@ -273,7 +312,7 @@ Instr::Pointer TexInstr::from_string(std::istream& is, ValueFactory& value_fctor int res_id = int_from_string_with_prefix(res_id_str, "RID:"); int sampler_id = int_from_string_with_prefix(sampler_id_str, "SID:"); - auto tex = new TexInstr( opcode, dest, dest_swz, src, sampler_id, res_id, nullptr); + auto tex = new TexInstr(opcode, dest, dest_swz, src, sampler_id, res_id, nullptr); while (!is.eof() && is.good()) { std::string next_token; @@ -292,26 +331,32 @@ Instr::Pointer TexInstr::from_string(std::istream& is, ValueFactory& value_fctor return tex; } -void TexInstr::read_tex_coord_normalitazion(const std::string& flags) +void +TexInstr::read_tex_coord_normalitazion(const std::string& flags) { assert(flags.length() == 4); - if (flags[0] == 'U') set_tex_flag(x_unnormalized); - if (flags[1] == 'U') set_tex_flag(y_unnormalized); - if (flags[2] == 'U') set_tex_flag(z_unnormalized); - if (flags[3] == 'U') set_tex_flag(w_unnormalized); + if (flags[0] == 'U') + set_tex_flag(x_unnormalized); + if (flags[1] == 'U') + set_tex_flag(y_unnormalized); + if (flags[2] == 'U') + set_tex_flag(z_unnormalized); + if (flags[3] == 'U') + set_tex_flag(w_unnormalized); } -void TexInstr::set_tex_param(const std::string& token) +void +TexInstr::set_tex_param(const std::string& token) { - if (token.substr(0,3) == "OX:") + if (token.substr(0, 3) == "OX:") set_offset(0, int_from_string_with_prefix(token, "OX:")); - else if (token.substr(0,3) == "OY:") + else if (token.substr(0, 3) == "OY:") set_offset(1, int_from_string_with_prefix(token, "OY:")); - else if (token.substr(0,3) == "OZ:") + else if (token.substr(0, 3) == "OZ:") set_offset(2, int_from_string_with_prefix(token, "OZ:")); - else if (token.substr(0,5) == "MODE:") + else if (token.substr(0, 5) == "MODE:") set_inst_mode(int_from_string_with_prefix(token, "MODE:")); - else if (token.substr(0,3) == "SO:") + else if (token.substr(0, 3) == "SO:") set_resource_offset(VirtualValue::from_string(token.substr(3))->as_register()); else { std::cerr << "Token '" << token << "': "; @@ -319,18 +364,18 @@ void TexInstr::set_tex_param(const std::string& token) } } -bool TexInstr::from_nir(nir_tex_instr *tex, Shader& shader) +bool +TexInstr::from_nir(nir_tex_instr *tex, Shader& shader) { Inputs src(*tex, shader.value_factory()); if (nir_tex_instr_src_index(tex, nir_tex_src_backend1) != -1) return emit_lowered_tex(tex, src, shader); - if (tex->sampler_dim == GLSL_SAMPLER_DIM_BUF) { switch (tex->op) { case nir_texop_txs: - return emit_tex_txs(tex, src, {0,1,2,3}, shader); + return emit_tex_txs(tex, src, {0, 1, 2, 3}, shader); case nir_texop_txf: return emit_buf_txf(tex, src, shader); default: @@ -343,17 +388,18 @@ bool TexInstr::from_nir(nir_tex_instr *tex, Shader& shader) case nir_texop_lod: return emit_tex_lod(tex, src, shader); case nir_texop_query_levels: - return emit_tex_txs(tex, src, {3,7,7,7}, shader); + return emit_tex_txs(tex, src, {3, 7, 7, 7}, shader); case nir_texop_texture_samples: return emit_tex_texture_samples(tex, src, shader); default: - return false; + return false; } } return true; } -bool TexInstr::replace_source(PRegister old_src, PVirtualValue new_src) +bool +TexInstr::replace_source(PRegister old_src, PVirtualValue new_src) { if (old_src->pin() != pin_free) return false; @@ -376,7 +422,8 @@ bool TexInstr::replace_source(PRegister old_src, PVirtualValue new_src) return success; } -uint8_t TexInstr::allowed_dest_chan_mask() const +uint8_t +TexInstr::allowed_dest_chan_mask() const { return m_src.free_chan_mask(); } @@ -398,20 +445,28 @@ get_sampler_id(int sampler_id, const nir_variable *deref) return result; } -void TexInstr::emit_set_gradients(nir_tex_instr* tex, int sampler_id, - Inputs& src, TexInstr *irt, Shader& shader) +void +TexInstr::emit_set_gradients( + nir_tex_instr *tex, int sampler_id, Inputs& src, TexInstr *irt, Shader& shader) { TexInstr *grad[2] = {nullptr, nullptr}; - RegisterVec4 empty_dst(0, false, {0,0,0,0}, pin_group); - grad[0] = new TexInstr(set_gradient_h, empty_dst, {7,7,7,7}, src.ddx, + RegisterVec4 empty_dst(0, false, {0, 0, 0, 0}, pin_group); + grad[0] = new TexInstr(set_gradient_h, + empty_dst, + {7, 7, 7, 7}, + src.ddx, sampler_id, sampler_id + R600_MAX_CONST_BUFFERS, src.resource_offset); grad[0]->set_rect_coordinate_flags(tex); grad[0]->set_always_keep(); - grad[1] = new TexInstr(set_gradient_v, empty_dst, {7,7,7,7}, src.ddy, - sampler_id, sampler_id + R600_MAX_CONST_BUFFERS, + grad[1] = new TexInstr(set_gradient_v, + empty_dst, + {7, 7, 7, 7}, + src.ddy, + sampler_id, + sampler_id + R600_MAX_CONST_BUFFERS, src.resource_offset); grad[1]->set_rect_coordinate_flags(tex); grad[1]->set_always_keep(); @@ -422,10 +477,11 @@ void TexInstr::emit_set_gradients(nir_tex_instr* tex, int sampler_id, shader.set_last_txd(irt); } -void TexInstr::emit_set_offsets(nir_tex_instr* tex, int sampler_id, - Inputs& src, TexInstr *irt, Shader& shader) +void +TexInstr::emit_set_offsets( + nir_tex_instr *tex, int sampler_id, Inputs& src, TexInstr *irt, Shader& shader) { - RegisterVec4::Swizzle swizzle = {4,4,4,4}; + RegisterVec4::Swizzle swizzle = {4, 4, 4, 4}; int src_components = tex->coord_components; if (tex->is_array) --src_components; @@ -438,26 +494,28 @@ void TexInstr::emit_set_offsets(nir_tex_instr* tex, int sampler_id, --noffsets; auto ofs = shader.value_factory().src_vec4(*src.offset, pin_group, swizzle); - RegisterVec4 empty_dst(0, false, {0,0,0,0}, pin_group); + RegisterVec4 empty_dst(0, false, {0, 0, 0, 0}, pin_group); - auto set_ofs = new TexInstr(TexInstr::set_offsets, empty_dst, {7,7,7,7}, - ofs, sampler_id, + auto set_ofs = new TexInstr(TexInstr::set_offsets, + empty_dst, + {7, 7, 7, 7}, + ofs, + sampler_id, sampler_id + R600_MAX_CONST_BUFFERS, src.resource_offset); set_ofs->set_always_keep(); irt->add_prepare_instr(set_ofs); } - -bool TexInstr::emit_lowered_tex(nir_tex_instr* tex, Inputs& src, Shader& shader) +bool +TexInstr::emit_lowered_tex(nir_tex_instr *tex, Inputs& src, Shader& shader) { assert(src.backend1); assert(src.backend2); auto& vf = shader.value_factory(); - sfn_log << SfnLog::instr << "emit '" - << *reinterpret_cast(tex) - << "' (" << __func__ << ")\n"; + sfn_log << SfnLog::instr << "emit '" << *reinterpret_cast(tex) << "' (" + << __func__ << ")\n"; auto sampler = get_sampler_id(tex->sampler_index, src.sampler_deref); assert(!sampler.indirect); @@ -476,20 +534,23 @@ bool TexInstr::emit_lowered_tex(nir_tex_instr* tex, Inputs& src, Shader& shader) auto src_coord = vf.src_vec4(*src.backend1, pin_group, src_swizzle); - RegisterVec4::Swizzle dst_swz = {0,1,2,3}; + RegisterVec4::Swizzle dst_swz = {0, 1, 2, 3}; if (dst_swz_packed) { for (int i = 0; i < 4; ++i) { dst_swz[i] = (dst_swz_packed >> (8 * i)) & 0xff; } } - auto irt = new TexInstr(src.opcode, dst, dst_swz, src_coord, sampler.id, + auto irt = new TexInstr(src.opcode, + dst, + dst_swz, + src_coord, + sampler.id, sampler.id + R600_MAX_CONST_BUFFERS, src.resource_offset); if (tex->op == nir_texop_txd) emit_set_gradients(tex, sampler.id, src, irt, shader); - if (!irt->set_coord_offsets(src.offset)) { assert(tex->op == nir_texop_tg4); emit_set_offsets(tex, sampler.id, src, irt, shader); @@ -500,14 +561,14 @@ bool TexInstr::emit_lowered_tex(nir_tex_instr* tex, Inputs& src, Shader& shader) irt->set_tex_flag(f); } - irt->set_inst_mode(inst_mode); shader.emit_instruction(irt); return true; } -bool TexInstr::emit_buf_txf(nir_tex_instr *tex, Inputs& src, Shader& shader) +bool +TexInstr::emit_buf_txf(nir_tex_instr *tex, Inputs& src, Shader& shader) { auto& vf = shader.value_factory(); auto dst = vf.dest_vec4(tex->dest, pin_group); @@ -523,9 +584,13 @@ bool TexInstr::emit_buf_txf(nir_tex_instr *tex, Inputs& src, Shader& shader) real_dst = &tmp; } - auto ir = new LoadFromBuffer(*real_dst, {0,1,2,3}, src.coord[0], 0, - tex->texture_index + R600_MAX_CONST_BUFFERS, - tex_offset, fmt_invalid); + auto ir = new LoadFromBuffer(*real_dst, + {0, 1, 2, 3}, + src.coord[0], + 0, + tex->texture_index + R600_MAX_CONST_BUFFERS, + tex_offset, + fmt_invalid); ir->set_fetch_flag(FetchInstr::use_const_field); shader.emit_instruction(ir); shader.set_flag(Shader::sh_uses_tex_buffer); @@ -536,37 +601,48 @@ bool TexInstr::emit_buf_txf(nir_tex_instr *tex, Inputs& src, Shader& shader) AluInstr *ir = nullptr; for (int i = 0; i < 4; ++i) { auto d = i < 3 ? dst[i] : tmp_w; - ir = new AluInstr(op2_and_int, d, tmp[i], + ir = new AluInstr(op2_and_int, + d, + tmp[i], vf.uniform(buf_sel, i, R600_BUFFER_INFO_CONST_BUFFER), AluInstr::write); shader.emit_instruction(ir); } ir->set_alu_flag(alu_last_instr); - shader.emit_instruction(new AluInstr(op2_or_int, dst[3], tmp_w, - vf.uniform(buf_sel + 1, 0, R600_BUFFER_INFO_CONST_BUFFER), - AluInstr::last_write)); + shader.emit_instruction( + new AluInstr(op2_or_int, + dst[3], + tmp_w, + vf.uniform(buf_sel + 1, 0, R600_BUFFER_INFO_CONST_BUFFER), + AluInstr::last_write)); } return true; } -bool TexInstr::emit_tex_texture_samples(nir_tex_instr* instr, Inputs& src, Shader& shader) +bool +TexInstr::emit_tex_texture_samples(nir_tex_instr *instr, Inputs& src, Shader& shader) { RegisterVec4 dest = shader.value_factory().dest_vec4(instr->dest, pin_chan); - RegisterVec4 help{0, true, {4,4,4,4}}; + RegisterVec4 help{ + 0, true, {4, 4, 4, 4} + }; int res_id = R600_MAX_CONST_BUFFERS + instr->sampler_index; // Fishy: should the zero be instr->sampler_index? - auto ir = new TexInstr(src.opcode, dest, {3, 7, 7, 7}, help, - 0, res_id, src.resource_offset); + auto ir = + new TexInstr(src.opcode, dest, {3, 7, 7, 7}, help, 0, res_id, src.resource_offset); shader.emit_instruction(ir); return true; } -bool TexInstr::emit_tex_txs(nir_tex_instr *tex, Inputs& src, - RegisterVec4::Swizzle dest_swz, Shader& shader) +bool +TexInstr::emit_tex_txs(nir_tex_instr *tex, + Inputs& src, + RegisterVec4::Swizzle dest_swz, + Shader& shader) { auto& vf = shader.value_factory(); @@ -574,18 +650,20 @@ bool TexInstr::emit_tex_txs(nir_tex_instr *tex, Inputs& src, if (tex->sampler_dim == GLSL_SAMPLER_DIM_BUF) { if (shader.chip_class() >= ISA_CC_EVERGREEN) { - shader.emit_instruction(new QueryBufferSizeInstr(dest, {0,7,7,7}, - tex->sampler_index + R600_MAX_CONST_BUFFERS)); + shader.emit_instruction(new QueryBufferSizeInstr( + dest, {0, 7, 7, 7}, tex->sampler_index + R600_MAX_CONST_BUFFERS)); } else { int id = 2 * tex->sampler_index + (512 + R600_BUFFER_INFO_OFFSET / 16) + 1; auto src = vf.uniform(id, 1, R600_BUFFER_INFO_CONST_BUFFER); - shader.emit_instruction(new AluInstr(op1_mov, dest[0], src, AluInstr::last_write)); + shader.emit_instruction( + new AluInstr(op1_mov, dest[0], src, AluInstr::last_write)); shader.set_flag(Shader::sh_uses_tex_buffer); } } else { auto src_lod = vf.temp_register(); - shader.emit_instruction(new AluInstr(op1_mov, src_lod, src.lod, AluInstr::last_write)); + shader.emit_instruction( + new AluInstr(op1_mov, src_lod, src.lod, AluInstr::last_write)); RegisterVec4 src_coord(src_lod, src_lod, src_lod, src_lod, pin_free); @@ -595,7 +673,10 @@ bool TexInstr::emit_tex_txs(nir_tex_instr *tex, Inputs& src, if (tex->is_array && tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE) dest_swz[2] = 7; - auto ir = new TexInstr(get_resinfo, dest, dest_swz, src_coord, + auto ir = new TexInstr(get_resinfo, + dest, + dest_swz, + src_coord, sampler.id, sampler.id + R600_MAX_CONST_BUFFERS, src.resource_offset); @@ -605,8 +686,8 @@ bool TexInstr::emit_tex_txs(nir_tex_instr *tex, Inputs& src, if (tex->is_array && tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE) { auto src_loc = vf.uniform(512 + R600_BUFFER_INFO_OFFSET / 16 + (sampler.id >> 2), - sampler.id & 3, R600_BUFFER_INFO_CONST_BUFFER); - + sampler.id & 3, + R600_BUFFER_INFO_CONST_BUFFER); auto alu = new AluInstr(op1_mov, dest[2], src_loc, AluInstr::last_write); shader.emit_instruction(alu); @@ -617,11 +698,12 @@ bool TexInstr::emit_tex_txs(nir_tex_instr *tex, Inputs& src, return true; } -auto TexInstr::prepare_source(nir_tex_instr *tex, const Inputs& inputs, Shader& shader) -> RegisterVec4 +auto +TexInstr::prepare_source(nir_tex_instr *tex, const Inputs& inputs, Shader& shader) + -> RegisterVec4 { - RegisterVec4::Swizzle target{7,7,7,7}; - PVirtualValue src[4]{nullptr,nullptr,nullptr,nullptr}; - + RegisterVec4::Swizzle target{7, 7, 7, 7}; + PVirtualValue src[4]{nullptr, nullptr, nullptr, nullptr}; for (unsigned i = 0; i < tex->coord_components; ++i) { target[i] = i; @@ -630,8 +712,8 @@ auto TexInstr::prepare_source(nir_tex_instr *tex, const Inputs& inputs, Shader& // array index always goes into z if (tex->is_array && tex->sampler_dim == GLSL_SAMPLER_DIM_1D) { - target[2] = 1; - target[1] = 7; + target[2] = 1; + target[1] = 7; src[2] = inputs.coord[1]; } @@ -639,7 +721,7 @@ auto TexInstr::prepare_source(nir_tex_instr *tex, const Inputs& inputs, Shader& if (tex->op == nir_texop_txl || tex->op == nir_texop_txb) { target[3] = 3; src[3] = tex->op == nir_texop_txl ? inputs.lod : inputs.bias; - if (tex->is_shadow){ + if (tex->is_shadow) { target[2] = 2; src[2] = inputs.comperator; } @@ -654,11 +736,11 @@ auto TexInstr::prepare_source(nir_tex_instr *tex, const Inputs& inputs, Shader& AluInstr *ir = nullptr; for (int i = 0; i < 4; ++i) { if (target[i] > 3) - continue; + continue; auto op = tex->is_array && i == 2 ? op1_rndne : op1_mov; - ir = new AluInstr(op, src_coord[i], src[i], AluInstr::write); + ir = new AluInstr(op, src_coord[i], src[i], AluInstr::write); shader.emit_instruction(ir); } @@ -669,21 +751,22 @@ auto TexInstr::prepare_source(nir_tex_instr *tex, const Inputs& inputs, Shader& } TexInstr::Inputs::Inputs(const nir_tex_instr& instr, ValueFactory& vf): - sampler_deref(nullptr), - texture_deref(nullptr), - bias(nullptr), - comperator(nullptr), - lod(nullptr), - offset(nullptr), - gather_comp(nullptr), - ms_index(nullptr), - texture_offset(nullptr), - resource_offset(nullptr), - backend1(nullptr), - backend2(nullptr), - opcode(ld) + sampler_deref(nullptr), + texture_deref(nullptr), + bias(nullptr), + comperator(nullptr), + lod(nullptr), + offset(nullptr), + gather_comp(nullptr), + ms_index(nullptr), + texture_offset(nullptr), + resource_offset(nullptr), + backend1(nullptr), + backend2(nullptr), + opcode(ld) { - //sfn_log << SfnLog::tex << "Get Inputs with " << instr.coord_components << " components\n"; + // sfn_log << SfnLog::tex << "Get Inputs with " << instr.coord_components + // << " components\n"; unsigned grad_components = instr.coord_components; if (instr.is_array && !instr.array_is_lowered_cube) @@ -693,26 +776,32 @@ TexInstr::Inputs::Inputs(const nir_tex_instr& instr, ValueFactory& vf): switch (instr.src[i].src_type) { case nir_tex_src_bias: bias = vf.src(instr.src[i], 0); - break; + break; case nir_tex_src_coord: { - coord = vf.src_vec4(instr.src[i].src, pin_none, swizzle_from_ncomps(instr.coord_components)); + coord = vf.src_vec4(instr.src[i].src, + pin_none, + swizzle_from_ncomps(instr.coord_components)); } break; case nir_tex_src_comparator: comperator = vf.src(instr.src[i], 0); - break; + break; case nir_tex_src_ddx: - ddx = vf.src_vec4(instr.src[i].src, pin_group, swizzle_from_ncomps(grad_components)); - break; + ddx = vf.src_vec4(instr.src[i].src, + pin_group, + swizzle_from_ncomps(grad_components)); + break; case nir_tex_src_ddy: - ddy = vf.src_vec4(instr.src[i].src, pin_group, swizzle_from_ncomps(grad_components)); - break; + ddy = vf.src_vec4(instr.src[i].src, + pin_group, + swizzle_from_ncomps(grad_components)); + break; case nir_tex_src_lod: lod = vf.src(instr.src[i].src, 0); - break; + break; case nir_tex_src_offset: offset = &instr.src[i].src; - break; + break; /* case nir_tex_src_sampler_deref: sampler_deref = get_deref_location(instr.src[i].src); break; @@ -722,13 +811,13 @@ TexInstr::Inputs::Inputs(const nir_tex_instr& instr, ValueFactory& vf): */ case nir_tex_src_ms_index: ms_index = vf.src(instr.src[i], 0); - break; + break; case nir_tex_src_texture_offset: texture_offset = vf.src(instr.src[i], 0); - break; + break; case nir_tex_src_sampler_offset: resource_offset = vf.src(instr.src[i], 0)->as_register(); - break; + break; case nir_tex_src_backend1: backend1 = &instr.src[i].src; break; @@ -744,11 +833,10 @@ TexInstr::Inputs::Inputs(const nir_tex_instr& instr, ValueFactory& vf): } opcode = get_opcode(instr); - - } -auto TexInstr::Inputs::get_opcode(const nir_tex_instr& instr) -> Opcode +auto +TexInstr::Inputs::get_opcode(const nir_tex_instr& instr) -> Opcode { switch (instr.op) { case nir_texop_tex: @@ -767,9 +855,8 @@ auto TexInstr::Inputs::get_opcode(const nir_tex_instr& instr) -> Opcode return instr.is_shadow ? sample_c_g : sample_g; case nir_texop_tg4: { auto var_offset = offset && nir_src_as_const_value(*offset) == nullptr; - return instr.is_shadow ? - (var_offset ? gather4_c_o : gather4_c) : - (var_offset ? gather4_o : gather4); + return instr.is_shadow ? (var_offset ? gather4_c_o : gather4_c) + : (var_offset ? gather4_o : gather4); } case nir_texop_txf_ms: return ld; @@ -782,7 +869,8 @@ auto TexInstr::Inputs::get_opcode(const nir_tex_instr& instr) -> Opcode } } -bool TexInstr::emit_tex_lod(nir_tex_instr* tex, Inputs& src, Shader& shader) +bool +TexInstr::emit_tex_lod(nir_tex_instr *tex, Inputs& src, Shader& shader) { auto& vf = shader.value_factory(); auto sampler = get_sampler_id(tex->sampler_index, src.sampler_deref); @@ -796,23 +884,25 @@ bool TexInstr::emit_tex_lod(nir_tex_instr* tex, Inputs& src, Shader& shader) AluInstr *ir = nullptr; for (unsigned i = 0; i < tex->coord_components; ++i) { - ir = new AluInstr(op1_mov, - src_coord[i], src.coord[i], - AluInstr::write); + ir = new AluInstr(op1_mov, src_coord[i], src.coord[i], AluInstr::write); shader.emit_instruction(ir); } if (ir) ir->set_alu_flag(alu_last_instr); - auto irt = new TexInstr(TexInstr::get_tex_lod, dst, {1,0,7,7}, src_coord, - sampler.id, sampler.id + R600_MAX_CONST_BUFFERS); + auto irt = new TexInstr(TexInstr::get_tex_lod, + dst, + {1, 0, 7, 7}, + src_coord, + sampler.id, + sampler.id + R600_MAX_CONST_BUFFERS); shader.emit_instruction(irt); return true; } - -RegisterVec4::Swizzle TexInstr::Inputs::swizzle_from_ncomps(int comps) const +RegisterVec4::Swizzle +TexInstr::Inputs::swizzle_from_ncomps(int comps) const { RegisterVec4::Swizzle swz; for (int i = 0; i < 4; ++i) @@ -820,7 +910,8 @@ RegisterVec4::Swizzle TexInstr::Inputs::swizzle_from_ncomps(int comps) const return swz; } -bool TexInstr::set_coord_offsets(nir_src *offset) +bool +TexInstr::set_coord_offsets(nir_src *offset) { if (!offset) return true; @@ -837,7 +928,8 @@ bool TexInstr::set_coord_offsets(nir_src *offset) return true; } -void TexInstr::set_rect_coordinate_flags(nir_tex_instr* instr) +void +TexInstr::set_rect_coordinate_flags(nir_tex_instr *instr) { if (instr->sampler_dim == GLSL_SAMPLER_DIM_RECT) { set_tex_flag(x_unnormalized); @@ -848,9 +940,10 @@ void TexInstr::set_rect_coordinate_flags(nir_tex_instr* instr) class LowerTexToBackend : public NirLowerInstruction { public: LowerTexToBackend(amd_gfx_level chip_class); + private: bool filter(const nir_instr *instr) const override; - nir_ssa_def *lower(nir_instr *instr) override; + nir_ssa_def *lower(nir_instr *instr) override; nir_ssa_def *lower_tex(nir_tex_instr *tex); nir_ssa_def *lower_txf(nir_tex_instr *tex); @@ -858,31 +951,31 @@ private: nir_ssa_def *lower_txf_ms(nir_tex_instr *tex); nir_ssa_def *lower_txf_ms_direct(nir_tex_instr *tex); - nir_ssa_def *prepare_coord(nir_tex_instr *tex, - int &unnormalized_mask, - int &used_coord_mask); - int get_src_coords(nir_tex_instr *tex, std::array &coord, + nir_ssa_def * + prepare_coord(nir_tex_instr *tex, int& unnormalized_mask, int& used_coord_mask); + int get_src_coords(nir_tex_instr *tex, + std::array& coord, bool round_array_index); - nir_ssa_def *prep_src(std::array &coord, - int &used_coord_mask); - nir_ssa_def *finalize(nir_tex_instr *tex, - nir_ssa_def *backend1, nir_ssa_def *backend2); + nir_ssa_def *prep_src(std::array& coord, int& used_coord_mask); + nir_ssa_def * + finalize(nir_tex_instr *tex, nir_ssa_def *backend1, nir_ssa_def *backend2); amd_gfx_level m_chip_class; }; -bool r600_nir_lower_tex_to_backend(nir_shader *shader, amd_gfx_level chip_class) +bool +r600_nir_lower_tex_to_backend(nir_shader *shader, amd_gfx_level chip_class) { return LowerTexToBackend(chip_class).run(shader); } LowerTexToBackend::LowerTexToBackend(amd_gfx_level chip_class): - m_chip_class(chip_class) + m_chip_class(chip_class) { - } -bool LowerTexToBackend::filter(const nir_instr *instr) const +bool +LowerTexToBackend::filter(const nir_instr *instr) const { if (instr->type != nir_instr_type_tex) return false; @@ -906,7 +999,8 @@ bool LowerTexToBackend::filter(const nir_instr *instr) const return nir_tex_instr_src_index(tex, nir_tex_src_backend1) == -1; } -nir_ssa_def *LowerTexToBackend::lower(nir_instr *instr) +nir_ssa_def * +LowerTexToBackend::lower(nir_instr *instr) { b->cursor = nir_before_instr(instr); @@ -931,27 +1025,23 @@ nir_ssa_def *LowerTexToBackend::lower(nir_instr *instr) } } -nir_ssa_def *LowerTexToBackend::lower_tex(nir_tex_instr *tex) +nir_ssa_def * +LowerTexToBackend::lower_tex(nir_tex_instr *tex) { int unnormalized_mask = 0; int used_coord_mask = 0; nir_ssa_def *backend1 = prepare_coord(tex, unnormalized_mask, used_coord_mask); - nir_ssa_def *backend2 = - nir_imm_ivec4(b, used_coord_mask, unnormalized_mask, 0, 0); + nir_ssa_def *backend2 = nir_imm_ivec4(b, used_coord_mask, unnormalized_mask, 0, 0); return finalize(tex, backend1, backend2); } -nir_ssa_def *LowerTexToBackend::lower_txf(nir_tex_instr *tex) +nir_ssa_def * +LowerTexToBackend::lower_txf(nir_tex_instr *tex) { - std::array new_coord = { - nullptr, - nullptr, - nullptr, - nullptr - }; + std::array new_coord = {nullptr, nullptr, nullptr, nullptr}; get_src_coords(tex, new_coord, false); @@ -960,44 +1050,34 @@ nir_ssa_def *LowerTexToBackend::lower_txf(nir_tex_instr *tex) int used_coord_mask = 0; nir_ssa_def *backend1 = prep_src(new_coord, used_coord_mask); - nir_ssa_def *backend2 = nir_imm_ivec4(b, used_coord_mask, - tex->is_array ? 0x4 : 0, 0, 0); + nir_ssa_def *backend2 = + nir_imm_ivec4(b, used_coord_mask, tex->is_array ? 0x4 : 0, 0, 0); return finalize(tex, backend1, backend2); } -nir_ssa_def *LowerTexToBackend::lower_tg4(nir_tex_instr *tex) +nir_ssa_def * +LowerTexToBackend::lower_tg4(nir_tex_instr *tex) { - std::array new_coord = { - nullptr, - nullptr, - nullptr, - nullptr - }; + std::array new_coord = {nullptr, nullptr, nullptr, nullptr}; get_src_coords(tex, new_coord, false); - uint32_t dest_swizzle = m_chip_class <= EVERGREEN ? - 1 | (2 << 8) | (0 << 16) | (3 << 24) : 0; + uint32_t dest_swizzle = + m_chip_class <= EVERGREEN ? 1 | (2 << 8) | (0 << 16) | (3 << 24) : 0; int used_coord_mask = 0; int unnormalized_mask = 0; nir_ssa_def *backend1 = prepare_coord(tex, unnormalized_mask, used_coord_mask); - nir_ssa_def *backend2 = nir_imm_ivec4(b, used_coord_mask, - unnormalized_mask, - tex->component, - dest_swizzle); + nir_ssa_def *backend2 = + nir_imm_ivec4(b, used_coord_mask, unnormalized_mask, tex->component, dest_swizzle); return finalize(tex, backend1, backend2); } -nir_ssa_def *LowerTexToBackend::lower_txf_ms(nir_tex_instr *tex) +nir_ssa_def * +LowerTexToBackend::lower_txf_ms(nir_tex_instr *tex) { - std::array new_coord = { - nullptr, - nullptr, - nullptr, - nullptr - }; + std::array new_coord = {nullptr, nullptr, nullptr, nullptr}; get_src_coords(tex, new_coord, false); @@ -1008,8 +1088,7 @@ nir_ssa_def *LowerTexToBackend::lower_txf_ms(nir_tex_instr *tex) if (offset_index >= 0) { auto offset = tex->src[offset_index].src.ssa; for (int i = 0; i < offset->num_components; ++i) { - new_coord[i] = nir_iadd(b, new_coord[i], - nir_channel(b, offset, i)); + new_coord[i] = nir_iadd(b, new_coord[i], nir_channel(b, offset, i)); } } @@ -1018,7 +1097,7 @@ nir_ssa_def *LowerTexToBackend::lower_txf_ms(nir_tex_instr *tex) int used_coord_mask = 0; nir_ssa_def *backend1 = prep_src(new_coord, used_coord_mask); - nir_ssa_def *backend2 = nir_imm_ivec4(b, used_coord_mask, 0xf, 1, 0); + nir_ssa_def *backend2 = nir_imm_ivec4(b, used_coord_mask, 0xf, 1, 0); nir_builder_instr_insert(b, &fetch_sample->instr); finalize(fetch_sample, backend1, backend2); @@ -1026,22 +1105,18 @@ nir_ssa_def *LowerTexToBackend::lower_txf_ms(nir_tex_instr *tex) new_coord[3] = nir_iand_imm(b, nir_ushr(b, nir_channel(b, &fetch_sample->dest.ssa, 0), - nir_ishl_imm(b, new_coord[3], 2) - ), 15); + nir_ishl_imm(b, new_coord[3], 2)), + 15); nir_ssa_def *backend1b = prep_src(new_coord, used_coord_mask); - nir_ssa_def *backend2b = nir_imm_ivec4(b, used_coord_mask, 0, 0, 0); + nir_ssa_def *backend2b = nir_imm_ivec4(b, used_coord_mask, 0, 0, 0); return finalize(tex, backend1b, backend2b); } -nir_ssa_def *LowerTexToBackend::lower_txf_ms_direct(nir_tex_instr *tex) +nir_ssa_def * +LowerTexToBackend::lower_txf_ms_direct(nir_tex_instr *tex) { - std::array new_coord = { - nullptr, - nullptr, - nullptr, - nullptr - }; + std::array new_coord = {nullptr, nullptr, nullptr, nullptr}; get_src_coords(tex, new_coord, false); @@ -1053,24 +1128,22 @@ nir_ssa_def *LowerTexToBackend::lower_txf_ms_direct(nir_tex_instr *tex) nir_ssa_def *backend2 = nir_imm_ivec4(b, used_coord_mask, 0, 0, 0); return finalize(tex, backend1, backend2); - } - -nir_ssa_def *LowerTexToBackend::finalize(nir_tex_instr *tex, nir_ssa_def *backend1, - nir_ssa_def *backend2) +nir_ssa_def * +LowerTexToBackend::finalize(nir_tex_instr *tex, + nir_ssa_def *backend1, + nir_ssa_def *backend2) { nir_tex_instr_add_src(tex, nir_tex_src_backend1, nir_src_for_ssa(backend1)); nir_tex_instr_add_src(tex, nir_tex_src_backend2, nir_src_for_ssa(backend2)); nir_tex_instr_remove_src(tex, nir_tex_src_coord); - static const nir_tex_src_type cleanup[] = { - nir_tex_src_coord, - nir_tex_src_lod, - nir_tex_src_bias, - nir_tex_src_comparator, - nir_tex_src_ms_index - }; + static const nir_tex_src_type cleanup[] = {nir_tex_src_coord, + nir_tex_src_lod, + nir_tex_src_bias, + nir_tex_src_comparator, + nir_tex_src_ms_index}; for (const auto type : cleanup) { int pos = nir_tex_instr_src_index(tex, type); @@ -1080,8 +1153,8 @@ nir_ssa_def *LowerTexToBackend::finalize(nir_tex_instr *tex, nir_ssa_def *backen return NIR_LOWER_INSTR_PROGRESS; } -nir_ssa_def *LowerTexToBackend::prep_src(std::array &coord, - int &used_coord_mask) +nir_ssa_def * +LowerTexToBackend::prep_src(std::array& coord, int& used_coord_mask) { for (int i = 0; i < 4; ++i) { if (coord[i]) @@ -1093,28 +1166,22 @@ nir_ssa_def *LowerTexToBackend::prep_src(std::array &coord, return nir_vec(b, coord.data(), 4); } -nir_ssa_def *LowerTexToBackend::prepare_coord(nir_tex_instr *tex, - int &unnormalized_mask, - int &used_coord_mask) +nir_ssa_def * +LowerTexToBackend::prepare_coord(nir_tex_instr *tex, + int& unnormalized_mask, + int& used_coord_mask) { - std::array new_coord = { - nullptr, - nullptr, - nullptr, - nullptr - }; + std::array new_coord = {nullptr, nullptr, nullptr, nullptr}; unnormalized_mask = get_src_coords(tex, new_coord, true); used_coord_mask = 0; - int comp_idx = tex->is_shadow ? - nir_tex_instr_src_index(tex, nir_tex_src_comparator): - -1; + int comp_idx = + tex->is_shadow ? nir_tex_instr_src_index(tex, nir_tex_src_comparator) : -1; if (tex->op == nir_texop_txl || tex->op == nir_texop_txb) { - int idx = tex->op == nir_texop_txl ? - nir_tex_instr_src_index(tex, nir_tex_src_lod) : - nir_tex_instr_src_index(tex, nir_tex_src_bias); + int idx = tex->op == nir_texop_txl ? nir_tex_instr_src_index(tex, nir_tex_src_lod) + : nir_tex_instr_src_index(tex, nir_tex_src_bias); assert(idx != -1); new_coord[3] = tex->src[idx].src.ssa; @@ -1126,21 +1193,17 @@ nir_ssa_def *LowerTexToBackend::prepare_coord(nir_tex_instr *tex, return prep_src(new_coord, used_coord_mask); } -int LowerTexToBackend::get_src_coords(nir_tex_instr *tex, - std::array& coord, - bool round_array_index) +int +LowerTexToBackend::get_src_coords(nir_tex_instr *tex, + std::array& coord, + bool round_array_index) { int unnormalized_mask = 0; auto coord_idx = nir_tex_instr_src_index(tex, nir_tex_src_coord); assert(coord_idx != -1); auto old_coord = tex->src[coord_idx]; - coord = { - nir_channel(b, old_coord.src.ssa, 0), - nullptr, - nullptr, - nullptr - }; + coord = {nir_channel(b, old_coord.src.ssa, 0), nullptr, nullptr, nullptr}; if (tex->coord_components > 1) { if (tex->is_array && tex->sampler_dim == GLSL_SAMPLER_DIM_1D) @@ -1150,7 +1213,7 @@ int LowerTexToBackend::get_src_coords(nir_tex_instr *tex, } if (tex->coord_components > 2) { - coord[2] = nir_channel(b, old_coord.src.ssa, 2); + coord[2] = nir_channel(b, old_coord.src.ssa, 2); } if (tex->is_array) { unnormalized_mask |= 0x4; @@ -1165,4 +1228,4 @@ int LowerTexToBackend::get_src_coords(nir_tex_instr *tex, return unnormalized_mask; } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_tex.h b/src/gallium/drivers/r600/sfn/sfn_instr_tex.h index 4b40a13991c..051dfee7c32 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instr_tex.h +++ b/src/gallium/drivers/r600/sfn/sfn_instr_tex.h @@ -28,9 +28,8 @@ #define INSTR_TEX_H #include "sfn_instr.h" -#include "sfn_valuefactory.h" - #include "sfn_shader.h" +#include "sfn_valuefactory.h" namespace r600 { @@ -54,7 +53,7 @@ public: sample_g = FETCH_OP_SAMPLE_G, sample_g_lb = FETCH_OP_SAMPLE_G_L, gather4 = FETCH_OP_GATHER4, - gather4_o = FETCH_OP_GATHER4_O, + gather4_o = FETCH_OP_GATHER4_O, sample_c = FETCH_OP_SAMPLE_C, sample_c_l = FETCH_OP_SAMPLE_C_L, @@ -63,7 +62,7 @@ public: sample_c_g = FETCH_OP_SAMPLE_C_G, sample_c_g_lb = FETCH_OP_SAMPLE_C_G_L, gather4_c = FETCH_OP_GATHER4_C, - gather4_c_o = FETCH_OP_GATHER4_C_O, + gather4_c_o = FETCH_OP_GATHER4_C_O, unknown = 255 }; @@ -76,17 +75,15 @@ public: num_tex_flag }; - static constexpr Flags TexFlags[] = { - x_unnormalized, - y_unnormalized, - z_unnormalized, - w_unnormalized, - grad_fine, - num_tex_flag - }; + static constexpr Flags TexFlags[] = {x_unnormalized, + y_unnormalized, + z_unnormalized, + w_unnormalized, + grad_fine, + num_tex_flag}; struct Inputs { - Inputs(const nir_tex_instr& instr, ValueFactory &vf); + Inputs(const nir_tex_instr& instr, ValueFactory& vf); const nir_variable *sampler_deref; const nir_variable *texture_deref; RegisterVec4 coord; @@ -106,37 +103,41 @@ public: RegisterVec4::Swizzle swizzle_from_ncomps(int comps) const; Opcode opcode; + private: auto get_opcode(const nir_tex_instr& instr) -> Opcode; }; - TexInstr(Opcode op, const RegisterVec4& dest, + TexInstr(Opcode op, + const RegisterVec4& dest, const RegisterVec4::Swizzle& dest_swizzle, - const RegisterVec4& src, unsigned sid, unsigned rid, + const RegisterVec4& src, + unsigned sid, + unsigned rid, PRegister sampler_offs = nullptr); TexInstr(const TexInstr& orig) = delete; TexInstr(const TexInstr&& orig) = delete; - TexInstr& operator =(const TexInstr& orig) = delete; - TexInstr& operator =(const TexInstr&& orig) = delete; + TexInstr& operator=(const TexInstr& orig) = delete; + TexInstr& operator=(const TexInstr&& orig) = delete; void accept(ConstInstrVisitor& visitor) const override; void accept(InstrVisitor& visitor) override; - const auto& src() const {return m_src;} - auto& src() {return m_src;} + const auto& src() const { return m_src; } + auto& src() { return m_src; } - unsigned opcode() const {return m_opcode;} - unsigned resource_id() const {return m_resource_id;} + unsigned opcode() const { return m_opcode; } + unsigned resource_id() const { return m_resource_id; } void set_offset(unsigned index, int32_t val); int get_offset(unsigned index) const; - void set_inst_mode(int inst_mode) { m_inst_mode = inst_mode;} - int inst_mode() const { return m_inst_mode;} + void set_inst_mode(int inst_mode) { m_inst_mode = inst_mode; } + int inst_mode() const { return m_inst_mode; } - void set_tex_flag(Flags flag) {m_tex_flags.set(flag);} - bool has_tex_flag(Flags flag) const {return m_tex_flags.test(flag);} + void set_tex_flag(Flags flag) { m_tex_flags.set(flag); } + bool has_tex_flag(Flags flag) const { return m_tex_flags.test(flag); } void set_gather_comp(int cmp); bool is_equal_to(const TexInstr& lhs) const; @@ -146,15 +147,15 @@ public: static bool from_nir(nir_tex_instr *tex, Shader& shader); - uint32_t slots() const override {return 1;}; + uint32_t slots() const override { return 1; }; - auto prepare_instr() const { return m_prepare_instr;} + auto prepare_instr() const { return m_prepare_instr; } bool replace_source(PRegister old_src, PVirtualValue new_src) override; uint8_t allowed_dest_chan_mask() const override; -private: +private: bool do_ready() const override; void do_print(std::ostream& os) const override; bool propagate_death() override; @@ -165,22 +166,26 @@ private: void read_tex_coord_normalitazion(const std::string& next_token); void set_tex_param(const std::string& next_token); - static auto prepare_source(nir_tex_instr *tex, const Inputs& inputs, Shader &shader) -> RegisterVec4; + static auto prepare_source(nir_tex_instr *tex, const Inputs& inputs, Shader& shader) + -> RegisterVec4; static bool emit_buf_txf(nir_tex_instr *tex, Inputs& src, Shader& shader); - static bool emit_tex_txs(nir_tex_instr *tex, Inputs& src, - RegisterVec4::Swizzle dest_swz, Shader& shader); - static bool emit_tex_lod(nir_tex_instr* tex, Inputs& src, Shader& shader); - static bool emit_tex_texture_samples(nir_tex_instr* instr, Inputs& src, Shader& shader); - static bool emit_lowered_tex(nir_tex_instr* instr, Inputs& src, Shader& shader); - static void emit_set_gradients(nir_tex_instr* tex, int sampler_id, - Inputs& src, TexInstr *irt, Shader& shader); - static void emit_set_offsets(nir_tex_instr* tex, int sampler_id, - Inputs& src, TexInstr *irt, Shader& shader); + static bool emit_tex_txs(nir_tex_instr *tex, + Inputs& src, + RegisterVec4::Swizzle dest_swz, + Shader& shader); + static bool emit_tex_lod(nir_tex_instr *tex, Inputs& src, Shader& shader); + static bool + emit_tex_texture_samples(nir_tex_instr *instr, Inputs& src, Shader& shader); + static bool emit_lowered_tex(nir_tex_instr *instr, Inputs& src, Shader& shader); + static void emit_set_gradients( + nir_tex_instr *tex, int sampler_id, Inputs& src, TexInstr *irt, Shader& shader); + static void emit_set_offsets( + nir_tex_instr *tex, int sampler_id, Inputs& src, TexInstr *irt, Shader& shader); bool set_coord_offsets(nir_src *offset); - void set_rect_coordinate_flags(nir_tex_instr* instr); - void add_prepare_instr(TexInstr *ir) {m_prepare_instr.push_back(ir);}; + void set_rect_coordinate_flags(nir_tex_instr *instr); + void add_prepare_instr(TexInstr *ir) { m_prepare_instr.push_back(ir); }; Opcode m_opcode; @@ -194,8 +199,9 @@ private: std::list m_prepare_instr; }; -bool r600_nir_lower_tex_to_backend(nir_shader *shader, amd_gfx_level chip_class); +bool +r600_nir_lower_tex_to_backend(nir_shader *shader, amd_gfx_level chip_class); -} +} // namespace r600 #endif // INSTR_TEX_H diff --git a/src/gallium/drivers/r600/sfn/sfn_instrfactory.cpp b/src/gallium/drivers/r600/sfn/sfn_instrfactory.cpp index e28446f3c5e..44542f93de4 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instrfactory.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_instrfactory.cpp @@ -26,21 +26,19 @@ #include "sfn_instrfactory.h" -#include "sfn_instr_alugroup.h" +#include "sfn_alu_defines.h" #include "sfn_debug.h" +#include "sfn_instr_alugroup.h" #include "sfn_instr_controlflow.h" #include "sfn_instr_export.h" #include "sfn_instr_fetch.h" #include "sfn_instr_lds.h" #include "sfn_instr_mem.h" #include "sfn_instr_tex.h" - -#include "sfn_alu_defines.h" - #include "sfn_shader.h" -#include #include +#include #include namespace r600 { @@ -49,12 +47,12 @@ using std::string; using std::vector; InstrFactory::InstrFactory(): - group(nullptr) + group(nullptr) { - } -PInst InstrFactory::from_string(const std::string& s, int nesting_depth) +PInst +InstrFactory::from_string(const std::string& s, int nesting_depth) { string type; std::istringstream is(s); @@ -115,7 +113,8 @@ PInst InstrFactory::from_string(const std::string& s, int nesting_depth) return result; } -bool InstrFactory::from_nir(nir_instr *instr, Shader& shader) +bool +InstrFactory::from_nir(nir_instr *instr, Shader& shader) { switch (instr->type) { case nir_instr_type_alu: @@ -132,11 +131,12 @@ bool InstrFactory::from_nir(nir_instr *instr, Shader& shader) return process_undef(nir_instr_as_ssa_undef(instr), shader); default: fprintf(stderr, "Instruction type %d not supported\n", instr->type); - return false; + return false; } } -bool InstrFactory::load_const(nir_load_const_instr *literal, Shader& shader) +bool +InstrFactory::load_const(nir_load_const_instr *literal, Shader& shader) { AluInstr *ir = nullptr; @@ -157,39 +157,50 @@ bool InstrFactory::load_const(nir_load_const_instr *literal, Shader& shader) uint32_t v = literal->value[i].i32; PVirtualValue src = nullptr; switch (v) { - case 0: src = m_value_factory.zero(); break; - case 1: src = m_value_factory.one_i(); break; - case 0xffffffff: src = m_value_factory.inline_const(ALU_SRC_M_1_INT, 0); break; - case 0x3f800000: src = m_value_factory.inline_const(ALU_SRC_1, 0); break; - case 0x3f000000: src = m_value_factory.inline_const(ALU_SRC_0_5, 0); break; - default: src = m_value_factory.literal(v); - } + case 0: + src = m_value_factory.zero(); + break; + case 1: + src = m_value_factory.one_i(); + break; + case 0xffffffff: + src = m_value_factory.inline_const(ALU_SRC_M_1_INT, 0); + break; + case 0x3f800000: + src = m_value_factory.inline_const(ALU_SRC_1, 0); + break; + case 0x3f000000: + src = m_value_factory.inline_const(ALU_SRC_0_5, 0); + break; + default: + src = m_value_factory.literal(v); + } ir = new AluInstr(op1_mov, dest, src, {alu_write}); shader.emit_instruction(ir); } if (ir) ir->set_alu_flag(alu_last_instr); - } return true; } -bool InstrFactory::process_jump(nir_jump_instr *instr, Shader& shader) +bool +InstrFactory::process_jump(nir_jump_instr *instr, Shader& shader) { ControlFlowInstr::CFType type; switch (instr->type) { case nir_jump_break: type = ControlFlowInstr::cf_loop_break; - break; + break; case nir_jump_continue: type = ControlFlowInstr::cf_loop_continue; - break; + break; default: { - nir_instr *i = reinterpret_cast(instr); - sfn_log << SfnLog::err << "Jump instrunction " << *i << " not supported\n"; + nir_instr *i = reinterpret_cast(instr); + sfn_log << SfnLog::err << "Jump instrunction " << *i << " not supported\n"; return false; } } @@ -199,16 +210,15 @@ bool InstrFactory::process_jump(nir_jump_instr *instr, Shader& shader) return true; } -bool InstrFactory::process_undef(nir_ssa_undef_instr *undef, Shader& shader) +bool +InstrFactory::process_undef(nir_ssa_undef_instr *undef, Shader& shader) { for (int i = 0; i < undef->def.num_components; ++i) { auto dest = shader.value_factory().undef(undef->def.index, i); - shader.emit_instruction(new AluInstr(op1_mov, dest, - value_factory().zero(), - AluInstr::last_write)); + shader.emit_instruction( + new AluInstr(op1_mov, dest, value_factory().zero(), AluInstr::last_write)); } return true; } - -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_instrfactory.h b/src/gallium/drivers/r600/sfn/sfn_instrfactory.h index 9b92b70cba9..277c228b9ba 100644 --- a/src/gallium/drivers/r600/sfn/sfn_instrfactory.h +++ b/src/gallium/drivers/r600/sfn/sfn_instrfactory.h @@ -30,7 +30,6 @@ #include "sfn_instr.h" #include "sfn_valuefactory.h" - #include namespace r600 { @@ -38,11 +37,11 @@ namespace r600 { class Shader; class InstrFactory : public Allocate { public: - InstrFactory(); + InstrFactory(); - PInst from_string(const std::string &s, int nesting_depth); + PInst from_string(const std::string& s, int nesting_depth); bool from_nir(nir_instr *instr, Shader& shader); - auto& value_factory() { return m_value_factory;} + auto& value_factory() { return m_value_factory; } private: bool load_const(nir_load_const_instr *lc, Shader& shader); @@ -51,10 +50,10 @@ private: Instr::Pointer export_from_string(std::istream& is, bool is_last); - ValueFactory m_value_factory; + ValueFactory m_value_factory; AluGroup *group; }; -} +} // namespace r600 #endif // INSTRFACTORY_H diff --git a/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator.cpp b/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator.cpp index a6ebea0a5f4..46d4084d15d 100644 --- a/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator.cpp @@ -25,16 +25,16 @@ */ #include "sfn_liverangeevaluator.h" -#include "sfn_liverangeevaluator_helpers.h" +#include "sfn_debug.h" #include "sfn_instr_alugroup.h" #include "sfn_instr_controlflow.h" #include "sfn_instr_export.h" #include "sfn_instr_fetch.h" #include "sfn_instr_mem.h" #include "sfn_instr_tex.h" +#include "sfn_liverangeevaluator_helpers.h" #include "sfn_shader.h" -#include "sfn_debug.h" #include #include @@ -56,7 +56,7 @@ public: void visit(ScratchIOInstr *instr) override; void visit(StreamOutInstr *instr) override; void visit(MemRingOutInstr *instr) override; - void visit(EmitVertexInstr *instr) override {(void)instr;} + void visit(EmitVertexInstr *instr) override { (void)instr; } void visit(GDSInstr *instr) override; void visit(WriteTFInstr *instr) override; void visit(LDSAtomicInstr *instr) override; @@ -64,13 +64,13 @@ public: void visit(RatInstr *instr) override; void finalize(); -private: +private: void record_write(const Register *reg); void record_read(const Register *reg, LiveRangeEntry::EUse use); void record_write(const RegisterVec4& reg, const RegisterVec4::Swizzle& swizzle); - void record_read(const RegisterVec4 ®, LiveRangeEntry::EUse use); + void record_read(const RegisterVec4& reg, LiveRangeEntry::EUse use); void scope_if(); void scope_else(); @@ -78,8 +78,8 @@ private: void scope_loop_begin(); void scope_loop_end(); void scope_loop_break(); - ProgramScope *create_scope(ProgramScope *parent, ProgramScopeType type, - int id, int nesting_depth, int line); + ProgramScope *create_scope( + ProgramScope *parent, ProgramScopeType type, int id, int nesting_depth, int line); std::vector> m_scopes; ProgramScope *m_current_scope; @@ -91,18 +91,14 @@ private: int m_loop_id{1}; }; -LiveRangeEvaluator::LiveRangeEvaluator() -{ +LiveRangeEvaluator::LiveRangeEvaluator() {} -} - - -LiveRangeMap LiveRangeEvaluator::run(Shader& sh) +LiveRangeMap +LiveRangeEvaluator::run(Shader& sh) { LiveRangeMap range_map = sh.prepare_live_range_map(); - LiveRangeInstrVisitor evaluator(range_map); for (auto& b : sh.func()) @@ -113,14 +109,15 @@ LiveRangeMap LiveRangeEvaluator::run(Shader& sh) return range_map; } -void LiveRangeInstrVisitor::finalize() +void +LiveRangeInstrVisitor::finalize() { m_current_scope->set_end(m_line); for (int i = 0; i < 4; ++i) { auto& live_ranges = m_live_range_map.component(i); - for(const auto& r : live_ranges) { + for (const auto& r : live_ranges) { if (r.m_register->live_end_pinned()) record_read(r.m_register, LiveRangeEntry::use_unspecified); } @@ -128,7 +125,8 @@ void LiveRangeInstrVisitor::finalize() auto& comp_access = m_register_access.component(i); for (size_t i = 0; i < comp_access.size(); ++i) { - sfn_log << SfnLog::merge << "Evaluae access for " << *live_ranges[i].m_register << "\n"; + sfn_log << SfnLog::merge << "Evaluae access for " << *live_ranges[i].m_register + << "\n"; auto& rca = comp_access[i]; rca.update_required_live_range(); @@ -140,13 +138,13 @@ void LiveRangeInstrVisitor::finalize() } LiveRangeInstrVisitor::LiveRangeInstrVisitor(LiveRangeMap& live_range_map): - m_live_range_map(live_range_map), - m_register_access(live_range_map.sizes()) + m_live_range_map(live_range_map), + m_register_access(live_range_map.sizes()) { if (sfn_log.has_debug_flag(SfnLog::merge)) { sfn_log << SfnLog::merge << "Have component register numbers: "; for (auto n : live_range_map.sizes()) - sfn_log << n << " "; + sfn_log << n << " "; sfn_log << "\n"; } @@ -155,7 +153,7 @@ LiveRangeInstrVisitor::LiveRangeInstrVisitor(LiveRangeMap& live_range_map): for (int i = 0; i < 4; ++i) { const auto& comp = live_range_map.component(i); - for(const auto& r : comp) { + for (const auto& r : comp) { if (r.m_register->live_start_pinned()) record_write(r.m_register); } @@ -163,7 +161,9 @@ LiveRangeInstrVisitor::LiveRangeInstrVisitor(LiveRangeMap& live_range_map): m_line = 1; } -void LiveRangeInstrVisitor::record_write(const RegisterVec4& reg, const RegisterVec4::Swizzle &swizzle) +void +LiveRangeInstrVisitor::record_write(const RegisterVec4& reg, + const RegisterVec4::Swizzle& swizzle) { for (int i = 0; i < 4; ++i) { if (swizzle[i] < 6 && reg[i]->chan() < 4) @@ -171,7 +171,8 @@ void LiveRangeInstrVisitor::record_write(const RegisterVec4& reg, const Register } } -void LiveRangeInstrVisitor::record_read(const RegisterVec4& reg, LiveRangeEntry::EUse use) +void +LiveRangeInstrVisitor::record_read(const RegisterVec4& reg, LiveRangeEntry::EUse use) { for (int i = 0; i < 4; ++i) { if (reg[i]->chan() < 4) @@ -179,54 +180,72 @@ void LiveRangeInstrVisitor::record_read(const RegisterVec4& reg, LiveRangeEntry: } } -void LiveRangeInstrVisitor::scope_if() +void +LiveRangeInstrVisitor::scope_if() { - m_current_scope = create_scope(m_current_scope, if_branch, m_if_id++, - m_current_scope->nesting_depth() + 1, m_line + 1); + m_current_scope = create_scope(m_current_scope, + if_branch, + m_if_id++, + m_current_scope->nesting_depth() + 1, + m_line + 1); } -void LiveRangeInstrVisitor::scope_else() +void +LiveRangeInstrVisitor::scope_else() { assert(m_current_scope->type() == if_branch); m_current_scope->set_end(m_line - 1); - m_current_scope = create_scope(m_current_scope->parent(), else_branch, m_current_scope->id(), - m_current_scope->nesting_depth() + 1, m_line + 1); + m_current_scope = create_scope(m_current_scope->parent(), + else_branch, + m_current_scope->id(), + m_current_scope->nesting_depth() + 1, + m_line + 1); } -void LiveRangeInstrVisitor::scope_endif() +void +LiveRangeInstrVisitor::scope_endif() { m_current_scope->set_end(m_line - 1); m_current_scope = m_current_scope->parent(); assert(m_current_scope); } -void LiveRangeInstrVisitor::scope_loop_begin() +void +LiveRangeInstrVisitor::scope_loop_begin() { - m_current_scope = create_scope(m_current_scope, loop_body, m_loop_id++, - m_current_scope->nesting_depth() + 1, m_line); + m_current_scope = create_scope(m_current_scope, + loop_body, + m_loop_id++, + m_current_scope->nesting_depth() + 1, + m_line); } -void LiveRangeInstrVisitor::scope_loop_end() +void +LiveRangeInstrVisitor::scope_loop_end() { m_current_scope->set_end(m_line); m_current_scope = m_current_scope->parent(); assert(m_current_scope); } -void LiveRangeInstrVisitor::scope_loop_break() +void +LiveRangeInstrVisitor::scope_loop_break() { m_current_scope->set_loop_break_line(m_line); } -ProgramScope *LiveRangeInstrVisitor::create_scope(ProgramScope *parent, ProgramScopeType type, - int id, int nesting_depth, int line) +ProgramScope * +LiveRangeInstrVisitor::create_scope( + ProgramScope *parent, ProgramScopeType type, int id, int nesting_depth, int line) { - m_scopes.emplace_back(std::make_unique(parent, type, id, nesting_depth, line)); + m_scopes.emplace_back( + std::make_unique(parent, type, id, nesting_depth, line)); return m_scopes[m_scopes.size() - 1].get(); } -void LiveRangeInstrVisitor::visit(AluInstr *instr) +void +LiveRangeInstrVisitor::visit(AluInstr *instr) { sfn_log << SfnLog::merge << "Visit " << *instr << "\n"; if (instr->has_alu_flag(alu_write)) @@ -240,14 +259,16 @@ void LiveRangeInstrVisitor::visit(AluInstr *instr) } } -void LiveRangeInstrVisitor::visit(AluGroup *group) +void +LiveRangeInstrVisitor::visit(AluGroup *group) { for (auto i : *group) if (i) i->accept(*this); } -void LiveRangeInstrVisitor::visit(TexInstr *instr) +void +LiveRangeInstrVisitor::visit(TexInstr *instr) { sfn_log << SfnLog::merge << "Visit " << *instr << "\n"; record_write(instr->dst(), instr->all_dest_swizzle()); @@ -259,14 +280,16 @@ void LiveRangeInstrVisitor::visit(TexInstr *instr) record_read(instr->resource_offset(), LiveRangeEntry::use_unspecified); } -void LiveRangeInstrVisitor::visit(ExportInstr *instr) +void +LiveRangeInstrVisitor::visit(ExportInstr *instr) { sfn_log << SfnLog::merge << "Visit " << *instr << "\n"; auto src = instr->value(); record_read(src, LiveRangeEntry::use_export); } -void LiveRangeInstrVisitor::visit(FetchInstr *instr) +void +LiveRangeInstrVisitor::visit(FetchInstr *instr) { sfn_log << SfnLog::merge << "Visit " << *instr << "\n"; record_write(instr->dst(), instr->all_dest_swizzle()); @@ -275,7 +298,8 @@ void LiveRangeInstrVisitor::visit(FetchInstr *instr) record_read(&src, LiveRangeEntry::use_unspecified); } -void LiveRangeInstrVisitor::visit(Block *instr) +void +LiveRangeInstrVisitor::visit(Block *instr) { sfn_log << SfnLog::merge << "Visit block\n"; for (auto i : *instr) { @@ -286,7 +310,8 @@ void LiveRangeInstrVisitor::visit(Block *instr) sfn_log << SfnLog::merge << "End block\n"; } -void LiveRangeInstrVisitor::visit(ScratchIOInstr *instr) +void +LiveRangeInstrVisitor::visit(ScratchIOInstr *instr) { auto& src = instr->value(); for (int i = 0; i < 4; ++i) { @@ -303,14 +328,16 @@ void LiveRangeInstrVisitor::visit(ScratchIOInstr *instr) record_read(addr, LiveRangeEntry::use_unspecified); } -void LiveRangeInstrVisitor::visit(StreamOutInstr *instr) +void +LiveRangeInstrVisitor::visit(StreamOutInstr *instr) { sfn_log << SfnLog::merge << "Visit " << *instr << "\n"; auto src = instr->value(); record_read(src, LiveRangeEntry::use_unspecified); } -void LiveRangeInstrVisitor::visit(MemRingOutInstr *instr) +void +LiveRangeInstrVisitor::visit(MemRingOutInstr *instr) { sfn_log << SfnLog::merge << "Visit " << *instr << "\n"; auto src = instr->value(); @@ -321,28 +348,43 @@ void LiveRangeInstrVisitor::visit(MemRingOutInstr *instr) record_read(idx->as_register(), LiveRangeEntry::use_unspecified); } -void LiveRangeInstrVisitor::visit(ControlFlowInstr *instr) +void +LiveRangeInstrVisitor::visit(ControlFlowInstr *instr) { switch (instr->cf_type()) { - case ControlFlowInstr::cf_else: scope_else(); break; - case ControlFlowInstr::cf_endif: scope_endif(); break; - case ControlFlowInstr::cf_loop_begin: scope_loop_begin(); break; - case ControlFlowInstr::cf_loop_end: scope_loop_end(); break; - case ControlFlowInstr::cf_loop_break: scope_loop_break(); break; - case ControlFlowInstr::cf_loop_continue: break; - case ControlFlowInstr::cf_wait_ack: break; + case ControlFlowInstr::cf_else: + scope_else(); + break; + case ControlFlowInstr::cf_endif: + scope_endif(); + break; + case ControlFlowInstr::cf_loop_begin: + scope_loop_begin(); + break; + case ControlFlowInstr::cf_loop_end: + scope_loop_end(); + break; + case ControlFlowInstr::cf_loop_break: + scope_loop_break(); + break; + case ControlFlowInstr::cf_loop_continue: + break; + case ControlFlowInstr::cf_wait_ack: + break; default: unreachable("Flow control unreachanble"); } } -void LiveRangeInstrVisitor::visit(IfInstr *instr) +void +LiveRangeInstrVisitor::visit(IfInstr *instr) { instr->predicate()->accept(*this); scope_if(); } -void LiveRangeInstrVisitor::visit(GDSInstr *instr) +void +LiveRangeInstrVisitor::visit(GDSInstr *instr) { sfn_log << SfnLog::merge << "Visit " << *instr << "\n"; record_read(instr->src(), LiveRangeEntry::use_unspecified); @@ -351,7 +393,8 @@ void LiveRangeInstrVisitor::visit(GDSInstr *instr) record_write(instr->dest()); } -void LiveRangeInstrVisitor::visit(RatInstr *instr) +void +LiveRangeInstrVisitor::visit(RatInstr *instr) { sfn_log << SfnLog::merge << "Visit " << *instr << "\n"; record_read(instr->value(), LiveRangeEntry::use_unspecified); @@ -362,23 +405,28 @@ void LiveRangeInstrVisitor::visit(RatInstr *instr) record_read(idx, LiveRangeEntry::use_unspecified); } - -void LiveRangeInstrVisitor::visit(WriteTFInstr *instr) +void +LiveRangeInstrVisitor::visit(WriteTFInstr *instr) { record_read(instr->value(), LiveRangeEntry::use_export); } -void LiveRangeInstrVisitor::visit(UNUSED LDSAtomicInstr *instr) -{ - unreachable("LDSAtomicInstr must be lowered before scheduling and live range evaluation"); -} - -void LiveRangeInstrVisitor::visit(UNUSED LDSReadInstr *instr) +void +LiveRangeInstrVisitor::visit(UNUSED LDSAtomicInstr *instr) { - unreachable("LDSReadInstr must be lowered before scheduling and live range evaluation"); + unreachable("LDSAtomicInstr must be lowered before scheduling and live " + "range evaluation"); } -void LiveRangeInstrVisitor::record_write(const Register *reg) +void +LiveRangeInstrVisitor::visit(UNUSED LDSReadInstr *instr) +{ + unreachable("LDSReadInstr must be lowered before scheduling and live " + "range evaluation"); +} + +void +LiveRangeInstrVisitor::record_write(const Register *reg) { auto addr = reg->get_addr(); if (addr && addr->as_register()) { @@ -395,19 +443,20 @@ void LiveRangeInstrVisitor::record_write(const Register *reg) } } else { auto& ra = m_register_access(*reg); - sfn_log << SfnLog::merge << *reg << " write:" << m_line << "\n"; + sfn_log << SfnLog::merge << *reg << " write:" << m_line << "\n"; ra.record_write(m_line, m_current_scope); } } -void LiveRangeInstrVisitor::record_read(const Register *reg, LiveRangeEntry::EUse use) +void +LiveRangeInstrVisitor::record_read(const Register *reg, LiveRangeEntry::EUse use) { if (!reg) return; auto addr = reg->get_addr(); if (addr && addr->as_register()) { - sfn_log << SfnLog::merge << "Record reading address register " << *addr << "\n"; + sfn_log << SfnLog::merge << "Record reading address register " << *addr << "\n"; auto& ra = m_register_access(*addr->as_register()); ra.record_read(m_line, m_current_scope, use); @@ -427,7 +476,8 @@ void LiveRangeInstrVisitor::record_read(const Register *reg, LiveRangeEntry::EUs } } -std::ostream& operator << (std::ostream& os, const LiveRangeMap& lrm) +std::ostream& +operator<<(std::ostream& os, const LiveRangeMap& lrm) { os << "Live ranges\n"; for (int i = 0; i < 4; ++i) { @@ -438,7 +488,8 @@ std::ostream& operator << (std::ostream& os, const LiveRangeMap& lrm) return os; } -bool operator == (const LiveRangeMap& lhs, const LiveRangeMap& rhs) +bool +operator==(const LiveRangeMap& lhs, const LiveRangeMap& rhs) { for (int i = 0; i < 4; ++i) { const auto& lc = lhs.component(i); @@ -450,10 +501,8 @@ bool operator == (const LiveRangeMap& lhs, const LiveRangeMap& rhs) const auto& lv = lc[j]; const auto& rv = rc[j]; - if (lv.m_start != rv.m_start || - lv.m_end != rv.m_end || - lv.m_color != rv.m_color || - !lv.m_register->equal_to(*rv.m_register)) + if (lv.m_start != rv.m_start || lv.m_end != rv.m_end || + lv.m_color != rv.m_color || !lv.m_register->equal_to(*rv.m_register)) return false; } } @@ -461,6 +510,4 @@ bool operator == (const LiveRangeMap& lhs, const LiveRangeMap& rhs) return true; } - -} - +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator.h b/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator.h index 91ccf37d771..2229d42b37a 100644 --- a/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator.h +++ b/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator.h @@ -29,21 +29,20 @@ #include "sfn_valuefactory.h" -#include #include +#include namespace r600 { class Shader; -class LiveRangeEvaluator { +class LiveRangeEvaluator { public: - LiveRangeEvaluator(); - LiveRangeMap run(Shader &sh); + LiveRangeMap run(Shader& sh); }; -} +} // namespace r600 #endif // LIFERANGEEVALUATOR_H diff --git a/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator_helpers.cpp b/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator_helpers.cpp index 66380baab36..7a692ddd173 100644 --- a/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator_helpers.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator_helpers.cpp @@ -27,53 +27,57 @@ #include "sfn_liverangeevaluator_helpers.h" #include "sfn_virtualvalues.h" - #include "util/u_math.h" -#include #include #include +#include namespace r600 { -ProgramScope::ProgramScope(ProgramScope *parent, ProgramScopeType type, int id, - int depth, int scope_begin): - scope_type(type), - scope_id(id), - scope_nesting_depth(depth), - scope_begin(scope_begin), - scope_end(-1), - break_loop_line(std::numeric_limits::max()), - parent_scope(parent) +ProgramScope::ProgramScope( + ProgramScope *parent, ProgramScopeType type, int id, int depth, int scope_begin): + scope_type(type), + scope_id(id), + scope_nesting_depth(depth), + scope_begin(scope_begin), + scope_end(-1), + break_loop_line(std::numeric_limits::max()), + parent_scope(parent) { } ProgramScope::ProgramScope(): - ProgramScope(nullptr, undefined_scope, -1, -1, -1) + ProgramScope(nullptr, undefined_scope, -1, -1, -1) { } -ProgramScopeType ProgramScope::type() const +ProgramScopeType +ProgramScope::type() const { return scope_type; } -ProgramScope *ProgramScope::parent() const +ProgramScope * +ProgramScope::parent() const { return parent_scope; } -int ProgramScope::nesting_depth() const +int +ProgramScope::nesting_depth() const { return scope_nesting_depth; } -bool ProgramScope::is_loop() const +bool +ProgramScope::is_loop() const { return (scope_type == loop_body); } -bool ProgramScope::is_in_loop() const +bool +ProgramScope::is_in_loop() const { if (scope_type == loop_body) return true; @@ -84,7 +88,8 @@ bool ProgramScope::is_in_loop() const return false; } -const ProgramScope *ProgramScope::innermost_loop() const +const ProgramScope * +ProgramScope::innermost_loop() const { if (scope_type == loop_body) return this; @@ -95,7 +100,8 @@ const ProgramScope *ProgramScope::innermost_loop() const return nullptr; } -const ProgramScope *ProgramScope::outermost_loop() const +const ProgramScope * +ProgramScope::outermost_loop() const { const ProgramScope *loop = nullptr; const ProgramScope *p = this; @@ -109,7 +115,8 @@ const ProgramScope *ProgramScope::outermost_loop() const return loop; } -bool ProgramScope::is_child_of_ifelse_id_sibling(const ProgramScope *scope) const +bool +ProgramScope::is_child_of_ifelse_id_sibling(const ProgramScope *scope) const { const ProgramScope *my_parent = in_parent_ifelse_scope(); while (my_parent) { @@ -124,7 +131,8 @@ bool ProgramScope::is_child_of_ifelse_id_sibling(const ProgramScope *scope) cons return false; } -bool ProgramScope::is_child_of(const ProgramScope *scope) const +bool +ProgramScope::is_child_of(const ProgramScope *scope) const { const ProgramScope *my_parent = parent(); while (my_parent) { @@ -135,7 +143,8 @@ bool ProgramScope::is_child_of(const ProgramScope *scope) const return false; } -const ProgramScope *ProgramScope::enclosing_conditional() const +const ProgramScope * +ProgramScope::enclosing_conditional() const { if (is_conditional()) return this; @@ -146,20 +155,21 @@ const ProgramScope *ProgramScope::enclosing_conditional() const return nullptr; } -bool ProgramScope::contains_range_of(const ProgramScope& other) const +bool +ProgramScope::contains_range_of(const ProgramScope& other) const { return (begin() <= other.begin()) && (end() >= other.end()); } -bool ProgramScope::is_conditional() const +bool +ProgramScope::is_conditional() const { - return scope_type == if_branch || - scope_type == else_branch || - scope_type == switch_case_branch || - scope_type == switch_default_branch; + return scope_type == if_branch || scope_type == else_branch || + scope_type == switch_case_branch || scope_type == switch_default_branch; } -const ProgramScope *ProgramScope::in_else_scope() const +const ProgramScope * +ProgramScope::in_else_scope() const { if (scope_type == else_branch) return this; @@ -170,7 +180,8 @@ const ProgramScope *ProgramScope::in_else_scope() const return nullptr; } -const ProgramScope *ProgramScope::in_parent_ifelse_scope() const +const ProgramScope * +ProgramScope::in_parent_ifelse_scope() const { if (parent_scope) return parent_scope->in_ifelse_scope(); @@ -178,10 +189,10 @@ const ProgramScope *ProgramScope::in_parent_ifelse_scope() const return nullptr; } -const ProgramScope *ProgramScope::in_ifelse_scope() const +const ProgramScope * +ProgramScope::in_ifelse_scope() const { - if (scope_type == if_branch || - scope_type == else_branch) + if (scope_type == if_branch || scope_type == else_branch) return this; if (parent_scope) @@ -190,20 +201,20 @@ const ProgramScope *ProgramScope::in_ifelse_scope() const return nullptr; } -bool ProgramScope::is_switchcase_scope_in_loop() const +bool +ProgramScope::is_switchcase_scope_in_loop() const { - return (scope_type == switch_case_branch || - scope_type == switch_default_branch) && - is_in_loop(); + return (scope_type == switch_case_branch || scope_type == switch_default_branch) && + is_in_loop(); } -bool ProgramScope::break_is_for_switchcase() const +bool +ProgramScope::break_is_for_switchcase() const { if (scope_type == loop_body) return false; - if (scope_type == switch_case_branch || - scope_type == switch_default_branch || + if (scope_type == switch_case_branch || scope_type == switch_default_branch || scope_type == switch_body) return true; @@ -213,28 +224,33 @@ bool ProgramScope::break_is_for_switchcase() const return false; } -int ProgramScope::id() const +int +ProgramScope::id() const { return scope_id; } -int ProgramScope::begin() const +int +ProgramScope::begin() const { return scope_begin; } -int ProgramScope::end() const +int +ProgramScope::end() const { return scope_end; } -void ProgramScope::set_end(int end) +void +ProgramScope::set_end(int end) { if (scope_end == -1) scope_end = end; } -void ProgramScope::set_loop_break_line(int line) +void +ProgramScope::set_loop_break_line(int line) { if (scope_type == loop_body) { break_loop_line = MIN2(break_loop_line, line); @@ -244,36 +260,36 @@ void ProgramScope::set_loop_break_line(int line) } } -int ProgramScope::loop_break_line() const +int +ProgramScope::loop_break_line() const { return break_loop_line; } RegisterCompAccess::RegisterCompAccess(LiveRange range): - last_read_scope(nullptr), - first_read_scope(nullptr), - first_write_scope(nullptr), - first_write(range.start), - last_read(range.end), - last_write(range.start), - first_read(std::numeric_limits::max()), - conditionality_in_loop_id(conditionality_untouched), - if_scope_write_flags(0), - next_ifelse_nesting_depth(0), - current_unpaired_if_write_scope(nullptr), - was_written_in_current_else_scope(false), - m_range(range) + last_read_scope(nullptr), + first_read_scope(nullptr), + first_write_scope(nullptr), + first_write(range.start), + last_read(range.end), + last_write(range.start), + first_read(std::numeric_limits::max()), + conditionality_in_loop_id(conditionality_untouched), + if_scope_write_flags(0), + next_ifelse_nesting_depth(0), + current_unpaired_if_write_scope(nullptr), + was_written_in_current_else_scope(false), + m_range(range) { - } RegisterCompAccess::RegisterCompAccess(): - RegisterCompAccess(LiveRange(-1,-1)) + RegisterCompAccess(LiveRange(-1, -1)) { } - -void RegisterCompAccess::record_read(int line, ProgramScope *scope, LiveRangeEntry::EUse use) +void +RegisterCompAccess::record_read(int line, ProgramScope *scope, LiveRangeEntry::EUse use) { last_read_scope = scope; if (use != LiveRangeEntry::use_unspecified) @@ -305,10 +321,10 @@ void RegisterCompAccess::record_read(int line, ProgramScope *scope, LiveRangeEnt if ((conditionality_in_loop_id != write_is_conditional) && (conditionality_in_loop_id != enclosing_loop->id())) { - if (current_unpaired_if_write_scope) { + if (current_unpaired_if_write_scope) { - /* Has been written in this or a parent scope? - this makes the temporary - * unconditionally set at this point. + /* Has been written in this or a parent scope? - this makes the + * temporary unconditionally set at this point. */ if (scope->is_child_of(current_unpaired_if_write_scope)) return; @@ -332,7 +348,8 @@ void RegisterCompAccess::record_read(int line, ProgramScope *scope, LiveRangeEnt } } -void RegisterCompAccess::record_write(int line, ProgramScope *scope) +void +RegisterCompAccess::record_write(int line, ProgramScope *scope) { last_write = line; @@ -368,11 +385,12 @@ void RegisterCompAccess::record_write(int line, ProgramScope *scope) */ const ProgramScope *ifelse_scope = scope->in_ifelse_scope(); if (ifelse_scope && ifelse_scope->innermost_loop() && - ifelse_scope->innermost_loop()->id() != conditionality_in_loop_id) + ifelse_scope->innermost_loop()->id() != conditionality_in_loop_id) record_ifelse_write(*ifelse_scope); } -void RegisterCompAccess::record_ifelse_write(const ProgramScope& scope) +void +RegisterCompAccess::record_ifelse_write(const ProgramScope& scope) { if (scope.type() == if_branch) { /* The first write in an IF branch within a loop implies unresolved @@ -387,7 +405,8 @@ void RegisterCompAccess::record_ifelse_write(const ProgramScope& scope) } } -void RegisterCompAccess::record_if_write(const ProgramScope& scope) +void +RegisterCompAccess::record_if_write(const ProgramScope& scope) { /* Don't record write if this IF scope if it ... * - is not the first write in this IF scope, @@ -399,19 +418,20 @@ void RegisterCompAccess::record_if_write(const ProgramScope& scope) * - is the first one (obviously), * - happens in an IF branch that is a child of the ELSE branch of the * last active IF/ELSE pair. In this case recording this write is used to - * established whether the write is (un-)conditional in the scope enclosing - * this outer IF/ELSE pair. + * established whether the write is (un-)conditional in the scope + * enclosing this outer IF/ELSE pair. */ if (!current_unpaired_if_write_scope || (current_unpaired_if_write_scope->id() != scope.id() && - scope.is_child_of_ifelse_id_sibling(current_unpaired_if_write_scope))) { + scope.is_child_of_ifelse_id_sibling(current_unpaired_if_write_scope))) { if_scope_write_flags |= 1 << next_ifelse_nesting_depth; current_unpaired_if_write_scope = &scope; next_ifelse_nesting_depth++; } } -void RegisterCompAccess::record_else_write(const ProgramScope& scope) +void +RegisterCompAccess::record_else_write(const ProgramScope& scope) { int mask = 1 << (next_ifelse_nesting_depth - 1); @@ -427,57 +447,57 @@ void RegisterCompAccess::record_else_write(const ProgramScope& scope) if_scope_write_flags &= ~mask; /* The following code deals with propagating unconditionality from - * inner levels of nested IF/ELSE to the outer levels like in - * - * 1: var t; - * 2: if (a) { <- start scope A - * 3: if (b) - * 4: t = ... - * 5: else - * 6: t = ... - * 7: } else { <- start scope B - * 8: if (c) - * 9: t = ... - * A: else <- start scope C - * B: t = ... - * C: } - * - */ + * inner levels of nested IF/ELSE to the outer levels like in + * + * 1: var t; + * 2: if (a) { <- start scope A + * 3: if (b) + * 4: t = ... + * 5: else + * 6: t = ... + * 7: } else { <- start scope B + * 8: if (c) + * 9: t = ... + * A: else <- start scope C + * B: t = ... + * C: } + * + */ const ProgramScope *parent_ifelse = scope.parent()->in_ifelse_scope(); if (1 << (next_ifelse_nesting_depth - 1) & if_scope_write_flags) { /* We are at the end of scope C and already recorded a write - * within an IF scope (A), the sibling of the parent ELSE scope B, - * and it is not yet resolved. Mark that as the last relevant - * IF scope. Below the write will be resolved for the A/B - * scope pair. - */ + * within an IF scope (A), the sibling of the parent ELSE scope B, + * and it is not yet resolved. Mark that as the last relevant + * IF scope. Below the write will be resolved for the A/B + * scope pair. + */ current_unpaired_if_write_scope = parent_ifelse; } else { current_unpaired_if_write_scope = nullptr; } /* Promote the first write scope to the enclosing scope because - * the current IF/ELSE pair is now irrelevant for the analysis. - * This is also required to evaluate the minimum life time for t in - * { - * var t; - * if (a) - * t = ... - * else - * t = ... - * x = t; - * ... - * } - */ + * the current IF/ELSE pair is now irrelevant for the analysis. + * This is also required to evaluate the minimum life time for t in + * { + * var t; + * if (a) + * t = ... + * else + * t = ... + * x = t; + * ... + * } + */ first_write_scope = scope.parent(); /* If some parent is IF/ELSE and in a loop then propagate the - * write to that scope. Otherwise the write is unconditional - * because it happens in both corresponding IF/ELSE branches - * in this loop, and hence, record the loop id to signal the - * resolution. - */ + * write to that scope. Otherwise the write is unconditional + * because it happens in both corresponding IF/ELSE branches + * in this loop, and hence, record the loop id to signal the + * resolution. + */ if (parent_ifelse && parent_ifelse->is_in_loop()) { record_ifelse_write(*parent_ifelse); } else { @@ -485,18 +505,20 @@ void RegisterCompAccess::record_else_write(const ProgramScope& scope) } } else { /* The temporary was not written in the IF branch corresponding - * to this ELSE branch, hence the write is conditional. - */ + * to this ELSE branch, hence the write is conditional. + */ conditionality_in_loop_id = write_is_conditional; } } -bool RegisterCompAccess::conditional_ifelse_write_in_loop() const +bool +RegisterCompAccess::conditional_ifelse_write_in_loop() const { return conditionality_in_loop_id <= conditionality_unresolved; } -void RegisterCompAccess::propagate_live_range_to_dominant_write_scope() +void +RegisterCompAccess::propagate_live_range_to_dominant_write_scope() { first_write = first_write_scope->begin(); int lr = first_write_scope->end(); @@ -505,7 +527,8 @@ void RegisterCompAccess::propagate_live_range_to_dominant_write_scope() last_read = lr; } -void RegisterCompAccess::update_required_live_range() +void +RegisterCompAccess::update_required_live_range() { bool keep_for_full_loop = false; @@ -531,7 +554,8 @@ void RegisterCompAccess::update_required_live_range() assert(first_write_scope || m_range.start >= 0); - /* The register was pre-defines, so th first write scope is the outerpost scopw */ + /* The register was pre-defines, so th first write scope is the outerpost + * scopw */ if (!first_write_scope) { first_write_scope = first_read_scope; while (first_write_scope->parent()) @@ -544,8 +568,7 @@ void RegisterCompAccess::update_required_live_range() /* We read before writing in a loop * hence the value must survive the loops */ - if ((first_read <= first_write) && - first_read_scope->is_in_loop()) { + if ((first_read <= first_write) && first_read_scope->is_in_loop()) { keep_for_full_loop = true; enclosing_scope_first_read = first_read_scope->outermost_loop(); } @@ -626,12 +649,10 @@ void RegisterCompAccess::update_required_live_range() m_range.end = last_read; } -const int -RegisterCompAccess::conditionality_untouched = std::numeric_limits::max(); - -const int -RegisterCompAccess::write_is_unconditional = std::numeric_limits::max() - 1; +const int RegisterCompAccess::conditionality_untouched = std::numeric_limits::max(); +const int RegisterCompAccess::write_is_unconditional = + std::numeric_limits::max() - 1; RegisterAccess::RegisterAccess(const std::array& sizes) { @@ -639,11 +660,12 @@ RegisterAccess::RegisterAccess(const std::array& sizes) m_access_record[i].resize(sizes[i]); } -RegisterCompAccess& RegisterAccess::operator() (const Register& reg) +RegisterCompAccess& +RegisterAccess::operator()(const Register& reg) { assert(reg.chan() < 4); assert(m_access_record[reg.chan()].size() > (size_t)reg.index()); return m_access_record[reg.chan()][reg.index()]; } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator_helpers.h b/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator_helpers.h index 53e1e1f3226..e8847c84402 100644 --- a/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator_helpers.h +++ b/src/gallium/drivers/r600/sfn/sfn_liverangeevaluator_helpers.h @@ -45,8 +45,7 @@ enum ProgramScopeType { class ProgramScope { public: ProgramScope(); - ProgramScope(ProgramScope *parent, ProgramScopeType type, int id, - int depth, int begin); + ProgramScope(ProgramScope *parent, ProgramScopeType type, int id, int depth, int begin); ProgramScopeType type() const; ProgramScope *parent() const; @@ -100,9 +99,10 @@ public: void update_required_live_range(); - const auto& range() { return m_range;} + const auto& range() { return m_range; } const auto& use_type() { return m_use_type; } + private: void propagate_live_range_to_dominant_write_scope(); bool conditional_ifelse_write_in_loop() const; @@ -120,20 +120,22 @@ private: int last_write; int first_read; - /* This member variable tracks the current resolution of conditional writing - * to this temporary in IF/ELSE clauses. + /* This member variable tracks the current resolution of conditional + * writing to this temporary in IF/ELSE clauses. * * The initial value "conditionality_untouched" indicates that this * temporary has not yet been written to within an if clause. * * A positive (other than "conditionality_untouched") number refers to the - * last loop id for which the write was resolved as unconditional. With each - * new loop this value will be overwitten by "conditionality_unresolved" - * on entering the first IF clause writing this temporary. + * last loop id for which the write was resolved as unconditional. With + * each new loop this value will be overwitten by + * "conditionality_unresolved" on entering the first IF clause writing this + * temporary. * * The value "conditionality_unresolved" indicates that no resolution has * been achieved so far. If the variable is set to this value at the end of - * the processing of the whole shader it also indicates a conditional write. + * the processing of the whole shader it also indicates a conditional + * write. * * The value "write_is_conditional" marks that the variable is written * conditionally (i.e. not in all relevant IF/ELSE code path pairs) in at @@ -176,7 +178,7 @@ public: RegisterAccess(const std::array& sizes); - RegisterCompAccess& operator() (const Register& reg); + RegisterCompAccess& operator()(const Register& reg); auto& component(int i) { return m_access_record[i]; } @@ -184,5 +186,5 @@ private: std::array m_access_record; }; -} +} // namespace r600 #endif // SFN_LIFERANGEEVALUATOR_HELPERS_H diff --git a/src/gallium/drivers/r600/sfn/sfn_memorypool.cpp b/src/gallium/drivers/r600/sfn/sfn_memorypool.cpp index bdc2bf41d1f..9fd9aaeb0db 100644 --- a/src/gallium/drivers/r600/sfn/sfn_memorypool.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_memorypool.cpp @@ -41,7 +41,7 @@ namespace r600 { #ifndef HAVE_MEMORY_RESOURCE /* Fallback memory resource if the C++17 memory resource is not * avaliable -*/ + */ struct MemoryBacking { ~MemoryBacking(); void *allocate(size_t size); @@ -60,74 +60,79 @@ public: MemoryBacking *pool; }; -MemoryPool::MemoryPool() noexcept : impl(nullptr) +MemoryPool::MemoryPool() noexcept: + impl(nullptr) { } -MemoryPool& MemoryPool::instance() +MemoryPool& +MemoryPool::instance() { - static thread_local MemoryPool me; - return me; + static thread_local MemoryPool me; + return me; } -void MemoryPool::free() +void +MemoryPool::free() { delete impl; impl = nullptr; } -void MemoryPool::initialize() +void +MemoryPool::initialize() { if (!impl) impl = new MemoryPoolImpl(); } -void *MemoryPool::allocate(size_t size) +void * +MemoryPool::allocate(size_t size) { assert(impl); return impl->pool->allocate(size); } -void *MemoryPool::allocate(size_t size, size_t align) +void * +MemoryPool::allocate(size_t size, size_t align) { assert(impl); return impl->pool->allocate(size, align); } -void MemoryPool::release_all() +void +MemoryPool::release_all() { instance().free(); } -void init_pool() +void +init_pool() { - MemoryPool::instance().initialize(); + MemoryPool::instance().initialize(); } -void release_pool() +void +release_pool() { - MemoryPool::release_all(); + MemoryPool::release_all(); } -void *Allocate::operator new(size_t size) +void * +Allocate::operator new(size_t size) { - return MemoryPool::instance().allocate(size); + return MemoryPool::instance().allocate(size); } -void Allocate::operator delete (void *p, size_t size) +void +Allocate::operator delete(void *p, size_t size) { - // MemoryPool::instance().deallocate(p, size); + // MemoryPool::instance().deallocate(p, size); } -MemoryPoolImpl::MemoryPoolImpl() -{ - pool = new MemoryBacking(); -} +MemoryPoolImpl::MemoryPoolImpl() { pool = new MemoryBacking(); } -MemoryPoolImpl::~MemoryPoolImpl() -{ - delete pool; -} +MemoryPoolImpl::~MemoryPoolImpl() { delete pool; } #ifndef HAVE_MEMORY_RESOURCE MemoryBacking::~MemoryBacking() @@ -136,14 +141,16 @@ MemoryBacking::~MemoryBacking() free(p); } -void *MemoryBacking::allocate(size_t size) +void * +MemoryBacking::allocate(size_t size) { void *retval = malloc(size); m_data.push_back(retval); return retval; } -void *MemoryBacking::allocate(size_t size, size_t align) +void * +MemoryBacking::allocate(size_t size, size_t align) { void *retval = aligned_alloc(align, size); m_data.push_back(retval); @@ -152,4 +159,4 @@ void *MemoryBacking::allocate(size_t size, size_t align) #endif -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_memorypool.h b/src/gallium/drivers/r600/sfn/sfn_memorypool.h index 44154102295..48bdb058ca2 100644 --- a/src/gallium/drivers/r600/sfn/sfn_memorypool.h +++ b/src/gallium/drivers/r600/sfn/sfn_memorypool.h @@ -33,58 +33,64 @@ #define R600_POINTER_TYPE(X) X * -namespace r600 { +namespace r600 { -void init_pool(); -void release_pool(); +void +init_pool(); +void +release_pool(); -class Allocate -{ +class Allocate { public: - void * operator new(size_t size); - void operator delete (void *p, size_t size); + void *operator new(size_t size); + void operator delete(void *p, size_t size); }; -class MemoryPool { +class MemoryPool { public: - static MemoryPool& instance(); - static void release_all(); + static MemoryPool& instance(); + static void release_all(); - void free(); - void initialize(); + void free(); + void initialize(); - void *allocate(size_t size); - void *allocate(size_t size, size_t align); + void *allocate(size_t size); + void *allocate(size_t size, size_t align); private: - MemoryPool() noexcept; + MemoryPool() noexcept; - struct MemoryPoolImpl* impl; + struct MemoryPoolImpl *impl; }; -template -struct Allocator { +template struct Allocator { using value_type = T; Allocator() = default; Allocator(const Allocator& other) = default; - template - Allocator(const Allocator& other) {(void)other;} + template Allocator(const Allocator& other) { (void)other; } - T *allocate(size_t n) { + T *allocate(size_t n) + { return (T *)MemoryPool::instance().allocate(n * sizeof(T), alignof(T)); } - void deallocate(void *p, size_t n) { - (void)p; (void)n; - //MemoryPool::instance().deallocate(p, n * sizeof(T), alignof(T)); + void deallocate(void *p, size_t n) + { + (void)p; + (void)n; + // MemoryPool::instance().deallocate(p, n * sizeof(T), alignof(T)); } - friend bool operator == (const Allocator& lhs, const Allocator& rhs) { - (void)lhs; (void)rhs; return true;} + friend bool operator==(const Allocator& lhs, const Allocator& rhs) + { + (void)lhs; + (void)rhs; + return true; + } }; -} +} // namespace r600 #endif // MEMORYPOOL_H diff --git a/src/gallium/drivers/r600/sfn/sfn_nir.cpp b/src/gallium/drivers/r600/sfn/sfn_nir.cpp index f44f1ca145f..fa7d51f67f7 100644 --- a/src/gallium/drivers/r600/sfn/sfn_nir.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_nir.cpp @@ -25,25 +25,22 @@ */ #include "sfn_nir.h" -#include "nir_builder.h" #include "../r600_pipe.h" #include "../r600_shader.h" - - -#include "util/u_prim.h" - -#include "sfn_shader.h" +#include "nir_builder.h" #include "sfn_assembler.h" #include "sfn_debug.h" -#include "sfn_liverangeevaluator.h" -#include "sfn_nir_lower_fs_out_to_vector.h" -#include "sfn_nir_lower_alu.h" -#include "sfn_nir_lower_tex.h" #include "sfn_instr_tex.h" +#include "sfn_liverangeevaluator.h" +#include "sfn_nir_lower_alu.h" +#include "sfn_nir_lower_fs_out_to_vector.h" +#include "sfn_nir_lower_tex.h" #include "sfn_optimizer.h" #include "sfn_ra.h" #include "sfn_scheduler.h" +#include "sfn_shader.h" +#include "util/u_prim.h" #include @@ -51,38 +48,36 @@ namespace r600 { using std::vector; - NirLowerInstruction::NirLowerInstruction(): - b(nullptr) + b(nullptr) { } -bool NirLowerInstruction::filter_instr(const nir_instr *instr, const void *data) +bool +NirLowerInstruction::filter_instr(const nir_instr *instr, const void *data) { - auto me = reinterpret_cast(data); + auto me = reinterpret_cast(data); return me->filter(instr); } -nir_ssa_def *NirLowerInstruction::lower_instr(nir_builder *b, nir_instr *instr, void *data) +nir_ssa_def * +NirLowerInstruction::lower_instr(nir_builder *b, nir_instr *instr, void *data) { - auto me = reinterpret_cast(data); + auto me = reinterpret_cast(data); me->set_builder(b); return me->lower(instr); } -bool NirLowerInstruction::run(nir_shader *shader) +bool +NirLowerInstruction::run(nir_shader *shader) { - return nir_shader_lower_instructions(shader, - filter_instr, - lower_instr, - (void *)this); + return nir_shader_lower_instructions(shader, filter_instr, lower_instr, (void *)this); } -AssemblyFromShader::~AssemblyFromShader() -{ -} +AssemblyFromShader::~AssemblyFromShader() {} -bool AssemblyFromShader::lower(const Shader& ir) +bool +AssemblyFromShader::lower(const Shader& ir) { return do_lower(ir); } @@ -96,28 +91,33 @@ r600_nir_lower_scratch_address_impl(nir_builder *b, nir_intrinsic_instr *instr) int align; if (instr->intrinsic == nir_intrinsic_store_scratch) { - align = instr->src[0].ssa->num_components; + align = instr->src[0].ssa->num_components; address_index = 1; - } else{ + } else { align = instr->dest.ssa.num_components; } nir_ssa_def *address = instr->src[address_index].ssa; - nir_ssa_def *new_address = nir_ishr(b, address, nir_imm_int(b, 4 * align)); + nir_ssa_def *new_address = nir_ishr(b, address, nir_imm_int(b, 4 * align)); - nir_instr_rewrite_src(&instr->instr, &instr->src[address_index], + nir_instr_rewrite_src(&instr->instr, + &instr->src[address_index], nir_src_for_ssa(new_address)); } -bool r600_lower_scratch_addresses(nir_shader *shader) +bool +r600_lower_scratch_addresses(nir_shader *shader) { bool progress = false; - nir_foreach_function(function, shader) { + nir_foreach_function(function, shader) + { nir_builder build; nir_builder_init(&build, function->impl); - nir_foreach_block(block, function->impl) { - nir_foreach_instr(instr, block) { + nir_foreach_block(block, function->impl) + { + nir_foreach_instr(instr, block) + { if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *op = nir_instr_as_intrinsic(instr); @@ -135,7 +135,8 @@ bool r600_lower_scratch_addresses(nir_shader *shader) static void insert_uniform_sorted(struct exec_list *var_list, nir_variable *new_var) { - nir_foreach_variable_in_list(var, var_list) { + nir_foreach_variable_in_list(var, var_list) + { if (var->data.binding > new_var->data.binding || (var->data.binding == new_var->data.binding && var->data.offset > new_var->data.offset)) { @@ -146,12 +147,14 @@ insert_uniform_sorted(struct exec_list *var_list, nir_variable *new_var) exec_list_push_tail(var_list, &new_var->node); } -void sort_uniforms(nir_shader *shader) +void +sort_uniforms(nir_shader *shader) { struct exec_list new_list; exec_list_make_empty(&new_list); - nir_foreach_uniform_variable_safe(var, shader) { + nir_foreach_uniform_variable_safe(var, shader) + { exec_node_remove(&var->node); insert_uniform_sorted(&new_list, var); } @@ -162,7 +165,8 @@ static void insert_fsoutput_sorted(struct exec_list *var_list, nir_variable *new_var) { - nir_foreach_variable_in_list(var, var_list) { + nir_foreach_variable_in_list(var, var_list) + { if (var->data.location > new_var->data.location || (var->data.location == new_var->data.location && var->data.index > new_var->data.index)) { @@ -174,19 +178,21 @@ insert_fsoutput_sorted(struct exec_list *var_list, nir_variable *new_var) exec_list_push_tail(var_list, &new_var->node); } -void sort_fsoutput(nir_shader *shader) +void +sort_fsoutput(nir_shader *shader) { struct exec_list new_list; exec_list_make_empty(&new_list); - nir_foreach_shader_out_variable_safe(var, shader) { + nir_foreach_shader_out_variable_safe(var, shader) + { exec_node_remove(&var->node); insert_fsoutput_sorted(&new_list, var); } unsigned driver_location = 0; - nir_foreach_variable_in_list(var, &new_list) - var->data.driver_location = driver_location++; + nir_foreach_variable_in_list(var, &new_list) var->data.driver_location = + driver_location++; exec_list_append(&shader->variables, &new_list); } @@ -194,12 +200,16 @@ void sort_fsoutput(nir_shader *shader) class LowerClipvertexWrite : public NirLowerInstruction { public: - LowerClipvertexWrite(int noutputs, pipe_stream_output_info& so_info) : - m_clipplane1(noutputs), - m_clipvtx(noutputs + 1), - m_so_info(so_info){} + LowerClipvertexWrite(int noutputs, pipe_stream_output_info& so_info): + m_clipplane1(noutputs), + m_clipvtx(noutputs + 1), + m_so_info(so_info) + { + } + private: - bool filter(const nir_instr *instr) const override { + bool filter(const nir_instr *instr) const override + { if (instr->type != nir_instr_type_intrinsic) return false; @@ -210,7 +220,8 @@ private: return nir_intrinsic_io_semantics(intr).location == VARYING_SLOT_CLIP_VERTEX; } - nir_ssa_def *lower(nir_instr *instr) override { + nir_ssa_def *lower(nir_instr *instr) override + { auto intr = nir_instr_as_intrinsic(instr); nir_ssa_def *output[8] = {nullptr}; @@ -231,7 +242,7 @@ private: for (int i = 0; i < 2; ++i) { auto clip_i = nir_vec(b, &output[4 * i], 4); - auto store = nir_store_output(b, clip_i, intr->src[1].ssa); + auto store = nir_store_output(b, clip_i, intr->src[1].ssa); nir_intrinsic_set_write_mask(store, 0xf); nir_intrinsic_set_base(store, clip_vertex_index); nir_io_semantics semantic = nir_intrinsic_io_semantics(intr); @@ -246,7 +257,7 @@ private: nir_intrinsic_set_base(intr, m_clipvtx); nir_ssa_def *result = NIR_LOWER_INSTR_PROGRESS_REPLACE; - for (unsigned i = 0; i < m_so_info.num_outputs; ++i) { + for (unsigned i = 0; i < m_so_info.num_outputs; ++i) { if (m_so_info.output[i].register_index == clip_vertex_index) { m_so_info.output[i].register_index = m_clipvtx; result = NIR_LOWER_INSTR_PROGRESS; @@ -259,8 +270,7 @@ private: pipe_stream_output_info& m_so_info; }; - -} +} // namespace r600 static nir_intrinsic_op r600_map_atomic(nir_intrinsic_op op) @@ -310,8 +320,7 @@ r600_lower_deref_instr(nir_builder *b, nir_instr *instr_, UNUSED void *cb_data) nir_deref_instr *deref = nir_src_as_deref(instr->src[0]); nir_variable *var = nir_deref_instr_get_variable(deref); - if (var->data.mode != nir_var_uniform && - var->data.mode != nir_var_mem_ssbo && + if (var->data.mode != nir_var_uniform && var->data.mode != nir_var_mem_ssbo && var->data.mode != nir_var_mem_shared) return false; /* atomics passed as function arguments can't be lowered */ @@ -329,8 +338,8 @@ r600_lower_deref_instr(nir_builder *b, nir_instr *instr_, UNUSED void *cb_data) if (glsl_type_is_array(d->type)) array_stride *= glsl_get_aoa_size(d->type); - offset = nir_iadd(b, offset, nir_imul(b, d->arr.index.ssa, - nir_imm_int(b, array_stride))); + offset = + nir_iadd(b, offset, nir_imul(b, d->arr.index.ssa, nir_imm_int(b, array_stride))); } /* Since the first source is a deref and the first source in the lowered @@ -338,8 +347,7 @@ r600_lower_deref_instr(nir_builder *b, nir_instr *instr_, UNUSED void *cb_data) * opcode. */ instr->intrinsic = op; - nir_instr_rewrite_src(&instr->instr, &instr->src[0], - nir_src_for_ssa(offset)); + nir_instr_rewrite_src(&instr->instr, &instr->src[0], nir_src_for_ssa(offset)); nir_intrinsic_set_base(instr, idx); nir_deref_instr_remove_if_unused(deref); @@ -347,10 +355,8 @@ r600_lower_deref_instr(nir_builder *b, nir_instr *instr_, UNUSED void *cb_data) return true; } - static bool -r600_lower_clipvertex_to_clipdist(nir_shader *sh, - pipe_stream_output_info& so_info) +r600_lower_clipvertex_to_clipdist(nir_shader *sh, pipe_stream_output_info& so_info) { if (!(sh->info.outputs_written & VARYING_BIT_CLIP_VERTEX)) return false; @@ -367,7 +373,8 @@ r600_nir_lower_atomics(nir_shader *shader) * binding, and we use an offset of one per counter */ int current_binding = -1; int current_offset = 0; - nir_foreach_variable_with_modes(var, shader, nir_var_uniform) { + nir_foreach_variable_with_modes(var, shader, nir_var_uniform) + { if (!var->type->contains_atomic()) continue; @@ -381,13 +388,13 @@ r600_nir_lower_atomics(nir_shader *shader) } } - return nir_shader_instructions_pass(shader, r600_lower_deref_instr, - nir_metadata_block_index | - nir_metadata_dominance, + return nir_shader_instructions_pass(shader, + r600_lower_deref_instr, + nir_metadata_block_index | nir_metadata_dominance, NULL); } -using r600::r600_lower_scratch_addresses; using r600::r600_lower_fs_out_to_vector; +using r600::r600_lower_scratch_addresses; using r600::r600_lower_ubo_to_align16; int @@ -398,15 +405,15 @@ r600_glsl_type_size(const struct glsl_type *type, bool is_bindless) void r600_get_natural_size_align_bytes(const struct glsl_type *type, - unsigned *size, unsigned *align) + unsigned *size, + unsigned *align) { if (type->base_type != GLSL_TYPE_ARRAY) { *align = 1; *size = 1; } else { unsigned elem_size, elem_align; - glsl_get_natural_size_align_bytes(type->fields.array, - &elem_size, &elem_align); + glsl_get_natural_size_align_bytes(type->fields.array, &elem_size, &elem_align); *align = 1; *size = type->length; } @@ -419,8 +426,10 @@ r600_lower_shared_io_impl(nir_function *func) nir_builder_init(&b, func->impl); bool progress = false; - nir_foreach_block(block, func->impl) { - nir_foreach_instr_safe(instr, block) { + nir_foreach_block(block, func->impl) + { + nir_foreach_instr_safe(instr, block) + { if (instr->type != nir_instr_type_intrinsic) continue; @@ -443,9 +452,8 @@ r600_lower_shared_io_impl(nir_function *func) } case 3: { auto addr2 = nir_iadd(&b, addr, nir_imm_ivec2(&b, 4, 8)); - addr = nir_vec3(&b, addr, - nir_channel(&b, addr2, 0), - nir_channel(&b, addr2, 1)); + addr = + nir_vec3(&b, addr, nir_channel(&b, addr2, 0), nir_channel(&b, addr2, 1)); break; } case 4: { @@ -454,11 +462,11 @@ r600_lower_shared_io_impl(nir_function *func) } } - auto load = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_local_shared_r600); + auto load = + nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_local_shared_r600); load->num_components = nir_dest_num_components(op->dest); load->src[0] = nir_src_for_ssa(addr); - nir_ssa_dest_init(&load->instr, &load->dest, - load->num_components, 32, NULL); + nir_ssa_dest_init(&load->instr, &load->dest, load->num_components, 32, NULL); nir_ssa_def_rewrite_uses(&op->dest.ssa, &load->dest.ssa); nir_builder_instr_insert(&b, &load->instr); } else { @@ -468,14 +476,17 @@ r600_lower_shared_io_impl(nir_function *func) if (!(nir_intrinsic_write_mask(op) & test_mask)) continue; - auto store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_local_shared_r600); + auto store = + nir_intrinsic_instr_create(b.shader, + nir_intrinsic_store_local_shared_r600); unsigned writemask = nir_intrinsic_write_mask(op) & test_mask; nir_intrinsic_set_write_mask(store, writemask); store->src[0] = nir_src_for_ssa(op->src[0].ssa); store->num_components = store->src[0].ssa->num_components; bool start_even = (writemask & (1u << (2 * i))); - auto addr2 = nir_iadd(&b, addr, nir_imm_int(&b, 8 * i + (start_even ? 0 : 4))); + auto addr2 = + nir_iadd(&b, addr, nir_imm_int(&b, 8 * i + (start_even ? 0 : 4))); store->src[1] = nir_src_for_ssa(addr2); nir_builder_instr_insert(&b, &store->instr); @@ -491,24 +502,26 @@ r600_lower_shared_io_impl(nir_function *func) static bool r600_lower_shared_io(nir_shader *nir) { - bool progress=false; - nir_foreach_function(function, nir) { - if (function->impl && - r600_lower_shared_io_impl(function)) - progress = true; - } - return progress; + bool progress = false; + nir_foreach_function(function, nir) + { + if (function->impl && r600_lower_shared_io_impl(function)) + progress = true; + } + return progress; } - static nir_ssa_def * r600_lower_fs_pos_input_impl(nir_builder *b, nir_instr *instr, void *_options) { (void)_options; auto old_ir = nir_instr_as_intrinsic(instr); auto load = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_input); - nir_ssa_dest_init(&load->instr, &load->dest, - old_ir->dest.ssa.num_components, old_ir->dest.ssa.bit_size, NULL); + nir_ssa_dest_init(&load->instr, + &load->dest, + old_ir->dest.ssa.num_components, + old_ir->dest.ssa.bit_size, + NULL); nir_intrinsic_set_io_semantics(load, nir_intrinsic_io_semantics(old_ir)); nir_intrinsic_set_base(load, nir_intrinsic_base(old_ir)); @@ -520,7 +533,8 @@ r600_lower_fs_pos_input_impl(nir_builder *b, nir_instr *instr, void *_options) return &load->dest.ssa; } -bool r600_lower_fs_pos_input_filter(const nir_instr *instr, const void *_options) +bool +r600_lower_fs_pos_input_filter(const nir_instr *instr, const void *_options) { (void)_options; @@ -535,7 +549,8 @@ bool r600_lower_fs_pos_input_filter(const nir_instr *instr, const void *_options } /* Strip the interpolator specification, it is not needed and irritates */ -bool r600_lower_fs_pos_input(nir_shader *shader) +bool +r600_lower_fs_pos_input(nir_shader *shader) { return nir_shader_lower_instructions(shader, r600_lower_fs_pos_input_filter, @@ -556,9 +571,9 @@ optimize_once(nir_shader *shader) NIR_PASS(progress, shader, nir_opt_remove_phis); if (nir_opt_trivial_continues(shader)) { - progress = true; - NIR_PASS(progress, shader, nir_copy_prop); - NIR_PASS(progress, shader, nir_opt_dce); + progress = true; + NIR_PASS(progress, shader, nir_copy_prop); + NIR_PASS(progress, shader, nir_opt_dce); } NIR_PASS(progress, shader, nir_opt_if, nir_opt_if_optimize_phi_true_false); @@ -573,10 +588,13 @@ optimize_once(nir_shader *shader) return progress; } -bool has_saturate(const nir_function *func) +bool +has_saturate(const nir_function *func) { - nir_foreach_block(block, func->impl) { - nir_foreach_instr(instr, block) { + nir_foreach_block(block, func->impl) + { + nir_foreach_instr(instr, block) + { if (instr->type == nir_instr_type_alu) { auto alu = nir_instr_as_alu(instr); if (alu->dest.saturate) @@ -587,24 +605,23 @@ bool has_saturate(const nir_function *func) return false; } -static bool r600_is_last_vertex_stage(nir_shader *nir, const r600_shader_key& key) +static bool +r600_is_last_vertex_stage(nir_shader *nir, const r600_shader_key& key) { if (nir->info.stage == MESA_SHADER_GEOMETRY) return true; - if (nir->info.stage == MESA_SHADER_TESS_EVAL && - !key.tes.as_es) + if (nir->info.stage == MESA_SHADER_TESS_EVAL && !key.tes.as_es) return true; - if (nir->info.stage == MESA_SHADER_VERTEX && - !key.vs.as_es && !key.vs.as_ls) + if (nir->info.stage == MESA_SHADER_VERTEX && !key.vs.as_es && !key.vs.as_ls) return true; return false; } -extern "C" -bool r600_lower_to_scalar_instr_filter(const nir_instr *instr, const void *) +extern "C" bool +r600_lower_to_scalar_instr_filter(const nir_instr *instr, const void *) { if (instr->type != nir_instr_type_alu) return true; @@ -636,29 +653,26 @@ bool r600_lower_to_scalar_instr_filter(const nir_instr *instr, const void *) } } - class MallocPoolRelease { public: - MallocPoolRelease() { - r600::init_pool(); - } - ~MallocPoolRelease() { - r600::release_pool(); - } + MallocPoolRelease() { r600::init_pool(); } + ~MallocPoolRelease() { r600::release_pool(); } }; -int r600_shader_from_nir(struct r600_context *rctx, - struct r600_pipe_shader *pipeshader, - r600_shader_key *key) +int +r600_shader_from_nir(struct r600_context *rctx, + struct r600_pipe_shader *pipeshader, + r600_shader_key *key) { MallocPoolRelease pool_release; struct r600_pipe_shader_selector *sel = pipeshader->selector; - bool lower_64bit = (rctx->b.gfx_level < CAYMAN && - (sel->nir->options->lower_int64_options || - sel->nir->options->lower_doubles_options) && - (sel->nir->info.bit_sizes_float | sel->nir->info.bit_sizes_int) & 64); + bool lower_64bit = + (rctx->b.gfx_level < CAYMAN && + (sel->nir->options->lower_int64_options || + sel->nir->options->lower_doubles_options) && + (sel->nir->info.bit_sizes_float | sel->nir->info.bit_sizes_int) & 64); if (rctx->screen->b.debug_flags & DBG_PREOPT_IR) { fprintf(stderr, "PRE-OPT-NIR-----------.------------------------------\n"); @@ -683,7 +697,8 @@ int r600_shader_from_nir(struct r600_context *rctx, if (lower_64bit) NIR_PASS_V(sel->nir, nir_lower_int64); - while(optimize_once(sel->nir)); + while (optimize_once(sel->nir)) + ; NIR_PASS_V(sel->nir, r600_lower_shared_io); NIR_PASS_V(sel->nir, r600_nir_lower_atomics); @@ -707,13 +722,14 @@ int r600_shader_from_nir(struct r600_context *rctx, NIR_PASS_V(sel->nir, nir_lower_fragcoord_wtrans); NIR_PASS_V(sel->nir, r600_lower_fs_out_to_vector); } - nir_variable_mode io_modes = nir_var_uniform | - nir_var_shader_in | - nir_var_shader_out; + nir_variable_mode io_modes = nir_var_uniform | nir_var_shader_in | nir_var_shader_out; NIR_PASS_V(sel->nir, nir_opt_combine_stores, nir_var_shader_out); - NIR_PASS_V(sel->nir, nir_lower_io, io_modes, r600_glsl_type_size, - nir_lower_io_lower_64bit_to_32); + NIR_PASS_V(sel->nir, + nir_lower_io, + io_modes, + r600_glsl_type_size, + nir_lower_io_lower_64bit_to_32); if (sel->nir->info.stage == MESA_SHADER_FRAGMENT) NIR_PASS_V(sel->nir, r600_lower_fs_pos_input); @@ -743,17 +759,19 @@ int r600_shader_from_nir(struct r600_context *rctx, if (sh->info.stage == MESA_SHADER_TESS_CTRL || sh->info.stage == MESA_SHADER_TESS_EVAL || (sh->info.stage == MESA_SHADER_VERTEX && key->vs.as_ls)) { - auto prim_type = sh->info.stage == MESA_SHADER_TESS_EVAL ? - u_tess_prim_from_shader(sh->info.tess._primitive_mode) : key->tcs.prim_mode; + auto prim_type = sh->info.stage == MESA_SHADER_TESS_EVAL + ? u_tess_prim_from_shader(sh->info.tess._primitive_mode) + : key->tcs.prim_mode; NIR_PASS_V(sh, r600_lower_tess_io, static_cast(prim_type)); } if (sh->info.stage == MESA_SHADER_TESS_CTRL) - NIR_PASS_V(sh, r600_append_tcs_TF_emission, - (pipe_prim_type)key->tcs.prim_mode); + NIR_PASS_V(sh, r600_append_tcs_TF_emission, (pipe_prim_type)key->tcs.prim_mode); if (sh->info.stage == MESA_SHADER_TESS_EVAL) { - NIR_PASS_V(sh, r600_lower_tess_coord, u_tess_prim_from_shader(sh->info.tess._primitive_mode)); + NIR_PASS_V(sh, + r600_lower_tess_coord, + u_tess_prim_from_shader(sh->info.tess._primitive_mode)); } NIR_PASS_V(sh, nir_lower_alu_to_scalar, r600_lower_to_scalar_instr_filter, NULL); @@ -762,7 +780,6 @@ int r600_shader_from_nir(struct r600_context *rctx, NIR_PASS_V(sh, r600_nir_lower_int_tg4); NIR_PASS_V(sh, r600::r600_nir_lower_tex_to_backend, rctx->b.gfx_level); - if ((sh->info.bit_sizes_float | sh->info.bit_sizes_int) & 64) { NIR_PASS_V(sh, r600::r600_nir_split_64bit_io); NIR_PASS_V(sh, r600::r600_split_64bit_alu_and_phi); @@ -779,25 +796,26 @@ int r600_shader_from_nir(struct r600_context *rctx, NIR_PASS_V(sh, r600::r600_split_64bit_uniforms_and_ubo); /* Lower to scalar to let some optimization work out better */ - while(optimize_once(sh)); + while (optimize_once(sh)) + ; if (lower_64bit) NIR_PASS_V(sh, r600::r600_merge_vec2_stores); NIR_PASS_V(sh, nir_remove_dead_variables, nir_var_shader_in, NULL); - NIR_PASS_V(sh, nir_remove_dead_variables, nir_var_shader_out, NULL); + NIR_PASS_V(sh, nir_remove_dead_variables, nir_var_shader_out, NULL); - - NIR_PASS_V(sh, nir_lower_vars_to_scratch, + NIR_PASS_V(sh, + nir_lower_vars_to_scratch, nir_var_function_temp, 40, r600_get_natural_size_align_bytes); - while (optimize_once(sh)); + while (optimize_once(sh)) + ; bool late_algebraic_progress; - do - { + do { late_algebraic_progress = false; NIR_PASS(late_algebraic_progress, sh, nir_opt_algebraic_late); NIR_PASS(late_algebraic_progress, sh, nir_opt_constant_folding); @@ -806,54 +824,61 @@ int r600_shader_from_nir(struct r600_context *rctx, NIR_PASS(late_algebraic_progress, sh, nir_opt_cse); } while (late_algebraic_progress); - NIR_PASS_V(sh, nir_lower_bool_to_int32); + NIR_PASS_V(sh, nir_lower_bool_to_int32); if (sh->info.stage == MESA_SHADER_FRAGMENT) r600::sort_fsoutput(sh); NIR_PASS_V(sh, nir_lower_locals_to_regs); - NIR_PASS_V(sh, nir_lower_to_source_mods, - (nir_lower_to_source_mods_flags)(nir_lower_float_source_mods | - nir_lower_64bit_source_mods)); + NIR_PASS_V(sh, + nir_lower_to_source_mods, + (nir_lower_to_source_mods_flags)(nir_lower_float_source_mods | + nir_lower_64bit_source_mods)); NIR_PASS_V(sh, nir_convert_from_ssa, true); NIR_PASS_V(sh, nir_opt_dce); if (rctx->screen->b.debug_flags & DBG_ALL_SHADERS) { - fprintf(stderr, "-- NIR --------------------------------------------------------\n"); - struct nir_function *func = (struct nir_function *)exec_list_get_head(&sh->functions); + fprintf(stderr, + "-- NIR --------------------------------------------------------\n"); + struct nir_function *func = + (struct nir_function *)exec_list_get_head(&sh->functions); nir_index_ssa_defs(func->impl); nir_print_shader(sh, stderr); - fprintf(stderr, "-- END --------------------------------------------------------\n"); + fprintf(stderr, + "-- END --------------------------------------------------------\n"); } memset(&pipeshader->shader, 0, sizeof(r600_shader)); pipeshader->scratch_space_needed = sh->scratch_size; - if (sh->info.stage == MESA_SHADER_TESS_EVAL || - sh->info.stage == MESA_SHADER_VERTEX || + if (sh->info.stage == MESA_SHADER_TESS_EVAL || sh->info.stage == MESA_SHADER_VERTEX || sh->info.stage == MESA_SHADER_GEOMETRY) { - pipeshader->shader.clip_dist_write |= ((1 << sh->info.clip_distance_array_size) - 1); + pipeshader->shader.clip_dist_write |= + ((1 << sh->info.clip_distance_array_size) - 1); pipeshader->shader.cull_dist_write = ((1 << sh->info.cull_distance_array_size) - 1) << sh->info.clip_distance_array_size; - pipeshader->shader.cc_dist_mask = (1 << (sh->info.cull_distance_array_size + - sh->info.clip_distance_array_size)) - 1; + pipeshader->shader.cc_dist_mask = + (1 << (sh->info.cull_distance_array_size + sh->info.clip_distance_array_size)) - + 1; } - struct r600_shader* gs_shader = nullptr; + struct r600_shader *gs_shader = nullptr; if (rctx->gs_shader) gs_shader = &rctx->gs_shader->current->shader; r600_screen *rscreen = rctx->screen; - r600::Shader *shader = r600::Shader::translate_from_nir(sh, &sel->so, gs_shader, - *key, rctx->isa->hw_class); + r600::Shader *shader = + r600::Shader::translate_from_nir(sh, &sel->so, gs_shader, *key, rctx->isa->hw_class); assert(shader); if (!shader) return -2; pipeshader->enabled_stream_buffers_mask = shader->enabled_stream_buffers_mask(); - pipeshader->selector->info.file_count[TGSI_FILE_HW_ATOMIC] += shader->atomic_file_count(); - pipeshader->selector->info.writes_memory = shader->has_flag(r600::Shader::sh_writes_memory); + pipeshader->selector->info.file_count[TGSI_FILE_HW_ATOMIC] += + shader->atomic_file_count(); + pipeshader->selector->info.writes_memory = + shader->has_flag(r600::Shader::sh_writes_memory); if (r600::sfn_log.has_debug_flag(r600::SfnLog::steps)) { std::cerr << "Shader after conversion from nir\n"; @@ -900,11 +925,12 @@ int r600_shader_from_nir(struct r600_context *rctx, scheduled_shader->get_shader_info(&pipeshader->shader); pipeshader->shader.uses_doubles = sh->info.bit_sizes_float & 64 ? 1 : 0; - r600_bytecode_init(&pipeshader->shader.bc, rscreen->b.gfx_level, rscreen->b.family, + r600_bytecode_init(&pipeshader->shader.bc, + rscreen->b.gfx_level, + rscreen->b.family, rscreen->has_compressed_msaa_texturing); - r600::sfn_log << r600::SfnLog::shader_info - << "pipeshader->shader.processor_type = " + r600::sfn_log << r600::SfnLog::shader_info << "pipeshader->shader.processor_type = " << pipeshader->shader.processor_type << "\n"; pipeshader->shader.bc.type = pipeshader->shader.processor_type; @@ -921,7 +947,8 @@ int r600_shader_from_nir(struct r600_context *rctx, } if (sh->info.stage == MESA_SHADER_GEOMETRY) { - r600::sfn_log << r600::SfnLog::shader_info << "Geometry shader, create copy shader\n"; + r600::sfn_log << r600::SfnLog::shader_info + << "Geometry shader, create copy shader\n"; generate_gs_copy_shader(rctx, pipeshader, &sel->so); assert(pipeshader->gs_copy_shader); } else { diff --git a/src/gallium/drivers/r600/sfn/sfn_nir.h b/src/gallium/drivers/r600/sfn/sfn_nir.h index 73ce5c4b109..cd2ae1ff5a4 100644 --- a/src/gallium/drivers/r600/sfn/sfn_nir.h +++ b/src/gallium/drivers/r600/sfn/sfn_nir.h @@ -32,53 +32,64 @@ #ifdef __cplusplus #include "sfn_shader.h" + #include namespace r600 { class NirLowerInstruction { public: - NirLowerInstruction(); + NirLowerInstruction(); - bool run(nir_shader *shader); + bool run(nir_shader *shader); private: - static bool filter_instr(const nir_instr *instr, const void *data); - static nir_ssa_def *lower_instr(nir_builder *b, nir_instr *instr, void *data); + static bool filter_instr(const nir_instr *instr, const void *data); + static nir_ssa_def *lower_instr(nir_builder *b, nir_instr *instr, void *data); - void set_builder(nir_builder *_b) { b = _b;} + void set_builder(nir_builder *_b) { b = _b; } + + virtual bool filter(const nir_instr *instr) const = 0; + virtual nir_ssa_def *lower(nir_instr *instr) = 0; - virtual bool filter(const nir_instr *instr) const = 0; - virtual nir_ssa_def *lower(nir_instr *instr) = 0; protected: - nir_builder *b; + nir_builder *b; }; -bool r600_lower_scratch_addresses(nir_shader *shader); +bool +r600_lower_scratch_addresses(nir_shader *shader); -bool r600_lower_ubo_to_align16(nir_shader *shader); +bool +r600_lower_ubo_to_align16(nir_shader *shader); -bool r600_nir_split_64bit_io(nir_shader *sh); +bool +r600_nir_split_64bit_io(nir_shader *sh); -bool r600_nir_64_to_vec2(nir_shader *sh); +bool +r600_nir_64_to_vec2(nir_shader *sh); -bool r600_merge_vec2_stores(nir_shader *shader); - -bool r600_split_64bit_uniforms_and_ubo(nir_shader *sh); -bool r600_lower_64bit_to_vec2(nir_shader *sh); -bool r600_split_64bit_alu_and_phi(nir_shader *sh); -bool r600_lower_clipvertex_to_clipdist(nir_shader *sh); +bool +r600_merge_vec2_stores(nir_shader *shader); +bool +r600_split_64bit_uniforms_and_ubo(nir_shader *sh); +bool +r600_lower_64bit_to_vec2(nir_shader *sh); +bool +r600_split_64bit_alu_and_phi(nir_shader *sh); +bool +r600_lower_clipvertex_to_clipdist(nir_shader *sh); class AssemblyFromShader { public: virtual ~AssemblyFromShader(); bool lower(const Shader& s); + private: - virtual bool do_lower(const Shader& s) = 0 ; + virtual bool do_lower(const Shader& s) = 0; }; -} +} // namespace r600 static inline nir_ssa_def * r600_imm_ivec3(nir_builder *build, int x, int y, int z) @@ -92,14 +103,16 @@ r600_imm_ivec3(nir_builder *build, int x, int y, int z) return nir_build_imm(build, 3, 32, v); } -bool r600_lower_tess_io(nir_shader *shader, enum pipe_prim_type prim_type); -bool r600_append_tcs_TF_emission(nir_shader *shader, enum pipe_prim_type prim_type); -bool r600_lower_tess_coord(nir_shader *sh, enum pipe_prim_type prim_type); +bool +r600_lower_tess_io(nir_shader *shader, enum pipe_prim_type prim_type); +bool +r600_append_tcs_TF_emission(nir_shader *shader, enum pipe_prim_type prim_type); +bool +r600_lower_tess_coord(nir_shader *sh, enum pipe_prim_type prim_type); bool r600_legalize_image_load_store(nir_shader *shader); - #else #include "gallium/drivers/r600/r600_shader.h" #endif @@ -108,17 +121,19 @@ r600_legalize_image_load_store(nir_shader *shader); extern "C" { #endif -bool r600_vectorize_vs_inputs(nir_shader *shader); +bool +r600_vectorize_vs_inputs(nir_shader *shader); -bool r600_lower_to_scalar_instr_filter(const nir_instr *instr, const void *); +bool +r600_lower_to_scalar_instr_filter(const nir_instr *instr, const void *); -int r600_shader_from_nir(struct r600_context *rctx, - struct r600_pipe_shader *pipeshader, - union r600_shader_key *key); +int +r600_shader_from_nir(struct r600_context *rctx, + struct r600_pipe_shader *pipeshader, + union r600_shader_key *key); #ifdef __cplusplus } #endif - #endif // SFN_NIR_H diff --git a/src/gallium/drivers/r600/sfn/sfn_nir_legalize_image_load_store.cpp b/src/gallium/drivers/r600/sfn/sfn_nir_legalize_image_load_store.cpp index 4211d1f0ad9..60c5a11470d 100644 --- a/src/gallium/drivers/r600/sfn/sfn_nir_legalize_image_load_store.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_nir_legalize_image_load_store.cpp @@ -24,15 +24,13 @@ * USE OR OTHER DEALINGS IN THE SOFTWARE. */ - -#include "sfn_nir.h" - #include "nir.h" #include "nir_builder.h" - +#include "sfn_nir.h" static nir_ssa_def * -r600_legalize_image_load_store_impl(nir_builder *b, nir_instr *instr, +r600_legalize_image_load_store_impl(nir_builder *b, + nir_instr *instr, UNUSED void *_options) { b->cursor = nir_before_instr(instr); @@ -45,10 +43,11 @@ r600_legalize_image_load_store_impl(nir_builder *b, nir_instr *instr, bool load_value = ir->intrinsic != nir_intrinsic_image_store; if (load_value) - default_value = nir_imm_zero(b, nir_dest_num_components(ir->dest), - nir_dest_bit_size(ir->dest)); + default_value = + nir_imm_zero(b, nir_dest_num_components(ir->dest), nir_dest_bit_size(ir->dest)); - auto image_exists = nir_ult(b, ir->src[0].ssa, nir_imm_int(b, b->shader->info.num_images)); + auto image_exists = + nir_ult(b, ir->src[0].ssa, nir_imm_int(b, b->shader->info.num_images)); nir_if *if_exists = nir_push_if(b, image_exists); @@ -57,8 +56,8 @@ r600_legalize_image_load_store_impl(nir_builder *b, nir_instr *instr, if (ir->intrinsic != nir_intrinsic_image_size) { /* Image exists start */ - auto new_index = nir_umin(b, ir->src[0].ssa, - nir_imm_int(b, b->shader->info.num_images - 1)); + auto new_index = + nir_umin(b, ir->src[0].ssa, nir_imm_int(b, b->shader->info.num_images - 1)); nir_instr_rewrite_src_ssa(instr, &ir->src[0], new_index); enum glsl_sampler_dim dim = nir_intrinsic_image_dim(ir); @@ -67,13 +66,16 @@ r600_legalize_image_load_store_impl(nir_builder *b, nir_instr *instr, switch (dim) { case GLSL_SAMPLER_DIM_BUF: case GLSL_SAMPLER_DIM_1D: - num_components = 1; break; + num_components = 1; + break; case GLSL_SAMPLER_DIM_2D: case GLSL_SAMPLER_DIM_RECT: case GLSL_SAMPLER_DIM_CUBE: - num_components = 2; break; + num_components = 2; + break; case GLSL_SAMPLER_DIM_3D: - num_components = 3; break; + num_components = 3; + break; default: unreachable("Unexpected image size"); } @@ -81,10 +83,15 @@ r600_legalize_image_load_store_impl(nir_builder *b, nir_instr *instr, if (num_components < 3 && nir_intrinsic_image_array(ir)) num_components++; - auto img_size = nir_image_size(b, num_components, 32, ir->src[0].ssa, nir_imm_int(b, 0), - dim, nir_intrinsic_image_array(ir), - nir_intrinsic_format(ir), - nir_intrinsic_access(ir)); + auto img_size = nir_image_size(b, + num_components, + 32, + ir->src[0].ssa, + nir_imm_int(b, 0), + dim, + nir_intrinsic_image_array(ir), + nir_intrinsic_format(ir), + nir_intrinsic_access(ir)); unsigned mask = (1 << num_components) - 1; unsigned num_src1_comp = MIN2(ir->src[1].ssa->num_components, num_components); @@ -95,10 +102,12 @@ r600_legalize_image_load_store_impl(nir_builder *b, nir_instr *instr, nir_channels(b, img_size, mask)); switch (num_components) { - case 2: in_range = nir_iand(b, nir_channel(b, in_range, 0), nir_channel(b, in_range, 1)); break; + case 2: + in_range = nir_iand(b, nir_channel(b, in_range, 0), nir_channel(b, in_range, 1)); + break; case 3: { - auto tmp = nir_iand(b, nir_channel(b, in_range, 0), nir_channel(b, in_range, 1)); - in_range = nir_iand(b, tmp, nir_channel(b, in_range, 2)); + auto tmp = nir_iand(b, nir_channel(b, in_range, 0), nir_channel(b, in_range, 1)); + in_range = nir_iand(b, tmp, nir_channel(b, in_range, 2)); break; } } @@ -144,8 +153,7 @@ r600_legalize_image_load_store_impl(nir_builder *b, nir_instr *instr, } static bool -r600_legalize_image_load_store_filter(const nir_instr *instr, - UNUSED const void *_options) +r600_legalize_image_load_store_filter(const nir_instr *instr, UNUSED const void *_options) { if (instr->type != nir_instr_type_intrinsic) return false; diff --git a/src/gallium/drivers/r600/sfn/sfn_nir_lower_64bit.cpp b/src/gallium/drivers/r600/sfn/sfn_nir_lower_64bit.cpp index 73a54681ff6..dbf5b4212f0 100644 --- a/src/gallium/drivers/r600/sfn/sfn_nir_lower_64bit.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_nir_lower_64bit.cpp @@ -24,47 +24,40 @@ * USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include "sfn_nir.h" - #include "nir.h" #include "nir_builder.h" +#include "sfn_nir.h" +#include #include #include -#include namespace r600 { +using std::make_pair; using std::map; using std::pair; -using std::make_pair; using std::vector; class LowerSplit64BitVar : public NirLowerInstruction { public: - ~LowerSplit64BitVar(); - using VarSplit = pair; + using VarSplit = pair; using VarMap = map; - nir_ssa_def * - split_double_load_deref(nir_intrinsic_instr *intr); + nir_ssa_def *split_double_load_deref(nir_intrinsic_instr *intr); - nir_ssa_def * - split_double_store_deref(nir_intrinsic_instr *intr); + nir_ssa_def *split_double_store_deref(nir_intrinsic_instr *intr); private: - nir_ssa_def * - split_load_deref_array(nir_intrinsic_instr *intr, nir_src& index); + nir_ssa_def *split_load_deref_array(nir_intrinsic_instr *intr, nir_src& index); - nir_ssa_def * - split_load_deref_var(nir_intrinsic_instr *intr); + nir_ssa_def *split_load_deref_var(nir_intrinsic_instr *intr); - nir_ssa_def * - split_store_deref_array(nir_intrinsic_instr *intr, nir_deref_instr *deref); + nir_ssa_def *split_store_deref_array(nir_intrinsic_instr *intr, + nir_deref_instr *deref); - nir_ssa_def * - split_store_deref_var(nir_intrinsic_instr *intr, nir_deref_instr *deref1); + nir_ssa_def *split_store_deref_var(nir_intrinsic_instr *intr, nir_deref_instr *deref1); VarSplit get_var_pair(nir_variable *old_var); @@ -73,27 +66,22 @@ private: nir_ssa_def *split_double_load(nir_intrinsic_instr *load1); - nir_ssa_def * - split_store_output(nir_intrinsic_instr *store1); + nir_ssa_def *split_store_output(nir_intrinsic_instr *store1); nir_ssa_def *split_double_load_uniform(nir_intrinsic_instr *intr); - nir_ssa_def * - split_double_load_ssbo(nir_intrinsic_instr *intr); + nir_ssa_def *split_double_load_ssbo(nir_intrinsic_instr *intr); - nir_ssa_def * - split_double_load_ubo(nir_intrinsic_instr *intr); + nir_ssa_def *split_double_load_ubo(nir_intrinsic_instr *intr); nir_ssa_def * split_reduction(nir_ssa_def *src[2][2], nir_op op1, nir_op op2, nir_op reduction); nir_ssa_def * - split_reduction3(nir_alu_instr *alu, - nir_op op1, nir_op op2, nir_op reduction); + split_reduction3(nir_alu_instr *alu, nir_op op1, nir_op op2, nir_op reduction); nir_ssa_def * - split_reduction4(nir_alu_instr *alu, - nir_op op1, nir_op op2, nir_op reduction); + split_reduction4(nir_alu_instr *alu, nir_op op1, nir_op op2, nir_op reduction); nir_ssa_def *split_bcsel(nir_alu_instr *alu); @@ -103,17 +91,17 @@ private: nir_ssa_def *lower(nir_instr *instr) override; VarMap m_varmap; - vector m_old_vars; + vector m_old_vars; vector m_old_stores; }; - class LowerLoad64Uniform : public NirLowerInstruction { bool filter(const nir_instr *instr) const override; nir_ssa_def *lower(nir_instr *instr) override; }; -bool LowerLoad64Uniform::filter(const nir_instr *instr) const +bool +LowerLoad64Uniform::filter(const nir_instr *instr) const { if (instr->type != nir_instr_type_intrinsic) return false; @@ -127,8 +115,8 @@ bool LowerLoad64Uniform::filter(const nir_instr *instr) const return nir_dest_bit_size(intr->dest) == 64; } - -nir_ssa_def *LowerLoad64Uniform::lower(nir_instr *instr) +nir_ssa_def * +LowerLoad64Uniform::lower(nir_instr *instr) { auto intr = nir_instr_as_intrinsic(instr); int old_components = nir_dest_num_components(intr->dest); @@ -138,8 +126,8 @@ nir_ssa_def *LowerLoad64Uniform::lower(nir_instr *instr) intr->dest.ssa.bit_size = 32; intr->num_components *= 2; - if (intr->intrinsic ==nir_intrinsic_load_ubo || - intr->intrinsic ==nir_intrinsic_load_ubo_vec4) + if (intr->intrinsic == nir_intrinsic_load_ubo || + intr->intrinsic == nir_intrinsic_load_ubo_vec4) nir_intrinsic_set_component(intr, 2 * nir_intrinsic_component(intr)); nir_ssa_def *result_vec[2] = {nullptr, nullptr}; @@ -155,13 +143,15 @@ nir_ssa_def *LowerLoad64Uniform::lower(nir_instr *instr) return nir_vec2(b, result_vec[0], result_vec[1]); } -bool r600_split_64bit_uniforms_and_ubo(nir_shader *sh) +bool +r600_split_64bit_uniforms_and_ubo(nir_shader *sh) { return LowerLoad64Uniform().run(sh); } class LowerSplit64op : public NirLowerInstruction { - bool filter(const nir_instr *instr) const override { + bool filter(const nir_instr *instr) const override + { switch (instr->type) { case nir_instr_type_alu: { auto alu = nir_instr_as_alu(instr); @@ -189,7 +179,8 @@ class LowerSplit64op : public NirLowerInstruction { } } - nir_ssa_def *lower(nir_instr *instr) override { + nir_ssa_def *lower(nir_instr *instr) override + { switch (instr->type) { case nir_instr_type_alu: { @@ -197,18 +188,23 @@ class LowerSplit64op : public NirLowerInstruction { switch (alu->op) { case nir_op_bcsel: { - auto lo = nir_bcsel(b, nir_ssa_for_src(b, alu->src[0].src, 1), - nir_unpack_64_2x32_split_x(b, nir_ssa_for_alu_src(b, alu, 1)), - nir_unpack_64_2x32_split_x(b, nir_ssa_for_alu_src(b, alu, 2))); - auto hi = nir_bcsel(b, nir_ssa_for_src(b, alu->src[0].src, 1), - nir_unpack_64_2x32_split_y(b, nir_ssa_for_alu_src(b, alu, 1)), - nir_unpack_64_2x32_split_y(b, nir_ssa_for_alu_src(b, alu, 2))); + auto lo = + nir_bcsel(b, + nir_ssa_for_src(b, alu->src[0].src, 1), + nir_unpack_64_2x32_split_x(b, nir_ssa_for_alu_src(b, alu, 1)), + nir_unpack_64_2x32_split_x(b, nir_ssa_for_alu_src(b, alu, 2))); + auto hi = + nir_bcsel(b, + nir_ssa_for_src(b, alu->src[0].src, 1), + nir_unpack_64_2x32_split_y(b, nir_ssa_for_alu_src(b, alu, 1)), + nir_unpack_64_2x32_split_y(b, nir_ssa_for_alu_src(b, alu, 2))); return nir_pack_64_2x32_split(b, lo, hi); } case nir_op_f2b1: { auto mask = nir_component_mask(nir_dest_num_components(alu->dest.dest)); - return nir_fneu(b, nir_channels(b, nir_ssa_for_alu_src(b, alu, 0), mask), - nir_imm_zero(b, nir_dest_num_components(alu->dest.dest), 64)); + return nir_fneu(b, + nir_channels(b, nir_ssa_for_alu_src(b, alu, 0), mask), + nir_imm_zero(b, nir_dest_num_components(alu->dest.dest), 64)); } case nir_op_f2i32: { auto src = nir_ssa_for_alu_src(b, alu, 0); @@ -224,12 +220,14 @@ class LowerSplit64op : public NirLowerInstruction { * For values > UINT_MAX the result is undefined */ auto src = nir_ssa_for_alu_src(b, alu, 0); auto gt0 = nir_flt(b, nir_imm_double(b, 0.0), src); - auto highval = nir_fmul_imm(b, src, 1.0/65536.0); + auto highval = nir_fmul_imm(b, src, 1.0 / 65536.0); auto fract = nir_ffract(b, highval); auto high = nir_f2u32(b, nir_f2f32(b, nir_fsub(b, highval, fract))); auto lowval = nir_fmul_imm(b, fract, 65536.0); auto low = nir_f2u32(b, nir_f2f32(b, lowval)); - return nir_bcsel(b, gt0, nir_ior(b, nir_ishl_imm(b, high, 16), low), + return nir_bcsel(b, + gt0, + nir_ior(b, nir_ishl_imm(b, high, 16), low), nir_imm_int(b, 0)); } case nir_op_f2i64: { @@ -242,11 +240,13 @@ class LowerSplit64op : public NirLowerInstruction { case nir_op_f2u64: { auto src = nir_ssa_for_alu_src(b, alu, 0); auto gt0 = nir_flt(b, nir_imm_double(b, 0.0), src); - auto highval = nir_fmul_imm(b, src, 1.0/(65536.0 * 65536.0)); + auto highval = nir_fmul_imm(b, src, 1.0 / (65536.0 * 65536.0)); auto fract = nir_ffract(b, highval); auto high = nir_f2u32(b, nir_fsub(b, highval, fract)); auto low = nir_f2u32(b, nir_fmul_imm(b, fract, 65536.0 * 65536.0)); - return nir_bcsel(b, gt0, nir_pack_64_2x32_split(b, low, high), + return nir_bcsel(b, + gt0, + nir_pack_64_2x32_split(b, low, high), nir_imm_zero(b, 1, 64)); } case nir_op_u2f64: { @@ -273,9 +273,12 @@ class LowerSplit64op : public NirLowerInstruction { auto phi = nir_instr_as_phi(instr); auto phi_lo = nir_phi_instr_create(b->shader); auto phi_hi = nir_phi_instr_create(b->shader); - nir_ssa_dest_init(&phi_lo->instr, &phi_lo->dest, phi->dest.ssa.num_components * 2, 32, ""); - nir_ssa_dest_init(&phi_hi->instr, &phi_hi->dest, phi->dest.ssa.num_components * 2, 32, ""); - nir_foreach_phi_src(s, phi) { + nir_ssa_dest_init( + &phi_lo->instr, &phi_lo->dest, phi->dest.ssa.num_components * 2, 32, ""); + nir_ssa_dest_init( + &phi_hi->instr, &phi_hi->dest, phi->dest.ssa.num_components * 2, 32, ""); + nir_foreach_phi_src(s, phi) + { auto lo = nir_unpack_32_2x16_split_x(b, nir_ssa_for_src(b, s->src, 1)); auto hi = nir_unpack_32_2x16_split_x(b, nir_ssa_for_src(b, s->src, 1)); nir_phi_instr_add_src(phi_lo, s->pred, nir_src_for_ssa(lo)); @@ -289,17 +292,17 @@ class LowerSplit64op : public NirLowerInstruction { } }; -bool r600_split_64bit_alu_and_phi(nir_shader *sh) +bool +r600_split_64bit_alu_and_phi(nir_shader *sh) { return LowerSplit64op().run(sh); } - bool LowerSplit64BitVar::filter(const nir_instr *instr) const { switch (instr->type) { - case nir_instr_type_intrinsic: { + case nir_instr_type_intrinsic: { auto intr = nir_instr_as_intrinsic(instr); switch (intr->intrinsic) { @@ -323,7 +326,7 @@ LowerSplit64BitVar::filter(const nir_instr *instr) const return false; } } - case nir_instr_type_alu: { + case nir_instr_type_alu: { auto alu = nir_instr_as_alu(instr); switch (alu->op) { case nir_op_bcsel: @@ -358,14 +361,17 @@ LowerSplit64BitVar::filter(const nir_instr *instr) const nir_ssa_def * LowerSplit64BitVar::merge_64bit_loads(nir_ssa_def *load1, - nir_ssa_def *load2, bool out_is_vec3) + nir_ssa_def *load2, + bool out_is_vec3) { if (out_is_vec3) - return nir_vec3(b, nir_channel(b, load1, 0), + return nir_vec3(b, + nir_channel(b, load1, 0), nir_channel(b, load1, 1), nir_channel(b, load2, 0)); else - return nir_vec4(b, nir_channel(b, load1, 0), + return nir_vec4(b, + nir_channel(b, load1, 0), nir_channel(b, load1, 1), nir_channel(b, load2, 0), nir_channel(b, load2, 1)); @@ -373,10 +379,10 @@ LowerSplit64BitVar::merge_64bit_loads(nir_ssa_def *load1, LowerSplit64BitVar::~LowerSplit64BitVar() { - for(auto&& v: m_old_vars) + for (auto&& v : m_old_vars) exec_node_remove(&v->node); - for(auto&& v: m_old_stores) + for (auto&& v : m_old_stores) nir_instr_remove(v); } @@ -419,18 +425,21 @@ LowerSplit64BitVar::split_load_deref_array(nir_intrinsic_instr *intr, nir_src& i auto deref1 = nir_build_deref_var(b, vars.first); auto deref_array1 = nir_build_deref_array(b, deref1, nir_ssa_for_src(b, index, 1)); - auto load1 = nir_build_load_deref(b, 2, 64, &deref_array1->dest.ssa, (enum gl_access_qualifier)0); + auto load1 = + nir_build_load_deref(b, 2, 64, &deref_array1->dest.ssa, (enum gl_access_qualifier)0); auto deref2 = nir_build_deref_var(b, vars.second); auto deref_array2 = nir_build_deref_array(b, deref2, nir_ssa_for_src(b, index, 1)); - auto load2 = nir_build_load_deref(b, old_components - 2, 64, &deref_array2->dest.ssa, (enum gl_access_qualifier)0); + auto load2 = nir_build_load_deref( + b, old_components - 2, 64, &deref_array2->dest.ssa, (enum gl_access_qualifier)0); return merge_64bit_loads(load1, load2, old_components == 3); } nir_ssa_def * -LowerSplit64BitVar::split_store_deref_array(nir_intrinsic_instr *intr, nir_deref_instr *deref) +LowerSplit64BitVar::split_store_deref_array(nir_intrinsic_instr *intr, + nir_deref_instr *deref) { auto old_var = nir_intrinsic_get_var(intr, 0); unsigned old_components = old_var->type->without_array()->components(); @@ -442,23 +451,32 @@ LowerSplit64BitVar::split_store_deref_array(nir_intrinsic_instr *intr, nir_deref auto vars = get_var_pair(old_var); auto deref1 = nir_build_deref_var(b, vars.first); - auto deref_array1 = nir_build_deref_array(b, deref1, nir_ssa_for_src(b, deref->arr.index, 1)); + auto deref_array1 = + nir_build_deref_array(b, deref1, nir_ssa_for_src(b, deref->arr.index, 1)); nir_build_store_deref(b, &deref_array1->dest.ssa, src_xy, 3); auto deref2 = nir_build_deref_var(b, vars.second); - auto deref_array2 = nir_build_deref_array(b, deref2, nir_ssa_for_src(b, deref->arr.index, 1)); + auto deref_array2 = + nir_build_deref_array(b, deref2, nir_ssa_for_src(b, deref->arr.index, 1)); if (old_components == 3) - nir_build_store_deref(b, &deref_array2->dest.ssa, nir_channel(b, intr->src[1].ssa, 2), 1); + nir_build_store_deref(b, + &deref_array2->dest.ssa, + nir_channel(b, intr->src[1].ssa, 2), + 1); else - nir_build_store_deref(b, &deref_array2->dest.ssa, nir_channels(b, intr->src[1].ssa, 0xc), 3); + nir_build_store_deref(b, + &deref_array2->dest.ssa, + nir_channels(b, intr->src[1].ssa, 0xc), + 3); return NIR_LOWER_INSTR_PROGRESS_REPLACE; } nir_ssa_def * -LowerSplit64BitVar::split_store_deref_var(nir_intrinsic_instr *intr, UNUSED nir_deref_instr *deref) +LowerSplit64BitVar::split_store_deref_var(nir_intrinsic_instr *intr, + UNUSED nir_deref_instr *deref) { auto old_var = nir_intrinsic_get_var(intr, 0); unsigned old_components = old_var->type->without_array()->components(); @@ -476,7 +494,10 @@ LowerSplit64BitVar::split_store_deref_var(nir_intrinsic_instr *intr, UNUSED nir_ if (old_components == 3) nir_build_store_deref(b, &deref2->dest.ssa, nir_channel(b, intr->src[1].ssa, 2), 1); else - nir_build_store_deref(b, &deref2->dest.ssa, nir_channels(b, intr->src[1].ssa, 0xc), 3); + nir_build_store_deref(b, + &deref2->dest.ssa, + nir_channels(b, intr->src[1].ssa, 0xc), + 3); return NIR_LOWER_INSTR_PROGRESS_REPLACE; } @@ -534,7 +555,6 @@ LowerSplit64BitVar::get_var_pair(nir_variable *old_var) return m_varmap[old_var->data.driver_location]; } - nir_ssa_def * LowerSplit64BitVar::split_double_load(nir_intrinsic_instr *load1) { @@ -555,7 +575,6 @@ LowerSplit64BitVar::split_double_load(nir_intrinsic_instr *load1) return merge_64bit_loads(&load1->dest.ssa, &load2->dest.ssa, old_components == 3); } - nir_ssa_def * LowerSplit64BitVar::split_store_output(nir_intrinsic_instr *store1) { @@ -584,12 +603,12 @@ LowerSplit64BitVar::split_store_output(nir_intrinsic_instr *store1) return NIR_LOWER_INSTR_PROGRESS; } - nir_ssa_def * LowerSplit64BitVar::split_double_load_uniform(nir_intrinsic_instr *intr) { unsigned second_components = nir_dest_num_components(intr->dest) - 2; - nir_intrinsic_instr *load2 = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_uniform); + nir_intrinsic_instr *load2 = + nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_uniform); load2->src[0] = nir_src_for_ssa(nir_iadd_imm(b, intr->src[0].ssa, 1)); nir_intrinsic_set_dest_type(load2, nir_intrinsic_dest_type(intr)); nir_intrinsic_set_base(load2, nir_intrinsic_base(intr)); @@ -602,11 +621,13 @@ LowerSplit64BitVar::split_double_load_uniform(nir_intrinsic_instr *intr) intr->dest.ssa.num_components = intr->num_components = 2; if (second_components == 1) - return nir_vec3(b, nir_channel(b, &intr->dest.ssa, 0), + return nir_vec3(b, + nir_channel(b, &intr->dest.ssa, 0), nir_channel(b, &intr->dest.ssa, 1), nir_channel(b, &load2->dest.ssa, 0)); else - return nir_vec4(b, nir_channel(b, &intr->dest.ssa, 0), + return nir_vec4(b, + nir_channel(b, &intr->dest.ssa, 0), nir_channel(b, &intr->dest.ssa, 1), nir_channel(b, &load2->dest.ssa, 0), nir_channel(b, &load2->dest.ssa, 1)); @@ -616,7 +637,8 @@ nir_ssa_def * LowerSplit64BitVar::split_double_load_ssbo(nir_intrinsic_instr *intr) { unsigned second_components = nir_dest_num_components(intr->dest) - 2; - nir_intrinsic_instr *load2 = nir_instr_as_intrinsic(nir_instr_clone(b->shader, &intr->instr)); + nir_intrinsic_instr *load2 = + nir_instr_as_intrinsic(nir_instr_clone(b->shader, &intr->instr)); auto new_src0 = nir_src_for_ssa(nir_iadd_imm(b, intr->src[0].ssa, 1)); nir_instr_rewrite_src(&load2->instr, &load2->src[0], new_src0); @@ -631,12 +653,12 @@ LowerSplit64BitVar::split_double_load_ssbo(nir_intrinsic_instr *intr) return merge_64bit_loads(&intr->dest.ssa, &load2->dest.ssa, second_components == 1); } - nir_ssa_def * LowerSplit64BitVar::split_double_load_ubo(nir_intrinsic_instr *intr) { unsigned second_components = nir_dest_num_components(intr->dest) - 2; - nir_intrinsic_instr *load2 = nir_instr_as_intrinsic(nir_instr_clone(b->shader, &intr->instr)); + nir_intrinsic_instr *load2 = + nir_instr_as_intrinsic(nir_instr_clone(b->shader, &intr->instr)); load2->src[0] = intr->src[0]; load2->src[1] = nir_src_for_ssa(nir_iadd_imm(b, intr->src[1].ssa, 16)); nir_intrinsic_set_range_base(load2, nir_intrinsic_range_base(intr) + 16); @@ -656,7 +678,10 @@ LowerSplit64BitVar::split_double_load_ubo(nir_intrinsic_instr *intr) } nir_ssa_def * -LowerSplit64BitVar::split_reduction(nir_ssa_def *src[2][2], nir_op op1, nir_op op2, nir_op reduction) +LowerSplit64BitVar::split_reduction(nir_ssa_def *src[2][2], + nir_op op1, + nir_op op2, + nir_op reduction) { auto cmp0 = nir_build_alu(b, op1, src[0][0], src[0][1], nullptr, nullptr); auto cmp1 = nir_build_alu(b, op2, src[1][0], src[1][1], nullptr, nullptr); @@ -665,30 +690,34 @@ LowerSplit64BitVar::split_reduction(nir_ssa_def *src[2][2], nir_op op1, nir_op o nir_ssa_def * LowerSplit64BitVar::split_reduction3(nir_alu_instr *alu, - nir_op op1, nir_op op2, nir_op reduction) + nir_op op1, + nir_op op2, + nir_op reduction) { nir_ssa_def *src[2][2]; src[0][0] = nir_channels(b, nir_ssa_for_src(b, alu->src[0].src, 2), 3); src[0][1] = nir_channels(b, nir_ssa_for_src(b, alu->src[1].src, 2), 3); - src[1][0] = nir_channel(b, nir_ssa_for_src(b, alu->src[0].src, 3), 2); - src[1][1] = nir_channel(b, nir_ssa_for_src(b, alu->src[1].src, 3), 2); + src[1][0] = nir_channel(b, nir_ssa_for_src(b, alu->src[0].src, 3), 2); + src[1][1] = nir_channel(b, nir_ssa_for_src(b, alu->src[1].src, 3), 2); return split_reduction(src, op1, op2, reduction); } nir_ssa_def * LowerSplit64BitVar::split_reduction4(nir_alu_instr *alu, - nir_op op1, nir_op op2, nir_op reduction) + nir_op op1, + nir_op op2, + nir_op reduction) { nir_ssa_def *src[2][2]; src[0][0] = nir_channels(b, nir_ssa_for_src(b, alu->src[0].src, 2), 3); src[0][1] = nir_channels(b, nir_ssa_for_src(b, alu->src[1].src, 2), 3); - src[1][0] = nir_channels(b, nir_ssa_for_src(b, alu->src[0].src, 4), 0xc); - src[1][1] = nir_channels(b, nir_ssa_for_src(b, alu->src[1].src, 4), 0xc); + src[1][0] = nir_channels(b, nir_ssa_for_src(b, alu->src[0].src, 4), 0xc); + src[1][1] = nir_channels(b, nir_ssa_for_src(b, alu->src[1].src, 4), 0xc); return split_reduction(src, op1, op2, reduction); } @@ -741,7 +770,7 @@ LowerSplit64BitVar::lower(nir_instr *instr) assert(0); } } - case nir_instr_type_alu: { + case nir_instr_type_alu: { auto alu = nir_instr_as_alu(instr); switch (alu->op) { case nir_op_bany_fnequal3: @@ -755,13 +784,25 @@ LowerSplit64BitVar::lower(nir_instr *instr) case nir_op_fdot3: return split_reduction3(alu, nir_op_fdot2, nir_op_fmul, nir_op_fadd); case nir_op_bany_fnequal4: - return split_reduction4(alu, nir_op_bany_fnequal2, nir_op_bany_fnequal2, nir_op_ior); + return split_reduction4(alu, + nir_op_bany_fnequal2, + nir_op_bany_fnequal2, + nir_op_ior); case nir_op_ball_fequal4: - return split_reduction4(alu, nir_op_ball_fequal2, nir_op_ball_fequal2, nir_op_iand); + return split_reduction4(alu, + nir_op_ball_fequal2, + nir_op_ball_fequal2, + nir_op_iand); case nir_op_bany_inequal4: - return split_reduction4(alu, nir_op_bany_inequal2, nir_op_bany_inequal2, nir_op_ior); + return split_reduction4(alu, + nir_op_bany_inequal2, + nir_op_bany_inequal2, + nir_op_ior); case nir_op_ball_iequal4: - return split_reduction4(alu, nir_op_bany_fnequal2, nir_op_bany_fnequal2, nir_op_ior); + return split_reduction4(alu, + nir_op_bany_fnequal2, + nir_op_bany_fnequal2, + nir_op_ior); case nir_op_fdot4: return split_reduction4(alu, nir_op_fdot2, nir_op_fdot2, nir_op_fadd); case nir_op_bcsel: @@ -807,7 +848,7 @@ bool Lower64BitToVec2::filter(const nir_instr *instr) const { switch (instr->type) { - case nir_instr_type_intrinsic: { + case nir_instr_type_intrinsic: { auto intr = nir_instr_as_intrinsic(instr); switch (intr->intrinsic) { @@ -838,11 +879,11 @@ Lower64BitToVec2::filter(const nir_instr *instr) const auto phi = nir_instr_as_phi(instr); return nir_dest_bit_size(phi->dest) == 64; } - case nir_instr_type_load_const: { + case nir_instr_type_load_const: { auto lc = nir_instr_as_load_const(instr); return lc->def.bit_size == 64; } - case nir_instr_type_ssa_undef: { + case nir_instr_type_ssa_undef: { auto undef = nir_instr_as_ssa_undef(instr); return undef->def.bit_size == 64; } @@ -855,7 +896,7 @@ nir_ssa_def * Lower64BitToVec2::lower(nir_instr *instr) { switch (instr->type) { - case nir_instr_type_intrinsic: { + case nir_instr_type_intrinsic: { auto intr = nir_instr_as_intrinsic(instr); switch (intr->intrinsic) { case nir_intrinsic_load_deref: @@ -904,10 +945,10 @@ Lower64BitToVec2::lower(nir_instr *instr) phi->dest.ssa.num_components = 2; return NIR_LOWER_INSTR_PROGRESS; } - case nir_instr_type_load_const: { + case nir_instr_type_load_const: { auto lc = nir_instr_as_load_const(instr); assert(lc->def.num_components < 3); - nir_const_value val[4] = { { 0 } }; + nir_const_value val[4] = {{0}}; for (uint i = 0; i < lc->def.num_components; ++i) { uint64_t v = lc->value[i].u64; val[0].u32 = v & 0xffffffff; @@ -916,7 +957,7 @@ Lower64BitToVec2::lower(nir_instr *instr) return nir_build_imm(b, 2 * lc->def.num_components, 32, val); } - case nir_instr_type_ssa_undef: { + case nir_instr_type_ssa_undef: { auto undef = nir_instr_as_ssa_undef(instr); undef->def.num_components *= 2; undef->def.bit_size = 32; @@ -925,10 +966,8 @@ Lower64BitToVec2::lower(nir_instr *instr) default: return nullptr; } - } - nir_ssa_def * Lower64BitToVec2::load_deref_64_to_vec2(nir_intrinsic_instr *intr) { @@ -941,8 +980,8 @@ Lower64BitToVec2::load_deref_64_to_vec2(nir_intrinsic_instr *intr) var->type = glsl_vec_type(components); } else if (deref->deref_type == nir_deref_type_array) { - var->type = glsl_array_type(glsl_vec_type(components), - var->type->array_size(), 0); + var->type = + glsl_array_type(glsl_vec_type(components), var->type->array_size(), 0); } else { nir_print_shader(b->shader, stderr); @@ -975,11 +1014,11 @@ Lower64BitToVec2::store_64_to_vec2(nir_intrinsic_instr *intr) if (deref->deref_type == nir_deref_type_var) { var->type = glsl_vec_type(components); } else if (deref->deref_type == nir_deref_type_array) { - var->type = glsl_array_type(glsl_vec_type(components), - var->type->array_size(), 0); + var->type = + glsl_array_type(glsl_vec_type(components), var->type->array_size(), 0); } else { - nir_print_shader(b->shader, stderr); - assert(0 && "Only lowring of var and array derefs supported\n"); + nir_print_shader(b->shader, stderr); + assert(0 && "Only lowring of var and array derefs supported\n"); } } deref->type = var->type; @@ -993,7 +1032,6 @@ Lower64BitToVec2::store_64_to_vec2(nir_intrinsic_instr *intr) return NIR_LOWER_INSTR_PROGRESS; } - nir_ssa_def * Lower64BitToVec2::load_uniform_64_to_vec2(nir_intrinsic_instr *intr) { @@ -1023,14 +1061,16 @@ Lower64BitToVec2::load_ssbo_64_to_vec2(nir_intrinsic_instr *intr) return NIR_LOWER_INSTR_PROGRESS; } -static bool store_64bit_intr(nir_src *src, void *state) +static bool +store_64bit_intr(nir_src *src, void *state) { bool *s = (bool *)state; *s = nir_src_bit_size(*src) == 64; return !*s; } -static bool double2vec2(nir_src *src, UNUSED void *state) +static bool +double2vec2(nir_src *src, UNUSED void *state) { if (nir_src_bit_size(*src) != 64) return true; @@ -1044,14 +1084,17 @@ static bool double2vec2(nir_src *src, UNUSED void *state) bool r600_nir_64_to_vec2(nir_shader *sh) { - vector intr64bit; - nir_foreach_function(function, sh) { + vector intr64bit; + nir_foreach_function(function, sh) + { if (function->impl) { nir_builder b; nir_builder_init(&b, function->impl); - nir_foreach_block(block, function->impl) { - nir_foreach_instr_safe(instr, block) { + nir_foreach_block(block, function->impl) + { + nir_foreach_instr_safe(instr, block) + { switch (instr->type) { case nir_instr_type_alu: { bool success = false; @@ -1074,12 +1117,10 @@ r600_nir_64_to_vec2(nir_shader *sh) } break; } - default: - ; + default:; } } - default: - ; + default:; } } } @@ -1090,7 +1131,7 @@ r600_nir_64_to_vec2(nir_shader *sh) if (result || !intr64bit.empty()) { - for(auto&& instr: intr64bit) { + for (auto&& instr : intr64bit) { if (instr->type == nir_instr_type_alu) { auto alu = nir_instr_as_alu(instr); auto alu_info = nir_op_infos[alu->op]; @@ -1138,35 +1179,38 @@ r600_nir_64_to_vec2(nir_shader *sh) } using std::map; -using std::vector; using std::pair; +using std::vector; class StoreMerger { public: StoreMerger(nir_shader *shader); void collect_stores(); bool combine(); - void combine_one_slot(vector& stores); + void combine_one_slot(vector& stores); - using StoreCombos = map>; + using StoreCombos = map>; StoreCombos m_stores; nir_shader *sh; }; StoreMerger::StoreMerger(nir_shader *shader): - sh(shader) + sh(shader) { } - -void StoreMerger::collect_stores() +void +StoreMerger::collect_stores() { unsigned vertex = 0; - nir_foreach_function(function, sh) { + nir_foreach_function(function, sh) + { if (function->impl) { - nir_foreach_block(block, function->impl) { - nir_foreach_instr_safe(instr, block) { + nir_foreach_block(block, function->impl) + { + nir_foreach_instr_safe(instr, block) + { if (instr->type != nir_instr_type_intrinsic) continue; @@ -1188,10 +1232,11 @@ void StoreMerger::collect_stores() } } -bool StoreMerger::combine() +bool +StoreMerger::combine() { bool progress = false; - for(auto&& i : m_stores) { + for (auto&& i : m_stores) { if (i.second.size() < 2) continue; @@ -1201,7 +1246,8 @@ bool StoreMerger::combine() return progress; } -void StoreMerger::combine_one_slot(vector& stores) +void +StoreMerger::combine_one_slot(vector& stores) { nir_ssa_def *srcs[4] = {nullptr}; @@ -1227,7 +1273,9 @@ void StoreMerger::combine_one_slot(vector& stores) auto new_src = nir_vec(&b, srcs, comps); - nir_instr_rewrite_src(&last_store->instr, &last_store->src[0], nir_src_for_ssa(new_src)); + nir_instr_rewrite_src(&last_store->instr, + &last_store->src[0], + nir_src_for_ssa(new_src)); last_store->num_components = comps; nir_intrinsic_set_component(last_store, first_comp); nir_intrinsic_set_write_mask(last_store, writemask); @@ -1236,7 +1284,8 @@ void StoreMerger::combine_one_slot(vector& stores) nir_instr_remove(&(*i)->instr); } -bool r600_merge_vec2_stores(nir_shader *shader) +bool +r600_merge_vec2_stores(nir_shader *shader) { r600::StoreMerger merger(shader); merger.collect_stores(); @@ -1271,9 +1320,9 @@ r600_lower_64bit_intrinsic(nir_builder *b, nir_intrinsic_instr *instr) if (has_dest) { if (nir_dest_bit_size(instr->dest) != 64) return false; - } else { + } else { if (nir_src_bit_size(instr->src[0]) != 64) - return false; + return false; } nir_intrinsic_instr *first = @@ -1325,18 +1374,20 @@ r600_lower_64bit_intrinsic(nir_builder *b, nir_intrinsic_instr *instr) b->cursor = nir_before_instr(&instr->instr); nir_ssa_def *src0 = instr->src[0].ssa; - nir_ssa_scalar channels[4] = { { 0 } }; + nir_ssa_scalar channels[4] = {{0}}; for (int i = 0; i < instr->num_components; i++) channels[i] = nir_get_ssa_scalar(src0, i); nir_intrinsic_set_write_mask(first, nir_intrinsic_write_mask(instr) & 3); nir_intrinsic_set_write_mask(second, nir_intrinsic_write_mask(instr) >> 2); - nir_instr_rewrite_src(&first->instr, &first->src[0], + nir_instr_rewrite_src(&first->instr, + &first->src[0], nir_src_for_ssa(nir_vec_scalars(b, channels, 2))); - nir_instr_rewrite_src(&second->instr, &second->src[0], - nir_src_for_ssa(nir_vec_scalars(b, &channels[2], - second->num_components))); + nir_instr_rewrite_src( + &second->instr, + &second->src[0], + nir_src_for_ssa(nir_vec_scalars(b, &channels[2], second->num_components))); } int offset_src = -1; @@ -1362,7 +1413,8 @@ r600_lower_64bit_intrinsic(nir_builder *b, nir_intrinsic_instr *instr) b->cursor = nir_before_instr(&second->instr); nir_ssa_def *second_offset = nir_iadd_imm(b, second->src[offset_src].ssa, offset_amount); - nir_instr_rewrite_src(&second->instr, &second->src[offset_src], + nir_instr_rewrite_src(&second->instr, + &second->src[offset_src], nir_src_for_ssa(second_offset)); } @@ -1391,8 +1443,7 @@ r600_lower_64bit_load_const(nir_builder *b, nir_load_const_instr *instr) b->cursor = nir_before_instr(&instr->instr); - nir_load_const_instr *first = - nir_load_const_instr_create(b->shader, 2, 64); + nir_load_const_instr *first = nir_load_const_instr_create(b->shader, 2, 64); nir_load_const_instr *second = nir_load_const_instr_create(b->shader, num_components - 2, 64); @@ -1437,12 +1488,8 @@ r600_lower_64bit_to_vec2(nir_shader *s) { return nir_shader_instructions_pass(s, r600_lower_64bit_to_vec2_instr, - nir_metadata_block_index | - nir_metadata_dominance, + nir_metadata_block_index | nir_metadata_dominance, NULL); } - } // end namespace r600 - - diff --git a/src/gallium/drivers/r600/sfn/sfn_nir_lower_alu.cpp b/src/gallium/drivers/r600/sfn/sfn_nir_lower_alu.cpp index 93f0aea823c..5620752b3f1 100644 --- a/src/gallium/drivers/r600/sfn/sfn_nir_lower_alu.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_nir_lower_alu.cpp @@ -25,6 +25,7 @@ */ #include "sfn_nir_lower_alu.h" + #include "sfn_nir.h" namespace r600 { @@ -35,8 +36,8 @@ private: nir_ssa_def *lower(nir_instr *instr) override; }; - -bool Lower2x16::filter(const nir_instr *instr) const +bool +Lower2x16::filter(const nir_instr *instr) const { if (instr->type != nir_instr_type_alu) return false; @@ -50,20 +51,22 @@ bool Lower2x16::filter(const nir_instr *instr) const } } -nir_ssa_def *Lower2x16::lower(nir_instr *instr) +nir_ssa_def * +Lower2x16::lower(nir_instr *instr) { nir_alu_instr *alu = nir_instr_as_alu(instr); switch (alu->op) { case nir_op_unpack_half_2x16: { nir_ssa_def *packed = nir_ssa_for_alu_src(b, alu, 0); - return nir_vec2(b, nir_unpack_half_2x16_split_x(b, packed), - nir_unpack_half_2x16_split_y(b, packed)); - + return nir_vec2(b, + nir_unpack_half_2x16_split_x(b, packed), + nir_unpack_half_2x16_split_y(b, packed)); } case nir_op_pack_half_2x16: { nir_ssa_def *src_vec2 = nir_ssa_for_alu_src(b, alu, 0); - return nir_pack_half_2x16_split(b, nir_channel(b, src_vec2, 0), + return nir_pack_half_2x16_split(b, + nir_channel(b, src_vec2, 0), nir_channel(b, src_vec2, 1)); } default: @@ -73,14 +76,19 @@ nir_ssa_def *Lower2x16::lower(nir_instr *instr) class LowerSinCos : public NirLowerInstruction { public: - LowerSinCos(amd_gfx_level gxf_level): m_gxf_level(gxf_level){} + LowerSinCos(amd_gfx_level gxf_level): + m_gxf_level(gxf_level) + { + } + private: bool filter(const nir_instr *instr) const override; nir_ssa_def *lower(nir_instr *instr) override; amd_gfx_level m_gxf_level; }; -bool LowerSinCos::filter(const nir_instr *instr) const +bool +LowerSinCos::filter(const nir_instr *instr) const { if (instr->type != nir_instr_type_alu) return false; @@ -95,12 +103,12 @@ bool LowerSinCos::filter(const nir_instr *instr) const } } -nir_ssa_def *LowerSinCos::lower(nir_instr *instr) +nir_ssa_def * +LowerSinCos::lower(nir_instr *instr) { auto alu = nir_instr_as_alu(instr); - assert(alu->op == nir_op_fsin || - alu->op == nir_op_fcos); + assert(alu->op == nir_op_fsin || alu->op == nir_op_fcos); auto fract = nir_ffract(b, nir_ffma(b, @@ -109,10 +117,9 @@ nir_ssa_def *LowerSinCos::lower(nir_instr *instr) nir_imm_float(b, 0.5))); auto normalized = - m_gxf_level != R600 ? - nir_fadd(b, fract, nir_imm_float(b, -0.5)) : - nir_ffma(b, fract, nir_imm_float(b, 2.0f * M_PI), - nir_imm_float(b, -M_PI)); + m_gxf_level != R600 + ? nir_fadd(b, fract, nir_imm_float(b, -0.5)) + : nir_ffma(b, fract, nir_imm_float(b, 2.0f * M_PI), nir_imm_float(b, -M_PI)); if (alu->op == nir_op_fsin) return nir_fsin_amd(b, normalized); @@ -120,16 +127,16 @@ nir_ssa_def *LowerSinCos::lower(nir_instr *instr) return nir_fcos_amd(b, normalized); } - } // namespace r600 - -bool r600_nir_lower_pack_unpack_2x16(nir_shader *shader) +bool +r600_nir_lower_pack_unpack_2x16(nir_shader *shader) { return r600::Lower2x16().run(shader); } -bool r600_nir_lower_trigen(nir_shader *shader, amd_gfx_level gfx_level) +bool +r600_nir_lower_trigen(nir_shader *shader, amd_gfx_level gfx_level) { return r600::LowerSinCos(gfx_level).run(shader); } diff --git a/src/gallium/drivers/r600/sfn/sfn_nir_lower_alu.h b/src/gallium/drivers/r600/sfn/sfn_nir_lower_alu.h index 01e9f0c0493..5dbe7c75770 100644 --- a/src/gallium/drivers/r600/sfn/sfn_nir_lower_alu.h +++ b/src/gallium/drivers/r600/sfn/sfn_nir_lower_alu.h @@ -27,13 +27,13 @@ #ifndef SFN_NIR_LOWER_ALU_H #define SFN_NIR_LOWER_ALU_H +#include "amd_family.h" #include "nir.h" -#include "amd_family.h" - -bool r600_nir_lower_pack_unpack_2x16(nir_shader *shader); - -bool r600_nir_lower_trigen(nir_shader *shader, enum amd_gfx_level gfx_level); +bool +r600_nir_lower_pack_unpack_2x16(nir_shader *shader); +bool +r600_nir_lower_trigen(nir_shader *shader, enum amd_gfx_level gfx_level); #endif // SFN_NIR_LOWER_ALU_H diff --git a/src/gallium/drivers/r600/sfn/sfn_nir_lower_fs_out_to_vector.cpp b/src/gallium/drivers/r600/sfn/sfn_nir_lower_fs_out_to_vector.cpp index b4a29731f73..9fe9d4c8b27 100644 --- a/src/gallium/drivers/r600/sfn/sfn_nir_lower_fs_out_to_vector.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_nir_lower_fs_out_to_vector.cpp @@ -30,19 +30,19 @@ #include "nir_deref.h" #include "util/u_math.h" +#include +#include #include #include -#include -#include namespace r600 { +using std::array; using std::multiset; using std::vector; -using std::array; -struct nir_intrinsic_instr_less { - bool operator () (const nir_intrinsic_instr *lhs, const nir_intrinsic_instr *rhs) const +struct nir_intrinsic_instr_less { + bool operator()(const nir_intrinsic_instr *lhs, const nir_intrinsic_instr *rhs) const { nir_variable *vlhs = nir_deref_instr_get_variable(nir_src_as_deref(lhs->src[0])); nir_variable *vrhs = nir_deref_instr_get_variable(nir_src_as_deref(rhs->src[0])); @@ -67,28 +67,34 @@ protected: void create_new_io_vars(nir_shader *shader); void create_new_io_var(nir_shader *shader, unsigned location, unsigned comps); - nir_deref_instr *clone_deref_array(nir_builder *b, nir_deref_instr *dst_tail, + nir_deref_instr *clone_deref_array(nir_builder *b, + nir_deref_instr *dst_tail, const nir_deref_instr *src_head); bool vectorize_block(nir_builder *b, nir_block *block); bool instr_can_rewrite(nir_instr *instr); - bool vec_instr_set_remove(nir_builder *b,nir_instr *instr); + bool vec_instr_set_remove(nir_builder *b, nir_instr *instr); - using InstrSet = multiset; + using InstrSet = multiset; using InstrSubSet = std::pair; - bool vec_instr_stack_pop(nir_builder *b, InstrSubSet& ir_set, - nir_intrinsic_instr *instr); + bool + vec_instr_stack_pop(nir_builder *b, InstrSubSet& ir_set, nir_intrinsic_instr *instr); array, 16> m_vars; InstrSet m_block_io; int m_next_index; + private: - virtual nir_variable_mode get_io_mode(nir_shader *shader) const = 0; - virtual bool instr_can_rewrite_type(nir_intrinsic_instr *intr) const = 0; + virtual nir_variable_mode get_io_mode(nir_shader *shader) const = 0; + virtual bool instr_can_rewrite_type(nir_intrinsic_instr *intr) const = 0; virtual bool var_can_rewrite_slot(nir_variable *var) const = 0; - virtual void create_new_io(nir_builder *b, nir_intrinsic_instr *intr, nir_variable *var, - nir_ssa_def **srcs, unsigned first_comp, unsigned num_comps) = 0; + virtual void create_new_io(nir_builder *b, + nir_intrinsic_instr *intr, + nir_variable *var, + nir_ssa_def **srcs, + unsigned first_comp, + unsigned num_comps) = 0; int m_base_slot; }; @@ -98,24 +104,32 @@ public: NirLowerFSOutToVector(); private: - nir_variable_mode get_io_mode(nir_shader *shader) const override; + nir_variable_mode get_io_mode(nir_shader *shader) const override; bool var_can_rewrite_slot(nir_variable *var) const override; - void create_new_io(nir_builder *b, nir_intrinsic_instr *intr, nir_variable *var, - nir_ssa_def **srcs, unsigned first_comp, unsigned num_comps) override; - bool instr_can_rewrite_type(nir_intrinsic_instr *intr) const override; + void create_new_io(nir_builder *b, + nir_intrinsic_instr *intr, + nir_variable *var, + nir_ssa_def **srcs, + unsigned first_comp, + unsigned num_comps) override; + bool instr_can_rewrite_type(nir_intrinsic_instr *intr) const override; - nir_ssa_def *create_combined_vector(nir_builder *b, nir_ssa_def **srcs, - int first_comp, int num_comp); + nir_ssa_def *create_combined_vector(nir_builder *b, + nir_ssa_def **srcs, + int first_comp, + int num_comp); }; -bool r600_lower_fs_out_to_vector(nir_shader *shader) +bool +r600_lower_fs_out_to_vector(nir_shader *shader) { NirLowerFSOutToVector processor; assert(shader->info.stage == MESA_SHADER_FRAGMENT); bool progress = false; - nir_foreach_function(function, shader) { + nir_foreach_function(function, shader) + { if (function->impl) progress |= processor.run(function->impl); } @@ -123,15 +137,16 @@ bool r600_lower_fs_out_to_vector(nir_shader *shader) } NirLowerIOToVector::NirLowerIOToVector(int base_slot): - m_next_index(0), - m_base_slot(base_slot) + m_next_index(0), + m_base_slot(base_slot) { - for(auto& a : m_vars) - for(auto& aa : a) + for (auto& a : m_vars) + for (auto& aa : a) aa = nullptr; } -bool NirLowerIOToVector::run(nir_function_impl *impl) +bool +NirLowerIOToVector::run(nir_function_impl *impl) { nir_builder b; nir_builder_init(&b, impl); @@ -148,12 +163,14 @@ bool NirLowerIOToVector::run(nir_function_impl *impl) return progress; } -void NirLowerIOToVector::create_new_io_vars(nir_shader *shader) +void +NirLowerIOToVector::create_new_io_vars(nir_shader *shader) { nir_variable_mode mode = get_io_mode(shader); bool can_rewrite_vars = false; - nir_foreach_variable_with_modes(var, shader, mode) { + nir_foreach_variable_with_modes(var, shader, mode) + { if (var_can_rewrite(var)) { can_rewrite_vars = true; unsigned loc = var->data.location - m_base_slot; @@ -187,7 +204,6 @@ void NirLowerIOToVector::create_new_io_vars(nir_shader *shader) for (unsigned n = 0; n < glsl_get_components(m_vars[i][k]->type); ++n) comps |= 1 << (m_vars[i][k]->data.location_frac + n); - } } if (comps) @@ -196,15 +212,15 @@ void NirLowerIOToVector::create_new_io_vars(nir_shader *shader) } bool -NirLowerIOToVector::var_can_merge(const nir_variable *lhs, - const nir_variable *rhs) +NirLowerIOToVector::var_can_merge(const nir_variable *lhs, const nir_variable *rhs) { return (glsl_get_base_type(lhs->type) == glsl_get_base_type(rhs->type)); } void NirLowerIOToVector::create_new_io_var(nir_shader *shader, - unsigned location, unsigned comps) + unsigned location, + unsigned comps) { unsigned num_comps = util_bitcount(comps); assert(num_comps > 1); @@ -228,7 +244,8 @@ NirLowerIOToVector::create_new_io_var(nir_shader *shader, } } -bool NirLowerIOToVector::var_can_rewrite(nir_variable *var) const +bool +NirLowerIOToVector::var_can_rewrite(nir_variable *var) const { /* Skip complex types we don't split in the first place */ if (!glsl_type_is_vector_or_scalar(glsl_without_array(var->type))) @@ -245,7 +262,8 @@ NirLowerIOToVector::vectorize_block(nir_builder *b, nir_block *block) { bool progress = false; - nir_foreach_instr_safe(instr, block) { + nir_foreach_instr_safe(instr, block) + { if (instr_can_rewrite(instr)) { instr->index = m_next_index++; nir_intrinsic_instr *ir = nir_instr_as_intrinsic(instr); @@ -258,7 +276,8 @@ NirLowerIOToVector::vectorize_block(nir_builder *b, nir_block *block) progress |= vectorize_block(b, child); } - nir_foreach_instr_reverse_safe(instr, block) { + nir_foreach_instr_reverse_safe(instr, block) + { progress |= vec_instr_set_remove(b, instr); } m_block_io.clear(); @@ -266,7 +285,8 @@ NirLowerIOToVector::vectorize_block(nir_builder *b, nir_block *block) return progress; } -bool NirLowerIOToVector::instr_can_rewrite(nir_instr *instr) +bool +NirLowerIOToVector::instr_can_rewrite(nir_instr *instr) { if (instr->type != nir_instr_type_intrinsic) return false; @@ -279,7 +299,8 @@ bool NirLowerIOToVector::instr_can_rewrite(nir_instr *instr) return instr_can_rewrite_type(intr); } -bool NirLowerIOToVector::vec_instr_set_remove(nir_builder *b,nir_instr *instr) +bool +NirLowerIOToVector::vec_instr_set_remove(nir_builder *b, nir_instr *instr) { if (!instr_can_rewrite(instr)) return false; @@ -293,8 +314,9 @@ bool NirLowerIOToVector::vec_instr_set_remove(nir_builder *b,nir_instr *instr) } nir_deref_instr * -NirLowerIOToVector::clone_deref_array(nir_builder *b, nir_deref_instr *dst_tail, - const nir_deref_instr *src_head) +NirLowerIOToVector::clone_deref_array(nir_builder *b, + nir_deref_instr *dst_tail, + const nir_deref_instr *src_head) { const nir_deref_instr *parent = nir_deref_instr_parent(src_head); @@ -305,33 +327,34 @@ NirLowerIOToVector::clone_deref_array(nir_builder *b, nir_deref_instr *dst_tail, dst_tail = clone_deref_array(b, dst_tail, parent); - return nir_build_deref_array(b, dst_tail, - nir_ssa_for_src(b, src_head->arr.index, 1)); + return nir_build_deref_array(b, dst_tail, nir_ssa_for_src(b, src_head->arr.index, 1)); } NirLowerFSOutToVector::NirLowerFSOutToVector(): - NirLowerIOToVector(FRAG_RESULT_COLOR) + NirLowerIOToVector(FRAG_RESULT_COLOR) { - } -bool NirLowerFSOutToVector::var_can_rewrite_slot(nir_variable *var) const +bool +NirLowerFSOutToVector::var_can_rewrite_slot(nir_variable *var) const { return ((var->data.mode == nir_var_shader_out) && ((var->data.location == FRAG_RESULT_COLOR) || - ((var->data.location >= FRAG_RESULT_DATA0) && - (var->data.location <= FRAG_RESULT_DATA7)))); + ((var->data.location >= FRAG_RESULT_DATA0) && + (var->data.location <= FRAG_RESULT_DATA7)))); } -bool NirLowerIOToVector::vec_instr_stack_pop(nir_builder *b, InstrSubSet &ir_set, - nir_intrinsic_instr *instr) +bool +NirLowerIOToVector::vec_instr_stack_pop(nir_builder *b, + InstrSubSet& ir_set, + nir_intrinsic_instr *instr) { - vector< nir_intrinsic_instr *> ir_sorted_set(ir_set.first, ir_set.second); - std::sort(ir_sorted_set.begin(), ir_sorted_set.end(), + vector ir_sorted_set(ir_set.first, ir_set.second); + std::sort(ir_sorted_set.begin(), + ir_sorted_set.end(), [](const nir_intrinsic_instr *lhs, const nir_intrinsic_instr *rhs) { - return lhs->instr.index > rhs->instr.index; - } - ); + return lhs->instr.index > rhs->instr.index; + }); nir_intrinsic_instr *intr = *ir_sorted_set.begin(); nir_variable *var = nir_deref_instr_get_variable(nir_src_as_deref(intr->src[0])); @@ -352,8 +375,7 @@ bool NirLowerIOToVector::vec_instr_stack_pop(nir_builder *b, InstrSubSet &ir_set } b->cursor = nir_after_instr(&intr->instr); - nir_ssa_undef_instr *instr_undef = - nir_ssa_undef_instr_create(b->shader, 1, 32); + nir_ssa_undef_instr *instr_undef = nir_ssa_undef_instr_create(b->shader, 1, 32); nir_builder_instr_insert(b, &instr_undef->instr); nir_ssa_def *srcs[4]; @@ -364,8 +386,7 @@ bool NirLowerIOToVector::vec_instr_stack_pop(nir_builder *b, InstrSubSet &ir_set for (auto k = ir_sorted_set.begin() + 1; k != ir_sorted_set.end(); ++k) { nir_intrinsic_instr *intr2 = *k; - nir_variable *var2 = - nir_deref_instr_get_variable(nir_src_as_deref(intr2->src[0])); + nir_variable *var2 = nir_deref_instr_get_variable(nir_src_as_deref(intr2->src[0])); unsigned loc2 = var->data.location - m_base_slot; if (m_vars[loc][var->data.location_frac] != @@ -373,7 +394,7 @@ bool NirLowerIOToVector::vec_instr_stack_pop(nir_builder *b, InstrSubSet &ir_set continue; } - assert(glsl_get_vector_elements(glsl_without_array(var2->type)) < 4); + assert(glsl_get_vector_elements(glsl_without_array(var2->type)) < 4); if (srcs[var2->data.location_frac] == &instr_undef->def) { assert(intr2->src[1].is_ssa); @@ -383,24 +404,27 @@ bool NirLowerIOToVector::vec_instr_stack_pop(nir_builder *b, InstrSubSet &ir_set nir_instr_remove(&intr2->instr); } - create_new_io(b, intr, new_var, srcs, new_var->data.location_frac, - num_comps); + create_new_io(b, intr, new_var, srcs, new_var->data.location_frac, num_comps); return true; } -nir_variable_mode NirLowerFSOutToVector::get_io_mode(nir_shader *shader) const +nir_variable_mode +NirLowerFSOutToVector::get_io_mode(nir_shader *shader) const { return nir_var_shader_out; } void -NirLowerFSOutToVector::create_new_io(nir_builder *b, nir_intrinsic_instr *intr, nir_variable *var, - nir_ssa_def **srcs, unsigned first_comp, unsigned num_comps) +NirLowerFSOutToVector::create_new_io(nir_builder *b, + nir_intrinsic_instr *intr, + nir_variable *var, + nir_ssa_def **srcs, + unsigned first_comp, + unsigned num_comps) { b->cursor = nir_before_instr(&intr->instr); - nir_intrinsic_instr *new_intr = - nir_intrinsic_instr_create(b->shader, intr->intrinsic); + nir_intrinsic_instr *new_intr = nir_intrinsic_instr_create(b->shader, intr->intrinsic); new_intr->num_components = num_comps; nir_intrinsic_set_write_mask(new_intr, (1 << num_comps) - 1); @@ -409,7 +433,8 @@ NirLowerFSOutToVector::create_new_io(nir_builder *b, nir_intrinsic_instr *intr, deref = clone_deref_array(b, deref, nir_src_as_deref(intr->src[0])); new_intr->src[0] = nir_src_for_ssa(&deref->dest.ssa); - new_intr->src[1] = nir_src_for_ssa(create_combined_vector(b, srcs, first_comp, num_comps)); + new_intr->src[1] = + nir_src_for_ssa(create_combined_vector(b, srcs, first_comp, num_comps)); nir_builder_instr_insert(b, &new_intr->instr); @@ -417,7 +442,8 @@ NirLowerFSOutToVector::create_new_io(nir_builder *b, nir_intrinsic_instr *intr, nir_instr_remove(&intr->instr); } -bool NirLowerFSOutToVector::instr_can_rewrite_type(nir_intrinsic_instr *intr) const +bool +NirLowerFSOutToVector::instr_can_rewrite_type(nir_intrinsic_instr *intr) const { if (intr->intrinsic != nir_intrinsic_store_deref) return false; @@ -429,26 +455,35 @@ bool NirLowerFSOutToVector::instr_can_rewrite_type(nir_intrinsic_instr *intr) co return var_can_rewrite(nir_deref_instr_get_variable(deref)); } -nir_ssa_def *NirLowerFSOutToVector::create_combined_vector(nir_builder *b, nir_ssa_def **srcs, - int first_comp, int num_comp) +nir_ssa_def * +NirLowerFSOutToVector::create_combined_vector(nir_builder *b, + nir_ssa_def **srcs, + int first_comp, + int num_comp) { nir_op op; switch (num_comp) { - case 2: op = nir_op_vec2; break; - case 3: op = nir_op_vec3; break; - case 4: op = nir_op_vec4; break; + case 2: + op = nir_op_vec2; + break; + case 3: + op = nir_op_vec3; + break; + case 4: + op = nir_op_vec4; + break; default: unreachable("combined vector must have 2 to 4 components"); } - nir_alu_instr * instr = nir_alu_instr_create(b->shader, op); + nir_alu_instr *instr = nir_alu_instr_create(b->shader, op); instr->exact = b->exact; int i = 0; unsigned k = 0; while (i < num_comp) { nir_ssa_def *s = srcs[first_comp + k]; - for(uint8_t kk = 0; kk < s->num_components && i < num_comp; ++kk) { - instr->src[i].src = nir_src_for_ssa(s); + for (uint8_t kk = 0; kk < s->num_components && i < num_comp; ++kk) { + instr->src[i].src = nir_src_for_ssa(s); instr->src[i].swizzle[0] = kk; ++i; } @@ -461,4 +496,4 @@ nir_ssa_def *NirLowerFSOutToVector::create_combined_vector(nir_builder *b, nir_s return &instr->dest.dest.ssa; } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_nir_lower_fs_out_to_vector.h b/src/gallium/drivers/r600/sfn/sfn_nir_lower_fs_out_to_vector.h index 016b7a22203..ce650f2ac95 100644 --- a/src/gallium/drivers/r600/sfn/sfn_nir_lower_fs_out_to_vector.h +++ b/src/gallium/drivers/r600/sfn/sfn_nir_lower_fs_out_to_vector.h @@ -31,7 +31,8 @@ namespace r600 { -bool r600_lower_fs_out_to_vector(nir_shader *sh); +bool +r600_lower_fs_out_to_vector(nir_shader *sh); } diff --git a/src/gallium/drivers/r600/sfn/sfn_nir_lower_tess_io.cpp b/src/gallium/drivers/r600/sfn/sfn_nir_lower_tess_io.cpp index c89d3a21b5c..886b64c9ea1 100644 --- a/src/gallium/drivers/r600/sfn/sfn_nir_lower_tess_io.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_nir_lower_tess_io.cpp @@ -26,7 +26,8 @@ #include "sfn_nir.h" -bool r600_lower_tess_io_filter(const nir_instr *instr, gl_shader_stage stage) +bool +r600_lower_tess_io_filter(const nir_instr *instr, gl_shader_stage stage) { if (instr->type != nir_instr_type_intrinsic) return false; @@ -45,8 +46,7 @@ bool r600_lower_tess_io_filter(const nir_instr *instr, gl_shader_stage stage) return true; case nir_intrinsic_store_output: return stage == MESA_SHADER_TESS_CTRL || stage == MESA_SHADER_VERTEX; - default: - ; + default:; } return false; } @@ -55,13 +55,13 @@ static nir_ssa_def * emit_load_param_base(nir_builder *b, nir_intrinsic_op op) { nir_intrinsic_instr *result = nir_intrinsic_instr_create(b->shader, op); - nir_ssa_dest_init(&result->instr, &result->dest, - 4, 32, NULL); + nir_ssa_dest_init(&result->instr, &result->dest, 4, 32, NULL); nir_builder_instr_insert(b, &result->instr); return &result->dest.ssa; } -static int get_tcs_varying_offset(nir_intrinsic_instr *op) +static int +get_tcs_varying_offset(nir_intrinsic_instr *op) { unsigned location = nir_intrinsic_io_semantics(op).location; @@ -89,11 +89,10 @@ static int get_tcs_varying_offset(nir_intrinsic_instr *op) case VARYING_SLOT_TESS_LEVEL_INNER: return 0x10; default: - if (location >= VARYING_SLOT_VAR0 && - location <= VARYING_SLOT_VAR31) + if (location >= VARYING_SLOT_VAR0 && location <= VARYING_SLOT_VAR31) return 0x10 * (location - VARYING_SLOT_VAR0) + 0x90; - if (location >= VARYING_SLOT_PATCH0) { + if (location >= VARYING_SLOT_PATCH0) { return 0x10 * (location - VARYING_SLOT_PATCH0) + 0x20; } } @@ -109,23 +108,24 @@ r600_umad_24(nir_builder *b, nir_ssa_def *op1, nir_ssa_def *op2, nir_ssa_def *op static inline nir_ssa_def * r600_tcs_base_address(nir_builder *b, nir_ssa_def *param_base, nir_ssa_def *rel_patch_id) { - return r600_umad_24(b, nir_channel(b, param_base, 0), + return r600_umad_24(b, + nir_channel(b, param_base, 0), rel_patch_id, nir_channel(b, param_base, 3)); } - static nir_ssa_def * -emil_lsd_in_addr(nir_builder *b, nir_ssa_def *base, nir_ssa_def *patch_id, nir_intrinsic_instr *op) +emil_lsd_in_addr(nir_builder *b, + nir_ssa_def *base, + nir_ssa_def *patch_id, + nir_intrinsic_instr *op) { - nir_ssa_def *addr = nir_build_alu(b, nir_op_umul24, - nir_channel(b, base, 0), - patch_id, NULL, NULL); + nir_ssa_def *addr = + nir_build_alu(b, nir_op_umul24, nir_channel(b, base, 0), patch_id, NULL, NULL); auto idx1 = nir_src_as_const_value(op->src[0]); if (!idx1 || idx1->u32 != 0) - addr = r600_umad_24(b, nir_channel(b, base, 1), - op->src[0].ssa, addr); + addr = r600_umad_24(b, nir_channel(b, base, 1), op->src[0].ssa, addr); auto offset = nir_imm_int(b, get_tcs_varying_offset(op)); @@ -137,39 +137,52 @@ emil_lsd_in_addr(nir_builder *b, nir_ssa_def *base, nir_ssa_def *patch_id, nir_i } static nir_ssa_def * -emil_lsd_out_addr(nir_builder *b, nir_ssa_def *base, nir_ssa_def *patch_id, nir_intrinsic_instr *op, - UNUSED nir_variable_mode mode, int src_offset) +emil_lsd_out_addr(nir_builder *b, + nir_ssa_def *base, + nir_ssa_def *patch_id, + nir_intrinsic_instr *op, + UNUSED nir_variable_mode mode, + int src_offset) { - nir_ssa_def *addr1 = r600_umad_24(b, nir_channel(b, base, 0), - patch_id, - nir_channel(b, base, 2)); - nir_ssa_def *addr2 = r600_umad_24(b, nir_channel(b, base, 1), - op->src[src_offset].ssa, addr1); + nir_ssa_def *addr1 = + r600_umad_24(b, nir_channel(b, base, 0), patch_id, nir_channel(b, base, 2)); + nir_ssa_def *addr2 = + r600_umad_24(b, nir_channel(b, base, 1), op->src[src_offset].ssa, addr1); int offset = get_tcs_varying_offset(op); - return nir_iadd(b, nir_iadd(b, addr2, - nir_ishl(b, op->src[src_offset + 1].ssa, nir_imm_int(b,4))), - nir_imm_int(b, offset)); + return nir_iadd(b, + nir_iadd(b, + addr2, + nir_ishl(b, op->src[src_offset + 1].ssa, nir_imm_int(b, 4))), + nir_imm_int(b, offset)); } -static nir_ssa_def *load_offset_group(nir_builder *b, int ncomponents) +static nir_ssa_def * +load_offset_group(nir_builder *b, int ncomponents) { switch (ncomponents) { /* tess outer offsets */ - case 1: return nir_imm_int(b, 0); - case 2: return nir_imm_ivec2(b, 0, 4); - case 3: return r600_imm_ivec3(b, 0, 4, 8); - case 4: return nir_imm_ivec4(b, 0, 4, 8, 12); + case 1: + return nir_imm_int(b, 0); + case 2: + return nir_imm_ivec2(b, 0, 4); + case 3: + return r600_imm_ivec3(b, 0, 4, 8); + case 4: + return nir_imm_ivec4(b, 0, 4, 8, 12); /* tess inner offsets */ - case 5: return nir_imm_int(b, 16); - case 6: return nir_imm_ivec2(b, 16, 20); + case 5: + return nir_imm_int(b, 16); + case 6: + return nir_imm_ivec2(b, 16, 20); default: debug_printf("Got %d components\n", ncomponents); unreachable("Unsupported component count"); } } -static nir_ssa_def *load_offset_group_from_mask(nir_builder *b, uint32_t mask) +static nir_ssa_def * +load_offset_group_from_mask(nir_builder *b, uint32_t mask) { auto full_mask = nir_imm_ivec4(b, 0, 4, 8, 12); return nir_channels(b, full_mask, mask); @@ -183,7 +196,8 @@ struct MaskQuery { uint32_t full_mask; }; -static bool update_alu_mask(nir_src *src, void *data) +static bool +update_alu_mask(nir_src *src, void *data) { auto mq = reinterpret_cast(data); @@ -195,14 +209,16 @@ static bool update_alu_mask(nir_src *src, void *data) return mq->mask != mq->full_mask; } -static uint32_t get_dest_usee_mask(nir_intrinsic_instr *op) +static uint32_t +get_dest_usee_mask(nir_intrinsic_instr *op) { assert(op->dest.is_ssa); MaskQuery mq = {0}; mq.full_mask = (1 << nir_dest_num_components(op->dest)) - 1; - nir_foreach_use(use_src, &op->dest.ssa) { + nir_foreach_use(use_src, &op->dest.ssa) + { auto use_instr = use_src->parent_instr; mq.ssa_index = use_src->ssa->index; @@ -214,7 +230,7 @@ static uint32_t get_dest_usee_mask(nir_intrinsic_instr *op) return 0xf; break; } - case nir_instr_type_intrinsic: { + case nir_instr_type_intrinsic: { auto intr = nir_instr_as_intrinsic(use_instr); switch (intr->intrinsic) { case nir_intrinsic_store_output: @@ -233,18 +249,19 @@ static uint32_t get_dest_usee_mask(nir_intrinsic_instr *op) default: return 0xf; } - } return mq.mask; } -static void replace_load_instr(nir_builder *b, nir_intrinsic_instr *op, nir_ssa_def *addr) +static void +replace_load_instr(nir_builder *b, nir_intrinsic_instr *op, nir_ssa_def *addr) { uint32_t mask = get_dest_usee_mask(op); if (mask) { nir_ssa_def *addr_outer = nir_iadd(b, addr, load_offset_group_from_mask(b, mask)); if (nir_intrinsic_component(op)) - addr_outer = nir_iadd(b, addr_outer, nir_imm_int(b, 4 * nir_intrinsic_component(op))); + addr_outer = + nir_iadd(b, addr_outer, nir_imm_int(b, 4 * nir_intrinsic_component(op))); auto new_load = nir_load_local_shared_r600(b, 32, addr_outer); @@ -267,9 +284,9 @@ static void replace_load_instr(nir_builder *b, nir_intrinsic_instr *op, nir_ssa_ static nir_ssa_def * r600_load_rel_patch_id(nir_builder *b) { - auto patch_id = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_tcs_rel_patch_id_r600); - nir_ssa_dest_init(&patch_id->instr, &patch_id->dest, - 1, 32, NULL); + auto patch_id = + nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_tcs_rel_patch_id_r600); + nir_ssa_dest_init(&patch_id->instr, &patch_id->dest, 1, 32, NULL); nir_builder_instr_insert(b, &patch_id->instr); return &patch_id->dest.ssa; } @@ -285,9 +302,10 @@ emit_store_lds(nir_builder *b, nir_intrinsic_instr *op, nir_ssa_def *addr) if (!(wmask)) continue; - uint32_t writemask = wmask >> nir_intrinsic_component(op); + uint32_t writemask = wmask >> nir_intrinsic_component(op); - auto store_tcs_out = nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_local_shared_r600); + auto store_tcs_out = + nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_local_shared_r600); nir_intrinsic_set_write_mask(store_tcs_out, writemask); store_tcs_out->src[0] = nir_src_for_ssa(op->src[0].ssa); store_tcs_out->num_components = store_tcs_out->src[0].ssa->num_components; @@ -301,29 +319,34 @@ emit_store_lds(nir_builder *b, nir_intrinsic_instr *op, nir_ssa_def *addr) } static nir_ssa_def * -emil_tcs_io_offset(nir_builder *b, nir_ssa_def *addr, nir_intrinsic_instr *op, int src_offset) +emil_tcs_io_offset(nir_builder *b, + nir_ssa_def *addr, + nir_intrinsic_instr *op, + int src_offset) { int offset = get_tcs_varying_offset(op); - return nir_iadd(b, nir_iadd(b, addr, - nir_ishl(b, op->src[src_offset].ssa, nir_imm_int(b,4))), - nir_imm_int(b, offset)); + return nir_iadd(b, + nir_iadd(b, + addr, + nir_ishl(b, op->src[src_offset].ssa, nir_imm_int(b, 4))), + nir_imm_int(b, offset)); } - inline unsigned outer_tf_components(pipe_prim_type prim_type) { switch (prim_type) { - case PIPE_PRIM_LINES: return 2; - case PIPE_PRIM_TRIANGLES: return 3; - case PIPE_PRIM_QUADS: return 4; + case PIPE_PRIM_LINES: + return 2; + case PIPE_PRIM_TRIANGLES: + return 3; + case PIPE_PRIM_QUADS: + return 4; default: return 0; } } - - static bool r600_lower_tess_io_impl(nir_builder *b, nir_instr *instr, enum pipe_prim_type prim_type) { @@ -334,12 +357,16 @@ r600_lower_tess_io_impl(nir_builder *b, nir_instr *instr, enum pipe_prim_type pr nir_intrinsic_instr *op = nir_instr_as_intrinsic(instr); if (b->shader->info.stage == MESA_SHADER_TESS_CTRL) { - load_in_param_base = emit_load_param_base(b, nir_intrinsic_load_tcs_in_param_base_r600); - load_out_param_base = emit_load_param_base(b, nir_intrinsic_load_tcs_out_param_base_r600); + load_in_param_base = + emit_load_param_base(b, nir_intrinsic_load_tcs_in_param_base_r600); + load_out_param_base = + emit_load_param_base(b, nir_intrinsic_load_tcs_out_param_base_r600); } else if (b->shader->info.stage == MESA_SHADER_TESS_EVAL) { - load_in_param_base = emit_load_param_base(b, nir_intrinsic_load_tcs_out_param_base_r600); + load_in_param_base = + emit_load_param_base(b, nir_intrinsic_load_tcs_out_param_base_r600); } else if (b->shader->info.stage == MESA_SHADER_VERTEX) { - load_out_param_base = emit_load_param_base(b, nir_intrinsic_load_tcs_in_param_base_r600); + load_out_param_base = + emit_load_param_base(b, nir_intrinsic_load_tcs_in_param_base_r600); } auto rel_patch_id = r600_load_rel_patch_id(b); @@ -362,29 +389,35 @@ r600_lower_tess_io_impl(nir_builder *b, nir_instr *instr, enum pipe_prim_type pr } case nir_intrinsic_load_per_vertex_input: { nir_ssa_def *addr = - b->shader->info.stage == MESA_SHADER_TESS_CTRL ? - emil_lsd_in_addr(b, load_in_param_base, rel_patch_id, op) : - emil_lsd_out_addr(b, load_in_param_base, rel_patch_id, op, nir_var_shader_in, 0); + b->shader->info.stage == MESA_SHADER_TESS_CTRL + ? emil_lsd_in_addr(b, load_in_param_base, rel_patch_id, op) + : emil_lsd_out_addr( + b, load_in_param_base, rel_patch_id, op, nir_var_shader_in, 0); replace_load_instr(b, op, addr); return true; } case nir_intrinsic_store_per_vertex_output: { - nir_ssa_def *addr = emil_lsd_out_addr(b, load_out_param_base, rel_patch_id, op, nir_var_shader_out, 1); + nir_ssa_def *addr = emil_lsd_out_addr( + b, load_out_param_base, rel_patch_id, op, nir_var_shader_out, 1); emit_store_lds(b, op, addr); nir_instr_remove(instr); return true; } case nir_intrinsic_load_per_vertex_output: { - nir_ssa_def *addr = emil_lsd_out_addr(b, load_out_param_base, rel_patch_id, op, nir_var_shader_out, 0); + nir_ssa_def *addr = emil_lsd_out_addr( + b, load_out_param_base, rel_patch_id, op, nir_var_shader_out, 0); replace_load_instr(b, op, addr); return true; } case nir_intrinsic_store_output: { - nir_ssa_def *addr = (b->shader->info.stage == MESA_SHADER_TESS_CTRL) ? - r600_tcs_base_address(b, load_out_param_base, rel_patch_id): - nir_build_alu(b, nir_op_umul24, - nir_channel(b, load_out_param_base, 1), - rel_patch_id, NULL, NULL); + nir_ssa_def *addr = (b->shader->info.stage == MESA_SHADER_TESS_CTRL) + ? r600_tcs_base_address(b, load_out_param_base, rel_patch_id) + : nir_build_alu(b, + nir_op_umul24, + nir_channel(b, load_out_param_base, 1), + rel_patch_id, + NULL, + NULL); addr = emil_tcs_io_offset(b, addr, op, 1); emit_store_lds(b, op, addr); nir_instr_remove(instr); @@ -414,13 +447,14 @@ r600_lower_tess_io_impl(nir_builder *b, nir_instr *instr, enum pipe_prim_type pr auto base = emit_load_param_base(b, nir_intrinsic_load_tcs_out_param_base_r600); auto rel_patch_id = r600_load_rel_patch_id(b); nir_ssa_def *addr0 = r600_tcs_base_address(b, base, rel_patch_id); - nir_ssa_def *addr_outer = nir_iadd(b, addr0, load_offset_group(b, tf_inner_address_offset + ncomps)); + nir_ssa_def *addr_outer = + nir_iadd(b, addr0, load_offset_group(b, tf_inner_address_offset + ncomps)); - auto tf = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_local_shared_r600); + auto tf = + nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_local_shared_r600); tf->num_components = ncomps; tf->src[0] = nir_src_for_ssa(addr_outer); - nir_ssa_dest_init(&tf->instr, &tf->dest, - tf->num_components, 32, NULL); + nir_ssa_dest_init(&tf->instr, &tf->dest, tf->num_components, 32, NULL); nir_builder_instr_insert(b, &tf->instr); if (ncomps < 4) { auto undef = nir_ssa_undef(b, 1, 32); @@ -435,23 +469,26 @@ r600_lower_tess_io_impl(nir_builder *b, nir_instr *instr, enum pipe_prim_type pr nir_instr_remove(instr); return true; } - default: - ; + default:; } return false; } -bool r600_lower_tess_io(nir_shader *shader, enum pipe_prim_type prim_type) +bool +r600_lower_tess_io(nir_shader *shader, enum pipe_prim_type prim_type) { bool progress = false; - nir_foreach_function(function, shader) { + nir_foreach_function(function, shader) + { if (function->impl) { nir_builder b; nir_builder_init(&b, function->impl); - nir_foreach_block(block, function->impl) { - nir_foreach_instr_safe(instr, block) { + nir_foreach_block(block, function->impl) + { + nir_foreach_instr_safe(instr, block) + { if (instr->type != nir_instr_type_intrinsic) continue; @@ -464,22 +501,29 @@ bool r600_lower_tess_io(nir_shader *shader, enum pipe_prim_type prim_type) return progress; } -bool r600_emit_tf(nir_builder *b, nir_ssa_def *val) +bool +r600_emit_tf(nir_builder *b, nir_ssa_def *val) { - nir_intrinsic_instr *store_tf = nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_tf_r600); + nir_intrinsic_instr *store_tf = + nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_tf_r600); store_tf->num_components = val->num_components; store_tf->src[0] = nir_src_for_ssa(val); nir_builder_instr_insert(b, &store_tf->instr); return true; } -bool r600_append_tcs_TF_emission(nir_shader *shader, enum pipe_prim_type prim_type) { +bool +r600_append_tcs_TF_emission(nir_shader *shader, enum pipe_prim_type prim_type) +{ if (shader->info.stage != MESA_SHADER_TESS_CTRL) return false; - nir_foreach_function(function, shader) { - nir_foreach_block(block, function->impl) { - nir_foreach_instr_safe(instr, block) { + nir_foreach_function(function, shader) + { + nir_foreach_block(block, function->impl) + { + nir_foreach_instr_safe(instr, block) + { if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); @@ -505,9 +549,9 @@ bool r600_append_tcs_TF_emission(nir_shader *shader, enum pipe_prim_type prim_ty b->cursor = nir_after_cf_list(&f->impl->body); - auto invocation_id = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_invocation_id); - nir_ssa_dest_init(&invocation_id->instr, &invocation_id->dest, - 1, 32, NULL); + auto invocation_id = + nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_invocation_id); + nir_ssa_dest_init(&invocation_id->instr, &invocation_id->dest, 1, 32, NULL); nir_builder_instr_insert(b, &invocation_id->instr); nir_push_if(b, nir_ieq_imm(b, &invocation_id->dest.ssa, 0)); @@ -517,22 +561,23 @@ bool r600_append_tcs_TF_emission(nir_shader *shader, enum pipe_prim_type prim_ty nir_ssa_def *addr0 = r600_tcs_base_address(b, base, rel_patch_id); nir_ssa_def *addr_outer = nir_iadd(b, addr0, load_offset_group(b, outer_comps)); - auto tf_outer = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_local_shared_r600); + auto tf_outer = + nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_local_shared_r600); tf_outer->num_components = outer_comps; tf_outer->src[0] = nir_src_for_ssa(addr_outer); - nir_ssa_dest_init(&tf_outer->instr, &tf_outer->dest, - tf_outer->num_components, 32, NULL); + nir_ssa_dest_init( + &tf_outer->instr, &tf_outer->dest, tf_outer->num_components, 32, NULL); nir_builder_instr_insert(b, &tf_outer->instr); std::vector tf_out; - - auto tf_out_base = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_tcs_tess_factor_base_r600); - nir_ssa_dest_init(&tf_out_base->instr, &tf_out_base->dest, - 1, 32, NULL); + auto tf_out_base = + nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_tcs_tess_factor_base_r600); + nir_ssa_dest_init(&tf_out_base->instr, &tf_out_base->dest, 1, 32, NULL); nir_builder_instr_insert(b, &tf_out_base->instr); - auto out_addr0 = nir_build_alu(b, nir_op_umad24, + auto out_addr0 = nir_build_alu(b, + nir_op_umad24, rel_patch_id, nir_imm_int(b, stride), &tf_out_base->dest.ssa, @@ -543,41 +588,47 @@ bool r600_append_tcs_TF_emission(nir_shader *shader, enum pipe_prim_type prim_ty if (prim_type == PIPE_PRIM_LINES) std::swap(chanx, chany); - - auto v0 = nir_vec4(b, out_addr0, nir_channel(b, &tf_outer->dest.ssa, chanx), + auto v0 = nir_vec4(b, + out_addr0, + nir_channel(b, &tf_outer->dest.ssa, chanx), nir_iadd(b, out_addr0, nir_imm_int(b, 4)), nir_channel(b, &tf_outer->dest.ssa, chany)); tf_out.push_back(v0); if (outer_comps > 2) { - auto v1 = (outer_comps > 3) ? nir_vec4(b, nir_iadd(b, out_addr0, nir_imm_int(b, 8)), + auto v1 = (outer_comps > 3) ? nir_vec4(b, + nir_iadd(b, out_addr0, nir_imm_int(b, 8)), nir_channel(b, &tf_outer->dest.ssa, 2), nir_iadd(b, out_addr0, nir_imm_int(b, 12)), - nir_channel(b, &tf_outer->dest.ssa, 3)) : - nir_vec2(b, nir_iadd(b, out_addr0, nir_imm_int(b, 8)), + nir_channel(b, &tf_outer->dest.ssa, 3)) + : nir_vec2(b, + nir_iadd(b, out_addr0, nir_imm_int(b, 8)), nir_channel(b, &tf_outer->dest.ssa, 2)); tf_out.push_back(v1); } if (inner_comps) { nir_ssa_def *addr1 = nir_iadd(b, addr0, load_offset_group(b, 4 + inner_comps)); - auto tf_inner = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_local_shared_r600); + auto tf_inner = + nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_local_shared_r600); tf_inner->num_components = inner_comps; tf_inner->src[0] = nir_src_for_ssa(addr1); - nir_ssa_dest_init(&tf_inner->instr, &tf_inner->dest, - tf_inner->num_components, 32, NULL); + nir_ssa_dest_init( + &tf_inner->instr, &tf_inner->dest, tf_inner->num_components, 32, NULL); nir_builder_instr_insert(b, &tf_inner->instr); - auto v2 = (inner_comps > 1) ? nir_vec4(b, nir_iadd(b, out_addr0, nir_imm_int(b, 16)), + auto v2 = (inner_comps > 1) ? nir_vec4(b, + nir_iadd(b, out_addr0, nir_imm_int(b, 16)), nir_channel(b, &tf_inner->dest.ssa, 0), nir_iadd(b, out_addr0, nir_imm_int(b, 20)), - nir_channel(b, &tf_inner->dest.ssa, 1)): - nir_vec2(b, nir_iadd(b, out_addr0, nir_imm_int(b, 12)), + nir_channel(b, &tf_inner->dest.ssa, 1)) + : nir_vec2(b, + nir_iadd(b, out_addr0, nir_imm_int(b, 12)), nir_channel(b, &tf_inner->dest.ssa, 0)); tf_out.push_back(v2); } - for (auto tf: tf_out) + for (auto tf : tf_out) r600_emit_tf(b, tf); nir_pop_if(b, nullptr); @@ -607,15 +658,19 @@ r600_lower_tess_coord_impl(nir_builder *b, UNUSED nir_instr *instr, void *_optio auto tc_y = nir_channel(b, tc_xy, 1); if (prim_type == PIPE_PRIM_TRIANGLES) - return nir_vec3(b, tc_x, tc_y, nir_fsub(b, nir_imm_float(b, 1.0), - nir_fadd(b, tc_x, tc_y))); + return nir_vec3(b, + tc_x, + tc_y, + nir_fsub(b, nir_imm_float(b, 1.0), nir_fadd(b, tc_x, tc_y))); else return nir_vec3(b, tc_x, tc_y, nir_imm_float(b, 0.0)); } - -bool r600_lower_tess_coord(nir_shader *sh, enum pipe_prim_type prim_type) +bool +r600_lower_tess_coord(nir_shader *sh, enum pipe_prim_type prim_type) { - return nir_shader_lower_instructions(sh, r600_lower_tess_coord_filter, - r600_lower_tess_coord_impl, &prim_type); + return nir_shader_lower_instructions(sh, + r600_lower_tess_coord_filter, + r600_lower_tess_coord_impl, + &prim_type); } diff --git a/src/gallium/drivers/r600/sfn/sfn_nir_lower_tex.cpp b/src/gallium/drivers/r600/sfn/sfn_nir_lower_tex.cpp index 26b8bb84bf1..c9ca3468852 100644 --- a/src/gallium/drivers/r600/sfn/sfn_nir_lower_tex.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_nir_lower_tex.cpp @@ -30,48 +30,52 @@ #include "nir_builder.h" #include "nir_builtin_builder.h" -static bool lower_coord_shift_normalized(nir_builder *b, nir_tex_instr *tex) +static bool +lower_coord_shift_normalized(nir_builder *b, nir_tex_instr *tex) { b->cursor = nir_before_instr(&tex->instr); - nir_ssa_def * size = nir_i2f32(b, nir_get_texture_size(b, tex)); + nir_ssa_def *size = nir_i2f32(b, nir_get_texture_size(b, tex)); nir_ssa_def *scale = nir_frcp(b, size); int coord_index = nir_tex_instr_src_index(tex, nir_tex_src_coord); nir_ssa_def *corr = nullptr; if (unlikely(tex->array_is_lowered_cube)) { - auto corr2 = nir_fadd(b, nir_channels(b, tex->src[coord_index].src.ssa, 3), + auto corr2 = nir_fadd(b, + nir_channels(b, tex->src[coord_index].src.ssa, 3), nir_fmul(b, nir_imm_float(b, -0.5f), scale)); - corr = nir_vec3(b, nir_channel(b, corr2, 0), nir_channel(b, corr2, 1), - nir_channel( - b, tex->src[coord_index].src.ssa, 2)); + corr = nir_vec3(b, + nir_channel(b, corr2, 0), + nir_channel(b, corr2, 1), + nir_channel(b, tex->src[coord_index].src.ssa, 2)); } else { corr = nir_fadd(b, nir_fmul(b, nir_imm_float(b, -0.5f), scale), tex->src[coord_index].src.ssa); } - nir_instr_rewrite_src(&tex->instr, &tex->src[coord_index].src, - nir_src_for_ssa(corr)); + nir_instr_rewrite_src(&tex->instr, &tex->src[coord_index].src, nir_src_for_ssa(corr)); return true; } -static bool lower_coord_shift_unnormalized(nir_builder *b, nir_tex_instr *tex) +static bool +lower_coord_shift_unnormalized(nir_builder *b, nir_tex_instr *tex) { b->cursor = nir_before_instr(&tex->instr); int coord_index = nir_tex_instr_src_index(tex, nir_tex_src_coord); nir_ssa_def *corr = nullptr; if (unlikely(tex->array_is_lowered_cube)) { - auto corr2 = nir_fadd(b, nir_channels(b, tex->src[coord_index].src.ssa, 3), + auto corr2 = nir_fadd(b, + nir_channels(b, tex->src[coord_index].src.ssa, 3), nir_imm_float(b, -0.5f)); - corr = nir_vec3(b, nir_channel(b, corr2, 0), nir_channel(b, corr2, 1), + corr = nir_vec3(b, + nir_channel(b, corr2, 0), + nir_channel(b, corr2, 1), nir_channel(b, tex->src[coord_index].src.ssa, 2)); } else { - corr = nir_fadd(b, tex->src[coord_index].src.ssa, - nir_imm_float(b, -0.5f)); + corr = nir_fadd(b, tex->src[coord_index].src.ssa, nir_imm_float(b, -0.5f)); } - nir_instr_rewrite_src(&tex->instr, &tex->src[coord_index].src, - nir_src_for_ssa(corr)); + nir_instr_rewrite_src(&tex->instr, &tex->src[coord_index].src, nir_src_for_ssa(corr)); return true; } @@ -82,12 +86,13 @@ r600_nir_lower_int_tg4_impl(nir_function_impl *impl) nir_builder_init(&b, impl); bool progress = false; - nir_foreach_block(block, impl) { - nir_foreach_instr_safe(instr, block) { + nir_foreach_block(block, impl) + { + nir_foreach_instr_safe(instr, block) + { if (instr->type == nir_instr_type_tex) { nir_tex_instr *tex = nir_instr_as_tex(instr); - if (tex->op == nir_texop_tg4 && - tex->sampler_dim != GLSL_SAMPLER_DIM_CUBE && + if (tex->op == nir_texop_tg4 && tex->sampler_dim != GLSL_SAMPLER_DIM_CUBE && nir_tex_instr_src_index(tex, nir_tex_src_backend1) < 0) { if (nir_alu_type_get_base_type(tex->dest_type) != nir_type_float) { if (tex->sampler_dim != GLSL_SAMPLER_DIM_RECT) @@ -114,12 +119,14 @@ r600_nir_lower_int_tg4_impl(nir_function_impl *impl) * the texel size. */ -bool r600_nir_lower_int_tg4(nir_shader *shader) +bool +r600_nir_lower_int_tg4(nir_shader *shader) { bool progress = false; bool need_lowering = false; - nir_foreach_uniform_variable(var, shader) { + nir_foreach_uniform_variable(var, shader) + { if (var->type->is_sampler()) { if (glsl_base_type_is_integer(var->type->sampled_type)) { need_lowering = true; @@ -128,7 +135,8 @@ bool r600_nir_lower_int_tg4(nir_shader *shader) } if (need_lowering) { - nir_foreach_function(function, shader) { + nir_foreach_function(function, shader) + { if (function->impl && r600_nir_lower_int_tg4_impl(function->impl)) progress = true; } @@ -137,8 +145,8 @@ bool r600_nir_lower_int_tg4(nir_shader *shader) return progress; } -static -bool lower_txl_txf_array_or_cube(nir_builder *b, nir_tex_instr *tex) +static bool +lower_txl_txf_array_or_cube(nir_builder *b, nir_tex_instr *tex) { assert(tex->op == nir_texop_txb || tex->op == nir_texop_txl); assert(nir_tex_instr_src_index(tex, nir_tex_src_ddx) < 0); @@ -149,32 +157,30 @@ bool lower_txl_txf_array_or_cube(nir_builder *b, nir_tex_instr *tex) int lod_idx = nir_tex_instr_src_index(tex, nir_tex_src_lod); int bias_idx = nir_tex_instr_src_index(tex, nir_tex_src_bias); int min_lod_idx = nir_tex_instr_src_index(tex, nir_tex_src_min_lod); - assert (lod_idx >= 0 || bias_idx >= 0); + assert(lod_idx >= 0 || bias_idx >= 0); nir_ssa_def *size = nir_i2f32(b, nir_get_texture_size(b, tex)); - nir_ssa_def *lod = (lod_idx >= 0) ? - nir_ssa_for_src(b, tex->src[lod_idx].src, 1) : - nir_get_texture_lod(b, tex); + nir_ssa_def *lod = (lod_idx >= 0) ? nir_ssa_for_src(b, tex->src[lod_idx].src, 1) + : nir_get_texture_lod(b, tex); if (bias_idx >= 0) - lod = nir_fadd(b, lod,nir_ssa_for_src(b, tex->src[bias_idx].src, 1)); + lod = nir_fadd(b, lod, nir_ssa_for_src(b, tex->src[bias_idx].src, 1)); if (min_lod_idx >= 0) lod = nir_fmax(b, lod, nir_ssa_for_src(b, tex->src[min_lod_idx].src, 1)); /* max lod? */ - nir_ssa_def *lambda_exp = nir_fexp2(b, lod); + nir_ssa_def *lambda_exp = nir_fexp2(b, lod); nir_ssa_def *scale = NULL; if (tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE) { - unsigned int swizzle[NIR_MAX_VEC_COMPONENTS] = {0,0,0,0}; - scale = nir_frcp(b, nir_channels(b, size, 1)); - scale = nir_swizzle(b, scale, swizzle, 3); - } else if (tex->is_array) { + unsigned int swizzle[NIR_MAX_VEC_COMPONENTS] = {0, 0, 0, 0}; + scale = nir_frcp(b, nir_channels(b, size, 1)); + scale = nir_swizzle(b, scale, swizzle, 3); + } else if (tex->is_array) { int cmp_mask = (1 << (size->num_components - 1)) - 1; - scale = nir_frcp(b, nir_channels(b, size, - (nir_component_mask_t)cmp_mask)); + scale = nir_frcp(b, nir_channels(b, size, (nir_component_mask_t)cmp_mask)); } nir_ssa_def *grad = nir_fmul(b, lambda_exp, scale); @@ -192,7 +198,6 @@ bool lower_txl_txf_array_or_cube(nir_builder *b, nir_tex_instr *tex) return true; } - static bool r600_nir_lower_txl_txf_array_or_cube_impl(nir_function_impl *impl) { @@ -200,8 +205,10 @@ r600_nir_lower_txl_txf_array_or_cube_impl(nir_function_impl *impl) nir_builder_init(&b, impl); bool progress = false; - nir_foreach_block(block, impl) { - nir_foreach_instr_safe(instr, block) { + nir_foreach_block(block, impl) + { + nir_foreach_instr_safe(instr, block) + { if (instr->type == nir_instr_type_tex) { nir_tex_instr *tex = nir_instr_as_tex(instr); @@ -219,7 +226,8 @@ bool r600_nir_lower_txl_txf_array_or_cube(nir_shader *shader) { bool progress = false; - nir_foreach_function(function, shader) { + nir_foreach_function(function, shader) + { if (function->impl && r600_nir_lower_txl_txf_array_or_cube_impl(function->impl)) progress = true; } @@ -268,23 +276,30 @@ r600_nir_lower_cube_to_2darray_impl(nir_builder *b, nir_instr *instr, void *_opt nir_ssa_def *z = nir_channel(b, cubed, 3); if (tex->is_array) { auto slice = nir_fround_even(b, nir_channel(b, tex->src[coord_idx].src.ssa, 3)); - z = nir_fmad(b, nir_fmax(b, slice, nir_imm_float(b, 0.0)), nir_imm_float(b, 8.0), - z); + z = + nir_fmad(b, nir_fmax(b, slice, nir_imm_float(b, 0.0)), nir_imm_float(b, 8.0), z); } if (tex->op == nir_texop_txd) { int ddx_idx = nir_tex_instr_src_index(tex, nir_tex_src_ddx); auto zero_dot_5 = nir_imm_float(b, 0.5); - nir_instr_rewrite_src(&tex->instr, &tex->src[ddx_idx].src, - nir_src_for_ssa(nir_fmul(b, nir_ssa_for_src(b, tex->src[ddx_idx].src, 3), zero_dot_5))); + nir_instr_rewrite_src( + &tex->instr, + &tex->src[ddx_idx].src, + nir_src_for_ssa( + nir_fmul(b, nir_ssa_for_src(b, tex->src[ddx_idx].src, 3), zero_dot_5))); int ddy_idx = nir_tex_instr_src_index(tex, nir_tex_src_ddy); - nir_instr_rewrite_src(&tex->instr, &tex->src[ddy_idx].src, - nir_src_for_ssa(nir_fmul(b, nir_ssa_for_src(b, tex->src[ddy_idx].src, 3), zero_dot_5))); + nir_instr_rewrite_src( + &tex->instr, + &tex->src[ddy_idx].src, + nir_src_for_ssa( + nir_fmul(b, nir_ssa_for_src(b, tex->src[ddy_idx].src, 3), zero_dot_5))); } auto new_coord = nir_vec3(b, nir_channel(b, xy, 0), nir_channel(b, xy, 1), z); - nir_instr_rewrite_src(&tex->instr, &tex->src[coord_idx].src, + nir_instr_rewrite_src(&tex->instr, + &tex->src[coord_idx].src, nir_src_for_ssa(new_coord)); tex->sampler_dim = GLSL_SAMPLER_DIM_2D; tex->is_array = true; @@ -300,5 +315,6 @@ r600_nir_lower_cube_to_2darray(nir_shader *shader) { return nir_shader_lower_instructions(shader, r600_nir_lower_cube_to_2darray_filer, - r600_nir_lower_cube_to_2darray_impl, nullptr); + r600_nir_lower_cube_to_2darray_impl, + nullptr); } diff --git a/src/gallium/drivers/r600/sfn/sfn_nir_lower_tex.h b/src/gallium/drivers/r600/sfn/sfn_nir_lower_tex.h index d195f88597d..885e3b29fad 100644 --- a/src/gallium/drivers/r600/sfn/sfn_nir_lower_tex.h +++ b/src/gallium/drivers/r600/sfn/sfn_nir_lower_tex.h @@ -29,8 +29,11 @@ struct nir_shader; -bool r600_nir_lower_int_tg4(nir_shader *nir); -bool r600_nir_lower_txl_txf_array_or_cube(nir_shader *shader); -bool r600_nir_lower_cube_to_2darray(nir_shader *shader); +bool +r600_nir_lower_int_tg4(nir_shader *nir); +bool +r600_nir_lower_txl_txf_array_or_cube(nir_shader *shader); +bool +r600_nir_lower_cube_to_2darray(nir_shader *shader); #endif // LALA_H diff --git a/src/gallium/drivers/r600/sfn/sfn_nir_vectorize_vs_inputs.c b/src/gallium/drivers/r600/sfn/sfn_nir_vectorize_vs_inputs.c index 35fd5cfffa1..fba5b5878ae 100644 --- a/src/gallium/drivers/r600/sfn/sfn_nir_vectorize_vs_inputs.c +++ b/src/gallium/drivers/r600/sfn/sfn_nir_vectorize_vs_inputs.c @@ -17,8 +17,8 @@ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. */ #include "nir.h" @@ -38,8 +38,9 @@ bool r600_vectorize_vs_inputs(nir_shader *shader); static nir_deref_instr * -r600_clone_deref_array(nir_builder *b, nir_deref_instr *dst_tail, - const nir_deref_instr *src_head) +r600_clone_deref_array(nir_builder *b, + nir_deref_instr *dst_tail, + const nir_deref_instr *src_head) { const nir_deref_instr *parent = nir_deref_instr_parent(src_head); @@ -50,8 +51,7 @@ r600_clone_deref_array(nir_builder *b, nir_deref_instr *dst_tail, dst_tail = r600_clone_deref_array(b, dst_tail, parent); - return nir_build_deref_array(b, dst_tail, - nir_ssa_for_src(b, src_head->arr.index, 1)); + return nir_build_deref_array(b, dst_tail, nir_ssa_for_src(b, src_head->arr.index, 1)); } static bool @@ -62,7 +62,6 @@ r600_variable_can_rewrite(nir_variable *var) if (!glsl_type_is_vector_or_scalar(glsl_without_array(var->type))) return false; - /* TODO: add 64/16bit support ? */ if (glsl_get_bit_size(glsl_without_array(var->type)) != 32) return false; @@ -102,10 +101,8 @@ r600_io_access_same_var(const nir_instr *instr1, const nir_instr *instr2) nir_intrinsic_instr *intr1 = nir_instr_as_intrinsic(instr1); nir_intrinsic_instr *intr2 = nir_instr_as_intrinsic(instr2); - nir_variable *var1 = - nir_deref_instr_get_variable(nir_src_as_deref(intr1->src[0])); - nir_variable *var2 = - nir_deref_instr_get_variable(nir_src_as_deref(intr2->src[0])); + nir_variable *var1 = nir_deref_instr_get_variable(nir_src_as_deref(intr1->src[0])); + nir_variable *var2 = nir_deref_instr_get_variable(nir_src_as_deref(intr2->src[0])); /* We don't handle combining vars of different base types, so skip those */ if (glsl_get_base_type(var1->type) != glsl_get_base_type(var2->type)) @@ -131,14 +128,19 @@ r600_vec_instr_stack_push(struct util_dynarray *stack, nir_instr *instr) util_dynarray_append(stack, nir_instr *, instr); } -static unsigned r600_correct_location(nir_variable *var) +static unsigned +r600_correct_location(nir_variable *var) { return var->data.location - VERT_ATTRIB_GENERIC0; } static void -r600_create_new_load(nir_builder *b, nir_intrinsic_instr *intr, nir_variable *var, - unsigned comp, unsigned num_comps, unsigned old_num_comps) +r600_create_new_load(nir_builder *b, + nir_intrinsic_instr *intr, + nir_variable *var, + unsigned comp, + unsigned num_comps, + unsigned old_num_comps) { unsigned channels[4]; @@ -146,10 +148,9 @@ r600_create_new_load(nir_builder *b, nir_intrinsic_instr *intr, nir_variable *va assert(intr->dest.is_ssa); - nir_intrinsic_instr *new_intr = - nir_intrinsic_instr_create(b->shader, intr->intrinsic); - nir_ssa_dest_init(&new_intr->instr, &new_intr->dest, num_comps, - intr->dest.ssa.bit_size, NULL); + nir_intrinsic_instr *new_intr = nir_intrinsic_instr_create(b->shader, intr->intrinsic); + nir_ssa_dest_init( + &new_intr->instr, &new_intr->dest, num_comps, intr->dest.ssa.bit_size, NULL); new_intr->num_components = num_comps; nir_deref_instr *deref = nir_build_deref_var(b, var); @@ -172,9 +173,9 @@ r600_create_new_load(nir_builder *b, nir_intrinsic_instr *intr, nir_variable *va nir_instr_remove(&intr->instr); } - static bool -r600_vec_instr_stack_pop(nir_builder *b, struct util_dynarray *stack, +r600_vec_instr_stack_pop(nir_builder *b, + struct util_dynarray *stack, nir_instr *instr, nir_variable *updated_vars[16][4]) { @@ -184,18 +185,15 @@ r600_vec_instr_stack_pop(nir_builder *b, struct util_dynarray *stack, assert(last->type == nir_instr_type_intrinsic); nir_intrinsic_instr *intr = nir_instr_as_intrinsic(last); - nir_variable *var = - nir_deref_instr_get_variable(nir_src_as_deref(intr->src[0])); + nir_variable *var = nir_deref_instr_get_variable(nir_src_as_deref(intr->src[0])); unsigned loc = r600_correct_location(var); nir_variable *new_var; new_var = updated_vars[loc][var->data.location_frac]; - unsigned num_comps = - glsl_get_vector_elements(glsl_without_array(new_var->type)); + unsigned num_comps = glsl_get_vector_elements(glsl_without_array(new_var->type)); - unsigned old_num_comps = - glsl_get_vector_elements(glsl_without_array(var->type)); + unsigned old_num_comps = glsl_get_vector_elements(glsl_without_array(var->type)); /* Don't bother walking the stack if this component can't be vectorised. */ if (old_num_comps > 3) { @@ -206,8 +204,8 @@ r600_vec_instr_stack_pop(nir_builder *b, struct util_dynarray *stack, return false; } - r600_create_new_load(b, intr, new_var, var->data.location_frac, - num_comps, old_num_comps); + r600_create_new_load( + b, intr, new_var, var->data.location_frac, num_comps, old_num_comps); return true; } @@ -231,8 +229,7 @@ r600_hash_instr(const nir_instr *instr) assert(instr->type == nir_instr_type_intrinsic); nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); - nir_variable *var = - nir_deref_instr_get_variable(nir_src_as_deref(intr->src[0])); + nir_variable *var = nir_deref_instr_get_variable(nir_src_as_deref(intr->src[0])); uint32_t hash = 0; @@ -274,7 +271,7 @@ r600_vec_instr_set_add(struct set *instr_set, nir_instr *instr) if (entry) { ralloc_free(new_stack); - struct util_dynarray *stack = (struct util_dynarray *) entry->key; + struct util_dynarray *stack = (struct util_dynarray *)entry->key; r600_vec_instr_stack_push(stack, instr); return; } @@ -285,7 +282,9 @@ r600_vec_instr_set_add(struct set *instr_set, nir_instr *instr) } static bool -r600_vec_instr_set_remove(nir_builder *b, struct set *instr_set, nir_instr *instr, +r600_vec_instr_set_remove(nir_builder *b, + struct set *instr_set, + nir_instr *instr, nir_variable *updated_vars[16][4]) { if (!r600_instr_can_rewrite(instr)) { @@ -308,7 +307,7 @@ r600_vec_instr_set_remove(nir_builder *b, struct set *instr_set, nir_instr *inst ralloc_free(temp); if (entry) { - struct util_dynarray *stack = (struct util_dynarray *) entry->key; + struct util_dynarray *stack = (struct util_dynarray *)entry->key; bool progress = r600_vec_instr_stack_pop(b, stack, instr, updated_vars); if (!util_dynarray_num_elements(stack, nir_instr *)) @@ -321,21 +320,22 @@ r600_vec_instr_set_remove(nir_builder *b, struct set *instr_set, nir_instr *inst } static bool -r600_vectorize_block(nir_builder *b, nir_block *block, struct set *instr_set, - nir_variable *updated_vars[16][4]) +r600_vectorize_block(nir_builder *b, + nir_block *block, + struct set *instr_set, + nir_variable *updated_vars[16][4]) { bool progress = false; - nir_foreach_instr_safe(instr, block) { - r600_vec_instr_set_add(instr_set, instr); - } + nir_foreach_instr_safe(instr, block) { r600_vec_instr_set_add(instr_set, instr); } for (unsigned i = 0; i < block->num_dom_children; i++) { nir_block *child = block->dom_children[i]; progress |= r600_vectorize_block(b, child, instr_set, updated_vars); } - nir_foreach_instr_reverse_safe(instr, block) { + nir_foreach_instr_reverse_safe(instr, block) + { progress |= r600_vec_instr_set_remove(b, instr_set, instr, updated_vars); } @@ -344,8 +344,9 @@ r600_vectorize_block(nir_builder *b, nir_block *block, struct set *instr_set, static void r600_create_new_io_var(nir_shader *shader, - nir_variable *vars[16][4], - unsigned location, unsigned comps) + nir_variable *vars[16][4], + unsigned location, + unsigned comps) { unsigned num_comps = util_bitcount(comps); assert(num_comps > 1); @@ -376,11 +377,13 @@ r600_variables_can_merge(const nir_variable *lhs, const nir_variable *rhs) } static void -r600_create_new_io_vars(nir_shader *shader, nir_variable_mode mode, - nir_variable *vars[16][4]) +r600_create_new_io_vars(nir_shader *shader, + nir_variable_mode mode, + nir_variable *vars[16][4]) { bool can_rewrite_vars = false; - nir_foreach_variable_with_modes(var, shader, mode) { + nir_foreach_variable_with_modes(var, shader, mode) + { if (r600_variable_can_rewrite(var)) { can_rewrite_vars = true; unsigned loc = r600_correct_location(var); @@ -415,7 +418,6 @@ r600_create_new_io_vars(nir_shader *shader, nir_variable_mode mode, for (unsigned n = 0; n < glsl_get_components(vars[i][k]->type); ++n) comps |= 1 << (vars[i][k]->data.location_frac + n); - } } if (comps) @@ -437,12 +439,11 @@ r600_vectorize_io_impl(nir_function_impl *impl) r600_create_new_io_vars(shader, nir_var_shader_in, updated_vars); struct set *instr_set = r600_vec_instr_set_create(); - bool progress = r600_vectorize_block(&b, nir_start_block(impl), instr_set, - updated_vars); + bool progress = + r600_vectorize_block(&b, nir_start_block(impl), instr_set, updated_vars); if (progress) { - nir_metadata_preserve(impl, nir_metadata_block_index | - nir_metadata_dominance); + nir_metadata_preserve(impl, nir_metadata_block_index | nir_metadata_dominance); } else { nir_metadata_preserve(impl, nir_metadata_all); } @@ -459,7 +460,8 @@ r600_vectorize_vs_inputs(nir_shader *shader) if (shader->info.stage != MESA_SHADER_VERTEX) return false; - nir_foreach_function(function, shader) { + nir_foreach_function(function, shader) + { if (function->impl) progress |= r600_vectorize_io_impl(function->impl); } diff --git a/src/gallium/drivers/r600/sfn/sfn_optimizer.cpp b/src/gallium/drivers/r600/sfn/sfn_optimizer.cpp index dbe911e8697..e74f486fd67 100644 --- a/src/gallium/drivers/r600/sfn/sfn_optimizer.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_optimizer.cpp @@ -26,24 +26,25 @@ #include "sfn_optimizer.h" +#include "sfn_debug.h" #include "sfn_instr_alugroup.h" #include "sfn_instr_controlflow.h" #include "sfn_instr_export.h" -#include "sfn_instr_tex.h" #include "sfn_instr_fetch.h" #include "sfn_instr_lds.h" +#include "sfn_instr_tex.h" #include "sfn_peephole.h" -#include "sfn_debug.h" #include namespace r600 { -bool optimize(Shader& shader) +bool +optimize(Shader& shader) { bool progress; - sfn_log << SfnLog::opt << "Shader before optimization\n"; + sfn_log << SfnLog::opt << "Shader before optimization\n"; if (sfn_log.has_debug_flag(SfnLog::opt)) { std::stringstream ss; shader.print(ss); @@ -70,28 +71,28 @@ public: void visit(AluInstr *instr) override; void visit(AluGroup *instr) override; - void visit(TexInstr *instr) override; - void visit(ExportInstr *instr) override {(void)instr;}; + void visit(TexInstr *instr) override; + void visit(ExportInstr *instr) override { (void)instr; }; void visit(FetchInstr *instr) override; void visit(Block *instr) override; - void visit(ControlFlowInstr *instr) override {(void)instr;}; - void visit(IfInstr *instr) override {(void)instr;}; - void visit(ScratchIOInstr *instr) override {(void)instr;}; - void visit(StreamOutInstr *instr) override {(void)instr;}; - void visit(MemRingOutInstr *instr) override {(void)instr;}; - void visit(EmitVertexInstr *instr) override {(void)instr;}; - void visit(GDSInstr *instr) override {(void)instr;}; - void visit(WriteTFInstr *instr) override {(void)instr;}; - void visit(LDSAtomicInstr *instr) override {(void)instr;}; + void visit(ControlFlowInstr *instr) override { (void)instr; }; + void visit(IfInstr *instr) override { (void)instr; }; + void visit(ScratchIOInstr *instr) override { (void)instr; }; + void visit(StreamOutInstr *instr) override { (void)instr; }; + void visit(MemRingOutInstr *instr) override { (void)instr; }; + void visit(EmitVertexInstr *instr) override { (void)instr; }; + void visit(GDSInstr *instr) override { (void)instr; }; + void visit(WriteTFInstr *instr) override { (void)instr; }; + void visit(LDSAtomicInstr *instr) override { (void)instr; }; void visit(LDSReadInstr *instr) override; - void visit(RatInstr *instr) override {(void)instr;}; - + void visit(RatInstr *instr) override { (void)instr; }; bool progress; }; -bool dead_code_elimination(Shader& shader) +bool +dead_code_elimination(Shader& shader) { DCEVisitor dce; @@ -105,9 +106,9 @@ bool dead_code_elimination(Shader& shader) sfn_log << SfnLog::opt << "finished dce run\n\n"; - } while (dce.progress); + } while (dce.progress); - sfn_log << SfnLog::opt << "Shader after DCE\n"; + sfn_log << SfnLog::opt << "Shader after DCE\n"; if (sfn_log.has_debug_flag(SfnLog::opt)) { std::stringstream ss; shader.print(ss); @@ -117,19 +118,20 @@ bool dead_code_elimination(Shader& shader) return dce.progress; } -DCEVisitor::DCEVisitor():progress(false) +DCEVisitor::DCEVisitor(): + progress(false) { } -void DCEVisitor::visit(AluInstr *instr) +void +DCEVisitor::visit(AluInstr *instr) { sfn_log << SfnLog::opt << "DCE: visit '" << *instr; if (instr->has_instr_flag(Instr::dead)) return; - if (instr->dest() && - (instr->dest()->has_uses()) ) { + if (instr->dest() && (instr->dest()->has_uses())) { sfn_log << SfnLog::opt << " dest used\n"; return; } @@ -148,8 +150,7 @@ void DCEVisitor::visit(AluInstr *instr) case op0_group_barrier: sfn_log << SfnLog::opt << " never kill\n"; return; - default: - ; + default:; } bool dead = instr->set_dead(); @@ -157,20 +158,23 @@ void DCEVisitor::visit(AluInstr *instr) progress |= dead; } -void DCEVisitor::visit(LDSReadInstr *instr) +void +DCEVisitor::visit(LDSReadInstr *instr) { sfn_log << SfnLog::opt << "visit " << *instr << "\n"; progress |= instr->remove_unused_components(); } -void DCEVisitor::visit(AluGroup *instr) +void +DCEVisitor::visit(AluGroup *instr) { /* Groups are created because the instructions are used together * so don't try to eliminate code there */ (void)instr; } -void DCEVisitor::visit(TexInstr *instr) +void +DCEVisitor::visit(TexInstr *instr) { auto& dest = instr->dst(); @@ -190,7 +194,8 @@ void DCEVisitor::visit(TexInstr *instr) progress |= instr->set_dead(); } -void DCEVisitor::visit(FetchInstr *instr) +void +DCEVisitor::visit(FetchInstr *instr) { auto& dest = instr->dst(); @@ -212,7 +217,8 @@ void DCEVisitor::visit(FetchInstr *instr) progress |= instr->set_dead(); } -void DCEVisitor::visit(Block *block) +void +DCEVisitor::visit(Block *block) { auto i = block->begin(); auto e = block->end(); @@ -237,26 +243,25 @@ public: void visit(ExportInstr *instr) override; void visit(FetchInstr *instr) override; void visit(Block *instr) override; - void visit(ControlFlowInstr *instr) override {(void)instr;} - void visit(IfInstr *instr) override {(void)instr;} - void visit(ScratchIOInstr *instr) override {(void)instr;} - void visit(StreamOutInstr *instr) override {(void)instr;} - void visit(MemRingOutInstr *instr) override {(void)instr;} - void visit(EmitVertexInstr *instr) override {(void)instr;} - void visit(GDSInstr *instr) override {(void)instr;}; - void visit(WriteTFInstr *instr) override {(void)instr;}; - void visit(RatInstr *instr) override {(void)instr;}; + void visit(ControlFlowInstr *instr) override { (void)instr; } + void visit(IfInstr *instr) override { (void)instr; } + void visit(ScratchIOInstr *instr) override { (void)instr; } + void visit(StreamOutInstr *instr) override { (void)instr; } + void visit(MemRingOutInstr *instr) override { (void)instr; } + void visit(EmitVertexInstr *instr) override { (void)instr; } + void visit(GDSInstr *instr) override { (void)instr; }; + void visit(WriteTFInstr *instr) override { (void)instr; }; + void visit(RatInstr *instr) override { (void)instr; }; // TODO: these two should use copy propagation - void visit(LDSAtomicInstr *instr) override {(void)instr;}; - void visit(LDSReadInstr *instr) override {(void)instr;}; + void visit(LDSAtomicInstr *instr) override { (void)instr; }; + void visit(LDSReadInstr *instr) override { (void)instr; }; void propagate_to(RegisterVec4& src, Instr *instr); bool progress; }; - class CopyPropBackVisitor : public InstrVisitor { public: CopyPropBackVisitor(); @@ -264,25 +269,26 @@ public: void visit(AluInstr *instr) override; void visit(AluGroup *instr) override; void visit(TexInstr *instr) override; - void visit(ExportInstr *instr) override {(void)instr;} + void visit(ExportInstr *instr) override { (void)instr; } void visit(FetchInstr *instr) override; void visit(Block *instr) override; - void visit(ControlFlowInstr *instr) override {(void)instr;} - void visit(IfInstr *instr) override {(void)instr;} - void visit(ScratchIOInstr *instr) override {(void)instr;} - void visit(StreamOutInstr *instr) override {(void)instr;} - void visit(MemRingOutInstr *instr) override {(void)instr;} - void visit(EmitVertexInstr *instr) override {(void)instr;} - void visit(GDSInstr *instr) override {(void)instr;}; - void visit(WriteTFInstr *instr) override {(void)instr;}; - void visit(LDSAtomicInstr *instr) override {(void)instr;}; - void visit(LDSReadInstr *instr) override {(void)instr;}; - void visit(RatInstr *instr) override {(void)instr;}; + void visit(ControlFlowInstr *instr) override { (void)instr; } + void visit(IfInstr *instr) override { (void)instr; } + void visit(ScratchIOInstr *instr) override { (void)instr; } + void visit(StreamOutInstr *instr) override { (void)instr; } + void visit(MemRingOutInstr *instr) override { (void)instr; } + void visit(EmitVertexInstr *instr) override { (void)instr; } + void visit(GDSInstr *instr) override { (void)instr; }; + void visit(WriteTFInstr *instr) override { (void)instr; }; + void visit(LDSAtomicInstr *instr) override { (void)instr; }; + void visit(LDSReadInstr *instr) override { (void)instr; }; + void visit(RatInstr *instr) override { (void)instr; }; bool progress; }; -bool copy_propagation_fwd(Shader& shader) +bool +copy_propagation_fwd(Shader& shader) { auto& root = shader.func(); CopyPropFwdVisitor copy_prop; @@ -291,30 +297,30 @@ bool copy_propagation_fwd(Shader& shader) copy_prop.progress = false; for (auto b : root) b->accept(copy_prop); - } while (copy_prop.progress); + } while (copy_prop.progress); - sfn_log << SfnLog::opt << "Shader after Copy Prop forward\n"; + sfn_log << SfnLog::opt << "Shader after Copy Prop forward\n"; if (sfn_log.has_debug_flag(SfnLog::opt)) { std::stringstream ss; shader.print(ss); sfn_log << ss.str() << "\n\n"; } - return copy_prop.progress; } -bool copy_propagation_backward(Shader& shader) +bool +copy_propagation_backward(Shader& shader) { CopyPropBackVisitor copy_prop; do { copy_prop.progress = false; - for (auto b: shader.func()) + for (auto b : shader.func()) b->accept(copy_prop); - } while (copy_prop.progress); + } while (copy_prop.progress); - sfn_log << SfnLog::opt << "Shader after Copy Prop backwards\n"; + sfn_log << SfnLog::opt << "Shader after Copy Prop backwards\n"; if (sfn_log.has_debug_flag(SfnLog::opt)) { std::stringstream ss; shader.print(ss); @@ -325,20 +331,18 @@ bool copy_propagation_backward(Shader& shader) } CopyPropFwdVisitor::CopyPropFwdVisitor(): - progress(false) -{} - -void CopyPropFwdVisitor::visit(AluInstr *instr) + progress(false) { - sfn_log << SfnLog::opt << "CopyPropFwdVisitor:[" - << instr->block_id() << ":" << instr->index() << "] " << *instr - << " dset=" << instr->dest() << " "; - +} +void +CopyPropFwdVisitor::visit(AluInstr *instr) +{ + sfn_log << SfnLog::opt << "CopyPropFwdVisitor:[" << instr->block_id() << ":" + << instr->index() << "] " << *instr << " dset=" << instr->dest() << " "; if (instr->dest()) { - sfn_log << SfnLog::opt << "has uses; " - << instr->dest()->uses().size(); + sfn_log << SfnLog::opt << "has uses; " << instr->dest()->uses().size(); } sfn_log << SfnLog::opt << "\n"; @@ -366,15 +370,13 @@ void CopyPropFwdVisitor::visit(AluInstr *instr) * 3: MOV SN.x, R0.x * * Here we can't prpagate the move in 1 to SN.x in 3 */ - if ((instr->block_id() == i->block_id() && - instr->index() < i->index())) { + if ((instr->block_id() == i->block_id() && instr->index() < i->index())) { can_propagate = true; if (dest->parents().size() > 1) { for (auto p : dest->parents()) { - if (p->block_id() == i->block_id() && - p->index() > instr->index()) { - can_propagate = false; - break; + if (p->block_id() == i->block_id() && p->index() > instr->index()) { + can_propagate = false; + break; } } } @@ -382,36 +384,37 @@ void CopyPropFwdVisitor::visit(AluInstr *instr) } if (can_propagate) { - sfn_log << SfnLog::opt << " Try replace in " - << i->block_id() << ":" << i->index() - << *i<< "\n"; + sfn_log << SfnLog::opt << " Try replace in " << i->block_id() << ":" + << i->index() << *i << "\n"; progress |= i->replace_source(dest, src); } } if (instr->dest()) { - sfn_log << SfnLog::opt << "has uses; " - << instr->dest()->uses().size(); + sfn_log << SfnLog::opt << "has uses; " << instr->dest()->uses().size(); } sfn_log << SfnLog::opt << " done\n"; } - -void CopyPropFwdVisitor::visit(AluGroup *instr) +void +CopyPropFwdVisitor::visit(AluGroup *instr) { (void)instr; } -void CopyPropFwdVisitor::visit(TexInstr *instr) +void +CopyPropFwdVisitor::visit(TexInstr *instr) { propagate_to(instr->src(), instr); } -void CopyPropFwdVisitor::visit(ExportInstr *instr) +void +CopyPropFwdVisitor::visit(ExportInstr *instr) { propagate_to(instr->value(), instr); } -void CopyPropFwdVisitor::propagate_to(RegisterVec4& src, Instr *instr) +void +CopyPropFwdVisitor::propagate_to(RegisterVec4& src, Instr *instr) { AluInstr *parents[4] = {nullptr}; for (int i = 0; i < 4; ++i) { @@ -430,8 +433,7 @@ void CopyPropFwdVisitor::propagate_to(RegisterVec4& src, Instr *instr) for (int i = 0; i < 4; ++i) { if (!parents[i]) continue; - if ((parents[i]->opcode() != op1_mov) || - parents[i]->has_alu_flag(alu_src0_neg) || + if ((parents[i]->opcode() != op1_mov) || parents[i]->has_alu_flag(alu_src0_neg) || parents[i]->has_alu_flag(alu_src0_abs) || parents[i]->has_alu_flag(alu_dst_clamp) || parents[i]->has_alu_flag(alu_src0_rel)) { @@ -468,30 +470,31 @@ void CopyPropFwdVisitor::propagate_to(RegisterVec4& src, Instr *instr) src.validate(); } -void CopyPropFwdVisitor::visit(FetchInstr *instr) +void +CopyPropFwdVisitor::visit(FetchInstr *instr) { (void)instr; } -void CopyPropFwdVisitor::visit(Block *instr) +void +CopyPropFwdVisitor::visit(Block *instr) { - for (auto& i: *instr) + for (auto& i : *instr) i->accept(*this); } CopyPropBackVisitor::CopyPropBackVisitor(): - progress(false) + progress(false) { - } -void CopyPropBackVisitor::visit(AluInstr *instr) +void +CopyPropBackVisitor::visit(AluInstr *instr) { bool local_progress = false; - sfn_log << SfnLog::opt << "CopyPropBackVisitor:[" - << instr->block_id() << ":" << instr->index() << "] " << *instr << "\n"; - + sfn_log << SfnLog::opt << "CopyPropBackVisitor:[" << instr->block_id() << ":" + << instr->index() << "] " << *instr << "\n"; if (!instr->can_propagate_dest()) { return; @@ -506,54 +509,56 @@ void CopyPropBackVisitor::visit(AluInstr *instr) return; auto dest = instr->dest(); - if (!dest || - !instr->has_alu_flag(alu_write)) { + if (!dest || !instr->has_alu_flag(alu_write)) { return; } if (!dest->is_ssa() && dest->parents().size() > 1) return; - for (auto& i: src_reg->parents()) { - sfn_log << SfnLog::opt << "Try replace dest in " - << i->block_id() << ":" << i->index() - << *i<< "\n"; + for (auto& i : src_reg->parents()) { + sfn_log << SfnLog::opt << "Try replace dest in " << i->block_id() << ":" + << i->index() << *i << "\n"; - if (i->replace_dest(dest, instr)) { - dest->del_parent(instr); - dest->add_parent(i); - for (auto d : instr->dependend_instr()) { - d->add_required_instr(i); - } - local_progress = true; - } - } + if (i->replace_dest(dest, instr)) { + dest->del_parent(instr); + dest->add_parent(i); + for (auto d : instr->dependend_instr()) { + d->add_required_instr(i); + } + local_progress = true; + } + } - if (local_progress) - instr->set_dead(); + if (local_progress) + instr->set_dead(); - progress |= local_progress; + progress |= local_progress; } -void CopyPropBackVisitor::visit(AluGroup *instr) +void +CopyPropBackVisitor::visit(AluGroup *instr) { - for (auto& i: *instr) { + for (auto& i : *instr) { if (i) i->accept(*this); } } -void CopyPropBackVisitor::visit(TexInstr *instr) +void +CopyPropBackVisitor::visit(TexInstr *instr) { (void)instr; } -void CopyPropBackVisitor::visit(FetchInstr *instr) +void +CopyPropBackVisitor::visit(FetchInstr *instr) { (void)instr; } -void CopyPropBackVisitor::visit(Block *instr) +void +CopyPropBackVisitor::visit(Block *instr) { for (auto i = instr->rbegin(); i != instr->rend(); ++i) if (!(*i)->is_dead()) @@ -562,10 +567,13 @@ void CopyPropBackVisitor::visit(Block *instr) class SimplifySourceVecVisitor : public InstrVisitor { public: - SimplifySourceVecVisitor():progress(false) {} + SimplifySourceVecVisitor(): + progress(false) + { + } - void visit(AluInstr *instr) override{(void)instr;} - void visit(AluGroup *instr) override{(void)instr;} + void visit(AluInstr *instr) override { (void)instr; } + void visit(AluGroup *instr) override { (void)instr; } void visit(TexInstr *instr) override; void visit(ExportInstr *instr) override; void visit(FetchInstr *instr) override; @@ -575,29 +583,31 @@ public: void visit(ScratchIOInstr *instr) override; void visit(StreamOutInstr *instr) override; void visit(MemRingOutInstr *instr) override; - void visit(EmitVertexInstr *instr) override {(void)instr;} - void visit(GDSInstr *instr) override {(void)instr;}; - void visit(WriteTFInstr *instr) override {(void)instr;}; - void visit(LDSAtomicInstr *instr) override {(void)instr;}; - void visit(LDSReadInstr *instr) override {(void)instr;}; - void visit(RatInstr *instr) override {(void)instr;}; + void visit(EmitVertexInstr *instr) override { (void)instr; } + void visit(GDSInstr *instr) override { (void)instr; }; + void visit(WriteTFInstr *instr) override { (void)instr; }; + void visit(LDSAtomicInstr *instr) override { (void)instr; }; + void visit(LDSReadInstr *instr) override { (void)instr; }; + void visit(RatInstr *instr) override { (void)instr; }; void replace_src(Instr *instr, RegisterVec4& reg4); bool progress; }; -bool simplify_source_vectors(Shader& sh) +bool +simplify_source_vectors(Shader& sh) { SimplifySourceVecVisitor visitor; - for (auto b: sh.func()) + for (auto b : sh.func()) b->accept(visitor); return visitor.progress; } -void SimplifySourceVecVisitor::visit(TexInstr *instr) +void +SimplifySourceVecVisitor::visit(TexInstr *instr) { if (instr->opcode() != TexInstr::get_resinfo) { @@ -622,15 +632,21 @@ void SimplifySourceVecVisitor::visit(TexInstr *instr) } } -void SimplifySourceVecVisitor::visit(ScratchIOInstr *instr) +void +SimplifySourceVecVisitor::visit(ScratchIOInstr *instr) { - (void) instr; + (void)instr; } class ReplaceConstSource : public AluInstrVisitor { public: ReplaceConstSource(Instr *old_use_, RegisterVec4& vreg_, int i): - old_use(old_use_), vreg(vreg_), index(i),success(false) {} + old_use(old_use_), + vreg(vreg_), + index(i), + success(false) + { + } using AluInstrVisitor::visit; @@ -642,12 +658,14 @@ public: bool success; }; -void SimplifySourceVecVisitor::visit(ExportInstr *instr) +void +SimplifySourceVecVisitor::visit(ExportInstr *instr) { replace_src(instr, instr->value()); } -void SimplifySourceVecVisitor::replace_src(Instr *instr, RegisterVec4& reg4) +void +SimplifySourceVecVisitor::replace_src(Instr *instr, RegisterVec4& reg4) { for (int i = 0; i < 4; ++i) { auto s = reg4[i]; @@ -673,23 +691,25 @@ void SimplifySourceVecVisitor::replace_src(Instr *instr, RegisterVec4& reg4) } } -void SimplifySourceVecVisitor::visit(StreamOutInstr *instr) +void +SimplifySourceVecVisitor::visit(StreamOutInstr *instr) { (void)instr; } -void SimplifySourceVecVisitor::visit(MemRingOutInstr *instr) +void +SimplifySourceVecVisitor::visit(MemRingOutInstr *instr) { (void)instr; } -void ReplaceConstSource::visit(AluInstr *alu) +void +ReplaceConstSource::visit(AluInstr *alu) { if (alu->opcode() != op1_mov) return; - if (alu->has_alu_flag(alu_src0_abs) || - alu->has_alu_flag(alu_src0_neg)) + if (alu->has_alu_flag(alu_src0_abs) || alu->has_alu_flag(alu_src0_neg)) return; auto src = alu->psrc(0); @@ -724,28 +744,30 @@ void ReplaceConstSource::visit(AluInstr *alu) } } -void SimplifySourceVecVisitor::visit(FetchInstr *instr) +void +SimplifySourceVecVisitor::visit(FetchInstr *instr) { - (void) instr; + (void)instr; } -void SimplifySourceVecVisitor::visit(Block *instr) +void +SimplifySourceVecVisitor::visit(Block *instr) { for (auto i = instr->rbegin(); i != instr->rend(); ++i) if (!(*i)->is_dead()) (*i)->accept(*this); } -void SimplifySourceVecVisitor::visit(ControlFlowInstr *instr) +void +SimplifySourceVecVisitor::visit(ControlFlowInstr *instr) { - (void) instr; + (void)instr; } -void SimplifySourceVecVisitor::visit(IfInstr *instr) +void +SimplifySourceVecVisitor::visit(IfInstr *instr) { - (void) instr; + (void)instr; } - - -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_optimizer.h b/src/gallium/drivers/r600/sfn/sfn_optimizer.h index 4a6f1206014..935edad4037 100644 --- a/src/gallium/drivers/r600/sfn/sfn_optimizer.h +++ b/src/gallium/drivers/r600/sfn/sfn_optimizer.h @@ -31,13 +31,18 @@ namespace r600 { -bool dead_code_elimination(Shader& shader); -bool copy_propagation_fwd(Shader& shader); -bool copy_propagation_backward(Shader& shader); -bool simplify_source_vectors(Shader& sh); +bool +dead_code_elimination(Shader& shader); +bool +copy_propagation_fwd(Shader& shader); +bool +copy_propagation_backward(Shader& shader); +bool +simplify_source_vectors(Shader& sh); -bool optimize(Shader& shader); +bool +optimize(Shader& shader); -} +} // namespace r600 #endif // OPTIMIZER_H diff --git a/src/gallium/drivers/r600/sfn/sfn_peephole.cpp b/src/gallium/drivers/r600/sfn/sfn_peephole.cpp index 74978029614..fc1d26f5bed 100644 --- a/src/gallium/drivers/r600/sfn/sfn_peephole.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_peephole.cpp @@ -28,41 +28,39 @@ namespace r600 { - class PeepholeVisitor : public InstrVisitor { public: void visit(AluInstr *instr) override; void visit(AluGroup *instr) override; - void visit(TexInstr *instr) override {(void)instr;}; - void visit(ExportInstr *instr) override {(void)instr;} - void visit(FetchInstr *instr) override {(void)instr;} + void visit(TexInstr *instr) override { (void)instr; }; + void visit(ExportInstr *instr) override { (void)instr; } + void visit(FetchInstr *instr) override { (void)instr; } void visit(Block *instr) override; - void visit(ControlFlowInstr *instr) override {(void)instr;} + void visit(ControlFlowInstr *instr) override { (void)instr; } void visit(IfInstr *instr) override; - void visit(ScratchIOInstr *instr) override {(void)instr;} - void visit(StreamOutInstr *instr) override {(void)instr;} - void visit(MemRingOutInstr *instr) override {(void)instr;} - void visit(EmitVertexInstr *instr) override {(void)instr;} - void visit(GDSInstr *instr) override {(void)instr;}; - void visit(WriteTFInstr *instr) override {(void)instr;}; - void visit(LDSAtomicInstr *instr) override {(void)instr;}; - void visit(LDSReadInstr *instr) override {(void)instr;}; - void visit(RatInstr *instr) override {(void)instr;}; + void visit(ScratchIOInstr *instr) override { (void)instr; } + void visit(StreamOutInstr *instr) override { (void)instr; } + void visit(MemRingOutInstr *instr) override { (void)instr; } + void visit(EmitVertexInstr *instr) override { (void)instr; } + void visit(GDSInstr *instr) override { (void)instr; }; + void visit(WriteTFInstr *instr) override { (void)instr; }; + void visit(LDSAtomicInstr *instr) override { (void)instr; }; + void visit(LDSReadInstr *instr) override { (void)instr; }; + void visit(RatInstr *instr) override { (void)instr; }; bool src_is_zero(PVirtualValue value); bool src_is_one(PVirtualValue value); void convert_to_mov(AluInstr *alu, int src_idx); - bool progress{false}; }; - -bool peephole(Shader& sh) +bool +peephole(Shader& sh) { PeepholeVisitor peephole; - for(auto b : sh.func()) + for (auto b : sh.func()) b->accept(peephole); return peephole.progress; } @@ -70,7 +68,9 @@ bool peephole(Shader& sh) class ReplacePredicate : public AluInstrVisitor { public: ReplacePredicate(AluInstr *pred): - m_pred(pred) {} + m_pred(pred) + { + } using AluInstrVisitor::visit; @@ -80,8 +80,8 @@ public: bool success{false}; }; - -void PeepholeVisitor::visit(AluInstr *instr) +void +PeepholeVisitor::visit(AluInstr *instr) { switch (instr->opcode()) { case op2_add: @@ -100,8 +100,7 @@ void PeepholeVisitor::visit(AluInstr *instr) break; case op3_muladd: case op3_muladd_ieee: - if (src_is_zero(instr->psrc(0)) || - src_is_zero(instr->psrc(1))) + if (src_is_zero(instr->psrc(0)) || src_is_zero(instr->psrc(1))) convert_to_mov(instr, 2); break; case op2_killne_int: @@ -115,38 +114,36 @@ void PeepholeVisitor::visit(AluInstr *instr) } } - default: - ; + default:; } } -bool PeepholeVisitor::src_is_zero(PVirtualValue value) +bool +PeepholeVisitor::src_is_zero(PVirtualValue value) { - if (value->as_inline_const() && - value->as_inline_const()->sel() == ALU_SRC_0) + if (value->as_inline_const() && value->as_inline_const()->sel() == ALU_SRC_0) return true; - if (value->as_literal() && - value->as_literal()->value() == 0) + if (value->as_literal() && value->as_literal()->value() == 0) return true; return false; } -bool PeepholeVisitor::src_is_one(PVirtualValue value) +bool +PeepholeVisitor::src_is_one(PVirtualValue value) { - if (value->as_inline_const() && - value->as_inline_const()->sel() == ALU_SRC_1) + if (value->as_inline_const() && value->as_inline_const()->sel() == ALU_SRC_1) return true; - if (value->as_literal() && - value->as_literal()->value() == 0x3f800000) + if (value->as_literal() && value->as_literal()->value() == 0x3f800000) return true; return false; } -void PeepholeVisitor::convert_to_mov(AluInstr *alu, int src_idx) +void +PeepholeVisitor::convert_to_mov(AluInstr *alu, int src_idx) { AluInstr::SrcValues new_src{alu->psrc(src_idx)}; alu->set_sources(new_src); @@ -154,26 +151,25 @@ void PeepholeVisitor::convert_to_mov(AluInstr *alu, int src_idx) progress = true; } - -void PeepholeVisitor::visit(AluGroup *instr) +void +PeepholeVisitor::visit(AluGroup *instr) { - } -void PeepholeVisitor::visit(Block *instr) +void +PeepholeVisitor::visit(Block *instr) { - for (auto& i: *instr) + for (auto& i : *instr) i->accept(*this); } - -void PeepholeVisitor::visit(IfInstr *instr) +void +PeepholeVisitor::visit(IfInstr *instr) { auto pred = instr->predicate(); auto& src1 = pred->src(1); - if (src1.as_inline_const() && - src1.as_inline_const()->sel() == ALU_SRC_0) { + if (src1.as_inline_const() && src1.as_inline_const()->sel() == ALU_SRC_0) { auto src0 = pred->src(0).as_register(); if (src0 && src0->is_ssa() && !src0->parents().empty()) { assert(src0->parents().size() == 1); @@ -186,52 +182,78 @@ void PeepholeVisitor::visit(IfInstr *instr) } } -static EAluOp pred_from_op(EAluOp pred_op, EAluOp op) +static EAluOp +pred_from_op(EAluOp pred_op, EAluOp op) { switch (pred_op) { case op2_pred_setne_int: switch (op) { - case op2_setge_dx10 : return op2_pred_setge; - case op2_setgt_dx10 : return op2_pred_setgt; - case op2_sete_dx10 : return op2_pred_sete; - case op2_setne_dx10 : return op2_pred_setne; + case op2_setge_dx10: + return op2_pred_setge; + case op2_setgt_dx10: + return op2_pred_setgt; + case op2_sete_dx10: + return op2_pred_sete; + case op2_setne_dx10: + return op2_pred_setne; - case op2_setge_int : return op2_pred_setge_int; - case op2_setgt_int : return op2_pred_setgt_int; - case op2_setge_uint : return op2_pred_setge_uint; - case op2_setgt_uint : return op2_pred_setgt_uint; - case op2_sete_int : return op2_prede_int; - case op2_setne_int : return op2_pred_setne_int; + case op2_setge_int: + return op2_pred_setge_int; + case op2_setgt_int: + return op2_pred_setgt_int; + case op2_setge_uint: + return op2_pred_setge_uint; + case op2_setgt_uint: + return op2_pred_setgt_uint; + case op2_sete_int: + return op2_prede_int; + case op2_setne_int: + return op2_pred_setne_int; default: return op0_nop; } case op2_prede_int: switch (op) { - case op2_sete_int : return op2_pred_setne_int; - case op2_setne_int : return op2_prede_int; + case op2_sete_int: + return op2_pred_setne_int; + case op2_setne_int: + return op2_prede_int; default: return op0_nop; } case op2_pred_setne: switch (op) { - case op2_setge : return op2_pred_setge; - case op2_setgt : return op2_pred_setgt; - case op2_sete : return op2_pred_sete; + case op2_setge: + return op2_pred_setge; + case op2_setgt: + return op2_pred_setgt; + case op2_sete: + return op2_pred_sete; default: return op0_nop; } case op2_killne_int: switch (op) { - case op2_setge_dx10 : return op2_killge; - case op2_setgt_dx10 : return op2_killgt; - case op2_sete_dx10 : return op2_kille; - case op2_setne_dx10 : return op2_killne; - case op2_setge_int : return op2_killge_int; - case op2_setgt_int : return op2_killgt_int; - case op2_setge_uint : return op2_killge_uint; - case op2_setgt_uint : return op2_killgt_uint; - case op2_sete_int : return op2_kille_int; - case op2_setne_int : return op2_killne_int; + case op2_setge_dx10: + return op2_killge; + case op2_setgt_dx10: + return op2_killgt; + case op2_sete_dx10: + return op2_kille; + case op2_setne_dx10: + return op2_killne; + case op2_setge_int: + return op2_killge_int; + case op2_setgt_int: + return op2_killgt_int; + case op2_setge_uint: + return op2_killge_uint; + case op2_setgt_uint: + return op2_killgt_uint; + case op2_sete_int: + return op2_kille_int; + case op2_setne_int: + return op2_killne_int; default: return op0_nop; } @@ -241,7 +263,8 @@ static EAluOp pred_from_op(EAluOp pred_op, EAluOp op) } } -void ReplacePredicate::visit(AluInstr *alu) +void +ReplacePredicate::visit(AluInstr *alu) { auto new_op = pred_from_op(m_pred->opcode(), alu->opcode()); @@ -282,4 +305,4 @@ void ReplacePredicate::visit(AluInstr *alu) success = true; } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_peephole.h b/src/gallium/drivers/r600/sfn/sfn_peephole.h index d2ff0d06328..f0d568cd214 100644 --- a/src/gallium/drivers/r600/sfn/sfn_peephole.h +++ b/src/gallium/drivers/r600/sfn/sfn_peephole.h @@ -31,9 +31,9 @@ namespace r600 { -bool peephole(Shader& sh); +bool +peephole(Shader& sh); } - #endif // PEEPHOLE_H diff --git a/src/gallium/drivers/r600/sfn/sfn_ra.cpp b/src/gallium/drivers/r600/sfn/sfn_ra.cpp index 8bdbb525d4d..6dac75e6f2b 100644 --- a/src/gallium/drivers/r600/sfn/sfn_ra.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_ra.cpp @@ -24,21 +24,23 @@ * USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include "sfn_debug.h" #include "sfn_ra.h" +#include "sfn_debug.h" + #include #include namespace r600 { -void ComponentInterference::prepare_row(int row) +void +ComponentInterference::prepare_row(int row) { m_rows.resize(row + 1); - } -void ComponentInterference::add(size_t idx1, size_t idx2) +void +ComponentInterference::add(size_t idx1, size_t idx2) { assert(idx1 > idx2); assert(m_rows.size() > idx1); @@ -46,30 +48,30 @@ void ComponentInterference::add(size_t idx1, size_t idx2) m_rows[idx2].push_back(idx1); } - Interference::Interference(LiveRangeMap& map): - m_map(map) + m_map(map) { initialize(); } -void Interference::initialize() +void +Interference::initialize() { - for(int i = 0; i < 4; ++i) { + for (int i = 0; i < 4; ++i) { initialize(m_components_maps[i], m_map.component(i)); } } -void Interference::initialize(ComponentInterference& comp_interference, - LiveRangeMap::ChannelLiveRange& clr) +void +Interference::initialize(ComponentInterference& comp_interference, + LiveRangeMap::ChannelLiveRange& clr) { for (size_t row = 0; row < clr.size(); ++row) { auto& row_entry = clr[row]; comp_interference.prepare_row(row); for (size_t col = 0; col < row; ++col) { auto& col_entry = clr[col]; - if (row_entry.m_end >= col_entry.m_start && - row_entry.m_start <= col_entry.m_end) + if (row_entry.m_end >= col_entry.m_start && row_entry.m_start <= col_entry.m_end) comp_interference.add(row, col); } } @@ -80,7 +82,8 @@ struct Group { std::array channels; }; -static inline bool operator < (const Group& lhs, const Group& rhs) +static inline bool +operator<(const Group& lhs, const Group& rhs) { return lhs.priority < rhs.priority; } @@ -88,7 +91,9 @@ static inline bool operator < (const Group& lhs, const Group& rhs) using GroupRegisters = std::priority_queue; static bool -group_allocation (LiveRangeMap& lrm, const Interference& interference, GroupRegisters& groups) +group_allocation(LiveRangeMap& lrm, + const Interference& interference, + GroupRegisters& groups) { int color = 0; // allocate grouped registers @@ -100,7 +105,8 @@ group_allocation (LiveRangeMap& lrm, const Interference& interference, GroupReg while (!group.channels[start_comp]) ++start_comp; - sfn_log << SfnLog::merge << "Color group with " << *group.channels[start_comp] << "\n"; + sfn_log << SfnLog::merge << "Color group with " << *group.channels[start_comp] + << "\n"; // don't restart registers for exports, we may be able tp merge the // export calls, is fthe registers are consecutive @@ -115,7 +121,7 @@ group_allocation (LiveRangeMap& lrm, const Interference& interference, GroupReg auto& adjecency = interference.row(start_comp, group.channels[comp]->index()); auto& regs = lrm.component(comp); - sfn_log << SfnLog::merge << "Try color "<< color; + sfn_log << SfnLog::merge << "Try color " << color; for (auto adj : adjecency) { if (regs[adj].m_color == color) { @@ -130,7 +136,8 @@ group_allocation (LiveRangeMap& lrm, const Interference& interference, GroupReg continue; } - /* First channel color found, check whether it can be used for all channels */ + /* First channel color found, check whether it can be used for all + * channels */ while (comp < 4) { sfn_log << SfnLog::merge << " interference: "; if (group.channels[comp]) { @@ -138,7 +145,8 @@ group_allocation (LiveRangeMap& lrm, const Interference& interference, GroupReg auto& adjecencies = interference.row(comp, group.channels[comp]->index()); for (auto adj_index : adjecencies) { - sfn_log << SfnLog::merge << *component_life_ranges[adj_index].m_register << " "; + sfn_log << SfnLog::merge << *component_life_ranges[adj_index].m_register + << " "; if (component_life_ranges[adj_index].m_color == color) { color_in_use = true; sfn_log << SfnLog::merge << "used"; @@ -180,7 +188,7 @@ group_allocation (LiveRangeMap& lrm, const Interference& interference, GroupReg } static bool -scalar_allocation (LiveRangeMap& lrm, const Interference& interference) +scalar_allocation(LiveRangeMap& lrm, const Interference& interference) { for (int comp = 0; comp < 4; ++comp) { auto& live_ranges = lrm.component(comp); @@ -188,8 +196,7 @@ scalar_allocation (LiveRangeMap& lrm, const Interference& interference) if (r.m_color != -1) continue; - if (r.m_start == -1 && - r.m_end == -1) + if (r.m_start == -1 && r.m_end == -1) continue; sfn_log << SfnLog::merge << "Color " << *r.m_register << "\n"; @@ -222,7 +229,8 @@ scalar_allocation (LiveRangeMap& lrm, const Interference& interference) return true; } -bool register_allocation(LiveRangeMap& lrm) +bool +register_allocation(LiveRangeMap& lrm) { Interference interference(lrm); @@ -232,9 +240,8 @@ bool register_allocation(LiveRangeMap& lrm) for (int i = 0; i < 4; ++i) { auto& comp = lrm.component(i); for (auto& entry : comp) { - sfn_log << SfnLog::merge << "Prepare RA for " - << *entry.m_register - << " [" << entry.m_start << ", " << entry.m_end << "]\n"; + sfn_log << SfnLog::merge << "Prepare RA for " << *entry.m_register << " [" + << entry.m_start << ", " << entry.m_end << "]\n"; auto pin = entry.m_register->pin(); if (entry.m_start == -1 && entry.m_end == -1) { if (pin == pin_group || pin == pin_chgr) @@ -245,23 +252,30 @@ bool register_allocation(LiveRangeMap& lrm) auto sel = entry.m_register->sel(); /* fully pinned registers contain system values with the * definite register index, and array values are allocated - * right after the system registers, so just reuse the IDs (for now) */ + * right after the system registers, so just reuse the IDs (for now) */ if (pin == pin_fully || pin == pin_array) { /* Must set all array element entries */ - sfn_log << SfnLog::merge << "Pin color " << sel << " to " << *entry.m_register << "\n"; + sfn_log << SfnLog::merge << "Pin color " << sel << " to " << *entry.m_register + << "\n"; entry.m_color = sel; } else if (pin == pin_group || pin == pin_chgr) { - /* Groups must all have the same sel() value, because they are used - * as vec4 registers */ + /* Groups must all have the same sel() value, because they are + * used as vec4 registers */ auto igroup = groups.find(sel); if (igroup != groups.end()) { igroup->second.channels[i] = entry.m_register; - assert(comp[entry.m_register->index()].m_register->index() == entry.m_register->index()); + assert(comp[entry.m_register->index()].m_register->index() == + entry.m_register->index()); } else { - int priority = entry.m_use.test(LiveRangeEntry::use_export) ? - entry.m_end : entry.m_start; - Group group{priority, {nullptr, nullptr, nullptr, nullptr}}; + int priority = entry.m_use.test(LiveRangeEntry::use_export) + ? -entry.m_end + : entry.m_start; + Group group{ + priority, {nullptr, nullptr, nullptr, nullptr} + }; group.channels[i] = entry.m_register; - assert(comp[group.channels[i]->index()].m_register->index() == entry.m_register->index()); + assert(comp[group.channels[i]->index()].m_register->index() == + entry.m_register->index()); groups[sel] = group; } } @@ -272,7 +286,7 @@ bool register_allocation(LiveRangeMap& lrm) for (auto& [sel, group] : groups) groups_sorted.push(group); - if (!group_allocation (lrm, interference, groups_sorted)) + if (!group_allocation(lrm, interference, groups_sorted)) return false; if (!scalar_allocation(lrm, interference)) @@ -291,4 +305,4 @@ bool register_allocation(LiveRangeMap& lrm) return true; } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_ra.h b/src/gallium/drivers/r600/sfn/sfn_ra.h index 08e42b222cf..fee4e327273 100644 --- a/src/gallium/drivers/r600/sfn/sfn_ra.h +++ b/src/gallium/drivers/r600/sfn/sfn_ra.h @@ -33,21 +33,21 @@ namespace r600 { -class ComponentInterference -{ +class ComponentInterference { public: - using Row = std::vector; void prepare_row(int row); void add(size_t idx1, size_t idx2); - auto row(int idx) const -> const Row& { - assert((size_t)idx < m_rows.size()); return m_rows[idx];} + auto row(int idx) const -> const Row& + { + assert((size_t)idx < m_rows.size()); + return m_rows[idx]; + } private: - std::vector m_rows; }; @@ -55,7 +55,8 @@ class Interference { public: Interference(LiveRangeMap& map); - const auto& row(int comp, int index) const { + const auto& row(int comp, int index) const + { assert(comp < 4); return m_components_maps[comp].row(index); } @@ -66,12 +67,11 @@ private: LiveRangeMap& m_map; std::array m_components_maps; - - }; -bool register_allocation(LiveRangeMap& lrm); +bool +register_allocation(LiveRangeMap& lrm); -} +} // namespace r600 #endif // INTERFERENCE_H diff --git a/src/gallium/drivers/r600/sfn/sfn_scheduler.cpp b/src/gallium/drivers/r600/sfn/sfn_scheduler.cpp index 7b105e26097..2afd2b166b0 100644 --- a/src/gallium/drivers/r600/sfn/sfn_scheduler.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_scheduler.cpp @@ -25,14 +25,15 @@ */ #include "sfn_scheduler.h" + +#include "sfn_debug.h" #include "sfn_instr_alugroup.h" #include "sfn_instr_controlflow.h" #include "sfn_instr_export.h" #include "sfn_instr_fetch.h" -#include "sfn_instr_mem.h" #include "sfn_instr_lds.h" +#include "sfn_instr_mem.h" #include "sfn_instr_tex.h" -#include "sfn_debug.h" #include #include @@ -43,9 +44,12 @@ class CollectInstructions : public InstrVisitor { public: CollectInstructions(ValueFactory& vf): - m_value_factory(vf) {} + m_value_factory(vf) + { + } - void visit(AluInstr *instr) override { + void visit(AluInstr *instr) override + { if (instr->has_alu_flag(alu_is_trans)) alu_trans.push_back(instr); else { @@ -55,78 +59,63 @@ public: alu_groups.push_back(instr->split(m_value_factory)); } } - void visit(AluGroup *instr) override { - alu_groups.push_back(instr); - } - void visit(TexInstr *instr) override { - tex.push_back(instr); - } - void visit(ExportInstr *instr) override { - exports.push_back(instr); - } - void visit(FetchInstr *instr) override { - fetches.push_back(instr); - } - void visit(Block *instr) override { - for (auto& i: *instr) + void visit(AluGroup *instr) override { alu_groups.push_back(instr); } + void visit(TexInstr *instr) override { tex.push_back(instr); } + void visit(ExportInstr *instr) override { exports.push_back(instr); } + void visit(FetchInstr *instr) override { fetches.push_back(instr); } + void visit(Block *instr) override + { + for (auto& i : *instr) i->accept(*this); } - void visit(ControlFlowInstr *instr) override { + void visit(ControlFlowInstr *instr) override + { assert(!m_cf_instr); m_cf_instr = instr; } - void visit(IfInstr *instr) override { + void visit(IfInstr *instr) override + { assert(!m_cf_instr); m_cf_instr = instr; } - void visit(EmitVertexInstr *instr) override { + void visit(EmitVertexInstr *instr) override + { assert(!m_cf_instr); m_cf_instr = instr; } - void visit(ScratchIOInstr *instr) override { - mem_write_instr.push_back(instr); - } + void visit(ScratchIOInstr *instr) override { mem_write_instr.push_back(instr); } - void visit(StreamOutInstr *instr) override { - mem_write_instr.push_back(instr); - } + void visit(StreamOutInstr *instr) override { mem_write_instr.push_back(instr); } - void visit(MemRingOutInstr *instr) override { - mem_ring_writes.push_back(instr); - } + void visit(MemRingOutInstr *instr) override { mem_ring_writes.push_back(instr); } - void visit(GDSInstr *instr) override { - gds_op.push_back(instr); - } + void visit(GDSInstr *instr) override { gds_op.push_back(instr); } - void visit(WriteTFInstr *instr) override { - write_tf.push_back(instr); - } + void visit(WriteTFInstr *instr) override { write_tf.push_back(instr); } - void visit(LDSReadInstr *instr) override { - std::vector buffer; + void visit(LDSReadInstr *instr) override + { + std::vector buffer; m_last_lds_instr = instr->split(buffer, m_last_lds_instr); - for (auto& i: buffer) { - i->accept(*this); - } - } - - void visit(LDSAtomicInstr *instr) override { - std::vector buffer; - m_last_lds_instr = instr->split(buffer, m_last_lds_instr); - for (auto& i: buffer) { + for (auto& i : buffer) { i->accept(*this); } } - void visit(RatInstr *instr) override { - rat_instr.push_back(instr); + void visit(LDSAtomicInstr *instr) override + { + std::vector buffer; + m_last_lds_instr = instr->split(buffer, m_last_lds_instr); + for (auto& i : buffer) { + i->accept(*this); + } } + void visit(RatInstr *instr) override { rat_instr.push_back(instr); } std::list alu_trans; std::list alu_vec; @@ -154,15 +143,16 @@ public: void finalize(); private: + void + schedule_block(Block& in_block, Shader::ShaderBlocks& out_blocks, ValueFactory& vf); - void schedule_block(Block& in_block, Shader::ShaderBlocks& out_blocks, ValueFactory& vf); - - bool collect_ready(CollectInstructions &available); + bool collect_ready(CollectInstructions& available); template bool collect_ready_type(std::list& ready, std::list& orig); - bool collect_ready_alu_vec(std::list& ready, std::list& available); + bool collect_ready_alu_vec(std::list& ready, + std::list& available); bool schedule_tex(Shader::ShaderBlocks& out_blocks); bool schedule_vtx(Shader::ShaderBlocks& out_blocks); @@ -179,13 +169,12 @@ private: bool schedule_alu_to_group_vec(AluGroup *group); bool schedule_alu_to_group_trans(AluGroup *group, std::list& readylist); - bool schedule_exports(Shader::ShaderBlocks& out_blocks, std::list& ready_list); + bool schedule_exports(Shader::ShaderBlocks& out_blocks, + std::list& ready_list); - template - bool schedule(std::list& ready_list); + template bool schedule(std::list& ready_list); - template - bool schedule_block(std::list& ready_list); + template bool schedule_block(std::list& ready_list); std::list alu_vec_ready; std::list alu_trans_ready; @@ -219,10 +208,10 @@ private: int m_lds_addr_count{0}; int m_alu_groups_schduled{0}; r600_chip_class m_chip_class; - }; -Shader *schedule(Shader *original) +Shader * +schedule(Shader *original) { Block::set_chipclass(original->chip_class()); AluGroup::set_chipclass(original->chip_class()); @@ -253,21 +242,22 @@ Shader *schedule(Shader *original) } BlockSheduler::BlockSheduler(r600_chip_class chip_class): - current_shed(sched_alu), - m_last_pos(nullptr), - m_last_pixel(nullptr), - m_last_param(nullptr), - m_current_block(nullptr), - m_chip_class(chip_class) + current_shed(sched_alu), + m_last_pos(nullptr), + m_last_pixel(nullptr), + m_last_param(nullptr), + m_current_block(nullptr), + m_chip_class(chip_class) { } -void BlockSheduler::run( Shader *shader) +void +BlockSheduler::run(Shader *shader) { Shader::ShaderBlocks scheduled_blocks; for (auto& block : shader->func()) { - sfn_log << SfnLog::schedule << "Process block " << block->id() <<"\n"; + sfn_log << SfnLog::schedule << "Process block " << block->id() << "\n"; if (sfn_log.has_debug_flag(SfnLog::schedule)) { std::stringstream ss; block->print(ss); @@ -279,12 +269,14 @@ void BlockSheduler::run( Shader *shader) shader->reset_function(scheduled_blocks); } -void BlockSheduler::schedule_block(Block& in_block, Shader::ShaderBlocks& out_blocks, ValueFactory& vf) +void +BlockSheduler::schedule_block(Block& in_block, + Shader::ShaderBlocks& out_blocks, + ValueFactory& vf) { assert(in_block.id() >= 0); - current_shed = sched_fetch; auto last_shed = sched_fetch; @@ -304,20 +296,17 @@ void BlockSheduler::schedule_block(Block& in_block, Shader::ShaderBlocks& out_bl sfn_log << SfnLog::schedule << " ALU V:" << alu_vec_ready.size() << "\n"; if (alu_trans_ready.size()) - sfn_log << SfnLog::schedule << " ALU T:" << alu_trans_ready.size() << "\n"; + sfn_log << SfnLog::schedule << " ALU T:" << alu_trans_ready.size() << "\n"; if (alu_groups_ready.size()) sfn_log << SfnLog::schedule << " ALU G:" << alu_groups_ready.size() << "\n"; if (exports_ready.size()) - sfn_log << SfnLog::schedule << " EXP:" << exports_ready.size() - << "\n"; + sfn_log << SfnLog::schedule << " EXP:" << exports_ready.size() << "\n"; if (tex_ready.size()) - sfn_log << SfnLog::schedule << " TEX:" << tex_ready.size() - << "\n"; + sfn_log << SfnLog::schedule << " TEX:" << tex_ready.size() << "\n"; if (fetches_ready.size()) - sfn_log << SfnLog::schedule << " FETCH:" << fetches_ready.size() - << "\n"; + sfn_log << SfnLog::schedule << " FETCH:" << fetches_ready.size() << "\n"; if (mem_ring_writes_ready.size()) sfn_log << SfnLog::schedule << " MEM_RING:" << mem_ring_writes_ready.size() << "\n"; @@ -333,7 +322,7 @@ void BlockSheduler::schedule_block(Block& in_block, Shader::ShaderBlocks& out_bl else if (rat_instr_ready.size() > 3) current_shed = sched_rat; else if (tex_ready.size() > (m_chip_class >= ISA_CC_EVERGREEN ? 15 : 7)) - current_shed = sched_tex; + current_shed = sched_tex; } switch (current_shed) { @@ -367,7 +356,8 @@ void BlockSheduler::schedule_block(Block& in_block, Shader::ShaderBlocks& out_bl current_shed = sched_mem_ring; continue; case sched_mem_ring: - if (mem_ring_writes_ready.empty() || !schedule_cf(out_blocks, mem_ring_writes_ready)) { + if (mem_ring_writes_ready.empty() || + !schedule_cf(out_blocks, mem_ring_writes_ready)) { current_shed = sched_write_tf; continue; } @@ -382,9 +372,9 @@ void BlockSheduler::schedule_block(Block& in_block, Shader::ShaderBlocks& out_bl break; case sched_rat: if (rat_instr_ready.empty() || !schedule_cf(out_blocks, rat_instr_ready)) { - current_shed = sched_free; - continue; - } + current_shed = sched_free; + continue; + } last_shed = current_shed; break; case sched_free: @@ -407,46 +397,46 @@ void BlockSheduler::schedule_block(Block& in_block, Shader::ShaderBlocks& out_bl if (!cir.alu_groups.empty()) { std::cerr << "Unscheduled ALU groups:\n"; for (auto& a : cir.alu_groups) { - std::cerr << " " << *a << "\n"; + std::cerr << " " << *a << "\n"; } fail = true; } - if (!cir.alu_vec.empty()){ + if (!cir.alu_vec.empty()) { std::cerr << "Unscheduled ALU vec ops:\n"; for (auto& a : cir.alu_vec) { - std::cerr << " " << *a << "\n"; + std::cerr << " " << *a << "\n"; } fail = true; } - if (!cir.alu_trans.empty()){ + if (!cir.alu_trans.empty()) { std::cerr << "Unscheduled ALU trans ops:\n"; for (auto& a : cir.alu_trans) { - std::cerr << " " << *a << "\n"; + std::cerr << " " << *a << "\n"; } fail = true; } - if (!cir.mem_write_instr.empty()){ + if (!cir.mem_write_instr.empty()) { std::cerr << "Unscheduled MEM ops:\n"; for (auto& a : cir.mem_write_instr) { - std::cerr << " " << *a << "\n"; + std::cerr << " " << *a << "\n"; } fail = true; } - if (!cir.fetches.empty()){ + if (!cir.fetches.empty()) { std::cerr << "Unscheduled Fetch ops:\n"; for (auto& a : cir.fetches) { - std::cerr << " " << *a << "\n"; + std::cerr << " " << *a << "\n"; } fail = true; } - if (!cir.tex.empty()){ + if (!cir.tex.empty()) { std::cerr << "Unscheduled Tex ops:\n"; for (auto& a : cir.tex) { - std::cerr << " " << *a << "\n"; + std::cerr << " " << *a << "\n"; } fail = true; } @@ -458,7 +448,7 @@ void BlockSheduler::schedule_block(Block& in_block, Shader::ShaderBlocks& out_bl assert(cir.mem_write_instr.empty()); assert(cir.mem_ring_writes.empty()); - assert (!fail); + assert(!fail); if (cir.m_cf_instr) { // Assert that if condition is ready @@ -469,7 +459,8 @@ void BlockSheduler::schedule_block(Block& in_block, Shader::ShaderBlocks& out_bl out_blocks.push_back(m_current_block); } -void BlockSheduler::finalize() +void +BlockSheduler::finalize() { if (m_last_pos) m_last_pos->set_is_last_export(true); @@ -479,18 +470,19 @@ void BlockSheduler::finalize() m_last_param->set_is_last_export(true); } -bool BlockSheduler::schedule_alu(Shader::ShaderBlocks& out_blocks) +bool +BlockSheduler::schedule_alu(Shader::ShaderBlocks& out_blocks) { bool success = false; AluGroup *group = nullptr; bool has_alu_ready = !alu_vec_ready.empty() || !alu_trans_ready.empty(); - bool has_lds_ready = !alu_vec_ready.empty() && - (*alu_vec_ready.begin())->has_lds_access(); + bool has_lds_ready = + !alu_vec_ready.empty() && (*alu_vec_ready.begin())->has_lds_access(); /* If we have ready ALU instructions we have to start a new ALU block */ - if (has_alu_ready || !alu_groups_ready.empty()) { + if (has_alu_ready || !alu_groups_ready.empty()) { if (m_current_block->type() != Block::alu) { start_new_block(out_blocks, Block::alu); m_alu_groups_schduled = 0; @@ -574,18 +566,17 @@ bool BlockSheduler::schedule_alu(Shader::ShaderBlocks& out_blocks) m_current_block->set_instr_flag(Instr::force_cf); } - return success; } -bool BlockSheduler::schedule_tex(Shader::ShaderBlocks& out_blocks) +bool +BlockSheduler::schedule_tex(Shader::ShaderBlocks& out_blocks) { - if (m_current_block->type() != Block::tex || m_current_block->remaining_slots() == 0) { + if (m_current_block->type() != Block::tex || m_current_block->remaining_slots() == 0) { start_new_block(out_blocks, Block::tex); m_current_block->set_instr_flag(Instr::force_cf); } - if (!tex_ready.empty() && m_current_block->remaining_slots() > 0) { auto ii = tex_ready.begin(); sfn_log << SfnLog::schedule << "Schedule: " << **ii << "\n"; @@ -606,7 +597,8 @@ bool BlockSheduler::schedule_tex(Shader::ShaderBlocks& out_blocks) return false; } -bool BlockSheduler::schedule_vtx(Shader::ShaderBlocks& out_blocks) +bool +BlockSheduler::schedule_vtx(Shader::ShaderBlocks& out_blocks) { if (m_current_block->type() != Block::vtx || m_current_block->remaining_slots() == 0) { start_new_block(out_blocks, Block::vtx); @@ -616,7 +608,8 @@ bool BlockSheduler::schedule_vtx(Shader::ShaderBlocks& out_blocks) } template -bool BlockSheduler::schedule_gds(Shader::ShaderBlocks& out_blocks, std::list& ready_list) +bool +BlockSheduler::schedule_gds(Shader::ShaderBlocks& out_blocks, std::list& ready_list) { bool was_full = m_current_block->remaining_slots() == 0; if (m_current_block->type() != Block::gds || was_full) { @@ -627,20 +620,22 @@ bool BlockSheduler::schedule_gds(Shader::ShaderBlocks& out_blocks, std::listempty()) { sfn_log << SfnLog::schedule << "Start new block\n"; assert(!m_current_block->lds_group_active()); out_blocks.push_back(m_current_block); - m_current_block = new Block(m_current_block->nesting_depth(), m_current_block->id()); + m_current_block = + new Block(m_current_block->nesting_depth(), m_current_block->id()); } m_current_block->set_type(type); } template -bool BlockSheduler::schedule_cf(Shader::ShaderBlocks& out_blocks, std::list& ready_list) +bool +BlockSheduler::schedule_cf(Shader::ShaderBlocks& out_blocks, std::list& ready_list) { if (ready_list.empty()) return false; @@ -649,20 +644,20 @@ bool BlockSheduler::schedule_cf(Shader::ShaderBlocks& out_blocks, std::list return schedule(ready_list); } - -bool BlockSheduler::schedule_alu_to_group_vec(AluGroup *group) +bool +BlockSheduler::schedule_alu_to_group_vec(AluGroup *group) { assert(group); assert(!alu_vec_ready.empty()); - bool success = false; + bool success = false; auto i = alu_vec_ready.begin(); auto e = alu_vec_ready.end(); while (i != e) { sfn_log << SfnLog::schedule << "Try schedule to vec " << **i; if (!m_current_block->try_reserve_kcache(**i)) { - sfn_log << SfnLog::schedule << " failed (kcache)\n"; + sfn_log << SfnLog::schedule << " failed (kcache)\n"; ++i; continue; } @@ -685,17 +680,19 @@ bool BlockSheduler::schedule_alu_to_group_vec(AluGroup *group) return success; } -bool BlockSheduler::schedule_alu_to_group_trans(AluGroup *group, std::list& readylist) +bool +BlockSheduler::schedule_alu_to_group_trans(AluGroup *group, + std::list& readylist) { assert(group); - bool success = false; + bool success = false; auto i = readylist.begin(); auto e = readylist.end(); while (i != e) { sfn_log << SfnLog::schedule << "Try schedule to trans " << **i; if (!m_current_block->try_reserve_kcache(**i)) { - sfn_log << SfnLog::schedule << " failed (kcache)\n"; + sfn_log << SfnLog::schedule << " failed (kcache)\n"; ++i; continue; } @@ -716,7 +713,8 @@ bool BlockSheduler::schedule_alu_to_group_trans(AluGroup *group, std::list -bool BlockSheduler::schedule(std::list& ready_list) +bool +BlockSheduler::schedule(std::list& ready_list) { if (!ready_list.empty() && m_current_block->remaining_slots() > 0) { auto ii = ready_list.begin(); @@ -730,7 +728,8 @@ bool BlockSheduler::schedule(std::list& ready_list) } template -bool BlockSheduler::schedule_block(std::list& ready_list) +bool +BlockSheduler::schedule_block(std::list& ready_list) { bool success = false; while (!ready_list.empty() && m_current_block->remaining_slots() > 0) { @@ -745,8 +744,9 @@ bool BlockSheduler::schedule_block(std::list& ready_list) return success; } - -bool BlockSheduler::schedule_exports(Shader::ShaderBlocks& out_blocks, std::list& ready_list) +bool +BlockSheduler::schedule_exports(Shader::ShaderBlocks& out_blocks, + std::list& ready_list) { if (m_current_block->type() != Block::cf) start_new_block(out_blocks, Block::cf); @@ -757,9 +757,15 @@ bool BlockSheduler::schedule_exports(Shader::ShaderBlocks& out_blocks, std::list (*ii)->set_scheduled(); m_current_block->push_back(*ii); switch ((*ii)->export_type()) { - case ExportInstr::pos: m_last_pos = *ii; break; - case ExportInstr::param: m_last_param = *ii; break; - case ExportInstr::pixel: m_last_pixel = *ii; break; + case ExportInstr::pos: + m_last_pos = *ii; + break; + case ExportInstr::param: + m_last_param = *ii; + break; + case ExportInstr::pixel: + m_last_pixel = *ii; + break; } (*ii)->set_is_last_export(false); ready_list.erase(ii); @@ -768,7 +774,8 @@ bool BlockSheduler::schedule_exports(Shader::ShaderBlocks& out_blocks, std::list return false; } -bool BlockSheduler::collect_ready(CollectInstructions &available) +bool +BlockSheduler::collect_ready(CollectInstructions& available) { sfn_log << SfnLog::schedule << "Ready instructions\n"; bool result = false; @@ -787,7 +794,9 @@ bool BlockSheduler::collect_ready(CollectInstructions &available) return result; } -bool BlockSheduler::collect_ready_alu_vec(std::list& ready, std::list& available) +bool +BlockSheduler::collect_ready_alu_vec(std::list& ready, + std::list& available) { auto i = available.begin(); auto e = available.end(); @@ -802,10 +811,10 @@ bool BlockSheduler::collect_ready_alu_vec(std::list& ready, std::lis int priority = 0; /* LDS fetches that use static offsets are usually ready ery fast, - * so that they would get schedules early, and this leaves the problem - * that we allocate too many registers with just constant values, - * and this will make problems wih RA. So limit the number of LDS - * address registers. + * so that they would get schedules early, and this leaves the + * problem that we allocate too many registers with just constant + * values, and this will make problems wih RA. So limit the number of + * LDS address registers. */ if ((*i)->has_alu_flag(alu_lds_address)) { if (m_lds_addr_count > 64) { @@ -823,7 +832,7 @@ bool BlockSheduler::collect_ready_alu_vec(std::list& ready, std::lis * for everything else we look at the register use */ if ((*i)->has_lds_access()) - priority = 100000; + priority = 100000; else if (AluGroup::has_t()) { auto opinfo = alu_ops.find((*i)->opcode()); assert(opinfo != alu_ops.end()); @@ -844,77 +853,65 @@ bool BlockSheduler::collect_ready_alu_vec(std::list& ready, std::lis ++i; } - for (auto& i: ready) + for (auto& i : ready) sfn_log << SfnLog::schedule << "V: " << *i << "\n"; ready.sort([](const AluInstr *lhs, const AluInstr *rhs) { - return lhs->priority() > rhs->priority();}); + return lhs->priority() > rhs->priority(); + }); - for (auto& i: ready) + for (auto& i : ready) sfn_log << SfnLog::schedule << "V (S): " << *i << "\n"; return !ready.empty(); } -template -struct type_char { - +template struct type_char { }; - -template <> -struct type_char { +template <> struct type_char { static constexpr const char value = 'A'; }; -template <> -struct type_char { +template <> struct type_char { static constexpr const char value = 'G'; }; -template <> -struct type_char { +template <> struct type_char { static constexpr const char value = 'E'; }; -template <> -struct type_char { +template <> struct type_char { static constexpr const char value = 'T'; }; -template <> -struct type_char { +template <> struct type_char { static constexpr const char value = 'F'; }; -template <> -struct type_char { +template <> struct type_char { static constexpr const char value = 'M'; }; -template <> -struct type_char { +template <> struct type_char { static constexpr const char value = 'R'; }; -template <> -struct type_char { +template <> struct type_char { static constexpr const char value = 'X'; }; -template <> -struct type_char { +template <> struct type_char { static constexpr const char value = 'S'; }; -template <> -struct type_char { +template <> struct type_char { static constexpr const char value = 'I'; }; - template -bool BlockSheduler::collect_ready_type(std::list& ready, std::list& available) +bool +BlockSheduler::collect_ready_type(std::list& ready, std::list& available) { auto i = available.begin(); auto e = available.end(); @@ -930,10 +927,10 @@ bool BlockSheduler::collect_ready_type(std::list& ready, std::list& av ++i; } - for (auto& i: ready) + for (auto& i : ready) sfn_log << SfnLog::schedule << type_char::value << "; " << *i << "\n"; return !ready.empty(); } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_scheduler.h b/src/gallium/drivers/r600/sfn/sfn_scheduler.h index 749b0a47431..e20950bcdec 100644 --- a/src/gallium/drivers/r600/sfn/sfn_scheduler.h +++ b/src/gallium/drivers/r600/sfn/sfn_scheduler.h @@ -31,8 +31,8 @@ namespace r600 { - -Shader *schedule(Shader *original); +Shader * +schedule(Shader *original); } diff --git a/src/gallium/drivers/r600/sfn/sfn_shader.cpp b/src/gallium/drivers/r600/sfn/sfn_shader.cpp index e82c32f2f3f..538be0c1003 100644 --- a/src/gallium/drivers/r600/sfn/sfn_shader.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_shader.cpp @@ -25,29 +25,25 @@ */ #include "sfn_shader.h" + +#include "gallium/drivers/r600/r600_shader.h" +#include "nir.h" +#include "sfn_debug.h" #include "sfn_instr.h" #include "sfn_instr_alugroup.h" +#include "sfn_instr_controlflow.h" #include "sfn_instr_export.h" #include "sfn_instr_fetch.h" #include "sfn_instr_lds.h" #include "sfn_instr_mem.h" - -#include "sfn_instr_controlflow.h" #include "sfn_liverangeevaluator.h" - #include "sfn_shader_cs.h" #include "sfn_shader_fs.h" -#include "sfn_shader_vs.h" #include "sfn_shader_gs.h" #include "sfn_shader_tess.h" - -#include "sfn_debug.h" -#include "gallium/drivers/r600/r600_shader.h" - +#include "sfn_shader_vs.h" #include "tgsi/tgsi_from_mesa.h" -#include "nir.h" - #include #include @@ -60,7 +56,9 @@ r600_get_varying_semantic(unsigned varying_location) { std::pair result; tgsi_get_gl_varying_semantic(static_cast(varying_location), - true, &result.first, &result.second); + true, + &result.first, + &result.second); if (result.first == TGSI_SEMANTIC_GENERIC) { result.second += 9; @@ -70,7 +68,8 @@ r600_get_varying_semantic(unsigned varying_location) return result; } -void ShaderIO::set_sid(int sid) +void +ShaderIO::set_sid(int sid) { m_sid = sid; switch (m_name) { @@ -81,24 +80,26 @@ void ShaderIO::set_sid(int sid) case TGSI_SEMANTIC_SAMPLEMASK: case TGSI_SEMANTIC_CLIPVERTEX: m_spi_sid = 0; - break; + break; case TGSI_SEMANTIC_GENERIC: case TGSI_SEMANTIC_TEXCOORD: case TGSI_SEMANTIC_PCOORD: m_spi_sid = m_sid + 1; - break; + break; default: /* For non-generic params - pack name and sid into 8 bits */ m_spi_sid = (0x80 | (m_name << 3) | m_sid) + 1; } } -void ShaderIO::override_spi_sid(int spi) +void +ShaderIO::override_spi_sid(int spi) { m_spi_sid = spi; } -void ShaderIO::print(std::ostream& os) const +void +ShaderIO::print(std::ostream& os) const { os << m_type << " LOC:" << m_location << " NAME:" << m_name; do_print(os); @@ -108,44 +109,42 @@ void ShaderIO::print(std::ostream& os) const } } - ShaderIO::ShaderIO(const char *type, int loc, int name): - m_type(type), - m_location(loc), - m_name(name) + m_type(type), + m_location(loc), + m_name(name) { } ShaderOutput::ShaderOutput(): - ShaderIO("OUTPUT", -1, -1) + ShaderIO("OUTPUT", -1, -1) { } ShaderOutput::ShaderOutput(int location, int name, int writemask): - ShaderIO("OUTPUT", location, name), - m_writemask(writemask) + ShaderIO("OUTPUT", location, name), + m_writemask(writemask) { - } -void ShaderOutput::do_print(std::ostream& os) const +void +ShaderOutput::do_print(std::ostream& os) const { os << " MASK:" << m_writemask; } - ShaderInput::ShaderInput(int location, int name): - ShaderIO("INPUT", location, name) + ShaderIO("INPUT", location, name) { } ShaderInput::ShaderInput(): - ShaderInput(-1, -1) + ShaderInput(-1, -1) { } - -void ShaderInput::do_print(std::ostream& os) const +void +ShaderInput::do_print(std::ostream& os) const { if (m_interpolator) os << " INTERP:" << m_interpolator; @@ -155,37 +154,43 @@ void ShaderInput::do_print(std::ostream& os) const os << " USE_CENTROID"; } -void ShaderInput::set_interpolator(int interp, int interp_loc, bool uses_interpolate_at_centroid) +void +ShaderInput::set_interpolator(int interp, + int interp_loc, + bool uses_interpolate_at_centroid) { m_interpolator = interp; m_interpolate_loc = interp_loc; m_uses_interpolate_at_centroid = uses_interpolate_at_centroid; } -void ShaderInput::set_uses_interpolate_at_centroid() +void +ShaderInput::set_uses_interpolate_at_centroid() { m_uses_interpolate_at_centroid = true; } Shader::Shader(const char *type_id): - m_current_block(nullptr), - m_type_id(type_id), - m_chip_class(ISA_CC_R600), - m_next_block(0) + m_current_block(nullptr), + m_type_id(type_id), + m_chip_class(ISA_CC_R600), + m_next_block(0) { m_instr_factory = new InstrFactory(); m_chain_instr.this_shader = this; start_new_block(0); } -void Shader::set_input_gpr(int driver_lcation, int gpr) +void +Shader::set_input_gpr(int driver_lcation, int gpr) { auto i = m_inputs.find(driver_lcation); assert(i != m_inputs.end()); i->second.set_gpr(gpr); } -bool Shader::add_info_from_string(std::istream& is) +bool +Shader::add_info_from_string(std::istream& is) { std::string type; is >> type; @@ -205,11 +210,11 @@ bool Shader::add_info_from_string(std::istream& is) if (type == "ARRAYS") return allocate_arrays_from_string(is); - return false; } -void Shader::emit_instruction_from_string(const std::string& s) +void +Shader::emit_instruction_from_string(const std::string& s) { sfn_log << SfnLog::instr << "Create Instr from '" << s << "'\n"; @@ -234,7 +239,8 @@ void Shader::emit_instruction_from_string(const std::string& s) } } -bool Shader::read_output(std::istream& is) +bool +Shader::read_output(std::istream& is) { string value; is >> value; @@ -259,8 +265,8 @@ bool Shader::read_output(std::istream& is) return true; } - -bool Shader::read_input(std::istream& is) +bool +Shader::read_input(std::istream& is) { string value; is >> value; @@ -304,7 +310,8 @@ bool Shader::read_input(std::istream& is) return true; } -bool Shader::allocate_registers_from_string(std::istream& is, Pin pin) +bool +Shader::allocate_registers_from_string(std::istream& is, Pin pin) { std::string line; if (!std::getline(is, line)) @@ -312,7 +319,7 @@ bool Shader::allocate_registers_from_string(std::istream& is, Pin pin) std::istringstream iline(line); - while (!iline.eof()) { + while (!iline.eof()) { string reg_str; iline >> reg_str; @@ -322,7 +329,7 @@ bool Shader::allocate_registers_from_string(std::istream& is, Pin pin) if (strchr(reg_str.c_str(), '@')) { value_factory().dest_from_string(reg_str); } else { - RegisterVec4::Swizzle swz = {0,1,2,3}; + RegisterVec4::Swizzle swz = {0, 1, 2, 3}; auto regs = value_factory().dest_vec4_from_string(reg_str, swz, pin); for (int i = 0; i < 4; ++i) { if (swz[i] < 4 && pin == pin_fully) { @@ -334,7 +341,8 @@ bool Shader::allocate_registers_from_string(std::istream& is, Pin pin) return true; } -bool Shader::allocate_arrays_from_string(std::istream& is) +bool +Shader::allocate_arrays_from_string(std::istream& is) { std::string line; if (!std::getline(is, line)) @@ -342,7 +350,7 @@ bool Shader::allocate_arrays_from_string(std::istream& is) std::istringstream iline(line); - while (!iline.eof()) { + while (!iline.eof()) { string reg_str; iline >> reg_str; @@ -354,7 +362,8 @@ bool Shader::allocate_arrays_from_string(std::istream& is) return true; } -bool Shader::read_chipclass(std::istream& is) +bool +Shader::read_chipclass(std::istream& is) { string name; is >> name; @@ -371,41 +380,55 @@ bool Shader::read_chipclass(std::istream& is) return true; } -void Shader::allocate_reserved_registers() +void +Shader::allocate_reserved_registers() { m_instr_factory->value_factory().set_virtual_register_base(0); auto reserved_registers_end = do_allocate_reserved_registers(); m_instr_factory->value_factory().set_virtual_register_base(reserved_registers_end); if (!m_atomics.empty()) { m_atomic_update = value_factory().temp_register(); - auto alu = new AluInstr(op1_mov, m_atomic_update, + auto alu = new AluInstr(op1_mov, + m_atomic_update, value_factory().one_i(), AluInstr::last_write); alu->set_alu_flag(alu_no_schedule_bias); emit_instruction(alu); } - if(m_flags.test(sh_needs_sbo_ret_address)) { + if (m_flags.test(sh_needs_sbo_ret_address)) { m_rat_return_address = value_factory().temp_register(0); auto temp0 = value_factory().temp_register(0); auto temp1 = value_factory().temp_register(1); auto temp2 = value_factory().temp_register(2); auto group = new AluGroup(); - group->add_instruction(new AluInstr(op1_mbcnt_32lo_accum_prev_int, temp0, value_factory().literal(-1), {alu_write})); - group->add_instruction(new AluInstr(op1_mbcnt_32hi_int, temp1, value_factory().literal(-1), {alu_write})); + group->add_instruction(new AluInstr( + op1_mbcnt_32lo_accum_prev_int, temp0, value_factory().literal(-1), {alu_write})); + group->add_instruction(new AluInstr( + op1_mbcnt_32hi_int, temp1, value_factory().literal(-1), {alu_write})); emit_instruction(group); - emit_instruction(new AluInstr(op3_muladd_uint24, temp2, value_factory().inline_const(ALU_SRC_SE_ID, 0), - value_factory().literal(256), value_factory().inline_const(ALU_SRC_HW_WAVE_ID, 0), {alu_write, alu_last_instr})); - emit_instruction(new AluInstr(op3_muladd_uint24, m_rat_return_address, - temp2, value_factory().literal(0x40), temp0, + emit_instruction(new AluInstr(op3_muladd_uint24, + temp2, + value_factory().inline_const(ALU_SRC_SE_ID, 0), + value_factory().literal(256), + value_factory().inline_const(ALU_SRC_HW_WAVE_ID, 0), + {alu_write, alu_last_instr})); + emit_instruction(new AluInstr(op3_muladd_uint24, + m_rat_return_address, + temp2, + value_factory().literal(0x40), + temp0, {alu_write, alu_last_instr})); } } -Shader *Shader::translate_from_nir(nir_shader *nir, const pipe_stream_output_info *so_info, - struct r600_shader* gs_shader, - r600_shader_key& key, r600_chip_class chip_class) +Shader * +Shader::translate_from_nir(nir_shader *nir, + const pipe_stream_output_info *so_info, + struct r600_shader *gs_shader, + r600_shader_key& key, + r600_chip_class chip_class) { Shader *shader = nullptr; @@ -415,13 +438,13 @@ Shader *Shader::translate_from_nir(nir_shader *nir, const pipe_stream_output_inf shader = new FragmentShaderEG(key); else shader = new FragmentShaderR600(key); - break; + break; case MESA_SHADER_VERTEX: shader = new VertexShader(so_info, gs_shader, key); - break; + break; case MESA_SHADER_GEOMETRY: shader = new GeometryShader(key); - break; + break; case MESA_SHADER_TESS_CTRL: shader = new TCSShader(key); break; @@ -445,29 +468,31 @@ Shader *Shader::translate_from_nir(nir_shader *nir, const pipe_stream_output_inf return shader; } -void Shader::set_info(nir_shader *nir) +void +Shader::set_info(nir_shader *nir) { m_scratch_size = nir->scratch_size; } -ValueFactory& Shader::value_factory() +ValueFactory& +Shader::value_factory() { return m_instr_factory->value_factory(); } - -bool Shader::process(nir_shader *nir) +bool +Shader::process(nir_shader *nir) { m_ssbo_image_offset = nir->info.num_images; if (nir->info.use_legacy_math_rules) set_flag(sh_legacy_math_rules); - nir_foreach_uniform_variable(var, nir) - scan_uniforms(var); + nir_foreach_uniform_variable(var, nir) scan_uniforms(var); // at this point all functions should be inlined - const nir_function *func = reinterpret_cast(exec_list_get_head_const(&nir->functions)); + const nir_function *func = + reinterpret_cast(exec_list_get_head_const(&nir->functions)); if (!scan_shader(func)) return false; @@ -477,7 +502,8 @@ bool Shader::process(nir_shader *nir) allocate_local_registers(&func->impl->registers); sfn_log << SfnLog::trans << "Process shader \n"; - foreach_list_typed(nir_cf_node, node, node, &func->impl->body) { + foreach_list_typed(nir_cf_node, node, node, &func->impl->body) + { if (!process_cf_node(node)) return false; } @@ -487,17 +513,21 @@ bool Shader::process(nir_shader *nir) return true; } -void Shader::allocate_local_registers(const exec_list *registers) +void +Shader::allocate_local_registers(const exec_list *registers) { if (value_factory().allocate_registers(registers)) m_indirect_files |= 1 << TGSI_FILE_TEMPORARY; } -bool Shader::scan_shader(const nir_function *func) +bool +Shader::scan_shader(const nir_function *func) { - nir_foreach_block(block, func->impl) { - nir_foreach_instr(instr, block) { + nir_foreach_block(block, func->impl) + { + nir_foreach_instr(instr, block) + { if (!scan_instruction(instr)) { fprintf(stderr, "Unhandled sysvalue access "); nir_print_instr(instr, stderr); @@ -525,7 +555,8 @@ bool Shader::scan_shader(const nir_function *func) return true; } -bool Shader::scan_uniforms(nir_variable *uniform) +bool +Shader::scan_uniforms(nir_variable *uniform) { if (uniform->type->contains_atomic()) { int natomics = uniform->type->atomic_size() / ATOMIC_COUNTER_SIZE; @@ -544,32 +575,30 @@ bool Shader::scan_uniforms(nir_variable *uniform) atom.start = uniform->data.offset >> 2; atom.end = atom.start + natomics - 1; - if (m_atomic_base_map.find(uniform->data.binding) == - m_atomic_base_map.end()) + if (m_atomic_base_map.find(uniform->data.binding) == m_atomic_base_map.end()) m_atomic_base_map[uniform->data.binding] = m_next_hwatomic_loc; m_next_hwatomic_loc += natomics; - m_atomic_file_count += atom.end - atom.start + 1; + m_atomic_file_count += atom.end - atom.start + 1; - sfn_log << SfnLog::io << "HW_ATOMIC file count: " - << m_atomic_file_count << "\n"; + sfn_log << SfnLog::io << "HW_ATOMIC file count: " << m_atomic_file_count << "\n"; m_atomics.push_back(atom); } - auto type = uniform->type->is_array() ? uniform->type->without_array(): uniform->type; + auto type = uniform->type->is_array() ? uniform->type->without_array() : uniform->type; if (type->is_image() || uniform->data.mode == nir_var_mem_ssbo) { m_flags.set(sh_uses_images); - if (uniform->type->is_array() && ! (uniform->data.mode == nir_var_mem_ssbo)) + if (uniform->type->is_array() && !(uniform->data.mode == nir_var_mem_ssbo)) m_indirect_files |= 1 << TGSI_FILE_IMAGE; } return true; } - -bool Shader::scan_instruction(nir_instr *instr) +bool +Shader::scan_instruction(nir_instr *instr) { if (do_scan_instruction(instr)) return true; @@ -614,38 +643,38 @@ bool Shader::scan_instruction(nir_instr *instr) case nir_intrinsic_memory_barrier: case nir_intrinsic_group_memory_barrier: m_chain_instr.prepare_mem_barrier = true; - default: - ; + default:; } return true; } -bool Shader::process_cf_node(nir_cf_node *node) +bool +Shader::process_cf_node(nir_cf_node *node) { SFN_TRACE_FUNC(SfnLog::flow, "CF"); switch (node->type) { case nir_cf_node_block: - return process_block(nir_cf_node_as_block(node)); + return process_block(nir_cf_node_as_block(node)); case nir_cf_node_if: - return process_if(nir_cf_node_as_if(node)); + return process_if(nir_cf_node_as_if(node)); case nir_cf_node_loop: - return process_loop(nir_cf_node_as_loop(node)); + return process_loop(nir_cf_node_as_loop(node)); default: - return false; + return false; } - } static bool -child_block_empty (const exec_list& list) +child_block_empty(const exec_list& list) { if (list.is_empty()) return true; bool result = true; - foreach_list_typed(nir_cf_node, n, node, &list) { + foreach_list_typed(nir_cf_node, n, node, &list) + { if (n->type == nir_cf_node_block) { if (!nir_cf_node_as_block(n)->instr_list.is_empty()) @@ -657,24 +686,28 @@ child_block_empty (const exec_list& list) return result; } -bool Shader::process_if(nir_if *if_stmt) +bool +Shader::process_if(nir_if *if_stmt) { SFN_TRACE_FUNC(SfnLog::flow, "IF"); if (!emit_if_start(if_stmt)) return false; - foreach_list_typed(nir_cf_node, n, node, &if_stmt->then_list) { + foreach_list_typed(nir_cf_node, n, node, &if_stmt->then_list) + { SFN_TRACE_FUNC(SfnLog::flow, "IF-then"); - if (!process_cf_node(n)) - return false; + if (!process_cf_node(n)) + return false; } if (!child_block_empty(if_stmt->else_list)) { if (!emit_control_flow(ControlFlowInstr::cf_else)) return false; - foreach_list_typed(nir_cf_node, n, node, &if_stmt->else_list) - if (!process_cf_node(n)) return false; + foreach_list_typed(nir_cf_node, + n, + node, + &if_stmt->else_list) if (!process_cf_node(n)) return false; } if (!emit_control_flow(ControlFlowInstr::cf_endif)) @@ -683,11 +716,15 @@ bool Shader::process_if(nir_if *if_stmt) return true; } -bool Shader::emit_if_start(nir_if *if_stmt) +bool +Shader::emit_if_start(nir_if *if_stmt) { auto value = value_factory().src(if_stmt->condition, 0); - AluInstr *pred = new AluInstr(op2_pred_setne_int, value_factory().temp_register(), - value, value_factory().zero(), AluInstr::last); + AluInstr *pred = new AluInstr(op2_pred_setne_int, + value_factory().temp_register(), + value, + value_factory().zero(), + AluInstr::last); pred->set_alu_flag(alu_update_exec); pred->set_alu_flag(alu_update_pred); pred->set_cf_type(cf_alu_push_before); @@ -698,7 +735,8 @@ bool Shader::emit_if_start(nir_if *if_stmt) return true; } -bool Shader::emit_control_flow(ControlFlowInstr::CFType type) +bool +Shader::emit_control_flow(ControlFlowInstr::CFType type) { auto ir = new ControlFlowInstr(type); emit_instruction(ir); @@ -708,29 +746,31 @@ bool Shader::emit_control_flow(ControlFlowInstr::CFType type) m_loops.push_back(ir); m_nloops++; depth = 1; - break; + break; case ControlFlowInstr::cf_loop_end: m_loops.pop_back(); FALLTHROUGH; case ControlFlowInstr::cf_endif: depth = -1; - break; - default: - ; + break; + default:; } start_new_block(depth); return true; } -bool Shader::process_loop(nir_loop *node) +bool +Shader::process_loop(nir_loop *node) { SFN_TRACE_FUNC(SfnLog::flow, "LOOP"); if (!emit_control_flow(ControlFlowInstr::cf_loop_begin)) return false; - foreach_list_typed(nir_cf_node, n, node, &node->body) - if (!process_cf_node(n)) return false; + foreach_list_typed(nir_cf_node, + n, + node, + &node->body) if (!process_cf_node(n)) return false; if (!emit_control_flow(ControlFlowInstr::cf_loop_end)) return false; @@ -738,28 +778,31 @@ bool Shader::process_loop(nir_loop *node) return true; } -bool Shader::process_block(nir_block *block) +bool +Shader::process_block(nir_block *block) { SFN_TRACE_FUNC(SfnLog::flow, "BLOCK"); - nir_foreach_instr(instr, block) { + nir_foreach_instr(instr, block) + { sfn_log << SfnLog::instr << "FROM:" << *instr << "\n"; bool r = process_instr(instr); if (!r) { - sfn_log << SfnLog::err << "R600: Unsupported instruction: " - << *instr << "\n"; + sfn_log << SfnLog::err << "R600: Unsupported instruction: " << *instr << "\n"; return false; } } return true; } -bool Shader::process_instr(nir_instr *instr) +bool +Shader::process_instr(nir_instr *instr) { return m_instr_factory->from_nir(instr, *this); } -bool Shader::process_intrinsic(nir_intrinsic_instr *intr) +bool +Shader::process_intrinsic(nir_intrinsic_instr *intr) { if (process_stage_intrinsic(intr)) return true; @@ -773,16 +816,26 @@ bool Shader::process_intrinsic(nir_intrinsic_instr *intr) return true; switch (intr->intrinsic) { - case nir_intrinsic_store_output: return store_output(intr); - case nir_intrinsic_load_input: return load_input(intr); - case nir_intrinsic_load_uniform: return load_uniform(intr); - case nir_intrinsic_load_ubo_vec4: return load_ubo(intr); - case nir_intrinsic_store_scratch: return emit_store_scratch(intr); - case nir_intrinsic_load_scratch: return emit_load_scratch(intr); - case nir_intrinsic_store_local_shared_r600: return emit_local_store(intr); - case nir_intrinsic_load_local_shared_r600: return emit_local_load(intr); - case nir_intrinsic_load_tcs_in_param_base_r600: return emit_load_tcs_param_base(intr, 0); - case nir_intrinsic_load_tcs_out_param_base_r600: return emit_load_tcs_param_base(intr, 16); + case nir_intrinsic_store_output: + return store_output(intr); + case nir_intrinsic_load_input: + return load_input(intr); + case nir_intrinsic_load_uniform: + return load_uniform(intr); + case nir_intrinsic_load_ubo_vec4: + return load_ubo(intr); + case nir_intrinsic_store_scratch: + return emit_store_scratch(intr); + case nir_intrinsic_load_scratch: + return emit_load_scratch(intr); + case nir_intrinsic_store_local_shared_r600: + return emit_local_store(intr); + case nir_intrinsic_load_local_shared_r600: + return emit_local_load(intr); + case nir_intrinsic_load_tcs_in_param_base_r600: + return emit_load_tcs_param_base(intr, 0); + case nir_intrinsic_load_tcs_out_param_base_r600: + return emit_load_tcs_param_base(intr, 16); // We only emit the group barrier, barriers across work groups // are not yet implemented case nir_intrinsic_control_barrier: @@ -812,12 +865,13 @@ bool Shader::process_intrinsic(nir_intrinsic_instr *intr) return emit_shader_clock(intr); default: - return false; + return false; } } static ESDOp -lds_op_from_intrinsic(nir_intrinsic_op op, bool ret) { +lds_op_from_intrinsic(nir_intrinsic_op op, bool ret) +{ switch (op) { case nir_intrinsic_shared_atomic_add: return ret ? LDS_ADD_RET : LDS_ADD; @@ -844,7 +898,8 @@ lds_op_from_intrinsic(nir_intrinsic_op op, bool ret) { } } -PRegister Shader::emit_load_to_register(PVirtualValue src) +PRegister +Shader::emit_load_to_register(PVirtualValue src) { assert(src); PRegister dest = src->as_register(); @@ -856,7 +911,8 @@ PRegister Shader::emit_load_to_register(PVirtualValue src) return dest; } -bool Shader::emit_atomic_local_shared(nir_intrinsic_instr* instr) +bool +Shader::emit_atomic_local_shared(nir_intrinsic_instr *instr) { bool uses_retval = !instr->dest.is_ssa || !list_is_empty(&instr->dest.ssa.uses); @@ -871,13 +927,15 @@ bool Shader::emit_atomic_local_shared(nir_intrinsic_instr* instr) AluInstr::SrcValues src; src.push_back(vf.src(instr->src[1], 0)); - if (unlikely(instr->intrinsic ==nir_intrinsic_shared_atomic_comp_swap)) + if (unlikely(instr->intrinsic == nir_intrinsic_shared_atomic_comp_swap)) src.push_back(vf.src(instr->src[2], 0)); emit_instruction(new LDSAtomicInstr(op, dest_value, address, src)); return true; } -auto Shader::evaluate_resource_offset(nir_intrinsic_instr *instr, int src_id) -> std::pair +auto +Shader::evaluate_resource_offset(nir_intrinsic_instr *instr, int src_id) + -> std::pair { auto& vf = value_factory(); @@ -893,22 +951,20 @@ auto Shader::evaluate_resource_offset(nir_intrinsic_instr *instr, int src_id) -> uav_id = uav_id_val->as_register(); } else { uav_id = vf.temp_register(); - emit_instruction(new AluInstr(op1_mov, uav_id, uav_id_val, - AluInstr::last_write)); + emit_instruction(new AluInstr(op1_mov, uav_id, uav_id_val, AluInstr::last_write)); } } return std::make_pair(offset, uav_id); } - -bool Shader::emit_store_scratch(nir_intrinsic_instr *intr) +bool +Shader::emit_store_scratch(nir_intrinsic_instr *intr) { auto& vf = m_instr_factory->value_factory(); int writemask = nir_intrinsic_write_mask(intr); - - RegisterVec4::Swizzle swz = {7,7,7,7}; + RegisterVec4::Swizzle swz = {7, 7, 7, 7}; for (unsigned i = 0; i < intr->num_components; ++i) swz[i] = (1 << i) & writemask ? i : 7; @@ -929,7 +985,6 @@ bool Shader::emit_store_scratch(nir_intrinsic_instr *intr) auto address = vf.src(intr->src[1], 0); - int align = nir_intrinsic_align_mul(intr); int align_offset = nir_intrinsic_align_offset(intr); @@ -949,12 +1004,13 @@ bool Shader::emit_store_scratch(nir_intrinsic_instr *intr) if (offset >= 0) { ws_ir = new ScratchIOInstr(value, offset, align, align_offset, writemask); } else { - auto addr_temp = vf.temp_register(0); + auto addr_temp = vf.temp_register(0); auto load_addr = new AluInstr(op1_mov, addr_temp, address, AluInstr::last_write); load_addr->set_alu_flag(alu_no_schedule_bias); emit_instruction(load_addr); - ws_ir = new ScratchIOInstr(value, addr_temp, align, align_offset, writemask, m_scratch_size); + ws_ir = new ScratchIOInstr( + value, addr_temp, align, align_offset, writemask, m_scratch_size); } emit_instruction(ws_ir); @@ -962,13 +1018,14 @@ bool Shader::emit_store_scratch(nir_intrinsic_instr *intr) return true; } -bool Shader::emit_load_scratch(nir_intrinsic_instr *intr) +bool +Shader::emit_load_scratch(nir_intrinsic_instr *intr) { auto addr = value_factory().src(intr->src[0], 0); auto dest = value_factory().dest_vec4(intr->dest, pin_group); if (chip_class() >= ISA_CC_R700) { - RegisterVec4::Swizzle dest_swz = {7,7,7,7}; + RegisterVec4::Swizzle dest_swz = {7, 7, 7, 7}; for (unsigned i = 0; i < intr->num_components; ++i) dest_swz[i] = i; @@ -980,7 +1037,6 @@ bool Shader::emit_load_scratch(nir_intrinsic_instr *intr) int align = nir_intrinsic_align_mul(intr); int align_offset = nir_intrinsic_align_offset(intr); - int offset = -1; if (addr->as_literal()) { offset = addr->as_literal()->value(); @@ -996,25 +1052,24 @@ bool Shader::emit_load_scratch(nir_intrinsic_instr *intr) if (offset >= 0) { ir = new ScratchIOInstr(dest, offset, align, align_offset, 0xf, true); } else { - auto addr_temp = value_factory().temp_register(0); + auto addr_temp = value_factory().temp_register(0); auto load_addr = new AluInstr(op1_mov, addr_temp, addr, AluInstr::last_write); load_addr->set_alu_flag(alu_no_schedule_bias); emit_instruction(load_addr); - ir = new ScratchIOInstr(dest, addr_temp, align, align_offset, 0xf, - m_scratch_size, true); + ir = new ScratchIOInstr( + dest, addr_temp, align, align_offset, 0xf, m_scratch_size, true); } emit_instruction(ir); } - m_flags.set(sh_needs_scratch_space); return true; - } -bool Shader::emit_local_store(nir_intrinsic_instr *instr) +bool +Shader::emit_local_store(nir_intrinsic_instr *instr) { unsigned write_mask = nir_intrinsic_write_mask(instr); @@ -1033,12 +1088,14 @@ bool Shader::emit_local_store(nir_intrinsic_instr *instr) } else { auto value = value_factory().src(instr->src[0], swizzle_base); auto value1 = value_factory().src(instr->src[0], swizzle_base + 1); - emit_instruction(new LDSAtomicInstr(LDS_WRITE_REL, nullptr, address, {value, value1})); + emit_instruction( + new LDSAtomicInstr(LDS_WRITE_REL, nullptr, address, {value, value1})); } return true; } -bool Shader::emit_local_load(nir_intrinsic_instr* instr) +bool +Shader::emit_local_load(nir_intrinsic_instr *instr) { auto address = value_factory().src_vec(instr->src[0], instr->num_components); auto dest_value = value_factory().dest_vec(instr->dest, instr->num_components); @@ -1046,17 +1103,20 @@ bool Shader::emit_local_load(nir_intrinsic_instr* instr) return true; } -void Shader::chain_scratch_read(Instr *instr) +void +Shader::chain_scratch_read(Instr *instr) { m_chain_instr.apply(instr, &m_chain_instr.last_scratch_instr); } -void Shader::chain_ssbo_read(Instr *instr) +void +Shader::chain_ssbo_read(Instr *instr) { m_chain_instr.apply(instr, &m_chain_instr.last_ssbo_instr); } -bool Shader::emit_wait_ack() +bool +Shader::emit_wait_ack() { start_new_block(0); emit_instruction(new ControlFlowInstr(ControlFlowInstr::cf_wait_ack)); @@ -1064,26 +1124,27 @@ bool Shader::emit_wait_ack() return true; } -void Shader::InstructionChain::visit(ScratchIOInstr *instr) +void +Shader::InstructionChain::visit(ScratchIOInstr *instr) { apply(instr, &last_scratch_instr); } -void Shader::InstructionChain::visit(GDSInstr *instr) +void +Shader::InstructionChain::visit(GDSInstr *instr) { apply(instr, &last_gds_instr); - Instr::Flags flag = instr->has_instr_flag(Instr::helper) ? - Instr::helper: Instr::vpm; + Instr::Flags flag = instr->has_instr_flag(Instr::helper) ? Instr::helper : Instr::vpm; for (auto& loop : this_shader->m_loops) { loop->set_instr_flag(flag); } } -void Shader::InstructionChain::visit(RatInstr *instr) +void +Shader::InstructionChain::visit(RatInstr *instr) { apply(instr, &last_ssbo_instr); - Instr::Flags flag = instr->has_instr_flag(Instr::helper) ? - Instr::helper: Instr::vpm; + Instr::Flags flag = instr->has_instr_flag(Instr::helper) ? Instr::helper : Instr::vpm; for (auto& loop : this_shader->m_loops) { loop->set_instr_flag(flag); } @@ -1095,35 +1156,41 @@ void Shader::InstructionChain::visit(RatInstr *instr) this_shader->start_new_block(0); } -void Shader::InstructionChain::apply(Instr *current, Instr **last) { +void +Shader::InstructionChain::apply(Instr *current, Instr **last) +{ if (*last) current->add_required_instr(*last); *last = current; } -void Shader::emit_instruction(PInst instr) +void +Shader::emit_instruction(PInst instr) { sfn_log << SfnLog::instr << " " << *instr << "\n"; instr->accept(m_chain_instr); m_current_block->push_back(instr); } -bool Shader::load_uniform(nir_intrinsic_instr *intr) +bool +Shader::load_uniform(nir_intrinsic_instr *intr) { auto literal = nir_src_as_const_value(intr->src[0]); if (literal) { AluInstr *ir = nullptr; - auto pin = intr->dest.is_ssa && nir_dest_num_components(intr->dest) == 1 ? - pin_free : pin_none; + auto pin = intr->dest.is_ssa && nir_dest_num_components(intr->dest) == 1 ? pin_free + : pin_none; for (unsigned i = 0; i < nir_dest_num_components(intr->dest); ++i) { - sfn_log << SfnLog::io << "uniform " - << intr->dest.ssa.index << " const["<< i << "]: "<< intr->const_index[i] << "\n"; + sfn_log << SfnLog::io << "uniform " << intr->dest.ssa.index << " const[" << i + << "]: " << intr->const_index[i] << "\n"; auto uniform = value_factory().uniform(intr, i); - ir = new AluInstr(op1_mov, value_factory().dest(intr->dest, i, pin), - uniform, {alu_write}); + ir = new AluInstr(op1_mov, + value_factory().dest(intr->dest, i, pin), + uniform, + {alu_write}); emit_instruction(ir); } if (ir) @@ -1135,8 +1202,11 @@ bool Shader::load_uniform(nir_intrinsic_instr *intr) } } -bool Shader::load_uniform_indirect(nir_intrinsic_instr *intr, PVirtualValue addr, - int offset , int buffer_id) +bool +Shader::load_uniform_indirect(nir_intrinsic_instr *intr, + PVirtualValue addr, + int offset, + int buffer_id) { auto addr_reg = addr->as_register(); if (!addr) { @@ -1147,22 +1217,27 @@ bool Shader::load_uniform_indirect(nir_intrinsic_instr *intr, PVirtualValue addr RegisterVec4 dest = value_factory().dest_vec4(intr->dest, pin_group); - auto ir = new LoadFromBuffer(dest, {0,1,2,3}, addr_reg, offset, buffer_id, - nullptr, fmt_32_32_32_32_float); + auto ir = new LoadFromBuffer( + dest, {0, 1, 2, 3}, addr_reg, offset, buffer_id, nullptr, fmt_32_32_32_32_float); emit_instruction(ir); m_flags.set(sh_indirect_const_file); return true; } -bool Shader::emit_load_tcs_param_base(nir_intrinsic_instr* instr, int offset) +bool +Shader::emit_load_tcs_param_base(nir_intrinsic_instr *instr, int offset) { auto src = value_factory().temp_register(); - emit_instruction(new AluInstr(op1_mov, src, value_factory().zero(), - AluInstr::last_write)); + emit_instruction( + new AluInstr(op1_mov, src, value_factory().zero(), AluInstr::last_write)); auto dest = value_factory().dest_vec4(instr->dest, pin_group); - auto fetch = new LoadFromBuffer(dest, {0,1,2,3}, src, offset, - R600_LDS_INFO_CONST_BUFFER, nullptr, + auto fetch = new LoadFromBuffer(dest, + {0, 1, 2, 3}, + src, + offset, + R600_LDS_INFO_CONST_BUFFER, + nullptr, fmt_32_32_32_32); fetch->set_fetch_flag(LoadFromBuffer::srf_mode); @@ -1171,20 +1246,25 @@ bool Shader::emit_load_tcs_param_base(nir_intrinsic_instr* instr, int offset) return true; } -bool Shader::emit_shader_clock(nir_intrinsic_instr* instr) +bool +Shader::emit_shader_clock(nir_intrinsic_instr *instr) { auto& vf = value_factory(); auto group = new AluGroup(); - group->add_instruction(new AluInstr(op1_mov, vf.dest(instr->dest, 0, pin_chan), - vf.inline_const(ALU_SRC_TIME_LO, 0), AluInstr::write)); - group->add_instruction(new AluInstr(op1_mov, vf.dest(instr->dest, 1, pin_chan), - vf.inline_const(ALU_SRC_TIME_HI, 0), AluInstr::last_write)); + group->add_instruction(new AluInstr(op1_mov, + vf.dest(instr->dest, 0, pin_chan), + vf.inline_const(ALU_SRC_TIME_LO, 0), + AluInstr::write)); + group->add_instruction(new AluInstr(op1_mov, + vf.dest(instr->dest, 1, pin_chan), + vf.inline_const(ALU_SRC_TIME_HI, 0), + AluInstr::last_write)); emit_instruction(group); return true; } - -bool Shader::emit_barrier(nir_intrinsic_instr* intr) +bool +Shader::emit_barrier(nir_intrinsic_instr *intr) { (void)intr; /* Put barrier into it's own block, so that optimizers and the @@ -1197,18 +1277,19 @@ bool Shader::emit_barrier(nir_intrinsic_instr* intr) return true; } -bool Shader::load_ubo(nir_intrinsic_instr *instr) +bool +Shader::load_ubo(nir_intrinsic_instr *instr) { auto bufid = nir_src_as_const_value(instr->src[0]); auto buf_offset = nir_src_as_const_value(instr->src[1]); if (!buf_offset) { - /* TODO: if bufid is constant then this can also be solved by using the CF indes - * on the ALU block, and this would probably make sense when there are more then one - * loads with the same buffer ID. */ + /* TODO: if bufid is constant then this can also be solved by using the + * CF indes on the ALU block, and this would probably make sense when + * there are more then one loads with the same buffer ID. */ auto addr = value_factory().src(instr->src[1], 0)->as_register(); - RegisterVec4::Swizzle dest_swz {7,7,7,7}; + RegisterVec4::Swizzle dest_swz{7, 7, 7, 7}; auto dest = value_factory().dest_vec4(instr->dest, pin_group); for (unsigned i = 0; i < nir_dest_num_components(instr->dest); ++i) { @@ -1217,12 +1298,12 @@ bool Shader::load_ubo(nir_intrinsic_instr *instr) LoadFromBuffer *ir; if (bufid) { - ir = new LoadFromBuffer(dest, dest_swz, addr, 0, 1 + bufid->u32, - nullptr, fmt_32_32_32_32_float); + ir = new LoadFromBuffer( + dest, dest_swz, addr, 0, 1 + bufid->u32, nullptr, fmt_32_32_32_32_float); } else { auto buffer_id = emit_load_to_register(value_factory().src(instr->src[0], 0)); - ir = new LoadFromBuffer(dest, dest_swz, addr, 0, 1, buffer_id, - fmt_32_32_32_32_float); + ir = new LoadFromBuffer( + dest, dest_swz, addr, 0, 1, buffer_id, fmt_32_32_32_32_float); } emit_instruction(ir); return true; @@ -1233,16 +1314,20 @@ bool Shader::load_ubo(nir_intrinsic_instr *instr) int buf_cmp = nir_intrinsic_component(instr); AluInstr *ir = nullptr; - auto pin = instr->dest.is_ssa && nir_dest_num_components(instr->dest) == 1 ? - pin_free : pin_none; + auto pin = instr->dest.is_ssa && nir_dest_num_components(instr->dest) == 1 + ? pin_free + : pin_none; for (unsigned i = 0; i < nir_dest_num_components(instr->dest); ++i) { - sfn_log << SfnLog::io << "UBO[" << bufid << "] " - << instr->dest.ssa.index << " const["<< i << "]: "<< instr->const_index[i] << "\n"; + sfn_log << SfnLog::io << "UBO[" << bufid << "] " << instr->dest.ssa.index + << " const[" << i << "]: " << instr->const_index[i] << "\n"; - auto uniform = value_factory().uniform(512 + buf_offset->u32, i + buf_cmp, bufid->u32 + 1); - ir = new AluInstr(op1_mov, value_factory().dest(instr->dest, i, pin), - uniform, {alu_write}); + auto uniform = + value_factory().uniform(512 + buf_offset->u32, i + buf_cmp, bufid->u32 + 1); + ir = new AluInstr(op1_mov, + value_factory().dest(instr->dest, i, pin), + uniform, + {alu_write}); emit_instruction(ir); } if (ir) @@ -1255,9 +1340,9 @@ bool Shader::load_ubo(nir_intrinsic_instr *instr) for (unsigned i = 0; i < nir_dest_num_components(instr->dest); ++i) { int cmp = buf_cmp + i; - auto u = new UniformValue(512 + buf_offset->u32, cmp, kc_id); + auto u = new UniformValue(512 + buf_offset->u32, cmp, kc_id); auto dest = value_factory().dest(instr->dest, i, pin_none); - ir = new AluInstr(op1_mov, dest, u, AluInstr::write); + ir = new AluInstr(op1_mov, dest, u, AluInstr::write); emit_instruction(ir); } if (ir) @@ -1267,30 +1352,33 @@ bool Shader::load_ubo(nir_intrinsic_instr *instr) } } -void Shader::start_new_block(int depth) +void +Shader::start_new_block(int depth) { int depth_offset = m_current_block ? m_current_block->nesting_depth() : 0; m_current_block = new Block(depth + depth_offset, m_next_block++); m_root.push_back(m_current_block); } -bool Shader::emit_simple_mov(nir_dest& dest, int chan, PVirtualValue src, Pin pin) +bool +Shader::emit_simple_mov(nir_dest& dest, int chan, PVirtualValue src, Pin pin) { auto dst = value_factory().dest(dest, chan, pin); emit_instruction(new AluInstr(op1_mov, dst, src, AluInstr::last_write)); return true; } -void Shader::print(std::ostream& os) const +void +Shader::print(std::ostream& os) const { print_header(os); - for (auto& [dummy, i]: m_inputs) { + for (auto& [dummy, i] : m_inputs) { i.print(os); os << "\n"; } - for (auto& [dummy, o]: m_outputs) { + for (auto& [dummy, o] : m_outputs) { o.print(os); os << "\n"; } @@ -1300,14 +1388,10 @@ void Shader::print(std::ostream& os) const b->print(os); } -const char *chip_class_names[] = { - "R600", - "R700", - "EVERGREEN", - "CAYMAN" -}; +const char *chip_class_names[] = {"R600", "R700", "EVERGREEN", "CAYMAN"}; -void Shader::print_header(std::ostream& os) const +void +Shader::print_header(std::ostream& os) const { assert(m_chip_class <= ISA_CC_CAYMAN); os << m_type_id << "\n"; @@ -1315,25 +1399,30 @@ void Shader::print_header(std::ostream& os) const print_properties(os); } -void Shader::print_properties(std::ostream& os) const +void +Shader::print_properties(std::ostream& os) const { do_print_properties(os); } -bool Shader::equal_to(const Shader& other) const +bool +Shader::equal_to(const Shader& other) const { if (m_root.size() != other.m_root.size()) return false; - return std::inner_product(m_root.begin(), m_root.end(), - other.m_root.begin(), - true, - [](bool lhs, bool rhs){ return lhs & rhs;}, - [](const Block::Pointer lhs, const Block::Pointer rhs) -> bool { - return lhs->is_equal_to(*rhs); - }); + return std::inner_product( + m_root.begin(), + m_root.end(), + other.m_root.begin(), + true, + [](bool lhs, bool rhs) { return lhs & rhs; }, + [](const Block::Pointer lhs, const Block::Pointer rhs) -> bool { + return lhs->is_equal_to(*rhs); + }); } -void Shader::get_shader_info(r600_shader *sh_info) +void +Shader::get_shader_info(r600_shader *sh_info) { sh_info->ninput = m_inputs.size(); int lds_pos = 0; @@ -1356,7 +1445,8 @@ void Shader::get_shader_info(r600_shader *sh_info) io.ring_offset = info.ring_offset(); io.uses_interpolate_at_centroid = info.uses_interpolate_at_centroid(); - sfn_log << SfnLog::io << "Emit Input [" << index << "] sid:" << io.sid << " spi_sid:" << io.spi_sid << "\n"; + sfn_log << SfnLog::io << "Emit Input [" << index << "] sid:" << io.sid + << " spi_sid:" << io.spi_sid << "\n"; assert(io.spi_sid >= 0); } @@ -1373,7 +1463,8 @@ void Shader::get_shader_info(r600_shader *sh_info) io.name = info.name(); io.write_mask = info.writemask(); - sfn_log << SfnLog::io << "Emit output[" << index << "] sid:" << io.sid << " spi_sid:" << io.spi_sid << "\n"; + sfn_log << SfnLog::io << "Emit output[" << index << "] sid:" << io.sid + << " spi_sid:" << io.spi_sid << "\n"; assert(io.spi_sid >= 0); } @@ -1384,7 +1475,7 @@ void Shader::get_shader_info(r600_shader *sh_info) sh_info->atomics[i] = m_atomics[i]; if (m_flags.test(sh_indirect_const_file)) - sh_info->indirect_files |= 1 << TGSI_FILE_CONSTANT; + sh_info->indirect_files |= 1 << TGSI_FILE_CONSTANT; if (m_flags.test(sh_indirect_atomic)) sh_info->indirect_files |= 1 << TGSI_FILE_HW_ATOMIC; @@ -1401,56 +1492,62 @@ void Shader::get_shader_info(r600_shader *sh_info) do_get_shader_info(sh_info); } -PRegister Shader::atomic_update() +PRegister +Shader::atomic_update() { assert(m_atomic_update); return m_atomic_update; } -int Shader::remap_atomic_base(int base) +int +Shader::remap_atomic_base(int base) { return m_atomic_base_map[base]; } -void Shader::do_get_shader_info(r600_shader *sh_info) +void +Shader::do_get_shader_info(r600_shader *sh_info) { sh_info->uses_atomics = m_nhwatomic > 0; } - -const ShaderInput& Shader::input(int base) const +const ShaderInput& +Shader::input(int base) const { auto io = m_inputs.find(base); assert(io != m_inputs.end()); return io->second; } -const ShaderOutput& Shader::output(int base) const +const ShaderOutput& +Shader::output(int base) const { auto io = m_outputs.find(base); assert(io != m_outputs.end()); return io->second; } -LiveRangeMap Shader::prepare_live_range_map() +LiveRangeMap +Shader::prepare_live_range_map() { return m_instr_factory->value_factory().prepare_live_range_map(); - } -void Shader::reset_function(ShaderBlocks& new_root) +void +Shader::reset_function(ShaderBlocks& new_root) { std::swap(m_root, new_root); } -void Shader::finalize() +void +Shader::finalize() { do_finalize(); } -void Shader::do_finalize() +void +Shader::do_finalize() { - } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_shader.h b/src/gallium/drivers/r600/sfn/sfn_shader.h index d7e69b4e334..2e5dd60cacc 100644 --- a/src/gallium/drivers/r600/sfn/sfn_shader.h +++ b/src/gallium/drivers/r600/sfn/sfn_shader.h @@ -27,10 +27,10 @@ #ifndef SFN_SHADER_H #define SFN_SHADER_H -#include "sfn_instr.h" -#include "sfn_instrfactory.h" -#include "sfn_instr_controlflow.h" #include "gallium/drivers/r600/r600_shader.h" +#include "sfn_instr.h" +#include "sfn_instr_controlflow.h" +#include "sfn_instrfactory.h" #include "sfn_liverangeevaluator.h" #include @@ -52,26 +52,25 @@ public: void override_spi_sid(int spi_sid); void print(std::ostream& os) const; - int spi_sid() const { return m_spi_sid;} - unsigned sid() const { return m_sid;} + int spi_sid() const { return m_spi_sid; } + unsigned sid() const { return m_sid; } - int location() const {return m_location;} - unsigned name() const { return m_name;} + int location() const { return m_location; } + unsigned name() const { return m_name; } - int pos() const { return m_pos;} - void set_pos(int pos) {m_pos = pos;} + int pos() const { return m_pos; } + void set_pos(int pos) { m_pos = pos; } - bool is_param() const { return m_is_param;} - void set_is_param(bool val) { m_is_param = val;} + bool is_param() const { return m_is_param; } + void set_is_param(bool val) { m_is_param = val; } - void set_gpr(int gpr) {m_gpr = gpr;} - int gpr() const {return m_gpr;} + void set_gpr(int gpr) { m_gpr = gpr; } + int gpr() const { return m_gpr; } protected: ShaderIO(const char *type, int loc, int name); private: - virtual void do_print(std::ostream& os) const = 0; const char *m_type; @@ -89,7 +88,7 @@ public: ShaderOutput(); ShaderOutput(int location, int name, int writemask); - int writemask() const { return m_writemask;} + int writemask() const { return m_writemask; } private: void do_print(std::ostream& os) const override; @@ -97,25 +96,24 @@ private: int m_writemask{0}; }; - -class ShaderInput : public ShaderIO { +class ShaderInput : public ShaderIO { public: ShaderInput(); ShaderInput(int location, int name); void set_interpolator(int interp, int interp_loc, bool uses_interpolate_at_centroid); void set_uses_interpolate_at_centroid(); - void set_need_lds_pos() { m_need_lds_pos = true;} - int ij_index() const { return m_ij_index;} + void set_need_lds_pos() { m_need_lds_pos = true; } + int ij_index() const { return m_ij_index; } - int interpolator() const{return m_interpolator;} - int interpolate_loc() const {return m_interpolate_loc;} - bool need_lds_pos() const {return m_need_lds_pos;} - int lds_pos() const {return m_lds_pos;} - void set_lds_pos(int pos) {m_lds_pos = pos;} + int interpolator() const { return m_interpolator; } + int interpolate_loc() const { return m_interpolate_loc; } + bool need_lds_pos() const { return m_need_lds_pos; } + int lds_pos() const { return m_lds_pos; } + void set_lds_pos(int pos) { m_lds_pos = pos; } - int ring_offset() const {return m_ring_offset;} - void set_ring_offset(int offs) {m_ring_offset = offs;} - bool uses_interpolate_at_centroid() const {return m_uses_interpolate_at_centroid;} + int ring_offset() const { return m_ring_offset; } + void set_ring_offset(int offs) { m_ring_offset = offs; } + bool uses_interpolate_at_centroid() const { return m_uses_interpolate_at_centroid; } private: void do_print(std::ostream& os) const override; @@ -142,8 +140,11 @@ public: bool add_info_from_string(std::istream& is); - static Shader *translate_from_nir(nir_shader *nir, const pipe_stream_output_info *so_info, r600_shader *gs_shader, - r600_shader_key& key, r600_chip_class chip_class); + static Shader *translate_from_nir(nir_shader *nir, + const pipe_stream_output_info *so_info, + r600_shader *gs_shader, + r600_shader_key& key, + r600_chip_class chip_class); bool process(nir_shader *nir); @@ -153,10 +154,10 @@ public: bool process_block(nir_block *node); bool process_instr(nir_instr *instr); void emit_instruction(PInst instr); - bool emit_atomic_local_shared(nir_intrinsic_instr* instr); + bool emit_atomic_local_shared(nir_intrinsic_instr *instr); - void print(std::ostream& os ) const; - void print_header(std::ostream& os ) const; + void print(std::ostream& os) const; + void print_header(std::ostream& os) const; bool process_intrinsic(nir_intrinsic_instr *intr); @@ -168,33 +169,29 @@ public: ValueFactory& value_factory(); - void add_output(const ShaderOutput& output) { - m_outputs[output.location()] = output; - } + void add_output(const ShaderOutput& output) { m_outputs[output.location()] = output; } - void add_input(const ShaderInput& input) { - m_inputs[input.location()] = input; - } + void add_input(const ShaderInput& input) { m_inputs[input.location()] = input; } void set_input_gpr(int driver_lcation, int gpr); - InputIterator find_input(int location) { return m_inputs.find(location);} + InputIterator find_input(int location) { return m_inputs.find(location); } - InputIterator input_not_found() {return m_inputs.end();} + InputIterator input_not_found() { return m_inputs.end(); } OutputIterator find_output(int location); - OutputIterator output_not_found() {return m_outputs.end();} + OutputIterator output_not_found() { return m_outputs.end(); } ShaderBlocks& func() { return m_root; } void reset_function(ShaderBlocks& new_root); - void emit_instruction_from_string(const std::string &s); + void emit_instruction_from_string(const std::string& s); void set_info(nir_shader *nir); void get_shader_info(r600_shader *sh_info); - r600_chip_class chip_class() const {return m_chip_class;}; - void set_chip_class(r600_chip_class cls) {m_chip_class = cls;}; + r600_chip_class chip_class() const { return m_chip_class; }; + void set_chip_class(r600_chip_class cls) { m_chip_class = cls; }; void start_new_block(int nesting_depth); @@ -202,17 +199,17 @@ public: LiveRangeMap prepare_live_range_map(); - void set_last_txd(Instr *txd){m_last_txd = txd;} - Instr *last_txd(){return m_last_txd;} + void set_last_txd(Instr *txd) { m_last_txd = txd; } + Instr *last_txd() { return m_last_txd; } // Needed for keeping the memory access in order void chain_scratch_read(Instr *instr); void chain_ssbo_read(Instr *instr); - virtual uint32_t enabled_stream_buffers_mask() const {return 0;}; + virtual uint32_t enabled_stream_buffers_mask() const { return 0; }; - size_t noutputs() const { return m_outputs.size();} - size_t ninputs() const { return m_inputs.size();} + size_t noutputs() const { return m_outputs.size(); } + size_t ninputs() const { return m_inputs.size(); } enum Flags { sh_indirect_const_file, @@ -229,16 +226,21 @@ public: sh_flags_count }; - void set_flag(Flags f) {m_flags.set(f);} - bool has_flag(Flags f) const {return m_flags.test(f);} + void set_flag(Flags f) { m_flags.set(f); } + bool has_flag(Flags f) const { return m_flags.test(f); } int atomic_file_count() const { return m_atomic_file_count; } PRegister atomic_update(); int remap_atomic_base(int base); - auto evaluate_resource_offset(nir_intrinsic_instr *instr, int src_id) -> std::pair; - int ssbo_image_offset() const {return m_ssbo_image_offset;} - PRegister rat_return_address() {assert(m_rat_return_address); return m_rat_return_address;} + auto evaluate_resource_offset(nir_intrinsic_instr *instr, int src_id) + -> std::pair; + int ssbo_image_offset() const { return m_ssbo_image_offset; } + PRegister rat_return_address() + { + assert(m_rat_return_address); + return m_rat_return_address; + } PRegister emit_load_to_register(PVirtualValue src); @@ -272,7 +274,7 @@ protected: template using IOMap = std::map, Allocator>>; - IOMap& inputs() {return m_inputs;} + IOMap& inputs() { return m_inputs; } private: virtual bool process_stage_intrinsic(nir_intrinsic_instr *intr) = 0; @@ -282,7 +284,10 @@ private: bool read_chipclass(std::istream& is); - bool load_uniform_indirect(nir_intrinsic_instr *intr, PVirtualValue addr, int offset , int buffer_id); + bool load_uniform_indirect(nir_intrinsic_instr *intr, + PVirtualValue addr, + int offset, + int buffer_id); bool scan_shader(const nir_function *impl); bool scan_uniforms(nir_variable *uniform); @@ -307,10 +312,10 @@ private: bool emit_store_scratch(nir_intrinsic_instr *intr); bool emit_load_scratch(nir_intrinsic_instr *intr); bool emit_local_store(nir_intrinsic_instr *intr); - bool emit_local_load(nir_intrinsic_instr* instr); - bool emit_load_tcs_param_base(nir_intrinsic_instr* instr, int offset); - bool emit_barrier(nir_intrinsic_instr* intr); - bool emit_shader_clock(nir_intrinsic_instr* instr); + bool emit_local_load(nir_intrinsic_instr *instr); + bool emit_load_tcs_param_base(nir_intrinsic_instr *instr, int offset); + bool emit_barrier(nir_intrinsic_instr *intr); + bool emit_shader_clock(nir_intrinsic_instr *instr); bool emit_wait_ack(); bool equal_to(const Shader& other) const; @@ -333,9 +338,9 @@ private: int m_next_block; bool m_indirect_const_file{false}; - Instr *m_last_txd {nullptr}; + Instr *m_last_txd{nullptr}; - uint32_t m_indirect_files{0}; + uint32_t m_indirect_files{0}; std::bitset m_flags; uint32_t nhwatomic_ranges{0}; std::vector m_atomics; @@ -353,20 +358,20 @@ private: class InstructionChain : public InstrVisitor { public: - void visit(AluInstr *instr) override {(void) instr;} - void visit(AluGroup *instr) override {(void) instr;} - void visit(TexInstr *instr) override {(void) instr;} - void visit(ExportInstr *instr) override {(void) instr;} - void visit(FetchInstr *instr) override {(void) instr;} - void visit(Block *instr) override {(void) instr;} - void visit(ControlFlowInstr *instr) override {(void) instr;} - void visit(IfInstr *instr) override {(void) instr;} - void visit(StreamOutInstr *instr) override {(void) instr;} - void visit(MemRingOutInstr *instr) override {(void) instr;} - void visit(EmitVertexInstr *instr) override {(void) instr;} - void visit(WriteTFInstr *instr) override {(void) instr;} - void visit(LDSAtomicInstr *instr) override {(void) instr;} - void visit(LDSReadInstr *instr) override {(void) instr;} + void visit(AluInstr *instr) override { (void)instr; } + void visit(AluGroup *instr) override { (void)instr; } + void visit(TexInstr *instr) override { (void)instr; } + void visit(ExportInstr *instr) override { (void)instr; } + void visit(FetchInstr *instr) override { (void)instr; } + void visit(Block *instr) override { (void)instr; } + void visit(ControlFlowInstr *instr) override { (void)instr; } + void visit(IfInstr *instr) override { (void)instr; } + void visit(StreamOutInstr *instr) override { (void)instr; } + void visit(MemRingOutInstr *instr) override { (void)instr; } + void visit(EmitVertexInstr *instr) override { (void)instr; } + void visit(WriteTFInstr *instr) override { (void)instr; } + void visit(LDSAtomicInstr *instr) override { (void)instr; } + void visit(LDSReadInstr *instr) override { (void)instr; } void visit(ScratchIOInstr *instr) override; void visit(GDSInstr *instr) override; @@ -385,10 +390,9 @@ private: std::list> m_loops; }; - std::pair r600_get_varying_semantic(unsigned varying_location); -} +} // namespace r600 #endif // SHADER_H diff --git a/src/gallium/drivers/r600/sfn/sfn_shader_cs.cpp b/src/gallium/drivers/r600/sfn/sfn_shader_cs.cpp index a3b079c324f..ad81f35c7c0 100644 --- a/src/gallium/drivers/r600/sfn/sfn_shader_cs.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_shader_cs.cpp @@ -25,23 +25,24 @@ */ #include "sfn_shader_cs.h" -#include "sfn_instr_fetch.h" +#include "sfn_instr_fetch.h" namespace r600 { ComputeShader::ComputeShader(UNUSED const r600_shader_key& key): - Shader("CS") + Shader("CS") { - } -bool ComputeShader::do_scan_instruction(UNUSED nir_instr *instr) +bool +ComputeShader::do_scan_instruction(UNUSED nir_instr *instr) { return false; } -int ComputeShader::do_allocate_reserved_registers() +int +ComputeShader::do_allocate_reserved_registers() { auto& vf = value_factory(); @@ -58,7 +59,8 @@ int ComputeShader::do_allocate_reserved_registers() return 2; } -bool ComputeShader::process_stage_intrinsic(nir_intrinsic_instr *instr) +bool +ComputeShader::process_stage_intrinsic(nir_intrinsic_instr *instr) { switch (instr->intrinsic) { case nir_intrinsic_load_local_invocation_id: @@ -72,50 +74,59 @@ bool ComputeShader::process_stage_intrinsic(nir_intrinsic_instr *instr) } } -void ComputeShader::do_get_shader_info(r600_shader *sh_info) +void +ComputeShader::do_get_shader_info(r600_shader *sh_info) { sh_info->processor_type = PIPE_SHADER_COMPUTE; } -bool ComputeShader::read_prop(UNUSED std::istream& is) +bool +ComputeShader::read_prop(UNUSED std::istream& is) { return true; } -void ComputeShader::do_print_properties(UNUSED std::ostream& os) const +void +ComputeShader::do_print_properties(UNUSED std::ostream& os) const { - } -bool ComputeShader::emit_load_num_workgroups(nir_intrinsic_instr* instr) +bool +ComputeShader::emit_load_num_workgroups(nir_intrinsic_instr *instr) { auto zero = value_factory().temp_register(); - emit_instruction(new AluInstr(op1_mov, zero, value_factory().inline_const(ALU_SRC_0, 0), - AluInstr::last_write)); + emit_instruction(new AluInstr( + op1_mov, zero, value_factory().inline_const(ALU_SRC_0, 0), AluInstr::last_write)); auto dest = value_factory().dest_vec4(instr->dest, pin_group); - auto ir = new LoadFromBuffer(dest, {0,1,2,7}, zero, 16, + auto ir = new LoadFromBuffer(dest, + {0, 1, 2, 7}, + zero, + 16, R600_BUFFER_INFO_CONST_BUFFER, - nullptr, fmt_32_32_32_32); + nullptr, + fmt_32_32_32_32); ir->set_fetch_flag(LoadFromBuffer::srf_mode); ir->reset_fetch_flag(LoadFromBuffer::format_comp_signed); ir->set_num_format(vtx_nf_int); emit_instruction(ir); return true; - } -bool ComputeShader::emit_load_3vec(nir_intrinsic_instr* instr, const std::array& src) +bool +ComputeShader::emit_load_3vec(nir_intrinsic_instr *instr, + const std::array& src) { auto& vf = value_factory(); for (int i = 0; i < 3; ++i) { auto dest = vf.dest(instr->dest, i, pin_none); - emit_instruction(new AluInstr(op1_mov, dest, src[i], i == 2 ? AluInstr::last_write : AluInstr::write)); + emit_instruction(new AluInstr( + op1_mov, dest, src[i], i == 2 ? AluInstr::last_write : AluInstr::write)); } return true; } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_shader_cs.h b/src/gallium/drivers/r600/sfn/sfn_shader_cs.h index 786aef093ac..ae4bb6031a0 100644 --- a/src/gallium/drivers/r600/sfn/sfn_shader_cs.h +++ b/src/gallium/drivers/r600/sfn/sfn_shader_cs.h @@ -31,8 +31,7 @@ namespace r600 { -class ComputeShader : public Shader -{ +class ComputeShader : public Shader { public: ComputeShader(const r600_shader_key& key); @@ -43,23 +42,25 @@ private: bool process_stage_intrinsic(nir_intrinsic_instr *intr) override; void do_get_shader_info(r600_shader *sh_info) override; - bool load_input(UNUSED nir_intrinsic_instr *intr) override { + bool load_input(UNUSED nir_intrinsic_instr *intr) override + { unreachable("compute shaders have bno inputs"); }; - bool store_output(UNUSED nir_intrinsic_instr *intr) override { + bool store_output(UNUSED nir_intrinsic_instr *intr) override + { unreachable("compute shaders have no outputs"); }; bool read_prop(std::istream& is) override; void do_print_properties(std::ostream& os) const override; - bool emit_load_num_workgroups(nir_intrinsic_instr* instr); - bool emit_load_3vec(nir_intrinsic_instr* instr, const std::array& src); + bool emit_load_num_workgroups(nir_intrinsic_instr *instr); + bool emit_load_3vec(nir_intrinsic_instr *instr, const std::array& src); - std::array m_workgroup_id{nullptr}; - std::array m_local_invocation_id{nullptr}; + std::array m_workgroup_id{nullptr}; + std::array m_local_invocation_id{nullptr}; }; -} +} // namespace r600 #endif // COMPUTE_H diff --git a/src/gallium/drivers/r600/sfn/sfn_shader_fs.cpp b/src/gallium/drivers/r600/sfn/sfn_shader_fs.cpp index f9d8d7080c3..b8cf3a46407 100644 --- a/src/gallium/drivers/r600/sfn/sfn_shader_fs.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_shader_fs.cpp @@ -24,15 +24,13 @@ * USE OR OTHER DEALINGS IN THE SOFTWARE. */ - -#include "sfn_debug.h" #include "sfn_shader_fs.h" +#include "sfn_debug.h" #include "sfn_instr_alugroup.h" -#include "sfn_instr_tex.h" -#include "sfn_instr_fetch.h" #include "sfn_instr_export.h" - +#include "sfn_instr_fetch.h" +#include "sfn_instr_tex.h" #include "tgsi/tgsi_from_mesa.h" #include @@ -42,22 +40,23 @@ namespace r600 { using std::string; FragmentShader::FragmentShader(const r600_shader_key& key): - Shader("FS"), - m_dual_source_blend(key.ps.dual_source_blend), - m_max_color_exports(MAX2(key.ps.nr_cbufs, 1)), - m_export_highest(0), - m_num_color_exports(0), - m_color_export_mask(0), - m_depth_exports(0), - m_last_pixel_export(nullptr), - m_pos_input(127, false), - m_fs_write_all(false), - m_apply_sample_mask(key.ps.apply_sample_id_mask), - m_rat_base(key.ps.nr_cbufs) + Shader("FS"), + m_dual_source_blend(key.ps.dual_source_blend), + m_max_color_exports(MAX2(key.ps.nr_cbufs, 1)), + m_export_highest(0), + m_num_color_exports(0), + m_color_export_mask(0), + m_depth_exports(0), + m_last_pixel_export(nullptr), + m_pos_input(127, false), + m_fs_write_all(false), + m_apply_sample_mask(key.ps.apply_sample_id_mask), + m_rat_base(key.ps.nr_cbufs) { } -void FragmentShader::do_get_shader_info(r600_shader *sh_info) +void +FragmentShader::do_get_shader_info(r600_shader *sh_info) { sh_info->processor_type = PIPE_SHADER_FRAGMENT; @@ -70,21 +69,20 @@ void FragmentShader::do_get_shader_info(r600_shader *sh_info) sh_info->rat_base = m_rat_base; sh_info->uses_kill = m_uses_discard; sh_info->gs_prim_id_input = m_gs_prim_id_input; - sh_info->ps_prim_id_input = m_ps_prim_id_input && - chip_class() >= ISA_CC_EVERGREEN; + sh_info->ps_prim_id_input = m_ps_prim_id_input && chip_class() >= ISA_CC_EVERGREEN; sh_info->nsys_inputs = m_nsys_inputs; sh_info->uses_helper_invocation = m_helper_invocation != nullptr; } - -bool FragmentShader::load_input(nir_intrinsic_instr *intr) +bool +FragmentShader::load_input(nir_intrinsic_instr *intr) { auto& vf = value_factory(); auto location = nir_intrinsic_io_semantics(intr).location; if (location == VARYING_SLOT_POS) { AluInstr *ir = nullptr; - for (unsigned i = 0; i < nir_dest_num_components(intr->dest) ; ++i) { + for (unsigned i = 0; i < nir_dest_num_components(intr->dest); ++i) { ir = new AluInstr(op1_mov, vf.dest(intr->dest, i, pin_none), m_pos_input[i], @@ -108,12 +106,13 @@ bool FragmentShader::load_input(nir_intrinsic_instr *intr) return load_input_hw(intr); } -bool FragmentShader::store_output(nir_intrinsic_instr *intr) +bool +FragmentShader::store_output(nir_intrinsic_instr *intr) { auto location = nir_intrinsic_io_semantics(intr).location; if (location == FRAG_RESULT_COLOR && !m_dual_source_blend) { - m_fs_write_all = true; + m_fs_write_all = true; } return emit_export_pixel(*intr); @@ -126,15 +125,15 @@ barycentric_ij_index(nir_intrinsic_instr *intr) switch (intr->intrinsic) { case nir_intrinsic_load_barycentric_sample: index = 0; - break; + break; case nir_intrinsic_load_barycentric_at_sample: case nir_intrinsic_load_barycentric_at_offset: case nir_intrinsic_load_barycentric_pixel: index = 1; - break; + break; case nir_intrinsic_load_barycentric_centroid: index = 2; - break; + break; default: unreachable("Unknown interpolator intrinsic"); } @@ -143,9 +142,9 @@ barycentric_ij_index(nir_intrinsic_instr *intr) case INTERP_MODE_NONE: case INTERP_MODE_SMOOTH: case INTERP_MODE_COLOR: - return index; + return index; case INTERP_MODE_NOPERSPECTIVE: - return index + 3; + return index + 3; case INTERP_MODE_FLAT: case INTERP_MODE_EXPLICIT: default: @@ -154,7 +153,8 @@ barycentric_ij_index(nir_intrinsic_instr *intr) return 0; } -bool FragmentShader::process_stage_intrinsic(nir_intrinsic_instr *intr) +bool +FragmentShader::process_stage_intrinsic(nir_intrinsic_instr *intr) { if (process_stage_intrinsic_hw(intr)) return true; @@ -166,7 +166,8 @@ bool FragmentShader::process_stage_intrinsic(nir_intrinsic_instr *intr) return load_interpolated_input(intr); case nir_intrinsic_discard_if: m_uses_discard = true; - emit_instruction(new AluInstr(op2_killne_int, nullptr, + emit_instruction(new AluInstr(op2_killne_int, + nullptr, value_factory().src(intr->src[0], 0), value_factory().zero(), {AluInstr::last})); @@ -174,7 +175,8 @@ bool FragmentShader::process_stage_intrinsic(nir_intrinsic_instr *intr) return true; case nir_intrinsic_discard: m_uses_discard = true; - emit_instruction(new AluInstr(op2_kille_int, nullptr, + emit_instruction(new AluInstr(op2_kille_int, + nullptr, value_factory().zero(), value_factory().zero(), {AluInstr::last})); @@ -188,7 +190,7 @@ bool FragmentShader::process_stage_intrinsic(nir_intrinsic_instr *intr) case nir_intrinsic_load_sample_id: return emit_simple_mov(intr->dest, 0, m_sample_id_reg); case nir_intrinsic_load_helper_invocation: - return emit_load_helper_invocation(intr); + return emit_load_helper_invocation(intr); case nir_intrinsic_load_sample_pos: return emit_load_sample_pos(intr); default: @@ -196,26 +198,26 @@ bool FragmentShader::process_stage_intrinsic(nir_intrinsic_instr *intr) } } -bool FragmentShader::load_interpolated_input(nir_intrinsic_instr *intr) +bool +FragmentShader::load_interpolated_input(nir_intrinsic_instr *intr) { auto& vf = value_factory(); unsigned loc = nir_intrinsic_io_semantics(intr).location; switch (loc) { case VARYING_SLOT_POS: for (unsigned i = 0; i < nir_dest_num_components(intr->dest); ++i) - vf.inject_value(intr->dest, i, m_pos_input[i]); + vf.inject_value(intr->dest, i, m_pos_input[i]); return true; case VARYING_SLOT_FACE: return false; - default: - ; + default:; } return load_interpolated_input_hw(intr); } - -int FragmentShader::do_allocate_reserved_registers() +int +FragmentShader::do_allocate_reserved_registers() { int next_register = allocate_interpolators_or_inputs(); @@ -224,7 +226,6 @@ int FragmentShader::do_allocate_reserved_registers() m_pos_input = value_factory().allocate_pinned_vec4(next_register++, false); for (int i = 0; i < 4; ++i) m_pos_input[i]->pin_live_range(true); - } int face_reg_index = -1; @@ -240,19 +241,19 @@ int FragmentShader::do_allocate_reserved_registers() face_reg_index = next_register++; m_sample_mask_reg = value_factory().allocate_pinned_register(face_reg_index, 2); m_sample_mask_reg->pin_live_range(true); - sfn_log << SfnLog::io << "Set sample mask in register to " << *m_sample_mask_reg << "\n"; + sfn_log << SfnLog::io << "Set sample mask in register to " << *m_sample_mask_reg + << "\n"; m_nsys_inputs = 1; ShaderInput input(ninputs(), TGSI_SEMANTIC_SAMPLEMASK); input.set_gpr(face_reg_index); add_input(input); } - if (m_sv_values.test(es_sample_id) || - m_sv_values.test(es_sample_mask_in)) { + if (m_sv_values.test(es_sample_id) || m_sv_values.test(es_sample_mask_in)) { int sample_id_reg = next_register++; m_sample_id_reg = value_factory().allocate_pinned_register(sample_id_reg, 3); m_sample_id_reg->pin_live_range(true); - sfn_log << SfnLog::io << "Set sample id register to " << *m_sample_id_reg << "\n"; + sfn_log << SfnLog::io << "Set sample id register to " << *m_sample_id_reg << "\n"; m_nsys_inputs++; ShaderInput input(ninputs(), TGSI_SEMANTIC_SAMPLEID); input.set_gpr(sample_id_reg); @@ -266,7 +267,8 @@ int FragmentShader::do_allocate_reserved_registers() return next_register; } -bool FragmentShader::do_scan_instruction(nir_instr *instr) +bool +FragmentShader::do_scan_instruction(nir_instr *instr) { if (instr->type != nir_instr_type_intrinsic) return false; @@ -305,7 +307,8 @@ bool FragmentShader::do_scan_instruction(nir_instr *instr) return true; } -bool FragmentShader::emit_load_sample_mask_in(nir_intrinsic_instr* instr) +bool +FragmentShader::emit_load_sample_mask_in(nir_intrinsic_instr *instr) { auto& vf = value_factory(); auto dest = vf.dest(instr->dest, 0, pin_free); @@ -313,20 +316,29 @@ bool FragmentShader::emit_load_sample_mask_in(nir_intrinsic_instr* instr) assert(m_sample_id_reg); assert(m_sample_mask_reg); - emit_instruction(new AluInstr(op2_lshl_int, tmp, vf.one_i(), m_sample_id_reg, AluInstr::last_write)); - emit_instruction(new AluInstr(op2_and_int, dest, tmp, m_sample_mask_reg, AluInstr::last_write)); + emit_instruction( + new AluInstr(op2_lshl_int, tmp, vf.one_i(), m_sample_id_reg, AluInstr::last_write)); + emit_instruction( + new AluInstr(op2_and_int, dest, tmp, m_sample_mask_reg, AluInstr::last_write)); return true; } -bool FragmentShader::emit_load_helper_invocation(nir_intrinsic_instr* instr) +bool +FragmentShader::emit_load_helper_invocation(nir_intrinsic_instr *instr) { assert(m_helper_invocation); auto& vf = value_factory(); - emit_instruction(new AluInstr(op1_mov, m_helper_invocation, vf.literal(-1), AluInstr::last_write)); + emit_instruction( + new AluInstr(op1_mov, m_helper_invocation, vf.literal(-1), AluInstr::last_write)); RegisterVec4 destvec{m_helper_invocation, nullptr, nullptr, nullptr, pin_group}; - auto vtx = new LoadFromBuffer(destvec, {4,7,7,7}, m_helper_invocation, 0, - R600_BUFFER_INFO_CONST_BUFFER, nullptr, fmt_32_32_32_32_float); + auto vtx = new LoadFromBuffer(destvec, + {4, 7, 7, 7}, + m_helper_invocation, + 0, + R600_BUFFER_INFO_CONST_BUFFER, + nullptr, + fmt_32_32_32_32_float); vtx->set_fetch_flag(FetchInstr::vpm); vtx->set_fetch_flag(FetchInstr::use_tc); vtx->set_always_keep(); @@ -339,15 +351,16 @@ bool FragmentShader::emit_load_helper_invocation(nir_intrinsic_instr* instr) return true; } -bool FragmentShader::scan_input(nir_intrinsic_instr *intr, int index_src_id) +bool +FragmentShader::scan_input(nir_intrinsic_instr *intr, int index_src_id) { auto index = nir_src_as_const_value(intr->src[index_src_id]); assert(index); - const unsigned location_offset = chip_class() < ISA_CC_EVERGREEN ? 32 : 0; + const unsigned location_offset = chip_class() < ISA_CC_EVERGREEN ? 32 : 0; bool uses_interpol_at_centroid = false; - unsigned location = nir_intrinsic_io_semantics(intr).location + index->u32; + unsigned location = nir_intrinsic_io_semantics(intr).location + index->u32; unsigned driver_location = nir_intrinsic_base(intr) + index->u32; auto semantic = r600_get_varying_semantic(location); tgsi_semantic name = (tgsi_semantic)semantic.first; @@ -358,7 +371,9 @@ bool FragmentShader::scan_input(nir_intrinsic_instr *intr, int index_src_id) m_pos_driver_loc = driver_location + location_offset; ShaderInput pos_input(m_pos_driver_loc, name); pos_input.set_sid(sid); - pos_input.set_interpolator(TGSI_INTERPOLATE_LINEAR, TGSI_INTERPOLATE_LOC_CENTER, false); + pos_input.set_interpolator(TGSI_INTERPOLATE_LINEAR, + TGSI_INTERPOLATE_LOC_CENTER, + false); add_input(pos_input); return true; } @@ -382,42 +397,41 @@ bool FragmentShader::scan_input(nir_intrinsic_instr *intr, int index_src_id) switch (parent->intrinsic) { case nir_intrinsic_load_barycentric_sample: tgsi_loc = TGSI_INTERPOLATE_LOC_SAMPLE; - break; + break; case nir_intrinsic_load_barycentric_at_sample: case nir_intrinsic_load_barycentric_at_offset: case nir_intrinsic_load_barycentric_pixel: tgsi_loc = TGSI_INTERPOLATE_LOC_CENTER; - break; + break; case nir_intrinsic_load_barycentric_centroid: tgsi_loc = TGSI_INTERPOLATE_LOC_CENTROID; uses_interpol_at_centroid = true; - break; + break; default: - std::cerr << "Instruction " << nir_intrinsic_infos[parent->intrinsic].name << " as parent of " - << nir_intrinsic_infos[intr->intrinsic].name + std::cerr << "Instruction " << nir_intrinsic_infos[parent->intrinsic].name + << " as parent of " << nir_intrinsic_infos[intr->intrinsic].name << " interpolator?\n"; assert(0); } switch (mode) { case INTERP_MODE_NONE: - if (name == TGSI_SEMANTIC_COLOR || - name == TGSI_SEMANTIC_BCOLOR) { + if (name == TGSI_SEMANTIC_COLOR || name == TGSI_SEMANTIC_BCOLOR) { tgsi_interpolate = TGSI_INTERPOLATE_COLOR; break; } FALLTHROUGH; case INTERP_MODE_SMOOTH: tgsi_interpolate = TGSI_INTERPOLATE_PERSPECTIVE; - break; + break; case INTERP_MODE_NOPERSPECTIVE: tgsi_interpolate = TGSI_INTERPOLATE_LINEAR; - break; + break; case INTERP_MODE_FLAT: - break; + break; case INTERP_MODE_COLOR: tgsi_interpolate = TGSI_INTERPOLATE_COLOR; - break; + break; case INTERP_MODE_EXPLICIT: default: assert(0); @@ -438,14 +452,14 @@ bool FragmentShader::scan_input(nir_intrinsic_instr *intr, int index_src_id) case TGSI_SEMANTIC_PCOORD: case TGSI_SEMANTIC_VIEWPORT_INDEX: case TGSI_SEMANTIC_CLIPDIST: { - sfn_log << SfnLog::io << " have IO at " << driver_location << "\n"; + sfn_log << SfnLog::io << " have IO at " << driver_location << "\n"; auto iinput = find_input(driver_location); if (iinput == input_not_found()) { ShaderInput input(driver_location, name); input.set_sid(sid); input.set_need_lds_pos(); input.set_interpolator(tgsi_interpolate, tgsi_loc, uses_interpol_at_centroid); - sfn_log << SfnLog::io << "add IO with LDS ID at " << input.location() << "\n"; + sfn_log << SfnLog::io << "add IO with LDS ID at " << input.location() << "\n"; add_input(input); assert(find_input(input.location()) != input_not_found()); } else { @@ -460,7 +474,8 @@ bool FragmentShader::scan_input(nir_intrinsic_instr *intr, int index_src_id) } } -bool FragmentShader::emit_export_pixel(nir_intrinsic_instr& intr) +bool +FragmentShader::emit_export_pixel(nir_intrinsic_instr& intr) { RegisterVec4::Swizzle swizzle; auto semantics = nir_intrinsic_io_semantics(&intr); @@ -469,14 +484,14 @@ bool FragmentShader::emit_export_pixel(nir_intrinsic_instr& intr) switch (semantics.location) { case FRAG_RESULT_DEPTH: - swizzle = {0,7,7,7}; - break; + swizzle = {0, 7, 7, 7}; + break; case FRAG_RESULT_STENCIL: - swizzle = {7,0,7,7}; - break; + swizzle = {7, 0, 7, 7}; + break; case FRAG_RESULT_SAMPLE_MASK: - swizzle = {7,7,0,7}; - break; + swizzle = {7, 7, 0, 7}; + break; default: for (int i = 0; i < 4; ++i) { swizzle[i] = (1 << i) & write_mask ? i : 7; @@ -492,21 +507,25 @@ bool FragmentShader::emit_export_pixel(nir_intrinsic_instr& intr) ShaderOutput output(driver_location, TGSI_SEMANTIC_COLOR, write_mask); add_output(output); - unsigned color_outputs = m_fs_write_all && chip_class() >= ISA_CC_R700 ? - m_max_color_exports : 1; + unsigned color_outputs = + m_fs_write_all && chip_class() >= ISA_CC_R700 ? m_max_color_exports : 1; for (unsigned k = 0; k < color_outputs; ++k) { - unsigned location = (m_dual_source_blend && (semantics.location == FRAG_RESULT_COLOR) - ? semantics.dual_source_blend_index : driver_location) + k - m_depth_exports; + unsigned location = + (m_dual_source_blend && (semantics.location == FRAG_RESULT_COLOR) + ? semantics.dual_source_blend_index + : driver_location) + + k - m_depth_exports; sfn_log << SfnLog::io << "Pixel output at loc:" << location << "\n"; if (location >= m_max_color_exports) { sfn_log << SfnLog::io << "Pixel output loc:" << location - << " dl:" << driver_location - << " skipped because we have only " << m_max_color_exports << " CBs\n"; - return true; ; + << " dl:" << driver_location << " skipped because we have only " + << m_max_color_exports << " CBs\n"; + return true; + ; } m_last_pixel_export = new ExportInstr(ExportInstr::pixel, location, value); @@ -549,23 +568,28 @@ bool FragmentShader::emit_export_pixel(nir_intrinsic_instr& intr) return true; } -bool FragmentShader::emit_load_sample_pos(nir_intrinsic_instr* instr) +bool +FragmentShader::emit_load_sample_pos(nir_intrinsic_instr *instr) { auto dest = value_factory().dest_vec4(instr->dest, pin_group); - - auto fetch = new LoadFromBuffer(dest, {0,1,2,3}, m_sample_id_reg, 0, + auto fetch = new LoadFromBuffer(dest, + {0, 1, 2, 3}, + m_sample_id_reg, + 0, R600_BUFFER_INFO_CONST_BUFFER, - nullptr, fmt_32_32_32_32_float); + nullptr, + fmt_32_32_32_32_float); fetch->set_fetch_flag(FetchInstr::srf_mode); emit_instruction(fetch); return true; } -void FragmentShader::do_finalize() +void +FragmentShader::do_finalize() { if (!m_last_pixel_export) { - RegisterVec4 value(0, false, {7,7,7,7}); + RegisterVec4 value(0, false, {7, 7, 7, 7}); m_last_pixel_export = new ExportInstr(ExportInstr::pixel, 0, value); emit_instruction(m_last_pixel_export); m_num_color_exports++; @@ -574,7 +598,8 @@ void FragmentShader::do_finalize() m_last_pixel_export->set_is_last_export(true); } -bool FragmentShader::read_prop(std::istream& is) +bool +FragmentShader::read_prop(std::istream& is) { string value; is >> value; @@ -601,32 +626,35 @@ bool FragmentShader::read_prop(std::istream& is) return true; } -void FragmentShader::do_print_properties(std::ostream& os) const +void +FragmentShader::do_print_properties(std::ostream& os) const { - os << "PROP MAX_COLOR_EXPORTS:" << m_max_color_exports << "\n"; - os << "PROP COLOR_EXPORTS:" << m_num_color_exports << "\n"; - os << "PROP COLOR_EXPORT_MASK:" << m_color_export_mask << "\n"; + os << "PROP MAX_COLOR_EXPORTS:" << m_max_color_exports << "\n"; + os << "PROP COLOR_EXPORTS:" << m_num_color_exports << "\n"; + os << "PROP COLOR_EXPORT_MASK:" << m_color_export_mask << "\n"; os << "PROP WRITE_ALL_COLORS:" << m_fs_write_all << "\n"; } -int FragmentShaderR600::allocate_interpolators_or_inputs() +int +FragmentShaderR600::allocate_interpolators_or_inputs() { int pos = 0; auto& vf = value_factory(); - for (auto& [index, inp]: inputs()) { + for (auto& [index, inp] : inputs()) { if (inp.need_lds_pos()) { RegisterVec4 input(vf.allocate_pinned_register(pos, 0), vf.allocate_pinned_register(pos, 1), vf.allocate_pinned_register(pos, 2), - vf.allocate_pinned_register(pos, 3), pin_fully); + vf.allocate_pinned_register(pos, 3), + pin_fully); inp.set_gpr(pos++); for (int i = 0; i < 4; ++i) { input[i]->pin_live_range(true); } - sfn_log << SfnLog::io << "Reseve input register at pos " << - index << " as " << input << " with register " << inp.gpr() << "\n"; + sfn_log << SfnLog::io << "Reseve input register at pos " << index << " as " + << input << " with register " << inp.gpr() << "\n"; m_interpolated_inputs[index] = input; } @@ -634,20 +662,25 @@ int FragmentShaderR600::allocate_interpolators_or_inputs() return pos; } -bool FragmentShaderR600::load_input_hw(nir_intrinsic_instr *intr) +bool +FragmentShaderR600::load_input_hw(nir_intrinsic_instr *intr) { auto& vf = value_factory(); AluInstr *ir = nullptr; for (unsigned i = 0; i < nir_dest_num_components(intr->dest); ++i) { - sfn_log << SfnLog::io << "Inject register " << *m_interpolated_inputs[nir_intrinsic_base(intr)][i] << "\n"; + sfn_log << SfnLog::io << "Inject register " + << *m_interpolated_inputs[nir_intrinsic_base(intr)][i] << "\n"; unsigned index = nir_intrinsic_component(intr) + i; - assert (index < 4); + assert(index < 4); if (intr->dest.is_ssa) { - vf.inject_value(intr->dest, i, m_interpolated_inputs[nir_intrinsic_base(intr)][index]); + vf.inject_value(intr->dest, + i, + m_interpolated_inputs[nir_intrinsic_base(intr)][index]); } else { - ir = new AluInstr(op1_mov, vf.dest(intr->dest, i, pin_none), - m_interpolated_inputs[nir_intrinsic_base(intr)][index], - AluInstr::write); + ir = new AluInstr(op1_mov, + vf.dest(intr->dest, i, pin_none), + m_interpolated_inputs[nir_intrinsic_base(intr)][index], + AluInstr::write); emit_instruction(ir); } } @@ -656,7 +689,8 @@ bool FragmentShaderR600::load_input_hw(nir_intrinsic_instr *intr) return true; } -bool FragmentShaderR600::process_stage_intrinsic_hw(nir_intrinsic_instr *intr) +bool +FragmentShaderR600::process_stage_intrinsic_hw(nir_intrinsic_instr *intr) { switch (intr->intrinsic) { case nir_intrinsic_load_barycentric_centroid: @@ -668,12 +702,14 @@ bool FragmentShaderR600::process_stage_intrinsic_hw(nir_intrinsic_instr *intr) } } -bool FragmentShaderR600::load_interpolated_input_hw(nir_intrinsic_instr *intr) +bool +FragmentShaderR600::load_interpolated_input_hw(nir_intrinsic_instr *intr) { return load_input_hw(intr); } -bool FragmentShaderEG::load_input_hw(nir_intrinsic_instr *intr) +bool +FragmentShaderEG::load_input_hw(nir_intrinsic_instr *intr) { auto& vf = value_factory(); auto io = input(nir_intrinsic_base(intr)); @@ -681,15 +717,17 @@ bool FragmentShaderEG::load_input_hw(nir_intrinsic_instr *intr) bool need_temp = comp > 0 || !intr->dest.is_ssa; AluInstr *ir = nullptr; - for (unsigned i = 0; i < nir_dest_num_components(intr->dest) ; ++i) { + for (unsigned i = 0; i < nir_dest_num_components(intr->dest); ++i) { if (need_temp) { auto tmp = vf.temp_register(comp + i); - ir = new AluInstr(op1_interp_load_p0, - tmp, - new InlineConstant(ALU_SRC_PARAM_BASE + io.lds_pos(), i + comp), - AluInstr::last_write); + ir = + new AluInstr(op1_interp_load_p0, + tmp, + new InlineConstant(ALU_SRC_PARAM_BASE + io.lds_pos(), i + comp), + AluInstr::last_write); emit_instruction(ir); - emit_instruction(new AluInstr(op1_mov, vf.dest(intr->dest, i, pin_chan), tmp, AluInstr::last_write)); + emit_instruction(new AluInstr( + op1_mov, vf.dest(intr->dest, i, pin_chan), tmp, AluInstr::last_write)); } else { ir = new AluInstr(op1_interp_load_p0, @@ -698,13 +736,13 @@ bool FragmentShaderEG::load_input_hw(nir_intrinsic_instr *intr) AluInstr::write); emit_instruction(ir); } - } ir->set_alu_flag(alu_last_instr); return true; } -int FragmentShaderEG::allocate_interpolators_or_inputs() +int +FragmentShaderEG::allocate_interpolators_or_inputs() { for (unsigned i = 0; i < s_max_interpolators; ++i) { if (interpolators_used(i)) { @@ -716,7 +754,8 @@ int FragmentShaderEG::allocate_interpolators_or_inputs() int num_baryc = 0; for (int i = 0; i < 6; ++i) { if (m_interpolator[i].enabled) { - sfn_log << SfnLog::io << "Interpolator " << i << " is enabled with ij=" << num_baryc <<" \n"; + sfn_log << SfnLog::io << "Interpolator " << i + << " is enabled with ij=" << num_baryc << " \n"; unsigned sel = num_baryc / 2; unsigned chan = 2 * (num_baryc % 2); @@ -732,7 +771,8 @@ int FragmentShaderEG::allocate_interpolators_or_inputs() return (num_baryc + 1) >> 1; } -bool FragmentShaderEG::process_stage_intrinsic_hw(nir_intrinsic_instr *intr) +bool +FragmentShaderEG::process_stage_intrinsic_hw(nir_intrinsic_instr *intr) { auto& vf = value_factory(); switch (intr->intrinsic) { @@ -753,7 +793,8 @@ bool FragmentShaderEG::process_stage_intrinsic_hw(nir_intrinsic_instr *intr) } } -bool FragmentShaderEG::load_interpolated_input_hw(nir_intrinsic_instr *intr) +bool +FragmentShaderEG::load_interpolated_input_hw(nir_intrinsic_instr *intr) { auto& vf = value_factory(); ASSERTED auto param = nir_src_as_const_value(intr->src[1]); @@ -788,17 +829,26 @@ bool FragmentShaderEG::load_interpolated_input_hw(nir_intrinsic_instr *intr) return true; } -bool FragmentShaderEG::load_interpolated(RegisterVec4& dest, const InterpolateParams& params, - int num_dest_comp, int start_comp) +bool +FragmentShaderEG::load_interpolated(RegisterVec4& dest, + const InterpolateParams& params, + int num_dest_comp, + int start_comp) { - sfn_log << SfnLog::io << "Using Interpolator (" << *params.j << ", " << *params.i << ")" << "\n"; + sfn_log << SfnLog::io << "Using Interpolator (" << *params.j << ", " << *params.i + << ")" + << "\n"; if (num_dest_comp == 1) { switch (start_comp) { - case 0: return load_interpolated_one_comp(dest, params, op2_interp_x); - case 1: return load_interpolated_two_comp_for_one(dest, params, op2_interp_xy, 1); - case 2: return load_interpolated_one_comp(dest, params, op2_interp_z); - case 3: return load_interpolated_two_comp_for_one(dest, params, op2_interp_zw, 3); + case 0: + return load_interpolated_one_comp(dest, params, op2_interp_x); + case 1: + return load_interpolated_two_comp_for_one(dest, params, op2_interp_xy, 1); + case 2: + return load_interpolated_one_comp(dest, params, op2_interp_z); + case 3: + return load_interpolated_two_comp_for_one(dest, params, op2_interp_zw, 3); default: assert(0); } @@ -806,10 +856,13 @@ bool FragmentShaderEG::load_interpolated(RegisterVec4& dest, const InterpolatePa if (num_dest_comp == 2) { switch (start_comp) { - case 0: return load_interpolated_two_comp(dest, params, op2_interp_xy, 0x3); - case 2: return load_interpolated_two_comp(dest, params, op2_interp_zw, 0xc); - case 1: return load_interpolated_one_comp(dest, params, op2_interp_z) && - load_interpolated_two_comp_for_one(dest, params, op2_interp_xy, 1); + case 0: + return load_interpolated_two_comp(dest, params, op2_interp_xy, 0x3); + case 2: + return load_interpolated_two_comp(dest, params, op2_interp_zw, 0xc); + case 1: + return load_interpolated_one_comp(dest, params, op2_interp_z) && + load_interpolated_two_comp_for_one(dest, params, op2_interp_xy, 1); default: assert(0); } @@ -817,23 +870,30 @@ bool FragmentShaderEG::load_interpolated(RegisterVec4& dest, const InterpolatePa if (num_dest_comp == 3 && start_comp == 0) return load_interpolated_two_comp(dest, params, op2_interp_xy, 0x3) && - load_interpolated_one_comp(dest, params, op2_interp_z); + load_interpolated_one_comp(dest, params, op2_interp_z); int full_write_mask = ((1 << num_dest_comp) - 1) << start_comp; - bool success = load_interpolated_two_comp(dest, params, op2_interp_zw, full_write_mask & 0xc); - success &= load_interpolated_two_comp(dest, params, op2_interp_xy, full_write_mask & 0x3); + bool success = + load_interpolated_two_comp(dest, params, op2_interp_zw, full_write_mask & 0xc); + success &= + load_interpolated_two_comp(dest, params, op2_interp_xy, full_write_mask & 0x3); return success; } - -bool FragmentShaderEG::load_barycentric_at_sample(nir_intrinsic_instr* instr) +bool +FragmentShaderEG::load_barycentric_at_sample(nir_intrinsic_instr *instr) { auto& vf = value_factory(); RegisterVec4 slope = vf.temp_vec4(pin_group); - auto src = emit_load_to_register(vf.src(instr->src[0], 0)); - auto fetch = new LoadFromBuffer(slope, {0, 1,2, 3}, src, 0, - R600_BUFFER_INFO_CONST_BUFFER, nullptr, fmt_32_32_32_32_float); + auto src = emit_load_to_register(vf.src(instr->src[0], 0)); + auto fetch = new LoadFromBuffer(slope, + {0, 1, 2, 3}, + src, + 0, + R600_BUFFER_INFO_CONST_BUFFER, + nullptr, + fmt_32_32_32_32_float); fetch->set_fetch_flag(FetchInstr::srf_mode); emit_instruction(fetch); @@ -853,7 +913,7 @@ bool FragmentShaderEG::load_barycentric_at_sample(nir_intrinsic_instr* instr) tex->set_tex_flag(TexInstr::w_unnormalized); emit_instruction(tex); - tex = new TexInstr(TexInstr::get_gradient_v, grad, {7,7,0,1}, interp, 0, 0); + tex = new TexInstr(TexInstr::get_gradient_v, grad, {7, 7, 0, 1}, interp, 0, 0); tex->set_tex_flag(TexInstr::x_unnormalized); tex->set_tex_flag(TexInstr::y_unnormalized); tex->set_tex_flag(TexInstr::z_unnormalized); @@ -864,16 +924,29 @@ bool FragmentShaderEG::load_barycentric_at_sample(nir_intrinsic_instr* instr) auto tmp0 = vf.temp_register(); auto tmp1 = vf.temp_register(); - emit_instruction(new AluInstr(op3_muladd, tmp0, grad[0], slope[2], interpolator.j, {alu_write})); - emit_instruction(new AluInstr(op3_muladd, tmp1, grad[1], slope[2], interpolator.i, {alu_write, alu_last_instr})); + emit_instruction( + new AluInstr(op3_muladd, tmp0, grad[0], slope[2], interpolator.j, {alu_write})); + emit_instruction(new AluInstr( + op3_muladd, tmp1, grad[1], slope[2], interpolator.i, {alu_write, alu_last_instr})); - emit_instruction(new AluInstr(op3_muladd, vf.dest(instr->dest, 0, pin_none), grad[3], slope[3], tmp1, {alu_write})); - emit_instruction(new AluInstr(op3_muladd, vf.dest(instr->dest, 1, pin_none), grad[2], slope[3], tmp0, {alu_write, alu_last_instr})); + emit_instruction(new AluInstr(op3_muladd, + vf.dest(instr->dest, 0, pin_none), + grad[3], + slope[3], + tmp1, + {alu_write})); + emit_instruction(new AluInstr(op3_muladd, + vf.dest(instr->dest, 1, pin_none), + grad[2], + slope[3], + tmp0, + {alu_write, alu_last_instr})); return true; } -bool FragmentShaderEG::load_barycentric_at_offset(nir_intrinsic_instr* instr) +bool +FragmentShaderEG::load_barycentric_at_offset(nir_intrinsic_instr *instr) { auto& vf = value_factory(); auto interpolator = m_interpolator[barycentric_ij_index(instr)]; @@ -881,7 +954,8 @@ bool FragmentShaderEG::load_barycentric_at_offset(nir_intrinsic_instr* instr) auto help = vf.temp_vec4(pin_group); RegisterVec4 interp(interpolator.j, interpolator.i, nullptr, nullptr, pin_group); - auto getgradh = new TexInstr(TexInstr::get_gradient_h, help, {0,1,7,7}, interp, 0, 0); + auto getgradh = + new TexInstr(TexInstr::get_gradient_h, help, {0, 1, 7, 7}, interp, 0, 0); getgradh->set_tex_flag(TexInstr::x_unnormalized); getgradh->set_tex_flag(TexInstr::y_unnormalized); getgradh->set_tex_flag(TexInstr::z_unnormalized); @@ -889,7 +963,8 @@ bool FragmentShaderEG::load_barycentric_at_offset(nir_intrinsic_instr* instr) getgradh->set_tex_flag(TexInstr::grad_fine); emit_instruction(getgradh); - auto getgradv = new TexInstr(TexInstr::get_gradient_v, help, {7,7,0,1}, interp, 0, 0); + auto getgradv = + new TexInstr(TexInstr::get_gradient_v, help, {7, 7, 0, 1}, interp, 0, 0); getgradv->set_tex_flag(TexInstr::x_unnormalized); getgradv->set_tex_flag(TexInstr::y_unnormalized); getgradv->set_tex_flag(TexInstr::z_unnormalized); @@ -901,17 +976,26 @@ bool FragmentShaderEG::load_barycentric_at_offset(nir_intrinsic_instr* instr) auto ofs_y = vf.src(instr->src[0], 1); auto tmp0 = vf.temp_register(); auto tmp1 = vf.temp_register(); - emit_instruction(new AluInstr(op3_muladd, tmp0, help[0], ofs_x, interpolator.j, {alu_write})); - emit_instruction(new AluInstr(op3_muladd, tmp1, help[1], ofs_x, interpolator.i, {alu_write, alu_last_instr})); - emit_instruction(new AluInstr(op3_muladd, vf.dest(instr->dest, 0, pin_none), help[3], ofs_y, tmp1, {alu_write})); - emit_instruction(new AluInstr(op3_muladd, vf.dest(instr->dest, 1, pin_none), help[2], ofs_y, tmp0, {alu_write, alu_last_instr})); + emit_instruction( + new AluInstr(op3_muladd, tmp0, help[0], ofs_x, interpolator.j, {alu_write})); + emit_instruction(new AluInstr( + op3_muladd, tmp1, help[1], ofs_x, interpolator.i, {alu_write, alu_last_instr})); + emit_instruction(new AluInstr( + op3_muladd, vf.dest(instr->dest, 0, pin_none), help[3], ofs_y, tmp1, {alu_write})); + emit_instruction(new AluInstr(op3_muladd, + vf.dest(instr->dest, 1, pin_none), + help[2], + ofs_y, + tmp0, + {alu_write, alu_last_instr})); return true; } -bool FragmentShaderEG::load_interpolated_one_comp(RegisterVec4& dest, - const InterpolateParams& params, - EAluOp op) +bool +FragmentShaderEG::load_interpolated_one_comp(RegisterVec4& dest, + const InterpolateParams& params, + EAluOp op) { auto group = new AluGroup(); bool success = true; @@ -922,11 +1006,11 @@ bool FragmentShaderEG::load_interpolated_one_comp(RegisterVec4& dest, if (op == op2_interp_z) chan += 2; - - ir = new AluInstr(op, dest[chan], + ir = new AluInstr(op, + dest[chan], i & 1 ? params.j : params.i, new InlineConstant(ALU_SRC_PARAM_BASE + params.base, chan), - i == 0 ? AluInstr::write : AluInstr::last); + i == 0 ? AluInstr::write : AluInstr::last); ir->set_bank_swizzle(alu_vec_210); success = group->add_instruction(ir); @@ -937,9 +1021,11 @@ bool FragmentShaderEG::load_interpolated_one_comp(RegisterVec4& dest, return success; } -bool FragmentShaderEG::load_interpolated_two_comp(RegisterVec4& dest, - const InterpolateParams& params, - EAluOp op, int writemask) +bool +FragmentShaderEG::load_interpolated_two_comp(RegisterVec4& dest, + const InterpolateParams& params, + EAluOp op, + int writemask) { auto group = new AluGroup(); bool success = true; @@ -947,8 +1033,10 @@ bool FragmentShaderEG::load_interpolated_two_comp(RegisterVec4& dest, AluInstr *ir = nullptr; assert(params.j); assert(params.i); - for (unsigned i = 0; i < 4 ; ++i) { - ir = new AluInstr(op, dest[i], i & 1 ? params.j : params.i, + for (unsigned i = 0; i < 4; ++i) { + ir = new AluInstr(op, + dest[i], + i & 1 ? params.j : params.i, new InlineConstant(ALU_SRC_PARAM_BASE + params.base, i), (writemask & (1 << i)) ? AluInstr::write : AluInstr::empty); ir->set_bank_swizzle(alu_vec_210); @@ -960,16 +1048,20 @@ bool FragmentShaderEG::load_interpolated_two_comp(RegisterVec4& dest, return success; } -bool FragmentShaderEG::load_interpolated_two_comp_for_one(RegisterVec4& dest, - const InterpolateParams& params, EAluOp op, - int comp) +bool +FragmentShaderEG::load_interpolated_two_comp_for_one(RegisterVec4& dest, + const InterpolateParams& params, + EAluOp op, + int comp) { auto group = new AluGroup(); bool success = true; AluInstr *ir = nullptr; - for (int i = 0; i < 4 ; ++i) { - ir = new AluInstr(op, dest[i], i & 1 ? params.j : params.i, + for (int i = 0; i < 4; ++i) { + ir = new AluInstr(op, + dest[i], + i & 1 ? params.j : params.i, new InlineConstant(ALU_SRC_PARAM_BASE + params.base, i), i == comp ? AluInstr::write : AluInstr::empty); ir->set_bank_swizzle(alu_vec_210); @@ -982,10 +1074,9 @@ bool FragmentShaderEG::load_interpolated_two_comp_for_one(RegisterVec4& dest, return success; } - FragmentShaderEG::Interpolator::Interpolator(): - enabled(false) + enabled(false) { } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_shader_fs.h b/src/gallium/drivers/r600/sfn/sfn_shader_fs.h index a6b492da25f..8c97e778c73 100644 --- a/src/gallium/drivers/r600/sfn/sfn_shader_fs.h +++ b/src/gallium/drivers/r600/sfn/sfn_shader_fs.h @@ -40,16 +40,15 @@ public: bool process_stage_intrinsic(nir_intrinsic_instr *intr) override; protected: - static const int s_max_interpolators = 6; - bool interpolators_used(int i) const {return m_interpolators_used.test(i);} -private: + bool interpolators_used(int i) const { return m_interpolators_used.test(i); } +private: bool load_interpolated_input(nir_intrinsic_instr *intr); virtual int allocate_interpolators_or_inputs() = 0; virtual bool load_input_hw(nir_intrinsic_instr *intr) = 0; - virtual bool process_stage_intrinsic_hw(nir_intrinsic_instr *intr) = 0; + virtual bool process_stage_intrinsic_hw(nir_intrinsic_instr *intr) = 0; virtual bool load_interpolated_input_hw(nir_intrinsic_instr *intr) = 0; bool do_scan_instruction(nir_instr *instr) override; @@ -59,11 +58,10 @@ private: bool scan_input(nir_intrinsic_instr *instr, int index_src_id); - bool emit_export_pixel(nir_intrinsic_instr& intr); - bool emit_load_sample_mask_in(nir_intrinsic_instr* instr); - bool emit_load_helper_invocation(nir_intrinsic_instr* instr); - bool emit_load_sample_pos(nir_intrinsic_instr* instr); + bool emit_load_sample_mask_in(nir_intrinsic_instr *instr); + bool emit_load_helper_invocation(nir_intrinsic_instr *instr); + bool emit_load_sample_pos(nir_intrinsic_instr *instr); void do_finalize() override; bool read_prop(std::istream& is) override; @@ -98,6 +96,7 @@ private: class FragmentShaderR600 : public FragmentShader { public: using FragmentShader::FragmentShader; + private: int allocate_interpolators_or_inputs() override; bool load_input_hw(nir_intrinsic_instr *intr) override; @@ -107,7 +106,6 @@ private: IOMap m_interpolated_inputs; }; - class FragmentShaderEG : public FragmentShader { public: using FragmentShader::FragmentShader; @@ -123,7 +121,7 @@ private: }; struct InterpolateParams { - PVirtualValue i,j; + PVirtualValue i, j; int base; }; @@ -133,20 +131,28 @@ private: bool load_interpolated_input_hw(nir_intrinsic_instr *intr) override; bool load_barycentric_pixel(nir_intrinsic_instr *intr); - bool load_barycentric_at_sample(nir_intrinsic_instr* instr); - bool load_barycentric_at_offset(nir_intrinsic_instr* instr); - bool load_interpolated(RegisterVec4& dest, const InterpolateParams& params, - int num_dest_comp, int start_comp); + bool load_barycentric_at_sample(nir_intrinsic_instr *instr); + bool load_barycentric_at_offset(nir_intrinsic_instr *instr); + bool load_interpolated(RegisterVec4& dest, + const InterpolateParams& params, + int num_dest_comp, + int start_comp); - bool load_interpolated_one_comp(RegisterVec4& dest, const InterpolateParams& params, EAluOp op); - bool load_interpolated_two_comp(RegisterVec4& dest, const InterpolateParams& params, EAluOp op, int writemask); - bool load_interpolated_two_comp_for_one(RegisterVec4& dest, const InterpolateParams& params, EAluOp op, + bool load_interpolated_one_comp(RegisterVec4& dest, + const InterpolateParams& params, + EAluOp op); + bool load_interpolated_two_comp(RegisterVec4& dest, + const InterpolateParams& params, + EAluOp op, + int writemask); + bool load_interpolated_two_comp_for_one(RegisterVec4& dest, + const InterpolateParams& params, + EAluOp op, int dest_slot); std::array m_interpolator; - }; -} +} // namespace r600 #endif diff --git a/src/gallium/drivers/r600/sfn/sfn_shader_gs.cpp b/src/gallium/drivers/r600/sfn/sfn_shader_gs.cpp index 1696738dfbf..b89c797b22d 100644 --- a/src/gallium/drivers/r600/sfn/sfn_shader_gs.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_shader_gs.cpp @@ -25,24 +25,25 @@ */ #include "sfn_shader_gs.h" -#include "sfn_instr_fetch.h" + #include "sfn_debug.h" +#include "sfn_instr_fetch.h" namespace r600 { GeometryShader::GeometryShader(const r600_shader_key& key): - Shader("GS"), - m_tri_strip_adj_fix(key.gs.tri_strip_adj_fix) + Shader("GS"), + m_tri_strip_adj_fix(key.gs.tri_strip_adj_fix) { - } -bool GeometryShader::do_scan_instruction(nir_instr *instr) +bool +GeometryShader::do_scan_instruction(nir_instr *instr) { if (instr->type != nir_instr_type_intrinsic) return false; - nir_intrinsic_instr *ii = nir_instr_as_intrinsic(instr); + nir_intrinsic_instr *ii = nir_instr_as_intrinsic(instr); switch (ii->intrinsic) { case nir_intrinsic_store_output: @@ -51,10 +52,11 @@ bool GeometryShader::do_scan_instruction(nir_instr *instr) return process_load_input(ii); default: return false; - } + } } -bool GeometryShader::process_store_output(nir_intrinsic_instr *instr) +bool +GeometryShader::process_store_output(nir_intrinsic_instr *instr) { auto location = nir_intrinsic_io_semantics(instr).location; auto index = nir_src_as_const_value(instr->src[1]); @@ -62,24 +64,15 @@ bool GeometryShader::process_store_output(nir_intrinsic_instr *instr) auto driver_location = nir_intrinsic_base(instr) + index->u32; - if (location == VARYING_SLOT_COL0 || - location == VARYING_SLOT_COL1 || - (location >= VARYING_SLOT_VAR0 && - location <= VARYING_SLOT_VAR31) || - (location >= VARYING_SLOT_TEX0 && - location <= VARYING_SLOT_TEX7) || - location == VARYING_SLOT_BFC0 || - location == VARYING_SLOT_BFC1 || - location == VARYING_SLOT_PNTC || - location == VARYING_SLOT_CLIP_VERTEX || - location == VARYING_SLOT_CLIP_DIST0 || - location == VARYING_SLOT_CLIP_DIST1 || - location == VARYING_SLOT_PRIMITIVE_ID || - location == VARYING_SLOT_POS || - location == VARYING_SLOT_PSIZ || - location == VARYING_SLOT_LAYER || - location == VARYING_SLOT_VIEWPORT || - location == VARYING_SLOT_FOGC) { + if (location == VARYING_SLOT_COL0 || location == VARYING_SLOT_COL1 || + (location >= VARYING_SLOT_VAR0 && location <= VARYING_SLOT_VAR31) || + (location >= VARYING_SLOT_TEX0 && location <= VARYING_SLOT_TEX7) || + location == VARYING_SLOT_BFC0 || location == VARYING_SLOT_BFC1 || + location == VARYING_SLOT_PNTC || location == VARYING_SLOT_CLIP_VERTEX || + location == VARYING_SLOT_CLIP_DIST0 || location == VARYING_SLOT_CLIP_DIST1 || + location == VARYING_SLOT_PRIMITIVE_ID || location == VARYING_SLOT_POS || + location == VARYING_SLOT_PSIZ || location == VARYING_SLOT_LAYER || + location == VARYING_SLOT_VIEWPORT || location == VARYING_SLOT_FOGC) { auto semantic = r600_get_varying_semantic(location); tgsi_semantic name = (tgsi_semantic)semantic.first; @@ -96,11 +89,10 @@ bool GeometryShader::process_store_output(nir_intrinsic_instr *instr) m_out_misc_write = true; } - if (location == VARYING_SLOT_CLIP_DIST0 || - location == VARYING_SLOT_CLIP_DIST1) { + if (location == VARYING_SLOT_CLIP_DIST0 || location == VARYING_SLOT_CLIP_DIST1) { auto write_mask = nir_intrinsic_write_mask(instr); m_cc_dist_mask |= write_mask << (4 * (location - VARYING_SLOT_CLIP_DIST0)); - m_clip_dist_write |= write_mask << (4 * (location - VARYING_SLOT_CLIP_DIST0)); + m_clip_dist_write |= write_mask << (4 * (location - VARYING_SLOT_CLIP_DIST0)); } if (m_noutputs <= driver_location && @@ -112,7 +104,8 @@ bool GeometryShader::process_store_output(nir_intrinsic_instr *instr) return false; } -bool GeometryShader::process_load_input(nir_intrinsic_instr *instr) +bool +GeometryShader::process_load_input(nir_intrinsic_instr *instr) { auto location = nir_intrinsic_io_semantics(instr).location; auto index = nir_src_as_const_value(instr->src[1]); @@ -120,21 +113,14 @@ bool GeometryShader::process_load_input(nir_intrinsic_instr *instr) auto driver_location = nir_intrinsic_base(instr) + index->u32; - if (location == VARYING_SLOT_POS || - location == VARYING_SLOT_PSIZ || - location == VARYING_SLOT_FOGC || - location == VARYING_SLOT_CLIP_VERTEX || - location == VARYING_SLOT_CLIP_DIST0 || - location == VARYING_SLOT_CLIP_DIST1 || - location == VARYING_SLOT_COL0 || - location == VARYING_SLOT_COL1 || - location == VARYING_SLOT_BFC0 || - location == VARYING_SLOT_BFC1 || + if (location == VARYING_SLOT_POS || location == VARYING_SLOT_PSIZ || + location == VARYING_SLOT_FOGC || location == VARYING_SLOT_CLIP_VERTEX || + location == VARYING_SLOT_CLIP_DIST0 || location == VARYING_SLOT_CLIP_DIST1 || + location == VARYING_SLOT_COL0 || location == VARYING_SLOT_COL1 || + location == VARYING_SLOT_BFC0 || location == VARYING_SLOT_BFC1 || location == VARYING_SLOT_PNTC || - (location >= VARYING_SLOT_VAR0 && - location <= VARYING_SLOT_VAR31) || - (location >= VARYING_SLOT_TEX0 && - location <= VARYING_SLOT_TEX7)) { + (location >= VARYING_SLOT_VAR0 && location <= VARYING_SLOT_VAR31) || + (location >= VARYING_SLOT_TEX0 && location <= VARYING_SLOT_TEX7)) { uint64_t bit = 1ull << location; if (!(bit & m_input_mask)) { @@ -151,10 +137,11 @@ bool GeometryShader::process_load_input(nir_intrinsic_instr *instr) return false; } -int GeometryShader::do_allocate_reserved_registers() +int +GeometryShader::do_allocate_reserved_registers() { - const int sel[6] = {0, 0 ,0, 1, 1, 1}; - const int chan[6] = {0, 1 ,3, 0, 1, 2}; + const int sel[6] = {0, 0, 0, 1, 1, 1}; + const int chan[6] = {0, 1, 3, 0, 1, 2}; /* Reserve registers used by the shaders (should check how many * components are actually used */ @@ -174,27 +161,26 @@ int GeometryShader::do_allocate_reserved_registers() for (int i = 0; i < 4; ++i) { m_export_base[i] = value_factory().temp_register(0, false); - emit_instruction(new AluInstr(op1_mov, m_export_base[i], zero, AluInstr::last_write)); + emit_instruction( + new AluInstr(op1_mov, m_export_base[i], zero, AluInstr::last_write)); } m_ring_item_sizes[0] = m_next_input_ring_offset; - /* GS thread with no output workaround - emit a cut at start of GS */ if (chip_class() == ISA_CC_R600) { emit_instruction(new EmitVertexInstr(0, true)); start_new_block(0); } - - if (m_tri_strip_adj_fix) emit_adj_fix(); return value_factory().next_register_index(); } -bool GeometryShader::process_stage_intrinsic(nir_intrinsic_instr *intr) +bool +GeometryShader::process_stage_intrinsic(nir_intrinsic_instr *intr) { switch (intr->intrinsic) { case nir_intrinsic_emit_vertex: @@ -207,20 +193,20 @@ bool GeometryShader::process_stage_intrinsic(nir_intrinsic_instr *intr) return emit_simple_mov(intr->dest, 0, m_invocation_id); case nir_intrinsic_load_per_vertex_input: return emit_load_per_vertex_input(intr); - default: - ; + default:; } return false; } -bool GeometryShader::emit_vertex(nir_intrinsic_instr* instr, bool cut) +bool +GeometryShader::emit_vertex(nir_intrinsic_instr *instr, bool cut) { int stream = nir_intrinsic_stream_id(instr); assert(stream < 4); auto cut_instr = new EmitVertexInstr(stream, cut); - for(auto v: m_streamout_data) { + for (auto v : m_streamout_data) { if (stream == 0 || v.first != VARYING_SLOT_POS) { v.second->patch_ring(stream, m_export_base[stream]); cut_instr->add_required_instr(v.second); @@ -234,7 +220,9 @@ bool GeometryShader::emit_vertex(nir_intrinsic_instr* instr, bool cut) start_new_block(0); if (!cut) { - auto ir = new AluInstr(op2_add_int, m_export_base[stream], m_export_base[stream], + auto ir = new AluInstr(op2_add_int, + m_export_base[stream], + m_export_base[stream], value_factory().literal(m_noutputs), AluInstr::last_write); emit_instruction(ir); @@ -243,7 +231,8 @@ bool GeometryShader::emit_vertex(nir_intrinsic_instr* instr, bool cut) return true; } -bool GeometryShader::store_output(nir_intrinsic_instr* instr) +bool +GeometryShader::store_output(nir_intrinsic_instr *instr) { if (nir_intrinsic_io_semantics(instr).location == VARYING_SLOT_CLIP_VERTEX) return true; @@ -256,9 +245,9 @@ bool GeometryShader::store_output(nir_intrinsic_instr* instr) uint32_t write_mask = nir_intrinsic_write_mask(instr); uint32_t shift = nir_intrinsic_component(instr); - RegisterVec4::Swizzle src_swz {7,7,7,7}; + RegisterVec4::Swizzle src_swz{7, 7, 7, 7}; for (unsigned i = shift; i < 4; ++i) { - src_swz[i] = (1 << i) & (write_mask << shift) ? i - shift: 7; + src_swz[i] = (1 << i) & (write_mask << shift) ? i - shift : 7; } auto out_value = value_factory().src_vec4(instr->src[0], pin_free, src_swz); @@ -271,7 +260,10 @@ bool GeometryShader::store_output(nir_intrinsic_instr* instr) if (!(write_mask & (1 << i))) continue; if (out_value[i + shift]->chan() < 4) { - ir = new AluInstr(op1_mov, tmp[i + shift], out_value[i + shift], AluInstr::write); + ir = new AluInstr(op1_mov, + tmp[i + shift], + out_value[i + shift], + AluInstr::write); } else if (value[i]->chan() < 4) { ir = new AluInstr(op1_mov, tmp[i + shift], value[i], AluInstr::write); } else @@ -279,13 +271,16 @@ bool GeometryShader::store_output(nir_intrinsic_instr* instr) emit_instruction(ir); } ir->set_alu_flag(alu_last_instr); - m_streamout_data[location] = new MemRingOutInstr(cf_mem_ring, MemRingOutInstr::mem_write_ind, tmp, + m_streamout_data[location] = new MemRingOutInstr(cf_mem_ring, + MemRingOutInstr::mem_write_ind, + tmp, 4 * driver_location, - instr->num_components, m_export_base[0]); + instr->num_components, + m_export_base[0]); } else { sfn_log << SfnLog::io << "None-streamout "; - bool need_copy = shift != 0; + bool need_copy = shift != 0; if (!need_copy) { for (int i = 0; i < 4; ++i) { if ((write_mask & (1 << i)) && (out_value[i]->chan() != i)) { @@ -304,26 +299,33 @@ bool GeometryShader::store_output(nir_intrinsic_instr* instr) } } ir->set_alu_flag(alu_last_instr); - m_streamout_data[location] = new MemRingOutInstr(cf_mem_ring, MemRingOutInstr::mem_write_ind, tmp, + m_streamout_data[location] = new MemRingOutInstr(cf_mem_ring, + MemRingOutInstr::mem_write_ind, + tmp, 4 * driver_location, - instr->num_components, m_export_base[0]); + instr->num_components, + m_export_base[0]); } else { for (auto i = 0; i < 4; ++i) out_value[i]->set_pin(pin_chgr); - m_streamout_data[location] = new MemRingOutInstr(cf_mem_ring, MemRingOutInstr::mem_write_ind, out_value, + m_streamout_data[location] = new MemRingOutInstr(cf_mem_ring, + MemRingOutInstr::mem_write_ind, + out_value, 4 * driver_location, - instr->num_components, m_export_base[0]); + instr->num_components, + m_export_base[0]); } } return true; } -bool GeometryShader::emit_load_per_vertex_input(nir_intrinsic_instr* instr) +bool +GeometryShader::emit_load_per_vertex_input(nir_intrinsic_instr *instr) { auto dest = value_factory().dest_vec4(instr->dest, pin_group); - RegisterVec4::Swizzle dest_swz{7,7,7,7}; + RegisterVec4::Swizzle dest_swz{7, 7, 7, 7}; for (unsigned i = 0; i < nir_dest_num_components(instr->dest); ++i) { dest_swz[i] = i + nir_intrinsic_component(instr); } @@ -337,16 +339,16 @@ bool GeometryShader::emit_load_per_vertex_input(nir_intrinsic_instr* instr) assert(literal_index->u32 < 6); assert(nir_intrinsic_io_semantics(instr).num_slots == 1); - - EVTXDataFormat fmt = chip_class() >= ISA_CC_EVERGREEN ? - fmt_invalid: - fmt_32_32_32_32_float; - + EVTXDataFormat fmt = + chip_class() >= ISA_CC_EVERGREEN ? fmt_invalid : fmt_32_32_32_32_float; auto addr = m_per_vertex_offsets[literal_index->u32]; - auto fetch = new LoadFromBuffer(dest, dest_swz, addr, + auto fetch = new LoadFromBuffer(dest, + dest_swz, + addr, 16 * nir_intrinsic_base(instr), - R600_GS_RING_CONST_BUFFER, nullptr, + R600_GS_RING_CONST_BUFFER, + nullptr, fmt); if (chip_class() >= ISA_CC_EVERGREEN) @@ -359,36 +361,42 @@ bool GeometryShader::emit_load_per_vertex_input(nir_intrinsic_instr* instr) return true; } -void GeometryShader::do_finalize() +void +GeometryShader::do_finalize() { - } -void GeometryShader::do_get_shader_info(r600_shader *sh_info) +void +GeometryShader::do_get_shader_info(r600_shader *sh_info) { sh_info->processor_type = PIPE_SHADER_GEOMETRY; - sh_info->ring_item_sizes[0] = m_ring_item_sizes[0]; + sh_info->ring_item_sizes[0] = m_ring_item_sizes[0]; sh_info->cc_dist_mask = m_cc_dist_mask; sh_info->clip_dist_write = m_clip_dist_write; } -bool GeometryShader::read_prop(std::istream& is) +bool +GeometryShader::read_prop(std::istream& is) { (void)is; return true; } -void GeometryShader::do_print_properties(std::ostream& os) const +void +GeometryShader::do_print_properties(std::ostream& os) const { (void)os; } -void GeometryShader::emit_adj_fix() +void +GeometryShader::emit_adj_fix() { auto adjhelp0 = value_factory().temp_register(); - emit_instruction(new AluInstr(op2_and_int, adjhelp0, - m_primitive_id, value_factory().one_i(), + emit_instruction(new AluInstr(op2_and_int, + adjhelp0, + m_primitive_id, + value_factory().one_i(), AluInstr::last_write)); int reg_indices[6]; @@ -402,8 +410,10 @@ void GeometryShader::emit_adj_fix() AluInstr *ir = nullptr; for (int i = 0; i < 6; i++) { adjhelp[i] = value_factory().temp_register(); - ir = new AluInstr(op3_cnde_int, adjhelp[i], - adjhelp0, m_per_vertex_offsets[i], + ir = new AluInstr(op3_cnde_int, + adjhelp[i], + adjhelp0, + m_per_vertex_offsets[i], m_per_vertex_offsets[rotate_indices[i]], AluInstr::write); @@ -415,5 +425,4 @@ void GeometryShader::emit_adj_fix() m_per_vertex_offsets[i] = adjhelp[i]; } - -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_shader_gs.h b/src/gallium/drivers/r600/sfn/sfn_shader_gs.h index f26d73adf1e..206e7713052 100644 --- a/src/gallium/drivers/r600/sfn/sfn_shader_gs.h +++ b/src/gallium/drivers/r600/sfn/sfn_shader_gs.h @@ -1,9 +1,8 @@ #ifndef SFN_GEOMETRYSHADER_H #define SFN_GEOMETRYSHADER_H - -#include "sfn_shader.h" #include "sfn_instr_export.h" +#include "sfn_shader.h" namespace r600 { @@ -11,8 +10,6 @@ class GeometryShader : public Shader { public: GeometryShader(const r600_shader_key& key); - - private: bool do_scan_instruction(nir_instr *instr) override; int do_allocate_reserved_registers() override; @@ -31,13 +28,14 @@ private: void emit_adj_fix(); - bool emit_load_per_vertex_input(nir_intrinsic_instr* instr); + bool emit_load_per_vertex_input(nir_intrinsic_instr *instr); - bool load_input(UNUSED nir_intrinsic_instr *intr) override { + bool load_input(UNUSED nir_intrinsic_instr *intr) override + { unreachable("load_input must be lowered in GS"); }; - bool store_output(nir_intrinsic_instr* instr) override; - bool emit_vertex(nir_intrinsic_instr* instr, bool cut); + bool store_output(nir_intrinsic_instr *instr) override; + bool emit_vertex(nir_intrinsic_instr *instr, bool cut); std::array m_per_vertex_offsets{nullptr}; PRegister m_primitive_id{nullptr}; @@ -62,6 +60,6 @@ private: std::map m_streamout_data; }; -} +} // namespace r600 #endif // GEOMETRYSHADER_H diff --git a/src/gallium/drivers/r600/sfn/sfn_shader_tess.cpp b/src/gallium/drivers/r600/sfn/sfn_shader_tess.cpp index b707bdc5317..e8b8a248a09 100644 --- a/src/gallium/drivers/r600/sfn/sfn_shader_tess.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_shader_tess.cpp @@ -24,10 +24,10 @@ * USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include "sfn_instr_export.h" #include "sfn_shader_tess.h" -#include "sfn_shader_vs.h" +#include "sfn_instr_export.h" +#include "sfn_shader_vs.h" #include @@ -36,18 +36,18 @@ namespace r600 { using std::string; TCSShader::TCSShader(const r600_shader_key& key): - Shader("TCS"), - m_tcs_prim_mode(key.tcs.prim_mode) + Shader("TCS"), + m_tcs_prim_mode(key.tcs.prim_mode) { - } -bool TCSShader::do_scan_instruction(nir_instr *instr) +bool +TCSShader::do_scan_instruction(nir_instr *instr) { if (instr->type != nir_instr_type_intrinsic) return false; - nir_intrinsic_instr *ii = nir_instr_as_intrinsic(instr); + nir_intrinsic_instr *ii = nir_instr_as_intrinsic(instr); switch (ii->intrinsic) { case nir_intrinsic_load_primitive_id: @@ -69,7 +69,8 @@ bool TCSShader::do_scan_instruction(nir_instr *instr) return true; } -int TCSShader::do_allocate_reserved_registers() +int +TCSShader::do_allocate_reserved_registers() { if (m_sv_values.test(es_primitive_id)) { m_primitive_id = value_factory().allocate_pinned_register(0, 0); @@ -82,7 +83,8 @@ int TCSShader::do_allocate_reserved_registers() } if (m_sv_values.test(es_rel_patch_id)) { - m_rel_patch_id = value_factory().allocate_pinned_register(0, 1);; + m_rel_patch_id = value_factory().allocate_pinned_register(0, 1); + ; m_rel_patch_id->pin_live_range(true); } @@ -91,10 +93,12 @@ int TCSShader::do_allocate_reserved_registers() m_tess_factor_base->pin_live_range(true); } - return value_factory().next_register_index();; + return value_factory().next_register_index(); + ; } -bool TCSShader::process_stage_intrinsic(nir_intrinsic_instr *instr) +bool +TCSShader::process_stage_intrinsic(nir_intrinsic_instr *instr) { switch (instr->intrinsic) { case nir_intrinsic_load_tcs_rel_patch_id_r600: @@ -112,22 +116,26 @@ bool TCSShader::process_stage_intrinsic(nir_intrinsic_instr *instr) } } -bool TCSShader::store_tess_factor(nir_intrinsic_instr* instr) +bool +TCSShader::store_tess_factor(nir_intrinsic_instr *instr) { bool two_parts = nir_src_num_components(instr->src[0]) == 4; auto value0 = value_factory().temp_vec4(pin_group, {0, 1, 7, 7}); - emit_instruction(new AluInstr(op1_mov, value0[0], value_factory().src(instr->src[0], 0), - AluInstr::write)); - emit_instruction(new AluInstr(op1_mov, value0[1], value_factory().src(instr->src[0], 1), + emit_instruction(new AluInstr( + op1_mov, value0[0], value_factory().src(instr->src[0], 0), AluInstr::write)); + emit_instruction(new AluInstr(op1_mov, + value0[1], + value_factory().src(instr->src[0], 1), two_parts ? AluInstr::write : AluInstr::last_write)); - if (two_parts) { auto value1 = value_factory().temp_vec4(pin_group, {2, 3, 7, 7}); - emit_instruction(new AluInstr(op1_mov, value1[0], value_factory().src(instr->src[0], 2), - AluInstr::write)); - emit_instruction(new AluInstr(op1_mov, value1[1], value_factory().src(instr->src[0], 3), + emit_instruction(new AluInstr( + op1_mov, value1[0], value_factory().src(instr->src[0], 2), AluInstr::write)); + emit_instruction(new AluInstr(op1_mov, + value1[1], + value_factory().src(instr->src[0], 3), AluInstr::last_write)); emit_instruction(new WriteTFInstr(value1)); } @@ -136,14 +144,15 @@ bool TCSShader::store_tess_factor(nir_intrinsic_instr* instr) return true; } - -void TCSShader::do_get_shader_info(r600_shader *sh_info) +void +TCSShader::do_get_shader_info(r600_shader *sh_info) { sh_info->processor_type = PIPE_SHADER_TESS_CTRL; sh_info->tcs_prim_mode = m_tcs_prim_mode; } -bool TCSShader::read_prop(std::istream& is) +bool +TCSShader::read_prop(std::istream& is) { string value; is >> value; @@ -164,16 +173,18 @@ bool TCSShader::read_prop(std::istream& is) return true; } -void TCSShader::do_print_properties(std::ostream& os) const +void +TCSShader::do_print_properties(std::ostream& os) const { os << "PROP TCS_PRIM_MODE:" << m_tcs_prim_mode << "\n"; } -TESShader::TESShader(const pipe_stream_output_info *so_info, const r600_shader *gs_shader, +TESShader::TESShader(const pipe_stream_output_info *so_info, + const r600_shader *gs_shader, const r600_shader_key& key): - VertexStageShader("TES"), - m_vs_as_gs_a(key.vs.as_gs_a), - m_tes_as_es(key.tes.as_es) + VertexStageShader("TES"), + m_vs_as_gs_a(key.vs.as_gs_a), + m_tes_as_es(key.tes.as_es) { if (key.tes.as_es) m_export_processor = new VertexExportForGS(this, gs_shader); @@ -181,7 +192,8 @@ TESShader::TESShader(const pipe_stream_output_info *so_info, const r600_shader * m_export_processor = new VertexExportForFs(this, so_info, key); } -bool TESShader::do_scan_instruction(nir_instr *instr) +bool +TESShader::do_scan_instruction(nir_instr *instr) { if (instr->type != nir_instr_type_intrinsic) return false; @@ -236,7 +248,8 @@ bool TESShader::do_scan_instruction(nir_instr *instr) return true; } -int TESShader::do_allocate_reserved_registers() +int +TESShader::do_allocate_reserved_registers() { if (m_sv_values.test(es_tess_coord)) { m_tess_coord[0] = value_factory().allocate_pinned_register(0, 0); @@ -257,12 +270,13 @@ int TESShader::do_allocate_reserved_registers() return value_factory().next_register_index(); } -bool TESShader::process_stage_intrinsic(nir_intrinsic_instr *intr) +bool +TESShader::process_stage_intrinsic(nir_intrinsic_instr *intr) { switch (intr->intrinsic) { case nir_intrinsic_load_tess_coord_r600: return emit_simple_mov(intr->dest, 0, m_tess_coord[0], pin_none) && - emit_simple_mov(intr->dest, 1, m_tess_coord[1], pin_none); + emit_simple_mov(intr->dest, 1, m_tess_coord[1], pin_none); case nir_intrinsic_load_primitive_id: return emit_simple_mov(intr->dest, 0, m_primitive_id); case nir_intrinsic_load_tcs_rel_patch_id_r600: @@ -274,27 +288,30 @@ bool TESShader::process_stage_intrinsic(nir_intrinsic_instr *intr) } } -void TESShader::do_get_shader_info(r600_shader *sh_info) +void +TESShader::do_get_shader_info(r600_shader *sh_info) { sh_info->processor_type = PIPE_SHADER_TESS_EVAL; m_export_processor->get_shader_info(sh_info); } -void TESShader::do_finalize() +void +TESShader::do_finalize() { m_export_processor->finalize(); } -bool TESShader::TESShader::read_prop(std::istream& is) +bool +TESShader::TESShader::read_prop(std::istream& is) { (void)is; return true; } -void TESShader::do_print_properties(std::ostream& os) const +void +TESShader::do_print_properties(std::ostream& os) const { (void)os; } - -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_shader_tess.h b/src/gallium/drivers/r600/sfn/sfn_shader_tess.h index 6e5db1bbe9a..2273bb31179 100644 --- a/src/gallium/drivers/r600/sfn/sfn_shader_tess.h +++ b/src/gallium/drivers/r600/sfn/sfn_shader_tess.h @@ -36,18 +36,21 @@ class VertexExportStage; class TCSShader : public Shader { public: TCSShader(const r600_shader_key& key); + private: bool do_scan_instruction(nir_instr *instr) override; int do_allocate_reserved_registers() override; bool process_stage_intrinsic(nir_intrinsic_instr *intr) override; void do_get_shader_info(r600_shader *sh_info) override; - bool store_tess_factor(nir_intrinsic_instr* instr); + bool store_tess_factor(nir_intrinsic_instr *instr); - bool load_input(nir_intrinsic_instr *intr) override { + bool load_input(nir_intrinsic_instr *intr) override + { unreachable("load_input must be lowered in TCS"); }; - bool store_output(nir_intrinsic_instr *intr) override { + bool store_output(nir_intrinsic_instr *intr) override + { unreachable("load_output must be lowered in TCS"); }; @@ -64,8 +67,10 @@ private: class TESShader : public VertexStageShader { public: - TESShader(const pipe_stream_output_info *so_info, const r600_shader *gs_shader, + TESShader(const pipe_stream_output_info *so_info, + const r600_shader *gs_shader, const r600_shader_key& key); + private: bool do_scan_instruction(nir_instr *instr) override; int do_allocate_reserved_registers() override; @@ -73,10 +78,12 @@ private: bool process_stage_intrinsic(nir_intrinsic_instr *intr) override; void do_get_shader_info(r600_shader *sh_info) override; - bool load_input(UNUSED nir_intrinsic_instr *intr) override { + bool load_input(UNUSED nir_intrinsic_instr *intr) override + { unreachable("load_input must be lowered in TES"); }; - bool store_output(UNUSED nir_intrinsic_instr *intr) override { + bool store_output(UNUSED nir_intrinsic_instr *intr) override + { unreachable("load_output must be lowered in TES"); }; @@ -85,7 +92,7 @@ private: void do_finalize() override; - PRegister m_tess_coord[2]= {nullptr, nullptr}; + PRegister m_tess_coord[2] = {nullptr, nullptr}; PRegister m_rel_patch_id{nullptr}; PRegister m_primitive_id{nullptr}; @@ -96,7 +103,6 @@ private: bool m_tes_as_es{false}; }; - -} +} // namespace r600 #endif // TCS_H diff --git a/src/gallium/drivers/r600/sfn/sfn_shader_vs.cpp b/src/gallium/drivers/r600/sfn/sfn_shader_vs.cpp index 9085326ba40..af3a9601e96 100644 --- a/src/gallium/drivers/r600/sfn/sfn_shader_vs.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_shader_vs.cpp @@ -24,59 +24,58 @@ * USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include "sfn_debug.h" #include "sfn_shader_vs.h" +#include "sfn_debug.h" #include "sfn_instr_alugroup.h" #include "sfn_instr_export.h" - #include "tgsi/tgsi_from_mesa.h" - namespace r600 { -uint32_t VertexStageShader::enabled_stream_buffers_mask() const +uint32_t +VertexStageShader::enabled_stream_buffers_mask() const { return m_enabled_stream_buffers_mask; } -void VertexStageShader::combine_enabled_stream_buffers_mask(uint32_t mask) +void +VertexStageShader::combine_enabled_stream_buffers_mask(uint32_t mask) { m_enabled_stream_buffers_mask = mask; } -bool VertexExportStage::store_output(nir_intrinsic_instr& intr) +bool +VertexExportStage::store_output(nir_intrinsic_instr& intr) { auto index = nir_src_as_const_value(intr.src[1]); assert(index && "Indirect outputs not supported"); - const store_loc store_info = { - nir_intrinsic_component(&intr), - nir_intrinsic_io_semantics(&intr).location, - (unsigned)nir_intrinsic_base(&intr) + index->u32, - 0 - }; + const store_loc store_info = {nir_intrinsic_component(&intr), + nir_intrinsic_io_semantics(&intr).location, + (unsigned)nir_intrinsic_base(&intr) + index->u32, + 0}; return do_store_output(store_info, intr); } VertexExportStage::VertexExportStage(VertexStageShader *parent): - m_parent(parent) + m_parent(parent) { - } VertexExportForFs::VertexExportForFs(VertexStageShader *parent, const pipe_stream_output_info *so_info, const r600_shader_key& key): - VertexExportStage(parent), - m_vs_as_gs_a(key.vs.as_gs_a), - m_vs_prim_id_out(key.vs.prim_id_out), - m_so_info(so_info) + VertexExportStage(parent), + m_vs_as_gs_a(key.vs.as_gs_a), + m_vs_prim_id_out(key.vs.prim_id_out), + m_so_info(so_info) { } -bool VertexExportForFs::do_store_output(const store_loc &store_info, nir_intrinsic_instr& intr) +bool +VertexExportForFs::do_store_output(const store_loc& store_info, nir_intrinsic_instr& intr) { switch (store_info.location) { @@ -86,13 +85,13 @@ bool VertexExportForFs::do_store_output(const store_loc &store_info, nir_intrins case VARYING_SLOT_POS: return emit_varying_pos(store_info, intr); case VARYING_SLOT_EDGE: { - std::array swizzle_override = {7 ,0, 7, 7}; + std::array swizzle_override = {7, 0, 7, 7}; return emit_varying_pos(store_info, intr, &swizzle_override); } case VARYING_SLOT_VIEWPORT: { std::array swizzle_override = {7, 7, 7, 0}; return emit_varying_pos(store_info, intr, &swizzle_override) && - emit_varying_param(store_info, intr); + emit_varying_param(store_info, intr); } case VARYING_SLOT_CLIP_VERTEX: return emit_clip_vertices(store_info, intr); @@ -103,43 +102,45 @@ bool VertexExportForFs::do_store_output(const store_loc &store_info, nir_intrins if (!nir_intrinsic_io_semantics(&intr).no_varying) success &= emit_varying_param(store_info, intr); return success; - } + } case VARYING_SLOT_LAYER: { m_out_misc_write = 1; m_vs_out_layer = 1; - std::array swz = {7,7,0,7}; + std::array swz = {7, 7, 0, 7}; return emit_varying_pos(store_info, intr, &swz) && - emit_varying_param(store_info, intr); + emit_varying_param(store_info, intr); } case VARYING_SLOT_VIEW_INDEX: - return emit_varying_pos(store_info, intr) && - emit_varying_param(store_info, intr); + return emit_varying_pos(store_info, intr) && emit_varying_param(store_info, intr); default: return emit_varying_param(store_info, intr); - return false; + return false; } } -bool VertexExportForFs::emit_clip_vertices(const store_loc &store_info, const nir_intrinsic_instr &instr) +bool +VertexExportForFs::emit_clip_vertices(const store_loc& store_info, + const nir_intrinsic_instr& instr) { auto& vf = m_parent->value_factory(); m_cc_dist_mask = 0xff; m_clip_dist_write = 0xff; - m_clip_vertex = vf.src_vec4(instr.src[store_info.data_loc], pin_group, {0,1,2,3}); + m_clip_vertex = vf.src_vec4(instr.src[store_info.data_loc], pin_group, {0, 1, 2, 3}); m_output_registers[nir_intrinsic_base(&instr)] = &m_clip_vertex; return true; } -void VertexExportForFs::get_shader_info(r600_shader *sh_info) const +void +VertexExportForFs::get_shader_info(r600_shader *sh_info) const { sh_info->cc_dist_mask = m_cc_dist_mask; sh_info->clip_dist_write = m_clip_dist_write; - sh_info->vs_as_gs_a = m_vs_as_gs_a; + sh_info->vs_as_gs_a = m_vs_as_gs_a; sh_info->vs_out_edgeflag = m_out_edgeflag; sh_info->vs_out_viewport = m_out_viewport; sh_info->vs_out_misc_write = m_out_misc_write; @@ -147,11 +148,13 @@ void VertexExportForFs::get_shader_info(r600_shader *sh_info) const sh_info->vs_out_layer = m_vs_out_layer; } -void VertexExportForFs::finalize() +void +VertexExportForFs::finalize() { if (m_vs_as_gs_a) { auto primid = m_parent->value_factory().temp_vec4(pin_group, {2, 7, 7, 7}); - m_parent->emit_instruction(new AluInstr(op1_mov, primid[0], m_parent->primitive_id(), AluInstr::last_write)); + m_parent->emit_instruction(new AluInstr( + op1_mov, primid[0], m_parent->primitive_id(), AluInstr::last_write)); int param = m_last_param_export ? m_last_param_export->location() + 1 : 0; m_last_param_export = new ExportInstr(ExportInstr::param, param, primid); @@ -164,13 +167,13 @@ void VertexExportForFs::finalize() } if (!m_last_pos_export) { - RegisterVec4 value(0, false, {7,7,7,7}); + RegisterVec4 value(0, false, {7, 7, 7, 7}); m_last_pos_export = new ExportInstr(ExportInstr::pos, 0, value); m_parent->emit_instruction(m_last_pos_export); } if (!m_last_param_export) { - RegisterVec4 value(0,false, {7,7,7,7}); + RegisterVec4 value(0, false, {7, 7, 7, 7}); m_last_param_export = new ExportInstr(ExportInstr::param, 0, value); m_parent->emit_instruction(m_last_param_export); } @@ -182,14 +185,17 @@ void VertexExportForFs::finalize() emit_stream(-1); } -void VertexShader::do_get_shader_info(r600_shader *sh_info) +void +VertexShader::do_get_shader_info(r600_shader *sh_info) { sh_info->processor_type = PIPE_SHADER_VERTEX; - m_export_stage->get_shader_info(sh_info); + m_export_stage->get_shader_info(sh_info); } -bool VertexExportForFs::emit_varying_pos(const store_loc &store_info, nir_intrinsic_instr& intr, - std::array *swizzle_override) +bool +VertexExportForFs::emit_varying_pos(const store_loc& store_info, + nir_intrinsic_instr& intr, + std::array *swizzle_override) { RegisterVec4::Swizzle swizzle; uint32_t write_mask = 0; @@ -214,8 +220,10 @@ bool VertexExportForFs::emit_varying_pos(const store_loc &store_info, nir_intrin m_out_edgeflag = true; auto src = m_parent->value_factory().src(intr.src[0], 0); auto clamped = m_parent->value_factory().temp_register(); - m_parent->emit_instruction(new AluInstr(op1_mov, clamped, src, {alu_write, alu_dst_clamp, alu_last_instr})); - auto alu = new AluInstr(op1_flt_to_int, out_value[1], clamped, AluInstr::last_write); + m_parent->emit_instruction( + new AluInstr(op1_mov, clamped, src, {alu_write, alu_dst_clamp, alu_last_instr})); + auto alu = + new AluInstr(op1_flt_to_int, out_value[1], clamped, AluInstr::last_write); if (m_parent->chip_class() < ISA_CC_EVERGREEN) alu->set_alu_flag(alu_is_trans); m_parent->emit_instruction(alu); @@ -238,19 +246,20 @@ bool VertexExportForFs::emit_varying_pos(const store_loc &store_info, nir_intrin case VARYING_SLOT_POS: break; case VARYING_SLOT_CLIP_DIST0: - case VARYING_SLOT_CLIP_DIST1: - m_cc_dist_mask |= write_mask << (4 * (store_info.location - VARYING_SLOT_CLIP_DIST0)); - m_clip_dist_write |= write_mask << (4 * (store_info.location - VARYING_SLOT_CLIP_DIST0)); + case VARYING_SLOT_CLIP_DIST1: + m_cc_dist_mask |= write_mask + << (4 * (store_info.location - VARYING_SLOT_CLIP_DIST0)); + m_clip_dist_write |= write_mask + << (4 * (store_info.location - VARYING_SLOT_CLIP_DIST0)); export_slot = m_cur_clip_pos++; break; default: - sfn_log << SfnLog::err << __func__ << "Unsupported location " - << store_info.location << "\n"; + sfn_log << SfnLog::err << __func__ << "Unsupported location " << store_info.location + << "\n"; return false; } - - m_last_pos_export = new ExportInstr(ExportInstr::pos, export_slot, value); + m_last_pos_export = new ExportInstr(ExportInstr::pos, export_slot, value); m_output_registers[nir_intrinsic_base(&intr)] = &m_last_pos_export->value(); @@ -259,17 +268,19 @@ bool VertexExportForFs::emit_varying_pos(const store_loc &store_info, nir_intrin return true; } - -bool VertexExportForFs::emit_varying_param(const store_loc &store_info, nir_intrinsic_instr& intr) +bool +VertexExportForFs::emit_varying_param(const store_loc& store_info, + nir_intrinsic_instr& intr) { - sfn_log << SfnLog::io << __func__ << ": emit DDL: " << store_info.driver_location << "\n"; + sfn_log << SfnLog::io << __func__ << ": emit DDL: " << store_info.driver_location + << "\n"; int write_mask = nir_intrinsic_write_mask(&intr) << store_info.frac; RegisterVec4::Swizzle swizzle; for (int i = 0; i < 4; ++i) swizzle[i] = ((1 << i) & write_mask) ? i - store_info.frac : 7; - Pin pin = util_bitcount(write_mask) > 1 ? pin_group: pin_free; + Pin pin = util_bitcount(write_mask) > 1 ? pin_group : pin_free; int export_slot = m_parent->output(nir_intrinsic_base(&intr)).pos(); auto value = m_parent->value_factory().temp_vec4(pin, swizzle); @@ -277,15 +288,17 @@ bool VertexExportForFs::emit_varying_param(const store_loc &store_info, nir_intr AluInstr *alu = nullptr; for (int i = 0; i < 4; ++i) { if (swizzle[i] < 4) { - alu = new AluInstr(op1_mov, value[i], m_parent->value_factory().src(intr.src[0], swizzle[i]), - AluInstr::write); + alu = new AluInstr(op1_mov, + value[i], + m_parent->value_factory().src(intr.src[0], swizzle[i]), + AluInstr::write); m_parent->emit_instruction(alu); } } if (alu) alu->set_alu_flag(alu_last_instr); - m_last_param_export = new ExportInstr(ExportInstr::param, export_slot, value); + m_last_param_export = new ExportInstr(ExportInstr::param, export_slot, value); m_output_registers[nir_intrinsic_base(&intr)] = &m_last_param_export->value(); m_parent->emit_instruction(m_last_param_export); @@ -293,7 +306,8 @@ bool VertexExportForFs::emit_varying_param(const store_loc &store_info, nir_intr return true; } -bool VertexExportForFs::emit_stream(int stream) +bool +VertexExportForFs::emit_stream(int stream) { assert(m_so_info); if (m_so_info->num_outputs > PIPE_MAX_SO_OUTPUTS) { @@ -316,16 +330,15 @@ bool VertexExportForFs::emit_stream(int stream) if (stream != -1 && stream != m_so_info->output[i].stream) continue; - sfn_log << SfnLog::instr << "Emit stream " << i - << " with register index " << m_so_info->output[i].register_index << " so_gpr:"; - + sfn_log << SfnLog::instr << "Emit stream " << i << " with register index " + << m_so_info->output[i].register_index << " so_gpr:"; so_gpr[i] = output_register(m_so_info->output[i].register_index); if (!so_gpr[i]) { sfn_log << SfnLog::err << "\nERR: register index " - << m_so_info->output[i].register_index - << " doesn't correspond to an output register\n"; + << m_so_info->output[i].register_index + << " doesn't correspond to an output register\n"; return false; } start_comp[i] = m_so_info->output[i].start_component; @@ -336,7 +349,8 @@ bool VertexExportForFs::emit_stream(int stream) * to store Y, Z, or W at buffer offset 0, we need to use MOV * to move it to X and output X. */ - bool need_copy = m_so_info->output[i].dst_offset < m_so_info->output[i].start_component; + bool need_copy = + m_so_info->output[i].dst_offset < m_so_info->output[i].start_component; int sc = m_so_info->output[i].start_component; for (int j = 0; j < m_so_info->output[i].num_components; j++) { @@ -346,7 +360,7 @@ bool VertexExportForFs::emit_stream(int stream) } } if (need_copy) { - RegisterVec4::Swizzle swizzle = {0,1,2,3}; + RegisterVec4::Swizzle swizzle = {0, 1, 2, 3}; for (auto j = m_so_info->output[i].num_components; j < 4; ++j) swizzle[j] = 7; tmp[i] = m_parent->value_factory().temp_vec4(pin_group, swizzle); @@ -362,41 +376,46 @@ bool VertexExportForFs::emit_stream(int stream) start_comp[i] = 0; so_gpr[i] = &tmp[i]; } - sfn_log << SfnLog::instr << *so_gpr[i] << "\n"; + sfn_log << SfnLog::instr << *so_gpr[i] << "\n"; } uint32_t enabled_stream_buffers_mask = 0; /* Write outputs to buffers. */ for (unsigned i = 0; i < m_so_info->num_outputs; i++) { - sfn_log << SfnLog::instr << "Write output buffer " << i - << " with register index " << m_so_info->output[i].register_index << "\n"; + sfn_log << SfnLog::instr << "Write output buffer " << i << " with register index " + << m_so_info->output[i].register_index << "\n"; auto out_stream = - new StreamOutInstr(*so_gpr[i], - m_so_info->output[i].num_components, - m_so_info->output[i].dst_offset - start_comp[i], - ((1 << m_so_info->output[i].num_components) - 1) << start_comp[i], - m_so_info->output[i].output_buffer, - m_so_info->output[i].stream); + new StreamOutInstr(*so_gpr[i], + m_so_info->output[i].num_components, + m_so_info->output[i].dst_offset - start_comp[i], + ((1 << m_so_info->output[i].num_components) - 1) + << start_comp[i], + m_so_info->output[i].output_buffer, + m_so_info->output[i].stream); m_parent->emit_instruction(out_stream); - enabled_stream_buffers_mask |= (1 << m_so_info->output[i].output_buffer) << m_so_info->output[i].stream * 4; + enabled_stream_buffers_mask |= (1 << m_so_info->output[i].output_buffer) + << m_so_info->output[i].stream * 4; } m_parent->combine_enabled_stream_buffers_mask(enabled_stream_buffers_mask); return true; } -const RegisterVec4 *VertexExportForFs::output_register(int loc) const +const RegisterVec4 * +VertexExportForFs::output_register(int loc) const { const RegisterVec4 *retval = nullptr; auto val = m_output_registers.find(loc); if (val != m_output_registers.end()) - retval = val->second; + retval = val->second; return retval; } -VertexShader::VertexShader(const pipe_stream_output_info *so_info, r600_shader *gs_shader, r600_shader_key& key): - VertexStageShader("VS"), - m_vs_as_gs_a(key.vs.as_gs_a) +VertexShader::VertexShader(const pipe_stream_output_info *so_info, + r600_shader *gs_shader, + r600_shader_key& key): + VertexStageShader("VS"), + m_vs_as_gs_a(key.vs.as_gs_a) { if (key.vs.as_es) m_export_stage = new VertexExportForGS(this, gs_shader); @@ -406,7 +425,8 @@ VertexShader::VertexShader(const pipe_stream_output_info *so_info, r600_shader * m_export_stage = new VertexExportForFs(this, so_info, key); } -bool VertexShader::do_scan_instruction(nir_instr *instr) +bool +VertexShader::do_scan_instruction(nir_instr *instr) { if (instr->type != nir_instr_type_intrinsic) return false; @@ -474,7 +494,8 @@ bool VertexShader::do_scan_instruction(nir_instr *instr) return true; } -bool VertexShader::load_input(nir_intrinsic_instr *intr) +bool +VertexShader::load_input(nir_intrinsic_instr *intr) { unsigned driver_location = nir_intrinsic_base(intr); unsigned location = nir_intrinsic_io_semantics(intr).location; @@ -489,7 +510,8 @@ bool VertexShader::load_input(nir_intrinsic_instr *intr) if (intr->dest.is_ssa) vf.inject_value(intr->dest, i, src); else { - ir = new AluInstr(op1_mov, vf.dest(intr->dest, i, pin_none), src, {alu_write}); + ir = + new AluInstr(op1_mov, vf.dest(intr->dest, i, pin_none), src, {alu_write}); emit_instruction(ir); } } @@ -505,8 +527,8 @@ bool VertexShader::load_input(nir_intrinsic_instr *intr) return false; } - -int VertexShader::do_allocate_reserved_registers() +int +VertexShader::do_allocate_reserved_registers() { if (m_sv_values.test(es_vertexid)) { m_vertex_id = value_factory().allocate_pinned_register(0, 0); @@ -532,12 +554,14 @@ int VertexShader::do_allocate_reserved_registers() return m_last_vertex_atribute_register + 1; } -bool VertexShader::store_output(nir_intrinsic_instr *intr) +bool +VertexShader::store_output(nir_intrinsic_instr *intr) { return m_export_stage->store_output(*intr); } -bool VertexShader::process_stage_intrinsic(nir_intrinsic_instr *intr) +bool +VertexShader::process_stage_intrinsic(nir_intrinsic_instr *intr) { switch (intr->intrinsic) { case nir_intrinsic_load_vertex_id: @@ -553,44 +577,48 @@ bool VertexShader::process_stage_intrinsic(nir_intrinsic_instr *intr) } } -void VertexShader::do_finalize() +void +VertexShader::do_finalize() { m_export_stage->finalize(); } -bool VertexShader::read_prop(std::istream& is) +bool +VertexShader::read_prop(std::istream& is) { (void)is; return false; } -void VertexShader::do_print_properties(std::ostream& os) const +void +VertexShader::do_print_properties(std::ostream& os) const { (void)os; } VertexExportForGS::VertexExportForGS(VertexStageShader *parent, - const r600_shader *gs_shader): - VertexExportStage(parent), - m_gs_shader(gs_shader) + const r600_shader *gs_shader): + VertexExportStage(parent), + m_gs_shader(gs_shader) { - } -bool VertexExportForGS::do_store_output(const store_loc &store_info, nir_intrinsic_instr& instr) +bool +VertexExportForGS::do_store_output(const store_loc& store_info, + nir_intrinsic_instr& instr) { int ring_offset = -1; auto out_io = m_parent->output(store_info.driver_location); sfn_log << SfnLog::io << "check output " << store_info.driver_location - << " name=" << out_io.name()<< " sid=" << out_io.sid() << "\n"; + << " name=" << out_io.name() << " sid=" << out_io.sid() << "\n"; for (unsigned k = 0; k < m_gs_shader->ninput; ++k) { auto& in_io = m_gs_shader->input[k]; - sfn_log << SfnLog::io << " against " << k << " name=" << in_io.name<< " sid=" << in_io.sid << "\n"; + sfn_log << SfnLog::io << " against " << k << " name=" << in_io.name + << " sid=" << in_io.sid << "\n"; - if (in_io.name == out_io.name() && - in_io.sid == out_io.sid()) { + if (in_io.name == out_io.name() && in_io.sid == out_io.sid()) { ring_offset = in_io.ring_offset; break; } @@ -603,21 +631,22 @@ bool VertexExportForGS::do_store_output(const store_loc &store_info, nir_intrins } if (ring_offset == -1) { - sfn_log << SfnLog::err << "VS defines output at " - << store_info.driver_location << "name=" << out_io.name() - << " sid=" << out_io.sid() << " that is not consumed as GS input\n"; + sfn_log << SfnLog::err << "VS defines output at " << store_info.driver_location + << "name=" << out_io.name() << " sid=" << out_io.sid() + << " that is not consumed as GS input\n"; return true; } - RegisterVec4::Swizzle src_swz = {7,7,7,7}; + RegisterVec4::Swizzle src_swz = {7, 7, 7, 7}; for (int i = 0; i < 4; ++i) src_swz[i] = i < instr.num_components ? i : 7; auto value = m_parent->value_factory().temp_vec4(pin_chgr, src_swz); AluInstr *ir = nullptr; - for (unsigned int i = 0; i < instr.num_components ; ++i) { - ir = new AluInstr(op1_mov, value[i], + for (unsigned int i = 0; i < instr.num_components; ++i) { + ir = new AluInstr(op1_mov, + value[i], m_parent->value_factory().src(instr.src[store_info.data_loc], i), AluInstr::write); m_parent->emit_instruction(ir); @@ -625,8 +654,8 @@ bool VertexExportForGS::do_store_output(const store_loc &store_info, nir_intrins if (ir) ir->set_alu_flag(alu_last_instr); - m_parent->emit_instruction(new MemRingOutInstr(cf_mem_ring, MemRingOutInstr::mem_write, value, - ring_offset >> 2, 4, nullptr)); + m_parent->emit_instruction(new MemRingOutInstr( + cf_mem_ring, MemRingOutInstr::mem_write, value, ring_offset >> 2, 4, nullptr)); if (store_info.location == VARYING_SLOT_CLIP_DIST0 || store_info.location == VARYING_SLOT_CLIP_DIST1) @@ -635,12 +664,13 @@ bool VertexExportForGS::do_store_output(const store_loc &store_info, nir_intrins return true; } -void VertexExportForGS::finalize() +void +VertexExportForGS::finalize() { - } -void VertexExportForGS::get_shader_info(r600_shader *sh_info) const +void +VertexExportForGS::get_shader_info(r600_shader *sh_info) const { sh_info->vs_out_viewport = m_vs_out_viewport; sh_info->vs_out_misc_write = m_vs_out_misc_write; @@ -648,30 +678,28 @@ void VertexExportForGS::get_shader_info(r600_shader *sh_info) const } VertexExportForTCS::VertexExportForTCS(VertexStageShader *parent): - VertexExportStage(parent) + VertexExportStage(parent) { - } -void VertexExportForTCS::finalize() +void +VertexExportForTCS::finalize() { - } -void VertexExportForTCS::get_shader_info(r600_shader *sh_info) const +void +VertexExportForTCS::get_shader_info(r600_shader *sh_info) const { sh_info->vs_as_ls = 1; } - -bool VertexExportForTCS::do_store_output(const store_loc &store_info, nir_intrinsic_instr& intr) +bool +VertexExportForTCS::do_store_output(const store_loc& store_info, + nir_intrinsic_instr& intr) { (void)store_info; (void)intr; return true; } - - -} - +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_shader_vs.h b/src/gallium/drivers/r600/sfn/sfn_shader_vs.h index 571f32a6a33..7b01f37e242 100644 --- a/src/gallium/drivers/r600/sfn/sfn_shader_vs.h +++ b/src/gallium/drivers/r600/sfn/sfn_shader_vs.h @@ -29,16 +29,15 @@ #include "sfn_shader.h" - - namespace r600 { class VertexStageShader : public Shader { protected: using Shader::Shader; + public: - PRegister primitive_id() const { return m_primitive_id;} - void set_primitive_id(PRegister prim_id) { m_primitive_id = prim_id;} + PRegister primitive_id() const { return m_primitive_id; } + void set_primitive_id(PRegister prim_id) { m_primitive_id = prim_id; } void combine_enabled_stream_buffers_mask(uint32_t mask); uint32_t enabled_stream_buffers_mask() const override; @@ -50,7 +49,6 @@ private: class VertexExportStage : public Allocate { public: - VertexExportStage(VertexStageShader *parent); bool store_output(nir_intrinsic_instr& intr); @@ -67,7 +65,8 @@ protected: int data_loc; }; - virtual bool do_store_output(const store_loc &store_info, nir_intrinsic_instr& intr) = 0; + virtual bool do_store_output(const store_loc& store_info, + nir_intrinsic_instr& intr) = 0; VertexStageShader *m_parent; @@ -78,8 +77,8 @@ class VertexExportForFs : public VertexExportStage { friend VertexExportStage; public: - - VertexExportForFs(VertexStageShader *parent, const pipe_stream_output_info *so_info, + VertexExportForFs(VertexStageShader *parent, + const pipe_stream_output_info *so_info, const r600_shader_key& key); void finalize() override; @@ -87,14 +86,14 @@ public: void get_shader_info(r600_shader *sh_info) const override; private: + bool do_store_output(const store_loc& store_info, nir_intrinsic_instr& intr) override; - bool do_store_output(const store_loc &store_info, nir_intrinsic_instr& intr) override; - - bool emit_varying_pos(const store_loc &store_info, nir_intrinsic_instr& intr, + bool emit_varying_pos(const store_loc& store_info, + nir_intrinsic_instr& intr, std::array *swizzle_override = nullptr); - bool emit_varying_param(const store_loc &store_info, nir_intrinsic_instr& intr); + bool emit_varying_param(const store_loc& store_info, nir_intrinsic_instr& intr); - bool emit_clip_vertices(const store_loc &store_info, const nir_intrinsic_instr &instr); + bool emit_clip_vertices(const store_loc& store_info, const nir_intrinsic_instr& instr); bool emit_stream(int stream); @@ -118,16 +117,18 @@ private: bool m_out_point_size{false}; RegisterVec4 m_clip_vertex; - const pipe_stream_output_info *m_so_info {nullptr}; + const pipe_stream_output_info *m_so_info{nullptr}; template - using unordered_map_alloc = std::unordered_map, std::equal_to, - Allocator>>; + using unordered_map_alloc = std::unordered_map, + std::equal_to, + Allocator>>; unordered_map_alloc m_output_registers; }; - class VertexExportForGS : public VertexExportStage { public: VertexExportForGS(VertexStageShader *parent, const r600_shader *gs_shader); @@ -136,7 +137,7 @@ public: void get_shader_info(r600_shader *sh_info) const override; private: - bool do_store_output(const store_loc &store_info, nir_intrinsic_instr& intr) override; + bool do_store_output(const store_loc& store_info, nir_intrinsic_instr& intr) override; unsigned m_num_clip_dist{0}; bool m_vs_out_viewport{false}; bool m_vs_out_misc_write{false}; @@ -149,13 +150,16 @@ public: VertexExportForTCS(VertexStageShader *parent); void finalize() override; void get_shader_info(r600_shader *sh_info) const override; + private: - bool do_store_output(const store_loc &store_info, nir_intrinsic_instr& intr) override; + bool do_store_output(const store_loc& store_info, nir_intrinsic_instr& intr) override; }; class VertexShader : public VertexStageShader { public: - VertexShader(const pipe_stream_output_info *so_info, r600_shader *gs_shader, r600_shader_key& key); + VertexShader(const pipe_stream_output_info *so_info, + r600_shader *gs_shader, + r600_shader_key& key); bool load_input(nir_intrinsic_instr *intr) override; bool store_output(nir_intrinsic_instr *intr) override; @@ -173,14 +177,14 @@ private: void do_print_properties(std::ostream& os) const override; void do_get_shader_info(r600_shader *sh_info) override; - VertexExportStage *m_export_stage {nullptr}; - int m_last_vertex_atribute_register {0}; + VertexExportStage *m_export_stage{nullptr}; + int m_last_vertex_atribute_register{0}; PRegister m_vertex_id{nullptr}; PRegister m_instance_id{nullptr}; PRegister m_rel_vertex_id{nullptr}; bool m_vs_as_gs_a; }; -} +} // namespace r600 #endif diff --git a/src/gallium/drivers/r600/sfn/sfn_valuefactory.cpp b/src/gallium/drivers/r600/sfn/sfn_valuefactory.cpp index 1b2aae5f38f..e2bee5a7265 100644 --- a/src/gallium/drivers/r600/sfn/sfn_valuefactory.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_valuefactory.cpp @@ -24,57 +24,60 @@ * USE OR OTHER DEALINGS IN THE SOFTWARE. */ - #include "sfn_valuefactory.h" -#include "sfn_instr.h" -#include "sfn_debug.h" #include "gallium/drivers/r600/r600_shader.h" +#include "sfn_debug.h" +#include "sfn_instr.h" -#include -#include -#include #include +#include +#include +#include namespace r600 { using std::istringstream; using std::string; - ValueFactory::ValueFactory(): - m_next_register_index(VirtualValue::virtual_register_base), - m_nowrite_idx(0) + m_next_register_index(VirtualValue::virtual_register_base), + m_nowrite_idx(0) { } -void ValueFactory::set_virtual_register_base(int base) +void +ValueFactory::set_virtual_register_base(int base) { m_next_register_index = base; } -bool ValueFactory::allocate_registers(const exec_list *registers) +bool +ValueFactory::allocate_registers(const exec_list *registers) { bool has_arrays = false; - struct array_entry { + struct array_entry { unsigned index; unsigned length; unsigned ncomponents; - bool operator ()(const array_entry& a, const array_entry& b) const { - return a.length < b.length || (a.length == b.length && a.ncomponents > b.ncomponents); + bool operator()(const array_entry& a, const array_entry& b) const + { + return a.length < b.length || + (a.length == b.length && a.ncomponents > b.ncomponents); } }; - using array_list = std::priority_queue, - array_entry>; + using array_list = + std::priority_queue, array_entry>; array_list arrays; - - foreach_list_typed(nir_register, reg, node, registers) { + foreach_list_typed(nir_register, reg, node, registers) + { if (reg->num_array_elems) { - array_entry ae = {reg->index, reg->num_array_elems, reg->bit_size / 32 * reg->num_components}; + array_entry ae = { + reg->index, reg->num_array_elems, reg->bit_size / 32 * reg->num_components}; arrays.push(ae); has_arrays = true; } @@ -88,11 +91,10 @@ bool ValueFactory::allocate_registers(const exec_list *registers) auto a = arrays.top(); arrays.pop(); - /* This is a bit hackish, return an id that encodes the array merge. To make sure - * that the mapping doesn't go wrong we have to make sure the arrays is longer than - * the number of instances in this arrays slot */ - if (a.ncomponents + ncomponents > 4 || - a.length > length) { + /* This is a bit hackish, return an id that encodes the array merge. To + * make sure that the mapping doesn't go wrong we have to make sure the + * arrays is longer than the number of instances in this arrays slot */ + if (a.ncomponents + ncomponents > 4 || a.length > length) { sel = m_next_register_index; ncomponents = 0; length = 0; @@ -102,35 +104,40 @@ bool ValueFactory::allocate_registers(const exec_list *registers) m_next_register_index += a.length; uint32_t frac = ncomponents; - auto array = new LocalArray( sel, a.ncomponents, a.length, frac); + auto array = new LocalArray(sel, a.ncomponents, a.length, frac); for (unsigned i = 0; i < a.ncomponents; ++i) { RegisterKey key(a.index, i, vp_array); m_channel_counts.inc_count(i); m_registers[key] = array; - sfn_log << SfnLog::reg << __func__ << ": Allocate array " << key << ":" << *array << "\n"; + sfn_log << SfnLog::reg << __func__ << ": Allocate array " << key << ":" << *array + << "\n"; } ncomponents += a.ncomponents; length = a.length; } - foreach_list_typed(nir_register, reg, node, registers) { + foreach_list_typed(nir_register, reg, node, registers) + { if (!reg->num_array_elems) { uint32_t sel = m_next_register_index++; unsigned num_components = reg->num_components * reg->bit_size / 32; for (auto chan = 0u; chan < num_components; ++chan) { RegisterKey key(reg->index, chan, vp_register); m_channel_counts.inc_count(chan); - m_registers[key] = new Register( sel, chan, num_components > 1 ? pin_none : pin_free); - sfn_log << SfnLog::reg << "allocate register " << key << ":" << *m_registers[key] << "\n"; + m_registers[key] = + new Register(sel, chan, num_components > 1 ? pin_none : pin_free); + sfn_log << SfnLog::reg << "allocate register " << key << ":" + << *m_registers[key] << "\n"; } } } return has_arrays; } -PRegister ValueFactory::allocate_pinned_register(int sel, int chan) +PRegister +ValueFactory::allocate_pinned_register(int sel, int chan) { if (m_next_register_index <= sel) m_next_register_index = sel + 1; @@ -140,19 +147,20 @@ PRegister ValueFactory::allocate_pinned_register(int sel, int chan) return reg; } -RegisterVec4 ValueFactory::allocate_pinned_vec4(int sel, bool is_ssa) +RegisterVec4 +ValueFactory::allocate_pinned_vec4(int sel, bool is_ssa) { if (m_next_register_index <= sel) m_next_register_index = sel + 1; - RegisterVec4 retval(sel, is_ssa, {0,1,2,3}, pin_fully); + RegisterVec4 retval(sel, is_ssa, {0, 1, 2, 3}, pin_fully); for (int i = 0; i < 4; ++i) m_pinned_registers.push_back(retval[i]); return retval; } - -void ValueFactory::inject_value(const nir_dest& dest, int chan, PVirtualValue value) +void +ValueFactory::inject_value(const nir_dest& dest, int chan, PVirtualValue value) { assert(dest.is_ssa); RegisterKey key(dest.ssa.index, chan, vp_ssa); @@ -161,25 +169,30 @@ void ValueFactory::inject_value(const nir_dest& dest, int chan, PVirtualValue va m_values[key] = value; } -PRegister ValueFactory::dest(const nir_alu_dest& dst, int chan, - Pin pin_channel, uint8_t chan_mask) +PRegister +ValueFactory::dest(const nir_alu_dest& dst, int chan, Pin pin_channel, uint8_t chan_mask) { sfn_log << SfnLog::reg << "Search (ref) " << &dst << "\n"; return dest(dst.dest, chan, pin_channel, chan_mask); } -class TranslateRegister: public RegisterVisitor { +class TranslateRegister : public RegisterVisitor { public: - void visit(VirtualValue& value) {(void)value;} - void visit(Register& value) {(void)value;}; - void visit(LocalArray& value) { m_value = value.element(m_offset, m_addr, m_chan);} - void visit(LocalArrayValue& value) {(void)value;} - void visit(UniformValue& value) {(void)value;} - void visit(LiteralConstant& value) {(void)value;} - void visit(InlineConstant& value) {(void)value;} + void visit(VirtualValue& value) { (void)value; } + void visit(Register& value) { (void)value; }; + void visit(LocalArray& value) { m_value = value.element(m_offset, m_addr, m_chan); } + void visit(LocalArrayValue& value) { (void)value; } + void visit(UniformValue& value) { (void)value; } + void visit(LiteralConstant& value) { (void)value; } + void visit(InlineConstant& value) { (void)value; } - TranslateRegister(int offset, PVirtualValue addr, int chan): m_addr(addr), - m_value(nullptr), m_offset(offset), m_chan(chan) {} + TranslateRegister(int offset, PVirtualValue addr, int chan): + m_addr(addr), + m_value(nullptr), + m_offset(offset), + m_chan(chan) + { + } PVirtualValue m_addr; PRegister m_value; @@ -187,8 +200,11 @@ public: int m_chan; }; -PRegister ValueFactory::resolve_array(nir_register *reg, nir_src *indirect, - int base_offset, int chan) +PRegister +ValueFactory::resolve_array(nir_register *reg, + nir_src *indirect, + int base_offset, + int chan) { PVirtualValue addr = nullptr; auto type = reg->num_array_elems ? vp_array : vp_register; @@ -214,50 +230,52 @@ PRegister ValueFactory::resolve_array(nir_register *reg, nir_src *indirect, } } -PRegister ValueFactory::dest(const nir_dest& dst, int chan, Pin pin_channel, - uint8_t chan_mask) +PRegister +ValueFactory::dest(const nir_dest& dst, int chan, Pin pin_channel, uint8_t chan_mask) { if (dst.is_ssa) { return dest(dst.ssa, chan, pin_channel, chan_mask); } else { - return resolve_array(dst.reg.reg, dst.reg.indirect, - dst.reg.base_offset, chan); + return resolve_array(dst.reg.reg, dst.reg.indirect, dst.reg.base_offset, chan); } } -void ValueFactory::allocate_const(nir_load_const_instr *load_const) +void +ValueFactory::allocate_const(nir_load_const_instr *load_const) { assert(load_const->def.bit_size == 32); for (int i = 0; i < load_const->def.num_components; ++i) { RegisterKey key(load_const->def.index, i, vp_ssa); m_values[key] = literal(load_const->value[i].i32); - sfn_log << SfnLog::reg << "Add const with key " << key << " as " << m_values[key] << "\n"; + sfn_log << SfnLog::reg << "Add const with key " << key << " as " << m_values[key] + << "\n"; } } -PVirtualValue ValueFactory::uniform(nir_intrinsic_instr *load_uniform, int chan) +PVirtualValue +ValueFactory::uniform(nir_intrinsic_instr *load_uniform, int chan) { auto literal = nir_src_as_const_value(load_uniform->src[0]); assert(literal); - int index = nir_intrinsic_base(load_uniform) + + literal->u32 + 512; + int index = nir_intrinsic_base(load_uniform) + +literal->u32 + 512; return uniform(index, chan, 0); } -PVirtualValue ValueFactory::uniform(uint32_t index, int chan, int kcache) +PVirtualValue +ValueFactory::uniform(uint32_t index, int chan, int kcache) { - return new UniformValue( index, chan, kcache); + return new UniformValue(index, chan, kcache); } -PRegister ValueFactory::temp_register(int pinned_channel, bool is_ssa) +PRegister +ValueFactory::temp_register(int pinned_channel, bool is_ssa) { int sel = m_next_register_index++; - int chan = (pinned_channel >= 0) ? - pinned_channel : m_channel_counts.least_used(0xf); + int chan = (pinned_channel >= 0) ? pinned_channel : m_channel_counts.least_used(0xf); - auto reg = new Register( sel, chan, - pinned_channel >= 0 ? pin_chan : pin_free); + auto reg = new Register(sel, chan, pinned_channel >= 0 ? pin_chan : pin_free); m_channel_counts.inc_count(chan); reg->set_is_ssa(is_ssa); @@ -265,7 +283,8 @@ PRegister ValueFactory::temp_register(int pinned_channel, bool is_ssa) return reg; } -RegisterVec4 ValueFactory::temp_vec4(Pin pin, const RegisterVec4::Swizzle &swizzle) +RegisterVec4 +ValueFactory::temp_vec4(Pin pin, const RegisterVec4::Swizzle& swizzle) { int sel = m_next_register_index++; @@ -275,14 +294,15 @@ RegisterVec4 ValueFactory::temp_vec4(Pin pin, const RegisterVec4::Swizzle &swizz PRegister vec4[4]; for (int i = 0; i < 4; ++i) { - vec4[i] = new Register( sel, swizzle[i], pin); + vec4[i] = new Register(sel, swizzle[i], pin); vec4[i]->set_is_ssa(true); m_registers[RegisterKey(sel, swizzle[i], vp_temp)] = vec4[i]; } return RegisterVec4(vec4[0], vec4[1], vec4[2], vec4[3], pin); } -RegisterVec4 ValueFactory::dest_vec4(const nir_dest& dst, Pin pin) +RegisterVec4 +ValueFactory::dest_vec4(const nir_dest& dst, Pin pin) { if (pin != pin_group && pin != pin_chgr) pin = pin_chan; @@ -312,22 +332,26 @@ RegisterVec4 ValueFactory::dest_vec4(const nir_dest& dst, Pin pin) unreachable("unsupported"); } -PVirtualValue ValueFactory::src(const nir_alu_src& alu_src, int chan) +PVirtualValue +ValueFactory::src(const nir_alu_src& alu_src, int chan) { return src(alu_src.src, alu_src.swizzle[chan]); } -PVirtualValue ValueFactory::src64(const nir_alu_src& alu_src, int chan, int comp) +PVirtualValue +ValueFactory::src64(const nir_alu_src& alu_src, int chan, int comp) { return src(alu_src.src, 2 * alu_src.swizzle[chan] + comp); } -PVirtualValue ValueFactory::src(const nir_src& src, int chan) +PVirtualValue +ValueFactory::src(const nir_src& src, int chan) { sfn_log << SfnLog::reg << "search (ref) " << (void *)&src << "\n"; if (src.is_ssa) { - sfn_log << SfnLog::reg << "search ssa " << src.ssa->index << " c:" << chan << " got "; + sfn_log << SfnLog::reg << "search ssa " << src.ssa->index << " c:" << chan + << " got "; auto val = ssa_src(*src.ssa, chan); sfn_log << *val << "\n"; return val; @@ -337,20 +361,21 @@ PVirtualValue ValueFactory::src(const nir_src& src, int chan) } } -PVirtualValue ValueFactory::src(const nir_tex_src& tex_src, int chan) +PVirtualValue +ValueFactory::src(const nir_tex_src& tex_src, int chan) { return src(tex_src.src, chan); } -PRegister ValueFactory::dummy_dest(unsigned chan) +PRegister +ValueFactory::dummy_dest(unsigned chan) { assert(chan < 4); return m_dummy_dest_pinned[chan]; } PRegister -ValueFactory::dest(const nir_ssa_def& ssa, int chan, Pin pin_channel, - uint8_t chan_mask) +ValueFactory::dest(const nir_ssa_def& ssa, int chan, Pin pin_channel, uint8_t chan_mask) { RegisterKey key(ssa.index, chan, vp_ssa); @@ -366,15 +391,15 @@ ValueFactory::dest(const nir_ssa_def& ssa, int chan, Pin pin_channel, sel = isel->second; else { sel = m_next_register_index++; - sfn_log << SfnLog::reg << "Assign " << sel << " to index " - << ssa.index << " in " << &m_ssa_index_to_sel << "\n"; + sfn_log << SfnLog::reg << "Assign " << sel << " to index " << ssa.index << " in " + << &m_ssa_index_to_sel << "\n"; m_ssa_index_to_sel[ssa.index] = sel; } if (pin_channel == pin_free) chan = m_channel_counts.least_used(chan_mask); - auto vreg = new Register( sel, chan, pin_channel); + auto vreg = new Register(sel, chan, pin_channel); m_channel_counts.inc_count(chan); vreg->set_is_ssa(true); m_registers[key] = vreg; @@ -382,22 +407,26 @@ ValueFactory::dest(const nir_ssa_def& ssa, int chan, Pin pin_channel, return vreg; } -PVirtualValue ValueFactory::zero() +PVirtualValue +ValueFactory::zero() { return inline_const(ALU_SRC_0, 0); } -PVirtualValue ValueFactory::one() +PVirtualValue +ValueFactory::one() { return inline_const(ALU_SRC_1, 0); } -PVirtualValue ValueFactory::one_i() +PVirtualValue +ValueFactory::one_i() { return inline_const(ALU_SRC_1_INT, 0); } -PRegister ValueFactory::undef(int index, int chan) +PRegister +ValueFactory::undef(int index, int chan) { RegisterKey key(index, chan, vp_ssa); PRegister reg = new Register(m_next_register_index++, 0, pin_free); @@ -424,45 +453,48 @@ ValueFactory::ssa_src(const nir_ssa_def& ssa, int chan) unreachable("Source values should always exist"); } -PRegister ValueFactory::local_register(const nir_reg_dest& dst, int chan) +PRegister +ValueFactory::local_register(const nir_reg_dest& dst, int chan) { - return resolve_array(dst.reg, dst.indirect, - dst.base_offset, chan); + return resolve_array(dst.reg, dst.indirect, dst.base_offset, chan); } -PRegister ValueFactory::local_register(const nir_reg_src& src, int chan) +PRegister +ValueFactory::local_register(const nir_reg_src& src, int chan) { - return resolve_array(src.reg, src.indirect, - src.base_offset, chan); + return resolve_array(src.reg, src.indirect, src.base_offset, chan); } -PVirtualValue ValueFactory::literal(uint32_t value) +PVirtualValue +ValueFactory::literal(uint32_t value) { auto iv = m_literal_values.find(value); if (iv != m_literal_values.end()) return iv->second; - auto v = new LiteralConstant( value); + auto v = new LiteralConstant(value); m_literal_values[value] = v; return v; } -PInlineConstant ValueFactory::inline_const(AluInlineConstants sel, int chan) +PInlineConstant +ValueFactory::inline_const(AluInlineConstants sel, int chan) { int hash = (sel << 3) | chan; auto iv = m_inline_constants.find(hash); - if (iv != m_inline_constants.end()) + if (iv != m_inline_constants.end()) return iv->second; - auto v = new InlineConstant( sel, chan); + auto v = new InlineConstant(sel, chan); m_inline_constants[hash] = v; return v; } - std::vector> ValueFactory::src_vec(const nir_src& source, int components) +std::vector> +ValueFactory::src_vec(const nir_src& source, int components) { std::vector> retval; retval.reserve(components); - for (int i = 0; i < components; ++i) + for (int i = 0; i < components; ++i) retval.push_back(src(source, i)); return retval; } @@ -472,12 +504,13 @@ ValueFactory::dest_vec(const nir_dest& dst, int num_components) { std::vector> retval; retval.reserve(num_components); - for (int i = 0; i < num_components; ++i) + for (int i = 0; i < num_components; ++i) retval.push_back(dest(dst, i, num_components > 1 ? pin_chan : pin_free)); return retval; } -RegisterVec4 ValueFactory::src_vec4(const nir_src& source, Pin pin, const RegisterVec4::Swizzle& swz) +RegisterVec4 +ValueFactory::src_vec4(const nir_src& source, Pin pin, const RegisterVec4::Swizzle& swz) { auto sx = swz[0] < 4 ? src(source, swz[0])->as_register() : nullptr; auto sy = swz[1] < 4 ? src(source, swz[1])->as_register() : nullptr; @@ -490,10 +523,14 @@ RegisterVec4 ValueFactory::src_vec4(const nir_src& source, Pin pin, const Regist if (sel < 0) unreachable("source vector without valid components"); - if (!sx) sx = new Register(sel, 7, pin); - if (!sy) sy = new Register(sel, 7, pin); - if (!sz) sz = new Register(sel, 7, pin); - if (!sw) sw = new Register(sel, 7, pin); + if (!sx) + sx = new Register(sel, 7, pin); + if (!sy) + sy = new Register(sel, 7, pin); + if (!sz) + sz = new Register(sel, 7, pin); + if (!sw) + sw = new Register(sel, 7, pin); return RegisterVec4(sx, sy, sz, sw, pin); } @@ -516,16 +553,24 @@ pin_from_string(const std::string& pinstr) return pin_none; } -static int chan_from_char(char chan) +static int +chan_from_char(char chan) { switch (chan) { - case 'x' : return 0; - case 'y' : return 1; - case 'z' : return 2; - case 'w' : return 3; - case '0' : return 4; - case '1' : return 5; - case '_' : return 7; + case 'x': + return 0; + case 'y': + return 1; + case 'z': + return 2; + case 'w': + return 3; + case '0': + return 4; + case '1': + return 5; + case '_': + return 7; } unreachable("Unknown swizzle char"); } @@ -559,10 +604,7 @@ split_register_string(const string& s, continue; } else if (s[i] == ']') { if (type != 3) - std::cerr << "s=" << s - << ": type=" << type - << ": i=" << i - << "\n"; + std::cerr << "s=" << s << ": type=" << type << ": i=" << i << "\n"; assert(type == 3); type = 4; @@ -570,10 +612,18 @@ split_register_string(const string& s, } switch (type) { - case 0: index_str.append(1, s[i]); break; - case 1: swizzle_str.append(1, s[i]); break; - case 2: pin_str.append(1, s[i]); break; - case 3: size_str.append(1, s[i]); break; + case 0: + index_str.append(1, s[i]); + break; + case 1: + swizzle_str.append(1, s[i]); + break; + case 2: + pin_str.append(1, s[i]); + break; + case 3: + size_str.append(1, s[i]); + break; default: unreachable("Malformed Array allocation string"); } @@ -581,7 +631,8 @@ split_register_string(const string& s, return true; } -PRegister ValueFactory::dest_from_string(const std::string& s) +PRegister +ValueFactory::dest_from_string(const std::string& s) { assert(s.length() >= 4); @@ -592,15 +643,14 @@ PRegister ValueFactory::dest_from_string(const std::string& s) string swizzle_str; string pin_str; - split_register_string(s, index_str, size_str, - swizzle_str, pin_str); + split_register_string(s, index_str, size_str, swizzle_str, pin_str); int sel = 0; if (s[0] == '_') { - /* Since these instructions still may use or switch to a different channel - * we have to create a new instance for each occurance */ + /* Since these instructions still may use or switch to a different + * channel we have to create a new instance for each occurance */ sel = std::numeric_limits::max() - m_nowrite_idx++; - } else { + } else { std::istringstream n(index_str); n >> sel; } @@ -610,29 +660,37 @@ PRegister ValueFactory::dest_from_string(const std::string& s) EValuePool pool = vp_temp; switch (s[0]) { - case 'A': pool = vp_array; break; - case 'R': pool = vp_register; break; - case '_': pool = vp_ignore; break; - case 'S': pool = vp_ssa; break; + case 'A': + pool = vp_array; + break; + case 'R': + pool = vp_register; + break; + case '_': + pool = vp_ignore; + break; + case 'S': + pool = vp_ssa; + break; default: - unreachable("Unknown value type"); + unreachable("Unknown value type"); } bool is_ssa = s[0] == 'S'; - RegisterKey key (sel, chan, pool); + RegisterKey key(sel, chan, pool); sfn_log << SfnLog::reg << "Search register with key " << key << "\n"; auto ireg = m_registers.find(key); - if ( ireg == m_registers.end()) { + if (ireg == m_registers.end()) { auto reg = new Register(sel, chan, p); reg->set_is_ssa(is_ssa); if (p == pin_fully) reg->pin_live_range(true); m_registers[key] = reg; return reg; - } else if (pool == vp_ignore) { + } else if (pool == vp_ignore) { assert(ireg->second->sel() == std::numeric_limits::max()); return ireg->second; } else { @@ -655,16 +713,22 @@ PRegister ValueFactory::dest_from_string(const std::string& s) } } -PVirtualValue ValueFactory::src_from_string(const std::string& s) +PVirtualValue +ValueFactory::src_from_string(const std::string& s) { switch (s[0]) { case 'A': case 'S': - case 'R': break; - case 'L': return LiteralConstant::from_string(s); - case 'K': return UniformValue::from_string(s); - case 'P': return InlineConstant::param_from_string(s); - case 'I': return InlineConstant::from_string(s); + case 'R': + break; + case 'L': + return LiteralConstant::from_string(s); + case 'K': + return UniformValue::from_string(s); + case 'P': + return InlineConstant::param_from_string(s); + case 'I': + return InlineConstant::from_string(s); default: std::cerr << "'" << s << "'"; @@ -678,8 +742,7 @@ PVirtualValue ValueFactory::src_from_string(const std::string& s) string swizzle_str; string pin_str; - split_register_string(s, index_str, size_str, - swizzle_str, pin_str); + split_register_string(s, index_str, size_str, swizzle_str, pin_str); int sel = 0; if (s[0] == '_') { @@ -694,15 +757,23 @@ PVirtualValue ValueFactory::src_from_string(const std::string& s) EValuePool pool = vp_temp; switch (s[0]) { - case 'A': pool = vp_array; break; - case 'R': pool = vp_register; break; - case '_': pool = vp_ignore; break; - case 'S': pool = vp_ssa; break; + case 'A': + pool = vp_array; + break; + case 'R': + pool = vp_register; + break; + case '_': + pool = vp_ignore; + break; + case 'S': + pool = vp_ssa; + break; default: - unreachable("Unknown value type"); + unreachable("Unknown value type"); } - RegisterKey key (sel, chan, pool); + RegisterKey key(sel, chan, pool); auto ireg = m_registers.find(key); if (ireg != m_registers.end()) { @@ -720,21 +791,22 @@ PVirtualValue ValueFactory::src_from_string(const std::string& s) } else { return ireg->second; } - } else { + } else { if (sel != std::numeric_limits::max()) { std::cerr << "register " << key << "not found \n"; unreachable("Source register should exist"); } else { - auto reg = new Register( sel, chan, p); + auto reg = new Register(sel, chan, p); m_registers[key] = reg; return reg; } } } -RegisterVec4 ValueFactory::dest_vec4_from_string(const std::string& s, - RegisterVec4::Swizzle& swz, - Pin pin) +RegisterVec4 +ValueFactory::dest_vec4_from_string(const std::string& s, + RegisterVec4::Swizzle& swz, + Pin pin) { bool is_ssa = false; int sel = sel_and_szw_from_string(s, swz, is_ssa); @@ -746,13 +818,13 @@ RegisterVec4 ValueFactory::dest_vec4_from_string(const std::string& s, if (swz[i] > 3) pool = vp_ignore; - RegisterKey key(sel, i, pool); + RegisterKey key(sel, i, pool); auto ireg = m_registers.find(key); if (ireg != m_registers.end()) { v[i] = ireg->second; assert(!is_ssa || pool == vp_ignore); } else { - v[i] = new Register( sel, i, pin); + v[i] = new Register(sel, i, pin); v[i]->set_is_ssa(is_ssa); m_registers[key] = v[i]; } @@ -760,7 +832,8 @@ RegisterVec4 ValueFactory::dest_vec4_from_string(const std::string& s, return RegisterVec4(v[0], v[1], v[2], v[3], pin); } -RegisterVec4 ValueFactory::src_vec4_from_string(const std::string& s) +RegisterVec4 +ValueFactory::src_vec4_from_string(const std::string& s) { RegisterVec4::Swizzle swz; bool is_ssa = false; @@ -771,7 +844,7 @@ RegisterVec4 ValueFactory::src_vec4_from_string(const std::string& s) PRegister used_reg = nullptr; for (int i = 0; i < 4; ++i) { if (swz[i] < 4) { - RegisterKey key(sel, swz[i], is_ssa ? vp_ssa : vp_register); + RegisterKey key(sel, swz[i], is_ssa ? vp_ssa : vp_register); auto ireg = m_registers.find(key); if (ireg == m_registers.end()) { std::cerr << s << ": Register with key " << key << " not found\n"; @@ -787,7 +860,7 @@ RegisterVec4 ValueFactory::src_vec4_from_string(const std::string& s) for (int i = 0; i < 4; ++i) { if (!v[i]) { - v[i] = new Register( sel, swz[i], pin); + v[i] = new Register(sel, swz[i], pin); v[i]->set_is_ssa(is_ssa); } else { if (v[i]->pin() == pin_none) @@ -797,7 +870,8 @@ RegisterVec4 ValueFactory::src_vec4_from_string(const std::string& s) return RegisterVec4(v[0], v[1], v[2], v[3], pin); } -LocalArray *ValueFactory::array_from_string(const std::string& s) +LocalArray * +ValueFactory::array_from_string(const std::string& s) { assert(s[0] == 'A'); string index_str; @@ -805,7 +879,6 @@ LocalArray *ValueFactory::array_from_string(const std::string& s) string swizzle_str; string pin_str; - int type = 0; for (unsigned i = 1; i < s.length(); ++i) { if (s[i] == '.') { @@ -824,10 +897,18 @@ LocalArray *ValueFactory::array_from_string(const std::string& s) } switch (type) { - case 0: index_str.append(1, s[i]); break; - case 1: swizzle_str.append(1, s[i]); break; - case 2: pin_str.append(1, s[i]); break; - case 3: size_str.append(1, s[i]); break; + case 0: + index_str.append(1, s[i]); + break; + case 1: + swizzle_str.append(1, s[i]); + break; + case 2: + pin_str.append(1, s[i]); + break; + case 3: + size_str.append(1, s[i]); + break; default: unreachable("Malformed Array allocation string"); } @@ -839,9 +920,9 @@ LocalArray *ValueFactory::array_from_string(const std::string& s) if (ncomp > 4 || ncomp <= 0) { std::cerr << "Error reading array from '" << s << ": "; - std::cerr << "index:'" << index_str << "' -> '" << sel - << "' size:'" << size_str << "' -> '" << size - << " swizzle:'" << swizzle_str << "' -> '" << ncomp << "'\n"; + std::cerr << "index:'" << index_str << "' -> '" << sel << "' size:'" << size_str + << "' -> '" << size << " swizzle:'" << swizzle_str << "' -> '" << ncomp + << "'\n"; assert(0); } @@ -850,7 +931,7 @@ LocalArray *ValueFactory::array_from_string(const std::string& s) long frac = first_swz - swz; assert(frac >= 0 && frac <= 4 - ncomp); - auto array = new LocalArray( sel, ncomp, size, frac); + auto array = new LocalArray(sel, ncomp, size, frac); for (int i = 0; i < ncomp; ++i) { RegisterKey key(sel, i + frac, vp_array); @@ -859,7 +940,8 @@ LocalArray *ValueFactory::array_from_string(const std::string& s) return array; } -void LiveRangeMap::append_register(Register *reg) +void +LiveRangeMap::append_register(Register *reg) { sfn_log << SfnLog::merge << __func__ << ": " << *reg << "\n"; @@ -870,15 +952,19 @@ void LiveRangeMap::append_register(Register *reg) ranges.emplace_back(entry); } -std::array LiveRangeMap::sizes() const +std::array +LiveRangeMap::sizes() const { std::array result; - std::transform(m_life_ranges.begin(), m_life_ranges.end(), - result.begin(), [](auto lr) {return lr.size(); }); + std::transform(m_life_ranges.begin(), + m_life_ranges.end(), + result.begin(), + [](auto lr) { return lr.size(); }); return result; } -LiveRangeMap ValueFactory::prepare_live_range_map() +LiveRangeMap +ValueFactory::prepare_live_range_map() { LiveRangeMap result; @@ -889,7 +975,7 @@ LiveRangeMap ValueFactory::prepare_live_range_map() if (key.value.pool == vp_array) { if (key.value.chan == 0) { auto array = static_cast(reg); - for (auto& a : *array) { + for (auto& a : *array) { result.append_register(a); } } @@ -905,18 +991,20 @@ LiveRangeMap ValueFactory::prepare_live_range_map() for (int i = 0; i < 4; ++i) { auto& comp = result.component(i); - std::sort(comp.begin(), comp.end(), + std::sort(comp.begin(), + comp.end(), [](const LiveRangeEntry& lhs, const LiveRangeEntry& rhs) { - return lhs.m_register->sel() < rhs.m_register->sel(); - }); - for(size_t j = 0; j < comp.size(); ++j) + return lhs.m_register->sel() < rhs.m_register->sel(); + }); + for (size_t j = 0; j < comp.size(); ++j) comp[j].m_register->set_index(j); } return result; } -void ValueFactory::clear_pins() +void +ValueFactory::clear_pins() { for (auto [key, reg] : m_registers) reg->set_pin(pin_none); @@ -925,7 +1013,8 @@ void ValueFactory::clear_pins() reg->set_pin(pin_none); } -void ValueFactory::clear() +void +ValueFactory::clear() { m_registers.clear(); m_values.clear(); @@ -934,9 +1023,10 @@ void ValueFactory::clear() m_ssa_index_to_sel.clear(); } -void ValueFactory::get_shader_info(r600_shader *sh_info) +void +ValueFactory::get_shader_info(r600_shader *sh_info) { - std::set arrays; + std::set arrays; for (auto& [key, reg] : m_registers) { if (key.value.pool == vp_array) @@ -951,13 +1041,10 @@ void ValueFactory::get_shader_info(r600_shader *sh_info) for (auto& arr : arrays) { sh_info->arrays->gpr_start = arr->sel(); sh_info->arrays->gpr_count = arr->size(); - sh_info->arrays->comp_mask = - ((1 << arr->nchannels()) - 1) - << arr->frac(); + sh_info->arrays->comp_mask = ((1 << arr->nchannels()) - 1) << arr->frac(); } sh_info->indirect_files |= 1 << TGSI_FILE_TEMPORARY; } } - -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_valuefactory.h b/src/gallium/drivers/r600/sfn/sfn_valuefactory.h index 51fd66e080d..69fd63e1e2b 100644 --- a/src/gallium/drivers/r600/sfn/sfn_valuefactory.h +++ b/src/gallium/drivers/r600/sfn/sfn_valuefactory.h @@ -24,20 +24,17 @@ * USE OR OTHER DEALINGS IN THE SOFTWARE. */ - #ifndef VALUEFACTORY_H #define VALUEFACTORY_H -#include "sfn_virtualvalues.h" -#include "sfn_alu_defines.h" - #include "nir.h" +#include "sfn_alu_defines.h" +#include "sfn_virtualvalues.h" -#include #include +#include #include #include -#include struct r600_shader; @@ -49,7 +46,10 @@ struct LiveRangeEntry { use_unspecified }; - LiveRangeEntry (Register *reg): m_register(reg) {} + LiveRangeEntry(Register *reg): + m_register(reg) + { + } int m_start{-1}; int m_end{-1}; int m_index{-1}; @@ -57,13 +57,15 @@ struct LiveRangeEntry { std::bitset m_use; Register *m_register; - void print(std::ostream& os) const { - os << *m_register << "(" << m_index << ", " << m_color << ") [" - << m_start << ":" << m_end << "]"; - } + void print(std::ostream& os) const + { + os << *m_register << "(" << m_index << ", " << m_color << ") [" << m_start << ":" + << m_end << "]"; + } }; -inline std::ostream& operator << (std::ostream& os, const LiveRangeEntry& lre) +inline std::ostream& +operator<<(std::ostream& os, const LiveRangeEntry& lre) { lre.print(os); return os; @@ -73,14 +75,16 @@ class LiveRangeMap { public: using ChannelLiveRange = std::vector; - LiveRangeEntry& operator()(int index, int chan) { - assert(chan < 4); + LiveRangeEntry& operator()(int index, int chan) + { + assert(chan < 4); return m_life_ranges[chan].at(index); } void append_register(Register *reg); - void set_life_range(const Register& reg, int start, int end) { + void set_life_range(const Register& reg, int start, int end) + { auto& entry = m_life_ranges[reg.chan()].at(reg.index()); entry.m_start = start; entry.m_end = end; @@ -88,24 +92,22 @@ public: std::array sizes() const; - ChannelLiveRange& component(int i) { - return m_life_ranges[i]; - } + ChannelLiveRange& component(int i) { return m_life_ranges[i]; } - const ChannelLiveRange& component(int i) const { - return m_life_ranges[i]; - } + const ChannelLiveRange& component(int i) const { return m_life_ranges[i]; } private: - std::array m_life_ranges; }; -std::ostream& operator << (std::ostream& os, const LiveRangeMap& lrm); +std::ostream& +operator<<(std::ostream& os, const LiveRangeMap& lrm); -bool operator == (const LiveRangeMap& lhs, const LiveRangeMap& rhs); +bool +operator==(const LiveRangeMap& lhs, const LiveRangeMap& rhs); -inline bool operator != (const LiveRangeMap& lhs, const LiveRangeMap& rhs) +inline bool +operator!=(const LiveRangeMap& lhs, const LiveRangeMap& rhs) { return !(lhs == rhs); } @@ -119,55 +121,65 @@ enum EValuePool { }; union RegisterKey { - struct { - uint32_t index; - uint32_t chan : 29; - EValuePool pool : 3; - } value; - uint64_t hash; + struct { + uint32_t index; + uint32_t chan : 29; + EValuePool pool : 3; + } value; + uint64_t hash; - RegisterKey(uint32_t index, uint32_t chan, EValuePool pool) - { - value.index = index; - value.chan = chan; - value.pool = pool; - } + RegisterKey(uint32_t index, uint32_t chan, EValuePool pool) + { + value.index = index; + value.chan = chan; + value.pool = pool; + } - void print(std::ostream& os) const { - os << "(" << value.index - << ", " << value.chan - << ", "; - switch (value.pool) { - case vp_ssa: os << "ssa"; break; - case vp_register: os << "reg"; break; - case vp_temp: os << "temp"; break; - case vp_array : os << "array"; break; - case vp_ignore : break; - } - os << ")"; - } + void print(std::ostream& os) const + { + os << "(" << value.index << ", " << value.chan << ", "; + switch (value.pool) { + case vp_ssa: + os << "ssa"; + break; + case vp_register: + os << "reg"; + break; + case vp_temp: + os << "temp"; + break; + case vp_array: + os << "array"; + break; + case vp_ignore: + break; + } + os << ")"; + } }; - -inline bool operator == (const RegisterKey& lhs, const RegisterKey& rhs) { - return lhs.hash == rhs.hash; +inline bool +operator==(const RegisterKey& lhs, const RegisterKey& rhs) +{ + return lhs.hash == rhs.hash; } -inline std::ostream& operator << (std::ostream& os, const RegisterKey& key) { - key.print(os); - return os; +inline std::ostream& +operator<<(std::ostream& os, const RegisterKey& key) +{ + key.print(os); + return os; } struct register_key_hash { - std::size_t operator () (const RegisterKey& key) const { - return key.hash; - } + std::size_t operator()(const RegisterKey& key) const { return key.hash; } }; class ChannelCounts { public: - void inc_count(int chan) {++m_counts[chan];} - int least_used(uint8_t mask) const { + void inc_count(int chan) { ++m_counts[chan]; } + int least_used(uint8_t mask) const + { int least_used = 0; uint32_t count = m_counts[0]; for (int i = 1; i < 4; ++i) { @@ -180,15 +192,18 @@ public: } return least_used; } - void print(std::ostream& os) const { - os << "CC:" << m_counts[0] << " " << m_counts[1] - << " " << m_counts[2] << " " << m_counts[3]; + void print(std::ostream& os) const + { + os << "CC:" << m_counts[0] << " " << m_counts[1] << " " << m_counts[2] << " " + << m_counts[3]; } + private: - std::array m_counts{0,0,0,0}; + std::array m_counts{0, 0, 0, 0}; }; -inline std::ostream& operator << (std::ostream& os, const ChannelCounts& cc) +inline std::ostream& +operator<<(std::ostream& os, const ChannelCounts& cc) { cc.print(os); return os; @@ -196,107 +211,121 @@ inline std::ostream& operator << (std::ostream& os, const ChannelCounts& cc) class ValueFactory : public Allocate { public: - ValueFactory(); + ValueFactory(); - void clear(); + void clear(); - ValueFactory(const ValueFactory& orig) = delete; - ValueFactory& operator = (const ValueFactory& orig) = delete; + ValueFactory(const ValueFactory& orig) = delete; + ValueFactory& operator=(const ValueFactory& orig) = delete; - void set_virtual_register_base(int base); + void set_virtual_register_base(int base); - bool allocate_registers(const exec_list *registers); - PRegister allocate_pinned_register(int sel, int chan); - RegisterVec4 allocate_pinned_vec4(int sel, bool is_ssa); + bool allocate_registers(const exec_list *registers); + PRegister allocate_pinned_register(int sel, int chan); + RegisterVec4 allocate_pinned_vec4(int sel, bool is_ssa); - void inject_value(const nir_dest& dest, int chan, PVirtualValue value); + void inject_value(const nir_dest& dest, int chan, PVirtualValue value); - std::vector> dest_vec(const nir_dest& dest, int num_components); - std::vector> dest_vector(const nir_src& src, - const std::vector& components); + std::vector> dest_vec(const nir_dest& dest, + int num_components); + std::vector> + dest_vector(const nir_src& src, const std::vector& components); + PRegister + dest(const nir_alu_dest& dest, int chan, Pin pin_channel, uint8_t chan_mask = 0xf); + PRegister + dest(const nir_dest& dest, int chan, Pin pin_channel, uint8_t chan_mask = 0xf); + PRegister + dest(const nir_ssa_def& dest, int chan, Pin pin_channel, uint8_t chan_mask = 0xf); - PRegister dest(const nir_alu_dest& dest, int chan, Pin pin_channel, uint8_t chan_mask = 0xf); - PRegister dest(const nir_dest& dest, int chan, Pin pin_channel, uint8_t chan_mask = 0xf); - PRegister dest(const nir_ssa_def& dest, int chan, Pin pin_channel, uint8_t chan_mask = 0xf); + PRegister dummy_dest(unsigned chan); + PRegister temp_register(int pinned_channel = -1, bool is_ssa = true); + RegisterVec4 temp_vec4(Pin pin, const RegisterVec4::Swizzle& swizzle = {0, 1, 2, 3}); + RegisterVec4 dest_vec4(const nir_dest& dest, Pin pin); - PRegister dummy_dest(unsigned chan); - PRegister temp_register(int pinned_channel = -1, bool is_ssa = true); - RegisterVec4 temp_vec4(Pin pin, const RegisterVec4::Swizzle& swizzle = {0,1,2,3}); - RegisterVec4 dest_vec4(const nir_dest& dest, Pin pin); + RegisterVec4 + src_vec4(const nir_src& src, Pin pin, const RegisterVec4::Swizzle& swz = {0, 1, 2, 3}); - RegisterVec4 src_vec4(const nir_src& src, Pin pin, const RegisterVec4::Swizzle &swz = {0,1,2,3}); + PVirtualValue src(const nir_alu_src& alu_src, int chan); + PVirtualValue src64(const nir_alu_src& alu_src, int chan, int comp); + PVirtualValue src(const nir_src& src, int chan); + PVirtualValue src(const nir_tex_src& tex_src, int chan); + PVirtualValue literal(uint32_t value); + PVirtualValue uniform(nir_intrinsic_instr *load_uniform, int chan); + PVirtualValue uniform(uint32_t index, int chan, int kcache); - PVirtualValue src(const nir_alu_src& alu_src, int chan); - PVirtualValue src64(const nir_alu_src& alu_src, int chan, int comp); - PVirtualValue src(const nir_src& src, int chan); - PVirtualValue src(const nir_tex_src& tex_src, int chan); - PVirtualValue literal(uint32_t value); - PVirtualValue uniform(nir_intrinsic_instr *load_uniform, int chan); - PVirtualValue uniform(uint32_t index, int chan, int kcache); + void allocate_const(nir_load_const_instr *load_const); - void allocate_const(nir_load_const_instr *load_const); + PRegister dest_from_string(const std::string& s); + RegisterVec4 dest_vec4_from_string(const std::string& s, + RegisterVec4::Swizzle& swz, + Pin pin = pin_none); + PVirtualValue src_from_string(const std::string& s); + RegisterVec4 src_vec4_from_string(const std::string& s); - PRegister dest_from_string(const std::string& s); - RegisterVec4 dest_vec4_from_string(const std::string& s, RegisterVec4::Swizzle &swz, - Pin pin = pin_none); - PVirtualValue src_from_string(const std::string& s); - RegisterVec4 src_vec4_from_string(const std::string& s); + LocalArray *array_from_string(const std::string& s); - LocalArray *array_from_string(const std::string& s); + std::vector> src_vec(const nir_src& src, + int components); - std::vector> src_vec(const nir_src& src, int components); + PInlineConstant inline_const(AluInlineConstants sel, int chan); - PInlineConstant inline_const(AluInlineConstants sel, int chan); + void get_shader_info(r600_shader *sh_info); - void get_shader_info(r600_shader *sh_info); + PRegister undef(int index, int chan); + PVirtualValue zero(); + PVirtualValue one(); + PVirtualValue one_i(); - PRegister undef(int index, int chan); - PVirtualValue zero(); - PVirtualValue one(); - PVirtualValue one_i(); + LiveRangeMap prepare_live_range_map(); - LiveRangeMap prepare_live_range_map(); + void clear_pins(); - void clear_pins(); + int next_register_index() const { return m_next_register_index; } - int next_register_index() const { return m_next_register_index; } private: + PVirtualValue ssa_src(const nir_ssa_def& dest, int chan); - PVirtualValue ssa_src(const nir_ssa_def &dest, int chan); + PRegister local_register(const nir_reg_dest& dest, int chan); + PRegister local_register(const nir_reg_src& dest, int chan); + PRegister + resolve_array(nir_register *reg, nir_src *indirect, int base_offset, int chan); - PRegister local_register(const nir_reg_dest& dest, int chan); - PRegister local_register(const nir_reg_src& dest, int chan); - PRegister resolve_array(nir_register *reg, nir_src *indirect, - int base_offset, int chan); + int m_next_register_index; + int m_next_temp_channel{0}; - int m_next_register_index; - int m_next_temp_channel{0}; + template + using unordered_map_alloc = std::unordered_map, + std::equal_to, + Allocator>>; - template - using unordered_map_alloc = std::unordered_map, std::equal_to, - Allocator>>; + template + using unordered_reg_map_alloc = std::unordered_map, + Allocator>>; - template - using unordered_reg_map_alloc = std::unordered_map, - Allocator>>; + using RegisterMap = unordered_reg_map_alloc; + using ROValueMap = unordered_reg_map_alloc; - using RegisterMap = unordered_reg_map_alloc; - using ROValueMap = unordered_reg_map_alloc; + RegisterMap m_registers; + std::list> m_pinned_registers; + ROValueMap m_values; + unordered_map_alloc m_literal_values; + unordered_map_alloc m_inline_constants; + unordered_map_alloc m_ssa_index_to_sel; - RegisterMap m_registers; - std::list> m_pinned_registers; - ROValueMap m_values; - unordered_map_alloc m_literal_values; - unordered_map_alloc m_inline_constants; - unordered_map_alloc m_ssa_index_to_sel; + uint32_t m_nowrite_idx; - uint32_t m_nowrite_idx; - - RegisterVec4 m_dummy_dest_pinned{126, pin_chan, {0,1,2,3}}; - ChannelCounts m_channel_counts; + RegisterVec4 m_dummy_dest_pinned{ + 126, pin_chan, {0, 1, 2, 3} + }; + ChannelCounts m_channel_counts; }; -} +} // namespace r600 #endif // VALUEFACTORY_H diff --git a/src/gallium/drivers/r600/sfn/sfn_virtualvalues.cpp b/src/gallium/drivers/r600/sfn/sfn_virtualvalues.cpp index 3b408954349..a21dc87180a 100644 --- a/src/gallium/drivers/r600/sfn/sfn_virtualvalues.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_virtualvalues.cpp @@ -25,60 +25,68 @@ */ #include "sfn_virtualvalues.h" -#include "sfn_alu_defines.h" -#include "sfn_valuefactory.h" -#include "sfn_instr.h" -#include "sfn_debug.h" +#include "sfn_alu_defines.h" +#include "sfn_debug.h" +#include "sfn_instr.h" +#include "sfn_valuefactory.h" #include "util/macros.h" -#include -#include #include +#include #include +#include #include namespace r600 { -std::ostream& operator << (std::ostream& os, Pin pin) +std::ostream& +operator<<(std::ostream& os, Pin pin) { -#define PRINT_PIN(X) case pin_ ## X : os << #X; break +#define PRINT_PIN(X) \ + case pin_##X: \ + os << #X; \ + break switch (pin) { - PRINT_PIN(chan); - PRINT_PIN(array); - PRINT_PIN(fully); - PRINT_PIN(group); - PRINT_PIN(chgr); - PRINT_PIN(free); + PRINT_PIN(chan); + PRINT_PIN(array); + PRINT_PIN(fully); + PRINT_PIN(group); + PRINT_PIN(chgr); + PRINT_PIN(free); case pin_none: - default: - ; + default:; } #undef PRINT_PIN return os; } VirtualValue::VirtualValue(int sel, int chan, Pin pin): - m_sel(sel), m_chan(chan), m_pins(pin) + m_sel(sel), + m_chan(chan), + m_pins(pin) { #if __cpp_exceptions >= 199711L - ASSERT_OR_THROW(m_sel < virtual_register_base || pin != pin_fully, "Register is virtual but pinned to sel"); + ASSERT_OR_THROW(m_sel < virtual_register_base || pin != pin_fully, + "Register is virtual but pinned to sel"); #endif } -bool VirtualValue::ready(int block, int index) const +bool +VirtualValue::ready(int block, int index) const { (void)block; (void)index; return true; } -bool VirtualValue::is_virtual() const +bool +VirtualValue::is_virtual() const { return m_sel >= virtual_register_base; } -class ValueComparer: public ConstRegisterVisitor { +class ValueComparer : public ConstRegisterVisitor { public: ValueComparer(); ValueComparer(const Register *value); @@ -96,6 +104,7 @@ public: void visit(const InlineConstant& other) override; bool m_result; + private: const Register *m_register; const LocalArray *m_array; @@ -105,27 +114,33 @@ private: const InlineConstant *m_inline_constant; }; -class ValueCompareCreater: public ConstRegisterVisitor { +class ValueCompareCreater : public ConstRegisterVisitor { public: - void visit(const Register& value) { compare = ValueComparer(&value);} - void visit(const LocalArray& value) {compare = ValueComparer(&value);} - void visit(const LocalArrayValue& value) {compare = ValueComparer(&value);} - void visit(const UniformValue& value) {compare = ValueComparer(&value);} - void visit(const LiteralConstant& value) {compare = ValueComparer(&value);} - void visit(const InlineConstant& value) {compare = ValueComparer(&value);} + void visit(const Register& value) { compare = ValueComparer(&value); } + void visit(const LocalArray& value) { compare = ValueComparer(&value); } + void visit(const LocalArrayValue& value) { compare = ValueComparer(&value); } + void visit(const UniformValue& value) { compare = ValueComparer(&value); } + void visit(const LiteralConstant& value) { compare = ValueComparer(&value); } + void visit(const InlineConstant& value) { compare = ValueComparer(&value); } ValueComparer compare; }; -VirtualValue::Pointer VirtualValue::from_string(const std::string& s) +VirtualValue::Pointer +VirtualValue::from_string(const std::string& s) { switch (s[0]) { case 'S': - case 'R': return Register::from_string(s); - case 'L': return LiteralConstant::from_string(s); - case 'K': return UniformValue::from_string(s); - case 'P': return InlineConstant::param_from_string(s); - case 'I': return InlineConstant::from_string(s); + case 'R': + return Register::from_string(s); + case 'L': + return LiteralConstant::from_string(s); + case 'K': + return UniformValue::from_string(s); + case 'P': + return InlineConstant::param_from_string(s); + case 'I': + return InlineConstant::from_string(s); default: std::cerr << "'" << s << "'"; @@ -133,11 +148,10 @@ VirtualValue::Pointer VirtualValue::from_string(const std::string& s) } } -bool VirtualValue::equal_to(const VirtualValue& other) const +bool +VirtualValue::equal_to(const VirtualValue& other) const { - bool result = m_sel == other.m_sel && - m_chan == other.m_chan && - m_pins == other.m_pins; + bool result = m_sel == other.m_sel && m_chan == other.m_chan && m_pins == other.m_pins; if (result) { ValueCompareCreater comp_creater; @@ -149,19 +163,23 @@ bool VirtualValue::equal_to(const VirtualValue& other) const return result; } -VirtualValue::Pointer VirtualValue::get_addr() const +VirtualValue::Pointer +VirtualValue::get_addr() const { - class GetAddressRegister: public ConstRegisterVisitor { + class GetAddressRegister : public ConstRegisterVisitor { public: - void visit(const VirtualValue& value) {(void)value;} - void visit(const Register& value) {(void)value;}; - void visit(const LocalArray& value) {(void)value;} - void visit(const LocalArrayValue& value) {m_result = value.addr();} - void visit(const UniformValue& value) {(void)value;} - void visit(const LiteralConstant& value) {(void)value;} - void visit(const InlineConstant& value) {(void)value;} + void visit(const VirtualValue& value) { (void)value; } + void visit(const Register& value) { (void)value; }; + void visit(const LocalArray& value) { (void)value; } + void visit(const LocalArrayValue& value) { m_result = value.addr(); } + void visit(const UniformValue& value) { (void)value; } + void visit(const LiteralConstant& value) { (void)value; } + void visit(const InlineConstant& value) { (void)value; } - GetAddressRegister() : m_result(nullptr) {} + GetAddressRegister(): + m_result(nullptr) + { + } PVirtualValue m_result; }; @@ -171,57 +189,65 @@ VirtualValue::Pointer VirtualValue::get_addr() const } Register::Register(int sel, int chan, Pin pin): - VirtualValue(sel, chan, pin) + VirtualValue(sel, chan, pin) { } -void Register::add_parent(Instr *instr) +void +Register::add_parent(Instr *instr) { m_parents.insert(instr); instr->add_use(); add_parent_to_array(instr); } -void Register::add_parent_to_array(Instr *instr) +void +Register::add_parent_to_array(Instr *instr) { (void)instr; } -void Register::del_parent(Instr *instr) +void +Register::del_parent(Instr *instr) { m_parents.erase(instr); instr->dec_use(); del_parent_from_array(instr); } -void Register::del_parent_from_array(Instr *instr) +void +Register::del_parent_from_array(Instr *instr) { (void)instr; } - -void Register::add_use(Instr *instr) +void +Register::add_use(Instr *instr) { - const auto& [itr, inserted] = m_uses.insert(instr); {} + const auto& [itr, inserted] = m_uses.insert(instr); + { + } if (inserted) { - for (auto& p: m_parents) + for (auto& p : m_parents) p->add_use(); } } -void Register::del_use(Instr *instr) +void +Register::del_use(Instr *instr) { sfn_log << SfnLog::opt << "Del use of " << *this << " in " << *instr << "\n"; if (m_uses.find(instr) != m_uses.end()) { m_uses.erase(instr); if (is_ssa()) - for (auto& p: m_parents) + for (auto& p : m_parents) p->dec_use(); } } -bool Register::ready(int block, int index) const +bool +Register::ready(int block, int index) const { for (auto p : m_parents) { if (p->block_id() <= block) { @@ -233,43 +259,48 @@ bool Register::ready(int block, int index) const return true; } -void Register::accept(RegisterVisitor& visitor) +void +Register::accept(RegisterVisitor& visitor) { visitor.visit(*this); } -void Register::accept(ConstRegisterVisitor& visitor) const +void +Register::accept(ConstRegisterVisitor& visitor) const { visitor.visit(*this); } - -void Register::pin_live_range(bool start, bool end) +void +Register::pin_live_range(bool start, bool end) { m_pin_start = start; m_pin_end = end; } -void Register::set_is_ssa(bool value) +void +Register::set_is_ssa(bool value) { m_is_ssa = value; } -void Register::print(std::ostream& os) const +void +Register::print(std::ostream& os) const { os << (m_is_ssa ? "S" : "R") << sel() << "." << chanchar[chan()]; - if (pin() != pin_none) + if (pin() != pin_none) os << "@" << pin(); } -Register::Pointer Register::from_string(const std::string &s) +Register::Pointer +Register::from_string(const std::string& s) { std::string numstr; char chan = 0; std::string pinstr; - assert(s[0] == 'R' || s[0] == '_' || s[0] == 'S' ); + assert(s[0] == 'R' || s[0] == '_' || s[0] == 'S'); int type = 0; for (unsigned i = 1; i < s.length(); ++i) { @@ -282,9 +313,15 @@ Register::Pointer Register::from_string(const std::string &s) } switch (type) { - case 0: numstr.append(1, s[i]); break; - case 1: chan = s[i]; break; - case 2: pinstr.append(1, s[i]); break; + case 0: + numstr.append(1, s[i]); + break; + case 1: + chan = s[i]; + break; + case 2: + pinstr.append(1, s[i]); + break; default: unreachable("Malformed register string"); } @@ -313,16 +350,30 @@ Register::Pointer Register::from_string(const std::string &s) p = pin_free; switch (chan) { - case 'x' : chan = 0; break; - case 'y' : chan = 1; break; - case 'z' : chan = 2; break; - case 'w' : chan = 3; break; - case '0' : chan = 4; break; - case '1' : chan = 5; break; - case '_' : chan = 7; break; + case 'x': + chan = 0; + break; + case 'y': + chan = 1; + break; + case 'z': + chan = 2; + break; + case 'w': + chan = 3; + break; + case '0': + chan = 4; + break; + case '1': + chan = 5; + break; + case '_': + chan = 7; + break; } - auto reg = new Register( sel, chan, p); + auto reg = new Register(sel, chan, p); reg->set_is_ssa(s[0] == 'S'); if (p == pin_fully || p == pin_array) reg->pin_live_range(true); @@ -330,25 +381,25 @@ Register::Pointer Register::from_string(const std::string &s) } RegisterVec4::RegisterVec4(): - m_sel(-1), - m_swz({7,7,7,7}), - m_values({nullptr, nullptr, nullptr, nullptr}) + m_sel(-1), + m_swz({7, 7, 7, 7}), + m_values({nullptr, nullptr, nullptr, nullptr}) { } RegisterVec4::RegisterVec4(int sel, bool is_ssa, const Swizzle& swz, Pin pin): - m_sel(sel), - m_swz(swz) + m_sel(sel), + m_swz(swz) { for (int i = 0; i < 4; ++i) { - m_values[i] = new Element( *this, new Register(m_sel, swz[i], pin)); + m_values[i] = new Element(*this, new Register(m_sel, swz[i], pin)); m_values[i]->value()->set_is_ssa(is_ssa); } } RegisterVec4::RegisterVec4(const RegisterVec4& orig): - m_sel(orig.m_sel), - m_swz(orig.m_swz) + m_sel(orig.m_sel), + m_swz(orig.m_swz) { for (int i = 0; i < 4; ++i) m_values[i] = new Element(*this, orig.m_values[i]->value()); @@ -370,12 +421,12 @@ RegisterVec4::RegisterVec4(PRegister x, PRegister y, PRegister z, PRegister w, P m_sel = 0; if (!(x && y && z && w)) - dummy = new Register (m_sel, 7, pin_none); + dummy = new Register(m_sel, 7, pin_none); - m_values[0] = new Element(*this, x ? x : dummy); - m_values[1] = new Element(*this, y ? y : dummy); - m_values[2] = new Element(*this, z ? z : dummy); - m_values[3] = new Element(*this, w ? w : dummy); + m_values[0] = new Element(*this, x ? x : dummy); + m_values[1] = new Element(*this, y ? y : dummy); + m_values[2] = new Element(*this, z ? z : dummy); + m_values[3] = new Element(*this, w ? w : dummy); for (int i = 0; i < 4; ++i) { if (m_values[0]->value()->pin() == pin_fully) { @@ -389,13 +440,12 @@ RegisterVec4::RegisterVec4(PRegister x, PRegister y, PRegister z, PRegister w, P case pin_none: case pin_free: m_values[i]->value()->set_pin(pin); - break; + break; case pin_chan: if (pin == pin_group) m_values[i]->value()->set_pin(pin_chgr); - break; - default: - ; + break; + default:; } m_swz[i] = m_values[i]->value()->chan(); @@ -403,32 +453,35 @@ RegisterVec4::RegisterVec4(PRegister x, PRegister y, PRegister z, PRegister w, P } } -void RegisterVec4::add_use(Instr *instr) +void +RegisterVec4::add_use(Instr *instr) { - for (auto& r: m_values) { + for (auto& r : m_values) { if (r->value()->chan() < 4) - r->value()->add_use(instr); + r->value()->add_use(instr); } } -void RegisterVec4::del_use(Instr *instr) +void +RegisterVec4::del_use(Instr *instr) { - for (auto& r: m_values) { + for (auto& r : m_values) { r->value()->del_use(instr); } } -bool RegisterVec4::has_uses() const +bool +RegisterVec4::has_uses() const { - for (auto& r: m_values) { + for (auto& r : m_values) { if (r->value()->has_uses()) return true; } return false; } - -int RegisterVec4::sel() const +int +RegisterVec4::sel() const { int comp = 0; while (comp < 4 && m_values[comp]->value()->chan() > 3) @@ -436,7 +489,8 @@ int RegisterVec4::sel() const return comp < 4 ? m_values[comp]->value()->sel() : 0; } -bool RegisterVec4::ready(int block_id, int index) const +bool +RegisterVec4::ready(int block_id, int index) const { for (int i = 0; i < 4; ++i) { if (m_values[i]->value()->chan() < 4) { @@ -447,14 +501,16 @@ bool RegisterVec4::ready(int block_id, int index) const return true; } -void RegisterVec4::print(std::ostream& os) const +void +RegisterVec4::print(std::ostream& os) const { os << (m_values[0]->value()->is_ssa() ? 'S' : 'R') << sel() << "."; for (int i = 0; i < 4; ++i) os << VirtualValue::chanchar[m_values[i]->value()->chan()]; } -bool operator == (const RegisterVec4& lhs, const RegisterVec4& rhs) +bool +operator==(const RegisterVec4& lhs, const RegisterVec4& rhs) { for (int i = 0; i < 4; ++i) { assert(lhs[i]); @@ -467,39 +523,43 @@ bool operator == (const RegisterVec4& lhs, const RegisterVec4& rhs) } RegisterVec4::Element::Element(const RegisterVec4& parent, int chan): - m_parent(parent), - m_value(new Register(parent.m_sel, chan, pin_none)) + m_parent(parent), + m_value(new Register(parent.m_sel, chan, pin_none)) { } RegisterVec4::Element::Element(const RegisterVec4& parent, PRegister value): - m_parent(parent), - m_value(value) + m_parent(parent), + m_value(value) { } LiteralConstant::LiteralConstant(uint32_t value): - VirtualValue(ALU_SRC_LITERAL, -1, pin_none), - m_value(value) + VirtualValue(ALU_SRC_LITERAL, -1, pin_none), + m_value(value) { } -void LiteralConstant::accept(RegisterVisitor& vistor) +void +LiteralConstant::accept(RegisterVisitor& vistor) { vistor.visit(*this); } -void LiteralConstant::accept(ConstRegisterVisitor& vistor) const +void +LiteralConstant::accept(ConstRegisterVisitor& vistor) const { vistor.visit(*this); } -void LiteralConstant::print(std::ostream& os) const +void +LiteralConstant::print(std::ostream& os) const { os << "L[0x" << std::hex << m_value << std::dec << "]"; } -LiteralConstant::Pointer LiteralConstant::from_string(const std::string& s) +LiteralConstant::Pointer +LiteralConstant::from_string(const std::string& s) { if (s[1] != '[') return nullptr; @@ -519,40 +579,39 @@ LiteralConstant::Pointer LiteralConstant::from_string(const std::string& s) uint32_t num; n >> std::hex >> num; - return new LiteralConstant( num); + return new LiteralConstant(num); } - // Inline constants usually don't care about the channel but // ALU_SRC_PV should be pinned, but we only emit these constants // very late, and based on the real register they replace InlineConstant::InlineConstant(int sel, int chan): - VirtualValue(sel, chan, pin_none) + VirtualValue(sel, chan, pin_none) { } -void InlineConstant::accept(RegisterVisitor& vistor) +void +InlineConstant::accept(RegisterVisitor& vistor) { vistor.visit(*this); } -void InlineConstant::accept(ConstRegisterVisitor& vistor) const +void +InlineConstant::accept(ConstRegisterVisitor& vistor) const { vistor.visit(*this); } -void InlineConstant::print(std::ostream& os) const +void +InlineConstant::print(std::ostream& os) const { auto ivalue = alu_src_const.find(static_cast(sel())); if (ivalue != alu_src_const.end()) { - os << "I[" << ivalue->second.descr<< "]"; + os << "I[" << ivalue->second.descr << "]"; if (ivalue->second.use_chan) os << "." << chanchar[chan()]; - } else if (sel() >= ALU_SRC_PARAM_BASE && - sel() < ALU_SRC_PARAM_BASE + 32 ) { - os << "Param" - << sel() - ALU_SRC_PARAM_BASE - << "." << chanchar[chan()]; + } else if (sel() >= ALU_SRC_PARAM_BASE && sel() < ALU_SRC_PARAM_BASE + 32) { + os << "Param" << sel() - ALU_SRC_PARAM_BASE << "." << chanchar[chan()]; } else { unreachable("Unknown inline constant"); } @@ -560,7 +619,8 @@ void InlineConstant::print(std::ostream& os) const std::map> InlineConstant::s_opmap; -InlineConstant::Pointer InlineConstant::from_string(const std::string& s) +InlineConstant::Pointer +InlineConstant::from_string(const std::string& s) { std::string namestr; char chan = 0; @@ -581,7 +641,7 @@ InlineConstant::Pointer InlineConstant::from_string(const std::string& s) AluInlineConstants value = ALU_SRC_UNKNOWN; bool use_chan = false; - if (entry == s_opmap.end()) { + if (entry == s_opmap.end()) { for (auto& [opcode, descr] : alu_src_const) { if (namestr == descr.descr) { value = opcode; @@ -601,21 +661,36 @@ InlineConstant::Pointer InlineConstant::from_string(const std::string& s) if (use_chan) { ASSERT_OR_THROW(s[i + 1] == '.', "inline const channel not started with '.'"); switch (s[i + 2]) { - case 'x': chan = 0; break; - case 'y': chan = 1; break; - case 'z': chan = 2; break; - case 'w': chan = 3; break; - case '0': chan = 4; break; - case '1': chan = 5; break; - case '_': chan = 7; break; + case 'x': + chan = 0; + break; + case 'y': + chan = 1; + break; + case 'z': + chan = 2; + break; + case 'w': + chan = 3; + break; + case '0': + chan = 4; + break; + case '1': + chan = 5; + break; + case '_': + chan = 7; + break; default: ASSERT_OR_THROW(0, "invalied inline const channel "); } } - return new InlineConstant( value, chan); + return new InlineConstant(value, chan); } -InlineConstant::Pointer InlineConstant::param_from_string(const std::string& s) +InlineConstant::Pointer +InlineConstant::param_from_string(const std::string& s) { assert(s.substr(0, 5) == "Param"); @@ -623,64 +698,76 @@ InlineConstant::Pointer InlineConstant::param_from_string(const std::string& s) int i = 5; while (isdigit(s[i])) { param *= 10; - param += s[i] - '0'; + param += s[i] - '0'; ++i; } int chan = 7; assert(s[i] == '.'); - switch (s[i+1]) { - case 'x': chan = 0; break; - case 'y': chan = 1; break; - case 'z': chan = 2; break; - case 'w': chan = 3; break; + switch (s[i + 1]) { + case 'x': + chan = 0; + break; + case 'y': + chan = 1; + break; + case 'z': + chan = 2; + break; + case 'w': + chan = 3; + break; default: unreachable("unsupported channel char"); } - return new InlineConstant( ALU_SRC_PARAM_BASE + param, chan); + return new InlineConstant(ALU_SRC_PARAM_BASE + param, chan); } UniformValue::UniformValue(int sel, int chan, int kcache_bank): - VirtualValue(sel, chan, pin_none), - m_kcache_bank(kcache_bank), - m_buf_addr(nullptr) + VirtualValue(sel, chan, pin_none), + m_kcache_bank(kcache_bank), + m_buf_addr(nullptr) { } UniformValue::UniformValue(int sel, int chan, PVirtualValue buf_addr): - VirtualValue(sel, chan, pin_none), - m_kcache_bank(0), - m_buf_addr(buf_addr) + VirtualValue(sel, chan, pin_none), + m_kcache_bank(0), + m_buf_addr(buf_addr) { } -void UniformValue::accept(RegisterVisitor& vistor) +void +UniformValue::accept(RegisterVisitor& vistor) { vistor.visit(*this); } -void UniformValue::accept(ConstRegisterVisitor& vistor) const +void +UniformValue::accept(ConstRegisterVisitor& vistor) const { vistor.visit(*this); } -PVirtualValue UniformValue::buf_addr() const +PVirtualValue +UniformValue::buf_addr() const { return m_buf_addr; } -void UniformValue::print(std::ostream& os) const +void +UniformValue::print(std::ostream& os) const { os << "KC" << m_kcache_bank; if (m_buf_addr) { - os << "[" << *m_buf_addr - << "]"; + os << "[" << *m_buf_addr << "]"; } os << "[" << (sel() - 512) << "]." << chanchar[chan()]; } -bool UniformValue::equal_buf_and_cache(const UniformValue& other) const +bool +UniformValue::equal_buf_and_cache(const UniformValue& other) const { bool result = m_kcache_bank == other.m_kcache_bank; if (result) { @@ -693,8 +780,8 @@ bool UniformValue::equal_buf_and_cache(const UniformValue& other) const return result; } - -UniformValue::Pointer UniformValue::from_string(const std::string& s) +UniformValue::Pointer +UniformValue::from_string(const std::string& s) { assert(s[1] == 'C'); std::istringstream is(s.substr(2)); @@ -716,10 +803,18 @@ UniformValue::Pointer UniformValue::from_string(const std::string& s) is >> c; int chan = 0; switch (c) { - case 'x': chan = 0; break; - case 'y': chan = 1; break; - case 'z': chan = 2; break; - case 'w': chan = 3; break; + case 'x': + chan = 0; + break; + case 'y': + chan = 1; + break; + case 'z': + chan = 2; + break; + case 'w': + chan = 3; + break; default: unreachable("Unknown channle when reading uniform"); } @@ -727,83 +822,102 @@ UniformValue::Pointer UniformValue::from_string(const std::string& s) } LocalArray::LocalArray(int base_sel, int nchannels, int size, int frac): - Register(base_sel, nchannels, pin_array), - m_base_sel(base_sel), - m_nchannels(nchannels), - m_size(size), - m_values(size * nchannels), - m_frac(frac) + Register(base_sel, nchannels, pin_array), + m_base_sel(base_sel), + m_nchannels(nchannels), + m_size(size), + m_values(size * nchannels), + m_frac(frac) { assert(nchannels <= 4); assert(nchannels + frac <= 4); - sfn_log << SfnLog::reg << "Allocate array A" << base_sel << "(" - << size << ", " << frac << ", " << nchannels << ")\n"; + sfn_log << SfnLog::reg << "Allocate array A" << base_sel << "(" << size << ", " << frac + << ", " << nchannels << ")\n"; for (int c = 0; c < nchannels; ++c) { for (unsigned i = 0; i < m_size; ++i) { - PRegister reg = new Register( base_sel + i, c + frac, pin_array); + PRegister reg = new Register(base_sel + i, c + frac, pin_array); m_values[m_size * c + i] = new LocalArrayValue(reg, *this); /* Pin the array register on the start, because currently we don't * don't track the first write to an array element as write to all - * array elements, and it seems that the one can not just use registers - * that are not written to in an array for other purpouses */ + * array elements, and it seems that the one can not just use + * registers that are not written to in an array for other purpouses + */ m_values[m_size * c + i]->pin_live_range(true); } } } -void LocalArray::accept(RegisterVisitor& vistor) +void +LocalArray::accept(RegisterVisitor& vistor) { vistor.visit(*this); } -void LocalArray::accept(ConstRegisterVisitor& vistor) const +void +LocalArray::accept(ConstRegisterVisitor& vistor) const { vistor.visit(*this); } -void LocalArray::print(std::ostream& os) const +void +LocalArray::print(std::ostream& os) const { - os << "A" << m_base_sel << "[0 " << ":" << m_values.size() << "]."; + os << "A" << m_base_sel << "[0 " + << ":" << m_values.size() << "]."; for (unsigned i = 0; i < m_nchannels; ++i) { os << chanchar[i]; } } - -size_t LocalArray::size() const +size_t +LocalArray::size() const { return m_size; } -uint32_t LocalArray::nchannels() const +uint32_t +LocalArray::nchannels() const { return m_nchannels; } -PRegister LocalArray::element(size_t offset, PVirtualValue indirect, uint32_t chan) +PRegister +LocalArray::element(size_t offset, PVirtualValue indirect, uint32_t chan) { ASSERT_OR_THROW(offset < m_size, "Array: index out of range"); ASSERT_OR_THROW(chan < m_nchannels, "Array: channel out of range"); - sfn_log << SfnLog::reg << "Request element A" << m_base_sel << "[" << offset; + sfn_log << SfnLog::reg << "Request element A" << m_base_sel << "[" << offset; if (indirect) - sfn_log << "+" << *indirect; + sfn_log << "+" << *indirect; sfn_log << SfnLog::reg << "]\n"; if (indirect) { - class ResolveDirectArrayElement: public ConstRegisterVisitor { + class ResolveDirectArrayElement : public ConstRegisterVisitor { public: - void visit(const Register& value) {(void) value;}; - void visit(const LocalArray& value) {(void)value; unreachable("An array can't be used as address");} - void visit(const LocalArrayValue& value) {(void) value;} - void visit(const UniformValue& value) {(void)value;} - void visit(const LiteralConstant& value) {offset = value.value(); is_contant = true;} - void visit(const InlineConstant& value) {(void)value;} + void visit(const Register& value) { (void)value; }; + void visit(const LocalArray& value) + { + (void)value; + unreachable("An array can't be used as address"); + } + void visit(const LocalArrayValue& value) { (void)value; } + void visit(const UniformValue& value) { (void)value; } + void visit(const LiteralConstant& value) + { + offset = value.value(); + is_contant = true; + } + void visit(const InlineConstant& value) { (void)value; } - ResolveDirectArrayElement(): offset(0), is_contant(false) {} + ResolveDirectArrayElement(): + offset(0), + is_contant(false) + { + } int offset; bool is_contant; @@ -821,7 +935,7 @@ PRegister LocalArray::element(size_t offset, PVirtualValue indirect, uint32_t ch LocalArrayValue *reg = m_values[m_size * chan + offset]; if (indirect) { - reg = new LocalArrayValue( reg, indirect, *this); + reg = new LocalArrayValue(reg, indirect, *this); m_values_indirect.push_back(reg); } @@ -829,7 +943,8 @@ PRegister LocalArray::element(size_t offset, PVirtualValue indirect, uint32_t ch return reg; } -bool LocalArray::ready_for_direct(int block, int index, int chan) const +bool +LocalArray::ready_for_direct(int block, int index, int chan) const { if (!Register::ready(block, index)) return false; @@ -845,7 +960,8 @@ bool LocalArray::ready_for_direct(int block, int index, int chan) const return true; } -bool LocalArray::ready_for_indirect(int block, int index, int chan) const +bool +LocalArray::ready_for_indirect(int block, int index, int chan) const { int offset = (chan - m_frac) * m_size; for (unsigned i = 0; i < m_size; ++i) { @@ -856,75 +972,80 @@ bool LocalArray::ready_for_indirect(int block, int index, int chan) const return ready_for_direct(block, index, chan); } - -LocalArrayValue::LocalArrayValue(PRegister reg, PVirtualValue index, - LocalArray& array): - Register(reg->sel(), reg->chan(), pin_array), - m_addr(index), - m_array(array) +LocalArrayValue::LocalArrayValue(PRegister reg, PVirtualValue index, LocalArray& array): + Register(reg->sel(), reg->chan(), pin_array), + m_addr(index), + m_array(array) { } -const Register& LocalArray::operator ()(size_t idx, size_t chan) const +const Register& +LocalArray::operator()(size_t idx, size_t chan) const { - return *m_values[m_size * (chan - m_frac) + idx]; + return *m_values[m_size * (chan - m_frac) + idx]; } LocalArrayValue::LocalArrayValue(PRegister reg, LocalArray& array): - LocalArrayValue(reg, nullptr, array) + LocalArrayValue(reg, nullptr, array) { - } - -PVirtualValue LocalArrayValue::addr() const +PVirtualValue +LocalArrayValue::addr() const { return m_addr; } -const LocalArray& LocalArrayValue::array() const +const LocalArray& +LocalArrayValue::array() const { return m_array; } - -void LocalArrayValue::forward_del_use(Instr *instr) +void +LocalArrayValue::forward_del_use(Instr *instr) { if (m_addr && m_addr->as_register()) m_addr->as_register()->del_use(instr); } -void LocalArrayValue::forward_add_use(Instr *instr) +void +LocalArrayValue::forward_add_use(Instr *instr) { if (m_addr && m_addr->as_register()) m_addr->as_register()->add_use(instr); } -void LocalArrayValue::accept(RegisterVisitor& vistor) +void +LocalArrayValue::accept(RegisterVisitor& vistor) { vistor.visit(*this); } -void LocalArrayValue::accept(ConstRegisterVisitor& vistor) const +void +LocalArrayValue::accept(ConstRegisterVisitor& vistor) const { vistor.visit(*this); } -void LocalArrayValue::add_parent_to_array(Instr *instr) +void +LocalArrayValue::add_parent_to_array(Instr *instr) { m_array.add_parent(instr); } -void LocalArrayValue::del_parent_from_array(Instr *instr) +void +LocalArrayValue::del_parent_from_array(Instr *instr) { m_array.del_parent(instr); } -void LocalArrayValue::print(std::ostream& os) const +void +LocalArrayValue::print(std::ostream& os) const { int offset = sel() - m_array.sel(); os << "A" << m_array.sel() << "["; - if ( offset > 0 && m_addr) + if (offset > 0 && m_addr) os << offset << "+" << *m_addr; else if (m_addr) os << *m_addr; @@ -933,99 +1054,110 @@ void LocalArrayValue::print(std::ostream& os) const os << "]." << chanchar[chan()]; } -bool LocalArrayValue::ready(int block, int index) const -{ - return m_addr ? - (m_array.ready_for_indirect(block, index, chan()) && m_addr->ready(block, index)): - m_array.ready_for_direct(block, index, chan()); +bool +LocalArrayValue::ready(int block, int index) const +{ + return m_addr ? (m_array.ready_for_indirect(block, index, chan()) && + m_addr->ready(block, index)) + : m_array.ready_for_direct(block, index, chan()); } -ValueComparer::ValueComparer() : - m_result(false), - m_register(nullptr), - m_array(nullptr), - m_array_value(nullptr), - m_uniform_value(nullptr), - m_literal_value(nullptr), - m_inline_constant(nullptr) -{} +ValueComparer::ValueComparer(): + m_result(false), + m_register(nullptr), + m_array(nullptr), + m_array_value(nullptr), + m_uniform_value(nullptr), + m_literal_value(nullptr), + m_inline_constant(nullptr) +{ +} ValueComparer::ValueComparer(const Register *value): - m_result(false), - m_register(value), - m_array(nullptr), - m_array_value(nullptr), - m_uniform_value(nullptr), - m_literal_value(nullptr), - m_inline_constant(nullptr) -{} + m_result(false), + m_register(value), + m_array(nullptr), + m_array_value(nullptr), + m_uniform_value(nullptr), + m_literal_value(nullptr), + m_inline_constant(nullptr) +{ +} ValueComparer::ValueComparer(const LocalArray *value): - m_result(false), - m_register(nullptr), - m_array(value), - m_array_value(nullptr), - m_uniform_value(nullptr), - m_literal_value(nullptr), - m_inline_constant(nullptr) -{} + m_result(false), + m_register(nullptr), + m_array(value), + m_array_value(nullptr), + m_uniform_value(nullptr), + m_literal_value(nullptr), + m_inline_constant(nullptr) +{ +} ValueComparer::ValueComparer(const LocalArrayValue *value): - m_result(false), - m_register(nullptr), - m_array(nullptr), - m_array_value(value), - m_uniform_value(nullptr), - m_literal_value(nullptr), - m_inline_constant(nullptr) -{} + m_result(false), + m_register(nullptr), + m_array(nullptr), + m_array_value(value), + m_uniform_value(nullptr), + m_literal_value(nullptr), + m_inline_constant(nullptr) +{ +} ValueComparer::ValueComparer(const UniformValue *value): - m_result(false), - m_register(nullptr), - m_array(nullptr), - m_array_value(nullptr), - m_uniform_value(value), - m_literal_value(nullptr), - m_inline_constant(nullptr) -{} + m_result(false), + m_register(nullptr), + m_array(nullptr), + m_array_value(nullptr), + m_uniform_value(value), + m_literal_value(nullptr), + m_inline_constant(nullptr) +{ +} ValueComparer::ValueComparer(const LiteralConstant *value): - m_result(false), - m_register(nullptr), - m_array(nullptr), - m_array_value(nullptr), - m_uniform_value(nullptr), - m_literal_value(value), - m_inline_constant(nullptr) -{} + m_result(false), + m_register(nullptr), + m_array(nullptr), + m_array_value(nullptr), + m_uniform_value(nullptr), + m_literal_value(value), + m_inline_constant(nullptr) +{ +} ValueComparer::ValueComparer(const InlineConstant *value): - m_result(false), - m_register(nullptr), - m_array(nullptr), - m_array_value(nullptr), - m_uniform_value(nullptr), - m_literal_value(nullptr), - m_inline_constant(value) -{} + m_result(false), + m_register(nullptr), + m_array(nullptr), + m_array_value(nullptr), + m_uniform_value(nullptr), + m_literal_value(nullptr), + m_inline_constant(value) +{ +} -void ValueComparer::visit(const Register& other) +void +ValueComparer::visit(const Register& other) { (void)other; m_result = !!m_register; }; -void ValueComparer::visit(const LocalArray& other) +void +ValueComparer::visit(const LocalArray& other) { m_result = false; if (m_array) { - m_result = m_array->size() == other.size() && - m_array->nchannels() == other.nchannels(); + m_result = + m_array->size() == other.size() && m_array->nchannels() == other.nchannels(); } }; -void ValueComparer::visit(const LocalArrayValue& other) +void +ValueComparer::visit(const LocalArrayValue& other) { m_result = false; if (m_array_value) { @@ -1042,7 +1174,8 @@ void ValueComparer::visit(const LocalArrayValue& other) } }; -void ValueComparer::visit(const UniformValue& value) +void +ValueComparer::visit(const UniformValue& value) { m_result = false; if (m_uniform_value) { @@ -1059,16 +1192,17 @@ void ValueComparer::visit(const UniformValue& value) } }; -void ValueComparer::visit(const LiteralConstant& other) +void +ValueComparer::visit(const LiteralConstant& other) { m_result = m_literal_value && (m_literal_value->value() == other.value()); }; -void ValueComparer::visit(const InlineConstant& other) +void +ValueComparer::visit(const InlineConstant& other) { (void)other; m_result = !!m_inline_constant; }; - } // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/sfn_virtualvalues.h b/src/gallium/drivers/r600/sfn/sfn_virtualvalues.h index d71c0fd4db9..f56a5ac1a93 100644 --- a/src/gallium/drivers/r600/sfn/sfn_virtualvalues.h +++ b/src/gallium/drivers/r600/sfn/sfn_virtualvalues.h @@ -26,21 +26,26 @@ #pragma once -#include "sfn_memorypool.h" #include "sfn_alu_defines.h" -#include -#include -#include -#include -#include +#include "sfn_memorypool.h" + #include #include +#include +#include +#include +#include +#include #if __cpp_exceptions >= 199711L #include -#define ASSERT_OR_THROW(EXPR, ERROR) if (!(EXPR)) throw std::invalid_argument(ERROR) +#define ASSERT_OR_THROW(EXPR, ERROR) \ + if (!(EXPR)) \ + throw std::invalid_argument(ERROR) #else -#define ASSERT_OR_THROW(EXPR, ERROR) if (!(EXPR)) unreachable(ERROR) +#define ASSERT_OR_THROW(EXPR, ERROR) \ + if (!(EXPR)) \ + unreachable(ERROR) #endif namespace r600 { @@ -55,7 +60,8 @@ enum Pin { pin_free }; -std::ostream& operator << (std::ostream& os, Pin pin); +std::ostream& +operator<<(std::ostream& os, Pin pin); class Register; class RegisterVisitor; @@ -65,11 +71,10 @@ class InlineConstant; class LiteralConstant; class UniformValue; -using InstructionSet = std::set, Allocator>; +using InstructionSet = std::set, Allocator>; class VirtualValue : public Allocate { public: - static const uint32_t virtual_register_base = 1024; static const uint32_t clause_temp_registers = 2; static const uint32_t gpr_register_end = 128 - 2 * clause_temp_registers; @@ -85,12 +90,11 @@ public: VirtualValue(const VirtualValue& orig) = default; int sel() const { return m_sel; } - int chan() const { return m_chan;} - Pin pin() const { return m_pins;}; + int chan() const { return m_chan; } + Pin pin() const { return m_pins; }; bool is_virtual() const; - void set_pin(Pin p) { m_pins = p;} - + void set_pin(Pin p) { m_pins = p; } virtual void accept(RegisterVisitor& vistor) = 0; virtual void accept(ConstRegisterVisitor& vistor) const = 0; @@ -101,16 +105,16 @@ public: static Pointer from_string(const std::string& s); - virtual Register *as_register() { return nullptr;} - virtual InlineConstant * as_inline_const() { return nullptr;} - virtual LiteralConstant *as_literal() { return nullptr;} - virtual UniformValue *as_uniform() { return nullptr;} + virtual Register *as_register() { return nullptr; } + virtual InlineConstant *as_inline_const() { return nullptr; } + virtual LiteralConstant *as_literal() { return nullptr; } + virtual UniformValue *as_uniform() { return nullptr; } virtual bool ready(int block, int index) const; static constexpr char chanchar[9] = "xyzw01?_"; protected: - void do_set_chan(int c) {m_chan = c;} + void do_set_chan(int c) { m_chan = c; } void set_sel_internal(int sel) { m_sel = sel; } private: @@ -120,21 +124,32 @@ private: }; using PVirtualValue = VirtualValue::Pointer; - -inline std::ostream& operator << (std::ostream& os, const VirtualValue& val) +inline std::ostream& +operator<<(std::ostream& os, const VirtualValue& val) { - val.print(os); - return os; + val.print(os); + return os; } -inline bool operator == (const VirtualValue& lhs, const VirtualValue& rhs) +inline bool +operator==(const VirtualValue& lhs, const VirtualValue& rhs) { - return lhs.equal_to(rhs); + return lhs.equal_to(rhs); } struct LiveRange { - LiveRange(): start(-1), end(-1), is_pinned(false) {} - LiveRange(int s, int e): start(s), end(e), is_pinned(false) {} + LiveRange(): + start(-1), + end(-1), + is_pinned(false) + { + } + LiveRange(int s, int e): + start(s), + end(e), + is_pinned(false) + { + } int start; int end; int is_pinned; @@ -142,111 +157,120 @@ struct LiveRange { class Register : public VirtualValue { public: - using Pointer = R600_POINTER_TYPE(Register); + using Pointer = R600_POINTER_TYPE(Register); - Register(int sel, int chan, Pin pin); - void accept(RegisterVisitor& vistor) override; - void accept(ConstRegisterVisitor& vistor) const override; - void print(std::ostream& os) const override; + Register(int sel, int chan, Pin pin); + void accept(RegisterVisitor& vistor) override; + void accept(ConstRegisterVisitor& vistor) const override; + void print(std::ostream& os) const override; - int live_start_pinned() const { return m_pin_start;} - int live_end_pinned() const { return m_pin_end;} + int live_start_pinned() const { return m_pin_start; } + int live_end_pinned() const { return m_pin_end; } - void pin_live_range(bool start, bool end = false); + void pin_live_range(bool start, bool end = false); - static Pointer from_string(const std::string& s); + static Pointer from_string(const std::string& s); - Register *as_register() override { return this;} + Register *as_register() override { return this; } - void set_is_ssa(bool value); + void set_is_ssa(bool value); - bool is_ssa() const { return m_is_ssa;} + bool is_ssa() const { return m_is_ssa; } - void add_parent(Instr *instr); - void del_parent(Instr *instr); - const InstructionSet& parents() const {return m_parents;} + void add_parent(Instr *instr); + void del_parent(Instr *instr); + const InstructionSet& parents() const { return m_parents; } - bool ready(int block, int index) const override; + bool ready(int block, int index) const override; - const InstructionSet& uses() const {return m_uses;} - void add_use(Instr *instr); - void del_use(Instr *instr); - bool has_uses() const {return !m_uses.empty() || pin() == pin_array;} - void set_chan(int c) {do_set_chan(c);} + const InstructionSet& uses() const { return m_uses; } + void add_use(Instr *instr); + void del_use(Instr *instr); + bool has_uses() const { return !m_uses.empty() || pin() == pin_array; } + void set_chan(int c) { do_set_chan(c); } - virtual VirtualValue *addr() const { return nullptr;} + virtual VirtualValue *addr() const { return nullptr; } - int index() const {return m_index;} - void set_index(int idx) {m_index = idx;} + int index() const { return m_index; } + void set_index(int idx) { m_index = idx; } - void set_sel(int new_sel) { set_sel_internal(new_sel); m_is_ssa = false;} + void set_sel(int new_sel) + { + set_sel_internal(new_sel); + m_is_ssa = false; + } private: - Register(const Register& orig) = delete; - Register(const Register&& orig) = delete; - Register& operator = (const Register& orig) = delete; - Register& operator = (Register&& orig) = delete; + Register(const Register& orig) = delete; + Register(const Register&& orig) = delete; + Register& operator=(const Register& orig) = delete; + Register& operator=(Register&& orig) = delete; - virtual void forward_del_use(Instr *instr) {(void)instr;} - virtual void forward_add_use(Instr *instr) {(void)instr;} - virtual void add_parent_to_array(Instr *instr); - virtual void del_parent_from_array(Instr *instr); + virtual void forward_del_use(Instr *instr) { (void)instr; } + virtual void forward_add_use(Instr *instr) { (void)instr; } + virtual void add_parent_to_array(Instr *instr); + virtual void del_parent_from_array(Instr *instr); - InstructionSet m_parents; - InstructionSet m_uses; + InstructionSet m_parents; + InstructionSet m_uses; - int m_index{-1}; + int m_index{-1}; - bool m_is_ssa {false}; - bool m_pin_start {false}; - bool m_pin_end {false}; + bool m_is_ssa{false}; + bool m_pin_start{false}; + bool m_pin_end{false}; }; using PRegister = Register::Pointer; -inline std::ostream& operator << (std::ostream& os, const Register& val) +inline std::ostream& +operator<<(std::ostream& os, const Register& val) { - val.print(os); - return os; + val.print(os); + return os; } class InlineConstant : public VirtualValue { public: - using Pointer = R600_POINTER_TYPE(InlineConstant); + using Pointer = R600_POINTER_TYPE(InlineConstant); - InlineConstant(int sel, int chan = 0); + InlineConstant(int sel, int chan = 0); - void accept(RegisterVisitor& vistor) override; - void accept(ConstRegisterVisitor& vistor) const override; - void print(std::ostream& os) const override; - static Pointer from_string(const std::string& s); - static Pointer param_from_string(const std::string& s); + void accept(RegisterVisitor& vistor) override; + void accept(ConstRegisterVisitor& vistor) const override; + void print(std::ostream& os) const override; + static Pointer from_string(const std::string& s); + static Pointer param_from_string(const std::string& s); + + InlineConstant *as_inline_const() override { return this; } - InlineConstant * as_inline_const() override { return this;} private: - InlineConstant(const InlineConstant& orig) = default; - static std::map> s_opmap; - + InlineConstant(const InlineConstant& orig) = default; + static std::map> s_opmap; }; using PInlineConstant = InlineConstant::Pointer; -inline std::ostream& operator << (std::ostream& os, const InlineConstant& val) +inline std::ostream& +operator<<(std::ostream& os, const InlineConstant& val) { - val.print(os); - return os; + val.print(os); + return os; } class RegisterVec4 { public: - using Swizzle = std::array; + using Swizzle = std::array; RegisterVec4(); - RegisterVec4(int sel, bool is_ssa = false, const Swizzle& swz = {0,1,2,3}, Pin pin = pin_group); + RegisterVec4(int sel, + bool is_ssa = false, + const Swizzle& swz = {0, 1, 2, 3}, + Pin pin = pin_group); RegisterVec4(PRegister x, PRegister y, PRegister z, PRegister w, Pin pin); RegisterVec4(const RegisterVec4& orig); RegisterVec4(RegisterVec4&& orig) = default; - RegisterVec4& operator = (RegisterVec4& orig) = default; - RegisterVec4& operator = (RegisterVec4&& orig) = default; + RegisterVec4& operator=(RegisterVec4& orig) = default; + RegisterVec4& operator=(RegisterVec4&& orig) = default; void add_use(Instr *instr); void del_use(Instr *instr); @@ -255,28 +279,26 @@ public: int sel() const; void print(std::ostream& os) const; - class Element : public Allocate { + class Element : public Allocate { public: Element(const RegisterVec4& parent, int chan); Element(const RegisterVec4& parent, PRegister value); PRegister value() { return m_value; } - void set_value(PRegister reg) { m_value = reg;} + void set_value(PRegister reg) { m_value = reg; } + private: const RegisterVec4& m_parent; PRegister m_value; - }; + }; - friend class Element; + friend class Element; - PRegister operator [] (int i) const { - return m_values[i]->value(); - } + PRegister operator[](int i) const { return m_values[i]->value(); } - PRegister operator [] (int i) { - return m_values[i]->value(); - } + PRegister operator[](int i) { return m_values[i]->value(); } - void set_value(int i, PRegister reg) { + void set_value(int i, PRegister reg) + { if (reg->chan() < 4) { m_sel = reg->sel(); } @@ -284,7 +306,8 @@ public: m_values[i]->set_value(reg); } - void validate() { + void validate() + { int sel = -1; for (int k = 0; k < 4; ++k) { if (sel < 0) { @@ -297,7 +320,8 @@ public: } } - uint8_t free_chan_mask() const { + uint8_t free_chan_mask() const + { int mask = 0xf; for (int i = 0; i < 4; ++i) { int chan = m_values[i]->value()->chan(); @@ -309,26 +333,29 @@ public: } bool ready(int block_id, int index) const; + private: int m_sel; Swizzle m_swz; std::array m_values; }; -bool operator == (const RegisterVec4& lhs, const RegisterVec4& rhs); +bool +operator==(const RegisterVec4& lhs, const RegisterVec4& rhs); -inline bool operator != (const RegisterVec4& lhs, const RegisterVec4& rhs) +inline bool +operator!=(const RegisterVec4& lhs, const RegisterVec4& rhs) { return !(lhs == rhs); } -inline std::ostream& operator << (std::ostream& os, const RegisterVec4& val) +inline std::ostream& +operator<<(std::ostream& os, const RegisterVec4& val) { - val.print(os); - return os; + val.print(os); + return os; } - class LiteralConstant : public VirtualValue { public: using Pointer = R600_POINTER_TYPE(LiteralConstant); @@ -337,9 +364,9 @@ public: void accept(RegisterVisitor& vistor) override; void accept(ConstRegisterVisitor& vistor) const override; void print(std::ostream& os) const override; - uint32_t value() const {return m_value;} + uint32_t value() const { return m_value; } static Pointer from_string(const std::string& s); - LiteralConstant *as_literal() override { return this;} + LiteralConstant *as_literal() override { return this; } private: LiteralConstant(const LiteralConstant& orig) = default; @@ -347,7 +374,6 @@ private: }; using PLiteralVirtualValue = LiteralConstant::Pointer; - class UniformValue : public VirtualValue { public: using Pointer = R600_POINTER_TYPE(UniformValue); @@ -360,7 +386,7 @@ public: void print(std::ostream& os) const override; int kcache_bank() const { return m_kcache_bank; } PVirtualValue buf_addr() const; - UniformValue *as_uniform() override { return this;} + UniformValue *as_uniform() override { return this; } bool equal_buf_and_cache(const UniformValue& other) const; static Pointer from_string(const std::string& s); @@ -371,17 +397,18 @@ private: }; using PUniformVirtualValue = UniformValue::Pointer; -inline std::ostream& operator << (std::ostream& os, const UniformValue& val) +inline std::ostream& +operator<<(std::ostream& os, const UniformValue& val) { - val.print(os); - return os; + val.print(os); + return os; } class LocalArrayValue; class LocalArray : public Register { public: using Pointer = R600_POINTER_TYPE(LocalArray); - using Values = std::vector >; + using Values = std::vector>; LocalArray(int base_sel, int nchannels, int size, int frac = 0); void accept(RegisterVisitor& vistor) override; @@ -394,67 +421,71 @@ public: size_t size() const; uint32_t nchannels() const; - uint32_t frac() const { return m_frac;} + uint32_t frac() const { return m_frac; } void add_parent_to_elements(Instr *instr); - const Register& operator ()(size_t idx, size_t chan) const; + const Register& operator()(size_t idx, size_t chan) const; - Values::iterator begin() { return m_values.begin();} - Values::iterator end() { return m_values.end();} + Values::iterator begin() { return m_values.begin(); } + Values::iterator end() { return m_values.end(); } private: - uint32_t m_base_sel; - uint32_t m_nchannels; - size_t m_size; - Values m_values; - Values m_values_indirect; - int m_frac; + uint32_t m_base_sel; + uint32_t m_nchannels; + size_t m_size; + Values m_values; + Values m_values_indirect; + int m_frac; }; -inline std::ostream& operator << (std::ostream& os, const LocalArray & val) +inline std::ostream& +operator<<(std::ostream& os, const LocalArray& val) { - val.print(os); - return os; + val.print(os); + return os; } class LocalArrayValue : public Register { public: - using Pointer = R600_POINTER_TYPE(LocalArrayValue); + using Pointer = R600_POINTER_TYPE(LocalArrayValue); - LocalArrayValue(PRegister reg, LocalArray& array); - LocalArrayValue(PRegister reg, PVirtualValue index, LocalArray &array); + LocalArrayValue(PRegister reg, LocalArray& array); + LocalArrayValue(PRegister reg, PVirtualValue index, LocalArray& array); - void accept(RegisterVisitor& vistor) override; - void accept(ConstRegisterVisitor& vistor) const override; - void print(std::ostream& os) const override; - bool ready(int block, int index) const override; + void accept(RegisterVisitor& vistor) override; + void accept(ConstRegisterVisitor& vistor) const override; + void print(std::ostream& os) const override; + bool ready(int block, int index) const override; + + VirtualValue *addr() const override; + const LocalArray& array() const; - VirtualValue *addr() const override; - const LocalArray& array() const; private: - void forward_del_use(Instr *instr) override; - void forward_add_use(Instr *instr) override; - void add_parent_to_array(Instr *instr) override; - void del_parent_from_array(Instr *instr) override; + void forward_del_use(Instr *instr) override; + void forward_add_use(Instr *instr) override; + void add_parent_to_array(Instr *instr) override; + void del_parent_from_array(Instr *instr) override; - PVirtualValue m_addr; - LocalArray& m_array; + PVirtualValue m_addr; + LocalArray& m_array; }; -inline std::ostream& operator << (std::ostream& os, const LocalArrayValue& val) +inline std::ostream& +operator<<(std::ostream& os, const LocalArrayValue& val) { - val.print(os); - return os; + val.print(os); + return os; } template -bool sfn_value_equal(const T* lhs, const T* rhs) +bool +sfn_value_equal(const T *lhs, const T *rhs) { if (lhs) { - if (!rhs) return - false; - if ( !lhs->equal_to(*rhs)) + if (!rhs) + return false; + if (!lhs->equal_to(*rhs)) return false; } else { if (rhs) @@ -465,23 +496,22 @@ bool sfn_value_equal(const T* lhs, const T* rhs) class RegisterVisitor { public: - virtual void visit(Register& value) = 0; - virtual void visit(LocalArray& value) = 0; - virtual void visit(LocalArrayValue& value) = 0; - virtual void visit(UniformValue& value) = 0; - virtual void visit(LiteralConstant& value) = 0; - virtual void visit(InlineConstant& value) = 0; + virtual void visit(Register& value) = 0; + virtual void visit(LocalArray& value) = 0; + virtual void visit(LocalArrayValue& value) = 0; + virtual void visit(UniformValue& value) = 0; + virtual void visit(LiteralConstant& value) = 0; + virtual void visit(InlineConstant& value) = 0; }; class ConstRegisterVisitor { public: - virtual void visit(const Register& value) = 0; - virtual void visit(const LocalArray& value) = 0; - virtual void visit(const LocalArrayValue& value) = 0; - virtual void visit(const UniformValue& value) = 0; - virtual void visit(const LiteralConstant& value) = 0; - virtual void visit(const InlineConstant& value) = 0; + virtual void visit(const Register& value) = 0; + virtual void visit(const LocalArray& value) = 0; + virtual void visit(const LocalArrayValue& value) = 0; + virtual void visit(const UniformValue& value) = 0; + virtual void visit(const LiteralConstant& value) = 0; + virtual void visit(const InlineConstant& value) = 0; }; -} - +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/tests/meson.build b/src/gallium/drivers/r600/sfn/tests/meson.build index d9d19f3c856..8bda27188bc 100644 --- a/src/gallium/drivers/r600/sfn/tests/meson.build +++ b/src/gallium/drivers/r600/sfn/tests/meson.build @@ -35,4 +35,3 @@ if with_tests ) endforeach endif - diff --git a/src/gallium/drivers/r600/sfn/tests/sfn_instr_test.cpp b/src/gallium/drivers/r600/sfn/tests/sfn_instr_test.cpp index cd78552c628..3539d488b47 100644 --- a/src/gallium/drivers/r600/sfn/tests/sfn_instr_test.cpp +++ b/src/gallium/drivers/r600/sfn/tests/sfn_instr_test.cpp @@ -11,22 +11,15 @@ using namespace r600; using std::vector; -class InstrTest : public ::testing::Test -{ - void SetUp() override { - init_pool(); - } +class InstrTest : public ::testing::Test { + void SetUp() override { init_pool(); } + + void TearDown() override { release_pool(); } - void TearDown() override { - release_pool(); - } protected: - void check(const Instr& lhs,const Instr& rhs) const { - EXPECT_EQ(lhs, rhs); - } + void check(const Instr& lhs, const Instr& rhs) const { EXPECT_EQ(lhs, rhs); } }; - TEST_F(InstrTest, test_alu_barrier) { AluInstr alu(op0_group_barrier); @@ -39,12 +32,12 @@ TEST_F(InstrTest, test_alu_barrier) EXPECT_EQ(alu, alu); } - TEST_F(InstrTest, test_alu_uni_op_mov) { AluInstr alu(op1_mov, - new Register( 128, 2, pin_none), - new Register( 129, 0, pin_chan), {alu_write}); + new Register(128, 2, pin_none), + new Register(129, 0, pin_chan), + {alu_write}); EXPECT_TRUE(alu.has_alu_flag(alu_write)); @@ -81,15 +74,14 @@ TEST_F(InstrTest, test_alu_uni_op_mov) alu.set_alu_flag(alu_src0_neg); EXPECT_TRUE(alu.has_alu_flag(alu_src0_neg)); - } TEST_F(InstrTest, test_alu_op2) { AluInstr alu(op2_add, - new Register( 130, 1, pin_none), - new Register( 129, 2, pin_chan), - new Register( 129, 3, pin_none), + new Register(130, 1, pin_none), + new Register(129, 2, pin_chan), + new Register(129, 3, pin_none), {alu_write, alu_last_instr}); EXPECT_TRUE(alu.has_alu_flag(alu_write)); @@ -138,10 +130,10 @@ TEST_F(InstrTest, test_alu_op2) TEST_F(InstrTest, test_alu_op3) { AluInstr alu(op3_cnde, - new Register( 130, 1, pin_none), - new Register( 129, 2, pin_chan), - new Register( 129, 3, pin_none), - new Register( 131, 1, pin_none), + new Register(130, 1, pin_none), + new Register(129, 2, pin_chan), + new Register(129, 3, pin_none), + new Register(131, 1, pin_none), {alu_write, alu_last_instr}); EXPECT_TRUE(alu.has_alu_flag(alu_write)); @@ -187,14 +179,13 @@ TEST_F(InstrTest, test_alu_op3) TEST_F(InstrTest, test_alu_op1_comp) { - auto r128z = new Register( 128, 2, pin_none); - auto r128zc = new Register( 128, 2, pin_chan); - auto r128y = new Register( 128, 1, pin_none); - auto r129x = new Register( 129, 0, pin_none); - auto r129xc = new Register( 129, 0, pin_chan); - auto r129y = new Register( 129, 1, pin_none); - auto r130x = new Register( 130, 0, pin_none); - + auto r128z = new Register(128, 2, pin_none); + auto r128zc = new Register(128, 2, pin_chan); + auto r128y = new Register(128, 1, pin_none); + auto r129x = new Register(129, 0, pin_none); + auto r129xc = new Register(129, 0, pin_chan); + auto r129y = new Register(129, 1, pin_none); + auto r130x = new Register(130, 0, pin_none); AluInstr alu1(op1_mov, r128z, r129x, {alu_write}); EXPECT_NE(alu1, AluInstr(op1_mov, r128y, r129x, {alu_write})); @@ -210,46 +201,63 @@ TEST_F(InstrTest, test_alu_op1_comp) TEST_F(InstrTest, test_alu_op2_comp) { - auto r128x = new Register( 128, 0, pin_none); - auto r128y = new Register( 128, 1, pin_none); - auto r128z = new Register( 128, 2, pin_none); + auto r128x = new Register(128, 0, pin_none); + auto r128y = new Register(128, 1, pin_none); + auto r128z = new Register(128, 2, pin_none); AluInstr alu1(op2_add, r128z, r128x, r128y, {alu_write}); - EXPECT_NE(alu1, AluInstr(op2_add, r128z, r128x, new Register( 129, 2, pin_none), {alu_write})); - EXPECT_NE(alu1, AluInstr(op2_add, r128z, r128x, new Register( 128, 0, pin_none), {alu_write})); - EXPECT_NE(alu1, AluInstr(op2_add, r128z, r128x, new Register( 128, 1, pin_chan), {alu_write})); + EXPECT_NE( + alu1, AluInstr(op2_add, r128z, r128x, new Register(129, 2, pin_none), {alu_write})); + EXPECT_NE( + alu1, AluInstr(op2_add, r128z, r128x, new Register(128, 0, pin_none), {alu_write})); + EXPECT_NE( + alu1, AluInstr(op2_add, r128z, r128x, new Register(128, 1, pin_chan), {alu_write})); } TEST_F(InstrTest, test_alu_op3_comp) { - auto r128x = new Register( 128, 0, pin_none); - auto r128y = new Register( 128, 1, pin_none); - auto r128z = new Register( 128, 2, pin_none); + auto r128x = new Register(128, 0, pin_none); + auto r128y = new Register(128, 1, pin_none); + auto r128z = new Register(128, 2, pin_none); AluInstr alu1(op3_muladd, r128z, r128x, r128y, r128y, {alu_write}); - EXPECT_NE(alu1, AluInstr(op3_muladd, r128z, r128x, r128y, new Register( 129, 2, pin_none), {alu_write})); - EXPECT_NE(alu1, AluInstr(op3_muladd, r128z, r128x, r128y, new Register( 128, 0, pin_none), {alu_write})); - EXPECT_NE(alu1, AluInstr(op3_muladd, r128z, r128x, r128y, new Register( 128, 1, pin_chan), {alu_write})); + EXPECT_NE( + alu1, + AluInstr( + op3_muladd, r128z, r128x, r128y, new Register(129, 2, pin_none), {alu_write})); + EXPECT_NE( + alu1, + AluInstr( + op3_muladd, r128z, r128x, r128y, new Register(128, 0, pin_none), {alu_write})); + EXPECT_NE( + alu1, + AluInstr( + op3_muladd, r128z, r128x, r128y, new Register(128, 1, pin_chan), {alu_write})); } TEST_F(InstrTest, test_alu_op3_ne) { - auto R130x = new Register( 130, 0, pin_none); - auto R130y = new Register( 130, 1, pin_none); - auto R130z = new Register( 130, 2, pin_none); - auto R131z = new Register( 131, 2, pin_none); - auto R131w = new Register( 131, 3, pin_none); + auto R130x = new Register(130, 0, pin_none); + auto R130y = new Register(130, 1, pin_none); + auto R130z = new Register(130, 2, pin_none); + auto R131z = new Register(131, 2, pin_none); + auto R131w = new Register(131, 3, pin_none); AluInstr alu(op3_cnde, R130x, R130y, R131z, R131w, {alu_write, alu_last_instr}); - EXPECT_NE(alu, AluInstr(op3_muladd, R130x, R130y, R131z, R131w, {alu_write, alu_last_instr})); + EXPECT_NE( + alu, AluInstr(op3_muladd, R130x, R130y, R131z, R131w, {alu_write, alu_last_instr})); - EXPECT_NE(alu, AluInstr(op3_cnde, R130z, R130y, R131z, R131w, {alu_write, alu_last_instr})); - EXPECT_NE(alu, AluInstr(op3_cnde, R130x, R130z, R131z, R131w, {alu_write, alu_last_instr})); - EXPECT_NE(alu, AluInstr(op3_cnde, R130x, R130y, R130z, R131w, {alu_write, alu_last_instr})); - EXPECT_NE(alu, AluInstr(op3_cnde, R130x, R130y, R131z, R130z, {alu_write, alu_last_instr})); + EXPECT_NE(alu, + AluInstr(op3_cnde, R130z, R130y, R131z, R131w, {alu_write, alu_last_instr})); + EXPECT_NE(alu, + AluInstr(op3_cnde, R130x, R130z, R131z, R131w, {alu_write, alu_last_instr})); + EXPECT_NE(alu, + AluInstr(op3_cnde, R130x, R130y, R130z, R131w, {alu_write, alu_last_instr})); + EXPECT_NE(alu, + AluInstr(op3_cnde, R130x, R130y, R131z, R130z, {alu_write, alu_last_instr})); EXPECT_NE(alu, AluInstr(op3_cnde, R130x, R130y, R131z, R131w, {alu_write})); AluInstr alu_cf_changes = alu; @@ -263,12 +271,11 @@ TEST_F(InstrTest, test_alu_op3_ne) EXPECT_NE(alu, alu_bs_changes); }; - TEST_F(InstrTest, test_alu_op1_ne) { - auto R130x = new Register( 130, 0, pin_none); - auto R130y = new Register( 130, 1, pin_none); - auto R130z = new Register( 130, 2, pin_none); + auto R130x = new Register(130, 0, pin_none); + auto R130y = new Register(130, 1, pin_none); + auto R130z = new Register(130, 2, pin_none); AluInstr alu(op1_mov, R130x, R130y, {alu_write, alu_last_instr}); @@ -291,23 +298,22 @@ TEST_F(InstrTest, test_alu_op1_ne) TEST_F(InstrTest, test_alu_dot4_grouped) { - auto R130x = new Register( 130, 0, pin_none); - auto R130y = new Register( 130, 1, pin_none); - auto R130z = new Register( 130, 2, pin_none); - auto R130w = new Register( 130, 3, pin_none); + auto R130x = new Register(130, 0, pin_none); + auto R130y = new Register(130, 1, pin_none); + auto R130z = new Register(130, 2, pin_none); + auto R130w = new Register(130, 3, pin_none); - auto R131x = new Register( 131, 0, pin_none); - auto R131y = new Register( 131, 1, pin_none); - auto R131z = new Register( 131, 2, pin_none); - auto R131w = new Register( 131, 3, pin_none); + auto R131x = new Register(131, 0, pin_none); + auto R131y = new Register(131, 1, pin_none); + auto R131z = new Register(131, 2, pin_none); + auto R131w = new Register(131, 3, pin_none); - auto R132x = new Register( 132, 0, pin_chan); - auto R132y = new Register( 132, 1, pin_chan); - auto R132z = new Register( 132, 2, pin_chan); - auto R132w = new Register( 132, 3, pin_chan); + auto R132x = new Register(132, 0, pin_chan); + auto R132y = new Register(132, 1, pin_chan); + auto R132z = new Register(132, 2, pin_chan); + auto R132w = new Register(132, 3, pin_chan); - AluInstr::SrcValues src({R130x, R130y, R130z, R130w, - R131x, R131y, R131z, R131w}); + AluInstr::SrcValues src({R130x, R130y, R130z, R130w, R131x, R131y, R131z, R131w}); AluInstr alu(op2_dot4_ieee, R132x, src, {alu_write, alu_last_instr}, 4); @@ -342,61 +348,60 @@ TEST_F(InstrTest, test_alu_dot4_grouped) EXPECT_EQ(i, group->end()); }; - - - #ifdef __cpp_exceptions TEST_F(InstrTest, test_alu_wrong_source_count) { EXPECT_THROW(AluInstr(op3_cnde, - new Register( 130, 1, pin_none), - new Register( 129, 2, pin_chan), - new Register( 129, 3, pin_none), - {alu_write, alu_last_instr}), std::invalid_argument); + new Register(130, 1, pin_none), + new Register(129, 2, pin_chan), + new Register(129, 3, pin_none), + {alu_write, alu_last_instr}), + std::invalid_argument); EXPECT_THROW(AluInstr(op3_cnde, - new Register( 130, 1, pin_none), - new Register( 129, 2, pin_chan), - {alu_write, alu_last_instr}), std::invalid_argument); + new Register(130, 1, pin_none), + new Register(129, 2, pin_chan), + {alu_write, alu_last_instr}), + std::invalid_argument); EXPECT_THROW(AluInstr(op1_mov, - new Register( 130, 1, pin_none), - new Register( 129, 2, pin_chan), - new Register( 129, 2, pin_chan), - {alu_write, alu_last_instr}), std::invalid_argument); + new Register(130, 1, pin_none), + new Register(129, 2, pin_chan), + new Register(129, 2, pin_chan), + {alu_write, alu_last_instr}), + std::invalid_argument); EXPECT_THROW(AluInstr(op2_add, - new Register( 130, 1, pin_none), - new Register( 129, 2, pin_chan), - {alu_write, alu_last_instr}), std::invalid_argument); + new Register(130, 1, pin_none), + new Register(129, 2, pin_chan), + {alu_write, alu_last_instr}), + std::invalid_argument); EXPECT_THROW(AluInstr(op2_add, - new Register( 130, 1, pin_none), - new Register( 129, 2, pin_chan), - new Register( 129, 2, pin_chan), - new Register( 129, 2, pin_chan), - {alu_write, alu_last_instr}), std::invalid_argument); + new Register(130, 1, pin_none), + new Register(129, 2, pin_chan), + new Register(129, 2, pin_chan), + new Register(129, 2, pin_chan), + {alu_write, alu_last_instr}), + std::invalid_argument); } TEST_F(InstrTest, test_alu_write_no_dest) { EXPECT_THROW(AluInstr(op2_add, nullptr, - new Register( 129, 2, pin_chan), - new Register( 129, 2, pin_chan), - {alu_write, alu_last_instr}), std::invalid_argument); + new Register(129, 2, pin_chan), + new Register(129, 2, pin_chan), + {alu_write, alu_last_instr}), + std::invalid_argument); } #endif TEST_F(InstrTest, test_tex_basic) { - TexInstr tex(TexInstr::sample, - RegisterVec4(129), - {0,1,2,3}, - RegisterVec4(130), - 1, - 17); + TexInstr tex( + TexInstr::sample, RegisterVec4(129), {0, 1, 2, 3}, RegisterVec4(130), 1, 17); EXPECT_EQ(tex.opcode(), TexInstr::sample); @@ -464,17 +469,12 @@ TEST_F(InstrTest, test_tex_basic) EXPECT_EQ(tex.get_offset(0), 4); EXPECT_EQ(tex.get_offset(1), -2); EXPECT_EQ(tex.get_offset(2), 6); - } TEST_F(InstrTest, test_tex_gather4) { - TexInstr tex(TexInstr::gather4, - RegisterVec4(131), - {0,1,2,3}, - RegisterVec4(132), - 2, - 19); + TexInstr tex( + TexInstr::gather4, RegisterVec4(131), {0, 1, 2, 3}, RegisterVec4(132), 2, 19); EXPECT_EQ(tex.opcode(), TexInstr::gather4); @@ -500,36 +500,81 @@ TEST_F(InstrTest, test_tex_gather4) tex.set_gather_comp(2); EXPECT_EQ(tex.inst_mode(), 2); - } TEST_F(InstrTest, test_tex_neq) { - TexInstr tex_ref(TexInstr::sample, - RegisterVec4(129), - {0,1,2,3}, - RegisterVec4(130), - 1, - 17); + TexInstr tex_ref( + TexInstr::sample, RegisterVec4(129), {0, 1, 2, 3}, RegisterVec4(130), 1, 17); EXPECT_EQ(tex_ref, tex_ref); + EXPECT_NE( + tex_ref, + TexInstr( + TexInstr::sample_c, RegisterVec4(129), {0, 1, 2, 3}, RegisterVec4(130), 1, 17)); + EXPECT_NE( + tex_ref, + TexInstr( + TexInstr::sample, RegisterVec4(130), {0, 1, 2, 3}, RegisterVec4(130), 1, 17)); + EXPECT_NE( + tex_ref, + TexInstr( + TexInstr::sample, RegisterVec4(130), {0, 1, 2, 3}, RegisterVec4(130), 1, 17)); - EXPECT_NE(tex_ref, TexInstr(TexInstr::sample_c, RegisterVec4(129), {0,1,2,3}, RegisterVec4(130), 1, 17)); - EXPECT_NE(tex_ref, TexInstr(TexInstr::sample, RegisterVec4(130), {0,1,2,3}, RegisterVec4(130), 1, 17)); - EXPECT_NE(tex_ref, TexInstr(TexInstr::sample, RegisterVec4(130), {0,1,2,3}, RegisterVec4(130), 1, 17)); + EXPECT_NE( + tex_ref, + TexInstr( + TexInstr::sample, RegisterVec4(129), {7, 1, 2, 3}, RegisterVec4(130), 1, 17)); + EXPECT_NE( + tex_ref, + TexInstr( + TexInstr::sample, RegisterVec4(129), {0, 7, 2, 3}, RegisterVec4(130), 1, 17)); + EXPECT_NE( + tex_ref, + TexInstr( + TexInstr::sample, RegisterVec4(129), {0, 1, 7, 3}, RegisterVec4(130), 1, 17)); + EXPECT_NE( + tex_ref, + TexInstr( + TexInstr::sample, RegisterVec4(129), {0, 1, 2, 7}, RegisterVec4(130), 1, 17)); - EXPECT_NE(tex_ref, TexInstr(TexInstr::sample, RegisterVec4(129), {7,1,2,3}, RegisterVec4(130), 1, 17)); - EXPECT_NE(tex_ref, TexInstr(TexInstr::sample, RegisterVec4(129), {0,7,2,3}, RegisterVec4(130), 1, 17)); - EXPECT_NE(tex_ref, TexInstr(TexInstr::sample, RegisterVec4(129), {0,1,7,3}, RegisterVec4(130), 1, 17)); - EXPECT_NE(tex_ref, TexInstr(TexInstr::sample, RegisterVec4(129), {0,1,2,7}, RegisterVec4(130), 1, 17)); + EXPECT_NE(tex_ref, + TexInstr(TexInstr::sample, + RegisterVec4(129), + {0, 1, 2, 3}, + RegisterVec4(130, false, {7, 1, 2, 3}), + 1, + 17)); + EXPECT_NE(tex_ref, + TexInstr(TexInstr::sample, + RegisterVec4(129), + {0, 1, 2, 3}, + RegisterVec4(130, false, {0, 7, 2, 3}), + 1, + 17)); + EXPECT_NE(tex_ref, + TexInstr(TexInstr::sample, + RegisterVec4(129), + {0, 1, 2, 3}, + RegisterVec4(130, false, {0, 1, 7, 3}), + 1, + 17)); + EXPECT_NE(tex_ref, + TexInstr(TexInstr::sample, + RegisterVec4(129), + {0, 1, 2, 3}, + RegisterVec4(130, false, {0, 1, 2, 7}), + 1, + 17)); - EXPECT_NE(tex_ref, TexInstr(TexInstr::sample, RegisterVec4(129), {0,1,2,3}, RegisterVec4(130, false, {7,1,2,3}), 1, 17)); - EXPECT_NE(tex_ref, TexInstr(TexInstr::sample, RegisterVec4(129), {0,1,2,3}, RegisterVec4(130, false, {0,7,2,3}), 1, 17)); - EXPECT_NE(tex_ref, TexInstr(TexInstr::sample, RegisterVec4(129), {0,1,2,3}, RegisterVec4(130, false, {0,1,7,3}), 1, 17)); - EXPECT_NE(tex_ref, TexInstr(TexInstr::sample, RegisterVec4(129), {0,1,2,3}, RegisterVec4(130, false, {0,1,2,7}), 1, 17)); - - EXPECT_NE(tex_ref, TexInstr(TexInstr::sample, RegisterVec4(129), {0,1,2,3}, RegisterVec4(130), 2, 17)); - EXPECT_NE(tex_ref, TexInstr(TexInstr::sample, RegisterVec4(129), {0,1,2,3}, RegisterVec4(130), 1, 18)); + EXPECT_NE( + tex_ref, + TexInstr( + TexInstr::sample, RegisterVec4(129), {0, 1, 2, 3}, RegisterVec4(130), 2, 17)); + EXPECT_NE( + tex_ref, + TexInstr( + TexInstr::sample, RegisterVec4(129), {0, 1, 2, 3}, RegisterVec4(130), 1, 18)); /* auto tex_with_sampler_offset = tex_ref; @@ -540,9 +585,11 @@ TEST_F(InstrTest, test_tex_neq) EXPECT_EQ(tex_ref, tex_cmp1); tex_cmp1.set_tex_flag(TexInstr::x_unnormalized); EXPECT_NE(tex_ref, tex_cmp1); - auto tex_cmp2 = tex_ref; tex_cmp2.set_tex_flag(TexInstr::y_unnormalized); EXPECT_NE(tex_ref, tex_cmp2); - auto tex_cmp3 = tex_ref; tex_cmp3.set_tex_flag(TexInstr::z_unnormalized); EXPECT_NE(tex_ref, tex_cmp3); - auto tex_cmp4 = tex_ref; tex_cmp4.set_tex_flag(TexInstr::w_unnormalized); EXPECT_NE(tex_ref, tex_cmp4); + auto tex_cmp2 = tex_ref; tex_cmp2.set_tex_flag(TexInstr::y_unnormalized); + EXPECT_NE(tex_ref, tex_cmp2); auto tex_cmp3 = tex_ref; + tex_cmp3.set_tex_flag(TexInstr::z_unnormalized); EXPECT_NE(tex_ref, tex_cmp3); auto + tex_cmp4 = tex_ref; tex_cmp4.set_tex_flag(TexInstr::w_unnormalized); EXPECT_NE(tex_ref, + tex_cmp4); for (int i = 0; i < 3; ++i) { auto tex_ofs = tex_ref; @@ -563,7 +610,6 @@ TEST_F(InstrTest, test_tex_neq) EXPECT_NE(tex_ref, tex_cmp_mode);*/ } - TEST_F(InstrTest, test_export_basic) { ExportInstr exp0(ExportInstr::param, 60, RegisterVec4(200)); @@ -592,19 +638,22 @@ TEST_F(InstrTest, test_export_basic) EXPECT_EQ(exp4.value(), RegisterVec4(201)); EXPECT_NE(exp0, exp4); - EXPECT_NE(exp0, ExportInstr(ExportInstr::param, 60, RegisterVec4(200, false, {7,1,2,3}))); - EXPECT_NE(exp0, ExportInstr(ExportInstr::param, 60, RegisterVec4(200, false, {0,7,2,3}))); - EXPECT_NE(exp0, ExportInstr(ExportInstr::param, 60, RegisterVec4(200, false, {0,1,7,3}))); - EXPECT_NE(exp0, ExportInstr(ExportInstr::param, 60, RegisterVec4(200, false, {0,1,2,7}))); + EXPECT_NE(exp0, + ExportInstr(ExportInstr::param, 60, RegisterVec4(200, false, {7, 1, 2, 3}))); + EXPECT_NE(exp0, + ExportInstr(ExportInstr::param, 60, RegisterVec4(200, false, {0, 7, 2, 3}))); + EXPECT_NE(exp0, + ExportInstr(ExportInstr::param, 60, RegisterVec4(200, false, {0, 1, 7, 3}))); + EXPECT_NE(exp0, + ExportInstr(ExportInstr::param, 60, RegisterVec4(200, false, {0, 1, 2, 7}))); } - TEST_F(InstrTest, test_fetch_basic) { FetchInstr fetch(vc_fetch, RegisterVec4(200), - {0,2,1,3}, - new Register( 201, 2, pin_none), + {0, 2, 1, 3}, + new Register(201, 2, pin_none), 0, vertex_data, fmt_8, @@ -613,8 +662,6 @@ TEST_F(InstrTest, test_fetch_basic) 1, nullptr); - - EXPECT_EQ(fetch.opcode(), vc_fetch); EXPECT_EQ(fetch.dst(), RegisterVec4(200)); EXPECT_EQ(fetch.dest_swizzle(0), 0); @@ -642,80 +689,155 @@ TEST_F(InstrTest, test_fetch_basic) EXPECT_FALSE(fetch.has_fetch_flag(static_cast(i))); } - EXPECT_NE(fetch, FetchInstr(vc_get_buf_resinfo, RegisterVec4(200),{0,2,1,3}, - new Register( 201, 2, pin_none), - 0, vertex_data, - fmt_8, vtx_nf_norm, vtx_es_none, 1, nullptr)); + EXPECT_NE(fetch, + FetchInstr(vc_get_buf_resinfo, + RegisterVec4(200), + {0, 2, 1, 3}, + new Register(201, 2, pin_none), + 0, + vertex_data, + fmt_8, + vtx_nf_norm, + vtx_es_none, + 1, + nullptr)); - EXPECT_NE(fetch, FetchInstr(vc_fetch, RegisterVec4(201),{0,2,1,3}, - new Register( 201, 2, pin_none), - 0, vertex_data, - fmt_8, vtx_nf_norm, vtx_es_none, - 1, nullptr)); + EXPECT_NE(fetch, + FetchInstr(vc_fetch, + RegisterVec4(201), + {0, 2, 1, 3}, + new Register(201, 2, pin_none), + 0, + vertex_data, + fmt_8, + vtx_nf_norm, + vtx_es_none, + 1, + nullptr)); + EXPECT_NE(fetch, + FetchInstr(vc_fetch, + RegisterVec4(200), + {1, 2, 0, 3}, + new Register(201, 2, pin_none), + 0, + vertex_data, + fmt_8, + vtx_nf_norm, + vtx_es_none, + 1, + nullptr)); - EXPECT_NE(fetch, FetchInstr(vc_fetch, RegisterVec4(200),{1,2,0,3}, - new Register( 201, 2, pin_none), - 0, vertex_data, - fmt_8, vtx_nf_norm, vtx_es_none, - 1, nullptr)); + EXPECT_NE(fetch, + FetchInstr(vc_fetch, + RegisterVec4(200), + {0, 2, 1, 3}, + new Register(200, 2, pin_none), + 0, + vertex_data, + fmt_8, + vtx_nf_norm, + vtx_es_none, + 1, + nullptr)); + EXPECT_NE(fetch, + FetchInstr(vc_fetch, + RegisterVec4(200), + {0, 2, 1, 3}, + new Register(201, 2, pin_none), + 8, + vertex_data, + fmt_8, + vtx_nf_norm, + vtx_es_none, + 1, + nullptr)); - EXPECT_NE(fetch, FetchInstr(vc_fetch, RegisterVec4(200),{0,2,1,3}, - new Register( 200, 2, pin_none), - 0, vertex_data, - fmt_8, vtx_nf_norm, vtx_es_none, - 1, nullptr)); + EXPECT_NE(fetch, + FetchInstr(vc_fetch, + RegisterVec4(200), + {0, 2, 1, 3}, + new Register(201, 2, pin_none), + 0, + instance_data, + fmt_8, + vtx_nf_norm, + vtx_es_none, + 1, + nullptr)); - EXPECT_NE(fetch, FetchInstr(vc_fetch, RegisterVec4(200),{0,2,1,3}, - new Register( 201, 2, pin_none), - 8, vertex_data, - fmt_8, vtx_nf_norm, vtx_es_none, - 1, nullptr)); + EXPECT_NE(fetch, + FetchInstr(vc_fetch, + RegisterVec4(200), + {0, 2, 1, 3}, + new Register(201, 2, pin_none), + 0, + vertex_data, + fmt_8_8, + vtx_nf_norm, + vtx_es_none, + 1, + nullptr)); + EXPECT_NE(fetch, + FetchInstr(vc_fetch, + RegisterVec4(200), + {0, 2, 1, 3}, + new Register(201, 2, pin_none), + 0, + vertex_data, + fmt_8, + vtx_nf_int, + vtx_es_none, + 1, + nullptr)); - EXPECT_NE(fetch, FetchInstr(vc_fetch, RegisterVec4(200),{0,2,1,3}, - new Register( 201, 2, pin_none), - 0, instance_data, - fmt_8, vtx_nf_norm, vtx_es_none, - 1, nullptr)); + EXPECT_NE(fetch, + FetchInstr(vc_fetch, + RegisterVec4(200), + {0, 2, 1, 3}, + new Register(201, 2, pin_none), + 0, + vertex_data, + fmt_8, + vtx_nf_norm, + vtx_es_8in16, + 1, + nullptr)); + EXPECT_NE(fetch, + FetchInstr(vc_fetch, + RegisterVec4(200), + {0, 2, 1, 3}, + new Register(201, 2, pin_none), + 0, + vertex_data, + fmt_8, + vtx_nf_norm, + vtx_es_none, + 2, + nullptr)); - EXPECT_NE(fetch, FetchInstr(vc_fetch, RegisterVec4(200),{0,2,1,3}, - new Register( 201, 2, pin_none), - 0, vertex_data, - fmt_8_8, vtx_nf_norm, vtx_es_none, - 1, nullptr)); - - EXPECT_NE(fetch, FetchInstr(vc_fetch, RegisterVec4(200),{0,2,1,3}, - new Register( 201, 2, pin_none), - 0, vertex_data, - fmt_8, vtx_nf_int, vtx_es_none, - 1, nullptr)); - - EXPECT_NE(fetch, FetchInstr(vc_fetch, RegisterVec4(200),{0,2,1,3}, - new Register( 201, 2, pin_none), - 0, vertex_data, - fmt_8, vtx_nf_norm, vtx_es_8in16, - 1, nullptr)); - - EXPECT_NE(fetch, FetchInstr(vc_fetch, RegisterVec4(200),{0,2,1,3}, - new Register( 201, 2, pin_none), - 0, vertex_data, - fmt_8, vtx_nf_norm, vtx_es_none, - 2, nullptr)); - - EXPECT_NE(fetch, FetchInstr(vc_fetch, RegisterVec4(200),{0,2,1,3}, - new Register( 201, 2, pin_none), - 0, vertex_data, - fmt_8, vtx_nf_norm, vtx_es_none, - 1, new Register( 1000, 0, pin_none))); + EXPECT_NE(fetch, + FetchInstr(vc_fetch, + RegisterVec4(200), + {0, 2, 1, 3}, + new Register(201, 2, pin_none), + 0, + vertex_data, + fmt_8, + vtx_nf_norm, + vtx_es_none, + 1, + new Register(1000, 0, pin_none))); auto fetch1 = fetch; fetch1.set_mfc(31); EXPECT_NE(fetch1, fetch); EXPECT_EQ(fetch1.mega_fetch_count(), 31); - EXPECT_TRUE(fetch1.has_fetch_flag(static_cast(FetchInstr::is_mega_fetch))); + EXPECT_TRUE( + fetch1.has_fetch_flag(static_cast(FetchInstr::is_mega_fetch))); auto fetch2 = fetch; fetch2.set_array_base(32); @@ -737,16 +859,15 @@ TEST_F(InstrTest, test_fetch_basic2) { FetchInstr fetch(vc_get_buf_resinfo, RegisterVec4(201), - {0,1,3,4}, - new Register( 202, 3, pin_none), + {0, 1, 3, 4}, + new Register(202, 3, pin_none), 1, no_index_offset, fmt_32_32, vtx_nf_int, vtx_es_8in16, 3, - new Register( 300, 1, pin_none)); - + new Register(300, 1, pin_none)); EXPECT_EQ(fetch.opcode(), vc_get_buf_resinfo); EXPECT_EQ(fetch.dst(), RegisterVec4(201)); @@ -779,7 +900,8 @@ TEST_F(InstrTest, test_fetch_basic2) fetch1.set_mfc(15); EXPECT_NE(fetch1, fetch); EXPECT_EQ(fetch1.mega_fetch_count(), 15); - EXPECT_TRUE(fetch1.has_fetch_flag(static_cast(FetchInstr::is_mega_fetch))); + EXPECT_TRUE( + fetch1.has_fetch_flag(static_cast(FetchInstr::is_mega_fetch))); auto fetch2 = fetch; fetch2.set_array_base(128); diff --git a/src/gallium/drivers/r600/sfn/tests/sfn_instrfromstring_test.cpp b/src/gallium/drivers/r600/sfn/tests/sfn_instrfromstring_test.cpp index 4d8124c92e0..470a5307637 100644 --- a/src/gallium/drivers/r600/sfn/tests/sfn_instrfromstring_test.cpp +++ b/src/gallium/drivers/r600/sfn/tests/sfn_instrfromstring_test.cpp @@ -1,12 +1,11 @@ -#include "../sfn_instrfactory.h" - #include "../sfn_instr_alu.h" #include "../sfn_instr_export.h" #include "../sfn_instr_fetch.h" #include "../sfn_instr_lds.h" #include "../sfn_instr_mem.h" #include "../sfn_instr_tex.h" +#include "../sfn_instrfactory.h" #include "gtest/gtest.h" #include @@ -17,8 +16,7 @@ using std::istringstream; using std::ostringstream; using std::string; -class TestInstrFromString : public ::testing::Test -{ +class TestInstrFromString : public ::testing::Test { public: TestInstrFromString(); @@ -35,7 +33,6 @@ protected: void TearDown() override; InstrFactory *m_instr_factory{nullptr}; - }; TEST_F(TestInstrFromString, test_alu_mov) @@ -43,8 +40,8 @@ TEST_F(TestInstrFromString, test_alu_mov) add_dest_from_string("R1999.x"); AluInstr expect(op1_mov, - new Register( 2000, 1, pin_none), - new Register( 1999, 0, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 0, pin_none), {alu_write, alu_last_instr}); check("ALU MOV R2000.y : R1999.x {WL}", expect); @@ -54,30 +51,27 @@ TEST_F(TestInstrFromString, test_alu_lds_read_ret) { add_dest_from_string("R1999.x"); - AluInstr expect(DS_OP_READ_RET, - {new Register( 1999, 0, pin_none)}, {}); + AluInstr expect(DS_OP_READ_RET, {new Register(1999, 0, pin_none)}, {}); check("ALU LDS READ_RET __.x : R1999.x {}", expect); } - TEST_F(TestInstrFromString, test_alu_mov_literal) { AluInstr expect(op1_mov, - new Register( 2000, 1, pin_none), - new LiteralConstant( 0x10), + new Register(2000, 1, pin_none), + new LiteralConstant(0x10), {alu_write, alu_last_instr}); check("ALU MOV R2000.y : L[0x10] {WL}", expect); } - TEST_F(TestInstrFromString, test_alu_mov_neg) { add_dest_from_string("R1999.x"); AluInstr expect(op1_mov, - new Register( 2000, 1, pin_none), - new Register( 1999, 0, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 0, pin_none), {alu_write, alu_last_instr, alu_src0_neg}); check("ALU MOV R2000.y : -R1999.x {WL}", expect); @@ -87,8 +81,8 @@ TEST_F(TestInstrFromString, test_alu_mov_abs) { add_dest_from_string("R1999.x"); AluInstr expect(op1_mov, - new Register( 2000, 1, pin_none), - new Register( 1999, 0, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 0, pin_none), {alu_write, alu_last_instr, alu_src0_abs}); check("ALU MOV R2000.y : |R1999.x| {WL}", expect); @@ -98,8 +92,8 @@ TEST_F(TestInstrFromString, test_alu_mov_neg_abs) { add_dest_from_string("R1999.x"); AluInstr expect(op1_mov, - new Register( 2000, 1, pin_none), - new Register( 1999, 0, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 0, pin_none), {alu_write, alu_src0_neg, alu_src0_abs}); check("ALU MOV R2000.y : -|R1999.x| {W}", expect); } @@ -110,12 +104,11 @@ TEST_F(TestInstrFromString, test_alu_add) add_dest_from_string("R1999.w"); AluInstr expect(op2_add, - new Register( 2000, 1, pin_none), - new Register( 1999, 3, pin_none), - new Register( 1998, 2, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 3, pin_none), + new Register(1998, 2, pin_none), {alu_last_instr}); check("ALU ADD __.y : R1999.w R1998.z {L}", expect); - } TEST_F(TestInstrFromString, test_alu_add_clmap) @@ -123,12 +116,11 @@ TEST_F(TestInstrFromString, test_alu_add_clmap) add_dest_from_string("R1998.z"); add_dest_from_string("R1999.w"); AluInstr expect(op2_add, - new Register( 2000, 1, pin_none), - new Register( 1999, 3, pin_none), - new Register( 1998, 2, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 3, pin_none), + new Register(1998, 2, pin_none), {alu_last_instr, alu_dst_clamp}); check("ALU ADD CLAMP __.y : R1999.w R1998.z {L}", expect); - } TEST_F(TestInstrFromString, test_alu_add_neg2) @@ -136,9 +128,9 @@ TEST_F(TestInstrFromString, test_alu_add_neg2) add_dest_from_string("R1998.z"); add_dest_from_string("R1999.w"); AluInstr expect(op2_add, - new Register( 2000, 1, pin_none), - new Register( 1999, 3, pin_none), - new Register( 1998, 2, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 3, pin_none), + new Register(1998, 2, pin_none), {alu_last_instr, alu_src1_neg}); check("ALU ADD __.y : R1999.w -R1998.z {L}", expect); } @@ -148,9 +140,9 @@ TEST_F(TestInstrFromString, test_alu_sete_update_pref) add_dest_from_string("R1998.z"); add_dest_from_string("R1999.w"); AluInstr expect(op2_sete, - new Register( 2000, 1, pin_none), - new Register( 1999, 3, pin_none), - new Register( 1998, 2, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 3, pin_none), + new Register(1998, 2, pin_none), {alu_last_instr, alu_src1_neg, alu_update_pred}); check("ALU SETE __.y : R1999.w -R1998.z {LP}", expect); } @@ -160,35 +152,33 @@ TEST_F(TestInstrFromString, test_alu_sete_update_pref_empty_dest) add_dest_from_string("R1998.z"); add_dest_from_string("R1999.w"); AluInstr expect(op2_sete, - new Register( 2000, 0, pin_none), - new Register( 1999, 3, pin_none), - new Register( 1998, 2, pin_none), + new Register(2000, 0, pin_none), + new Register(1999, 3, pin_none), + new Register(1998, 2, pin_none), {alu_last_instr, alu_update_pred}); check("ALU SETE __.x : R1999.w R1998.z {LP}", expect); } - TEST_F(TestInstrFromString, test_alu_setne_update_exec) { add_dest_from_string("R1998.z"); add_dest_from_string("R1999.w"); AluInstr expect(op2_setne, - new Register( 2000, 1, pin_none), - new Register( 1999, 3, pin_none), - new Register( 1998, 2, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 3, pin_none), + new Register(1998, 2, pin_none), {alu_last_instr, alu_src1_neg, alu_update_exec}); check("ALU SETNE __.y : R1999.w -R1998.z {LE}", expect); } - TEST_F(TestInstrFromString, test_alu_add_abs2) { add_dest_from_string("R1998.z"); add_dest_from_string("R1999.w"); AluInstr expect(op2_add, - new Register( 2000, 1, pin_none), - new Register( 1999, 3, pin_none), - new Register( 1998, 2, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 3, pin_none), + new Register(1998, 2, pin_none), {alu_write, alu_last_instr, alu_src1_abs}); check("ALU ADD R2000.y : R1999.w |R1998.z| {WL}", expect); } @@ -198,24 +188,23 @@ TEST_F(TestInstrFromString, test_alu_add_abs2_neg2) add_dest_from_string("R1998.z"); add_dest_from_string("R1999.w"); AluInstr expect(op2_add, - new Register( 2000, 1, pin_none), - new Register( 1999, 3, pin_none), - new Register( 1998, 2, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 3, pin_none), + new Register(1998, 2, pin_none), {alu_write, alu_last_instr, alu_src1_abs, alu_src1_neg}); check("ALU ADD R2000.y : R1999.w -|R1998.z| {WL}", expect); } - TEST_F(TestInstrFromString, test_alu_muladd) { add_dest_from_string("R1998.z"); add_dest_from_string("R1999.w"); add_dest_from_string("R2000.y"); AluInstr expect(op3_muladd_ieee, - new Register( 2000, 1, pin_none), - new Register( 1999, 3, pin_none), - new Register( 1998, 2, pin_none), - new Register( 2000, 1, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 3, pin_none), + new Register(1998, 2, pin_none), + new Register(2000, 1, pin_none), {alu_write, alu_last_instr}); check("ALU MULADD_IEEE R2000.y : R1999.w R1998.z R2000.y {WL}", expect); } @@ -226,24 +215,23 @@ TEST_F(TestInstrFromString, test_alu_muladd_neg3) add_dest_from_string("R1999.w"); add_dest_from_string("R2000.y"); AluInstr expect(op3_muladd_ieee, - new Register( 2000, 1, pin_none), - new Register( 1999, 3, pin_none), - new Register( 1998, 2, pin_none), - new Register( 2000, 1, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 3, pin_none), + new Register(1998, 2, pin_none), + new Register(2000, 1, pin_none), {alu_last_instr, alu_src2_neg}); check("ALU MULADD_IEEE __.y : R1999.w R1998.z -R2000.y {L}", expect); } - TEST_F(TestInstrFromString, test_alu_mov_bs) { add_dest_from_string("R1999.x"); for (auto& [expect_bs, str] : AluInstr::bank_swizzle_map) { - auto init = std::string("ALU MOV R2000.y : R1999.x {WL} ") + str; + auto init = std::string("ALU MOV R2000.y : R1999.x {WL} ") + str; AluInstr expect(op1_mov, - new Register( 2000, 1, pin_none), - new Register( 1999, 0, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 0, pin_none), {alu_write, alu_last_instr}); expect.set_bank_swizzle(expect_bs); @@ -261,19 +249,21 @@ TEST_F(TestInstrFromString, test_alu_dot4_ieee) add_dest_from_string("R198.y"); add_dest_from_string("R198.z"); add_dest_from_string("R198.w"); - auto init = std::string("ALU DOT4_IEEE R2000.y : R199.x R198.w + R199.y R198.z + R199.z R198.y + R199.w R198.x {WL}"); + auto init = std::string("ALU DOT4_IEEE R2000.y : R199.x R198.w + R199.y R198.z + " + "R199.z R198.y + R199.w R198.x {WL}"); AluInstr expect(op2_dot4_ieee, - new Register( 2000, 1, pin_none), - {new Register( 199, 0, pin_none), - new Register( 198, 3, pin_none), - new Register( 199, 1, pin_none), - new Register( 198, 2, pin_none), - new Register( 199, 2, pin_none), - new Register( 198, 1, pin_none), - new Register( 199, 3, pin_none), - new Register( 198, 0, pin_none)}, - {alu_write, alu_last_instr}, 4); + new Register(2000, 1, pin_none), + {new Register(199, 0, pin_none), + new Register(198, 3, pin_none), + new Register(199, 1, pin_none), + new Register(198, 2, pin_none), + new Register(199, 2, pin_none), + new Register(198, 1, pin_none), + new Register(199, 3, pin_none), + new Register(198, 0, pin_none)}, + {alu_write, alu_last_instr}, + 4); check(init, expect); } @@ -282,11 +272,11 @@ TEST_F(TestInstrFromString, test_alu_mov_cf) { add_dest_from_string("R1999.x"); for (auto& [expect_cf, str] : AluInstr::cf_map) { - auto init = std::string("ALU MOV R2000.y : R1999.x {WL} ") + str; + auto init = std::string("ALU MOV R2000.y : R1999.x {WL} ") + str; AluInstr expect(op1_mov, - new Register( 2000, 1, pin_none), - new Register( 1999, 0, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 0, pin_none), {alu_write, alu_last_instr}); expect.set_cf_type(expect_cf); @@ -297,42 +287,41 @@ TEST_F(TestInstrFromString, test_alu_mov_cf) TEST_F(TestInstrFromString, test_alu_interp_xy) { add_dest_from_string("R0.y@fully"); - auto init = std::string("ALU INTERP_ZW R1024.z@chan : R0.y@fully Param0.z {W} VEC_210"); + auto init = + std::string("ALU INTERP_ZW R1024.z@chan : R0.y@fully Param0.z {W} VEC_210"); AluInstr expect(op2_interp_zw, - new Register( 1024, 2, pin_chan), - new Register( 0, 1, pin_fully), - new InlineConstant( ALU_SRC_PARAM_BASE, 2), + new Register(1024, 2, pin_chan), + new Register(0, 1, pin_fully), + new InlineConstant(ALU_SRC_PARAM_BASE, 2), {alu_write}); expect.set_bank_swizzle(alu_vec_210); check(init, expect); } - TEST_F(TestInstrFromString, test_alu_interp_xy_no_write) { add_dest_from_string("R0.x@fully"); auto init = std::string("ALU INTERP_XY __.x@chan : R0.x@fully Param0.z {} VEC_210"); AluInstr expect(op2_interp_xy, - new Register( 1024, 0, pin_chan), - new Register( 0, 0, pin_fully), - new InlineConstant( ALU_SRC_PARAM_BASE, 2), + new Register(1024, 0, pin_chan), + new Register(0, 0, pin_fully), + new InlineConstant(ALU_SRC_PARAM_BASE, 2), {}); expect.set_bank_swizzle(alu_vec_210); check(init, expect); } - TEST_F(TestInstrFromString, test_alu_mov_cf_bs) { add_dest_from_string("R1999.x"); auto init = std::string("ALU MOV R2000.y : R1999.x {WL} VEC_210 POP_AFTER"); AluInstr expect(op1_mov, - new Register( 2000, 1, pin_none), - new Register( 1999, 0, pin_none), + new Register(2000, 1, pin_none), + new Register(1999, 0, pin_none), {alu_write, alu_last_instr}); expect.set_cf_type(cf_alu_pop_after); expect.set_bank_swizzle(alu_vec_210); @@ -343,7 +332,8 @@ TEST_F(TestInstrFromString, test_tex_sample_basic) { add_dest_vec4_from_string("R2000.xyzw"); auto init = std::string("TEX SAMPLE R1000.xyzw : R2000.xyzw RID:10 SID:1 NNNN"); - TexInstr expect(TexInstr::sample, RegisterVec4(1000), {0,1,2,3}, RegisterVec4(2000), 1, 10); + TexInstr expect( + TexInstr::sample, RegisterVec4(1000), {0, 1, 2, 3}, RegisterVec4(2000), 1, 10); check(init, expect); } @@ -351,16 +341,19 @@ TEST_F(TestInstrFromString, test_tex_ld_basic) { add_dest_vec4_from_string("R2002.xyzw"); auto init = std::string("TEX LD R1001.xyzw : R2002.xyzw RID:27 SID:7 NNNN"); - TexInstr expect(TexInstr::ld, RegisterVec4(1001), {0,1,2,3}, RegisterVec4(2002), 7, 27); + TexInstr expect( + TexInstr::ld, RegisterVec4(1001), {0, 1, 2, 3}, RegisterVec4(2002), 7, 27); check(init, expect); } TEST_F(TestInstrFromString, test_tex_sample_with_offset) { add_dest_vec4_from_string("R2002.xyzw"); - auto init = std::string("TEX SAMPLE R1001.xyzw : R2002.xyzw RID:27 SID:2 OX:1 OY:-2 OZ:5 NNNN"); + auto init = + std::string("TEX SAMPLE R1001.xyzw : R2002.xyzw RID:27 SID:2 OX:1 OY:-2 OZ:5 NNNN"); - TexInstr expect(TexInstr::sample, RegisterVec4(1001), {0,1,2,3}, RegisterVec4(2002), 2, 27); + TexInstr expect( + TexInstr::sample, RegisterVec4(1001), {0, 1, 2, 3}, RegisterVec4(2002), 2, 27); expect.set_offset(0, 1); expect.set_offset(1, -2); expect.set_offset(2, 5); @@ -371,16 +364,20 @@ TEST_F(TestInstrFromString, test_tex_sample_with_offset) TEST_F(TestInstrFromString, test_tex_gather4_x) { add_dest_vec4_from_string("R2002.xyzw"); - auto init = std::string("TEX GATHER4 R1001.xyzw : R2002.xyzw RID:7 SID:27 MODE:0 NNNN"); - TexInstr expect(TexInstr::gather4, RegisterVec4(1001), {0,1,2,3}, RegisterVec4(2002), 27, 7); + auto init = + std::string("TEX GATHER4 R1001.xyzw : R2002.xyzw RID:7 SID:27 MODE:0 NNNN"); + TexInstr expect( + TexInstr::gather4, RegisterVec4(1001), {0, 1, 2, 3}, RegisterVec4(2002), 27, 7); check(init, expect); } TEST_F(TestInstrFromString, test_tex_gather4_y) { add_dest_vec4_from_string("R2002.xyzw"); - auto init = std::string("TEX GATHER4 R1001.xyzw : R2002.xyzw RID:7 SID:27 MODE:1 NNNN"); - TexInstr expect(TexInstr::gather4, RegisterVec4(1001), {0,1,2,3}, RegisterVec4(2002), 27, 7); + auto init = + std::string("TEX GATHER4 R1001.xyzw : R2002.xyzw RID:7 SID:27 MODE:1 NNNN"); + TexInstr expect( + TexInstr::gather4, RegisterVec4(1001), {0, 1, 2, 3}, RegisterVec4(2002), 27, 7); expect.set_gather_comp(1); check(init, expect); } @@ -388,9 +385,15 @@ TEST_F(TestInstrFromString, test_tex_gather4_y) TEST_F(TestInstrFromString, test_tex_sampler_with_offset) { add_dest_vec4_from_string("R2002.xyzw"); - auto init = std::string("TEX SAMPLE R1001.xyzw : R2002.xyzw RID:7 SID:27 SO:R200.z NNNN"); - TexInstr expect(TexInstr::sample, RegisterVec4(1001), {0,1,2,3}, - RegisterVec4(2002), 27, 7, new Register( 200, 2, pin_none)); + auto init = + std::string("TEX SAMPLE R1001.xyzw : R2002.xyzw RID:7 SID:27 SO:R200.z NNNN"); + TexInstr expect(TexInstr::sample, + RegisterVec4(1001), + {0, 1, 2, 3}, + RegisterVec4(2002), + 27, + 7, + new Register(200, 2, pin_none)); check(init, expect); } @@ -419,15 +422,14 @@ TEST_F(TestInstrFromString, test_export_last_pixel_0) check("EXPORT_DONE PIXEL 0 R1002.zwxy", expect); } - TEST_F(TestInstrFromString, test_fetch_basic) { add_dest_from_string("R201.z"); FetchInstr expect(vc_fetch, RegisterVec4(1002), - {0,4,5,1}, - new Register( 201, 2, pin_none), + {0, 4, 5, 1}, + new Register(201, 2, pin_none), 0, vertex_data, fmt_8, @@ -442,12 +444,15 @@ TEST_F(TestInstrFromString, test_fetch_basic) TEST_F(TestInstrFromString, test_query_buffer_size) { - QueryBufferSizeInstr expect(RegisterVec4(1002),RegisterVec4::Swizzle({0,1,2,3}), 1); + QueryBufferSizeInstr expect(RegisterVec4(1002), + RegisterVec4::Swizzle({0, 1, 2, 3}), + 1); check("GET_BUF_RESINFO R1002.xyzw : RID:1", expect); FetchInstr expect_fetch(vc_get_buf_resinfo, - RegisterVec4(1002),RegisterVec4::Swizzle({0,1,2,3}), - new Register( 0, 7, pin_fully), + RegisterVec4(1002), + RegisterVec4::Swizzle({0, 1, 2, 3}), + new Register(0, 7, pin_fully), 0, no_index_offset, fmt_32_32_32_32, @@ -464,22 +469,27 @@ TEST_F(TestInstrFromString, test_load_from_buffer) add_dest_from_string("R201.x"); add_dest_from_string("R202.x"); string init = "LOAD_BUF R200.xzwy : R201.x + 16b RID:10 + R202.x"; - LoadFromBuffer expect(RegisterVec4(200), RegisterVec4::Swizzle({0,2,3,1}), - new Register( 201, 0, pin_none), 16, 10, - new Register( 202, 0, pin_none), fmt_32_32_32_32_float); + LoadFromBuffer expect(RegisterVec4(200), + RegisterVec4::Swizzle({0, 2, 3, 1}), + new Register(201, 0, pin_none), + 16, + 10, + new Register(202, 0, pin_none), + fmt_32_32_32_32_float); check(init, expect); auto instr = from_string(init); FetchInstr expect_fetch(vc_fetch, - RegisterVec4(200),RegisterVec4::Swizzle({0,2,3,1}), - new Register( 201, 0, pin_none), + RegisterVec4(200), + RegisterVec4::Swizzle({0, 2, 3, 1}), + new Register(201, 0, pin_none), 16, no_index_offset, fmt_32_32_32_32_float, vtx_nf_scaled, vtx_es_none, 10, - new Register( 202, 0, pin_none)); + new Register(202, 0, pin_none)); expect_fetch.set_fetch_flag(FetchInstr::format_comp_signed); expect_fetch.set_mfc(16); check(*instr, expect_fetch); @@ -491,13 +501,16 @@ TEST_F(TestInstrFromString, test_load_from_scratch) add_dest_from_string("R201.x"); string init = "READ_SCRATCH R200.xzwy : R201.x SIZE:20 ES:3"; - LoadFromScratch expect(RegisterVec4(200), RegisterVec4::Swizzle({0,2,3,1}), - new Register( 201, 0, pin_none), 20); + LoadFromScratch expect(RegisterVec4(200), + RegisterVec4::Swizzle({0, 2, 3, 1}), + new Register(201, 0, pin_none), + 20); check(init, expect); FetchInstr expect_fetch(vc_read_scratch, - RegisterVec4(200),RegisterVec4::Swizzle({0,2,3,1}), - new Register( 201, 0, pin_none), + RegisterVec4(200), + RegisterVec4::Swizzle({0, 2, 3, 1}), + new Register(201, 0, pin_none), 0, no_index_offset, fmt_32_32_32_32, @@ -543,23 +556,22 @@ TEST_F(TestInstrFromString, test_write_scratch_to_index) string init2 = "WRITE_SCRATCH @R4.x[20] R2.xy__ AL:4 ALO:16"; ScratchIOInstr expect2(RegisterVec4(2), new Register(4, 0, pin_none), 4, 16, 0x3, 20); check(init2, expect2); - - } - - TEST_F(TestInstrFromString, test_load_from_scratch_fixed_offset) { string init = "READ_SCRATCH R200.xzwy : L[0xA] SIZE:40 ES:3"; - LoadFromScratch expect(RegisterVec4(200), RegisterVec4::Swizzle({0,2,3,1}), - new LiteralConstant( 10), 40); + LoadFromScratch expect(RegisterVec4(200), + RegisterVec4::Swizzle({0, 2, 3, 1}), + new LiteralConstant(10), + 40); check(init, expect); FetchInstr expect_fetch(vc_read_scratch, - RegisterVec4(200),RegisterVec4::Swizzle({0,2,3,1}), - new Register( 0, 7, pin_none), + RegisterVec4(200), + RegisterVec4::Swizzle({0, 2, 3, 1}), + new Register(0, 7, pin_none), 0, no_index_offset, fmt_32_32_32_32, @@ -579,14 +591,14 @@ TEST_F(TestInstrFromString, test_load_from_scratch_fixed_offset) check(init, expect_fetch); } - TEST_F(TestInstrFromString, test_lds_read_3_values) { add_dest_from_string("R5.x@free"); add_dest_from_string("R5.y@free"); add_dest_from_string("R5.z@free"); - auto init = "LDS_READ [ R10.x@free R11.x@free R12.x@free ] : [ R5.x@free R5.y@free R5.z@free ]"; + auto init = + "LDS_READ [ R10.x@free R11.x@free R12.x@free ] : [ R5.x@free R5.y@free R5.z@free ]"; std::vector> dests(3); std::vector> srcs(3); @@ -625,7 +637,8 @@ TEST_F(TestInstrFromString, test_lds_write_1_value) add_dest_from_string("R1.x"); add_dest_from_string("R2.y"); - LDSAtomicInstr expect(DS_OP_WRITE, nullptr, + LDSAtomicInstr expect(DS_OP_WRITE, + nullptr, new Register(1, 0, pin_none), {new Register(2, 1, pin_none)}); @@ -639,10 +652,10 @@ TEST_F(TestInstrFromString, test_lds_write_2_value) add_dest_from_string("R1.x"); add_dest_from_string("R2.y"); - LDSAtomicInstr expect(DS_OP_WRITE2, nullptr, + LDSAtomicInstr expect(DS_OP_WRITE2, + nullptr, new Register(1, 0, pin_none), - {new Register(2, 1, pin_none), - new UniformValue(513, 2, 0) }); + {new Register(2, 1, pin_none), new UniformValue(513, 2, 0)}); check(init, expect); } @@ -677,34 +690,33 @@ TEST_F(TestInstrFromString, test_lds_write_atomic_add) check(init, expect); } - TEST_F(TestInstrFromString, test_writeTF) { auto init = "WRITE_TF R1.xyzw"; add_dest_vec4_from_string("R1.xyzw"); - WriteTFInstr expect(RegisterVec4(1, true, {0,1,2,3}, pin_group)); + WriteTFInstr expect(RegisterVec4(1, true, {0, 1, 2, 3}, pin_group)); check(init, expect); } -TestInstrFromString::TestInstrFromString() -{ +TestInstrFromString::TestInstrFromString() {} -} - -PInst TestInstrFromString::from_string(const std::string& s) +PInst +TestInstrFromString::from_string(const std::string& s) { return m_instr_factory->from_string(s, 0); } -void TestInstrFromString::check(const Instr& eval, const Instr& expect) +void +TestInstrFromString::check(const Instr& eval, const Instr& expect) { EXPECT_EQ(eval, expect); } -void TestInstrFromString::check(const string& init, const Instr& expect) +void +TestInstrFromString::check(const string& init, const Instr& expect) { auto instr = from_string(init); ASSERT_TRUE(instr); @@ -715,26 +727,30 @@ void TestInstrFromString::check(const string& init, const Instr& expect) EXPECT_EQ(os.str(), init); } -void TestInstrFromString::add_dest_from_string(const char *init) +void +TestInstrFromString::add_dest_from_string(const char *init) { m_instr_factory->value_factory().dest_from_string(init); } -void TestInstrFromString::add_dest_vec4_from_string(const char *init) +void +TestInstrFromString::add_dest_vec4_from_string(const char *init) { RegisterVec4::Swizzle dummy; m_instr_factory->value_factory().dest_vec4_from_string(init, dummy); } -void TestInstrFromString::SetUp() +void +TestInstrFromString::SetUp() { MemoryPool::instance().initialize(); m_instr_factory = new InstrFactory; } -void TestInstrFromString::TearDown() +void +TestInstrFromString::TearDown() { MemoryPool::instance().free(); } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/tests/sfn_liverange_test.cpp b/src/gallium/drivers/r600/sfn/tests/sfn_liverange_test.cpp index 1a9b5e11d81..208c38bf8c8 100644 --- a/src/gallium/drivers/r600/sfn/tests/sfn_liverange_test.cpp +++ b/src/gallium/drivers/r600/sfn/tests/sfn_liverange_test.cpp @@ -1,12 +1,10 @@ -#include "../sfn_shader.h" #include "../sfn_liverangeevaluator.h" +#include "../sfn_shader.h" #include "sfn_test_shaders.h" #include "gtest/gtest.h" #include -#include "../sfn_liverangeevaluator.h" - namespace r600 { using std::ostringstream; @@ -14,9 +12,7 @@ using std::ostringstream; class LiveRangeTests : public TestShader { protected: - void check(const char *shader, LiveRangeMap& expect); - }; using SimpleTest = testing::Test; @@ -71,7 +67,6 @@ TEST_F(LiveRangeTests, SimpleAssignments) LiveRangeMap expect = vf.prepare_live_range_map(); - expect.set_life_range(*r1x, 2, 3); for (int i = 0; i < 4; ++i) expect.set_life_range(*r0[i], 1, 3); @@ -152,7 +147,6 @@ TEST_F(LiveRangeTests, SimpleAInterpolation) expect.set_life_range(*r5[1], 6, 7); expect.set_life_range(*r5[3], 6, 7); - expect.set_life_range(*r6[0], 7, 8); expect.set_life_range(*r6[1], 7, 8); expect.set_life_range(*r6[2], 7, 8); @@ -175,7 +169,6 @@ TEST_F(LiveRangeTests, SimpleArrayAccess) auto s3 = vf.dest_vec4_from_string("S3.xy01", dummy, pin_chgr); - LiveRangeMap expect = vf.prepare_live_range_map(); expect.set_life_range(*array->element(0, nullptr, 0), 0, 4); @@ -195,7 +188,8 @@ TEST_F(LiveRangeTests, SimpleArrayAccess) check(shader_with_dest_array2_scheduled, expect); } -void LiveRangeTests::check(const char *shader, LiveRangeMap& expect) +void +LiveRangeTests::check(const char *shader, LiveRangeMap& expect) { auto sh = from_string(shader); ASSERT_TRUE(sh); @@ -211,7 +205,6 @@ void LiveRangeTests::check(const char *shader, LiveRangeMap& expect) expect_str << expect; EXPECT_EQ(eval_str.str(), expect_str.str()); - } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/tests/sfn_optimizer_test.cpp b/src/gallium/drivers/r600/sfn/tests/sfn_optimizer_test.cpp index 12f7288dfcd..710f3bc407d 100644 --- a/src/gallium/drivers/r600/sfn/tests/sfn_optimizer_test.cpp +++ b/src/gallium/drivers/r600/sfn/tests/sfn_optimizer_test.cpp @@ -1,9 +1,9 @@ -#include "sfn_test_shaders.h" -#include "../sfn_shader.h" #include "../sfn_optimizer.h" #include "../sfn_ra.h" #include "../sfn_scheduler.h" +#include "../sfn_shader.h" +#include "sfn_test_shaders.h" using namespace r600; using std::ostringstream; @@ -11,12 +11,10 @@ using std::ostringstream; class TestShaderFromNir : public TestShader { protected: - void check(Shader *s, const char *expect_str); void ra_check(Shader *s, const char *expect_str); }; - TEST_F(TestShaderFromNir, SimpleDCE) { auto sh = from_string(red_triangle_fs_expect_from_nir); @@ -25,7 +23,6 @@ TEST_F(TestShaderFromNir, SimpleDCE) check(sh, red_triangle_fs_expect_from_nir_dce); } - TEST_F(TestShaderFromNir, CopyPropagationForwardBackward) { auto sh = from_string(add_add_1_expect_from_nir); @@ -49,7 +46,6 @@ TEST_F(TestShaderFromNir, CopyPropagationBackwardDCE) check(sh, add_add_1_expect_from_nir_copy_prop_fwd_dce_bwd); } - TEST_F(TestShaderFromNir, FullOPtimize) { auto sh = from_string(basic_interpolation_orig); @@ -84,7 +80,6 @@ TEST_F(TestShaderFromNir, CombinePinFlags) check(sh, shader_group_chan_pin_combined); } - TEST_F(TestShaderFromNir, FullOPtimizeLoop) { auto sh = from_string(vs_nexted_loop_from_nir_expect); @@ -121,7 +116,6 @@ TEST_F(TestShaderFromNir, RA_with_dest_array) auto lrm = r600::LiveRangeEvaluator().run(*sh); EXPECT_TRUE(r600::register_allocation(lrm)); - ra_check(sh, shader_with_dest_array2_scheduled_ra); } @@ -134,7 +128,6 @@ TEST_F(TestShaderFromNir, RA_with_chan_group) ra_check(sh, shader_group_chan_pin_combined_sheduled_ra); } - TEST_F(TestShaderFromNir, TES_opt) { auto sh = from_string(tes_pre_op); @@ -151,7 +144,6 @@ TEST_F(TestShaderFromNir, TES_scheduled) check(schedule(sh), tes_optimized_sched); } - /* TEST_F(TestShaderFromNir, ShaderClone) { @@ -176,7 +168,6 @@ TEST_F(TestShaderFromNir, ShaderSchedule) auto sh = from_string(basic_interpolation_orig); check(schedule(sh), basic_interpolation_expect_from_nir_sched); - } TEST_F(TestShaderFromNir, ShaderScheduleCayman) @@ -186,7 +177,6 @@ TEST_F(TestShaderFromNir, ShaderScheduleCayman) check(schedule(sh), basic_interpolation_expect_from_nir_sched_cayman); } - TEST_F(TestShaderFromNir, ShaderScheduleOptimizedCayman) { auto sh = from_string(basic_interpolation_orig_cayman); @@ -196,18 +186,16 @@ TEST_F(TestShaderFromNir, ShaderScheduleOptimizedCayman) check(schedule(sh), basic_interpolation_expect_opt_sched_cayman); } - TEST_F(TestShaderFromNir, CopyPropLegalConst) { - auto sh = from_string(dot4_pre); + auto sh = from_string(dot4_pre); - copy_propagation_fwd(*sh); - dead_code_elimination(*sh); + copy_propagation_fwd(*sh); + dead_code_elimination(*sh); - check(sh, dot4_copy_prop_dce); + check(sh, dot4_copy_prop_dce); } - TEST_F(TestShaderFromNir, FullOPtimize_glxgears_vs2) { @@ -231,8 +219,6 @@ TEST_F(TestShaderFromNir, test_dont_kill_dual_use) check(schedule(sh), shader_copy_prop_dont_kill_double_use_expect); } - - TEST_F(TestShaderFromNir, test_schedule_with_bany) { @@ -241,7 +227,6 @@ TEST_F(TestShaderFromNir, test_schedule_with_bany) check(schedule(sh), shader_with_bany_expect_opt_sched_eg); } - TEST_F(TestShaderFromNir, GroupAndChanCombine) { auto sh = from_string(shader_group_chan_pin_to_combine_2); @@ -283,8 +268,8 @@ TEST_F(TestShaderFromNir, fs_shed_tex_coord) check(schedule(sh), fs_sched_tex_coord_expect); } - -void TestShaderFromNir::check(Shader *s, const char *expect_orig) +void +TestShaderFromNir::check(Shader *s, const char *expect_orig) { ostringstream test_str; s->print(test_str); @@ -297,7 +282,8 @@ void TestShaderFromNir::check(Shader *s, const char *expect_orig) EXPECT_EQ(test_str.str(), expect_str.str()); } -void TestShaderFromNir::ra_check(Shader *s, const char *expect_orig) +void +TestShaderFromNir::ra_check(Shader *s, const char *expect_orig) { s->value_factory().clear_pins(); ostringstream test_str; @@ -311,5 +297,3 @@ void TestShaderFromNir::ra_check(Shader *s, const char *expect_orig) EXPECT_EQ(test_str.str(), expect_str.str()); } - - diff --git a/src/gallium/drivers/r600/sfn/tests/sfn_shaderfromstring_test.cpp b/src/gallium/drivers/r600/sfn/tests/sfn_shaderfromstring_test.cpp index 1d13c756436..aac6918b9b2 100644 --- a/src/gallium/drivers/r600/sfn/tests/sfn_shaderfromstring_test.cpp +++ b/src/gallium/drivers/r600/sfn/tests/sfn_shaderfromstring_test.cpp @@ -1,9 +1,8 @@ -#include "../sfn_instrfactory.h" - #include "../sfn_instr_alu.h" -#include "../sfn_instr_tex.h" #include "../sfn_instr_export.h" +#include "../sfn_instr_tex.h" +#include "../sfn_instrfactory.h" #include "gtest/gtest.h" #include @@ -14,33 +13,32 @@ using std::istringstream; using std::string; using std::vector; -class TestShaderFromString : public ::testing::Test -{ +class TestShaderFromString : public ::testing::Test { public: - void SetUp() override { + void SetUp() override + { init_pool(); m_instr_factory = new InstrFactory(); } - void TearDown() override { - release_pool(); - } + void TearDown() override { release_pool(); } TestShaderFromString(); std::vector from_string(const std::string& s); protected: - void check(const vector& eval, const std::vector>& expect); + void check(const vector& eval, + const std::vector>& expect); + private: InstrFactory *m_instr_factory = nullptr; }; - TEST_F(TestShaderFromString, test_simple_fs) { auto init_str = - R"( + R"( # load constant color ALU MOV R2000.x@group : L[0x38000000] {W} @@ -52,48 +50,37 @@ ALU MOV R2000.w@group : L[0x38F00000] {WL} EXPORT_DONE PIXEL 0 R2000.xyzw )"; - auto shader = from_string(init_str); std::vector> expect; expect.push_back(new AluInstr(op1_mov, - new Register( 2000, 0, pin_group), + new Register(2000, 0, pin_group), new LiteralConstant(0x38000000), {alu_write})); - expect.push_back(new AluInstr(op1_mov, - new Register( 2000, 1, pin_group), - new LiteralConstant( 0x0), - {alu_write})); + expect.push_back(new AluInstr( + op1_mov, new Register(2000, 1, pin_group), new LiteralConstant(0x0), {alu_write})); + + expect.push_back(new AluInstr( + op1_mov, new Register(2000, 2, pin_group), new LiteralConstant(0x0), {alu_write})); expect.push_back(new AluInstr(op1_mov, - new Register( 2000, 2, pin_group), - new LiteralConstant( 0x0), - {alu_write})); - - expect.push_back(new AluInstr(op1_mov, - new Register( 2000, 3, pin_group), - new LiteralConstant( 0x38F00000), + new Register(2000, 3, pin_group), + new LiteralConstant(0x38F00000), {alu_write, alu_last_instr})); - auto exp = new ExportInstr( - ExportInstr::pixel, 0, RegisterVec4(2000, false)); + auto exp = new ExportInstr(ExportInstr::pixel, 0, RegisterVec4(2000, false)); exp->set_is_last_export(true); expect.push_back(exp); check(shader, expect); - } +TestShaderFromString::TestShaderFromString() {} - -TestShaderFromString::TestShaderFromString() -{ - -} - -std::vector TestShaderFromString::from_string(const std::string& s) +std::vector +TestShaderFromString::from_string(const std::string& s) { istringstream is(s); string line; @@ -112,8 +99,9 @@ std::vector TestShaderFromString::from_string(const std::string& s) return shader; } -void TestShaderFromString::check(const vector& eval, - const std::vector>& expect) +void +TestShaderFromString::check(const vector& eval, + const std::vector>& expect) { ASSERT_EQ(eval.size(), expect.size()); diff --git a/src/gallium/drivers/r600/sfn/tests/sfn_test_shaders.cpp b/src/gallium/drivers/r600/sfn/tests/sfn_test_shaders.cpp index db4663d8ed7..a693cb46cd9 100644 --- a/src/gallium/drivers/r600/sfn/tests/sfn_test_shaders.cpp +++ b/src/gallium/drivers/r600/sfn/tests/sfn_test_shaders.cpp @@ -1,17 +1,18 @@ #include "sfn_test_shaders.h" + +#include "../sfn_memorypool.h" #include "../sfn_shader_fs.h" #include "../sfn_shader_gs.h" #include "../sfn_shader_tess.h" #include "../sfn_shader_vs.h" -#include "../sfn_memorypool.h" namespace r600 { using std::istringstream; using std::string; -const char *red_triangle_fs_nir = -R"(shader: MESA_SHADER_FRAGMENT +const char *red_triangle_fs_nir = + R"(shader: MESA_SHADER_FRAGMENT name: TTN inputs: 0 outputs: 1 @@ -60,9 +61,8 @@ ALU MOV S0.w@group : I[1.0] {WL} EXPORT_DONE PIXEL 0 S0.xyzw )"; - const char *add_add_1_nir = -R"(shader: MESA_SHADER_FRAGMENT + R"(shader: MESA_SHADER_FRAGMENT name: GLSL3 inputs: 0 outputs: 1 @@ -86,7 +86,7 @@ impl main { })"; const char *add_add_1_expect_from_nir = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -108,9 +108,8 @@ ALU MOV S4.w@group : S2.w {WL} EXPORT_DONE PIXEL 0 S4.xyzw )"; - const char *add_add_1_expect_from_nir_copy_prop_fwd = -R"( + R"( FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 @@ -134,7 +133,7 @@ EXPORT_DONE PIXEL 0 S4.xyzw )"; const char *add_add_1_expect_from_nir_copy_prop_fwd_dce = -R"( + R"( FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 @@ -151,9 +150,8 @@ ALU MOV S4.w@group : KC0[0].w {WL} EXPORT_DONE PIXEL 0 S4.xyzw )"; - const char *add_add_1_expect_from_nir_copy_prop_fwd_dce_bwd = -R"( + R"( FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 @@ -170,9 +168,8 @@ ALU MOV S4.w@group : KC0[0].w {WL} EXPORT_DONE PIXEL 0 S4.xyzw )"; - const char *basic_interpolation_nir = -R"(shader: MESA_SHADER_FRAGMENT + R"(shader: MESA_SHADER_FRAGMENT name: TTN inputs: 1 outputs: 1 @@ -199,7 +196,7 @@ impl main { })"; const char *basic_interpolation_expect_from_nir = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -240,9 +237,8 @@ ALU MOV S9.w@group : S7.x@free {WL} TEX LD S10.xyzw : S9.xy_w RID:18 SID:0 NNNN EXPORT_DONE PIXEL 0 S10.xyzw)"; - const char *basic_interpolation_translated_1 = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -277,10 +273,8 @@ ALU MOV S5.w@group : S3.z {WL} TEX LD S6.xyzw : S5.xy_w RID:18 SID:0 NNNN EXPORT_DONE PIXEL 0 S6.xyzw)"; - - const char *basic_interpolation_2 = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -305,9 +299,8 @@ ALU_GROUP_END EXPORT_DONE PIXEL 0 S2.xyzw )"; - const char *basic_interpolation_orig = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -344,7 +337,7 @@ EXPORT_DONE PIXEL 0 S1029.xyzw )"; const char *basic_interpolation_expect_from_nir_sched = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -391,9 +384,8 @@ EXPORT_DONE PIXEL 0 S1029.xyzw BLOCK_END )"; - const char *basic_interpolation_orig_cayman = -R"(FS + R"(FS CHIPCLASS CAYMAN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -430,7 +422,7 @@ EXPORT_DONE PIXEL 0 S1029.xyzw )"; const char *basic_interpolation_expect_from_nir_sched_cayman = -R"(FS + R"(FS CHIPCLASS CAYMAN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -480,7 +472,7 @@ BLOCK_END )"; const char *basic_interpolation_expect_opt_sched_cayman = -R"(FS + R"(FS CHIPCLASS CAYMAN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -518,7 +510,7 @@ BLOCK_END )"; const char *basic_interpolation_expect_from_nir_opt = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -548,7 +540,7 @@ EXPORT_DONE PIXEL 0 S1029.xyzw )"; const char *dot4_pre = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -573,7 +565,7 @@ EXPORT_DONE PIXEL 0 S4.xyzw )"; const char *dot4_copy_prop_dce = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -590,7 +582,7 @@ EXPORT_DONE PIXEL 0 S3.xxxx )"; const char *glxgears_vs2_nir = -R"(shader: MESA_SHADER_VERTEX + R"(shader: MESA_SHADER_VERTEX name: ARB0 inputs: 2 outputs: 2 @@ -648,7 +640,7 @@ impl main { })"; const char *glxgears_vs2_from_nir_expect = -R"(VS + R"(VS CHIPCLASS EVERGREEN INPUT LOC:0 NAME:0 INPUT LOC:1 NAME:1 @@ -755,9 +747,8 @@ ALU MOV S40.z@group : S38.z {W} ALU MOV S40.w@group : S39.x@free {WL} EXPORT_DONE PARAM 0 S40.xyzw)"; - const char *glxgears_vs2_from_nir_expect_cayman = -R"(VS + R"(VS CHIPCLASS CAYMAN INPUT LOC:0 NAME:0 INPUT LOC:1 NAME:1 @@ -864,9 +855,8 @@ ALU MOV S40.z@group : S38.z {W} ALU MOV S40.w@group : S39.x@free {WL} EXPORT_DONE PARAM 0 S40.xyzw)"; - const char *glxgears_vs2_from_nir_expect_optimized = -R"(VS + R"(VS CHIPCLASS EVERGREEN INPUT LOC:0 NAME:0 INPUT LOC:1 NAME:1 @@ -911,10 +901,8 @@ EXPORT_DONE POS 0 S15.xyzw ALU MOV CLAMP S40.w@group : KC0[2].w {WL} EXPORT_DONE PARAM 0 S40.xyzw)"; - - const char *vs_nexted_loop_nir = -R"(shader: MESA_SHADER_VERTEX + R"(shader: MESA_SHADER_VERTEX name: GLSL3 inputs: 1 outputs: 2 @@ -1029,7 +1017,7 @@ impl main { })"; const char *vs_nexted_loop_from_nir_expect = -R"(VS + R"(VS CHIPCLASS EVERGREEN INPUT LOC:0 NAME:0 OUTPUT LOC:0 NAME:0 MASK:15 SID:0 SPI_SID:0 @@ -1098,7 +1086,7 @@ EXPORT_DONE PARAM 0 S34.xyzw )"; const char *vs_nexted_loop_from_nir_expect_opt = -R"( + R"( VS CHIPCLASS EVERGREEN INPUT LOC:0 NAME:0 @@ -1146,7 +1134,7 @@ EXPORT_DONE PARAM 0 S34.xy0w )"; const char *shader_with_local_array_nir = -R"( + R"( shader: MESA_SHADER_FRAGMENT name: GLSL3 inputs: 2 @@ -1227,7 +1215,7 @@ impl main { )"; const char *shader_with_local_array_expect = -R"(FS + R"(FS CHIPCLASS EVERGREEN CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 @@ -1318,7 +1306,7 @@ ALU MOV S35.w@group : S34.y {WL} EXPORT_DONE PIXEL 0 S35.xyzw)"; const char *test_schedule_group = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -1349,7 +1337,7 @@ EXPORT_DONE PIXEL 0 S12.xyzw )"; const char *test_schedule_group_expect = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -1387,9 +1375,8 @@ EXPORT_DONE PIXEL 0 S12.xy01 BLOCK_END )"; - const char *shader_with_bany_nir = -R"(shader: MESA_SHADER_FRAGMENT + R"(shader: MESA_SHADER_FRAGMENT source_sha1: {0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000} name: GLSL3 inputs: 0 @@ -1432,9 +1419,8 @@ impl main { block block_1: })"; - const char *shader_with_bany_expect_eg = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -1527,7 +1513,7 @@ EXPORT_DONE PIXEL 0 S35.xyzw )"; const char *shader_with_bany_expect_opt_sched_eg = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -1624,7 +1610,7 @@ BLOCK_END )"; const char *shader_copy_prop_dont_kill_double_use = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -1660,9 +1646,8 @@ ALU MOV S15.w@group : S2.x {WL} EXPORT_DONE PIXEL 0 S15.xyzw )"; - const char *shader_copy_prop_dont_kill_double_use_expect = -R"( + R"( FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 @@ -1704,9 +1689,8 @@ EXPORT_DONE PIXEL 0 S15.xyz1 BLOCK_END )"; - const char *shader_with_dest_array = -R"(VS + R"(VS CHIPCLASS EVERGREEN INPUT LOC:0 NAME:0 OUTPUT LOC:0 NAME:0 MASK:15 @@ -1845,7 +1829,7 @@ EXPORT_DONE PARAM 3 S49.xyzw )"; const char *shader_with_dest_array_opt_expect = -R"(VS + R"(VS CHIPCLASS EVERGREEN INPUT LOC:0 NAME:0 OUTPUT LOC:0 NAME:0 MASK:15 @@ -1921,7 +1905,7 @@ EXPORT_DONE PARAM 3 S49.xyzw )"; const char *shader_with_dest_array_opt_scheduled = -R"(VS + R"(VS CHIPCLASS EVERGREEN INPUT LOC:0 NAME:0 OUTPUT LOC:0 NAME:0 MASK:15 @@ -2034,9 +2018,8 @@ EXPORT_DONE PARAM 3 S49.xyzw BLOCK_END )"; - const char *shader_with_dest_array2 = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2064,7 +2047,7 @@ BLOCK_END )"; const char *shader_with_dest_array2_scheduled = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2102,7 +2085,7 @@ BLOCK_END )"; const char *shader_with_dest_array2_scheduled_ra = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2140,7 +2123,7 @@ BLOCK_END )"; const char *shader_group_chan_pin_to_combine = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2169,9 +2152,8 @@ ALU MOV S2.w@group : S1.w@chan {WL} VEC_210 EXPORT_DONE PIXEL 0 S2.xyzw )"; - const char *shader_group_chan_pin_combined = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2197,7 +2179,7 @@ EXPORT_DONE PIXEL 0 S1.xyzw )"; const char *shader_group_chan_pin_combined_sheduled = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2223,7 +2205,7 @@ EXPORT_DONE PIXEL 0 S2.xyzw )"; const char *shader_group_chan_pin_combined_sheduled_ra = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2248,9 +2230,8 @@ ALU_GROUP_END EXPORT_DONE PIXEL 0 R1.xyzw )"; - const char *shader_group_chan_pin_to_combine_2 = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2274,7 +2255,7 @@ EXPORT_DONE PIXEL 0 S5.xyzw )"; const char *shader_group_chan_pin_to_combine_2_opt = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2290,9 +2271,8 @@ ALU MOV S5.z@group : S3.x@free {W} EXPORT_DONE PIXEL 0 S5.xyzw )"; - const char *fs_with_grand_and_abs = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2346,9 +2326,8 @@ ALU MOV S25.w@group : S4.x@free {WL} EXPORT_DONE PIXEL 0 S25.xyzw )"; - const char *fs_opt_tex_coord_init = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2380,7 +2359,7 @@ ALU ADD S5.w@group : S3.w@group S4.w@group {W} EXPORT_DONE PIXEL 0 S5.xyzw)"; const char *fs_opt_tex_coord_expect = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2410,7 +2389,7 @@ ALU ADD S5.w@group : S3.w@group S4.w@group {W} EXPORT_DONE PIXEL 0 S5.xyzw)"; const char *fs_sched_tex_coord_init = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2444,9 +2423,8 @@ ALU ADD S6.z@group : S5.z@group S4.z@group {W} ALU ADD S6.w@group : S5.w@group S4.w@group {W} EXPORT_DONE PIXEL 0 S5.xyzw)"; - const char *fs_sched_tex_coord_expect = -R"(FS + R"(FS CHIPCLASS EVERGREEN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2491,9 +2469,8 @@ BLOCK_START EXPORT_DONE PIXEL 0 S5.xyzw BLOCK_END)"; - const char *fs_with_loop_multislot_reuse = -R"(FS + R"(FS CHIPCLASS CAYMAN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2516,7 +2493,7 @@ EXPORT_DONE PIXEL 0 R1.xxxx )"; const char *fs_with_loop_multislot_reuse_scheduled = -R"(FS + R"(FS CHIPCLASS CAYMAN PROP MAX_COLOR_EXPORTS:1 PROP COLOR_EXPORTS:1 @@ -2550,9 +2527,8 @@ LOOP_END EXPORT_DONE PIXEL 0 R1.xxxx )"; - const char *gs_abs_float_nir = -R"(shader: MESA_SHADER_GEOMETRY + R"(shader: MESA_SHADER_GEOMETRY source_sha1: {0xdfd2ba73, 0x5eff5b0c, 0x577ee695, 0xb65ae49e, 0xecc34679} name: GLSL4 inputs: 1 @@ -2605,7 +2581,7 @@ impl main { })"; const char *gs_abs_float_expect = -R"(GS + R"(GS CHIPCLASS EVERGREEN INPUT LOC:0 NAME:5 SID:9 SPI_SID:10 OUTPUT LOC:0 NAME:0 MASK:15 @@ -2649,9 +2625,8 @@ EMIT_VERTEX @0 ALU ADD_INT S24.x@chan : S23.x@chan L[0x2] {WL} )"; - const char *vtx_for_tcs_nir = -R"(shader: MESA_SHADER_VERTEX + R"(shader: MESA_SHADER_VERTEX source_sha1: {0xbd6100f2, 0xc71e7b0e, 0x74662024, 0x261073d8, 0xeae01762} name: GLSL5 inputs: 0 @@ -2683,9 +2658,8 @@ impl main { block block_1: })"; - const char *vtx_for_tcs_from_nir_expect = -R"(VS + R"(VS CHIPCLASS EVERGREEN REGISTERS R0.x@fully R0.y@fully SHADER @@ -2707,9 +2681,8 @@ ALU MOV S11.x@free : L[0x8] {WL} ALU ADD_INT S12.x@free : S11.x@free S10.x@free {WL} LDS WRITE_REL __.x [ S12.x@free ] : S6.z S6.w)"; - const char *vtx_for_tcs_inp = -R"(VS + R"(VS CHIPCLASS EVERGREEN REGISTERS R0.x@fully R0.y@fully SHADER @@ -2732,7 +2705,7 @@ ALU ADD_INT S12.x@free : S11.x@free S10.x@free {WL} LDS WRITE_REL __.x [ S12.x@free ] : S6.z S6.w)"; const char *vtx_for_tcs_opt = -R"(VS + R"(VS CHIPCLASS EVERGREEN REGISTERS R0.x@fully R0.y@fully SHADER @@ -2746,7 +2719,7 @@ ALU ADD_INT S12.x@free : L[0x8] S10.x@free {WL} LDS WRITE_REL __.x [ S12.x@free ] : I[0] I[1.0])"; const char *vtx_for_tcs_pre_sched = -R"(VS + R"(VS CHIPCLASS EVERGREEN REGISTERS R0.x@fully R0.y@fully SHADER @@ -2761,7 +2734,7 @@ ALU ADD_INT S12.x@free : L[0x8] S10.x@free {WL} LDS WRITE_REL __.x [ S12.x@free ] : I[0] I[1.0])"; const char *vtx_for_tcs_sched = -R"(VS + R"(VS CHIPCLASS EVERGREEN REGISTERS R0.x@fully R0.y@fully SHADER @@ -2796,7 +2769,7 @@ ALU_GROUP_END BLOCK_END)"; const char *tcs_nir = -R"(shader: MESA_SHADER_TESS_CTRL + R"(shader: MESA_SHADER_TESS_CTRL source_sha1: {0xc83b0de6, 0x36934b97, 0xccddb436, 0xb0952cb0, 0x07a450a1} name: GLSL5 inputs: 1 @@ -2869,7 +2842,7 @@ impl main { })"; const char *tcs_from_nir_expect = -R"(TCS + R"(TCS CHIPCLASS EVERGREEN PROP TCS_PRIM_MODE:4 REGISTERS R0.x@fully R0.y@fully R0.z@fully R0.w@fully @@ -2963,7 +2936,7 @@ IF (( ALU PRED_SETNE_INT __.z@free : S28.x@free I[0] {LEP} PUSH_BEFORE )) ENDIF)"; const char *tes_nir = -R"(shader: MESA_SHADER_TESS_EVAL + R"(shader: MESA_SHADER_TESS_EVAL source_sha1: {0x2db04154, 0x4884cf59, 0x50e43ee6, 0x4bb239d7, 0x0b502229} name: GLSL5 inputs: 1 @@ -2997,7 +2970,7 @@ impl main { })"; const char *tes_from_nir_expect = -R"(TES + R"(TES CHIPCLASS EVERGREEN OUTPUT LOC:0 NAME:0 MASK:15 REGISTERS R0.x@fully R0.y@fully R0.z@fully @@ -3029,9 +3002,8 @@ LDS_READ [ S17.x@group S17.y@group S17.z@group S17.w@group ] : [ S16.x S16.y S16 EXPORT_DONE POS 0 S17.xyzw EXPORT_DONE PARAM 0 R0.____)"; - const char *tes_pre_op = -R"(TES + R"(TES CHIPCLASS EVERGREEN OUTPUT LOC:0 NAME:0 MASK:15 REGISTERS R0.x@fully R0.y@fully R0.z@fully @@ -3064,7 +3036,7 @@ EXPORT_DONE POS 0 S1040.xyzw EXPORT_DONE PARAM 0 R0.____)"; const char *tes_optimized = -R"(TES + R"(TES CHIPCLASS EVERGREEN OUTPUT LOC:0 NAME:0 MASK:15 REGISTERS R0.x@fully R0.y@fully R0.z@fully @@ -3087,7 +3059,7 @@ EXPORT_DONE POS 0 S1040.xyzw EXPORT_DONE PARAM 0 R0.____)"; const char *tes_optimized_pre_sched = -R"(TES + R"(TES CHIPCLASS EVERGREEN OUTPUT LOC:0 NAME:0 MASK:15 REGISTERS R0.x@fully R0.y@fully R0.z@fully @@ -3110,7 +3082,7 @@ EXPORT_DONE POS 0 S1040.xyzw EXPORT_DONE PARAM 0 R0.____)"; const char *tes_optimized_sched = -R"(TES + R"(TES CHIPCLASS EVERGREEN OUTPUT LOC:0 NAME:0 MASK:15 REGISTERS R0.x@fully R0.y@fully R0.z@fully @@ -3179,32 +3151,37 @@ EXPORT_DONE POS 0 S1040.xyzw EXPORT_DONE PARAM 0 R0.____ BLOCK_END)"; -void TestShader::SetUp() +void +TestShader::SetUp() { init_pool(); SetUpMore(); } -void TestShader::TearDown() +void +TestShader::TearDown() { TearDownMore(); release_pool(); } -void TestShader::SetUpMore() +void +TestShader::SetUpMore() { } -void TestShader::TearDownMore() +void +TestShader::TearDownMore() { } -Shader *TestShader::from_string(const std::string& s) +Shader * +TestShader::from_string(const std::string& s) { istringstream is(s); string line; - r600_shader_key key = { { 0 } }; + r600_shader_key key = {{0}}; key.ps.nr_cbufs = 1; do { @@ -3213,15 +3190,15 @@ Shader *TestShader::from_string(const std::string& s) Shader *shader = nullptr; - if (line.substr(0,2) == "FS") + if (line.substr(0, 2) == "FS") shader = new FragmentShaderEG(key); - else if (line.substr(0,2) == "VS") - shader = new VertexShader(nullptr, nullptr, key); - else if (line.substr(0,2) == "GS") + else if (line.substr(0, 2) == "VS") + shader = new VertexShader(nullptr, nullptr, key); + else if (line.substr(0, 2) == "GS") shader = new GeometryShader(key); - else if (line.substr(0,3) == "TCS") + else if (line.substr(0, 3) == "TCS") shader = new TCSShader(key); - else if (line.substr(0,3) == "TES") + else if (line.substr(0, 3) == "TES") shader = new TESShader(nullptr, nullptr, key); else return nullptr; @@ -3232,7 +3209,7 @@ Shader *TestShader::from_string(const std::string& s) if (line[0] == '#') continue; - if (line.substr(0,6) == "SHADER") + if (line.substr(0, 6) == "SHADER") break; istringstream ls(line); @@ -3254,4 +3231,4 @@ Shader *TestShader::from_string(const std::string& s) return shader; } -} +} // namespace r600 diff --git a/src/gallium/drivers/r600/sfn/tests/sfn_test_shaders.h b/src/gallium/drivers/r600/sfn/tests/sfn_test_shaders.h index e56506b2df6..8ec81fbb5a7 100644 --- a/src/gallium/drivers/r600/sfn/tests/sfn_test_shaders.h +++ b/src/gallium/drivers/r600/sfn/tests/sfn_test_shaders.h @@ -109,13 +109,10 @@ class TestShader : public ::testing::Test { virtual void SetUpMore(); virtual void TearDownMore(); - protected: Shader *from_string(const std::string& s); }; - -} - +} // namespace r600 #endif // SFN_TEST_SHADERS_H diff --git a/src/gallium/drivers/r600/sfn/tests/sfn_value_test.cpp b/src/gallium/drivers/r600/sfn/tests/sfn_value_test.cpp index f8b7c48351c..edd2024c16b 100644 --- a/src/gallium/drivers/r600/sfn/tests/sfn_value_test.cpp +++ b/src/gallium/drivers/r600/sfn/tests/sfn_value_test.cpp @@ -24,28 +24,20 @@ * USE OR OTHER DEALINGS IN THE SOFTWARE. */ - -#include "../sfn_virtualvalues.h" #include "../sfn_alu_defines.h" #include "../sfn_debug.h" +#include "../sfn_virtualvalues.h" #include "gtest/gtest.h" using namespace r600; -class ValueTest : public ::testing::Test -{ - void SetUp() override { - init_pool(); - } +class ValueTest : public ::testing::Test { + void SetUp() override { init_pool(); } - void TearDown() override { - release_pool(); - } + void TearDown() override { release_pool(); } }; - - TEST_F(ValueTest, gpr_register_fully_pinned) { Register reg(1, 2, pin_fully); @@ -105,7 +97,7 @@ TEST_F(ValueTest, uniform_value) EXPECT_FALSE(reg1.buf_addr()); EXPECT_FALSE(reg1.is_virtual()); - auto addr = new Register( 1024, 0, pin_none); + auto addr = new Register(1024, 0, pin_none); ASSERT_TRUE(addr); UniformValue reg_with_buffer_addr(513, 0, addr); @@ -182,7 +174,7 @@ TEST_F(ValueTest, array) EXPECT_EQ(elm1->pin(), pin_array); EXPECT_FALSE(elm1->get_addr()); - auto addr = new Register( 2000, 0, pin_none); + auto addr = new Register(2000, 0, pin_none); ASSERT_TRUE(addr); auto elm_indirect = array.element(0, addr, 1); @@ -200,7 +192,7 @@ TEST_F(ValueTest, array) EXPECT_EQ(elm_addr->pin(), pin_none); // A constant addr should resolve directly - auto addr2 = new LiteralConstant( 3); + auto addr2 = new LiteralConstant(3); ASSERT_TRUE(addr2); auto elm_direct = array.element(0, addr2, 0); @@ -215,7 +207,7 @@ TEST_F(ValueTest, array) EXPECT_THROW(array.element(12, nullptr, 0), std::invalid_argument); EXPECT_THROW(array.element(3, nullptr, 2), std::invalid_argument); - auto addr3 = new LiteralConstant( 12); + auto addr3 = new LiteralConstant(12); ASSERT_TRUE(addr3); EXPECT_THROW(array.element(0, addr3, 0), std::invalid_argument); #endif @@ -232,13 +224,11 @@ TEST_F(ValueTest, reg_from_string) EXPECT_EQ(*Register::from_string("R1000.y@chan"), Register(1000, 1, pin_chan)); EXPECT_EQ(*Register::from_string("R1000.y@free"), Register(1000, 1, pin_free)); - EXPECT_EQ(*VirtualValue::from_string("L[0x1]"), LiteralConstant(1)); EXPECT_EQ(*VirtualValue::from_string("L[0x2]"), LiteralConstant(2)); EXPECT_EQ(*VirtualValue::from_string("L[0xA]"), LiteralConstant(10)); EXPECT_EQ(*VirtualValue::from_string("I[0]"), InlineConstant(ALU_SRC_0)); - EXPECT_EQ(*VirtualValue::from_string("I[HW_WAVE_ID]"), InlineConstant(ALU_SRC_HW_WAVE_ID)); - - + EXPECT_EQ(*VirtualValue::from_string("I[HW_WAVE_ID]"), + InlineConstant(ALU_SRC_HW_WAVE_ID)); } diff --git a/src/gallium/drivers/r600/sfn/tests/sfn_valuefactory_test.cpp b/src/gallium/drivers/r600/sfn/tests/sfn_valuefactory_test.cpp index 77ff8369932..1145197f59b 100644 --- a/src/gallium/drivers/r600/sfn/tests/sfn_valuefactory_test.cpp +++ b/src/gallium/drivers/r600/sfn/tests/sfn_valuefactory_test.cpp @@ -24,18 +24,17 @@ * USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include "../sfn_valuefactory.h" #include "../sfn_alu_defines.h" #include "../sfn_debug.h" - +#include "../sfn_valuefactory.h" #include "nir_builder.h" -#include "gtest/gtest.h" - #include "ralloc.h" +#include "gtest/gtest.h" + using namespace r600; -class ValuefactoryTest: public ::testing::Test { +class ValuefactoryTest : public ::testing::Test { public: ValuefactoryTest(); @@ -47,7 +46,6 @@ protected: ValueFactory *factory; nir_builder b; nir_shader_compiler_options options; - }; TEST_F(ValuefactoryTest, test_create_ssa) @@ -74,7 +72,7 @@ TEST_F(ValuefactoryTest, test_create_ssa) TEST_F(ValuefactoryTest, test_create_register_1) { nir_src src1 = NIR_SRC_INIT; - src1.reg.reg = nir_local_reg_create(b.impl); + src1.reg.reg = nir_local_reg_create(b.impl); src1.reg.reg->num_components = 1; nir_src src2 = NIR_SRC_INIT; @@ -99,7 +97,7 @@ TEST_F(ValuefactoryTest, test_create_register_1) TEST_F(ValuefactoryTest, test_create_register_array_direct_access) { nir_dest dst = NIR_DEST_INIT; - dst.reg.reg = nir_local_reg_create(b.impl); + dst.reg.reg = nir_local_reg_create(b.impl); dst.reg.reg->num_components = 2; dst.reg.reg->num_array_elems = 10; @@ -124,14 +122,12 @@ TEST_F(ValuefactoryTest, test_create_register_array_direct_access) EXPECT_EQ(regy->sel(), 1024 + 5); EXPECT_EQ(regy->chan(), 1); EXPECT_EQ(regy->pin(), pin_array); - } - TEST_F(ValuefactoryTest, test_create_register_array_indirect_access) { nir_dest dst = NIR_DEST_INIT; - dst.reg.reg = nir_local_reg_create(b.impl); + dst.reg.reg = nir_local_reg_create(b.impl); dst.reg.reg->num_components = 3; dst.reg.reg->num_array_elems = 10; @@ -173,7 +169,6 @@ TEST_F(ValuefactoryTest, test_create_register_array_indirect_access) EXPECT_EQ(regy->chan(), 1); EXPECT_EQ(*regy_addr, *addr_reg); EXPECT_EQ(regy->pin(), pin_array); - } TEST_F(ValuefactoryTest, test_create_ssa_pinned_chan) @@ -194,7 +189,6 @@ TEST_F(ValuefactoryTest, test_create_ssa_pinned_chan) EXPECT_EQ(value->pin(), pin_chan); } - TEST_F(ValuefactoryTest, test_create_ssa_pinned_chan_and_reg) { auto c1 = nir_imm_float(&b, 2.0); @@ -213,7 +207,6 @@ TEST_F(ValuefactoryTest, test_create_ssa_pinned_chan_and_reg) EXPECT_EQ(value->pin(), pin_chan); } - TEST_F(ValuefactoryTest, test_create_const) { auto c1 = nir_imm_int(&b, 2); @@ -246,40 +239,46 @@ TEST_F(ValuefactoryTest, test_create_sysvalue) EXPECT_EQ(ic->chan(), 0); } - -class GetKCache: public ConstRegisterVisitor { +class GetKCache : public ConstRegisterVisitor { public: - void visit(const VirtualValue& value) {(void)value;} - void visit(const Register& value) {(void)value;}; - void visit(const LocalArray& value) {(void)value;} - void visit(const LocalArrayValue& value) {(void)value;} - void visit(const UniformValue& value) {(void)value; m_result = value.kcache_bank();} - void visit(const LiteralConstant& value) {(void)value;} - void visit(const InlineConstant& value) {(void)value;} + void visit(const VirtualValue& value) { (void)value; } + void visit(const Register& value) { (void)value; }; + void visit(const LocalArray& value) { (void)value; } + void visit(const LocalArrayValue& value) { (void)value; } + void visit(const UniformValue& value) + { + (void)value; + m_result = value.kcache_bank(); + } + void visit(const LiteralConstant& value) { (void)value; } + void visit(const InlineConstant& value) { (void)value; } - GetKCache() : m_result(0) {} + GetKCache(): + m_result(0) + { + } int m_result; }; ValuefactoryTest::ValuefactoryTest() { - memset(&options, 0, sizeof (options)); + memset(&options, 0, sizeof(options)); init_pool(); } - -void ValuefactoryTest::SetUp() +void +ValuefactoryTest::SetUp() { glsl_type_singleton_init_or_ref(); b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, &options, "test shader"); factory = new ValueFactory(); } -void ValuefactoryTest::TearDown() +void +ValuefactoryTest::TearDown() { ralloc_free(b.shader); glsl_type_singleton_decref(); release_pool(); } -