diff --git a/src/intel/compiler/brw_eu_defines.h b/src/intel/compiler/brw_eu_defines.h index da12d1b968b..264994803b4 100644 --- a/src/intel/compiler/brw_eu_defines.h +++ b/src/intel/compiler/brw_eu_defines.h @@ -489,6 +489,14 @@ enum opcode { */ SHADER_OPCODE_FIND_LAST_LIVE_CHANNEL, + /** + * Return the current execution mask and assign it to the first component + * of the destination. + * + * \sa opcode::FS_OPCODE_LOAD_LIVE_CHANNELS + */ + SHADER_OPCODE_LOAD_LIVE_CHANNELS, + /** * Return the current execution mask in the specified flag subregister. * Can be CSE'ed more easily than a plain MOV from the ce0 ARF register. diff --git a/src/intel/compiler/brw_fs_cse.cpp b/src/intel/compiler/brw_fs_cse.cpp index 31e5a1b6983..296c517362c 100644 --- a/src/intel/compiler/brw_fs_cse.cpp +++ b/src/intel/compiler/brw_fs_cse.cpp @@ -78,6 +78,7 @@ is_expression(const fs_visitor *v, const fs_inst *const inst) case FS_OPCODE_LINTERP: case SHADER_OPCODE_FIND_LIVE_CHANNEL: case SHADER_OPCODE_FIND_LAST_LIVE_CHANNEL: + case SHADER_OPCODE_LOAD_LIVE_CHANNELS: case FS_OPCODE_LOAD_LIVE_CHANNELS: case SHADER_OPCODE_BROADCAST: case SHADER_OPCODE_MOV_INDIRECT: diff --git a/src/intel/compiler/brw_fs_generator.cpp b/src/intel/compiler/brw_fs_generator.cpp index e3bd734e9d8..15c70a23071 100644 --- a/src/intel/compiler/brw_fs_generator.cpp +++ b/src/intel/compiler/brw_fs_generator.cpp @@ -2269,6 +2269,10 @@ fs_generator::generate_code(const cfg_t *cfg, int dispatch_width, brw_find_live_channel(p, dst, true); break; + case SHADER_OPCODE_LOAD_LIVE_CHANNELS: + unreachable("Should be lowered by lower_find_live_channel()"); + break; + case FS_OPCODE_LOAD_LIVE_CHANNELS: { assert(devinfo->ver >= 8); assert(inst->force_writemask_all && inst->group == 0); diff --git a/src/intel/compiler/brw_fs_lower.cpp b/src/intel/compiler/brw_fs_lower.cpp index e573fb21b56..96992837f44 100644 --- a/src/intel/compiler/brw_fs_lower.cpp +++ b/src/intel/compiler/brw_fs_lower.cpp @@ -473,7 +473,8 @@ brw_fs_lower_find_live_channel(fs_visitor &s) foreach_block_and_inst_safe(block, fs_inst, inst, s.cfg) { if (inst->opcode != SHADER_OPCODE_FIND_LIVE_CHANNEL && - inst->opcode != SHADER_OPCODE_FIND_LAST_LIVE_CHANNEL) + inst->opcode != SHADER_OPCODE_FIND_LAST_LIVE_CHANNEL && + inst->opcode != SHADER_OPCODE_LOAD_LIVE_CHANNELS) continue; bool first = inst->opcode == SHADER_OPCODE_FIND_LIVE_CHANNEL; @@ -515,13 +516,25 @@ brw_fs_lower_find_live_channel(fs_visitor &s) exec_mask = mask; } - if (first) { + switch (inst->opcode) { + case SHADER_OPCODE_FIND_LIVE_CHANNEL: ubld.FBL(inst->dst, exec_mask); - } else { + break; + + case SHADER_OPCODE_FIND_LAST_LIVE_CHANNEL: { fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 1); ubld.UNDEF(tmp); ubld.LZD(tmp, exec_mask); ubld.ADD(inst->dst, negate(tmp), brw_imm_uw(31)); + break; + } + + case SHADER_OPCODE_LOAD_LIVE_CHANNELS: + ubld.MOV(inst->dst, exec_mask); + break; + + default: + unreachable("Impossible."); } inst->remove(block); diff --git a/src/intel/compiler/brw_fs_nir.cpp b/src/intel/compiler/brw_fs_nir.cpp index 7f043c0fbe9..2c5c3927124 100644 --- a/src/intel/compiler/brw_fs_nir.cpp +++ b/src/intel/compiler/brw_fs_nir.cpp @@ -7282,6 +7282,21 @@ fs_nir_emit_intrinsic(nir_to_brw_state &ntb, } case nir_intrinsic_ballot: { + if (instr->def.bit_size > 32) { + dest.type = BRW_REGISTER_TYPE_UQ; + } else { + dest.type = BRW_REGISTER_TYPE_UD; + } + + /* Implement a fast-path for ballot(true). */ + if (nir_src_is_const(instr->src[0]) && + nir_src_as_bool(instr->src[0])) { + fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD); + bld.exec_all().emit(SHADER_OPCODE_LOAD_LIVE_CHANNELS, tmp); + bld.MOV(dest, fs_reg(component(tmp, 0))); + break; + } + const fs_reg value = retype(get_nir_src(ntb, instr->src[0]), BRW_REGISTER_TYPE_UD); struct brw_reg flag = brw_flag_reg(0, 0); @@ -7291,12 +7306,6 @@ fs_nir_emit_intrinsic(nir_to_brw_state &ntb, bld.exec_all().group(1, 0).MOV(flag, retype(brw_imm_ud(0u), flag.type)); bld.CMP(bld.null_reg_ud(), value, brw_imm_ud(0u), BRW_CONDITIONAL_NZ); - - if (instr->def.bit_size > 32) { - dest.type = BRW_REGISTER_TYPE_UQ; - } else { - dest.type = BRW_REGISTER_TYPE_UD; - } bld.MOV(dest, flag); break; } diff --git a/src/intel/compiler/brw_ir_performance.cpp b/src/intel/compiler/brw_ir_performance.cpp index 9da5104dd6a..e94006cca65 100644 --- a/src/intel/compiler/brw_ir_performance.cpp +++ b/src/intel/compiler/brw_ir_performance.cpp @@ -723,6 +723,7 @@ namespace { case SHADER_OPCODE_FIND_LIVE_CHANNEL: case SHADER_OPCODE_FIND_LAST_LIVE_CHANNEL: + case SHADER_OPCODE_LOAD_LIVE_CHANNELS: if (devinfo->ver >= 11) return calculate_desc(info, EU_UNIT_FPU, 2, 0, 0, 2, 0, 0, 10, 6 /* XXX */, 14 /* XXX */, 0, 0); diff --git a/src/intel/compiler/brw_schedule_instructions.cpp b/src/intel/compiler/brw_schedule_instructions.cpp index ff2cc6ff997..01d1243bc77 100644 --- a/src/intel/compiler/brw_schedule_instructions.cpp +++ b/src/intel/compiler/brw_schedule_instructions.cpp @@ -1176,6 +1176,7 @@ has_cross_lane_access(const fs_inst *inst) inst->opcode == SHADER_OPCODE_CLUSTER_BROADCAST || inst->opcode == SHADER_OPCODE_SHUFFLE || inst->opcode == FS_OPCODE_LOAD_LIVE_CHANNELS || + inst->opcode == SHADER_OPCODE_LOAD_LIVE_CHANNELS || inst->opcode == SHADER_OPCODE_FIND_LAST_LIVE_CHANNEL || inst->opcode == SHADER_OPCODE_FIND_LIVE_CHANNEL) return true; diff --git a/src/intel/compiler/brw_shader.cpp b/src/intel/compiler/brw_shader.cpp index ed260ebe468..f56ae8d68d1 100644 --- a/src/intel/compiler/brw_shader.cpp +++ b/src/intel/compiler/brw_shader.cpp @@ -376,8 +376,10 @@ brw_instruction_name(const struct brw_isa_info *isa, enum opcode op) return "find_live_channel"; case SHADER_OPCODE_FIND_LAST_LIVE_CHANNEL: return "find_last_live_channel"; - case FS_OPCODE_LOAD_LIVE_CHANNELS: + case SHADER_OPCODE_LOAD_LIVE_CHANNELS: return "load_live_channels"; + case FS_OPCODE_LOAD_LIVE_CHANNELS: + return "fs_load_live_channels"; case SHADER_OPCODE_BROADCAST: return "broadcast";