diff --git a/src/intel/compiler/brw_builder.h b/src/intel/compiler/brw_builder.h index 4a0d869b1ab..577f595b626 100644 --- a/src/intel/compiler/brw_builder.h +++ b/src/intel/compiler/brw_builder.h @@ -1069,3 +1069,11 @@ void brw_check_dynamic_msaa_flag(const brw_builder &bld, const struct brw_wm_prog_data *wm_prog_data, enum intel_msaa_flags flag); + +inline brw_inst * +brw_transform_inst(const brw_builder &bld, brw_inst *inst, + enum opcode new_opcode, + unsigned new_num_srcs = UINT_MAX) +{ + return brw_transform_inst(*bld.shader, inst, new_opcode, new_num_srcs); +} diff --git a/src/intel/compiler/brw_cfg.cpp b/src/intel/compiler/brw_cfg.cpp index a6e0a9485e7..3d4d9fb033f 100644 --- a/src/intel/compiler/brw_cfg.cpp +++ b/src/intel/compiler/brw_cfg.cpp @@ -94,8 +94,7 @@ void bblock_t::remove(brw_inst *inst) { if (brw_exec_list_is_singular(&instructions)) { - inst->opcode = BRW_OPCODE_NOP; - inst->resize_sources(0); + inst = brw_transform_inst(*cfg->s, inst, BRW_OPCODE_NOP); inst->dst = brw_reg(); inst->size_written = 0; return; diff --git a/src/intel/compiler/brw_compile_fs.cpp b/src/intel/compiler/brw_compile_fs.cpp index 15a16c305e3..dd10fdd830f 100644 --- a/src/intel/compiler/brw_compile_fs.cpp +++ b/src/intel/compiler/brw_compile_fs.cpp @@ -1321,11 +1321,10 @@ brw_assign_urb_setup(brw_shader &s) /* Offset all the urb_setup[] index by the actual position of the * setup regs, now that the location of the constants has been chosen. */ - foreach_block_and_inst(block, brw_inst, inst, s.cfg) { + foreach_block_and_inst_safe(block, brw_inst, inst, s.cfg) { if (inst->opcode == FS_OPCODE_READ_ATTRIBUTE_PAYLOAD) { brw_reg offset = inst->src[0]; - inst->resize_sources(3); - inst->opcode = SHADER_OPCODE_MOV_INDIRECT; + inst = brw_transform_inst(s, inst, SHADER_OPCODE_MOV_INDIRECT, 3); inst->src[0] = retype(brw_vec8_grf(urb_start, 0), BRW_TYPE_UD); inst->src[1] = offset; inst->src[2] = brw_imm_ud(REG_SIZE * 2 * 32); diff --git a/src/intel/compiler/brw_inst.cpp b/src/intel/compiler/brw_inst.cpp index 53087e0e5da..852de8d7c8b 100644 --- a/src/intel/compiler/brw_inst.cpp +++ b/src/intel/compiler/brw_inst.cpp @@ -80,19 +80,37 @@ brw_clone_inst(brw_shader &s, const brw_inst *inst) return clone; } -void -brw_inst::resize_sources(uint8_t num_sources) +static unsigned +brw_num_sources_for_opcode(const brw_shader &s, enum opcode opcode) { - if (this->sources < num_sources) { - brw_reg *new_src = ralloc_array(this, brw_reg, num_sources); + const struct opcode_desc *desc = + brw_opcode_desc(&s.compiler->isa, opcode); + if (desc) + return desc->nsrc; + if (opcode == SHADER_OPCODE_SEND) + return SEND_NUM_SRCS; + return -1; +} - for (unsigned i = 0; i < this->sources; i++) - new_src[i] = this->src[i]; +brw_inst * +brw_transform_inst(brw_shader &s, brw_inst *inst, enum opcode new_opcode, + unsigned new_num_sources) +{ + inst->opcode = new_opcode; + if (new_num_sources == UINT_MAX) + new_num_sources = brw_num_sources_for_opcode(s, new_opcode); + assert(new_num_sources != UINT_MAX); - this->src = new_src; + if (new_num_sources > inst->sources) { + brw_reg *new_src = ralloc_array(inst, brw_reg, new_num_sources); + for (unsigned i = 0; i < inst->sources; i++) + new_src[i] = inst->src[i]; + inst->src = new_src; } - this->sources = num_sources; + inst->sources = new_num_sources; + + return inst; } bool diff --git a/src/intel/compiler/brw_inst.h b/src/intel/compiler/brw_inst.h index 3aac3f78e32..bc64c80f6fc 100644 --- a/src/intel/compiler/brw_inst.h +++ b/src/intel/compiler/brw_inst.h @@ -37,6 +37,7 @@ #define MAX_VGRF_SIZE(devinfo) ((devinfo)->ver >= 20 ? 40 : 20) struct bblock_t; +struct brw_shader; struct brw_inst : public brw_exec_node { public: @@ -47,8 +48,6 @@ public: static void* operator new(size_t size, void *ptr) { return ptr; } static void operator delete(void *p) {} - void resize_sources(uint8_t num_sources); - bool is_send() const; bool is_payload(unsigned arg) const; bool is_partial_write(unsigned grf_size = REG_SIZE) const; diff --git a/src/intel/compiler/brw_lower.cpp b/src/intel/compiler/brw_lower.cpp index 59c0c7fe6c8..96b07015a4a 100644 --- a/src/intel/compiler/brw_lower.cpp +++ b/src/intel/compiler/brw_lower.cpp @@ -162,10 +162,9 @@ brw_lower_csel(brw_shader &s) ibld.CMP(retype(brw_null_reg(), orig_type), inst->src[2], zero, inst->conditional_mod); - inst->opcode = BRW_OPCODE_SEL; + inst = brw_transform_inst(s, inst, BRW_OPCODE_SEL, 2); inst->predicate = BRW_PREDICATE_NORMAL; inst->conditional_mod = BRW_CONDITIONAL_NONE; - inst->resize_sources(2); progress = true; } else if (new_type != orig_type) { inst->src[0].type = new_type; @@ -364,10 +363,9 @@ lower_derivative(brw_shader &s, brw_inst *inst, ubld.emit(SHADER_OPCODE_QUAD_SWIZZLE, tmp0, inst->src[0], brw_imm_ud(swz0)); ubld.emit(SHADER_OPCODE_QUAD_SWIZZLE, tmp1, inst->src[0], brw_imm_ud(swz1)); - inst->resize_sources(2); + inst = brw_transform_inst(s, inst, BRW_OPCODE_ADD); inst->src[0] = negate(tmp0); inst->src[1] = tmp1; - inst->opcode = BRW_OPCODE_ADD; return true; } @@ -384,7 +382,7 @@ brw_lower_derivatives(brw_shader &s) if (s.devinfo->verx10 < 125) return false; - foreach_block_and_inst(block, brw_inst, inst, s.cfg) { + foreach_block_and_inst_safe(block, brw_inst, inst, s.cfg) { if (inst->opcode == FS_OPCODE_DDX_COARSE) progress |= lower_derivative(s, inst, BRW_SWIZZLE_XXXX, BRW_SWIZZLE_YYYY); @@ -615,8 +613,7 @@ brw_lower_bfloat_conversion(brw_shader &s, brw_inst *inst) * not work since it doesn't preserve -0.0f! */ assert(inst->src[0].type == BRW_TYPE_F); - inst->resize_sources(2); - inst->opcode = BRW_OPCODE_ADD; + inst = brw_transform_inst(s, inst, BRW_OPCODE_ADD); inst->src[1] = brw_imm_f(-0.0f); return true; diff --git a/src/intel/compiler/brw_lower_logical_sends.cpp b/src/intel/compiler/brw_lower_logical_sends.cpp index 83ea4ae0bd1..d2701019836 100644 --- a/src/intel/compiler/brw_lower_logical_sends.cpp +++ b/src/intel/compiler/brw_lower_logical_sends.cpp @@ -50,7 +50,7 @@ lower_urb_read_logical_send(const brw_builder &bld, brw_inst *inst) brw_reg payload = retype(brw_allocate_vgrf_units(*bld.shader, header_size), BRW_TYPE_F); bld.LOAD_PAYLOAD(payload, payload_sources, header_size, header_size); - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->header_size = header_size; inst->sfid = BRW_SFID_URB; @@ -65,8 +65,6 @@ lower_urb_read_logical_send(const brw_builder &bld, brw_inst *inst) inst->ex_mlen = 0; inst->send_is_volatile = true; - inst->resize_sources(SEND_NUM_SRCS); - inst->src[SEND_SRC_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = payload; @@ -116,15 +114,14 @@ lower_urb_read_logical_send_xe2(const brw_builder &bld, brw_inst *inst) LSC_CACHE(devinfo, LOAD, L1UC_L3UC)); /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->mlen = lsc_msg_addr_len(devinfo, LSC_ADDR_SIZE_A32, inst->exec_size); inst->ex_mlen = 0; inst->header_size = 0; inst->send_has_side_effects = true; inst->send_is_volatile = false; - inst->resize_sources(SEND_NUM_SRCS); - inst->src[SEND_SRC_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = payload; @@ -169,7 +166,7 @@ lower_urb_write_logical_send(const brw_builder &bld, brw_inst *inst) delete [] payload_sources; - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->header_size = header_size; inst->dst = brw_null_reg(); @@ -185,8 +182,6 @@ lower_urb_write_logical_send(const brw_builder &bld, brw_inst *inst) inst->ex_mlen = 0; inst->send_has_side_effects = true; - inst->resize_sources(SEND_NUM_SRCS); - inst->src[SEND_SRC_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = payload; @@ -254,15 +249,13 @@ lower_urb_write_logical_send_xe2(const brw_builder &bld, brw_inst *inst) /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->mlen = lsc_msg_addr_len(devinfo, LSC_ADDR_SIZE_A32, inst->exec_size); inst->ex_mlen = ex_mlen; inst->header_size = 0; inst->send_has_side_effects = true; inst->send_is_volatile = false; - inst->resize_sources(SEND_NUM_SRCS); - inst->src[SEND_SRC_DESC] = desc; inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = payload; @@ -502,9 +495,9 @@ lower_fb_write_logical_send(const brw_builder &bld, brw_inst *inst, } inst->ex_desc = ex_desc; - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->sfid = BRW_SFID_RENDER_CACHE; - inst->resize_sources(SEND_NUM_SRCS); + inst->src[SEND_SRC_DESC] = desc; inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = payload; @@ -588,8 +581,7 @@ lower_fb_read_logical_send(const brw_builder &bld, brw_inst *inst, component(header, 0), brw_imm_ud(~INTEL_MASK(14, 11))); - inst->opcode = SHADER_OPCODE_SEND; - inst->resize_sources(SEND_NUM_SRCS); + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->src[SEND_SRC_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = header; @@ -1172,8 +1164,8 @@ lower_sampler_logical_send(const brw_builder &bld, brw_inst *inst, } /* Generate the SEND. */ - inst->opcode = SHADER_OPCODE_SEND; - inst->resize_sources(SEND_NUM_SRCS); + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->mlen = mlen; inst->header_size = header_size; inst->sfid = BRW_SFID_SAMPLER; @@ -1681,7 +1673,9 @@ lower_lsc_memory_logical_send(const brw_builder &bld, brw_inst *inst) setup_lsc_surface_descriptors(bld, inst, inst->desc, binding, base_offset); - inst->opcode = SHADER_OPCODE_SEND; + + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->mlen = lsc_msg_addr_len(devinfo, addr_size, inst->exec_size * coord_components); inst->ex_mlen = ex_mlen; @@ -1689,8 +1683,6 @@ lower_lsc_memory_logical_send(const brw_builder &bld, brw_inst *inst) inst->send_has_side_effects = has_side_effects; inst->send_is_volatile = !has_side_effects || volatile_access; - inst->resize_sources(SEND_NUM_SRCS); - /* Finally, the payload */ inst->src[SEND_SRC_PAYLOAD1] = payload; inst->src[SEND_SRC_PAYLOAD2] = payload2; @@ -1965,7 +1957,7 @@ lower_hdc_memory_logical_send(const brw_builder &bld, brw_inst *inst) assert(sfid); /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->sfid = sfid; inst->mlen = mlen; inst->ex_mlen = ex_mlen; @@ -1978,8 +1970,6 @@ lower_hdc_memory_logical_send(const brw_builder &bld, brw_inst *inst) inst->exec_size = components > 8 ? 16 : 8; } - inst->resize_sources(SEND_NUM_SRCS); - /* Set up descriptors */ switch (binding_type) { case LSC_ADDR_SURFTYPE_FLAT: @@ -2046,9 +2036,9 @@ lower_lsc_varying_pull_constant_logical_send(const brw_builder &bld, assert(alignment_B.file == IMM); unsigned alignment = alignment_B.ud; - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->sfid = BRW_SFID_UGM; - inst->resize_sources(SEND_NUM_SRCS); inst->send_ex_bso = surf_type == LSC_ADDR_SURFTYPE_BSS && compiler->extended_bindless_surface_offset; @@ -2130,9 +2120,9 @@ lower_varying_pull_constant_logical_send(const brw_builder &bld, brw_inst *inst) assert(inst->src[PULL_VARYING_CONSTANT_SRC_ALIGNMENT].file == IMM); unsigned alignment = inst->src[PULL_VARYING_CONSTANT_SRC_ALIGNMENT].ud; - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->mlen = inst->exec_size / 8; - inst->resize_sources(SEND_NUM_SRCS); /* src[SEND_SRC_DESC/EX_DESC] are filled by setup_surface_descriptors() */ inst->src[SEND_SRC_PAYLOAD1] = ubo_offset; @@ -2300,7 +2290,7 @@ lower_interpolator_logical_send(const brw_builder &bld, brw_inst *inst, } } - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->sfid = BRW_SFID_PIXEL_INTERPOLATOR; inst->desc = desc_imm; inst->ex_desc = 0; @@ -2309,8 +2299,6 @@ lower_interpolator_logical_send(const brw_builder &bld, brw_inst *inst, inst->send_has_side_effects = false; inst->send_is_volatile = false; - inst->resize_sources(SEND_NUM_SRCS); - inst->src[SEND_SRC_DESC] = component(desc, 0); inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = payload; @@ -2379,7 +2367,7 @@ lower_btd_logical_send(const brw_builder &bld, brw_inst *inst) } /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->mlen = mlen; inst->ex_mlen = ex_mlen; inst->header_size = 0; /* HW docs require has_header = false */ @@ -2391,8 +2379,6 @@ lower_btd_logical_send(const brw_builder &bld, brw_inst *inst) inst->desc = brw_btd_spawn_desc(devinfo, inst->exec_size, GEN_RT_BTD_MESSAGE_SPAWN); - inst->resize_sources(SEND_NUM_SRCS); - inst->src[SEND_SRC_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = header; @@ -2487,7 +2473,8 @@ lower_trace_ray_logical_send(const brw_builder &bld, brw_inst *inst) } /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->mlen = mlen; inst->ex_mlen = ex_mlen; inst->header_size = 0; /* HW docs require has_header = false */ @@ -2498,8 +2485,6 @@ lower_trace_ray_logical_send(const brw_builder &bld, brw_inst *inst) inst->sfid = BRW_SFID_RAY_TRACE_ACCELERATOR; inst->desc = brw_rt_trace_ray_desc(devinfo, inst->exec_size); - inst->resize_sources(SEND_NUM_SRCS); - inst->src[SEND_SRC_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = header; @@ -2519,9 +2504,9 @@ lower_get_buffer_size(const brw_builder &bld, brw_inst *inst) brw_reg surface_handle = inst->src[GET_BUFFER_SIZE_SRC_SURFACE_HANDLE]; brw_reg lod = bld.move_to_vgrf(inst->src[GET_BUFFER_SIZE_SRC_LOD], 1); - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->mlen = inst->exec_size / 8; - inst->resize_sources(SEND_NUM_SRCS); inst->ex_mlen = 0; inst->ex_desc = 0; @@ -2555,8 +2540,8 @@ lower_lsc_memory_fence_and_interlock(const brw_builder &bld, brw_inst *inst) assert(inst->size_written == reg_unit(devinfo) * REG_SIZE); - inst->opcode = SHADER_OPCODE_SEND; - inst->resize_sources(SEND_NUM_SRCS); + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->check_tdr = interlock; inst->send_has_side_effects = true; @@ -2631,8 +2616,8 @@ lower_hdc_memory_fence_and_interlock(const brw_builder &bld, brw_inst *inst) assert(inst->size_written == (commit_enable ? REG_SIZE : 0)); - inst->opcode = SHADER_OPCODE_SEND; - inst->resize_sources(SEND_NUM_SRCS); + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->check_tdr = interlock; inst->send_has_side_effects = true; @@ -2824,7 +2809,8 @@ brw_lower_uniform_pull_constant_loads(brw_shader &s) LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS)); /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(ubld, inst, SHADER_OPCODE_SEND); + inst->mlen = lsc_msg_addr_len(devinfo, LSC_ADDR_SIZE_A32, 1); inst->send_ex_bso = surface_handle.file != BAD_FILE && s.compiler->extended_bindless_surface_offset; @@ -2835,8 +2821,6 @@ brw_lower_uniform_pull_constant_loads(brw_shader &s) inst->exec_size = 1; /* Finally, the payload */ - - inst->resize_sources(SEND_NUM_SRCS); setup_lsc_surface_descriptors(ubld, inst, inst->desc, surface.file != BAD_FILE ? surface : surface_handle, 0); @@ -2854,8 +2838,9 @@ brw_lower_uniform_pull_constant_loads(brw_shader &s) ubld.group(1, 0).MOV(component(header, 2), brw_imm_ud(offset_B.ud / 16)); + inst = brw_transform_inst(ubld, inst, SHADER_OPCODE_SEND); + inst->sfid = BRW_SFID_HDC_READ_ONLY; - inst->opcode = SHADER_OPCODE_SEND; inst->header_size = 1; inst->mlen = 1; @@ -2863,8 +2848,6 @@ brw_lower_uniform_pull_constant_loads(brw_shader &s) brw_dp_oword_block_rw_desc(devinfo, true /* align_16B */, size_B.ud / 4, false /* write */); - inst->resize_sources(SEND_NUM_SRCS); - setup_surface_descriptors(ubld, inst, desc, surface, surface_handle); inst->src[SEND_SRC_PAYLOAD1] = header; diff --git a/src/intel/compiler/brw_opt.cpp b/src/intel/compiler/brw_opt.cpp index 6a63777d017..b0dc1817ccb 100644 --- a/src/intel/compiler/brw_opt.cpp +++ b/src/intel/compiler/brw_opt.cpp @@ -500,9 +500,8 @@ brw_opt_eliminate_find_live_channel(brw_shader &s) case SHADER_OPCODE_FIND_LIVE_CHANNEL: if (depth == 0) { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); - inst->resize_sources(1); inst->src[0] = brw_imm_ud(0u); inst->force_writemask_all = true; @@ -528,14 +527,15 @@ brw_opt_eliminate_find_live_channel(brw_shader &s) inst->dst.file == bcast->src[1].file && inst->dst.nr == bcast->src[1].nr && inst->dst.offset == bcast->src[1].offset) { - bcast->opcode = BRW_OPCODE_MOV; + + bcast = brw_transform_inst(s, bcast, BRW_OPCODE_MOV); + if (!is_uniform(bcast->src[0])) bcast->src[0] = component(bcast->src[0], 0); bcast->force_writemask_all = true; bcast->exec_size = 8 * reg_unit(s.devinfo); assert(bcast->size_written == bcast->dst.component_size(bcast->exec_size)); - bcast->resize_sources(1); } } break; @@ -645,7 +645,9 @@ brw_opt_send_to_send_gather(brw_shader &s) continue; } - inst->resize_sources(SEND_GATHER_SRC_PAYLOAD + num_payload_sources); + inst = brw_transform_inst(s, inst, SHADER_OPCODE_SEND_GATHER, + SEND_GATHER_SRC_PAYLOAD + num_payload_sources); + /* Sources 0 and 1 remain the same. Source 2 will be filled * after register allocation. */ @@ -660,7 +662,6 @@ brw_opt_send_to_send_gather(brw_shader &s) } assert(idx == inst->sources); - inst->opcode = SHADER_OPCODE_SEND_GATHER; inst->mlen = 0; inst->ex_mlen = 0; @@ -763,8 +764,8 @@ brw_opt_send_gather_to_send(brw_shader &s) continue; } - inst->resize_sources(SEND_NUM_SRCS); - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(s, inst, SHADER_OPCODE_SEND); + inst->src[SEND_SRC_PAYLOAD1] = payload1; inst->src[SEND_SRC_PAYLOAD2] = payload2; inst->mlen = payload1_len * unit; diff --git a/src/intel/compiler/brw_opt_algebraic.cpp b/src/intel/compiler/brw_opt_algebraic.cpp index 298a4a2dadc..4a17a5855cc 100644 --- a/src/intel/compiler/brw_opt_algebraic.cpp +++ b/src/intel/compiler/brw_opt_algebraic.cpp @@ -86,7 +86,7 @@ brw_imm_for_type(uint64_t value, enum brw_reg_type type) /** * Converts a MAD to an ADD by folding the multiplicand sources. */ -static void +static brw_inst * fold_multiplicands_of_MAD(brw_shader &s, brw_inst *inst) { assert(inst->opcode == BRW_OPCODE_MAD); @@ -128,8 +128,7 @@ fold_multiplicands_of_MAD(brw_shader &s, brw_inst *inst) } } - inst->opcode = BRW_OPCODE_ADD; - inst->resize_sources(2); + return brw_transform_inst(s, inst, BRW_OPCODE_ADD); } bool @@ -188,7 +187,7 @@ brw_opt_constant_fold_instruction(brw_shader &s, brw_inst *inst) !brw_type_is_vector_imm(inst->src[0].type) && !brw_type_is_vector_imm(inst->src[1].type) && !brw_type_is_vector_imm(inst->src[2].type)) { - fold_multiplicands_of_MAD(s, inst); + inst = fold_multiplicands_of_MAD(s, inst); assert(inst->opcode == BRW_OPCODE_ADD); ASSERTED bool folded = brw_opt_constant_fold_instruction(s, inst); @@ -279,9 +278,9 @@ brw_opt_constant_fold_instruction(brw_shader &s, brw_inst *inst) case SHADER_OPCODE_BROADCAST: if (inst->src[0].file == IMM) { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); + inst->force_writemask_all = true; - inst->resize_sources(1); /* The destination of BROADCAST will always be is_scalar, so the * allocation will always be REG_SIZE * reg_unit. Adjust the @@ -316,9 +315,9 @@ brw_opt_constant_fold_instruction(brw_shader &s, brw_inst *inst) if (result.file != BAD_FILE) { assert(result.file == IMM); - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); + inst->src[0] = result; - inst->resize_sources(1); return true; } @@ -341,8 +340,7 @@ brw_opt_algebraic(brw_shader &s) case BRW_OPCODE_ADD: if (brw_type_is_int(inst->src[1].type) && inst->src[1].is_zero()) { - inst->opcode = BRW_OPCODE_MOV; - inst->resize_sources(1); + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); progress = true; } @@ -373,14 +371,12 @@ brw_opt_algebraic(brw_shader &s) assert(src.file != BAD_FILE); if (uint32_t(sum) == 0) { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); inst->src[0] = src; - inst->resize_sources(1); } else { - inst->opcode = BRW_OPCODE_ADD; + inst = brw_transform_inst(s, inst, BRW_OPCODE_ADD); inst->src[0] = src; inst->src[1] = brw_imm_ud(sum); - inst->resize_sources(2); } progress = true; @@ -390,9 +386,8 @@ brw_opt_algebraic(brw_shader &s) */ for (unsigned i = 0; i < 3; i++) { if (inst->src[i].is_zero()) { - inst->opcode = BRW_OPCODE_ADD; inst->src[i] = inst->src[2]; - inst->resize_sources(2); + inst = brw_transform_inst(s, inst, BRW_OPCODE_ADD); progress = true; break; } @@ -462,12 +457,14 @@ brw_opt_algebraic(brw_shader &s) break; for (unsigned i = 0; i < 2; i++) { + bool found = false; + /* a * 1 = a */ if (inst->src[i].is_one()) { - inst->opcode = BRW_OPCODE_MOV; + found = true; } else if (inst->src[i].is_negative_one()) { /* a * -1 = -a */ - inst->opcode = BRW_OPCODE_MOV; + found = true; /* If the source other than the -1 is immediate, just * toggling the negation flag will not work. Due to the @@ -478,12 +475,12 @@ brw_opt_algebraic(brw_shader &s) inst->src[1 - i].negate = !inst->src[1 - i].negate; } - if (inst->opcode == BRW_OPCODE_MOV) { + if (found) { /* If the literal 1 was src0, put the old src1 in src0. */ if (i == 0) inst->src[0] = inst->src[1]; - inst->resize_sources(1); + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); progress = true; break; } @@ -507,7 +504,7 @@ brw_opt_algebraic(brw_shader &s) if (!inst->src[0].negate) inst->conditional_mod = brw_negate_cmod(inst->conditional_mod); - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); inst->src[0].negate = false; progress = true; } @@ -520,12 +517,11 @@ brw_opt_algebraic(brw_shader &s) * or 'OR r0, ~r1, ~r1' should become a NOT instead of a MOV. */ if (inst->src[0].negate) { - inst->opcode = BRW_OPCODE_NOT; + inst = brw_transform_inst(s, inst, BRW_OPCODE_NOT); inst->src[0].negate = false; } else { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); } - inst->resize_sources(1); progress = true; break; } @@ -552,11 +548,10 @@ brw_opt_algebraic(brw_shader &s) if (inst->src[0].equals(inst->src[1]) && (!brw_type_is_float(inst->dst.type) || inst->conditional_mod == BRW_CONDITIONAL_NONE)) { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); inst->predicate = BRW_PREDICATE_NONE; inst->predicate_inverse = false; inst->conditional_mod = BRW_CONDITIONAL_NONE; - inst->resize_sources(1); progress = true; } else if (inst->saturate && inst->src[1].file == IMM) { switch (inst->conditional_mod) { @@ -565,9 +560,8 @@ brw_opt_algebraic(brw_shader &s) switch (inst->src[1].type) { case BRW_TYPE_F: if (inst->src[1].f >= 1.0f) { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); inst->conditional_mod = BRW_CONDITIONAL_NONE; - inst->resize_sources(1); progress = true; } break; @@ -580,9 +574,8 @@ brw_opt_algebraic(brw_shader &s) switch (inst->src[1].type) { case BRW_TYPE_F: if (inst->src[1].f <= 0.0f) { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); inst->conditional_mod = BRW_CONDITIONAL_NONE; - inst->resize_sources(1); progress = true; } break; @@ -653,15 +646,13 @@ brw_opt_algebraic(brw_shader &s) break; case BRW_CONDITIONAL_G: /* This is a contradtion. -abs(x) cannot be > 0. */ - inst->opcode = BRW_OPCODE_MOV; inst->src[0] = inst->src[1]; - inst->resize_sources(1); + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); progress = true; break; case BRW_CONDITIONAL_LE: /* This is a tautology. -abs(x) must be <= 0. */ - inst->opcode = BRW_OPCODE_MOV; - inst->resize_sources(1); + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); progress = true; break; case BRW_CONDITIONAL_Z: @@ -692,19 +683,18 @@ brw_opt_algebraic(brw_shader &s) } if (inst->src[1].is_one()) { - inst->opcode = BRW_OPCODE_ADD; inst->src[1] = inst->src[2]; - inst->resize_sources(2); + inst = brw_transform_inst(s, inst, BRW_OPCODE_ADD); progress = true; } else if (inst->src[2].is_one()) { - inst->opcode = BRW_OPCODE_ADD; - inst->resize_sources(2); + inst = brw_transform_inst(s, inst, BRW_OPCODE_ADD); progress = true; } break; case SHADER_OPCODE_BROADCAST: if (is_uniform(inst->src[0])) { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); + inst->force_writemask_all = true; /* The destination of BROADCAST will always be is_scalar, so the @@ -713,10 +703,8 @@ brw_opt_algebraic(brw_shader &s) */ inst->exec_size = 8 * reg_unit(devinfo); assert(inst->size_written == inst->dst.component_size(inst->exec_size)); - inst->resize_sources(1); progress = true; } else if (inst->src[1].file == IMM) { - inst->opcode = BRW_OPCODE_MOV; /* It's possible that the selected component will be too large and * overflow the register. This can happen if someone does a * readInvocation() from GLSL or SPIR-V and provides an OOB @@ -731,21 +719,21 @@ brw_opt_algebraic(brw_shader &s) inst->force_writemask_all = true; inst->exec_size = 8 * reg_unit(devinfo); assert(inst->size_written == inst->dst.component_size(inst->exec_size)); - inst->resize_sources(1); + + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); + progress = true; } break; case SHADER_OPCODE_SHUFFLE: if (is_uniform(inst->src[0])) { - inst->opcode = BRW_OPCODE_MOV; - inst->resize_sources(1); + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); progress = true; } else if (inst->src[1].file == IMM) { const unsigned comp = inst->src[1].ud & (inst->exec_size - 1); - inst->opcode = BRW_OPCODE_MOV; inst->src[0] = component(inst->src[0], comp); - inst->resize_sources(1); + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); progress = true; } break; @@ -770,8 +758,7 @@ brw_opt_algebraic(brw_shader &s) } if (progress) - s.invalidate_analysis(BRW_DEPENDENCY_INSTRUCTION_DATA_FLOW | - BRW_DEPENDENCY_INSTRUCTION_DETAIL); + s.invalidate_analysis(BRW_DEPENDENCY_INSTRUCTIONS); return progress; } diff --git a/src/intel/compiler/brw_opt_copy_propagation.cpp b/src/intel/compiler/brw_opt_copy_propagation.cpp index cf4451e9c46..0d1bf4aeaa9 100644 --- a/src/intel/compiler/brw_opt_copy_propagation.cpp +++ b/src/intel/compiler/brw_opt_copy_propagation.cpp @@ -1409,7 +1409,7 @@ opt_copy_propagation_local(brw_shader &s, linear_ctx *lin_ctx, const struct intel_device_info *devinfo = s.devinfo; bool progress = false; - foreach_inst_in_block(brw_inst, inst, block) { + foreach_inst_in_block_safe(brw_inst, inst, block) { /* The non-defs copy propagation passes should not be called while * LOAD_REG instructions still exist. */ diff --git a/src/intel/compiler/brw_opt_dead_code_eliminate.cpp b/src/intel/compiler/brw_opt_dead_code_eliminate.cpp index 0c852ddf24b..5e7470a32cf 100644 --- a/src/intel/compiler/brw_opt_dead_code_eliminate.cpp +++ b/src/intel/compiler/brw_opt_dead_code_eliminate.cpp @@ -134,7 +134,7 @@ brw_opt_dead_code_eliminate(brw_shader &s) if (inst->dst.is_null() && can_eliminate(devinfo, inst, flag_live) && !(inst->opcode == BRW_OPCODE_NOP && brw_exec_list_is_singular(&block->instructions))) { - inst->opcode = BRW_OPCODE_NOP; + inst = brw_transform_inst(s, inst, BRW_OPCODE_NOP); progress = true; } diff --git a/src/intel/compiler/brw_opt_register_coalesce.cpp b/src/intel/compiler/brw_opt_register_coalesce.cpp index 00357f38b13..f5442d50a84 100644 --- a/src/intel/compiler/brw_opt_register_coalesce.cpp +++ b/src/intel/compiler/brw_opt_register_coalesce.cpp @@ -268,12 +268,12 @@ brw_opt_register_coalesce(brw_shader &s) int *src_var = new int[live.max_vgrf_size]; const brw_def_analysis &defs = s.def_analysis.require(); - foreach_block_and_inst(block, brw_inst, inst, s.cfg) { + foreach_block_and_inst_safe(block, brw_inst, inst, s.cfg) { if (!is_coalesce_candidate(&s, inst)) continue; if (is_nop_mov(inst)) { - inst->opcode = BRW_OPCODE_NOP; + inst = brw_transform_inst(s, inst, BRW_OPCODE_NOP); progress = true; continue; } @@ -365,7 +365,7 @@ brw_opt_register_coalesce(brw_shader &s) continue; if (mov[i]->conditional_mod == BRW_CONDITIONAL_NONE) { - mov[i]->opcode = BRW_OPCODE_NOP; + mov[i] = brw_transform_inst(s, mov[i], BRW_OPCODE_NOP); mov[i]->dst = reg_undef; for (int j = 0; j < mov[i]->sources; j++) { mov[i]->src[j] = reg_undef; diff --git a/src/intel/compiler/brw_shader.h b/src/intel/compiler/brw_shader.h index 69269c0a854..8b99054dd48 100644 --- a/src/intel/compiler/brw_shader.h +++ b/src/intel/compiler/brw_shader.h @@ -374,3 +374,14 @@ bool brw_lower_load_reg(brw_shader &s); brw_inst *brw_new_inst(brw_shader &s, enum opcode opcode, unsigned exec_size, const brw_reg &dst, unsigned num_srcs); brw_inst *brw_clone_inst(brw_shader &s, const brw_inst *inst); + +/* Transform the opcode/num_sources of an instruction. All the fields in + * brw_inst are maintained and any previous sources still visible. Additional + * sources will be uninitialized. + * + * If new_num_srcs is UINT_MAX a default will be picked based on the opcode. + * Not all opcodes have a default. + */ +brw_inst *brw_transform_inst(brw_shader &s, brw_inst *inst, enum opcode new_opcode, + unsigned new_num_srcs = UINT_MAX); +