From e194909b3f9a15891c6684ea3728de837b86ac87 Mon Sep 17 00:00:00 2001 From: Caio Oliveira Date: Sat, 9 Aug 2025 16:23:01 -0700 Subject: [PATCH] brw: Add and use brw_transform_inst() The new function takes care of changing an instruction opcode and sources, which will allow later patches to tweak how allocations are done in those cases. Like the instruction allocation, this also takes a shader (or a builder, for it to get a shader). Reviewed-by: Lionel Landwerlin Reviewed-by: Kenneth Graunke Part-of: --- src/intel/compiler/brw_builder.h | 8 ++ src/intel/compiler/brw_cfg.cpp | 3 +- src/intel/compiler/brw_compile_fs.cpp | 5 +- src/intel/compiler/brw_inst.cpp | 34 ++++++-- src/intel/compiler/brw_inst.h | 3 +- src/intel/compiler/brw_lower.cpp | 11 +-- .../compiler/brw_lower_logical_sends.cpp | 81 +++++++----------- src/intel/compiler/brw_opt.cpp | 17 ++-- src/intel/compiler/brw_opt_algebraic.cpp | 83 ++++++++----------- .../compiler/brw_opt_copy_propagation.cpp | 2 +- .../compiler/brw_opt_dead_code_eliminate.cpp | 2 +- .../compiler/brw_opt_register_coalesce.cpp | 6 +- src/intel/compiler/brw_shader.h | 11 +++ 13 files changed, 134 insertions(+), 132 deletions(-) diff --git a/src/intel/compiler/brw_builder.h b/src/intel/compiler/brw_builder.h index 4a0d869b1ab..577f595b626 100644 --- a/src/intel/compiler/brw_builder.h +++ b/src/intel/compiler/brw_builder.h @@ -1069,3 +1069,11 @@ void brw_check_dynamic_msaa_flag(const brw_builder &bld, const struct brw_wm_prog_data *wm_prog_data, enum intel_msaa_flags flag); + +inline brw_inst * +brw_transform_inst(const brw_builder &bld, brw_inst *inst, + enum opcode new_opcode, + unsigned new_num_srcs = UINT_MAX) +{ + return brw_transform_inst(*bld.shader, inst, new_opcode, new_num_srcs); +} diff --git a/src/intel/compiler/brw_cfg.cpp b/src/intel/compiler/brw_cfg.cpp index a6e0a9485e7..3d4d9fb033f 100644 --- a/src/intel/compiler/brw_cfg.cpp +++ b/src/intel/compiler/brw_cfg.cpp @@ -94,8 +94,7 @@ void bblock_t::remove(brw_inst *inst) { if (brw_exec_list_is_singular(&instructions)) { - inst->opcode = BRW_OPCODE_NOP; - inst->resize_sources(0); + inst = brw_transform_inst(*cfg->s, inst, BRW_OPCODE_NOP); inst->dst = brw_reg(); inst->size_written = 0; return; diff --git a/src/intel/compiler/brw_compile_fs.cpp b/src/intel/compiler/brw_compile_fs.cpp index 15a16c305e3..dd10fdd830f 100644 --- a/src/intel/compiler/brw_compile_fs.cpp +++ b/src/intel/compiler/brw_compile_fs.cpp @@ -1321,11 +1321,10 @@ brw_assign_urb_setup(brw_shader &s) /* Offset all the urb_setup[] index by the actual position of the * setup regs, now that the location of the constants has been chosen. */ - foreach_block_and_inst(block, brw_inst, inst, s.cfg) { + foreach_block_and_inst_safe(block, brw_inst, inst, s.cfg) { if (inst->opcode == FS_OPCODE_READ_ATTRIBUTE_PAYLOAD) { brw_reg offset = inst->src[0]; - inst->resize_sources(3); - inst->opcode = SHADER_OPCODE_MOV_INDIRECT; + inst = brw_transform_inst(s, inst, SHADER_OPCODE_MOV_INDIRECT, 3); inst->src[0] = retype(brw_vec8_grf(urb_start, 0), BRW_TYPE_UD); inst->src[1] = offset; inst->src[2] = brw_imm_ud(REG_SIZE * 2 * 32); diff --git a/src/intel/compiler/brw_inst.cpp b/src/intel/compiler/brw_inst.cpp index 53087e0e5da..852de8d7c8b 100644 --- a/src/intel/compiler/brw_inst.cpp +++ b/src/intel/compiler/brw_inst.cpp @@ -80,19 +80,37 @@ brw_clone_inst(brw_shader &s, const brw_inst *inst) return clone; } -void -brw_inst::resize_sources(uint8_t num_sources) +static unsigned +brw_num_sources_for_opcode(const brw_shader &s, enum opcode opcode) { - if (this->sources < num_sources) { - brw_reg *new_src = ralloc_array(this, brw_reg, num_sources); + const struct opcode_desc *desc = + brw_opcode_desc(&s.compiler->isa, opcode); + if (desc) + return desc->nsrc; + if (opcode == SHADER_OPCODE_SEND) + return SEND_NUM_SRCS; + return -1; +} - for (unsigned i = 0; i < this->sources; i++) - new_src[i] = this->src[i]; +brw_inst * +brw_transform_inst(brw_shader &s, brw_inst *inst, enum opcode new_opcode, + unsigned new_num_sources) +{ + inst->opcode = new_opcode; + if (new_num_sources == UINT_MAX) + new_num_sources = brw_num_sources_for_opcode(s, new_opcode); + assert(new_num_sources != UINT_MAX); - this->src = new_src; + if (new_num_sources > inst->sources) { + brw_reg *new_src = ralloc_array(inst, brw_reg, new_num_sources); + for (unsigned i = 0; i < inst->sources; i++) + new_src[i] = inst->src[i]; + inst->src = new_src; } - this->sources = num_sources; + inst->sources = new_num_sources; + + return inst; } bool diff --git a/src/intel/compiler/brw_inst.h b/src/intel/compiler/brw_inst.h index 3aac3f78e32..bc64c80f6fc 100644 --- a/src/intel/compiler/brw_inst.h +++ b/src/intel/compiler/brw_inst.h @@ -37,6 +37,7 @@ #define MAX_VGRF_SIZE(devinfo) ((devinfo)->ver >= 20 ? 40 : 20) struct bblock_t; +struct brw_shader; struct brw_inst : public brw_exec_node { public: @@ -47,8 +48,6 @@ public: static void* operator new(size_t size, void *ptr) { return ptr; } static void operator delete(void *p) {} - void resize_sources(uint8_t num_sources); - bool is_send() const; bool is_payload(unsigned arg) const; bool is_partial_write(unsigned grf_size = REG_SIZE) const; diff --git a/src/intel/compiler/brw_lower.cpp b/src/intel/compiler/brw_lower.cpp index 59c0c7fe6c8..96b07015a4a 100644 --- a/src/intel/compiler/brw_lower.cpp +++ b/src/intel/compiler/brw_lower.cpp @@ -162,10 +162,9 @@ brw_lower_csel(brw_shader &s) ibld.CMP(retype(brw_null_reg(), orig_type), inst->src[2], zero, inst->conditional_mod); - inst->opcode = BRW_OPCODE_SEL; + inst = brw_transform_inst(s, inst, BRW_OPCODE_SEL, 2); inst->predicate = BRW_PREDICATE_NORMAL; inst->conditional_mod = BRW_CONDITIONAL_NONE; - inst->resize_sources(2); progress = true; } else if (new_type != orig_type) { inst->src[0].type = new_type; @@ -364,10 +363,9 @@ lower_derivative(brw_shader &s, brw_inst *inst, ubld.emit(SHADER_OPCODE_QUAD_SWIZZLE, tmp0, inst->src[0], brw_imm_ud(swz0)); ubld.emit(SHADER_OPCODE_QUAD_SWIZZLE, tmp1, inst->src[0], brw_imm_ud(swz1)); - inst->resize_sources(2); + inst = brw_transform_inst(s, inst, BRW_OPCODE_ADD); inst->src[0] = negate(tmp0); inst->src[1] = tmp1; - inst->opcode = BRW_OPCODE_ADD; return true; } @@ -384,7 +382,7 @@ brw_lower_derivatives(brw_shader &s) if (s.devinfo->verx10 < 125) return false; - foreach_block_and_inst(block, brw_inst, inst, s.cfg) { + foreach_block_and_inst_safe(block, brw_inst, inst, s.cfg) { if (inst->opcode == FS_OPCODE_DDX_COARSE) progress |= lower_derivative(s, inst, BRW_SWIZZLE_XXXX, BRW_SWIZZLE_YYYY); @@ -615,8 +613,7 @@ brw_lower_bfloat_conversion(brw_shader &s, brw_inst *inst) * not work since it doesn't preserve -0.0f! */ assert(inst->src[0].type == BRW_TYPE_F); - inst->resize_sources(2); - inst->opcode = BRW_OPCODE_ADD; + inst = brw_transform_inst(s, inst, BRW_OPCODE_ADD); inst->src[1] = brw_imm_f(-0.0f); return true; diff --git a/src/intel/compiler/brw_lower_logical_sends.cpp b/src/intel/compiler/brw_lower_logical_sends.cpp index 83ea4ae0bd1..d2701019836 100644 --- a/src/intel/compiler/brw_lower_logical_sends.cpp +++ b/src/intel/compiler/brw_lower_logical_sends.cpp @@ -50,7 +50,7 @@ lower_urb_read_logical_send(const brw_builder &bld, brw_inst *inst) brw_reg payload = retype(brw_allocate_vgrf_units(*bld.shader, header_size), BRW_TYPE_F); bld.LOAD_PAYLOAD(payload, payload_sources, header_size, header_size); - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->header_size = header_size; inst->sfid = BRW_SFID_URB; @@ -65,8 +65,6 @@ lower_urb_read_logical_send(const brw_builder &bld, brw_inst *inst) inst->ex_mlen = 0; inst->send_is_volatile = true; - inst->resize_sources(SEND_NUM_SRCS); - inst->src[SEND_SRC_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = payload; @@ -116,15 +114,14 @@ lower_urb_read_logical_send_xe2(const brw_builder &bld, brw_inst *inst) LSC_CACHE(devinfo, LOAD, L1UC_L3UC)); /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->mlen = lsc_msg_addr_len(devinfo, LSC_ADDR_SIZE_A32, inst->exec_size); inst->ex_mlen = 0; inst->header_size = 0; inst->send_has_side_effects = true; inst->send_is_volatile = false; - inst->resize_sources(SEND_NUM_SRCS); - inst->src[SEND_SRC_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = payload; @@ -169,7 +166,7 @@ lower_urb_write_logical_send(const brw_builder &bld, brw_inst *inst) delete [] payload_sources; - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->header_size = header_size; inst->dst = brw_null_reg(); @@ -185,8 +182,6 @@ lower_urb_write_logical_send(const brw_builder &bld, brw_inst *inst) inst->ex_mlen = 0; inst->send_has_side_effects = true; - inst->resize_sources(SEND_NUM_SRCS); - inst->src[SEND_SRC_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = payload; @@ -254,15 +249,13 @@ lower_urb_write_logical_send_xe2(const brw_builder &bld, brw_inst *inst) /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->mlen = lsc_msg_addr_len(devinfo, LSC_ADDR_SIZE_A32, inst->exec_size); inst->ex_mlen = ex_mlen; inst->header_size = 0; inst->send_has_side_effects = true; inst->send_is_volatile = false; - inst->resize_sources(SEND_NUM_SRCS); - inst->src[SEND_SRC_DESC] = desc; inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = payload; @@ -502,9 +495,9 @@ lower_fb_write_logical_send(const brw_builder &bld, brw_inst *inst, } inst->ex_desc = ex_desc; - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->sfid = BRW_SFID_RENDER_CACHE; - inst->resize_sources(SEND_NUM_SRCS); + inst->src[SEND_SRC_DESC] = desc; inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = payload; @@ -588,8 +581,7 @@ lower_fb_read_logical_send(const brw_builder &bld, brw_inst *inst, component(header, 0), brw_imm_ud(~INTEL_MASK(14, 11))); - inst->opcode = SHADER_OPCODE_SEND; - inst->resize_sources(SEND_NUM_SRCS); + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->src[SEND_SRC_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = header; @@ -1172,8 +1164,8 @@ lower_sampler_logical_send(const brw_builder &bld, brw_inst *inst, } /* Generate the SEND. */ - inst->opcode = SHADER_OPCODE_SEND; - inst->resize_sources(SEND_NUM_SRCS); + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->mlen = mlen; inst->header_size = header_size; inst->sfid = BRW_SFID_SAMPLER; @@ -1681,7 +1673,9 @@ lower_lsc_memory_logical_send(const brw_builder &bld, brw_inst *inst) setup_lsc_surface_descriptors(bld, inst, inst->desc, binding, base_offset); - inst->opcode = SHADER_OPCODE_SEND; + + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->mlen = lsc_msg_addr_len(devinfo, addr_size, inst->exec_size * coord_components); inst->ex_mlen = ex_mlen; @@ -1689,8 +1683,6 @@ lower_lsc_memory_logical_send(const brw_builder &bld, brw_inst *inst) inst->send_has_side_effects = has_side_effects; inst->send_is_volatile = !has_side_effects || volatile_access; - inst->resize_sources(SEND_NUM_SRCS); - /* Finally, the payload */ inst->src[SEND_SRC_PAYLOAD1] = payload; inst->src[SEND_SRC_PAYLOAD2] = payload2; @@ -1965,7 +1957,7 @@ lower_hdc_memory_logical_send(const brw_builder &bld, brw_inst *inst) assert(sfid); /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->sfid = sfid; inst->mlen = mlen; inst->ex_mlen = ex_mlen; @@ -1978,8 +1970,6 @@ lower_hdc_memory_logical_send(const brw_builder &bld, brw_inst *inst) inst->exec_size = components > 8 ? 16 : 8; } - inst->resize_sources(SEND_NUM_SRCS); - /* Set up descriptors */ switch (binding_type) { case LSC_ADDR_SURFTYPE_FLAT: @@ -2046,9 +2036,9 @@ lower_lsc_varying_pull_constant_logical_send(const brw_builder &bld, assert(alignment_B.file == IMM); unsigned alignment = alignment_B.ud; - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->sfid = BRW_SFID_UGM; - inst->resize_sources(SEND_NUM_SRCS); inst->send_ex_bso = surf_type == LSC_ADDR_SURFTYPE_BSS && compiler->extended_bindless_surface_offset; @@ -2130,9 +2120,9 @@ lower_varying_pull_constant_logical_send(const brw_builder &bld, brw_inst *inst) assert(inst->src[PULL_VARYING_CONSTANT_SRC_ALIGNMENT].file == IMM); unsigned alignment = inst->src[PULL_VARYING_CONSTANT_SRC_ALIGNMENT].ud; - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->mlen = inst->exec_size / 8; - inst->resize_sources(SEND_NUM_SRCS); /* src[SEND_SRC_DESC/EX_DESC] are filled by setup_surface_descriptors() */ inst->src[SEND_SRC_PAYLOAD1] = ubo_offset; @@ -2300,7 +2290,7 @@ lower_interpolator_logical_send(const brw_builder &bld, brw_inst *inst, } } - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->sfid = BRW_SFID_PIXEL_INTERPOLATOR; inst->desc = desc_imm; inst->ex_desc = 0; @@ -2309,8 +2299,6 @@ lower_interpolator_logical_send(const brw_builder &bld, brw_inst *inst, inst->send_has_side_effects = false; inst->send_is_volatile = false; - inst->resize_sources(SEND_NUM_SRCS); - inst->src[SEND_SRC_DESC] = component(desc, 0); inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = payload; @@ -2379,7 +2367,7 @@ lower_btd_logical_send(const brw_builder &bld, brw_inst *inst) } /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); inst->mlen = mlen; inst->ex_mlen = ex_mlen; inst->header_size = 0; /* HW docs require has_header = false */ @@ -2391,8 +2379,6 @@ lower_btd_logical_send(const brw_builder &bld, brw_inst *inst) inst->desc = brw_btd_spawn_desc(devinfo, inst->exec_size, GEN_RT_BTD_MESSAGE_SPAWN); - inst->resize_sources(SEND_NUM_SRCS); - inst->src[SEND_SRC_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = header; @@ -2487,7 +2473,8 @@ lower_trace_ray_logical_send(const brw_builder &bld, brw_inst *inst) } /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->mlen = mlen; inst->ex_mlen = ex_mlen; inst->header_size = 0; /* HW docs require has_header = false */ @@ -2498,8 +2485,6 @@ lower_trace_ray_logical_send(const brw_builder &bld, brw_inst *inst) inst->sfid = BRW_SFID_RAY_TRACE_ACCELERATOR; inst->desc = brw_rt_trace_ray_desc(devinfo, inst->exec_size); - inst->resize_sources(SEND_NUM_SRCS); - inst->src[SEND_SRC_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_EX_DESC] = brw_imm_ud(0); inst->src[SEND_SRC_PAYLOAD1] = header; @@ -2519,9 +2504,9 @@ lower_get_buffer_size(const brw_builder &bld, brw_inst *inst) brw_reg surface_handle = inst->src[GET_BUFFER_SIZE_SRC_SURFACE_HANDLE]; brw_reg lod = bld.move_to_vgrf(inst->src[GET_BUFFER_SIZE_SRC_LOD], 1); - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->mlen = inst->exec_size / 8; - inst->resize_sources(SEND_NUM_SRCS); inst->ex_mlen = 0; inst->ex_desc = 0; @@ -2555,8 +2540,8 @@ lower_lsc_memory_fence_and_interlock(const brw_builder &bld, brw_inst *inst) assert(inst->size_written == reg_unit(devinfo) * REG_SIZE); - inst->opcode = SHADER_OPCODE_SEND; - inst->resize_sources(SEND_NUM_SRCS); + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->check_tdr = interlock; inst->send_has_side_effects = true; @@ -2631,8 +2616,8 @@ lower_hdc_memory_fence_and_interlock(const brw_builder &bld, brw_inst *inst) assert(inst->size_written == (commit_enable ? REG_SIZE : 0)); - inst->opcode = SHADER_OPCODE_SEND; - inst->resize_sources(SEND_NUM_SRCS); + inst = brw_transform_inst(bld, inst, SHADER_OPCODE_SEND); + inst->check_tdr = interlock; inst->send_has_side_effects = true; @@ -2824,7 +2809,8 @@ brw_lower_uniform_pull_constant_loads(brw_shader &s) LSC_CACHE(devinfo, LOAD, L1STATE_L3MOCS)); /* Update the original instruction. */ - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(ubld, inst, SHADER_OPCODE_SEND); + inst->mlen = lsc_msg_addr_len(devinfo, LSC_ADDR_SIZE_A32, 1); inst->send_ex_bso = surface_handle.file != BAD_FILE && s.compiler->extended_bindless_surface_offset; @@ -2835,8 +2821,6 @@ brw_lower_uniform_pull_constant_loads(brw_shader &s) inst->exec_size = 1; /* Finally, the payload */ - - inst->resize_sources(SEND_NUM_SRCS); setup_lsc_surface_descriptors(ubld, inst, inst->desc, surface.file != BAD_FILE ? surface : surface_handle, 0); @@ -2854,8 +2838,9 @@ brw_lower_uniform_pull_constant_loads(brw_shader &s) ubld.group(1, 0).MOV(component(header, 2), brw_imm_ud(offset_B.ud / 16)); + inst = brw_transform_inst(ubld, inst, SHADER_OPCODE_SEND); + inst->sfid = BRW_SFID_HDC_READ_ONLY; - inst->opcode = SHADER_OPCODE_SEND; inst->header_size = 1; inst->mlen = 1; @@ -2863,8 +2848,6 @@ brw_lower_uniform_pull_constant_loads(brw_shader &s) brw_dp_oword_block_rw_desc(devinfo, true /* align_16B */, size_B.ud / 4, false /* write */); - inst->resize_sources(SEND_NUM_SRCS); - setup_surface_descriptors(ubld, inst, desc, surface, surface_handle); inst->src[SEND_SRC_PAYLOAD1] = header; diff --git a/src/intel/compiler/brw_opt.cpp b/src/intel/compiler/brw_opt.cpp index 6a63777d017..b0dc1817ccb 100644 --- a/src/intel/compiler/brw_opt.cpp +++ b/src/intel/compiler/brw_opt.cpp @@ -500,9 +500,8 @@ brw_opt_eliminate_find_live_channel(brw_shader &s) case SHADER_OPCODE_FIND_LIVE_CHANNEL: if (depth == 0) { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); - inst->resize_sources(1); inst->src[0] = brw_imm_ud(0u); inst->force_writemask_all = true; @@ -528,14 +527,15 @@ brw_opt_eliminate_find_live_channel(brw_shader &s) inst->dst.file == bcast->src[1].file && inst->dst.nr == bcast->src[1].nr && inst->dst.offset == bcast->src[1].offset) { - bcast->opcode = BRW_OPCODE_MOV; + + bcast = brw_transform_inst(s, bcast, BRW_OPCODE_MOV); + if (!is_uniform(bcast->src[0])) bcast->src[0] = component(bcast->src[0], 0); bcast->force_writemask_all = true; bcast->exec_size = 8 * reg_unit(s.devinfo); assert(bcast->size_written == bcast->dst.component_size(bcast->exec_size)); - bcast->resize_sources(1); } } break; @@ -645,7 +645,9 @@ brw_opt_send_to_send_gather(brw_shader &s) continue; } - inst->resize_sources(SEND_GATHER_SRC_PAYLOAD + num_payload_sources); + inst = brw_transform_inst(s, inst, SHADER_OPCODE_SEND_GATHER, + SEND_GATHER_SRC_PAYLOAD + num_payload_sources); + /* Sources 0 and 1 remain the same. Source 2 will be filled * after register allocation. */ @@ -660,7 +662,6 @@ brw_opt_send_to_send_gather(brw_shader &s) } assert(idx == inst->sources); - inst->opcode = SHADER_OPCODE_SEND_GATHER; inst->mlen = 0; inst->ex_mlen = 0; @@ -763,8 +764,8 @@ brw_opt_send_gather_to_send(brw_shader &s) continue; } - inst->resize_sources(SEND_NUM_SRCS); - inst->opcode = SHADER_OPCODE_SEND; + inst = brw_transform_inst(s, inst, SHADER_OPCODE_SEND); + inst->src[SEND_SRC_PAYLOAD1] = payload1; inst->src[SEND_SRC_PAYLOAD2] = payload2; inst->mlen = payload1_len * unit; diff --git a/src/intel/compiler/brw_opt_algebraic.cpp b/src/intel/compiler/brw_opt_algebraic.cpp index 298a4a2dadc..4a17a5855cc 100644 --- a/src/intel/compiler/brw_opt_algebraic.cpp +++ b/src/intel/compiler/brw_opt_algebraic.cpp @@ -86,7 +86,7 @@ brw_imm_for_type(uint64_t value, enum brw_reg_type type) /** * Converts a MAD to an ADD by folding the multiplicand sources. */ -static void +static brw_inst * fold_multiplicands_of_MAD(brw_shader &s, brw_inst *inst) { assert(inst->opcode == BRW_OPCODE_MAD); @@ -128,8 +128,7 @@ fold_multiplicands_of_MAD(brw_shader &s, brw_inst *inst) } } - inst->opcode = BRW_OPCODE_ADD; - inst->resize_sources(2); + return brw_transform_inst(s, inst, BRW_OPCODE_ADD); } bool @@ -188,7 +187,7 @@ brw_opt_constant_fold_instruction(brw_shader &s, brw_inst *inst) !brw_type_is_vector_imm(inst->src[0].type) && !brw_type_is_vector_imm(inst->src[1].type) && !brw_type_is_vector_imm(inst->src[2].type)) { - fold_multiplicands_of_MAD(s, inst); + inst = fold_multiplicands_of_MAD(s, inst); assert(inst->opcode == BRW_OPCODE_ADD); ASSERTED bool folded = brw_opt_constant_fold_instruction(s, inst); @@ -279,9 +278,9 @@ brw_opt_constant_fold_instruction(brw_shader &s, brw_inst *inst) case SHADER_OPCODE_BROADCAST: if (inst->src[0].file == IMM) { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); + inst->force_writemask_all = true; - inst->resize_sources(1); /* The destination of BROADCAST will always be is_scalar, so the * allocation will always be REG_SIZE * reg_unit. Adjust the @@ -316,9 +315,9 @@ brw_opt_constant_fold_instruction(brw_shader &s, brw_inst *inst) if (result.file != BAD_FILE) { assert(result.file == IMM); - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); + inst->src[0] = result; - inst->resize_sources(1); return true; } @@ -341,8 +340,7 @@ brw_opt_algebraic(brw_shader &s) case BRW_OPCODE_ADD: if (brw_type_is_int(inst->src[1].type) && inst->src[1].is_zero()) { - inst->opcode = BRW_OPCODE_MOV; - inst->resize_sources(1); + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); progress = true; } @@ -373,14 +371,12 @@ brw_opt_algebraic(brw_shader &s) assert(src.file != BAD_FILE); if (uint32_t(sum) == 0) { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); inst->src[0] = src; - inst->resize_sources(1); } else { - inst->opcode = BRW_OPCODE_ADD; + inst = brw_transform_inst(s, inst, BRW_OPCODE_ADD); inst->src[0] = src; inst->src[1] = brw_imm_ud(sum); - inst->resize_sources(2); } progress = true; @@ -390,9 +386,8 @@ brw_opt_algebraic(brw_shader &s) */ for (unsigned i = 0; i < 3; i++) { if (inst->src[i].is_zero()) { - inst->opcode = BRW_OPCODE_ADD; inst->src[i] = inst->src[2]; - inst->resize_sources(2); + inst = brw_transform_inst(s, inst, BRW_OPCODE_ADD); progress = true; break; } @@ -462,12 +457,14 @@ brw_opt_algebraic(brw_shader &s) break; for (unsigned i = 0; i < 2; i++) { + bool found = false; + /* a * 1 = a */ if (inst->src[i].is_one()) { - inst->opcode = BRW_OPCODE_MOV; + found = true; } else if (inst->src[i].is_negative_one()) { /* a * -1 = -a */ - inst->opcode = BRW_OPCODE_MOV; + found = true; /* If the source other than the -1 is immediate, just * toggling the negation flag will not work. Due to the @@ -478,12 +475,12 @@ brw_opt_algebraic(brw_shader &s) inst->src[1 - i].negate = !inst->src[1 - i].negate; } - if (inst->opcode == BRW_OPCODE_MOV) { + if (found) { /* If the literal 1 was src0, put the old src1 in src0. */ if (i == 0) inst->src[0] = inst->src[1]; - inst->resize_sources(1); + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); progress = true; break; } @@ -507,7 +504,7 @@ brw_opt_algebraic(brw_shader &s) if (!inst->src[0].negate) inst->conditional_mod = brw_negate_cmod(inst->conditional_mod); - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); inst->src[0].negate = false; progress = true; } @@ -520,12 +517,11 @@ brw_opt_algebraic(brw_shader &s) * or 'OR r0, ~r1, ~r1' should become a NOT instead of a MOV. */ if (inst->src[0].negate) { - inst->opcode = BRW_OPCODE_NOT; + inst = brw_transform_inst(s, inst, BRW_OPCODE_NOT); inst->src[0].negate = false; } else { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); } - inst->resize_sources(1); progress = true; break; } @@ -552,11 +548,10 @@ brw_opt_algebraic(brw_shader &s) if (inst->src[0].equals(inst->src[1]) && (!brw_type_is_float(inst->dst.type) || inst->conditional_mod == BRW_CONDITIONAL_NONE)) { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); inst->predicate = BRW_PREDICATE_NONE; inst->predicate_inverse = false; inst->conditional_mod = BRW_CONDITIONAL_NONE; - inst->resize_sources(1); progress = true; } else if (inst->saturate && inst->src[1].file == IMM) { switch (inst->conditional_mod) { @@ -565,9 +560,8 @@ brw_opt_algebraic(brw_shader &s) switch (inst->src[1].type) { case BRW_TYPE_F: if (inst->src[1].f >= 1.0f) { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); inst->conditional_mod = BRW_CONDITIONAL_NONE; - inst->resize_sources(1); progress = true; } break; @@ -580,9 +574,8 @@ brw_opt_algebraic(brw_shader &s) switch (inst->src[1].type) { case BRW_TYPE_F: if (inst->src[1].f <= 0.0f) { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); inst->conditional_mod = BRW_CONDITIONAL_NONE; - inst->resize_sources(1); progress = true; } break; @@ -653,15 +646,13 @@ brw_opt_algebraic(brw_shader &s) break; case BRW_CONDITIONAL_G: /* This is a contradtion. -abs(x) cannot be > 0. */ - inst->opcode = BRW_OPCODE_MOV; inst->src[0] = inst->src[1]; - inst->resize_sources(1); + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); progress = true; break; case BRW_CONDITIONAL_LE: /* This is a tautology. -abs(x) must be <= 0. */ - inst->opcode = BRW_OPCODE_MOV; - inst->resize_sources(1); + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); progress = true; break; case BRW_CONDITIONAL_Z: @@ -692,19 +683,18 @@ brw_opt_algebraic(brw_shader &s) } if (inst->src[1].is_one()) { - inst->opcode = BRW_OPCODE_ADD; inst->src[1] = inst->src[2]; - inst->resize_sources(2); + inst = brw_transform_inst(s, inst, BRW_OPCODE_ADD); progress = true; } else if (inst->src[2].is_one()) { - inst->opcode = BRW_OPCODE_ADD; - inst->resize_sources(2); + inst = brw_transform_inst(s, inst, BRW_OPCODE_ADD); progress = true; } break; case SHADER_OPCODE_BROADCAST: if (is_uniform(inst->src[0])) { - inst->opcode = BRW_OPCODE_MOV; + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); + inst->force_writemask_all = true; /* The destination of BROADCAST will always be is_scalar, so the @@ -713,10 +703,8 @@ brw_opt_algebraic(brw_shader &s) */ inst->exec_size = 8 * reg_unit(devinfo); assert(inst->size_written == inst->dst.component_size(inst->exec_size)); - inst->resize_sources(1); progress = true; } else if (inst->src[1].file == IMM) { - inst->opcode = BRW_OPCODE_MOV; /* It's possible that the selected component will be too large and * overflow the register. This can happen if someone does a * readInvocation() from GLSL or SPIR-V and provides an OOB @@ -731,21 +719,21 @@ brw_opt_algebraic(brw_shader &s) inst->force_writemask_all = true; inst->exec_size = 8 * reg_unit(devinfo); assert(inst->size_written == inst->dst.component_size(inst->exec_size)); - inst->resize_sources(1); + + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); + progress = true; } break; case SHADER_OPCODE_SHUFFLE: if (is_uniform(inst->src[0])) { - inst->opcode = BRW_OPCODE_MOV; - inst->resize_sources(1); + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); progress = true; } else if (inst->src[1].file == IMM) { const unsigned comp = inst->src[1].ud & (inst->exec_size - 1); - inst->opcode = BRW_OPCODE_MOV; inst->src[0] = component(inst->src[0], comp); - inst->resize_sources(1); + inst = brw_transform_inst(s, inst, BRW_OPCODE_MOV); progress = true; } break; @@ -770,8 +758,7 @@ brw_opt_algebraic(brw_shader &s) } if (progress) - s.invalidate_analysis(BRW_DEPENDENCY_INSTRUCTION_DATA_FLOW | - BRW_DEPENDENCY_INSTRUCTION_DETAIL); + s.invalidate_analysis(BRW_DEPENDENCY_INSTRUCTIONS); return progress; } diff --git a/src/intel/compiler/brw_opt_copy_propagation.cpp b/src/intel/compiler/brw_opt_copy_propagation.cpp index cf4451e9c46..0d1bf4aeaa9 100644 --- a/src/intel/compiler/brw_opt_copy_propagation.cpp +++ b/src/intel/compiler/brw_opt_copy_propagation.cpp @@ -1409,7 +1409,7 @@ opt_copy_propagation_local(brw_shader &s, linear_ctx *lin_ctx, const struct intel_device_info *devinfo = s.devinfo; bool progress = false; - foreach_inst_in_block(brw_inst, inst, block) { + foreach_inst_in_block_safe(brw_inst, inst, block) { /* The non-defs copy propagation passes should not be called while * LOAD_REG instructions still exist. */ diff --git a/src/intel/compiler/brw_opt_dead_code_eliminate.cpp b/src/intel/compiler/brw_opt_dead_code_eliminate.cpp index 0c852ddf24b..5e7470a32cf 100644 --- a/src/intel/compiler/brw_opt_dead_code_eliminate.cpp +++ b/src/intel/compiler/brw_opt_dead_code_eliminate.cpp @@ -134,7 +134,7 @@ brw_opt_dead_code_eliminate(brw_shader &s) if (inst->dst.is_null() && can_eliminate(devinfo, inst, flag_live) && !(inst->opcode == BRW_OPCODE_NOP && brw_exec_list_is_singular(&block->instructions))) { - inst->opcode = BRW_OPCODE_NOP; + inst = brw_transform_inst(s, inst, BRW_OPCODE_NOP); progress = true; } diff --git a/src/intel/compiler/brw_opt_register_coalesce.cpp b/src/intel/compiler/brw_opt_register_coalesce.cpp index 00357f38b13..f5442d50a84 100644 --- a/src/intel/compiler/brw_opt_register_coalesce.cpp +++ b/src/intel/compiler/brw_opt_register_coalesce.cpp @@ -268,12 +268,12 @@ brw_opt_register_coalesce(brw_shader &s) int *src_var = new int[live.max_vgrf_size]; const brw_def_analysis &defs = s.def_analysis.require(); - foreach_block_and_inst(block, brw_inst, inst, s.cfg) { + foreach_block_and_inst_safe(block, brw_inst, inst, s.cfg) { if (!is_coalesce_candidate(&s, inst)) continue; if (is_nop_mov(inst)) { - inst->opcode = BRW_OPCODE_NOP; + inst = brw_transform_inst(s, inst, BRW_OPCODE_NOP); progress = true; continue; } @@ -365,7 +365,7 @@ brw_opt_register_coalesce(brw_shader &s) continue; if (mov[i]->conditional_mod == BRW_CONDITIONAL_NONE) { - mov[i]->opcode = BRW_OPCODE_NOP; + mov[i] = brw_transform_inst(s, mov[i], BRW_OPCODE_NOP); mov[i]->dst = reg_undef; for (int j = 0; j < mov[i]->sources; j++) { mov[i]->src[j] = reg_undef; diff --git a/src/intel/compiler/brw_shader.h b/src/intel/compiler/brw_shader.h index 69269c0a854..8b99054dd48 100644 --- a/src/intel/compiler/brw_shader.h +++ b/src/intel/compiler/brw_shader.h @@ -374,3 +374,14 @@ bool brw_lower_load_reg(brw_shader &s); brw_inst *brw_new_inst(brw_shader &s, enum opcode opcode, unsigned exec_size, const brw_reg &dst, unsigned num_srcs); brw_inst *brw_clone_inst(brw_shader &s, const brw_inst *inst); + +/* Transform the opcode/num_sources of an instruction. All the fields in + * brw_inst are maintained and any previous sources still visible. Additional + * sources will be uninitialized. + * + * If new_num_srcs is UINT_MAX a default will be picked based on the opcode. + * Not all opcodes have a default. + */ +brw_inst *brw_transform_inst(brw_shader &s, brw_inst *inst, enum opcode new_opcode, + unsigned new_num_srcs = UINT_MAX); +