diff --git a/src/gallium/drivers/iris/iris_context.h b/src/gallium/drivers/iris/iris_context.h index 17f8d331404..c401fdf5b61 100644 --- a/src/gallium/drivers/iris/iris_context.h +++ b/src/gallium/drivers/iris/iris_context.h @@ -304,6 +304,137 @@ PRAGMA_DIAGNOSTIC_POP /** @} */ +struct iris_ubo_range +{ + uint16_t block; + + /* In units of 32-byte registers */ + uint8_t start; + uint8_t length; +}; + +struct iris_fs_data { + int urb_setup[VARYING_SLOT_MAX]; + uint8_t urb_setup_attribs[VARYING_SLOT_MAX]; + uint8_t urb_setup_attribs_count; + + uint64_t inputs; + unsigned num_varying_inputs; + + unsigned msaa_flags_param; + uint32_t flat_inputs; + + uint8_t computed_depth_mode; + uint8_t max_polygons; + uint8_t dispatch_multi; + + bool computed_stencil; + bool early_fragment_tests; + bool post_depth_coverage; + bool inner_coverage; + bool dispatch_8; + bool dispatch_16; + bool dispatch_32; + bool dual_src_blend; + bool uses_pos_offset; + bool uses_omask; + bool uses_kill; + bool uses_src_depth; + bool uses_src_w; + bool uses_sample_mask; + bool uses_vmask; + bool has_side_effects; + bool pulls_bary; + + bool uses_nonperspective_interp_modes; + + bool is_per_sample; +}; + +struct iris_push_const_block { + unsigned dwords; /* Dword count, not reg aligned */ + unsigned regs; + unsigned size; /* Bytes, register aligned */ +}; + +struct iris_cs_data { + struct { + struct iris_push_const_block cross_thread; + struct iris_push_const_block per_thread; + } push; + + unsigned local_size[3]; + unsigned prog_offset[3]; + unsigned prog_mask; + + uint8_t generate_local_id; + enum intel_compute_walk_order walk_order; + + bool uses_barrier; + bool first_param_is_builtin_subgroup_id; +}; + +static inline uint32_t +iris_cs_data_prog_offset(const struct iris_cs_data *prog_data, + unsigned dispatch_width) +{ + assert(dispatch_width == 8 || + dispatch_width == 16 || + dispatch_width == 32); + const unsigned index = dispatch_width / 16; + assert(prog_data->prog_mask & (1 << index)); + return prog_data->prog_offset[index]; +} + +struct iris_vue_data { + struct intel_vue_map vue_map; + unsigned urb_read_length; + uint32_t cull_distance_mask; + unsigned urb_entry_size; + enum intel_shader_dispatch_mode dispatch_mode; + bool include_vue_handles; +}; + +struct iris_vs_data { + struct iris_vue_data base; + + bool uses_vertexid; + bool uses_instanceid; + bool uses_firstvertex; + bool uses_baseinstance; + bool uses_drawid; +}; + +struct iris_tcs_data { + struct iris_vue_data base; + + int instances; + int patch_count_threshold; + bool include_primitive_id; +}; + +struct iris_tes_data { + struct iris_vue_data base; + + enum intel_tess_partitioning partitioning; + enum intel_tess_output_topology output_topology; + enum intel_tess_domain domain; + bool include_primitive_id; +}; + +struct iris_gs_data { + struct iris_vue_data base; + + unsigned vertices_in; + unsigned output_vertex_size_hwords; + unsigned output_topology; + unsigned control_data_header_size_hwords; + unsigned control_data_format; + int static_vertex_count; + int invocations; + bool include_primitive_id; +}; + struct iris_depth_stencil_alpha_state; /** @@ -533,7 +664,7 @@ struct iris_compiled_shader { void *map; /** The program data (owned by the program cache hash table) */ - struct brw_stage_prog_data *prog_data; + struct brw_stage_prog_data *brw_prog_data; /** A list of system values to be uploaded as uniforms. */ enum brw_param_builtin *system_values; @@ -555,6 +686,29 @@ struct iris_compiled_shader { gl_shader_stage stage; + /** + * Data derived from prog_data. + */ + struct iris_ubo_range ubo_ranges[4]; + + unsigned nr_params; + unsigned total_scratch; + unsigned total_shared; + unsigned program_size; + unsigned const_data_offset; + unsigned dispatch_grf_start_reg; + bool has_ubo_pull; + bool use_alt_mode; + + union { + struct iris_fs_data fs; + struct iris_cs_data cs; + struct iris_vs_data vs; + struct iris_tcs_data tcs; + struct iris_tes_data tes; + struct iris_gs_data gs; + }; + /** * Shader packets and other data derived from prog_data. These must be * completely determined from prog_data. @@ -569,6 +723,43 @@ KSP(const struct iris_compiled_shader *shader) return iris_bo_offset_from_base_address(res->bo) + shader->assembly.offset; } +#define DEFINE_IRIS_SHADER_DATA(TYPE, STAGE, FIELD) \ +static inline TYPE * \ +iris_ ## FIELD ## _data(struct iris_compiled_shader *shader) \ +{ \ + assert(shader->stage == STAGE); \ + return &shader->FIELD; \ +} \ +static inline const TYPE * \ +iris_ ## FIELD ## _data_const(const struct iris_compiled_shader *shader) \ +{ \ + assert(shader->stage == STAGE); \ + return &shader->FIELD; \ +} + +DEFINE_IRIS_SHADER_DATA(struct iris_fs_data, MESA_SHADER_FRAGMENT, fs) +DEFINE_IRIS_SHADER_DATA(struct iris_cs_data, MESA_SHADER_COMPUTE, cs) +DEFINE_IRIS_SHADER_DATA(struct iris_vs_data, MESA_SHADER_VERTEX, vs) +DEFINE_IRIS_SHADER_DATA(struct iris_tcs_data, MESA_SHADER_TESS_CTRL, tcs) +DEFINE_IRIS_SHADER_DATA(struct iris_tes_data, MESA_SHADER_TESS_EVAL, tes) +DEFINE_IRIS_SHADER_DATA(struct iris_gs_data, MESA_SHADER_GEOMETRY, gs) + +#undef DEFINE_IRIS_SHADER_DATA + +static inline struct iris_vue_data * +iris_vue_data(struct iris_compiled_shader *shader) +{ + switch (shader->stage) { + case MESA_SHADER_VERTEX: return &shader->vs.base; + case MESA_SHADER_TESS_CTRL: return &shader->tcs.base; + case MESA_SHADER_TESS_EVAL: return &shader->tes.base; + case MESA_SHADER_GEOMETRY: return &shader->gs.base; + default: + unreachable("invalid shader stage for vue prog data"); + return NULL; + } +} + /** * API context state that is replicated per shader stage. */ @@ -1033,7 +1224,8 @@ void iris_init_resource_functions(struct pipe_context *ctx); void iris_init_perfquery_functions(struct pipe_context *ctx); void iris_update_compiled_shaders(struct iris_context *ice); void iris_update_compiled_compute_shader(struct iris_context *ice); -void iris_fill_cs_push_const_buffer(struct brw_cs_prog_data *cs_prog_data, +void iris_fill_cs_push_const_buffer(struct iris_screen *screen, + struct iris_compiled_shader *shader, unsigned threads, uint32_t *dst); @@ -1116,6 +1308,18 @@ uint32_t iris_group_index_to_bti(const struct iris_binding_table *bt, uint32_t iris_bti_to_group_index(const struct iris_binding_table *bt, enum iris_surface_group group, uint32_t bti); +void iris_apply_brw_prog_data(struct iris_compiled_shader *shader, + const struct brw_stage_prog_data *prog_data); +struct intel_cs_dispatch_info +iris_get_cs_dispatch_info(const struct intel_device_info *devinfo, + const struct iris_compiled_shader *shader, + const uint32_t block[3]); +unsigned +iris_cs_push_const_total_size(const struct iris_compiled_shader *shader, + unsigned threads); +uint32_t +iris_fs_barycentric_modes(const struct iris_compiled_shader *shader, + enum intel_msaa_flags pushed_msaa_flags); /* iris_disk_cache.c */ @@ -1149,7 +1353,7 @@ struct iris_compiled_shader *iris_create_shader_variant(const struct iris_screen const void *key); void iris_finalize_program(struct iris_compiled_shader *shader, - struct brw_stage_prog_data *prog_data, + struct brw_stage_prog_data *brw_prog_data, uint32_t *streamout, enum brw_param_builtin *system_values, unsigned num_system_values, @@ -1248,8 +1452,8 @@ iris_execute_indirect_draw_supported(const struct iris_context *ice, const struct pipe_draw_info *draw) { const struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen; - const struct brw_vs_prog_data *vs_prog_data = (void *) - ice->shaders.prog[MESA_SHADER_VERTEX]->prog_data; + const struct iris_vs_data *vs_data = + iris_vs_data(ice->shaders.prog[MESA_SHADER_VERTEX]); const bool is_multiview = draw->view_mask != 0; const size_t struct_size = draw->index_size ? sizeof(uint32_t) * 5 : @@ -1262,9 +1466,9 @@ iris_execute_indirect_draw_supported(const struct iris_context *ice, (indirect && !indirect->count_from_stream_output) && !is_multiview && - !(vs_prog_data->uses_firstvertex || - vs_prog_data->uses_baseinstance || - vs_prog_data->uses_drawid)); + !(vs_data->uses_firstvertex || + vs_data->uses_baseinstance || + vs_data->uses_drawid)); } #ifdef genX diff --git a/src/gallium/drivers/iris/iris_disk_cache.c b/src/gallium/drivers/iris/iris_disk_cache.c index 47eeabb7ccd..260f1624c37 100644 --- a/src/gallium/drivers/iris/iris_disk_cache.c +++ b/src/gallium/drivers/iris/iris_disk_cache.c @@ -86,7 +86,7 @@ iris_disk_cache_store(struct disk_cache *cache, return; gl_shader_stage stage = ish->nir->info.stage; - const struct brw_stage_prog_data *prog_data = shader->prog_data; + const struct brw_stage_prog_data *prog_data = shader->brw_prog_data; cache_key cache_key; iris_disk_cache_compute_key(cache, ish, prog_key, prog_key_size, cache_key); @@ -116,12 +116,12 @@ iris_disk_cache_store(struct disk_cache *cache, size_t prog_data_s = brw_prog_data_size(stage); union brw_any_prog_data serializable; assert(prog_data_s <= sizeof(serializable)); - memcpy(&serializable, shader->prog_data, prog_data_s); + memcpy(&serializable, shader->brw_prog_data, prog_data_s); serializable.base.param = NULL; serializable.base.relocs = NULL; blob_write_bytes(&blob, &serializable, prog_data_s); - blob_write_bytes(&blob, shader->map, shader->prog_data->program_size); + blob_write_bytes(&blob, shader->map, shader->program_size); blob_write_uint32(&blob, shader->num_system_values); blob_write_bytes(&blob, shader->system_values, shader->num_system_values * sizeof(enum brw_param_builtin)); @@ -228,9 +228,9 @@ iris_disk_cache_retrieve(struct iris_screen *screen, if (stage == MESA_SHADER_VERTEX || stage == MESA_SHADER_TESS_EVAL || stage == MESA_SHADER_GEOMETRY) { - struct brw_vue_prog_data *vue_prog_data = (void *) prog_data; - so_decls = screen->vtbl.create_so_decl_list(&ish->stream_output, - &vue_prog_data->vue_map); + struct intel_vue_map *vue_map = + &brw_vue_prog_data(prog_data)->vue_map; + so_decls = screen->vtbl.create_so_decl_list(&ish->stream_output, vue_map); } /* System values and uniforms are stored in constant buffer 0, the @@ -245,6 +245,8 @@ iris_disk_cache_retrieve(struct iris_screen *screen, if (num_system_values || kernel_input_size) num_cbufs++; + iris_apply_brw_prog_data(shader, prog_data); + iris_finalize_program(shader, prog_data, so_decls, system_values, num_system_values, kernel_input_size, num_cbufs, &bt); diff --git a/src/gallium/drivers/iris/iris_indirect_gen.c b/src/gallium/drivers/iris/iris_indirect_gen.c index 9383f3e9559..ab38c9f1891 100644 --- a/src/gallium/drivers/iris/iris_indirect_gen.c +++ b/src/gallium/drivers/iris/iris_indirect_gen.c @@ -263,14 +263,14 @@ emit_indirect_generate_draw(struct iris_batch *batch, raster.CullMode = CULLMODE_NONE; } - const struct brw_wm_prog_data *wm_prog_data = (void *) - ice->draw.generation.shader->prog_data; + const struct iris_compiled_shader *shader = ice->draw.generation.shader; + const struct iris_fs_data *fs_data = iris_fs_data_const(shader); iris_emit_cmd(batch, GENX(3DSTATE_SBE), sbe) { sbe.VertexURBEntryReadOffset = 1; - sbe.NumberofSFOutputAttributes = wm_prog_data->num_varying_inputs; - sbe.VertexURBEntryReadLength = MAX2((wm_prog_data->num_varying_inputs + 1) / 2, 1); - sbe.ConstantInterpolationEnable = wm_prog_data->flat_inputs; + sbe.NumberofSFOutputAttributes = fs_data->num_varying_inputs; + sbe.VertexURBEntryReadLength = MAX2((fs_data->num_varying_inputs + 1) / 2, 1); + sbe.ConstantInterpolationEnable = fs_data->flat_inputs; sbe.ForceVertexURBEntryReadLength = true; sbe.ForceVertexURBEntryReadOffset = true; #if GFX_VER >= 9 @@ -280,21 +280,22 @@ emit_indirect_generate_draw(struct iris_batch *batch, } iris_emit_cmd(batch, GENX(3DSTATE_WM), wm) { - if (wm_prog_data->has_side_effects || wm_prog_data->uses_kill) + if (fs_data->has_side_effects || fs_data->uses_kill) wm.ForceThreadDispatchEnable = ForceON; } iris_emit_cmd(batch, GENX(3DSTATE_PS), ps) { + struct brw_wm_prog_data *wm_prog_data = brw_wm_prog_data(shader->brw_prog_data); intel_set_ps_dispatch_state(&ps, devinfo, wm_prog_data, 1 /* rasterization_samples */, 0 /* msaa_flags */); - ps.VectorMaskEnable = wm_prog_data->uses_vmask; + ps.VectorMaskEnable = fs_data->uses_vmask; ps.BindingTableEntryCount = GFX_VER == 9 ? 1 : 0; #if GFX_VER < 20 - ps.PushConstantEnable = wm_prog_data->base.nr_params > 0 || - wm_prog_data->base.ubo_ranges[0].length; + ps.PushConstantEnable = shader->nr_params > 0 || + shader->ubo_ranges[0].length; #endif ps.DispatchGRFStartRegisterForConstantSetupData0 = @@ -321,17 +322,17 @@ emit_indirect_generate_draw(struct iris_batch *batch, iris_emit_cmd(batch, GENX(3DSTATE_PS_EXTRA), psx) { psx.PixelShaderValid = true; #if GFX_VER < 20 - psx.AttributeEnable = wm_prog_data->num_varying_inputs > 0; + psx.AttributeEnable = fs_data->num_varying_inputs > 0; #endif - psx.PixelShaderIsPerSample = wm_prog_data->persample_dispatch; - psx.PixelShaderComputedDepthMode = wm_prog_data->computed_depth_mode; + psx.PixelShaderIsPerSample = fs_data->is_per_sample; + psx.PixelShaderComputedDepthMode = fs_data->computed_depth_mode; #if GFX_VER >= 9 #if GFX_VER >= 20 - assert(!wm_prog_data->pulls_bary); + assert(!fs_data->pulls_bary); #else - psx.PixelShaderPullsBary = wm_prog_data->pulls_bary; + psx.PixelShaderPullsBary = fs_data->pulls_bary; #endif - psx.PixelShaderComputesStencil = wm_prog_data->computed_stencil; + psx.PixelShaderComputesStencil = fs_data->computed_stencil; #endif psx.PixelShaderHasUAV = GFX_VER == 8; } diff --git a/src/gallium/drivers/iris/iris_program.c b/src/gallium/drivers/iris/iris_program.c index 5b14a7b8253..37d6312cb3c 100644 --- a/src/gallium/drivers/iris/iris_program.c +++ b/src/gallium/drivers/iris/iris_program.c @@ -70,9 +70,205 @@ get_new_program_id(struct iris_screen *screen) return p_atomic_inc_return(&screen->program_id); } +static void +iris_apply_brw_wm_prog_data(struct iris_compiled_shader *shader, + const struct brw_wm_prog_data *brw) +{ + assert(shader->stage == MESA_SHADER_FRAGMENT); + struct iris_fs_data *iris = &shader->fs; + + STATIC_ASSERT(ARRAY_SIZE(iris->urb_setup) == ARRAY_SIZE(brw->urb_setup)); + STATIC_ASSERT(ARRAY_SIZE(iris->urb_setup_attribs) == ARRAY_SIZE(brw->urb_setup_attribs)); + memcpy(iris->urb_setup, brw->urb_setup, sizeof(iris->urb_setup)); + memcpy(iris->urb_setup_attribs, brw->urb_setup_attribs, brw->urb_setup_attribs_count); + iris->urb_setup_attribs_count = brw->urb_setup_attribs_count; + + iris->num_varying_inputs = brw->num_varying_inputs; + iris->msaa_flags_param = brw->msaa_flags_param; + iris->flat_inputs = brw->flat_inputs; + iris->inputs = brw->inputs; + iris->computed_depth_mode = brw->computed_depth_mode; + iris->max_polygons = brw->max_polygons; + iris->dispatch_multi = brw->dispatch_multi; + iris->computed_stencil = brw->computed_stencil; + iris->early_fragment_tests = brw->early_fragment_tests; + iris->post_depth_coverage = brw->post_depth_coverage; + iris->inner_coverage = brw->inner_coverage; + iris->dispatch_8 = brw->dispatch_8; + iris->dispatch_16 = brw->dispatch_16; + iris->dispatch_32 = brw->dispatch_32; + iris->dual_src_blend = brw->dual_src_blend; + iris->uses_pos_offset = brw->uses_pos_offset; + iris->uses_omask = brw->uses_omask; + iris->uses_kill = brw->uses_kill; + iris->uses_src_depth = brw->uses_src_depth; + iris->uses_src_w = brw->uses_src_w; + iris->uses_sample_mask = brw->uses_sample_mask; + iris->uses_vmask = brw->uses_vmask; + iris->has_side_effects = brw->has_side_effects; + iris->pulls_bary = brw->pulls_bary; + + iris->uses_nonperspective_interp_modes = brw->uses_nonperspective_interp_modes; + + iris->is_per_sample = brw_wm_prog_data_is_persample(brw, 0); +} + +static void +iris_apply_brw_cs_prog_data(struct iris_compiled_shader *shader, + const struct brw_cs_prog_data *brw) +{ + assert(shader->stage == MESA_SHADER_COMPUTE); + struct iris_cs_data *iris = &shader->cs; + + iris->push.cross_thread.dwords = brw->push.cross_thread.dwords; + iris->push.cross_thread.regs = brw->push.cross_thread.regs; + iris->push.cross_thread.size = brw->push.cross_thread.size; + + iris->push.per_thread.dwords = brw->push.per_thread.dwords; + iris->push.per_thread.regs = brw->push.per_thread.regs; + iris->push.per_thread.size = brw->push.per_thread.size; + + iris->local_size[0] = brw->local_size[0]; + iris->local_size[1] = brw->local_size[1]; + iris->local_size[2] = brw->local_size[2]; + iris->prog_offset[0] = brw->prog_offset[0]; + iris->prog_offset[1] = brw->prog_offset[1]; + iris->prog_offset[2] = brw->prog_offset[2]; + + iris->generate_local_id = brw->generate_local_id; + iris->walk_order = brw->walk_order; + iris->uses_barrier = brw->uses_barrier; + iris->prog_mask = brw->prog_mask; + + iris->first_param_is_builtin_subgroup_id = + brw->base.nr_params > 0 && + brw->base.param[0] == BRW_PARAM_BUILTIN_SUBGROUP_ID; +} + +static void +iris_apply_brw_vue_prog_data(const struct brw_vue_prog_data *brw, + struct iris_vue_data *iris) +{ + memcpy(&iris->vue_map, &brw->vue_map, sizeof(struct intel_vue_map)); + + iris->urb_read_length = brw->urb_read_length; + iris->cull_distance_mask = brw->cull_distance_mask; + iris->urb_entry_size = brw->urb_entry_size; + iris->dispatch_mode = brw->dispatch_mode; + iris->include_vue_handles = brw->include_vue_handles; +} + +static void +iris_apply_brw_vs_prog_data(struct iris_compiled_shader *shader, + const struct brw_vs_prog_data *brw) +{ + assert(shader->stage == MESA_SHADER_VERTEX); + struct iris_vs_data *iris = &shader->vs; + + iris_apply_brw_vue_prog_data(&brw->base, &iris->base); + + iris->uses_vertexid = brw->uses_vertexid; + iris->uses_instanceid = brw->uses_instanceid; + iris->uses_firstvertex = brw->uses_firstvertex; + iris->uses_baseinstance = brw->uses_baseinstance; + iris->uses_drawid = brw->uses_drawid; +} + +static void +iris_apply_brw_tcs_prog_data(struct iris_compiled_shader *shader, + const struct brw_tcs_prog_data *brw) +{ + assert(shader->stage == MESA_SHADER_TESS_CTRL); + struct iris_tcs_data *iris = &shader->tcs; + + iris_apply_brw_vue_prog_data(&brw->base, &iris->base); + + iris->instances = brw->instances; + iris->patch_count_threshold = brw->patch_count_threshold; + iris->include_primitive_id = brw->include_primitive_id; +} + +static void +iris_apply_brw_tes_prog_data(struct iris_compiled_shader *shader, + const struct brw_tes_prog_data *brw) +{ + assert(shader->stage == MESA_SHADER_TESS_EVAL); + struct iris_tes_data *iris = &shader->tes; + + iris_apply_brw_vue_prog_data(&brw->base, &iris->base); + + iris->partitioning = brw->partitioning; + iris->output_topology = brw->output_topology; + iris->domain = brw->domain; + iris->include_primitive_id = brw->include_primitive_id; +} + +static void +iris_apply_brw_gs_prog_data(struct iris_compiled_shader *shader, + const struct brw_gs_prog_data *brw) +{ + assert(shader->stage == MESA_SHADER_GEOMETRY); + struct iris_gs_data *iris = &shader->gs; + + iris_apply_brw_vue_prog_data(&brw->base, &iris->base); + + iris->vertices_in = brw->vertices_in; + iris->output_vertex_size_hwords = brw->output_vertex_size_hwords; + iris->output_topology = brw->output_topology; + iris->control_data_header_size_hwords = brw->control_data_header_size_hwords; + iris->control_data_format = brw->control_data_format; + iris->static_vertex_count = brw->static_vertex_count; + iris->invocations = brw->invocations; + iris->include_primitive_id = brw->include_primitive_id; +} + +void +iris_apply_brw_prog_data(struct iris_compiled_shader *shader, + const struct brw_stage_prog_data *brw) +{ + STATIC_ASSERT(ARRAY_SIZE(brw->ubo_ranges) == ARRAY_SIZE(shader->ubo_ranges)); + for (int i = 0; i < ARRAY_SIZE(shader->ubo_ranges); i++) { + shader->ubo_ranges[i].block = brw->ubo_ranges[i].block; + shader->ubo_ranges[i].start = brw->ubo_ranges[i].start; + shader->ubo_ranges[i].length = brw->ubo_ranges[i].length; + } + + shader->nr_params = brw->nr_params; + shader->total_scratch = brw->total_scratch; + shader->total_shared = brw->total_shared; + shader->program_size = brw->program_size; + shader->const_data_offset = brw->const_data_offset; + shader->dispatch_grf_start_reg = brw->dispatch_grf_start_reg; + shader->has_ubo_pull = brw->has_ubo_pull; + shader->use_alt_mode = brw->use_alt_mode; + + switch (shader->stage) { + case MESA_SHADER_FRAGMENT: + iris_apply_brw_wm_prog_data(shader, brw_wm_prog_data_const(brw)); + break; + case MESA_SHADER_COMPUTE: + iris_apply_brw_cs_prog_data(shader, brw_cs_prog_data_const(brw)); + break; + case MESA_SHADER_VERTEX: + iris_apply_brw_vs_prog_data(shader, brw_vs_prog_data_const(brw)); + break; + case MESA_SHADER_TESS_CTRL: + iris_apply_brw_tcs_prog_data(shader, brw_tcs_prog_data_const(brw)); + break; + case MESA_SHADER_TESS_EVAL: + iris_apply_brw_tes_prog_data(shader, brw_tes_prog_data_const(brw)); + break; + case MESA_SHADER_GEOMETRY: + iris_apply_brw_gs_prog_data(shader, brw_gs_prog_data_const(brw)); + break; + default: + unreachable("invalid shader stage"); + } +} + void iris_finalize_program(struct iris_compiled_shader *shader, - struct brw_stage_prog_data *prog_data, + struct brw_stage_prog_data *brw_prog_data, uint32_t *streamout, enum brw_param_builtin *system_values, unsigned num_system_values, @@ -80,7 +276,7 @@ iris_finalize_program(struct iris_compiled_shader *shader, unsigned num_cbufs, const struct iris_binding_table *bt) { - shader->prog_data = prog_data; + shader->brw_prog_data = brw_prog_data; shader->streamout = streamout; shader->system_values = system_values; shader->num_system_values = num_system_values; @@ -88,9 +284,9 @@ iris_finalize_program(struct iris_compiled_shader *shader, shader->num_cbufs = num_cbufs; shader->bt = *bt; - ralloc_steal(shader, shader->prog_data); - ralloc_steal(shader->prog_data, (void *)prog_data->relocs); - ralloc_steal(shader->prog_data, prog_data->param); + ralloc_steal(shader, shader->brw_prog_data); + ralloc_steal(shader->brw_prog_data, (void *)brw_prog_data->relocs); + ralloc_steal(shader->brw_prog_data, brw_prog_data->param); ralloc_steal(shader, shader->streamout); ralloc_steal(shader, shader->system_values); } @@ -1293,10 +1489,6 @@ iris_compile_vs(struct iris_screen *screen, const struct brw_compiler *compiler = screen->compiler; const struct intel_device_info *devinfo = screen->devinfo; void *mem_ctx = ralloc_context(NULL); - struct brw_vs_prog_data *vs_prog_data = - rzalloc(mem_ctx, struct brw_vs_prog_data); - struct brw_vue_prog_data *vue_prog_data = &vs_prog_data->base; - struct brw_stage_prog_data *prog_data = &vue_prog_data->base; enum brw_param_builtin *system_values; unsigned num_system_values; unsigned num_cbufs; @@ -1316,8 +1508,6 @@ iris_compile_vs(struct iris_screen *screen, } } - prog_data->use_alt_mode = nir->info.use_legacy_math_rules; - iris_setup_uniforms(devinfo, mem_ctx, nir, 0, &system_values, &num_system_values, &num_cbufs); @@ -1325,10 +1515,15 @@ iris_compile_vs(struct iris_screen *screen, iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0, num_system_values, num_cbufs); - brw_nir_analyze_ubo_ranges(compiler, nir, prog_data->ubo_ranges); + struct brw_vs_prog_data *brw_prog_data = + rzalloc(mem_ctx, struct brw_vs_prog_data); + + brw_prog_data->base.base.use_alt_mode = nir->info.use_legacy_math_rules; + + brw_nir_analyze_ubo_ranges(compiler, nir, brw_prog_data->base.base.ubo_ranges); brw_compute_vue_map(devinfo, - &vue_prog_data->vue_map, nir->info.outputs_written, + &brw_prog_data->base.vue_map, nir->info.outputs_written, nir->info.separate_shader, /* pos_slots */ 1); struct brw_vs_prog_key brw_key = iris_to_brw_vs_key(screen, key); @@ -1341,7 +1536,7 @@ iris_compile_vs(struct iris_screen *screen, .source_hash = ish->source_hash, }, .key = &brw_key, - .prog_data = vs_prog_data, + .prog_data = brw_prog_data, }; const unsigned *program = brw_compile_vs(compiler, ¶ms); @@ -1359,12 +1554,14 @@ iris_compile_vs(struct iris_screen *screen, iris_debug_recompile(screen, dbg, ish, &brw_key.base); + iris_apply_brw_prog_data(shader, &brw_prog_data->base.base); + uint32_t *so_decls = screen->vtbl.create_so_decl_list(&ish->stream_output, - &vue_prog_data->vue_map); + &iris_vue_data(shader)->vue_map); - iris_finalize_program(shader, prog_data, so_decls, system_values, - num_system_values, 0, num_cbufs, &bt); + iris_finalize_program(shader, &brw_prog_data->base.base, so_decls, + system_values, num_system_values, 0, num_cbufs, &bt); iris_upload_shader(screen, ish, shader, NULL, uploader, IRIS_CACHE_VS, sizeof(*key), key, program); @@ -1414,7 +1611,7 @@ iris_update_compiled_vs(struct iris_context *ice) shs->sysvals_need_upload = true; unsigned urb_entry_size = shader ? - ((struct brw_vue_prog_data *) shader->prog_data)->urb_entry_size : 0; + iris_vue_data(shader)->urb_entry_size : 0; check_urb_size(ice, urb_entry_size, MESA_SHADER_VERTEX); } } @@ -1478,10 +1675,6 @@ iris_compile_tcs(struct iris_screen *screen, { const struct brw_compiler *compiler = screen->compiler; void *mem_ctx = ralloc_context(NULL); - struct brw_tcs_prog_data *tcs_prog_data = - rzalloc(mem_ctx, struct brw_tcs_prog_data); - struct brw_vue_prog_data *vue_prog_data = &tcs_prog_data->base; - struct brw_stage_prog_data *prog_data = &vue_prog_data->base; const struct intel_device_info *devinfo = screen->devinfo; enum brw_param_builtin *system_values = NULL; unsigned num_system_values = 0; @@ -1507,7 +1700,10 @@ iris_compile_tcs(struct iris_screen *screen, &num_system_values, &num_cbufs); iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0, num_system_values, num_cbufs); - brw_nir_analyze_ubo_ranges(compiler, nir, prog_data->ubo_ranges); + + struct brw_tcs_prog_data *brw_prog_data = + rzalloc(mem_ctx, struct brw_tcs_prog_data); + brw_nir_analyze_ubo_ranges(compiler, nir, brw_prog_data->base.base.ubo_ranges); struct brw_compile_tcs_params params = { .base = { @@ -1517,7 +1713,7 @@ iris_compile_tcs(struct iris_screen *screen, .source_hash = source_hash, }, .key = &brw_key, - .prog_data = tcs_prog_data, + .prog_data = brw_prog_data, }; const unsigned *program = brw_compile_tcs(compiler, ¶ms); @@ -1535,7 +1731,9 @@ iris_compile_tcs(struct iris_screen *screen, iris_debug_recompile(screen, dbg, ish, &brw_key.base); - iris_finalize_program(shader, prog_data, NULL, system_values, + iris_apply_brw_prog_data(shader, &brw_prog_data->base.base); + + iris_finalize_program(shader, &brw_prog_data->base.base, NULL, system_values, num_system_values, 0, num_cbufs, &bt); iris_upload_shader(screen, ish, shader, passthrough_ht, uploader, @@ -1622,7 +1820,7 @@ iris_update_compiled_tcs(struct iris_context *ice) shs->sysvals_need_upload = true; unsigned urb_entry_size = shader ? - ((struct brw_vue_prog_data *) shader->prog_data)->urb_entry_size : 0; + iris_vue_data(shader)->urb_entry_size : 0; check_urb_size(ice, urb_entry_size, MESA_SHADER_TESS_CTRL); } } @@ -1639,10 +1837,6 @@ iris_compile_tes(struct iris_screen *screen, { const struct brw_compiler *compiler = screen->compiler; void *mem_ctx = ralloc_context(NULL); - struct brw_tes_prog_data *tes_prog_data = - rzalloc(mem_ctx, struct brw_tes_prog_data); - struct brw_vue_prog_data *vue_prog_data = &tes_prog_data->base; - struct brw_stage_prog_data *prog_data = &vue_prog_data->base; enum brw_param_builtin *system_values; const struct intel_device_info *devinfo = screen->devinfo; unsigned num_system_values; @@ -1668,7 +1862,10 @@ iris_compile_tes(struct iris_screen *screen, iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0, num_system_values, num_cbufs); - brw_nir_analyze_ubo_ranges(compiler, nir, prog_data->ubo_ranges); + struct brw_tes_prog_data *brw_prog_data = + rzalloc(mem_ctx, struct brw_tes_prog_data); + + brw_nir_analyze_ubo_ranges(compiler, nir, brw_prog_data->base.base.ubo_ranges); struct intel_vue_map input_vue_map; brw_compute_tess_vue_map(&input_vue_map, key->inputs_read, @@ -1684,7 +1881,7 @@ iris_compile_tes(struct iris_screen *screen, .source_hash = ish->source_hash, }, .key = &brw_key, - .prog_data = tes_prog_data, + .prog_data = brw_prog_data, .input_vue_map = &input_vue_map, }; @@ -1703,11 +1900,13 @@ iris_compile_tes(struct iris_screen *screen, iris_debug_recompile(screen, dbg, ish, &brw_key.base); + iris_apply_brw_prog_data(shader, &brw_prog_data->base.base); + uint32_t *so_decls = screen->vtbl.create_so_decl_list(&ish->stream_output, - &vue_prog_data->vue_map); + &iris_vue_data(shader)->vue_map); - iris_finalize_program(shader, prog_data, so_decls, system_values, + iris_finalize_program(shader, &brw_prog_data->base.base, so_decls, system_values, num_system_values, 0, num_cbufs, &bt); iris_upload_shader(screen, ish, shader, NULL, uploader, IRIS_CACHE_TES, @@ -1758,7 +1957,7 @@ iris_update_compiled_tes(struct iris_context *ice) shs->sysvals_need_upload = true; unsigned urb_entry_size = shader ? - ((struct brw_vue_prog_data *) shader->prog_data)->urb_entry_size : 0; + iris_vue_data(shader)->urb_entry_size : 0; check_urb_size(ice, urb_entry_size, MESA_SHADER_TESS_EVAL); } @@ -1783,10 +1982,6 @@ iris_compile_gs(struct iris_screen *screen, const struct brw_compiler *compiler = screen->compiler; const struct intel_device_info *devinfo = screen->devinfo; void *mem_ctx = ralloc_context(NULL); - struct brw_gs_prog_data *gs_prog_data = - rzalloc(mem_ctx, struct brw_gs_prog_data); - struct brw_vue_prog_data *vue_prog_data = &gs_prog_data->base; - struct brw_stage_prog_data *prog_data = &vue_prog_data->base; enum brw_param_builtin *system_values; unsigned num_system_values; unsigned num_cbufs; @@ -1811,10 +2006,13 @@ iris_compile_gs(struct iris_screen *screen, iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0, num_system_values, num_cbufs); - brw_nir_analyze_ubo_ranges(compiler, nir, prog_data->ubo_ranges); + struct brw_gs_prog_data *brw_prog_data = + rzalloc(mem_ctx, struct brw_gs_prog_data); + + brw_nir_analyze_ubo_ranges(compiler, nir, brw_prog_data->base.base.ubo_ranges); brw_compute_vue_map(devinfo, - &vue_prog_data->vue_map, nir->info.outputs_written, + &brw_prog_data->base.vue_map, nir->info.outputs_written, nir->info.separate_shader, /* pos_slots */ 1); struct brw_gs_prog_key brw_key = iris_to_brw_gs_key(screen, key); @@ -1827,7 +2025,7 @@ iris_compile_gs(struct iris_screen *screen, .source_hash = ish->source_hash, }, .key = &brw_key, - .prog_data = gs_prog_data, + .prog_data = brw_prog_data, }; const unsigned *program = brw_compile_gs(compiler, ¶ms); @@ -1845,11 +2043,13 @@ iris_compile_gs(struct iris_screen *screen, iris_debug_recompile(screen, dbg, ish, &brw_key.base); + iris_apply_brw_prog_data(shader, &brw_prog_data->base.base); + uint32_t *so_decls = screen->vtbl.create_so_decl_list(&ish->stream_output, - &vue_prog_data->vue_map); + &iris_vue_data(shader)->vue_map); - iris_finalize_program(shader, prog_data, so_decls, system_values, + iris_finalize_program(shader, &brw_prog_data->base.base, so_decls, system_values, num_system_values, 0, num_cbufs, &bt); iris_upload_shader(screen, ish, shader, NULL, uploader, IRIS_CACHE_GS, @@ -1903,7 +2103,7 @@ iris_update_compiled_gs(struct iris_context *ice) shs->sysvals_need_upload = true; unsigned urb_entry_size = shader ? - ((struct brw_vue_prog_data *) shader->prog_data)->urb_entry_size : 0; + iris_vue_data(shader)->urb_entry_size : 0; check_urb_size(ice, urb_entry_size, MESA_SHADER_GEOMETRY); } } @@ -1921,9 +2121,6 @@ iris_compile_fs(struct iris_screen *screen, { const struct brw_compiler *compiler = screen->compiler; void *mem_ctx = ralloc_context(NULL); - struct brw_wm_prog_data *fs_prog_data = - rzalloc(mem_ctx, struct brw_wm_prog_data); - struct brw_stage_prog_data *prog_data = &fs_prog_data->base; enum brw_param_builtin *system_values; const struct intel_device_info *devinfo = screen->devinfo; unsigned num_system_values; @@ -1932,7 +2129,10 @@ iris_compile_fs(struct iris_screen *screen, nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir); const struct iris_fs_prog_key *const key = &shader->key.fs; - prog_data->use_alt_mode = nir->info.use_legacy_math_rules; + struct brw_wm_prog_data *brw_prog_data = + rzalloc(mem_ctx, struct brw_wm_prog_data); + + brw_prog_data->base.use_alt_mode = nir->info.use_legacy_math_rules; iris_setup_uniforms(devinfo, mem_ctx, nir, 0, &system_values, &num_system_values, &num_cbufs); @@ -1955,7 +2155,7 @@ iris_compile_fs(struct iris_screen *screen, MAX2(key->nr_color_regions, null_rts), num_system_values, num_cbufs); - brw_nir_analyze_ubo_ranges(compiler, nir, prog_data->ubo_ranges); + brw_nir_analyze_ubo_ranges(compiler, nir, brw_prog_data->base.ubo_ranges); struct brw_wm_prog_key brw_key = iris_to_brw_fs_key(screen, key); @@ -1967,7 +2167,7 @@ iris_compile_fs(struct iris_screen *screen, .source_hash = ish->source_hash, }, .key = &brw_key, - .prog_data = fs_prog_data, + .prog_data = brw_prog_data, .allow_spilling = true, .max_polygons = UCHAR_MAX, @@ -1989,7 +2189,9 @@ iris_compile_fs(struct iris_screen *screen, iris_debug_recompile(screen, dbg, ish, &brw_key.base); - iris_finalize_program(shader, prog_data, NULL, system_values, + iris_apply_brw_prog_data(shader, &brw_prog_data->base); + + iris_finalize_program(shader, &brw_prog_data->base, NULL, system_values, num_system_values, 0, num_cbufs, &bt); iris_upload_shader(screen, ish, shader, NULL, uploader, IRIS_CACHE_FS, @@ -2017,7 +2219,7 @@ iris_update_compiled_fs(struct iris_context *ice) screen->vtbl.populate_fs_key(ice, &ish->nir->info, &key); struct intel_vue_map *last_vue_map = - &brw_vue_prog_data(ice->shaders.last_vue_shader->prog_data)->vue_map; + &iris_vue_data(ice->shaders.last_vue_shader)->vue_map; if (ish->nos & (1ull << IRIS_NOS_LAST_VUE_MAP)) key.input_slots_valid = last_vue_map->slots_valid; @@ -2061,10 +2263,10 @@ static void update_last_vue_map(struct iris_context *ice, struct iris_compiled_shader *shader) { - struct brw_vue_prog_data *vue_prog_data = (void *) shader->prog_data; - struct intel_vue_map *vue_map = &vue_prog_data->vue_map; - struct intel_vue_map *old_map = !ice->shaders.last_vue_shader ? NULL : - &brw_vue_prog_data(ice->shaders.last_vue_shader->prog_data)->vue_map; + const struct intel_vue_map *vue_map = &iris_vue_data(shader)->vue_map; + const struct intel_vue_map *old_map = + !ice->shaders.last_vue_shader ? NULL : + &iris_vue_data(ice->shaders.last_vue_shader)->vue_map; const uint64_t changed_slots = (old_map ? old_map->slots_valid : 0ull) ^ vue_map->slots_valid; @@ -2092,7 +2294,7 @@ iris_update_pull_constant_descriptors(struct iris_context *ice, { struct iris_compiled_shader *shader = ice->shaders.prog[stage]; - if (!shader || !shader->prog_data->has_ubo_pull) + if (!shader || !shader->has_ubo_pull) return; struct iris_shader_state *shs = &ice->state.shaders[stage]; @@ -2163,12 +2365,12 @@ iris_update_compiled_shaders(struct iris_context *ice) bool points_or_lines = false; if (gs) { - const struct brw_gs_prog_data *gs_prog_data = (void *) gs->prog_data; + const struct iris_gs_data *gs_data = iris_gs_data_const(gs); points_or_lines = - gs_prog_data->output_topology == _3DPRIM_POINTLIST || - gs_prog_data->output_topology == _3DPRIM_LINESTRIP; + gs_data->output_topology == _3DPRIM_POINTLIST || + gs_data->output_topology == _3DPRIM_LINESTRIP; } else if (tes) { - const struct brw_tes_prog_data *tes_data = (void *) tes->prog_data; + const struct iris_tes_data *tes_data = iris_tes_data_const(tes); points_or_lines = tes_data->output_topology == INTEL_TESS_OUTPUT_TOPOLOGY_LINE || tes_data->output_topology == INTEL_TESS_OUTPUT_TOPOLOGY_POINT; @@ -2217,9 +2419,6 @@ iris_compile_cs(struct iris_screen *screen, { const struct brw_compiler *compiler = screen->compiler; void *mem_ctx = ralloc_context(NULL); - struct brw_cs_prog_data *cs_prog_data = - rzalloc(mem_ctx, struct brw_cs_prog_data); - struct brw_stage_prog_data *prog_data = &cs_prog_data->base; enum brw_param_builtin *system_values; const struct intel_device_info *devinfo = screen->devinfo; unsigned num_system_values; @@ -2228,7 +2427,8 @@ iris_compile_cs(struct iris_screen *screen, nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir); const struct iris_cs_prog_key *const key = &shader->key.cs; - NIR_PASS_V(nir, brw_nir_lower_cs_intrinsics, devinfo, cs_prog_data); + NIR_PASS_V(nir, brw_nir_lower_cs_intrinsics, devinfo, + brw_cs_prog_data(shader->brw_prog_data)); iris_setup_uniforms(devinfo, mem_ctx, nir, ish->kernel_input_size, &system_values, &num_system_values, &num_cbufs); @@ -2239,6 +2439,9 @@ iris_compile_cs(struct iris_screen *screen, struct brw_cs_prog_key brw_key = iris_to_brw_cs_key(screen, key); + struct brw_cs_prog_data *brw_prog_data = + rzalloc(mem_ctx, struct brw_cs_prog_data); + struct brw_compile_cs_params params = { .base = { .mem_ctx = mem_ctx, @@ -2247,7 +2450,7 @@ iris_compile_cs(struct iris_screen *screen, .source_hash = ish->source_hash, }, .key = &brw_key, - .prog_data = cs_prog_data, + .prog_data = brw_prog_data, }; const unsigned *program = brw_compile_cs(compiler, ¶ms); @@ -2264,7 +2467,9 @@ iris_compile_cs(struct iris_screen *screen, iris_debug_recompile(screen, dbg, ish, &brw_key.base); - iris_finalize_program(shader, prog_data, NULL, system_values, + iris_apply_brw_prog_data(shader, &brw_prog_data->base); + + iris_finalize_program(shader, &brw_prog_data->base, NULL, system_values, num_system_values, ish->kernel_input_size, num_cbufs, &bt); @@ -2322,14 +2527,16 @@ iris_update_compiled_compute_shader(struct iris_context *ice) } void -iris_fill_cs_push_const_buffer(struct brw_cs_prog_data *cs_prog_data, +iris_fill_cs_push_const_buffer(struct iris_screen *screen, + struct iris_compiled_shader *shader, unsigned threads, uint32_t *dst) { - assert(brw_cs_push_const_total_size(cs_prog_data, threads) > 0); - assert(cs_prog_data->push.cross_thread.size == 0); - assert(cs_prog_data->push.per_thread.dwords == 1); - assert(cs_prog_data->base.param[0] == BRW_PARAM_BUILTIN_SUBGROUP_ID); + struct iris_cs_data *cs_data = iris_cs_data(shader); + assert(iris_cs_push_const_total_size(shader, threads) > 0); + assert(cs_data->push.cross_thread.size == 0); + assert(cs_data->push.per_thread.dwords == 1); + assert(cs_data->first_param_is_builtin_subgroup_id); for (unsigned t = 0; t < threads; t++) dst[8 * t] = t; } @@ -2536,7 +2743,7 @@ iris_get_compute_state_info(struct pipe_context *ctx, void *state, list_for_each_entry_safe(struct iris_compiled_shader, shader, &ish->variants, link) { info->private_memory = MAX2(info->private_memory, - shader->prog_data->total_scratch); + shader->total_scratch); } } @@ -2547,7 +2754,6 @@ iris_get_compute_state_subgroup_size(struct pipe_context *ctx, void *state, struct iris_context *ice = (void *) ctx; struct iris_screen *screen = (void *) ctx->screen; struct u_upload_mgr *uploader = ice->shaders.uploader_driver; - const struct intel_device_info *devinfo = screen->devinfo; struct iris_uncompiled_shader *ish = state; struct iris_cs_prog_key key = { KEY_INIT(base) }; @@ -2563,9 +2769,7 @@ iris_get_compute_state_subgroup_size(struct pipe_context *ctx, void *state, iris_compile_cs(screen, uploader, &ice->dbg, ish, shader); } - struct brw_cs_prog_data *cs_prog_data = (void *) shader->prog_data; - - return brw_cs_get_dispatch_info(devinfo, cs_prog_data, block).simd_size; + return iris_get_cs_dispatch_info(screen->devinfo, shader, block).simd_size; } static void @@ -3041,3 +3245,30 @@ iris_init_program_functions(struct pipe_context *ctx) ctx->get_compute_state_info = iris_get_compute_state_info; ctx->get_compute_state_subgroup_size = iris_get_compute_state_subgroup_size; } + +struct intel_cs_dispatch_info +iris_get_cs_dispatch_info(const struct intel_device_info *devinfo, + const struct iris_compiled_shader *shader, + const uint32_t block[3]) +{ + struct brw_cs_prog_data *cs_prog_data = + brw_cs_prog_data(shader->brw_prog_data); + return brw_cs_get_dispatch_info(devinfo, cs_prog_data, block); +} + +unsigned +iris_cs_push_const_total_size(const struct iris_compiled_shader *shader, + unsigned threads) +{ + struct brw_cs_prog_data *cs_prog_data = + brw_cs_prog_data(shader->brw_prog_data); + return brw_cs_push_const_total_size(cs_prog_data, threads); +} + +uint32_t +iris_fs_barycentric_modes(const struct iris_compiled_shader *shader, + enum intel_msaa_flags pushed_msaa_flags) +{ + const struct brw_wm_prog_data *brw = brw_wm_prog_data(shader->brw_prog_data); + return wm_prog_data_barycentric_modes(brw, pushed_msaa_flags); +} diff --git a/src/gallium/drivers/iris/iris_program_cache.c b/src/gallium/drivers/iris/iris_program_cache.c index 474fbedb935..62dbd3354fe 100644 --- a/src/gallium/drivers/iris/iris_program_cache.c +++ b/src/gallium/drivers/iris/iris_program_cache.c @@ -160,15 +160,15 @@ iris_upload_shader(struct iris_screen *screen, { const struct intel_device_info *devinfo = screen->devinfo; - u_upload_alloc(uploader, 0, shader->prog_data->program_size, 64, + u_upload_alloc(uploader, 0, shader->program_size, 64, &shader->assembly.offset, &shader->assembly.res, &shader->map); - memcpy(shader->map, assembly, shader->prog_data->program_size); + memcpy(shader->map, assembly, shader->program_size); struct iris_resource *res = (void *) shader->assembly.res; uint64_t shader_data_addr = res->bo->address + shader->assembly.offset + - shader->prog_data->const_data_offset; + shader->const_data_offset; struct brw_shader_reloc_value reloc_values[] = { { @@ -181,7 +181,7 @@ iris_upload_shader(struct iris_screen *screen, }, }; brw_write_shader_relocs(&screen->compiler->isa, shader->map, - shader->prog_data, reloc_values, + shader->brw_prog_data, reloc_values, ARRAY_SIZE(reloc_values)); /* Store the 3DSTATE shader packets and other derived state. */ @@ -212,7 +212,7 @@ iris_blorp_lookup_shader(struct blorp_batch *blorp_batch, struct iris_bo *bo = iris_resource_bo(shader->assembly.res); *kernel_out = iris_bo_offset_from_base_address(bo) + shader->assembly.offset; - *((void **) prog_data_out) = shader->prog_data; + *((void **) prog_data_out) = shader->brw_prog_data; iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE); @@ -242,6 +242,8 @@ iris_blorp_upload_shader(struct blorp_batch *blorp_batch, uint32_t stage, iris_create_shader_variant(screen, ice->shaders.cache, stage, IRIS_CACHE_BLORP, key_size, key); + iris_apply_brw_prog_data(shader, prog_data); + iris_finalize_program(shader, prog_data, NULL, NULL, 0, 0, 0, &bt); iris_upload_shader(screen, NULL, shader, ice->shaders.cache, @@ -251,7 +253,7 @@ iris_blorp_upload_shader(struct blorp_batch *blorp_batch, uint32_t stage, struct iris_bo *bo = iris_resource_bo(shader->assembly.res); *kernel_out = iris_bo_offset_from_base_address(bo) + shader->assembly.offset; - *((void **) prog_data_out) = shader->prog_data; + *((void **) prog_data_out) = shader->brw_prog_data; iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE); @@ -419,6 +421,7 @@ iris_ensure_indirect_generation_shader(struct iris_batch *batch) MESA_SHADER_FRAGMENT, IRIS_CACHE_BLORP, sizeof(key), &key); + iris_apply_brw_prog_data(shader, &prog_data->base); iris_finalize_program(shader, &prog_data->base, NULL, NULL, 0, 0, 0, &bt); iris_upload_shader(screen, NULL, shader, ice->shaders.cache, diff --git a/src/gallium/drivers/iris/iris_state.c b/src/gallium/drivers/iris/iris_state.c index c33c01f82d0..c5dc1e888e6 100644 --- a/src/gallium/drivers/iris/iris_state.c +++ b/src/gallium/drivers/iris/iris_state.c @@ -96,7 +96,6 @@ #include "util/u_trace_gallium.h" #include "drm-uapi/i915_drm.h" #include "nir.h" -#include "intel/compiler/brw_compiler.h" #include "intel/common/intel_aux_map.h" #include "intel/common/intel_l3_config.h" #include "intel/common/intel_sample_positions.h" @@ -1930,8 +1929,8 @@ want_pma_fix(struct iris_context *ice) { UNUSED struct iris_screen *screen = (void *) ice->ctx.screen; UNUSED const struct intel_device_info *devinfo = screen->devinfo; - const struct brw_wm_prog_data *wm_prog_data = (void *) - ice->shaders.prog[MESA_SHADER_FRAGMENT]->prog_data; + const struct iris_fs_data *fs_data = + iris_fs_data(ice->shaders.prog[MESA_SHADER_FRAGMENT]); const struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer; const struct iris_depth_stencil_alpha_state *cso_zsa = ice->state.cso_zsa; const struct iris_blend_state *cso_blend = ice->state.cso_blend; @@ -2008,7 +2007,7 @@ want_pma_fix(struct iris_context *ice) return false; /* 3DSTATE_WM::EDSC_Mode != EDSC_PREPS */ - if (wm_prog_data->early_fragment_tests) + if (fs_data->early_fragment_tests) return false; /* 3DSTATE_WM::ForceKillPix != ForceOff && @@ -2018,7 +2017,7 @@ want_pma_fix(struct iris_context *ice) * 3DSTATE_PS_BLEND::AlphaTestEnable || * 3DSTATE_WM_CHROMAKEY::ChromaKeyKillEnable) */ - bool killpixels = wm_prog_data->uses_kill || wm_prog_data->uses_omask || + bool killpixels = fs_data->uses_kill || fs_data->uses_omask || cso_blend->alpha_to_coverage || cso_zsa->alpha_enabled; /* The Gfx8 depth PMA equation becomes: @@ -2042,7 +2041,7 @@ want_pma_fix(struct iris_context *ice) if (!cso_zsa->depth_test_enabled) return false; - return wm_prog_data->computed_depth_mode != PSCDEPTH_OFF || + return fs_data->computed_depth_mode != PSCDEPTH_OFF || (killpixels && (cso_zsa->depth_writes_enabled || (sres && cso_zsa->stencil_writes_enabled))); } @@ -4626,15 +4625,15 @@ iris_emit_sbe_swiz(struct iris_batch *batch, unsigned sprite_coord_enables) { struct GENX(SF_OUTPUT_ATTRIBUTE_DETAIL) attr_overrides[16] = {}; - const struct brw_wm_prog_data *wm_prog_data = (void *) - ice->shaders.prog[MESA_SHADER_FRAGMENT]->prog_data; + const struct iris_fs_data *fs_data = + iris_fs_data(ice->shaders.prog[MESA_SHADER_FRAGMENT]); const struct iris_rasterizer_state *cso_rast = ice->state.cso_rast; /* XXX: this should be generated when putting programs in place */ - for (uint8_t idx = 0; idx < wm_prog_data->urb_setup_attribs_count; idx++) { - const uint8_t fs_attr = wm_prog_data->urb_setup_attribs[idx]; - const int input_index = wm_prog_data->urb_setup[fs_attr]; + for (uint8_t idx = 0; idx < fs_data->urb_setup_attribs_count; idx++) { + const uint8_t fs_attr = fs_data->urb_setup_attribs[idx]; + const int input_index = fs_data->urb_setup[fs_attr]; if (input_index < 0 || input_index >= 16) continue; @@ -4719,12 +4718,12 @@ iris_is_drawing_points(const struct iris_context *ice) } if (ice->shaders.prog[MESA_SHADER_GEOMETRY]) { - const struct brw_gs_prog_data *gs_prog_data = - (void *) ice->shaders.prog[MESA_SHADER_GEOMETRY]->prog_data; - return gs_prog_data->output_topology == _3DPRIM_POINTLIST; + const struct iris_gs_data *gs_data = + iris_gs_data(ice->shaders.prog[MESA_SHADER_GEOMETRY]); + return gs_data->output_topology == _3DPRIM_POINTLIST; } else if (ice->shaders.prog[MESA_SHADER_TESS_EVAL]) { - const struct brw_tes_prog_data *tes_data = - (void *) ice->shaders.prog[MESA_SHADER_TESS_EVAL]->prog_data; + const struct iris_tes_data *tes_data = + iris_tes_data(ice->shaders.prog[MESA_SHADER_TESS_EVAL]); return tes_data->output_topology == INTEL_TESS_OUTPUT_TOPOLOGY_POINT; } else { return ice->state.prim_mode == MESA_PRIM_POINTS; @@ -4732,18 +4731,18 @@ iris_is_drawing_points(const struct iris_context *ice) } static unsigned -iris_calculate_point_sprite_overrides(const struct brw_wm_prog_data *prog_data, +iris_calculate_point_sprite_overrides(const struct iris_fs_data *fs_data, const struct iris_rasterizer_state *cso) { unsigned overrides = 0; - if (prog_data->urb_setup[VARYING_SLOT_PNTC] != -1) - overrides |= 1 << prog_data->urb_setup[VARYING_SLOT_PNTC]; + if (fs_data->urb_setup[VARYING_SLOT_PNTC] != -1) + overrides |= 1 << fs_data->urb_setup[VARYING_SLOT_PNTC]; for (int i = 0; i < 8; i++) { if ((cso->sprite_coord_enable & (1 << i)) && - prog_data->urb_setup[VARYING_SLOT_TEX0 + i] != -1) - overrides |= 1 << prog_data->urb_setup[VARYING_SLOT_TEX0 + i]; + fs_data->urb_setup[VARYING_SLOT_TEX0 + i] != -1) + overrides |= 1 << fs_data->urb_setup[VARYING_SLOT_TEX0 + i]; } return overrides; @@ -4753,30 +4752,30 @@ static void iris_emit_sbe(struct iris_batch *batch, const struct iris_context *ice) { const struct iris_rasterizer_state *cso_rast = ice->state.cso_rast; - const struct brw_wm_prog_data *wm_prog_data = (void *) - ice->shaders.prog[MESA_SHADER_FRAGMENT]->prog_data; + const struct iris_fs_data *fs_data = + iris_fs_data(ice->shaders.prog[MESA_SHADER_FRAGMENT]); const struct intel_vue_map *last_vue_map = - &brw_vue_prog_data(ice->shaders.last_vue_shader->prog_data)->vue_map; + &iris_vue_data(ice->shaders.last_vue_shader)->vue_map; unsigned urb_read_offset, urb_read_length; - iris_compute_sbe_urb_read_interval(wm_prog_data->inputs, + iris_compute_sbe_urb_read_interval(fs_data->inputs, last_vue_map, cso_rast->light_twoside, &urb_read_offset, &urb_read_length); unsigned sprite_coord_overrides = iris_is_drawing_points(ice) ? - iris_calculate_point_sprite_overrides(wm_prog_data, cso_rast) : 0; + iris_calculate_point_sprite_overrides(fs_data, cso_rast) : 0; iris_emit_cmd(batch, GENX(3DSTATE_SBE), sbe) { sbe.AttributeSwizzleEnable = true; - sbe.NumberofSFOutputAttributes = wm_prog_data->num_varying_inputs; + sbe.NumberofSFOutputAttributes = fs_data->num_varying_inputs; sbe.PointSpriteTextureCoordinateOrigin = cso_rast->sprite_coord_mode; sbe.VertexURBEntryReadOffset = urb_read_offset; sbe.VertexURBEntryReadLength = urb_read_length; sbe.ForceVertexURBEntryReadOffset = true; sbe.ForceVertexURBEntryReadLength = true; - sbe.ConstantInterpolationEnable = wm_prog_data->flat_inputs; + sbe.ConstantInterpolationEnable = fs_data->flat_inputs; sbe.PointSpriteTextureCoordinateEnable = sprite_coord_overrides; #if GFX_VER >= 9 for (int i = 0; i < 32; i++) { @@ -4787,10 +4786,10 @@ iris_emit_sbe(struct iris_batch *batch, const struct iris_context *ice) /* Ask the hardware to supply PrimitiveID if the fragment shader * reads it but a previous stage didn't write one. */ - if ((wm_prog_data->inputs & VARYING_BIT_PRIMITIVE_ID) && + if ((fs_data->inputs & VARYING_BIT_PRIMITIVE_ID) && last_vue_map->varying_to_slot[VARYING_SLOT_PRIMITIVE_ID] == -1) { sbe.PrimitiveIDOverrideAttributeSelect = - wm_prog_data->urb_setup[VARYING_SLOT_PRIMITIVE_ID]; + fs_data->urb_setup[VARYING_SLOT_PRIMITIVE_ID]; sbe.PrimitiveIDOverrideComponentX = true; sbe.PrimitiveIDOverrideComponentY = true; sbe.PrimitiveIDOverrideComponentZ = true; @@ -4922,17 +4921,17 @@ encode_sampler_count(const struct iris_compiled_shader *shader) pkt.KernelStartPointer = KSP(shader); \ pkt.BindingTableEntryCount = shader->bt.size_bytes / 4; \ pkt.SamplerCount = encode_sampler_count(shader); \ - pkt.FloatingPointMode = prog_data->use_alt_mode; \ + pkt.FloatingPointMode = shader->use_alt_mode; \ \ pkt.DispatchGRFStartRegisterForURBData = \ - prog_data->dispatch_grf_start_reg; \ - pkt.prefix##URBEntryReadLength = vue_prog_data->urb_read_length; \ + shader->dispatch_grf_start_reg; \ + pkt.prefix##URBEntryReadLength = vue_data->urb_read_length; \ pkt.prefix##URBEntryReadOffset = 0; \ \ pkt.StatisticsEnable = true; \ pkt.Enable = true; \ \ - if (prog_data->total_scratch) { \ + if (shader->total_scratch) { \ INIT_THREAD_SCRATCH_SIZE(pkt) \ } @@ -4948,7 +4947,7 @@ encode_sampler_count(const struct iris_compiled_shader *shader) } #else #define INIT_THREAD_SCRATCH_SIZE(pkt) \ - pkt.PerThreadScratchSpace = ffs(prog_data->total_scratch) - 11; + pkt.PerThreadScratchSpace = ffs(shader->total_scratch) - 11; #define MERGE_SCRATCH_ADDR(name) \ { \ uint32_t pkt2[GENX(name##_length)] = {0}; \ @@ -4968,8 +4967,7 @@ static void iris_store_vs_state(const struct intel_device_info *devinfo, struct iris_compiled_shader *shader) { - struct brw_stage_prog_data *prog_data = shader->prog_data; - struct brw_vue_prog_data *vue_prog_data = (void *) prog_data; + struct iris_vue_data *vue_data = iris_vue_data(shader); iris_pack_command(GENX(3DSTATE_VS), shader->derived_data, vs) { INIT_THREAD_DISPATCH_FIELDS(vs, Vertex, MESA_SHADER_VERTEX); @@ -4978,7 +4976,7 @@ iris_store_vs_state(const struct intel_device_info *devinfo, vs.SIMD8DispatchEnable = true; #endif vs.UserClipDistanceCullTestEnableBitmask = - vue_prog_data->cull_distance_mask; + vue_data->cull_distance_mask; } } @@ -4989,9 +4987,8 @@ static void iris_store_tcs_state(const struct intel_device_info *devinfo, struct iris_compiled_shader *shader) { - struct brw_stage_prog_data *prog_data = shader->prog_data; - struct brw_vue_prog_data *vue_prog_data = (void *) prog_data; - struct brw_tcs_prog_data *tcs_prog_data = (void *) prog_data; + struct iris_tcs_data *tcs_data = iris_tcs_data(shader); + struct iris_vue_data *vue_data = &tcs_data->base; iris_pack_command(GENX(3DSTATE_HS), shader->derived_data, hs) { INIT_THREAD_DISPATCH_FIELDS(hs, Vertex, MESA_SHADER_TESS_CTRL); @@ -5003,12 +5000,12 @@ iris_store_tcs_state(const struct intel_device_info *devinfo, * the number of instance count. The number of max threads must be * more than 2 times the number of instance count. */ - assert((devinfo->max_tcs_threads / 2) > tcs_prog_data->instances); - hs.DispatchGRFStartRegisterForURBData = prog_data->dispatch_grf_start_reg & 0x1f; - hs.DispatchGRFStartRegisterForURBData5 = prog_data->dispatch_grf_start_reg >> 5; + assert((devinfo->max_tcs_threads / 2) > tcs_data->instances); + hs.DispatchGRFStartRegisterForURBData = shader->dispatch_grf_start_reg & 0x1f; + hs.DispatchGRFStartRegisterForURBData5 = shader->dispatch_grf_start_reg >> 5; #endif - hs.InstanceCount = tcs_prog_data->instances - 1; + hs.InstanceCount = tcs_data->instances - 1; hs.MaximumNumberofThreads = devinfo->max_tcs_threads - 1; hs.IncludeVertexHandles = true; @@ -5016,14 +5013,14 @@ iris_store_tcs_state(const struct intel_device_info *devinfo, /* Patch Count threshold specifies the maximum number of patches that * will be accumulated before a thread dispatch is forced. */ - hs.PatchCountThreshold = tcs_prog_data->patch_count_threshold; + hs.PatchCountThreshold = tcs_data->patch_count_threshold; #endif #if GFX_VER >= 9 #if GFX_VER < 20 - hs.DispatchMode = vue_prog_data->dispatch_mode; + hs.DispatchMode = vue_data->dispatch_mode; #endif - hs.IncludePrimitiveID = tcs_prog_data->include_primitive_id; + hs.IncludePrimitiveID = tcs_data->include_primitive_id; #endif } } @@ -5035,9 +5032,8 @@ static void iris_store_tes_state(const struct intel_device_info *devinfo, struct iris_compiled_shader *shader) { - struct brw_stage_prog_data *prog_data = shader->prog_data; - struct brw_vue_prog_data *vue_prog_data = (void *) prog_data; - struct brw_tes_prog_data *tes_prog_data = (void *) prog_data; + struct iris_tes_data *tes_data = iris_tes_data(shader); + struct iris_vue_data *vue_data = &tes_data->base; uint32_t *ds_state = (void *) shader->derived_data; uint32_t *te_state = ds_state + GENX(3DSTATE_DS_length); @@ -5048,19 +5044,19 @@ iris_store_tes_state(const struct intel_device_info *devinfo, ds.DispatchMode = DISPATCH_MODE_SIMD8_SINGLE_PATCH; ds.MaximumNumberofThreads = devinfo->max_tes_threads - 1; ds.ComputeWCoordinateEnable = - tes_prog_data->domain == INTEL_TESS_DOMAIN_TRI; + tes_data->domain == INTEL_TESS_DOMAIN_TRI; #if GFX_VER >= 12 - ds.PrimitiveIDNotRequired = !tes_prog_data->include_primitive_id; + ds.PrimitiveIDNotRequired = !tes_data->include_primitive_id; #endif ds.UserClipDistanceCullTestEnableBitmask = - vue_prog_data->cull_distance_mask; + vue_data->cull_distance_mask; } iris_pack_command(GENX(3DSTATE_TE), te_state, te) { - te.Partitioning = tes_prog_data->partitioning; - te.OutputTopology = tes_prog_data->output_topology; - te.TEDomain = tes_prog_data->domain; + te.Partitioning = tes_data->partitioning; + te.OutputTopology = tes_data->output_topology; + te.TEDomain = tes_data->domain; te.TEEnable = true; te.MaximumTessellationFactorOdd = 63.0; te.MaximumTessellationFactorNotOdd = 64.0; @@ -5096,42 +5092,38 @@ static void iris_store_gs_state(const struct intel_device_info *devinfo, struct iris_compiled_shader *shader) { - struct brw_stage_prog_data *prog_data = shader->prog_data; - struct brw_vue_prog_data *vue_prog_data = (void *) prog_data; - struct brw_gs_prog_data *gs_prog_data = (void *) prog_data; + struct iris_gs_data *gs_data = iris_gs_data(shader); + struct iris_vue_data *vue_data = &gs_data->base; iris_pack_command(GENX(3DSTATE_GS), shader->derived_data, gs) { INIT_THREAD_DISPATCH_FIELDS(gs, Vertex, MESA_SHADER_GEOMETRY); - gs.OutputVertexSize = gs_prog_data->output_vertex_size_hwords * 2 - 1; - gs.OutputTopology = gs_prog_data->output_topology; - gs.ControlDataHeaderSize = - gs_prog_data->control_data_header_size_hwords; - gs.InstanceControl = gs_prog_data->invocations - 1; + gs.OutputVertexSize = gs_data->output_vertex_size_hwords * 2 - 1; + gs.OutputTopology = gs_data->output_topology; + gs.ControlDataHeaderSize = gs_data->control_data_header_size_hwords; + gs.InstanceControl = gs_data->invocations - 1; #if GFX_VER < 20 gs.DispatchMode = DISPATCH_MODE_SIMD8; #endif - gs.IncludePrimitiveID = gs_prog_data->include_primitive_id; - gs.ControlDataFormat = gs_prog_data->control_data_format; + gs.IncludePrimitiveID = gs_data->include_primitive_id; + gs.ControlDataFormat = gs_data->control_data_format; gs.ReorderMode = TRAILING; - gs.ExpectedVertexCount = gs_prog_data->vertices_in; + gs.ExpectedVertexCount = gs_data->vertices_in; gs.MaximumNumberofThreads = GFX_VER == 8 ? (devinfo->max_gs_threads / 2 - 1) : (devinfo->max_gs_threads - 1); - if (gs_prog_data->static_vertex_count != -1) { + if (gs_data->static_vertex_count != -1) { gs.StaticOutput = true; - gs.StaticOutputVertexCount = gs_prog_data->static_vertex_count; + gs.StaticOutputVertexCount = gs_data->static_vertex_count; } - gs.IncludeVertexHandles = vue_prog_data->include_vue_handles; + gs.IncludeVertexHandles = vue_data->include_vue_handles; - gs.UserClipDistanceCullTestEnableBitmask = - vue_prog_data->cull_distance_mask; + gs.UserClipDistanceCullTestEnableBitmask = vue_data->cull_distance_mask; const int urb_entry_write_offset = 1; const uint32_t urb_entry_output_length = - DIV_ROUND_UP(vue_prog_data->vue_map.num_slots, 2) - - urb_entry_write_offset; + DIV_ROUND_UP(vue_data->vue_map.num_slots, 2) - urb_entry_write_offset; gs.VertexURBEntryOutputReadOffset = urb_entry_write_offset; gs.VertexURBEntryOutputLength = MAX2(urb_entry_output_length, 1); @@ -5145,22 +5137,21 @@ static void iris_store_fs_state(const struct intel_device_info *devinfo, struct iris_compiled_shader *shader) { - struct brw_stage_prog_data *prog_data = shader->prog_data; - struct brw_wm_prog_data *wm_prog_data = (void *) shader->prog_data; + struct iris_fs_data *fs_data = iris_fs_data(shader); uint32_t *ps_state = (void *) shader->derived_data; uint32_t *psx_state = ps_state + GENX(3DSTATE_PS_length); iris_pack_command(GENX(3DSTATE_PS), ps_state, ps) { - ps.VectorMaskEnable = wm_prog_data->uses_vmask; + ps.VectorMaskEnable = fs_data->uses_vmask; ps.BindingTableEntryCount = shader->bt.size_bytes / 4; ps.SamplerCount = encode_sampler_count(shader); - ps.FloatingPointMode = prog_data->use_alt_mode; + ps.FloatingPointMode = shader->use_alt_mode; ps.MaximumNumberofThreadsPerPSD = devinfo->max_threads_per_psd - (GFX_VER == 8 ? 2 : 1); #if GFX_VER < 20 - ps.PushConstantEnable = prog_data->ubo_ranges[0].length > 0; + ps.PushConstantEnable = shader->ubo_ranges[0].length > 0; #endif /* From the documentation for this packet: @@ -5177,33 +5168,32 @@ iris_store_fs_state(const struct intel_device_info *devinfo, * look useful at the moment. We might need this in future. */ ps.PositionXYOffsetSelect = - wm_prog_data->uses_pos_offset ? POSOFFSET_SAMPLE : POSOFFSET_NONE; + fs_data->uses_pos_offset ? POSOFFSET_SAMPLE : POSOFFSET_NONE; - if (prog_data->total_scratch) { + if (shader->total_scratch) { INIT_THREAD_SCRATCH_SIZE(ps); } } iris_pack_command(GENX(3DSTATE_PS_EXTRA), psx_state, psx) { psx.PixelShaderValid = true; - psx.PixelShaderComputedDepthMode = wm_prog_data->computed_depth_mode; - psx.PixelShaderKillsPixel = wm_prog_data->uses_kill; + psx.PixelShaderComputedDepthMode = fs_data->computed_depth_mode; + psx.PixelShaderKillsPixel = fs_data->uses_kill; #if GFX_VER < 20 - psx.AttributeEnable = wm_prog_data->num_varying_inputs != 0; + psx.AttributeEnable = fs_data->num_varying_inputs != 0; #endif - psx.PixelShaderUsesSourceDepth = wm_prog_data->uses_src_depth; - psx.PixelShaderUsesSourceW = wm_prog_data->uses_src_w; - psx.PixelShaderIsPerSample = - brw_wm_prog_data_is_persample(wm_prog_data, 0); - psx.oMaskPresenttoRenderTarget = wm_prog_data->uses_omask; + psx.PixelShaderUsesSourceDepth = fs_data->uses_src_depth; + psx.PixelShaderUsesSourceW = fs_data->uses_src_w; + psx.PixelShaderIsPerSample = fs_data->is_per_sample; + psx.oMaskPresenttoRenderTarget = fs_data->uses_omask; #if GFX_VER >= 9 #if GFX_VER >= 20 - assert(!wm_prog_data->pulls_bary); + assert(!fs_data->pulls_bary); #else - psx.PixelShaderPullsBary = wm_prog_data->pulls_bary; + psx.PixelShaderPullsBary = fs_data->pulls_bary; #endif - psx.PixelShaderComputesStencil = wm_prog_data->computed_stencil; + psx.PixelShaderComputesStencil = fs_data->computed_stencil; #endif } } @@ -5217,20 +5207,20 @@ static void iris_store_cs_state(const struct intel_device_info *devinfo, struct iris_compiled_shader *shader) { - struct brw_cs_prog_data *cs_prog_data = (void *) shader->prog_data; + struct iris_cs_data *cs_data = iris_cs_data(shader); void *map = shader->derived_data; iris_pack_state(GENX(INTERFACE_DESCRIPTOR_DATA), map, desc) { #if GFX_VERx10 < 125 - desc.ConstantURBEntryReadLength = cs_prog_data->push.per_thread.regs; + desc.ConstantURBEntryReadLength = cs_data->push.per_thread.regs; desc.CrossThreadConstantDataReadLength = - cs_prog_data->push.cross_thread.regs; + cs_data->push.cross_thread.regs; #else - assert(cs_prog_data->push.per_thread.regs == 0); - assert(cs_prog_data->push.cross_thread.regs == 0); + assert(cs_data->push.per_thread.regs == 0); + assert(cs_data->push.cross_thread.regs == 0); #endif #if GFX_VERx10 <= 125 - desc.BarrierEnable = cs_prog_data->uses_barrier; + desc.BarrierEnable = cs_data->uses_barrier; #endif /* Typically set to 0 to avoid prefetching on every thread dispatch. */ desc.BindingTableEntryCount = devinfo->verx10 == 125 ? @@ -5597,7 +5587,6 @@ iris_populate_binding_table(struct iris_context *ice, return; struct iris_binding_table *bt = &shader->bt; - UNUSED struct brw_stage_prog_data *prog_data = shader->prog_data; struct iris_shader_state *shs = &ice->state.shaders[stage]; uint32_t surf_base_offset = GFX_VER < 11 ? binder->bo->address : 0; @@ -5743,19 +5732,19 @@ pin_depth_and_stencil_buffers(struct iris_batch *batch, static uint32_t pin_scratch_space(struct iris_context *ice, struct iris_batch *batch, - const struct brw_stage_prog_data *prog_data, + const struct iris_compiled_shader *shader, gl_shader_stage stage) { uint32_t scratch_addr = 0; - if (prog_data->total_scratch > 0) { + if (shader->total_scratch > 0) { struct iris_bo *scratch_bo = - iris_get_scratch_space(ice, prog_data->total_scratch, stage); + iris_get_scratch_space(ice, shader->total_scratch, stage); iris_use_pinned_bo(batch, scratch_bo, true, IRIS_DOMAIN_NONE); #if GFX_VERx10 >= 125 const struct iris_state_ref *ref = - iris_get_scratch_surf(ice, prog_data->total_scratch); + iris_get_scratch_surf(ice, shader->total_scratch); iris_use_pinned_bo(batch, iris_resource_bo(ref->res), false, IRIS_DOMAIN_NONE); scratch_addr = ref->offset + @@ -5843,10 +5832,8 @@ iris_restore_render_saved_bos(struct iris_context *ice, if (!shader) continue; - struct brw_stage_prog_data *prog_data = (void *) shader->prog_data; - for (int i = 0; i < 4; i++) { - const struct brw_ubo_range *range = &prog_data->ubo_ranges[i]; + const struct iris_ubo_range *range = &shader->ubo_ranges[i]; if (range->length == 0) continue; @@ -5890,7 +5877,7 @@ iris_restore_render_saved_bos(struct iris_context *ice, struct iris_bo *bo = iris_resource_bo(shader->assembly.res); iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE); - pin_scratch_space(ice, batch, shader->prog_data, stage); + pin_scratch_space(ice, batch, shader, stage); } } } @@ -5963,7 +5950,7 @@ iris_restore_compute_saved_bos(struct iris_context *ice, iris_use_pinned_bo(batch, curbe_bo, false, IRIS_DOMAIN_NONE); } - pin_scratch_space(ice, batch, shader->prog_data, stage); + pin_scratch_space(ice, batch, shader, stage); } } } @@ -6288,13 +6275,12 @@ setup_constant_buffers(struct iris_context *ice, { struct iris_shader_state *shs = &ice->state.shaders[stage]; struct iris_compiled_shader *shader = ice->shaders.prog[stage]; - struct brw_stage_prog_data *prog_data = (void *) shader->prog_data; uint32_t push_range_sum = 0; int n = 0; for (int i = 0; i < 4; i++) { - const struct brw_ubo_range *range = &prog_data->ubo_ranges[i]; + const struct iris_ubo_range *range = &shader->ubo_ranges[i]; if (range->length == 0) continue; @@ -6564,7 +6550,7 @@ iris_preemption_streamout_wa(struct iris_context *ice, static void shader_program_needs_wa_14015055625(struct iris_context *ice, struct iris_batch *batch, - const struct brw_stage_prog_data *prog_data, + struct iris_compiled_shader *shader, gl_shader_stage stage, bool *program_needs_wa_14015055625) { @@ -6573,13 +6559,13 @@ shader_program_needs_wa_14015055625(struct iris_context *ice, switch (stage) { case MESA_SHADER_TESS_CTRL: { - struct brw_tcs_prog_data *tcs_prog_data = (void *) prog_data; - *program_needs_wa_14015055625 |= tcs_prog_data->include_primitive_id; + struct iris_tcs_data *tcs_data = iris_tcs_data(shader); + *program_needs_wa_14015055625 |= tcs_data->include_primitive_id; break; } case MESA_SHADER_TESS_EVAL: { - struct brw_tes_prog_data *tes_prog_data = (void *) prog_data; - *program_needs_wa_14015055625 |= tes_prog_data->include_primitive_id; + struct iris_tes_data *tes_data = iris_tes_data(shader); + *program_needs_wa_14015055625 |= tes_data->include_primitive_id; break; } default: @@ -6588,11 +6574,10 @@ shader_program_needs_wa_14015055625(struct iris_context *ice, struct iris_compiled_shader *gs_shader = ice->shaders.prog[MESA_SHADER_GEOMETRY]; - const struct brw_gs_prog_data *gs_prog_data = - gs_shader ? (void *) gs_shader->prog_data : NULL; + const struct iris_gs_data *gs_data = + gs_shader ? iris_gs_data(gs_shader) : NULL; - *program_needs_wa_14015055625 |= - gs_prog_data && gs_prog_data->include_primitive_id; + *program_needs_wa_14015055625 |= gs_data && gs_data->include_primitive_id; } static void @@ -6700,8 +6685,8 @@ iris_upload_dirty_render_state(struct iris_context *ice, struct iris_genx_state *genx = ice->state.genx; struct iris_binder *binder = &ice->state.binder; - struct brw_wm_prog_data *wm_prog_data = (void *) - ice->shaders.prog[MESA_SHADER_FRAGMENT]->prog_data; + struct iris_fs_data *fs_data = + iris_fs_data(ice->shaders.prog[MESA_SHADER_FRAGMENT]); /* When MSAA is enabled, instead of using BLENDFACTOR_ZERO use * CONST_COLOR, CONST_ALPHA and supply zero by using blend constants. @@ -6830,9 +6815,9 @@ iris_upload_dirty_render_state(struct iris_context *ice, if (!ice->shaders.prog[i]) { ice->shaders.urb.cfg.size[i] = 1; } else { - struct brw_vue_prog_data *vue_prog_data = - (void *) ice->shaders.prog[i]->prog_data; - ice->shaders.urb.cfg.size[i] = vue_prog_data->urb_entry_size; + struct iris_vue_data *vue_data = + iris_vue_data(ice->shaders.prog[i]); + ice->shaders.urb.cfg.size[i] = vue_data->urb_entry_size; } assert(ice->shaders.urb.cfg.size[i] != 0); } @@ -7135,8 +7120,8 @@ iris_upload_dirty_render_state(struct iris_context *ice, #if INTEL_WA_14015055625_GFX_VER /* Check if FS stage will use primitive ID overrides for Wa_14015055625. */ const struct intel_vue_map *last_vue_map = - &brw_vue_prog_data(ice->shaders.last_vue_shader->prog_data)->vue_map; - if ((wm_prog_data->inputs & VARYING_BIT_PRIMITIVE_ID) && + &iris_vue_data(ice->shaders.last_vue_shader)->vue_map; + if ((fs_data->inputs & VARYING_BIT_PRIMITIVE_ID) && last_vue_map->varying_to_slot[VARYING_SLOT_PRIMITIVE_ID] == -1 && intel_needs_workaround(batch->screen->devinfo, 14015055625)) { program_needs_wa_14015055625 = true; @@ -7150,15 +7135,14 @@ iris_upload_dirty_render_state(struct iris_context *ice, struct iris_compiled_shader *shader = ice->shaders.prog[stage]; if (shader) { - struct brw_stage_prog_data *prog_data = shader->prog_data; struct iris_resource *cache = (void *) shader->assembly.res; iris_use_pinned_bo(batch, cache->bo, false, IRIS_DOMAIN_NONE); uint32_t scratch_addr = - pin_scratch_space(ice, batch, prog_data, stage); + pin_scratch_space(ice, batch, shader, stage); #if INTEL_WA_14015055625_GFX_VER - shader_program_needs_wa_14015055625(ice, batch, prog_data, stage, + shader_program_needs_wa_14015055625(ice, batch, shader, stage, &program_needs_wa_14015055625); #endif @@ -7168,14 +7152,15 @@ iris_upload_dirty_render_state(struct iris_context *ice, uint32_t ps_state[GENX(3DSTATE_PS_length)] = {0}; _iris_pack_command(batch, GENX(3DSTATE_PS), ps_state, ps) { + struct brw_wm_prog_data *wm_prog_data = brw_wm_prog_data(shader->brw_prog_data); intel_set_ps_dispatch_state(&ps, batch->screen->devinfo, wm_prog_data, util_framebuffer_get_num_samples(cso_fb), 0 /* msaa_flags */); #if GFX_VER == 12 - assert(wm_prog_data->dispatch_multi == 0 || - (wm_prog_data->dispatch_multi == 16 && wm_prog_data->max_polygons == 2)); - ps.DualSIMD8DispatchEnable = wm_prog_data->dispatch_multi; + assert(fs_data->dispatch_multi == 0 || + (fs_data->dispatch_multi == 16 && fs_data->max_polygons == 2)); + ps.DualSIMD8DispatchEnable = fs_data->dispatch_multi; /* XXX - No major improvement observed from enabling * overlapping subspans, but it could be helpful * in theory when the requirements listed on the @@ -7213,18 +7198,18 @@ iris_upload_dirty_render_state(struct iris_context *ice, uint32_t psx_state[GENX(3DSTATE_PS_EXTRA_length)] = {0}; iris_pack_command(GENX(3DSTATE_PS_EXTRA), psx_state, psx) { #if GFX_VER >= 9 - if (!wm_prog_data->uses_sample_mask) + if (!fs_data->uses_sample_mask) psx.InputCoverageMaskState = ICMS_NONE; - else if (wm_prog_data->post_depth_coverage) + else if (fs_data->post_depth_coverage) psx.InputCoverageMaskState = ICMS_DEPTH_COVERAGE; - else if (wm_prog_data->inner_coverage && + else if (fs_data->inner_coverage && cso->conservative_rasterization) psx.InputCoverageMaskState = ICMS_INNER_CONSERVATIVE; else psx.InputCoverageMaskState = ICMS_NORMAL; #else psx.PixelShaderUsesInputCoverageMask = - wm_prog_data->uses_sample_mask; + fs_data->uses_sample_mask; #endif } @@ -7461,7 +7446,7 @@ iris_upload_dirty_render_state(struct iris_context *ice, cl.PerspectiveDivideDisable = ice->state.window_space_position; cl.ViewportXYClipTestEnable = !points_or_lines; - cl.NonPerspectiveBarycentricEnable = wm_prog_data->uses_nonperspective_interp_modes; + cl.NonPerspectiveBarycentricEnable = fs_data->uses_nonperspective_interp_modes; cl.ForceZeroRTAIndexEnable = cso_fb->layers <= 1; cl.MaximumVPIndex = ice->state.num_viewports - 1; @@ -7517,17 +7502,17 @@ iris_upload_dirty_render_state(struct iris_context *ice, wm.StatisticsEnable = ice->state.statistics_counters_enabled; wm.BarycentricInterpolationMode = - wm_prog_data_barycentric_modes(wm_prog_data, 0); + iris_fs_barycentric_modes(ice->shaders.prog[MESA_SHADER_FRAGMENT], 0); - if (wm_prog_data->early_fragment_tests) + if (fs_data->early_fragment_tests) wm.EarlyDepthStencilControl = EDSC_PREPS; - else if (wm_prog_data->has_side_effects) + else if (fs_data->has_side_effects) wm.EarlyDepthStencilControl = EDSC_PSEXEC; else wm.EarlyDepthStencilControl = EDSC_NORMAL; /* We could skip this bit if color writes are enabled. */ - if (wm_prog_data->has_side_effects || wm_prog_data->uses_kill) + if (fs_data->has_side_effects || fs_data->uses_kill) wm.ForceThreadDispatchEnable = ForceON; } iris_emit_merge(batch, cso->wm, dynamic_wm, ARRAY_SIZE(cso->wm)); @@ -7570,7 +7555,7 @@ iris_upload_dirty_render_state(struct iris_context *ice, * undefined anyway, so simply disable blending to avoid the hang. */ pb.ColorBufferBlendEnable = (cso_blend->blend_enables & 1) && - (!cso_blend->dual_color_blending || wm_prog_data->dual_src_blend); + (!cso_blend->dual_color_blending || fs_data->dual_src_blend); } iris_emit_merge(batch, cso_blend->ps_blend, dynamic_pb, @@ -7964,19 +7949,19 @@ iris_upload_dirty_render_state(struct iris_context *ice, } if (dirty & IRIS_DIRTY_VF_SGVS) { - const struct brw_vs_prog_data *vs_prog_data = (void *) - ice->shaders.prog[MESA_SHADER_VERTEX]->prog_data; + const struct iris_vs_data *vs_data = + iris_vs_data(ice->shaders.prog[MESA_SHADER_VERTEX]); struct iris_vertex_element_state *cso = ice->state.cso_vertex_elements; iris_emit_cmd(batch, GENX(3DSTATE_VF_SGVS), sgv) { - if (vs_prog_data->uses_vertexid) { + if (vs_data->uses_vertexid) { sgv.VertexIDEnable = true; sgv.VertexIDComponentNumber = 2; sgv.VertexIDElementOffset = cso->count - ice->state.vs_needs_edge_flag; } - if (vs_prog_data->uses_instanceid) { + if (vs_data->uses_instanceid) { sgv.InstanceIDEnable = true; sgv.InstanceIDComponentNumber = 3; sgv.InstanceIDElementOffset = @@ -8750,13 +8735,10 @@ static void iris_emit_execute_indirect_dispatch(struct iris_context *ice, const struct GENX(INTERFACE_DESCRIPTOR_DATA) idd) { const struct iris_screen *screen = batch->screen; - const struct intel_device_info *devinfo = screen->devinfo; struct iris_compiled_shader *shader = ice->shaders.prog[MESA_SHADER_COMPUTE]; - struct brw_stage_prog_data *prog_data = shader->prog_data; - struct brw_cs_prog_data *cs_prog_data = (void *) prog_data; const struct intel_cs_dispatch_info dispatch = - brw_cs_get_dispatch_info(devinfo, cs_prog_data, grid->block); + iris_get_cs_dispatch_info(screen->devinfo, shader, grid->block); struct iris_bo *indirect = iris_resource_bo(grid->indirect); const int dispatch_size = dispatch.simd_size / 16; @@ -8794,10 +8776,9 @@ iris_upload_compute_walker(struct iris_context *ice, struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_COMPUTE]; struct iris_compiled_shader *shader = ice->shaders.prog[MESA_SHADER_COMPUTE]; - struct brw_stage_prog_data *prog_data = shader->prog_data; - struct brw_cs_prog_data *cs_prog_data = (void *) prog_data; + const struct iris_cs_data *cs_data = iris_cs_data(shader); const struct intel_cs_dispatch_info dispatch = - brw_cs_get_dispatch_info(devinfo, cs_prog_data, grid->block); + iris_get_cs_dispatch_info(devinfo, shader, grid->block); trace_intel_begin_compute(&batch->trace); @@ -8805,7 +8786,7 @@ iris_upload_compute_walker(struct iris_context *ice, iris_emit_cmd(batch, GENX(CFE_STATE), cfe) { cfe.MaximumNumberofThreads = devinfo->max_cs_threads * devinfo->subslice_total; - uint32_t scratch_addr = pin_scratch_space(ice, batch, prog_data, + uint32_t scratch_addr = pin_scratch_space(ice, batch, shader, MESA_SHADER_COMPUTE); cfe.ScratchSpaceBuffer = scratch_addr >> 4; } @@ -8815,7 +8796,7 @@ iris_upload_compute_walker(struct iris_context *ice, idd.KernelStartPointer = KSP(shader); idd.NumberofThreadsinGPGPUThreadGroup = dispatch.threads; idd.SharedLocalMemorySize = - encode_slm_size(GFX_VER, prog_data->total_shared); + encode_slm_size(GFX_VER, shader->total_shared); idd.SamplerStatePointer = shs->sampler_table.offset; idd.SamplerCount = encode_sampler_count(shader), idd.BindingTablePointer = binder->bt_offset[MESA_SHADER_COMPUTE]; @@ -8823,7 +8804,7 @@ iris_upload_compute_walker(struct iris_context *ice, idd.BindingTableEntryCount = devinfo->verx10 == 125 ? 0 : MIN2(shader->bt.size_bytes / 4, 31); idd.PreferredSLMAllocationSize = preferred_slm_allocation_size(devinfo); - idd.NumberOfBarriers = cs_prog_data->uses_barrier; + idd.NumberOfBarriers = cs_data->uses_barrier; iris_measure_snapshot(ice, batch, INTEL_SNAPSHOT_COMPUTE, NULL, NULL, NULL); @@ -8853,14 +8834,14 @@ iris_upload_compute_walker(struct iris_context *ice, cw.InterfaceDescriptor = idd; #if GFX_VERx10 >= 125 - cw.GenerateLocalID = cs_prog_data->generate_local_id != 0; - cw.EmitLocal = cs_prog_data->generate_local_id; - cw.WalkOrder = cs_prog_data->walk_order; - cw.TileLayout = cs_prog_data->walk_order == INTEL_WALK_ORDER_YXZ ? + cw.GenerateLocalID = cs_data->generate_local_id != 0; + cw.EmitLocal = cs_data->generate_local_id; + cw.WalkOrder = cs_data->walk_order; + cw.TileLayout = cs_data->walk_order == INTEL_WALK_ORDER_YXZ ? TileY32bpe : Linear; #endif - assert(brw_cs_push_const_total_size(cs_prog_data, dispatch.threads) == 0); + assert(iris_cs_push_const_total_size(shader, dispatch.threads) == 0); } } @@ -8883,15 +8864,14 @@ iris_upload_gpgpu_walker(struct iris_context *ice, ice->shaders.uncompiled[MESA_SHADER_COMPUTE]; struct iris_compiled_shader *shader = ice->shaders.prog[MESA_SHADER_COMPUTE]; - struct brw_stage_prog_data *prog_data = shader->prog_data; - struct brw_cs_prog_data *cs_prog_data = (void *) prog_data; + struct iris_cs_data *cs_data = iris_cs_data(shader); const struct intel_cs_dispatch_info dispatch = - brw_cs_get_dispatch_info(devinfo, cs_prog_data, grid->block); + iris_get_cs_dispatch_info(screen->devinfo, shader, grid->block); trace_intel_begin_compute(&batch->trace); if ((stage_dirty & IRIS_STAGE_DIRTY_CS) || - cs_prog_data->local_size[0] == 0 /* Variable local group size */) { + cs_data->local_size[0] == 0 /* Variable local group size */) { /* The MEDIA_VFE_STATE documentation for Gfx8+ says: * * "A stalling PIPE_CONTROL is required before MEDIA_VFE_STATE unless @@ -8905,11 +8885,11 @@ iris_upload_gpgpu_walker(struct iris_context *ice, PIPE_CONTROL_CS_STALL); iris_emit_cmd(batch, GENX(MEDIA_VFE_STATE), vfe) { - if (prog_data->total_scratch) { + if (shader->total_scratch) { uint32_t scratch_addr = - pin_scratch_space(ice, batch, prog_data, MESA_SHADER_COMPUTE); + pin_scratch_space(ice, batch, shader, MESA_SHADER_COMPUTE); - vfe.PerThreadScratchSpace = ffs(prog_data->total_scratch) - 11; + vfe.PerThreadScratchSpace = ffs(shader->total_scratch) - 11; vfe.ScratchSpaceBasePointer = rw_bo(NULL, scratch_addr, IRIS_DOMAIN_NONE); } @@ -8927,20 +8907,20 @@ iris_upload_gpgpu_walker(struct iris_context *ice, vfe.URBEntryAllocationSize = 2; vfe.CURBEAllocationSize = - ALIGN(cs_prog_data->push.per_thread.regs * dispatch.threads + - cs_prog_data->push.cross_thread.regs, 2); + ALIGN(cs_data->push.per_thread.regs * dispatch.threads + + cs_data->push.cross_thread.regs, 2); } } /* TODO: Combine subgroup-id with cbuf0 so we can push regular uniforms */ if ((stage_dirty & IRIS_STAGE_DIRTY_CS) || - cs_prog_data->local_size[0] == 0 /* Variable local group size */) { + cs_data->local_size[0] == 0 /* Variable local group size */) { uint32_t curbe_data_offset = 0; - assert(cs_prog_data->push.cross_thread.dwords == 0 && - cs_prog_data->push.per_thread.dwords == 1 && - cs_prog_data->base.param[0] == BRW_PARAM_BUILTIN_SUBGROUP_ID); + assert(cs_data->push.cross_thread.dwords == 0 && + cs_data->push.per_thread.dwords == 1 && + cs_data->first_param_is_builtin_subgroup_id); const unsigned push_const_size = - brw_cs_push_const_total_size(cs_prog_data, dispatch.threads); + iris_cs_push_const_total_size(shader, dispatch.threads); uint32_t *curbe_data_map = stream_state(batch, ice->state.dynamic_uploader, &ice->state.last_res.cs_thread_ids, @@ -8948,7 +8928,7 @@ iris_upload_gpgpu_walker(struct iris_context *ice, &curbe_data_offset); assert(curbe_data_map); memset(curbe_data_map, 0x5a, ALIGN(push_const_size, 64)); - iris_fill_cs_push_const_buffer(cs_prog_data, dispatch.threads, + iris_fill_cs_push_const_buffer(screen, shader, dispatch.threads, curbe_data_map); iris_emit_cmd(batch, GENX(MEDIA_CURBE_LOAD), curbe) { @@ -8976,8 +8956,7 @@ iris_upload_gpgpu_walker(struct iris_context *ice, idd.SharedLocalMemorySize = encode_slm_size(GFX_VER, ish->kernel_shared_size + grid->variable_shared_mem); idd.KernelStartPointer = - KSP(shader) + brw_cs_prog_data_prog_offset(cs_prog_data, - dispatch.simd_size); + KSP(shader) + iris_cs_data_prog_offset(cs_data, dispatch.simd_size); idd.SamplerStatePointer = shs->sampler_table.offset; idd.BindingTablePointer = binder->bt_offset[MESA_SHADER_COMPUTE] >> IRIS_BT_OFFSET_SHIFT;