diff --git a/src/freedreno/vulkan/tu_cmd_buffer.cc b/src/freedreno/vulkan/tu_cmd_buffer.cc index d7138361907..51cda50842a 100644 --- a/src/freedreno/vulkan/tu_cmd_buffer.cc +++ b/src/freedreno/vulkan/tu_cmd_buffer.cc @@ -660,7 +660,7 @@ tu6_update_msaa_disable(struct tu_cmd_buffer *cmd) tu6_primtype_line(cmd->state.primtype) || (tu6_primtype_patches(cmd->state.primtype) && cmd->state.pipeline && - cmd->state.pipeline->tess.patch_type == IR3_TESS_ISOLINES); + cmd->state.pipeline->base.tess.patch_type == IR3_TESS_ISOLINES); bool msaa_disable = is_line && cmd->state.line_mode == BRESENHAM; if (cmd->state.msaa_disable != msaa_disable) { @@ -2841,14 +2841,14 @@ tu_CmdBindPipeline(VkCommandBuffer commandBuffer, TU_FROM_HANDLE(tu_pipeline, pipeline, _pipeline); if (pipelineBindPoint == VK_PIPELINE_BIND_POINT_COMPUTE) { - cmd->state.compute_pipeline = pipeline; + cmd->state.compute_pipeline = tu_pipeline_to_compute(pipeline); tu_cs_emit_state_ib(&cmd->cs, pipeline->program.state); return; } assert(pipelineBindPoint == VK_PIPELINE_BIND_POINT_GRAPHICS); - cmd->state.pipeline = pipeline; + cmd->state.pipeline = tu_pipeline_to_graphics(pipeline); cmd->state.dirty |= TU_CMD_DIRTY_DESC_SETS | TU_CMD_DIRTY_SHADER_CONSTS | TU_CMD_DIRTY_LRZ | TU_CMD_DIRTY_VS_PARAMS | TU_CMD_DIRTY_FS_PARAMS; @@ -5008,14 +5008,14 @@ tu6_update_simplified_stencil_state(struct tu_cmd_buffer *cmd) } bool stencil_front_writemask = - (cmd->state.pipeline->dynamic_state_mask & BIT(VK_DYNAMIC_STATE_STENCIL_WRITE_MASK)) ? + (cmd->state.pipeline->base.dynamic_state_mask & BIT(VK_DYNAMIC_STATE_STENCIL_WRITE_MASK)) ? (cmd->state.dynamic_stencil_wrmask & 0xff) : - (cmd->state.pipeline->ds.stencil_wrmask & 0xff); + (cmd->state.pipeline->base.ds.stencil_wrmask & 0xff); bool stencil_back_writemask = - (cmd->state.pipeline->dynamic_state_mask & BIT(VK_DYNAMIC_STATE_STENCIL_WRITE_MASK)) ? + (cmd->state.pipeline->base.dynamic_state_mask & BIT(VK_DYNAMIC_STATE_STENCIL_WRITE_MASK)) ? ((cmd->state.dynamic_stencil_wrmask & 0xff00) >> 8) : - (cmd->state.pipeline->ds.stencil_wrmask & 0xff00) >> 8; + (cmd->state.pipeline->base.ds.stencil_wrmask & 0xff00) >> 8; VkStencilOp front_fail_op = (VkStencilOp) ((cmd->state.rb_stencil_cntl & A6XX_RB_STENCIL_CONTROL_FAIL__MASK) >> A6XX_RB_STENCIL_CONTROL_FAIL__SHIFT); @@ -5074,26 +5074,26 @@ tu6_build_depth_plane_z_mode(struct tu_cmd_buffer *cmd, struct tu_cs *cs) bool depth_write = tu6_writes_depth(cmd, depth_test_enable); bool stencil_write = tu6_writes_stencil(cmd); - if ((cmd->state.pipeline->lrz.fs.has_kill || - cmd->state.pipeline->output.subpass_feedback_loop_ds) && + if ((cmd->state.pipeline->base.lrz.fs.has_kill || + cmd->state.pipeline->base.output.subpass_feedback_loop_ds) && (depth_write || stencil_write)) { zmode = (cmd->state.lrz.valid && cmd->state.lrz.enabled) ? A6XX_EARLY_LRZ_LATE_Z : A6XX_LATE_Z; } - bool force_late_z = cmd->state.pipeline->lrz.force_late_z || + bool force_late_z = cmd->state.pipeline->base.lrz.force_late_z || /* If enabled dynamically, alpha-to-coverage can behave like a discard. */ - ((cmd->state.pipeline->dynamic_state_mask & + ((cmd->state.pipeline->base.dynamic_state_mask & BIT(TU_DYNAMIC_STATE_ALPHA_TO_COVERAGE)) && cmd->state.alpha_to_coverage); - if ((force_late_z && !cmd->state.pipeline->lrz.fs.force_early_z) || + if ((force_late_z && !cmd->state.pipeline->base.lrz.fs.force_early_z) || !depth_test_enable) zmode = A6XX_LATE_Z; /* User defined early tests take precedence above all else */ - if (cmd->state.pipeline->lrz.fs.early_fragment_tests) + if (cmd->state.pipeline->base.lrz.fs.early_fragment_tests) zmode = A6XX_EARLY_Z; tu_cs_emit_pkt4(cs, REG_A6XX_GRAS_SU_DEPTH_PLANE_CNTL, 1); @@ -5106,7 +5106,7 @@ tu6_build_depth_plane_z_mode(struct tu_cmd_buffer *cmd, struct tu_cs *cs) static void tu6_emit_blend(struct tu_cs *cs, struct tu_cmd_buffer *cmd) { - struct tu_pipeline *pipeline = cmd->state.pipeline; + struct tu_pipeline *pipeline = &cmd->state.pipeline->base; uint32_t color_write_enable = cmd->state.pipeline_color_write_enable; if (pipeline->dynamic_state_mask & @@ -5294,7 +5294,7 @@ static uint32_t fs_params_offset(struct tu_cmd_buffer *cmd) { const struct tu_program_descriptor_linkage *link = - &cmd->state.pipeline->program.link[MESA_SHADER_FRAGMENT]; + &cmd->state.pipeline->base.program.link[MESA_SHADER_FRAGMENT]; const struct ir3_const_state *const_state = &link->const_state; if (const_state->num_driver_params <= IR3_DP_FS_DYNAMIC) @@ -5310,7 +5310,7 @@ static uint32_t fs_params_size(struct tu_cmd_buffer *cmd) { const struct tu_program_descriptor_linkage *link = - &cmd->state.pipeline->program.link[MESA_SHADER_FRAGMENT]; + &cmd->state.pipeline->base.program.link[MESA_SHADER_FRAGMENT]; const struct ir3_const_state *const_state = &link->const_state; return DIV_ROUND_UP(const_state->num_driver_params - IR3_DP_FS_DYNAMIC, 4); @@ -5350,7 +5350,7 @@ tu6_emit_fs_params(struct tu_cmd_buffer *cmd) return; } - struct tu_pipeline *pipeline = cmd->state.pipeline; + struct tu_pipeline *pipeline = &cmd->state.pipeline->base; unsigned num_units = fs_params_size(cmd); @@ -5412,7 +5412,7 @@ tu6_draw_common(struct tu_cmd_buffer *cmd, /* note: draw_count is 0 for indirect */ uint32_t draw_count) { - const struct tu_pipeline *pipeline = cmd->state.pipeline; + const struct tu_pipeline *pipeline = &cmd->state.pipeline->base; struct tu_render_pass_state *rp = &cmd->state.rp; /* Fill draw stats for autotuner */ @@ -5590,18 +5590,18 @@ tu6_draw_common(struct tu_cmd_buffer *cmd, if (dirty & TU_CMD_DIRTY_BLEND) { struct tu_cs cs = tu_cmd_dynamic_state(cmd, TU_DYNAMIC_STATE_BLEND, - 8 + 3 * cmd->state.pipeline->blend.num_rts); + 8 + 3 * pipeline->blend.num_rts); tu6_emit_blend(&cs, cmd); } if (dirty & TU_CMD_DIRTY_PATCH_CONTROL_POINTS) { - bool tess = cmd->state.pipeline->active_stages & + bool tess = pipeline->active_stages & VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT; uint32_t state_size = TU6_EMIT_PATCH_CONTROL_POINTS_DWORDS( pipeline->program.hs_param_dwords); struct tu_cs cs = tu_cmd_dynamic_state( cmd, TU_DYNAMIC_STATE_PATCH_CONTROL_POINTS, tess ? state_size : 0); - tu6_emit_patch_control_points(&cs, cmd->state.pipeline, + tu6_emit_patch_control_points(&cs, &cmd->state.pipeline->base, cmd->state.patch_control_points); } @@ -5723,7 +5723,7 @@ tu6_draw_common(struct tu_cmd_buffer *cmd, static uint32_t tu_draw_initiator(struct tu_cmd_buffer *cmd, enum pc_di_src_sel src_sel) { - const struct tu_pipeline *pipeline = cmd->state.pipeline; + const struct tu_pipeline *pipeline = &cmd->state.pipeline->base; enum pc_di_primtype primtype = cmd->state.primtype; if (primtype == DI_PT_PATCHES0) @@ -5763,7 +5763,7 @@ static uint32_t vs_params_offset(struct tu_cmd_buffer *cmd) { const struct tu_program_descriptor_linkage *link = - &cmd->state.pipeline->program.link[MESA_SHADER_VERTEX]; + &cmd->state.pipeline->base.program.link[MESA_SHADER_VERTEX]; const struct ir3_const_state *const_state = &link->const_state; if (const_state->offsets.driver_param >= link->constlen) @@ -5879,7 +5879,7 @@ tu_CmdDrawMultiEXT(VkCommandBuffer commandBuffer, return; bool has_tess = - cmd->state.pipeline->active_stages & VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT; + cmd->state.pipeline->base.active_stages & VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT; uint32_t max_vertex_count = 0; if (has_tess) { @@ -5949,7 +5949,7 @@ tu_CmdDrawMultiIndexedEXT(VkCommandBuffer commandBuffer, return; bool has_tess = - cmd->state.pipeline->active_stages & VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT; + cmd->state.pipeline->base.active_stages & VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT; uint32_t max_index_count = 0; if (has_tess) { @@ -6178,15 +6178,15 @@ struct tu_dispatch_info static void tu_emit_compute_driver_params(struct tu_cmd_buffer *cmd, - struct tu_cs *cs, struct tu_pipeline *pipeline, + struct tu_cs *cs, struct tu_compute_pipeline *pipeline, const struct tu_dispatch_info *info) { gl_shader_stage type = MESA_SHADER_COMPUTE; const struct tu_program_descriptor_linkage *link = - &pipeline->program.link[type]; + &pipeline->base.program.link[type]; const struct ir3_const_state *const_state = &link->const_state; uint32_t offset = const_state->offsets.driver_param; - unsigned subgroup_size = pipeline->compute.subgroup_size; + unsigned subgroup_size = pipeline->subgroup_size; unsigned subgroup_shift = util_logbase2(subgroup_size); if (link->constlen <= offset) @@ -6294,10 +6294,10 @@ tu_dispatch(struct tu_cmd_buffer *cmd, return; struct tu_cs *cs = &cmd->cs; - struct tu_pipeline *pipeline = cmd->state.compute_pipeline; + struct tu_compute_pipeline *pipeline = cmd->state.compute_pipeline; bool emit_instrlen_workaround = - pipeline->program.cs_instrlen > + pipeline->instrlen > cmd->device->physical_device->info->a6xx.instr_cache_size; /* There appears to be a HW bug where in some rare circumstances it appears @@ -6316,7 +6316,7 @@ tu_dispatch(struct tu_cmd_buffer *cmd, * See https://gitlab.freedesktop.org/mesa/mesa/-/issues/5892 */ if (emit_instrlen_workaround) { - tu_cs_emit_regs(cs, A6XX_SP_FS_INSTRLEN(pipeline->program.cs_instrlen)); + tu_cs_emit_regs(cs, A6XX_SP_FS_INSTRLEN(pipeline->instrlen)); tu6_emit_event_write(cmd, cs, LABEL); } @@ -6326,13 +6326,13 @@ tu_dispatch(struct tu_cmd_buffer *cmd, tu_emit_cache_flush(cmd); /* note: no reason to have this in a separate IB */ - tu_cs_emit_state_ib(cs, tu6_emit_consts(cmd, pipeline, true)); + tu_cs_emit_state_ib(cs, tu6_emit_consts(cmd, &pipeline->base, true)); tu_emit_compute_driver_params(cmd, cs, pipeline, info); if (cmd->state.dirty & TU_CMD_DIRTY_COMPUTE_DESC_SETS) { tu6_emit_descriptor_sets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE); - tu_cs_emit_state_ib(cs, pipeline->load_state); + tu_cs_emit_state_ib(cs, pipeline->base.load_state); } cmd->state.dirty &= ~TU_CMD_DIRTY_COMPUTE_DESC_SETS; @@ -6340,7 +6340,7 @@ tu_dispatch(struct tu_cmd_buffer *cmd, tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1); tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_COMPUTE)); - const uint32_t *local_size = pipeline->compute.local_size; + const uint32_t *local_size = pipeline->local_size; const uint32_t *num_groups = info->blocks; tu_cs_emit_regs(cs, A6XX_HLSQ_CS_NDRANGE_0(.kerneldim = 3, diff --git a/src/freedreno/vulkan/tu_cmd_buffer.h b/src/freedreno/vulkan/tu_cmd_buffer.h index 70aafc68afc..1b181dffbbd 100644 --- a/src/freedreno/vulkan/tu_cmd_buffer.h +++ b/src/freedreno/vulkan/tu_cmd_buffer.h @@ -418,8 +418,8 @@ struct tu_cmd_state { uint32_t dirty; - struct tu_pipeline *pipeline; - struct tu_pipeline *compute_pipeline; + struct tu_graphics_pipeline *pipeline; + struct tu_compute_pipeline *compute_pipeline; struct tu_render_pass_state rp; diff --git a/src/freedreno/vulkan/tu_lrz.cc b/src/freedreno/vulkan/tu_lrz.cc index f009ebae543..c672bd43320 100644 --- a/src/freedreno/vulkan/tu_lrz.cc +++ b/src/freedreno/vulkan/tu_lrz.cc @@ -559,7 +559,7 @@ static struct A6XX_GRAS_LRZ_CNTL tu6_calculate_lrz_state(struct tu_cmd_buffer *cmd, const uint32_t a) { - struct tu_pipeline *pipeline = cmd->state.pipeline; + struct tu_pipeline *pipeline = &cmd->state.pipeline->base; bool z_test_enable = (bool) (cmd->state.rb_depth_cntl & A6XX_RB_DEPTH_CNTL_Z_TEST_ENABLE); bool z_write_enable = (bool) (cmd->state.rb_depth_cntl & A6XX_RB_DEPTH_CNTL_Z_WRITE_ENABLE); bool z_bounds_enable = (bool) (cmd->state.rb_depth_cntl & A6XX_RB_DEPTH_CNTL_Z_BOUNDS_ENABLE); @@ -600,7 +600,7 @@ tu6_calculate_lrz_state(struct tu_cmd_buffer *cmd, /* See comment in tu_pipeline about disabling LRZ write for blending. */ bool reads_dest = !!(pipeline->lrz.lrz_status & TU_LRZ_READS_DEST); - if (gras_lrz_cntl.lrz_write && cmd->state.pipeline->dynamic_state_mask & + if (gras_lrz_cntl.lrz_write && pipeline->dynamic_state_mask & (BIT(TU_DYNAMIC_STATE_LOGIC_OP) | BIT(TU_DYNAMIC_STATE_BLEND_ENABLE))) { if (cmd->state.logic_op_enabled && cmd->state.rop_reads_dst) { @@ -616,7 +616,7 @@ tu6_calculate_lrz_state(struct tu_cmd_buffer *cmd, } } - if ((cmd->state.pipeline->dynamic_state_mask & BIT(TU_DYNAMIC_STATE_BLEND))) { + if ((pipeline->dynamic_state_mask & BIT(TU_DYNAMIC_STATE_BLEND))) { for (unsigned i = 0; i < cmd->state.subpass->color_count; i++) { unsigned a = cmd->state.subpass->color_attachments[i].attachment; if (a == VK_ATTACHMENT_UNUSED) @@ -641,17 +641,17 @@ tu6_calculate_lrz_state(struct tu_cmd_buffer *cmd, } } - if ((cmd->state.pipeline->dynamic_state_mask & + if ((pipeline->dynamic_state_mask & BIT(TU_DYNAMIC_STATE_COLOR_WRITE_ENABLE)) && (cmd->state.color_write_enable & MASK(cmd->state.subpass->color_count)) != - MASK(cmd->state.pipeline->blend.num_rts)) { + MASK(pipeline->blend.num_rts)) { if (gras_lrz_cntl.lrz_write) { perf_debug( cmd->device, "disabling lrz write due to dynamic color write enables (%x/%x)", cmd->state.color_write_enable, - MASK(cmd->state.pipeline->blend.num_rts)); + MASK(pipeline->blend.num_rts)); } gras_lrz_cntl.lrz_write = false; reads_dest = true; diff --git a/src/freedreno/vulkan/tu_pipeline.cc b/src/freedreno/vulkan/tu_pipeline.cc index e8cdcb0acfe..d5d4a0a41f2 100644 --- a/src/freedreno/vulkan/tu_pipeline.cc +++ b/src/freedreno/vulkan/tu_pipeline.cc @@ -284,7 +284,7 @@ struct tu_pipeline_builder #define MAX_LIBRARIES 4 unsigned num_libraries; - struct tu_pipeline *libraries[MAX_LIBRARIES]; + struct tu_graphics_lib_pipeline *libraries[MAX_LIBRARIES]; /* This is just the state that we are compiling now, whereas the final * pipeline will include the state from the libraries. @@ -2435,6 +2435,14 @@ contains_all_shader_state(VkGraphicsPipelineLibraryFlagsEXT state) VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT); } +static bool +pipeline_contains_all_shader_state(struct tu_pipeline *pipeline) +{ + return pipeline->type == TU_PIPELINE_GRAPHICS || + pipeline->type == TU_PIPELINE_COMPUTE || + contains_all_shader_state(tu_pipeline_to_graphics_lib(pipeline)->state); +} + /* Return true if this pipeline contains all of the GPL stages listed but none * of the libraries it uses do, so this is "the first time" that all of them * are defined together. This is useful for state that needs to be combined @@ -2446,7 +2454,8 @@ set_combined_state(struct tu_pipeline_builder *builder, struct tu_pipeline *pipeline, VkGraphicsPipelineLibraryFlagsEXT state) { - if ((pipeline->state & state) != state) + if (pipeline->type == TU_PIPELINE_GRAPHICS_LIB && + (tu_pipeline_to_graphics_lib(pipeline)->state & state) != state) return false; for (unsigned i = 0; i < builder->num_libraries; i++) { @@ -3142,7 +3151,7 @@ tu_pipeline_builder_compile_shaders(struct tu_pipeline_builder *builder, if (builder->create_info->flags & VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT) { for (unsigned i = 0; i < builder->num_libraries; i++) { - struct tu_pipeline *library = builder->libraries[i]; + struct tu_graphics_lib_pipeline *library = builder->libraries[i]; for (unsigned j = 0; j < ARRAY_SIZE(library->shaders); j++) { if (library->shaders[j].nir) { @@ -3448,7 +3457,7 @@ done:; * retain it ourselves in case another pipeline includes us with LTO. */ for (unsigned i = 0; i < builder->num_libraries; i++) { - struct tu_pipeline *library = builder->libraries[i]; + struct tu_graphics_lib_pipeline *library = builder->libraries[i]; for (gl_shader_stage stage = MESA_SHADER_VERTEX; stage < ARRAY_SIZE(library->shaders); stage = (gl_shader_stage) (stage + 1)) { @@ -3462,7 +3471,7 @@ done:; if (!(builder->create_info->flags & VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT)) { for (unsigned i = 0; i < builder->num_libraries; i++) { - struct tu_pipeline *library = builder->libraries[i]; + struct tu_graphics_lib_pipeline *library = builder->libraries[i]; for (gl_shader_stage stage = MESA_SHADER_VERTEX; stage < ARRAY_SIZE(library->shaders); stage = (gl_shader_stage) (stage + 1)) { @@ -3495,8 +3504,8 @@ done:; pipeline->active_desc_sets = compiled_shaders->active_desc_sets; for (unsigned i = 0; i < builder->num_libraries; i++) { - struct tu_pipeline *library = builder->libraries[i]; - pipeline->active_desc_sets |= library->active_desc_sets; + struct tu_graphics_lib_pipeline *library = builder->libraries[i]; + pipeline->active_desc_sets |= library->base.active_desc_sets; } if (compiled_shaders && compiled_shaders->variants[MESA_SHADER_TESS_CTRL]) { @@ -3504,7 +3513,7 @@ done:; compiled_shaders->variants[MESA_SHADER_TESS_CTRL]->key.tessellation; } - if (contains_all_shader_state(pipeline->state)) { + if (pipeline_contains_all_shader_state(pipeline)) { struct ir3_shader_variant *vs = builder->variants[MESA_SHADER_VERTEX]; @@ -3527,17 +3536,19 @@ done:; vk_pipeline_cache_object_unref(&builder->device->vk, &nir_shaders->base); } else { - pipeline->compiled_shaders = compiled_shaders; - pipeline->nir_shaders = nir_shaders; - pipeline->ir3_key = ir3_key; + struct tu_graphics_lib_pipeline *library = + tu_pipeline_to_graphics_lib(pipeline); + library->compiled_shaders = compiled_shaders; + library->nir_shaders = nir_shaders; + library->ir3_key = ir3_key; for (gl_shader_stage stage = MESA_SHADER_VERTEX; - stage < ARRAY_SIZE(pipeline->shaders); + stage < ARRAY_SIZE(library->shaders); stage = (gl_shader_stage) (stage + 1)) { - pipeline->shaders[stage].nir = post_link_nir[stage]; - pipeline->shaders[stage].key = keys[stage]; - pipeline->shaders[stage].const_state = builder->const_state[stage]; - pipeline->shaders[stage].variant = builder->variants[stage]; - pipeline->shaders[stage].safe_const_variant = + library->shaders[stage].nir = post_link_nir[stage]; + library->shaders[stage].key = keys[stage]; + library->shaders[stage].const_state = builder->const_state[stage]; + library->shaders[stage].variant = builder->variants[stage]; + library->shaders[stage].safe_const_variant = safe_const_variants[stage]; } } @@ -3845,36 +3856,38 @@ tu_pipeline_builder_parse_libraries(struct tu_pipeline_builder *builder, builder->num_libraries = library_info->libraryCount; for (unsigned i = 0; i < library_info->libraryCount; i++) { TU_FROM_HANDLE(tu_pipeline, library, library_info->pLibraries[i]); - builder->libraries[i] = library; + builder->libraries[i] = tu_pipeline_to_graphics_lib(library); } } /* Merge in the state from libraries. The program state is a bit special * and is handled separately. */ - pipeline->state = builder->state; + if (pipeline->type == TU_PIPELINE_GRAPHICS_LIB) + tu_pipeline_to_graphics_lib(pipeline)->state = builder->state; for (unsigned i = 0; i < builder->num_libraries; i++) { - struct tu_pipeline *library = builder->libraries[i]; - pipeline->state |= library->state; + struct tu_graphics_lib_pipeline *library = builder->libraries[i]; + if (pipeline->type == TU_PIPELINE_GRAPHICS_LIB) + tu_pipeline_to_graphics_lib(pipeline)->state |= library->state; uint64_t library_dynamic_state = 0; if (library->state & VK_GRAPHICS_PIPELINE_LIBRARY_VERTEX_INPUT_INTERFACE_BIT_EXT) { - pipeline->vi = library->vi; - pipeline->ia = library->ia; + pipeline->vi = library->base.vi; + pipeline->ia = library->base.ia; library_dynamic_state |= BIT(TU_DYNAMIC_STATE_VERTEX_INPUT) | BIT(TU_DYNAMIC_STATE_VB_STRIDE) | BIT(TU_DYNAMIC_STATE_PRIMITIVE_TOPOLOGY) | BIT(TU_DYNAMIC_STATE_PRIMITIVE_RESTART_ENABLE); - pipeline->shared_consts = library->shared_consts; + pipeline->shared_consts = library->base.shared_consts; } if (library->state & VK_GRAPHICS_PIPELINE_LIBRARY_PRE_RASTERIZATION_SHADERS_BIT_EXT) { - pipeline->tess = library->tess; - pipeline->rast = library->rast; - pipeline->viewport = library->viewport; + pipeline->tess = library->base.tess; + pipeline->rast = library->base.rast; + pipeline->viewport = library->base.viewport; library_dynamic_state |= BIT(VK_DYNAMIC_STATE_VIEWPORT) | BIT(VK_DYNAMIC_STATE_SCISSOR) | @@ -3893,27 +3906,27 @@ tu_pipeline_builder_parse_libraries(struct tu_pipeline_builder *builder, if (library->state & VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT) { - pipeline->ds = library->ds; - pipeline->fs = library->fs; - pipeline->lrz.fs = library->lrz.fs; - pipeline->lrz.lrz_status |= library->lrz.lrz_status; - pipeline->lrz.force_late_z |= library->lrz.force_late_z; + pipeline->ds = library->base.ds; + pipeline->fs = library->base.fs; + pipeline->lrz.fs = library->base.lrz.fs; + pipeline->lrz.lrz_status |= library->base.lrz.lrz_status; + pipeline->lrz.force_late_z |= library->base.lrz.force_late_z; library_dynamic_state |= BIT(VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK) | BIT(VK_DYNAMIC_STATE_STENCIL_WRITE_MASK) | BIT(VK_DYNAMIC_STATE_STENCIL_REFERENCE) | BIT(TU_DYNAMIC_STATE_DS) | BIT(VK_DYNAMIC_STATE_DEPTH_BOUNDS); - pipeline->shared_consts = library->shared_consts; + pipeline->shared_consts = library->base.shared_consts; } if (library->state & VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_OUTPUT_INTERFACE_BIT_EXT) { - pipeline->blend = library->blend; - pipeline->output = library->output; - pipeline->lrz.lrz_status |= library->lrz.lrz_status; - pipeline->lrz.force_late_z |= library->lrz.force_late_z; - pipeline->prim_order = library->prim_order; + pipeline->blend = library->base.blend; + pipeline->output = library->base.output; + pipeline->lrz.lrz_status |= library->base.lrz.lrz_status; + pipeline->lrz.force_late_z |= library->base.lrz.force_late_z; + pipeline->prim_order = library->base.prim_order; library_dynamic_state |= BIT(VK_DYNAMIC_STATE_BLEND_CONSTANTS) | BIT(TU_DYNAMIC_STATE_SAMPLE_LOCATIONS) | @@ -3932,7 +3945,7 @@ tu_pipeline_builder_parse_libraries(struct tu_pipeline_builder *builder, VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT) && (library->state & VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_OUTPUT_INTERFACE_BIT_EXT)) { - pipeline->prim_order = library->prim_order; + pipeline->prim_order = library->base.prim_order; } if ((library->state & @@ -3941,23 +3954,23 @@ tu_pipeline_builder_parse_libraries(struct tu_pipeline_builder *builder, VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_OUTPUT_INTERFACE_BIT_EXT) && (library->state & VK_GRAPHICS_PIPELINE_LIBRARY_PRE_RASTERIZATION_SHADERS_BIT_EXT)) { - pipeline->rast_ds = library->rast_ds; + pipeline->rast_ds = library->base.rast_ds; } pipeline->dynamic_state_mask = (pipeline->dynamic_state_mask & ~library_dynamic_state) | - (library->dynamic_state_mask & library_dynamic_state); + (library->base.dynamic_state_mask & library_dynamic_state); - u_foreach_bit (i, library_dynamic_state & ~library->dynamic_state_mask) { + u_foreach_bit (i, library_dynamic_state & ~library->base.dynamic_state_mask) { if (i >= TU_DYNAMIC_STATE_COUNT) break; - pipeline->dynamic_state[i] = library->dynamic_state[i]; + pipeline->dynamic_state[i] = library->base.dynamic_state[i]; } if (contains_all_shader_state(library->state)) { - pipeline->program = library->program; - pipeline->load_state = library->load_state; + pipeline->program = library->base.program; + pipeline->load_state = library->base.load_state; } } } @@ -3977,7 +3990,7 @@ tu_pipeline_builder_parse_layout(struct tu_pipeline_builder *builder, builder->layout = *layout; } else { for (unsigned i = 0; i < builder->num_libraries; i++) { - struct tu_pipeline *library = builder->libraries[i]; + struct tu_graphics_lib_pipeline *library = builder->libraries[i]; builder->layout.num_sets = MAX2(builder->layout.num_sets, library->num_sets); for (unsigned j = 0; j < library->num_sets; j++) { @@ -3985,22 +3998,24 @@ tu_pipeline_builder_parse_layout(struct tu_pipeline_builder *builder, builder->layout.set[i].layout = library->layouts[i]; } - builder->layout.push_constant_size = pipeline->push_constant_size; - builder->layout.independent_sets |= pipeline->independent_sets; + builder->layout.push_constant_size = library->push_constant_size; + builder->layout.independent_sets |= library->independent_sets; } tu_pipeline_layout_init(&builder->layout); } - if (builder->create_info->flags & VK_PIPELINE_CREATE_LIBRARY_BIT_KHR) { - pipeline->num_sets = builder->layout.num_sets; - for (unsigned i = 0; i < pipeline->num_sets; i++) { - pipeline->layouts[i] = builder->layout.set[i].layout; - if (pipeline->layouts[i]) - vk_descriptor_set_layout_ref(&pipeline->layouts[i]->vk); + if (pipeline->type == TU_PIPELINE_GRAPHICS_LIB) { + struct tu_graphics_lib_pipeline *library = + tu_pipeline_to_graphics_lib(pipeline); + library->num_sets = builder->layout.num_sets; + for (unsigned i = 0; i < library->num_sets; i++) { + library->layouts[i] = builder->layout.set[i].layout; + if (library->layouts[i]) + vk_descriptor_set_layout_ref(&library->layouts[i]->vk); } - pipeline->push_constant_size = builder->layout.push_constant_size; - pipeline->independent_sets = builder->layout.independent_sets; + library->push_constant_size = builder->layout.push_constant_size; + library->independent_sets = builder->layout.independent_sets; } } @@ -4842,16 +4857,21 @@ tu_pipeline_finish(struct tu_pipeline *pipeline, if (pipeline->pvtmem_bo) tu_bo_finish(dev, pipeline->pvtmem_bo); - if (pipeline->compiled_shaders) - vk_pipeline_cache_object_unref(&dev->vk, - &pipeline->compiled_shaders->base); + if (pipeline->type == TU_PIPELINE_GRAPHICS_LIB) { + struct tu_graphics_lib_pipeline *library = + tu_pipeline_to_graphics_lib(pipeline); + if (library->compiled_shaders) + vk_pipeline_cache_object_unref(&dev->vk, + &library->compiled_shaders->base); - if (pipeline->nir_shaders) - vk_pipeline_cache_object_unref(&dev->vk, &pipeline->nir_shaders->base); + if (library->nir_shaders) + vk_pipeline_cache_object_unref(&dev->vk, + &library->nir_shaders->base); - for (unsigned i = 0; i < pipeline->num_sets; i++) { - if (pipeline->layouts[i]) - vk_descriptor_set_layout_unref(&dev->vk, &pipeline->layouts[i]->vk); + for (unsigned i = 0; i < library->num_sets; i++) { + if (library->layouts[i]) + vk_descriptor_set_layout_unref(&dev->vk, &library->layouts[i]->vk); + } } ralloc_free(pipeline->executables_mem_ctx); @@ -4882,11 +4902,23 @@ tu_pipeline_builder_build(struct tu_pipeline_builder *builder, { VkResult result; - *pipeline = (struct tu_pipeline *) vk_object_zalloc( - &builder->device->vk, builder->alloc, sizeof(**pipeline), - VK_OBJECT_TYPE_PIPELINE); - if (!*pipeline) - return VK_ERROR_OUT_OF_HOST_MEMORY; + if (builder->create_info->flags & VK_PIPELINE_CREATE_LIBRARY_BIT_KHR) { + *pipeline = (struct tu_pipeline *) vk_object_zalloc( + &builder->device->vk, builder->alloc, + sizeof(struct tu_graphics_lib_pipeline), + VK_OBJECT_TYPE_PIPELINE); + if (!*pipeline) + return VK_ERROR_OUT_OF_HOST_MEMORY; + (*pipeline)->type = TU_PIPELINE_GRAPHICS_LIB; + } else { + *pipeline = (struct tu_pipeline *) vk_object_zalloc( + &builder->device->vk, builder->alloc, + sizeof(struct tu_graphics_pipeline), + VK_OBJECT_TYPE_PIPELINE); + if (!*pipeline) + return VK_ERROR_OUT_OF_HOST_MEMORY; + (*pipeline)->type = TU_PIPELINE_GRAPHICS; + } (*pipeline)->executables_mem_ctx = ralloc_context(NULL); util_dynarray_init(&(*pipeline)->executables, (*pipeline)->executables_mem_ctx); @@ -4908,7 +4940,7 @@ tu_pipeline_builder_build(struct tu_pipeline_builder *builder, (*pipeline)->active_stages = stages; for (unsigned i = 0; i < builder->num_libraries; i++) - (*pipeline)->active_stages |= builder->libraries[i]->active_stages; + (*pipeline)->active_stages |= builder->libraries[i]->base.active_stages; /* Compile and upload shaders unless a library has already done that. */ if ((*pipeline)->program.state.size == 0) { @@ -5308,7 +5340,7 @@ tu_compute_pipeline_create(VkDevice device, cache = cache ? cache : dev->mem_cache; - struct tu_pipeline *pipeline; + struct tu_compute_pipeline *pipeline; *pPipeline = VK_NULL_HANDLE; @@ -5321,14 +5353,15 @@ tu_compute_pipeline_create(VkDevice device, int64_t pipeline_start = os_time_get_nano(); - pipeline = (struct tu_pipeline *) vk_object_zalloc( + pipeline = (struct tu_compute_pipeline *) vk_object_zalloc( &dev->vk, pAllocator, sizeof(*pipeline), VK_OBJECT_TYPE_PIPELINE); if (!pipeline) return VK_ERROR_OUT_OF_HOST_MEMORY; + pipeline->base.type = TU_PIPELINE_COMPUTE; - pipeline->executables_mem_ctx = ralloc_context(NULL); - util_dynarray_init(&pipeline->executables, pipeline->executables_mem_ctx); - pipeline->active_stages = VK_SHADER_STAGE_COMPUTE_BIT; + pipeline->base.executables_mem_ctx = ralloc_context(NULL); + util_dynarray_init(&pipeline->base.executables, pipeline->base.executables_mem_ctx); + pipeline->base.active_stages = VK_SHADER_STAGE_COMPUTE_BIT; struct tu_shader_key key = { }; tu_shader_key_init(&key, stage_info, dev); @@ -5357,7 +5390,7 @@ tu_compute_pipeline_create(VkDevice device, } if (tu6_shared_constants_enable(layout, dev->compiler)) { - pipeline->shared_consts = (struct tu_push_constant_range) { + pipeline->base.shared_consts = (struct tu_push_constant_range) { .lo = 0, .dwords = layout->push_constant_size / 4, }; @@ -5378,7 +5411,7 @@ tu_compute_pipeline_create(VkDevice device, MESA_SHADER_COMPUTE); nir_initial_disasm = executable_info ? - nir_shader_as_str(nir, pipeline->executables_mem_ctx) : NULL; + nir_shader_as_str(nir, pipeline->base.executables_mem_ctx) : NULL; struct tu_shader *shader = tu_shader_create(dev, nir, &key, layout, pAllocator); @@ -5420,43 +5453,43 @@ tu_compute_pipeline_create(VkDevice device, creation_feedback->pPipelineStageCreationFeedbacks[0] = pipeline_feedback; } - pipeline->active_desc_sets = compiled->active_desc_sets; + pipeline->base.active_desc_sets = compiled->active_desc_sets; v = compiled->variants[MESA_SHADER_COMPUTE]; - tu_pipeline_set_linkage(&pipeline->program.link[MESA_SHADER_COMPUTE], + tu_pipeline_set_linkage(&pipeline->base.program.link[MESA_SHADER_COMPUTE], &compiled->const_state[MESA_SHADER_COMPUTE], v); - result = tu_pipeline_allocate_cs(dev, pipeline, layout, NULL, v); + result = tu_pipeline_allocate_cs(dev, &pipeline->base, layout, NULL, v); if (result != VK_SUCCESS) goto fail; - shader_iova = tu_upload_variant(pipeline, v); + shader_iova = tu_upload_variant(&pipeline->base, v); struct tu_pvtmem_config pvtmem; - tu_setup_pvtmem(dev, pipeline, &pvtmem, v->pvtmem_size, v->pvtmem_per_wave); + tu_setup_pvtmem(dev, &pipeline->base, &pvtmem, v->pvtmem_size, v->pvtmem_per_wave); for (int i = 0; i < 3; i++) - pipeline->compute.local_size[i] = v->local_size[i]; + pipeline->local_size[i] = v->local_size[i]; - pipeline->compute.subgroup_size = v->info.subgroup_size; + pipeline->subgroup_size = v->info.subgroup_size; struct tu_cs prog_cs; additional_reserve_size = tu_xs_get_additional_cs_size_dwords(v); - tu_cs_begin_sub_stream(&pipeline->cs, 64 + additional_reserve_size, &prog_cs); + tu_cs_begin_sub_stream(&pipeline->base.cs, 64 + additional_reserve_size, &prog_cs); tu6_emit_cs_config(&prog_cs, v, &pvtmem, shader_iova); - pipeline->program.state = tu_cs_end_draw_state(&pipeline->cs, &prog_cs); + pipeline->base.program.state = tu_cs_end_draw_state(&pipeline->base.cs, &prog_cs); - tu6_emit_load_state(pipeline, layout); + tu6_emit_load_state(&pipeline->base, layout); - tu_append_executable(pipeline, v, nir_initial_disasm); + tu_append_executable(&pipeline->base, v, nir_initial_disasm); - pipeline->program.cs_instrlen = v->instrlen; + pipeline->instrlen = v->instrlen; vk_pipeline_cache_object_unref(&dev->vk, &compiled->base); ralloc_free(pipeline_mem_ctx); - *pPipeline = tu_pipeline_to_handle(pipeline); + *pPipeline = tu_pipeline_to_handle(&pipeline->base); return VK_SUCCESS; diff --git a/src/freedreno/vulkan/tu_pipeline.h b/src/freedreno/vulkan/tu_pipeline.h index fd6f481d39c..90cd25443bc 100644 --- a/src/freedreno/vulkan/tu_pipeline.h +++ b/src/freedreno/vulkan/tu_pipeline.h @@ -120,9 +120,16 @@ struct tu_pipeline_executable { char *disasm; }; +enum tu_pipeline_type { + TU_PIPELINE_GRAPHICS, + TU_PIPELINE_GRAPHICS_LIB, + TU_PIPELINE_COMPUTE, +}; + struct tu_pipeline { struct vk_object_base base; + enum tu_pipeline_type type; struct tu_cs cs; struct tu_suballoc_bo bo; @@ -139,8 +146,6 @@ struct tu_pipeline uint64_t dynamic_state_mask; struct tu_draw_state dynamic_state[TU_DYNAMIC_STATE_COUNT]; - VkGraphicsPipelineLibraryFlagsEXT state; - /* for dynamic states which use the same register: */ struct { uint32_t gras_su_cntl, gras_su_cntl_mask; @@ -232,7 +237,6 @@ struct tu_pipeline uint32_t hs_param_stride; uint32_t hs_param_dwords; uint32_t hs_vertices_out; - uint32_t cs_instrlen; bool writes_viewport; bool per_samp; @@ -254,12 +258,6 @@ struct tu_pipeline bool upper_left_domain_origin; } tess; - struct - { - uint32_t local_size[3]; - uint32_t subgroup_size; - } compute; - struct tu_lrz_pipeline lrz; struct { @@ -272,11 +270,20 @@ struct tu_pipeline bool per_view_viewport; } viewport; - /* Used only for libraries. compiled_shaders only contains variants compiled - * by this pipeline, and it owns them, so when it is freed they disappear. - * Similarly, nir_shaders owns the link-time NIR. shaders points to the - * shaders from this pipeline and all libraries included in it, for - * convenience. + void *executables_mem_ctx; + /* tu_pipeline_executable */ + struct util_dynarray executables; +}; + +struct tu_graphics_lib_pipeline { + struct tu_pipeline base; + + VkGraphicsPipelineLibraryFlagsEXT state; + + /* compiled_shaders only contains variants compiled by this pipeline, and + * it owns them, so when it is freed they disappear. Similarly, + * nir_shaders owns the link-time NIR. shaders points to the shaders from + * this pipeline and all libraries included in it, for convenience. */ struct tu_compiled_shaders *compiled_shaders; struct tu_nir_shaders *nir_shaders; @@ -289,21 +296,40 @@ struct tu_pipeline struct ir3_shader_key ir3_key; - /* Used for libraries, to stitch together an overall layout for the final - * pipeline. - */ + /* Used to stitch together an overall layout for the final pipeline. */ struct tu_descriptor_set_layout *layouts[MAX_SETS]; unsigned num_sets; unsigned push_constant_size; bool independent_sets; - - void *executables_mem_ctx; - /* tu_pipeline_executable */ - struct util_dynarray executables; }; + +struct tu_graphics_pipeline { + struct tu_pipeline base; +}; + +struct tu_compute_pipeline { + struct tu_pipeline base; + + uint32_t local_size[3]; + uint32_t subgroup_size; + uint32_t instrlen; +}; + VK_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline, base, VkPipeline, VK_OBJECT_TYPE_PIPELINE) +#define TU_DECL_PIPELINE_DOWNCAST(pipe_type, pipe_enum) \ + static inline struct tu_##pipe_type##_pipeline * \ + tu_pipeline_to_##pipe_type(struct tu_pipeline *pipeline) \ + { \ + assert(pipeline->type == pipe_enum); \ + return (struct tu_##pipe_type##_pipeline *) pipeline; \ + } + +TU_DECL_PIPELINE_DOWNCAST(graphics, TU_PIPELINE_GRAPHICS) +TU_DECL_PIPELINE_DOWNCAST(graphics_lib, TU_PIPELINE_GRAPHICS_LIB) +TU_DECL_PIPELINE_DOWNCAST(compute, TU_PIPELINE_COMPUTE) + void tu6_emit_viewport(struct tu_cs *cs, const VkViewport *viewport, uint32_t num_viewport, bool z_negative_one_to_one);