From 0f7590af818f31acda7b54c5a4f352412c4b2f5b Mon Sep 17 00:00:00 2001 From: Kenneth Graunke Date: Wed, 22 Oct 2025 14:54:15 -0700 Subject: [PATCH] brw, anv, iris: Switch to reversed patch header layouts These are a ton more convenient. When the TCS and TES were linked together, the legacy layouts were a hassle, but didn't impose any significant cost. With unlinked TCS and TES, the legacy layouts involve significant runtime code for scrambling the data, whereas the reversed layouts are substantially less overhead. Reviewed-by: Alyssa Rosenzweig Part-of: --- src/gallium/drivers/iris/iris_state.c | 4 + src/intel/compiler/brw/brw_compile_tcs.cpp | 31 ++++-- src/intel/compiler/brw/brw_nir.c | 116 +++++++++++++++++++-- src/intel/vulkan/anv_private.h | 1 + src/intel/vulkan/genX_gfx_state.c | 8 ++ 5 files changed, 146 insertions(+), 14 deletions(-) diff --git a/src/gallium/drivers/iris/iris_state.c b/src/gallium/drivers/iris/iris_state.c index d8593f8947d..3f1c4377d8c 100644 --- a/src/gallium/drivers/iris/iris_state.c +++ b/src/gallium/drivers/iris/iris_state.c @@ -5258,6 +5258,10 @@ iris_store_tes_state(const struct intel_device_info *devinfo, #endif te.OutputTopology = tes_data->output_topology; te.TEDomain = tes_data->domain; +#if GFX_VER >= 12 + te.PatchHeaderLayout = tes_data->domain == INTEL_TESS_DOMAIN_TRI ? + REVERSED_TRI_INSIDE_SEPARATE : REVERSED; +#endif te.TEEnable = true; te.MaximumTessellationFactorOdd = 63.0; te.MaximumTessellationFactorNotOdd = 64.0; diff --git a/src/intel/compiler/brw/brw_compile_tcs.cpp b/src/intel/compiler/brw/brw_compile_tcs.cpp index 0c6099cd07b..37d123d6045 100644 --- a/src/intel/compiler/brw/brw_compile_tcs.cpp +++ b/src/intel/compiler/brw/brw_compile_tcs.cpp @@ -99,18 +99,37 @@ brw_emit_tcs_thread_end(brw_shader &s) const brw_builder bld = brw_builder(&s); - /* Emit a URB write to end the thread. On Broadwell, we use this to write - * zero to the "TR DS Cache Disable" bit (we haven't implemented a fancy - * algorithm to set it optimally). On other platforms, we simply write - * zero to a reserved/MBZ patch header DWord which has no consequence. + /* Otherwise, we emit a URB write which writes zero to a reserved/MBZ + * patch header DWord to end the thread. We use DWord 0 for legacy + * layouts, and DWord 6 for reversed layouts. */ + const bool reversed = s.devinfo->ver >= 12; + unsigned components = 1; + unsigned offset = 0; + brw_reg srcs[URB_LOGICAL_NUM_SRCS]; srcs[URB_LOGICAL_SRC_HANDLE] = s.tcs_payload().patch_urb_output; - srcs[URB_LOGICAL_SRC_CHANNEL_MASK] = brw_imm_ud(WRITEMASK_X); srcs[URB_LOGICAL_SRC_DATA] = brw_imm_ud(0); + + if (s.devinfo->ver >= 20) { + offset = 28; /* .z of slot 1 */ + } else if (reversed) { + srcs[URB_LOGICAL_SRC_CHANNEL_MASK] = brw_imm_ud(WRITEMASK_Z); + + brw_reg zeroes[3] = { brw_imm_ud(0), brw_imm_ud(0), brw_imm_ud(0) }; + srcs[URB_LOGICAL_SRC_DATA] = bld.vgrf(BRW_TYPE_UD, 3); + bld.VEC(srcs[URB_LOGICAL_SRC_DATA], zeroes, 3); + + components = 3; + offset = 1; + } else { + srcs[URB_LOGICAL_SRC_CHANNEL_MASK] = brw_imm_ud(WRITEMASK_X); + } + brw_urb_inst *urb = bld.URB_WRITE(srcs, ARRAY_SIZE(srcs)); urb->eot = true; - urb->components = 1; + urb->offset = offset; + urb->components = components; } static void diff --git a/src/intel/compiler/brw/brw_nir.c b/src/intel/compiler/brw/brw_nir.c index c81a2554ad8..c23024568e1 100644 --- a/src/intel/compiler/brw/brw_nir.c +++ b/src/intel/compiler/brw/brw_nir.c @@ -66,6 +66,12 @@ is_output(nir_intrinsic_instr *intrin) intrin->intrinsic == nir_intrinsic_store_per_view_output; } +static nir_def * +output_handle(nir_builder *b) +{ + return nir_load_urb_output_handle_intel(b); +} + static bool remap_tess_levels_legacy_static(nir_builder *b, nir_intrinsic_instr *intr, @@ -307,16 +313,108 @@ remap_tess_levels_legacy_dynamic(nir_shader *nir, return true; } +struct remap_tesslevel_cb_data { + const struct intel_device_info *devinfo; + enum tess_primitive_mode prim_mode; +}; + +static bool +remap_tess_levels_reversed(nir_builder *b, + nir_intrinsic_instr *io, + void *data) +{ + const struct remap_tesslevel_cb_data *cb_data = data; + const struct intel_device_info *devinfo = cb_data->devinfo; + + /* The Gfx12+ reversed patch header layouts are: + * + * [ 7 6 5 4 | 3 2 1 0] + * [__ __ iy ix | w z y x] quad reversed + * [__ __ __ __ | ix z y x] tri reversed + * [__ __ __ ix | __ z y x] tri reversed inside separate + * [__ __ __ __ | __ __ x y] isoline reversed + * + * By using the separate layout for triangles, no remapping is required + * except that isolines is backwards for some reason. We flip it here. + */ + + if (!nir_intrinsic_has_io_semantics(io) || + nir_intrinsic_io_semantics(io).location != + VARYING_SLOT_TESS_LEVEL_OUTER) + return false; + + b->cursor = nir_after_instr(&io->instr); + + nir_def *is_isoline; + if (cb_data->prim_mode == TESS_PRIMITIVE_UNSPECIFIED) { + nir_def *tess_config = nir_load_tess_config_intel(b); + is_isoline = nir_test_mask(b, tess_config, INTEL_TESS_CONFIG_ISOLINES); + } else { + is_isoline = nir_imm_true(b); + } + + const unsigned yx[2] = { 1, 0 }; + + if (io->intrinsic == nir_intrinsic_store_output) { + /* Flip isolines source: xy__ -> yx__ */ + const unsigned mask = nir_intrinsic_write_mask(io); + const unsigned revmask = (mask & ~WRITEMASK_XY) | + (mask & WRITEMASK_X) << 1 | (mask & WRITEMASK_Y) >> 1; + + nir_def *new_val = + nir_bcsel(b, is_isoline, + nir_pad_vector(b, nir_swizzle(b, io->src[0].ssa, yx, 2), + nir_src_num_components(io->src[0])), + io->src[0].ssa); + + if (devinfo->ver >= 20) { + nir_store_urb_lsc_intel(b, new_val, output_handle(b), + .base = mask == WRITEMASK_X ? 4 : 0); + } else { + nir_store_urb_vec4_intel(b, new_val, output_handle(b), + nir_imm_int(b, 0), + nir_bcsel(b, is_isoline, + nir_imm_int(b, revmask), + nir_imm_int(b, mask))); + } + nir_instr_remove(&io->instr); + } else { + /* Just leave these as load intrinsics and let the generic remapper + * take care of that part. + */ + nir_def *new_val = + nir_bcsel(b, is_isoline, nir_swizzle(b, &io->def, yx, 2), &io->def); + nir_def_rewrite_uses_after(&io->def, new_val); + } + + return true; +} + static bool remap_tess_levels(nir_shader *nir, const struct intel_device_info *devinfo, enum tess_primitive_mode prim) { - return prim == TESS_PRIMITIVE_UNSPECIFIED ? - remap_tess_levels_legacy_dynamic(nir, devinfo) : - nir_shader_intrinsics_pass(nir, remap_tess_levels_legacy_static, - nir_metadata_control_flow, - (void *)(uintptr_t) prim); + /* Pre-Gfx12 use legacy patch header layouts */ + if (devinfo->ver < 12) { + return prim == TESS_PRIMITIVE_UNSPECIFIED ? + remap_tess_levels_legacy_dynamic(nir, devinfo) : + nir_shader_intrinsics_pass(nir, remap_tess_levels_legacy_static, + nir_metadata_control_flow, + (void *)(uintptr_t) prim); + } + + /* With the reversed layouts, remapping is only required for + * isolines (or unspecified, which might be isolines). + */ + if (prim != TESS_PRIMITIVE_ISOLINES && prim != TESS_PRIMITIVE_UNSPECIFIED) + return false; + + struct remap_tesslevel_cb_data cb = { + .devinfo = devinfo, .prim_mode = prim + }; + return nir_shader_intrinsics_pass(nir, remap_tess_levels_reversed, + nir_metadata_control_flow, &cb); } struct remap_patch_cb_data { @@ -328,16 +426,18 @@ static bool remap_patch_urb_offsets_instr(nir_builder *b, nir_intrinsic_instr *intrin, void *data) { struct remap_patch_cb_data *cb_data = data; + const struct intel_device_info *devinfo = cb_data->devinfo; const struct intel_vue_map *vue_map = cb_data->vue_map; if (!(b->shader->info.stage == MESA_SHADER_TESS_CTRL && is_output(intrin)) && !(b->shader->info.stage == MESA_SHADER_TESS_EVAL && is_input(intrin))) return false; - /* Handled in a different pass */ + /* Legacy tess levels are handled in a different pass */ nir_io_semantics io_sem = nir_intrinsic_io_semantics(intrin); - if (io_sem.location == VARYING_SLOT_TESS_LEVEL_INNER || - io_sem.location == VARYING_SLOT_TESS_LEVEL_OUTER) + if (devinfo->ver < 12 && + (io_sem.location == VARYING_SLOT_TESS_LEVEL_INNER || + io_sem.location == VARYING_SLOT_TESS_LEVEL_OUTER)) return false; gl_varying_slot varying = nir_intrinsic_base(intrin); diff --git a/src/intel/vulkan/anv_private.h b/src/intel/vulkan/anv_private.h index 8b0989cc6b5..533049220ca 100644 --- a/src/intel/vulkan/anv_private.h +++ b/src/intel/vulkan/anv_private.h @@ -2173,6 +2173,7 @@ struct anv_gfx_dynamic_state { /* 3DSTATE_TE */ struct { uint32_t TEDomain; + uint32_t PatchHeaderLayout; uint32_t Partitioning; uint32_t OutputTopology; uint32_t TessellationDistributionMode; diff --git a/src/intel/vulkan/genX_gfx_state.c b/src/intel/vulkan/genX_gfx_state.c index 544774aac5e..636496a9474 100644 --- a/src/intel/vulkan/genX_gfx_state.c +++ b/src/intel/vulkan/genX_gfx_state.c @@ -1354,6 +1354,11 @@ update_te(struct anv_gfx_dynamic_state *hw_state, tes_prog_data->tess_info); SET(TE, te.TEDomain, brw_tess_info_domain(tess_info)); +#if GFX_VER >= 12 + SET(TE, te.PatchHeaderLayout, + tess_info.primitive_mode == TESS_PRIMITIVE_TRIANGLES ? + REVERSED_TRI_INSIDE_SEPARATE : REVERSED); +#endif SET(TE, te.Partitioning, brw_tess_info_partitioning(tess_info)); if (dyn->ts.domain_origin == VK_TESSELLATION_DOMAIN_ORIGIN_LOWER_LEFT) { SET(TE, te.OutputTopology, brw_tess_info_output_topology(tess_info)); @@ -3058,6 +3063,9 @@ cmd_buffer_repack_gfx_state(struct anv_gfx_dynamic_state *hw_state, anv_gfx_pack_merge(te, GENX(3DSTATE_TE), MESA_SHADER_TESS_EVAL, ds.te, te) { SET(te, te, TEDomain); +#if GFX_VER >= 12 + SET(te, te, PatchHeaderLayout); +#endif SET(te, te, Partitioning); SET(te, te, OutputTopology); #if GFX_VERx10 >= 125