From e8914c62ac9bd7f37d3e6ed5f5c5822a0af54826 Mon Sep 17 00:00:00 2001 From: Dave Airlie Date: Mon, 9 Jun 2025 17:33:10 +1000 Subject: [PATCH] nak: add support for cbuf2 format to non uniform lowering. This adds the hopper/blackwell support. Part-of: --- src/nouveau/compiler/nak_nir.c | 2 +- .../compiler/nak_nir_lower_non_uniform_ldcx.c | 69 ++++++++++++------- src/nouveau/compiler/nak_private.h | 2 +- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/src/nouveau/compiler/nak_nir.c b/src/nouveau/compiler/nak_nir.c index 5d6f43d2669..24262eeff30 100644 --- a/src/nouveau/compiler/nak_nir.c +++ b/src/nouveau/compiler/nak_nir.c @@ -1106,7 +1106,7 @@ nak_postprocess_nir(nir_shader *nir, if (nak->sm >= 73) { OPT(nir, nak_nir_mark_lcssa_invariants); - if (OPT(nir, nak_nir_lower_non_uniform_ldcx)) { + if (OPT(nir, nak_nir_lower_non_uniform_ldcx, nak)) { OPT(nir, nir_copy_prop); OPT(nir, nir_opt_dce); } diff --git a/src/nouveau/compiler/nak_nir_lower_non_uniform_ldcx.c b/src/nouveau/compiler/nak_nir_lower_non_uniform_ldcx.c index c19b068f187..3a9f7848ace 100644 --- a/src/nouveau/compiler/nak_nir_lower_non_uniform_ldcx.c +++ b/src/nouveau/compiler/nak_nir_lower_non_uniform_ldcx.c @@ -15,7 +15,8 @@ */ static void -lower_ldcx_to_global(nir_builder *b, nir_intrinsic_instr *load) +lower_ldcx_to_global(nir_builder *b, nir_intrinsic_instr *load, + const struct nak_compiler *nak) { assert(load->intrinsic == nir_intrinsic_ldcx_nv); @@ -26,17 +27,31 @@ lower_ldcx_to_global(nir_builder *b, nir_intrinsic_instr *load) nir_def *offset = load->src[1].ssa; assert(offset->bit_size == 32); - /* Base address shifted by 4 is the bottom 45 bits */ - nir_def *addr_4_lo = nir_unpack_64_2x32_split_x(b, handle); - nir_def *addr_4_hi = nir_iand_imm(b, nir_unpack_64_2x32_split_y(b, handle), + nir_def *addr; + nir_def *size; + if (nak->sm >= 100) { + /* Base address shifted by 6 is the bottom 51 bits */ + nir_def *addr_6_lo = nir_unpack_64_2x32_split_x(b, handle); + nir_def *addr_6_hi = nir_iand_imm(b, nir_unpack_64_2x32_split_y(b, handle), + BITFIELD_MASK(19)); + nir_def *addr_6 = nir_pack_64_2x32_split(b, addr_6_lo, addr_6_hi); + addr = nir_ishl_imm(b, addr_6, 6); + + /* Size shifted by 2 is the top 13 bits */ + size = nir_unpack_64_2x32_split_y(b, handle); + size = nir_ishl_imm(b, nir_ushr_imm(b, size, 19), 4); + } else { + /* Base address shifted by 4 is the bottom 45 bits */ + nir_def *addr_4_lo = nir_unpack_64_2x32_split_x(b, handle); + nir_def *addr_4_hi = nir_iand_imm(b, nir_unpack_64_2x32_split_y(b, handle), BITFIELD_MASK(13)); - nir_def *addr_4 = nir_pack_64_2x32_split(b, addr_4_lo, addr_4_hi); - nir_def *addr = nir_ishl_imm(b, addr_4, 4); - - /* Size shifted by 2 is the top 19 bits */ - nir_def *size = nir_unpack_64_2x32_split_y(b, handle); - size = nir_ishl_imm(b, nir_ushr_imm(b, size, 13), 4); + nir_def *addr_4 = nir_pack_64_2x32_split(b, addr_4_lo, addr_4_hi); + addr = nir_ishl_imm(b, addr_4, 4); + /* Size shifted by 2 is the top 19 bits */ + size = nir_unpack_64_2x32_split_y(b, handle); + size = nir_ishl_imm(b, nir_ushr_imm(b, size, 13), 4); + } /* At this point we can assume the offset is aligned so we only need a * simple less-than check here. */ @@ -343,7 +358,8 @@ try_remat_ldcx_alu_use(nir_builder *b, nir_alu_instr *alu, uint8_t src_idx, static bool lower_ldcx_block(nir_builder *b, nir_block *block, - struct non_uniform_section *nus) + struct non_uniform_section *nus, + const struct nak_compiler *nak) { bool progress = false; @@ -370,7 +386,7 @@ lower_ldcx_block(nir_builder *b, nir_block *block, if (nus == NULL) { /* Uniform control-flow */ if (nir_intrinsic_access(load) & ACCESS_NON_UNIFORM) { - lower_ldcx_to_global(b, load); + lower_ldcx_to_global(b, load, nak); progress = true; } else if (handle->divergent) { b->cursor = nir_before_instr(&load->instr); @@ -386,7 +402,7 @@ lower_ldcx_block(nir_builder *b, nir_block *block, assert(!ugpr->divergent); nir_src_rewrite(&load->src[0], ugpr); } else { - lower_ldcx_to_global(b, load); + lower_ldcx_to_global(b, load, nak); } progress = true; } @@ -403,7 +419,8 @@ lower_ldcx_block(nir_builder *b, nir_block *block, static bool lower_non_uniform_cf_node(nir_builder *b, nir_cf_node *node, - nir_block *pred, nir_block *succ) + nir_block *pred, nir_block *succ, + const struct nak_compiler *nak) { bool progress = false; @@ -417,7 +434,7 @@ lower_non_uniform_cf_node(nir_builder *b, nir_cf_node *node, progress |= sort_and_mark_live_handles(b, &nus); nir_foreach_block_in_cf_node_safe(block, node) - progress |= lower_ldcx_block(b, block, &nus); + progress |= lower_ldcx_block(b, block, &nus, nak); non_uniform_section_finish(&nus); @@ -425,7 +442,8 @@ lower_non_uniform_cf_node(nir_builder *b, nir_cf_node *node, } static bool -lower_cf_list(nir_builder *b, struct exec_list *cf_list) +lower_cf_list(nir_builder *b, struct exec_list *cf_list, + const struct nak_compiler *nak) { bool progress = false; @@ -434,17 +452,17 @@ lower_cf_list(nir_builder *b, struct exec_list *cf_list) switch (node->type) { case nir_cf_node_block: block = nir_cf_node_as_block(node); - progress |= lower_ldcx_block(b, block, NULL); + progress |= lower_ldcx_block(b, block, NULL, nak); continue; case nir_cf_node_if: { nir_if *nif = nir_cf_node_as_if(node); if (nir_src_is_divergent(&nif->condition)) { nir_block *succ = nir_cf_node_as_block(nir_cf_node_next(node)); - progress |= lower_non_uniform_cf_node(b, node, block, succ); + progress |= lower_non_uniform_cf_node(b, node, block, succ, nak); } else { - progress |= lower_cf_list(b, &nif->then_list); - progress |= lower_cf_list(b, &nif->else_list); + progress |= lower_cf_list(b, &nif->then_list, nak); + progress |= lower_cf_list(b, &nif->else_list, nak); } break; } @@ -453,10 +471,10 @@ lower_cf_list(nir_builder *b, struct exec_list *cf_list) nir_loop *loop = nir_cf_node_as_loop(node); if (nir_loop_is_divergent(loop)) { nir_block *succ = nir_cf_node_as_block(nir_cf_node_next(node)); - progress |= lower_non_uniform_cf_node(b, node, block, succ); + progress |= lower_non_uniform_cf_node(b, node, block, succ, nak); } else { - progress |= lower_cf_list(b, &loop->body); - progress |= lower_cf_list(b, &loop->continue_list); + progress |= lower_cf_list(b, &loop->body, nak); + progress |= lower_cf_list(b, &loop->continue_list, nak); } break; } @@ -470,7 +488,8 @@ lower_cf_list(nir_builder *b, struct exec_list *cf_list) } bool -nak_nir_lower_non_uniform_ldcx(nir_shader *nir) +nak_nir_lower_non_uniform_ldcx(nir_shader *nir, + const struct nak_compiler *nak) { /* Real functions are going to make hash of this */ nir_function_impl *impl = nir_shader_get_entrypoint(nir); @@ -479,6 +498,6 @@ nak_nir_lower_non_uniform_ldcx(nir_shader *nir) /* We use block indices to determine when something is a predecessor */ nir_metadata_require(impl, nir_metadata_block_index | nir_metadata_divergence); - bool progress = lower_cf_list(&b, &impl->body); + bool progress = lower_cf_list(&b, &impl->body, nak); return nir_progress(progress, impl, nir_metadata_none); } diff --git a/src/nouveau/compiler/nak_private.h b/src/nouveau/compiler/nak_private.h index d1cf3e3f46c..1087823000c 100644 --- a/src/nouveau/compiler/nak_private.h +++ b/src/nouveau/compiler/nak_private.h @@ -308,7 +308,7 @@ enum nak_fs_out { bool nak_nir_rematerialize_load_const(nir_shader *nir); bool nak_nir_mark_lcssa_invariants(nir_shader *nir); -bool nak_nir_lower_non_uniform_ldcx(nir_shader *nir); +bool nak_nir_lower_non_uniform_ldcx(nir_shader *nir, const struct nak_compiler *nak); bool nak_nir_add_barriers(nir_shader *nir, const struct nak_compiler *nak); bool nak_nir_lower_cf(nir_shader *nir);