From cc6e43c04964f1ab3460a70c8ccb82a91ea708c5 Mon Sep 17 00:00:00 2001 From: Konstantin Seurer Date: Mon, 11 Nov 2024 09:20:44 +0100 Subject: [PATCH] lavapipe: Lower descriptor sets in NIR This moves lowering vulkan resource indices out of gallivm into lavapipe where it should be. Reviewed-by: Mike Blumenkrantz Part-of: --- src/compiler/nir/nir_intrinsics.py | 2 + .../auxiliary/gallivm/lp_bld_jit_sample.c | 35 ++--- .../auxiliary/gallivm/lp_bld_jit_types.c | 17 ++- .../auxiliary/gallivm/lp_bld_nir_soa.c | 90 +++++-------- .../drivers/zink/ci/zink-lvp-skips.txt | 3 + .../lavapipe/lvp_ray_tracing_pipeline.c | 16 +-- .../nir/lvp_nir_lower_pipeline_layout.c | 121 ++++++++++-------- .../lavapipe/nir/lvp_nir_lower_ray_queries.c | 13 +- 8 files changed, 133 insertions(+), 164 deletions(-) diff --git a/src/compiler/nir/nir_intrinsics.py b/src/compiler/nir/nir_intrinsics.py index efc0b046139..a5655f8ff74 100644 --- a/src/compiler/nir/nir_intrinsics.py +++ b/src/compiler/nir/nir_intrinsics.py @@ -1934,6 +1934,8 @@ system_value("ray_tracing_stack_base_lvp", 1) system_value("shader_call_data_offset_lvp", 1) +intrinsic("load_const_buf_base_addr_lvp", src_comp=[1], bit_sizes=[64], dest_comp=1, flags=[CAN_ELIMINATE, CAN_REORDER]) + # Broadcom-specific instrinc for tile buffer color reads. # # The hardware requires that we read the samples and components of a pixel diff --git a/src/gallium/auxiliary/gallivm/lp_bld_jit_sample.c b/src/gallium/auxiliary/gallivm/lp_bld_jit_sample.c index 09e58652a91..ecf25618148 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_jit_sample.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_jit_sample.c @@ -174,16 +174,12 @@ lp_bld_llvm_sampler_soa_emit_fetch_texel(const struct lp_build_sampler_soa *base struct lp_build_if_state if_state; lp_build_if(&if_state, gallivm, any_active); - LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr); - - LLVMValueRef texture_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->texture_resource, LP_MAX_TGSI_CONST_BUFFERS); - enum lp_sampler_op_type op_type = (params->sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT; uint32_t functions_offset = op_type == LP_SAMPLER_OP_FETCH ? offsetof(struct lp_texture_functions, fetch_functions) : offsetof(struct lp_texture_functions, sample_functions); LLVMValueRef texture_base_ptr = load_texture_functions_ptr( - gallivm, texture_descriptor, offsetof(struct lp_descriptor, functions), functions_offset); + gallivm, params->texture_resource, offsetof(struct lp_descriptor, functions), functions_offset); LLVMTypeRef texture_function_type = lp_build_sample_function_type(gallivm, params->sample_key); LLVMTypeRef texture_function_ptr_type = LLVMPointerType(texture_function_type, 0); @@ -200,7 +196,7 @@ lp_bld_llvm_sampler_soa_emit_fetch_texel(const struct lp_build_sampler_soa *base texture_functions = texture_base; sampler_desc_ptr = LLVMGetUndef(LLVMInt64TypeInContext(gallivm->context)); } else { - sampler_desc_ptr = lp_llvm_descriptor_base(gallivm, consts, params->sampler_resource, LP_MAX_TGSI_CONST_BUFFERS); + sampler_desc_ptr = params->sampler_resource; LLVMValueRef sampler_index_offset = lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, texture.sampler_index)); LLVMValueRef sampler_index_ptr = LLVMBuildAdd(builder, sampler_desc_ptr, sampler_index_offset, ""); @@ -222,7 +218,7 @@ lp_bld_llvm_sampler_soa_emit_fetch_texel(const struct lp_build_sampler_soa *base LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS]; uint32_t num_args = 0; - args[num_args++] = texture_descriptor; + args[num_args++] = params->texture_resource; args[num_args++] = sampler_desc_ptr; LLVMTypeRef coord_type; @@ -351,15 +347,11 @@ lp_bld_llvm_sampler_soa_emit_size_query(const struct lp_build_sampler_soa *base, struct lp_build_if_state if_state; lp_build_if(&if_state, gallivm, any_active); - LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr); - - LLVMValueRef texture_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS); - uint32_t functions_offset = params->samples_only ? offsetof(struct lp_texture_functions, samples_function) : offsetof(struct lp_texture_functions, size_function); LLVMValueRef texture_base_ptr = load_texture_functions_ptr( - gallivm, texture_descriptor, offsetof(struct lp_descriptor, functions), functions_offset); + gallivm, params->resource, offsetof(struct lp_descriptor, functions), functions_offset); LLVMTypeRef texture_function_type = lp_build_size_function_type(gallivm, params); LLVMTypeRef texture_function_ptr_type = LLVMPointerType(texture_function_type, 0); @@ -371,7 +363,7 @@ lp_bld_llvm_sampler_soa_emit_size_query(const struct lp_build_sampler_soa *base, LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS]; uint32_t num_args = 0; - args[num_args++] = texture_descriptor; + args[num_args++] = params->resource; if (!params->samples_only) args[num_args++] = params->explicit_lod; @@ -461,18 +453,11 @@ lp_bld_llvm_image_soa_emit_op(const struct lp_build_image_soa *base, LLVMValueRef any_active = LLVMBuildICmp(builder, LLVMIntNE, bitmask, LLVMConstInt(bitmask_type, 0, false), "any_active"); - LLVMValueRef binding_index = LLVMBuildExtractValue(builder, params->resource, 1, ""); - LLVMValueRef inbounds = LLVMBuildICmp(builder, LLVMIntSGE, binding_index, lp_build_const_int32(gallivm, 0), "inbounds"); - struct lp_build_if_state if_state; - lp_build_if(&if_state, gallivm, LLVMBuildAnd(builder, any_active, inbounds, "")); - - LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr); - - LLVMValueRef image_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS); + lp_build_if(&if_state, gallivm, any_active); LLVMValueRef image_base_ptr = load_texture_functions_ptr( - gallivm, image_descriptor, offsetof(struct lp_descriptor, functions), + gallivm, params->resource, offsetof(struct lp_descriptor, functions), offsetof(struct lp_texture_functions, image_functions)); LLVMTypeRef image_function_type = lp_build_image_function_type(gallivm, params, params->ms_index); @@ -500,7 +485,7 @@ lp_bld_llvm_image_soa_emit_op(const struct lp_build_image_soa *base, LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS] = { 0 }; uint32_t num_args = 0; - args[num_args++] = image_descriptor; + args[num_args++] = params->resource; if (params->img_op != LP_IMG_LOAD && params->img_op != LP_IMG_LOAD_SPARSE) args[num_args++] = params->exec_mask; @@ -598,9 +583,7 @@ lp_bld_llvm_image_soa_emit_size_query(const struct lp_build_image_soa *base, if (params->resource) { LLVMValueRef old_texture = gallivm->texture_descriptor; - - LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr); - gallivm->texture_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS); + gallivm->texture_descriptor = params->resource; enum pipe_format format = params->format; if (format == PIPE_FORMAT_NONE) diff --git a/src/gallium/auxiliary/gallivm/lp_bld_jit_types.c b/src/gallium/auxiliary/gallivm/lp_bld_jit_types.c index 8e2651dd404..8621c06561e 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_jit_types.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_jit_types.c @@ -93,17 +93,16 @@ lp_llvm_buffer_member(struct gallivm_state *gallivm, LLVMTypeRef buffer_type = lp_build_create_jit_buffer_type(gallivm); LLVMValueRef ptr; - if (LLVMGetTypeKind(LLVMTypeOf(buffers_offset)) == LLVMArrayTypeKind) { - LLVMValueRef desc_ptr = lp_llvm_descriptor_base(gallivm, buffers_ptr, buffers_offset, buffers_limit); + if (LLVMGetTypeKind(LLVMTypeOf(buffers_offset)) == LLVMIntegerTypeKind && + LLVMGetIntTypeWidth(LLVMTypeOf(buffers_offset)) == 64) { + LLVMTypeRef ptr_type = LLVMPointerType(buffer_type, 0); + ptr = LLVMBuildIntToPtr(builder, buffers_offset, ptr_type, ""); - LLVMTypeRef buffer_ptr_type = LLVMPointerType(buffer_type, 0); - desc_ptr = LLVMBuildIntToPtr(builder, desc_ptr, buffer_ptr_type, ""); + LLVMValueRef indices[2]; + indices[0] = lp_build_const_int32(gallivm, 0); + indices[1] = lp_build_const_int32(gallivm, member_index); - LLVMValueRef indices[2] = { - lp_build_const_int32(gallivm, 0), - lp_build_const_int32(gallivm, member_index), - }; - ptr = LLVMBuildGEP2(builder, buffer_type, desc_ptr, indices, ARRAY_SIZE(indices), ""); + ptr = LLVMBuildGEP2(builder, buffer_type, ptr, indices, ARRAY_SIZE(indices), ""); } else { LLVMValueRef indices[3]; diff --git a/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c b/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c index f644e2120bb..11d931e0012 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c @@ -1099,23 +1099,6 @@ lp_offset_in_range(struct lp_build_nir_soa_context *bld, return LLVMBuildAnd(gallivm->builder, fetch_in_bounds, fetch_non_negative, ""); } -static LLVMValueRef -build_resource_to_scalar(struct lp_build_nir_soa_context *bld, LLVMValueRef resource) -{ - struct gallivm_state *gallivm = bld->base.gallivm; - - LLVMValueRef invocation = first_active_invocation(bld); - - LLVMValueRef set = LLVMBuildExtractValue(gallivm->builder, resource, 0, ""); - set = LLVMBuildExtractElement(gallivm->builder, set, invocation, ""); - - LLVMValueRef binding = LLVMBuildExtractValue(gallivm->builder, resource, 1, ""); - binding = LLVMBuildExtractElement(gallivm->builder, binding, invocation, ""); - - LLVMValueRef components[2] = { set, binding }; - return lp_nir_array_build_gather_values(gallivm->builder, components, 2); -} - static void emit_load_ubo(struct lp_build_nir_soa_context *bld, unsigned nc, unsigned bit_size, @@ -1124,9 +1107,6 @@ static void emit_load_ubo(struct lp_build_nir_soa_context *bld, LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]) { - if (LLVMGetTypeKind(LLVMTypeOf(index)) == LLVMArrayTypeKind) - index = build_resource_to_scalar(bld, index); - struct gallivm_state *gallivm = bld->base.gallivm; LLVMBuilderRef builder = gallivm->builder; struct lp_build_context *uint_bld = &bld->uint_bld; @@ -1185,6 +1165,18 @@ static void emit_load_ubo(struct lp_build_nir_soa_context *bld, } } +static LLVMValueRef +load_ubo_base_addr(struct lp_build_nir_soa_context *bld, LLVMValueRef index) +{ + struct gallivm_state *gallivm = bld->base.gallivm; + + index = LLVMBuildExtractElement(gallivm->builder, index, first_active_invocation(bld), ""); + + LLVMValueRef base = lp_llvm_buffer_base(gallivm, bld->consts_ptr, index, LP_MAX_TGSI_CONST_BUFFERS); + base = LLVMBuildPtrToInt(gallivm->builder, base, LLVMInt64TypeInContext(gallivm->context), ""); + return lp_build_broadcast_scalar(&bld->int64_bld, base); +} + static void emit_load_const(struct lp_build_nir_soa_context *bld, const nir_load_const_instr *instr, @@ -1215,30 +1207,10 @@ ssbo_base_pointer(struct lp_build_nir_soa_context *bld, struct gallivm_state *gallivm = bld->base.gallivm; uint32_t shift_val = bit_size_to_shift_size(bit_size); - LLVMValueRef ssbo_idx; - LLVMValueRef buffers; - uint32_t buffers_limit; - if (LLVMGetTypeKind(LLVMTypeOf(index)) == LLVMArrayTypeKind) { - LLVMValueRef set = LLVMBuildExtractValue(gallivm->builder, index, 0, ""); - set = LLVMBuildExtractElement(gallivm->builder, set, invocation, ""); + LLVMValueRef ssbo_idx = LLVMBuildExtractElement(gallivm->builder, index, invocation, ""); - LLVMValueRef binding = LLVMBuildExtractValue(gallivm->builder, index, 1, ""); - binding = LLVMBuildExtractElement(gallivm->builder, binding, invocation, ""); - - LLVMValueRef components[2] = { set, binding }; - ssbo_idx = lp_nir_array_build_gather_values(gallivm->builder, components, 2); - - buffers = bld->consts_ptr; - buffers_limit = LP_MAX_TGSI_CONST_BUFFERS; - } else { - ssbo_idx = LLVMBuildExtractElement(gallivm->builder, index, invocation, ""); - - buffers = bld->ssbo_ptr; - buffers_limit = LP_MAX_TGSI_SHADER_BUFFERS; - } - - LLVMValueRef ssbo_size_ptr = lp_llvm_buffer_num_elements(gallivm, buffers, ssbo_idx, buffers_limit); - LLVMValueRef ssbo_ptr = lp_llvm_buffer_base(gallivm, buffers, ssbo_idx, buffers_limit); + LLVMValueRef ssbo_size_ptr = lp_llvm_buffer_num_elements(gallivm, bld->ssbo_ptr, ssbo_idx, LP_MAX_TGSI_SHADER_BUFFERS); + LLVMValueRef ssbo_ptr = lp_llvm_buffer_base(gallivm, bld->ssbo_ptr, ssbo_idx, LP_MAX_TGSI_SHADER_BUFFERS); if (bounds) *bounds = LLVMBuildAShr(gallivm->builder, ssbo_size_ptr, lp_build_const_int32(gallivm, shift_val), ""); @@ -1647,7 +1619,8 @@ static void emit_image_op(struct lp_build_nir_soa_context *bld, first_active_invocation(bld), ""); if (params->resource) - params->resource = build_resource_to_scalar(bld, params->resource); + params->resource = LLVMBuildExtractElement(gallivm->builder, params->resource, + first_active_invocation(bld), ""); bld->image->emit_op(bld->image, bld->base.gallivm, @@ -1775,10 +1748,12 @@ static void emit_tex(struct lp_build_nir_soa_context *bld, } if (params->texture_resource) - params->texture_resource = build_resource_to_scalar(bld, params->texture_resource); + params->texture_resource = LLVMBuildExtractElement(gallivm->builder, params->texture_resource, + first_active_invocation(bld), ""); if (params->sampler_resource) - params->sampler_resource = build_resource_to_scalar(bld, params->sampler_resource); + params->sampler_resource = LLVMBuildExtractElement(gallivm->builder, params->sampler_resource, + first_active_invocation(bld), ""); params->type = bld->base.type; bld->sampler->emit_tex_sample(bld->sampler, @@ -1799,7 +1774,8 @@ static void emit_tex_size(struct lp_build_nir_soa_context *bld, params->exec_mask = mask_vec(bld); if (params->resource) - params->resource = build_resource_to_scalar(bld, params->resource); + params->resource = LLVMBuildExtractElement(bld->base.gallivm->builder, params->resource, + first_active_invocation(bld), ""); bld->sampler->emit_size_query(bld->sampler, bld->base.gallivm, @@ -4156,8 +4132,7 @@ visit_load_ssbo(struct lp_build_nir_soa_context *bld, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]) { LLVMValueRef idx = get_src(bld, instr->src[0]); - if (nir_src_num_components(instr->src[0]) == 1) - idx = cast_type(bld, idx, nir_type_uint, 32); + idx = cast_type(bld, idx, nir_type_uint, nir_src_bit_size(instr->src[0])); LLVMValueRef offset = get_src(bld, instr->src[1]); @@ -4175,8 +4150,7 @@ visit_store_ssbo(struct lp_build_nir_soa_context *bld, LLVMValueRef val = get_src(bld, instr->src[0]); LLVMValueRef idx = get_src(bld, instr->src[1]); - if (nir_src_num_components(instr->src[1]) == 1) - idx = cast_type(bld, idx, nir_type_uint, 32); + idx = cast_type(bld, idx, nir_type_uint, nir_src_bit_size(instr->src[1])); LLVMValueRef offset = get_src(bld, instr->src[2]); int writemask = instr->const_index[0]; @@ -4194,8 +4168,7 @@ visit_get_ssbo_size(struct lp_build_nir_soa_context *bld, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]) { LLVMValueRef idx = get_src(bld, instr->src[0]); - if (nir_src_num_components(instr->src[0]) == 1) - idx = cast_type(bld, idx, nir_type_uint, 32); + idx = cast_type(bld, idx, nir_type_uint, nir_src_bit_size(instr->src[0])); struct lp_build_context *bld_broad = &bld->uint_bld; @@ -4211,8 +4184,7 @@ visit_ssbo_atomic(struct lp_build_nir_soa_context *bld, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]) { LLVMValueRef idx = get_src(bld, instr->src[0]); - if (nir_src_num_components(instr->src[0]) == 1) - idx = cast_type(bld, idx, nir_type_uint, 32); + idx = cast_type(bld, idx, nir_type_uint, nir_src_bit_size(instr->src[0])); LLVMValueRef offset = get_src(bld, instr->src[1]); LLVMValueRef val = get_src(bld, instr->src[2]); @@ -4228,7 +4200,7 @@ visit_ssbo_atomic(struct lp_build_nir_soa_context *bld, static void img_params_init_resource(struct lp_build_nir_soa_context *bld, struct lp_img_params *params, nir_src src) { - if (nir_src_num_components(src) == 1) { + if (nir_src_bit_size(src) < 64) { if (nir_src_is_const(src)) params->image_index = nir_src_as_int(src); else @@ -4243,7 +4215,7 @@ img_params_init_resource(struct lp_build_nir_soa_context *bld, struct lp_img_par static void sampler_size_params_init_resource(struct lp_build_nir_soa_context *bld, struct lp_sampler_size_query_params *params, nir_src src) { - if (nir_src_num_components(src) == 1) { + if (nir_src_bit_size(src) < 64) { if (nir_src_is_const(src)) params->texture_unit = nir_src_as_int(src); else @@ -5209,6 +5181,10 @@ visit_intrinsic(struct lp_build_nir_soa_context *bld, break; } + case nir_intrinsic_load_const_buf_base_addr_lvp: { + result[0] = load_ubo_base_addr(bld, get_src(bld, instr->src[0])); + break; + } default: fprintf(stderr, "Unsupported intrinsic: "); nir_print_instr(&instr->instr, stderr); diff --git a/src/gallium/drivers/zink/ci/zink-lvp-skips.txt b/src/gallium/drivers/zink/ci/zink-lvp-skips.txt index 7226486adfb..842998b3990 100644 --- a/src/gallium/drivers/zink/ci/zink-lvp-skips.txt +++ b/src/gallium/drivers/zink/ci/zink-lvp-skips.txt @@ -45,3 +45,6 @@ spec@arb_texture_rg@fbo-blending-formats #these need format conversions that gallium doesn't implement yet spec@arb_texture_buffer_object@formats.*arb.* + +# Crashes when accessing a descriptor out of bounds +spec@arb_shader_image_load_store@invalid diff --git a/src/gallium/frontends/lavapipe/lvp_ray_tracing_pipeline.c b/src/gallium/frontends/lavapipe/lvp_ray_tracing_pipeline.c index fb576ba5f86..7f5f77b9728 100644 --- a/src/gallium/frontends/lavapipe/lvp_ray_tracing_pipeline.c +++ b/src/gallium/frontends/lavapipe/lvp_ray_tracing_pipeline.c @@ -724,17 +724,9 @@ lvp_trace_ray(nir_builder *b, struct lvp_ray_tracing_pipeline_compiler *compiler nir_store_var(b, state->shader_call_data_offset, nir_iadd_imm(b, payload, -stack_size), 0x1); - nir_def *bvh_base = accel_struct; - if (bvh_base->bit_size != 64) { - assert(bvh_base->num_components >= 2); - bvh_base = nir_load_ubo( - b, 1, 64, nir_channel(b, accel_struct, 0), - nir_imul_imm(b, nir_channel(b, accel_struct, 1), sizeof(struct lp_descriptor)), .range = ~0); - } - lvp_ray_traversal_state_init(b->impl, &state->traversal); - nir_store_var(b, state->bvh_base, bvh_base, 0x1); + nir_store_var(b, state->bvh_base, accel_struct, 0x1); nir_store_var(b, state->flags, flags, 0x1); nir_store_var(b, state->cull_mask, cull_mask, 0x1); nir_store_var(b, state->sbt_offset, sbt_offset, 0x1); @@ -745,7 +737,7 @@ lvp_trace_ray(nir_builder *b, struct lvp_ray_tracing_pipeline_compiler *compiler nir_store_var(b, state->dir, dir, 0x7); nir_store_var(b, state->tmax, tmax, 0x1); - nir_store_var(b, state->traversal.bvh_base, bvh_base, 0x1); + nir_store_var(b, state->traversal.bvh_base, accel_struct, 0x1); nir_store_var(b, state->traversal.origin, origin, 0x7); nir_store_var(b, state->traversal.dir, dir, 0x7); nir_store_var(b, state->traversal.inv_dir, nir_frcp(b, dir), 0x7); @@ -770,7 +762,7 @@ lvp_trace_ray(nir_builder *b, struct lvp_ray_tracing_pipeline_compiler *compiler }; struct lvp_ray_traversal_args args = { - .root_bvh_base = bvh_base, + .root_bvh_base = accel_struct, .flags = flags, .cull_mask = nir_ishl_imm(b, cull_mask, 24), .origin = origin, @@ -784,7 +776,7 @@ lvp_trace_ray(nir_builder *b, struct lvp_ray_tracing_pipeline_compiler *compiler .data = compiler, }; - nir_push_if(b, nir_ine_imm(b, bvh_base, 0)); + nir_push_if(b, nir_ine_imm(b, accel_struct, 0)); lvp_build_ray_traversal(b, &args); nir_pop_if(b, NULL); diff --git a/src/gallium/frontends/lavapipe/nir/lvp_nir_lower_pipeline_layout.c b/src/gallium/frontends/lavapipe/nir/lvp_nir_lower_pipeline_layout.c index 5769b2334fe..ce3a9f52e32 100644 --- a/src/gallium/frontends/lavapipe/nir/lvp_nir_lower_pipeline_layout.c +++ b/src/gallium/frontends/lavapipe/nir/lvp_nir_lower_pipeline_layout.c @@ -24,34 +24,6 @@ #include "lvp_private.h" #include "lvp_nir.h" -static bool -lower_vulkan_resource_index(const nir_instr *instr, const void *data_cb) -{ - if (instr->type == nir_instr_type_intrinsic) { - nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); - switch (intrin->intrinsic) { - case nir_intrinsic_vulkan_resource_index: - case nir_intrinsic_vulkan_resource_reindex: - case nir_intrinsic_load_vulkan_descriptor: - case nir_intrinsic_get_ssbo_size: - case nir_intrinsic_image_deref_sparse_load: - case nir_intrinsic_image_deref_load: - case nir_intrinsic_image_deref_store: - case nir_intrinsic_image_deref_atomic: - case nir_intrinsic_image_deref_atomic_swap: - case nir_intrinsic_image_deref_size: - case nir_intrinsic_image_deref_samples: - return true; - default: - return false; - } - } - if (instr->type == nir_instr_type_tex) { - return true; - } - return false; -} - static nir_def *lower_vri_intrin_vri(struct nir_builder *b, nir_instr *instr, void *data_cb) { @@ -84,6 +56,34 @@ static nir_def *lower_vri_intrin_lvd(struct nir_builder *b, return intrin->src[0].ssa; } +static void +lower_buffer(nir_builder *b, nir_intrinsic_instr *intr, uint32_t src_index) +{ + if (nir_src_num_components(intr->src[src_index]) == 1) + return; + + nir_def *set = nir_channel(b, intr->src[src_index].ssa, 0); + nir_def *binding = nir_channel(b, intr->src[src_index].ssa, 1); + + nir_def *base = nir_load_const_buf_base_addr_lvp(b, set); + nir_def *offset = nir_imul_imm(b, binding, sizeof(struct lp_descriptor)); + nir_def *descriptor = nir_iadd(b, base, nir_u2u64(b, offset)); + nir_src_rewrite(&intr->src[src_index], descriptor); +} + +static void +lower_accel_struct(nir_builder *b, nir_intrinsic_instr *intr, uint32_t src_index) +{ + if (nir_src_bit_size(intr->src[src_index]) == 64) + return; + + nir_def *set = nir_channel(b, intr->src[src_index].ssa, 0); + nir_def *binding = nir_channel(b, intr->src[src_index].ssa, 1); + + nir_def *offset = nir_imul_imm(b, binding, sizeof(struct lp_descriptor)); + nir_src_rewrite(&intr->src[src_index], nir_load_ubo(b, 1, 64, set, offset, .range = UINT32_MAX)); +} + static nir_def * vulkan_resource_from_deref(nir_builder *b, nir_deref_instr *deref, const struct lvp_pipeline_layout *layout, unsigned plane) @@ -103,11 +103,11 @@ vulkan_resource_from_deref(nir_builder *b, nir_deref_instr *deref, const struct const struct lvp_descriptor_set_binding_layout *binding = get_binding_layout(layout, var->data.descriptor_set, var->data.binding); uint32_t binding_base = binding->descriptor_index + plane; - index = nir_imul_imm(b, index, binding->stride); + index = nir_iadd_imm(b, nir_imul_imm(b, index, binding->stride), binding_base); - return nir_vec3(b, nir_imm_int(b, var->data.descriptor_set + 1), - nir_iadd_imm(b, index, binding_base), - nir_imm_int(b, 0)); + nir_def *set = nir_load_const_buf_base_addr_lvp(b, nir_imm_int(b, var->data.descriptor_set + 1)); + nir_def *offset = nir_imul_imm(b, index, sizeof(struct lp_descriptor)); + return nir_iadd(b, set, nir_u2u64(b, offset)); } static void lower_vri_instr_tex(struct nir_builder *b, @@ -179,28 +179,50 @@ lower_load_ubo(nir_builder *b, nir_intrinsic_instr *intrin, void *data_cb) return true; } -static nir_def *lower_vri_instr(struct nir_builder *b, - nir_instr *instr, void *data_cb) +static bool +lower_vri_instr(struct nir_builder *b, nir_instr *instr, void *data_cb) { + b->cursor = nir_before_instr(instr); + if (instr->type == nir_instr_type_intrinsic) { nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); switch (intrin->intrinsic) { case nir_intrinsic_vulkan_resource_index: - return lower_vri_intrin_vri(b, instr, data_cb); + nir_def_rewrite_uses(&intrin->def, lower_vri_intrin_vri(b, instr, data_cb)); + return true; case nir_intrinsic_vulkan_resource_reindex: - return lower_vri_intrin_vrri(b, instr, data_cb); + nir_def_rewrite_uses(&intrin->def, lower_vri_intrin_vrri(b, instr, data_cb)); + return true; case nir_intrinsic_load_vulkan_descriptor: - return lower_vri_intrin_lvd(b, instr, data_cb); + nir_def_rewrite_uses(&intrin->def, lower_vri_intrin_lvd(b, instr, data_cb)); + return true; + + + case nir_intrinsic_load_ubo: + lower_buffer(b, intrin, 0); + return true; + + case nir_intrinsic_load_ssbo: + case nir_intrinsic_ssbo_atomic: + case nir_intrinsic_ssbo_atomic_swap: + case nir_intrinsic_get_ssbo_size: + lower_buffer(b, intrin, 0); + return true; + + case nir_intrinsic_store_ssbo: + lower_buffer(b, intrin, 1); + return true; + + case nir_intrinsic_trace_ray: + lower_accel_struct(b, intrin, 0); + return true; + + case nir_intrinsic_rq_initialize: + lower_accel_struct(b, intrin, 1); + return true; - case nir_intrinsic_get_ssbo_size: { - /* Ignore the offset component. */ - b->cursor = nir_before_instr(instr); - nir_def *resource = intrin->src[0].ssa; - nir_src_rewrite(&intrin->src[0], resource); - return NULL; - } case nir_intrinsic_image_deref_sparse_load: case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_store: @@ -208,21 +230,20 @@ static nir_def *lower_vri_instr(struct nir_builder *b, case nir_intrinsic_image_deref_atomic_swap: case nir_intrinsic_image_deref_size: case nir_intrinsic_image_deref_samples: - b->cursor = nir_before_instr(instr); lower_image_intrinsic(b, intrin, data_cb); - return NULL; + return true; default: - return NULL; + return false; } } if (instr->type == nir_instr_type_tex) { - b->cursor = nir_before_instr(instr); lower_vri_instr_tex(b, nir_instr_as_tex(instr), data_cb); + return true; } - return NULL; + return false; } void lvp_lower_pipeline_layout(const struct lvp_device *device, @@ -232,5 +253,5 @@ void lvp_lower_pipeline_layout(const struct lvp_device *device, nir_shader_intrinsics_pass(shader, lower_load_ubo, nir_metadata_control_flow, layout); - nir_shader_lower_instructions(shader, lower_vulkan_resource_index, lower_vri_instr, layout); + nir_shader_instructions_pass(shader, lower_vri_instr, nir_metadata_control_flow, layout); } diff --git a/src/gallium/frontends/lavapipe/nir/lvp_nir_lower_ray_queries.c b/src/gallium/frontends/lavapipe/nir/lvp_nir_lower_ray_queries.c index b956e2dce25..2389c96ccec 100644 --- a/src/gallium/frontends/lavapipe/nir/lvp_nir_lower_ray_queries.c +++ b/src/gallium/frontends/lavapipe/nir/lvp_nir_lower_ray_queries.c @@ -325,22 +325,15 @@ lower_rq_initialize(nir_builder *b, nir_def *index, nir_intrinsic_instr *instr, 0x1); nir_def *accel_struct = instr->src[1].ssa; - nir_def *bvh_base = accel_struct; - if (bvh_base->bit_size != 64) { - assert(bvh_base->num_components >= 2); - bvh_base = nir_load_ubo( - b, 1, 64, nir_channel(b, accel_struct, 0), - nir_imul_imm(b, nir_channel(b, accel_struct, 1), sizeof(struct lp_descriptor)), .range = ~0); - } - rq_store_var(b, index, vars->root_bvh_base, bvh_base, 0x1); - rq_store_var(b, index, vars->trav.bvh_base, bvh_base, 1); + rq_store_var(b, index, vars->root_bvh_base, accel_struct, 0x1); + rq_store_var(b, index, vars->trav.bvh_base, accel_struct, 1); rq_store_var(b, index, vars->trav.current_node, nir_imm_int(b, LVP_BVH_ROOT_NODE), 0x1); rq_store_var(b, index, vars->trav.stack_ptr, nir_imm_int(b, 0), 0x1); rq_store_var(b, index, vars->trav.stack_base, nir_imm_int(b, -1), 0x1); - rq_store_var(b, index, vars->incomplete, nir_ine_imm(b, bvh_base, 0), 0x1); + rq_store_var(b, index, vars->incomplete, nir_ine_imm(b, accel_struct, 0), 0x1); } static nir_def *