diff --git a/src/gallium/drivers/radeonsi/si_compute.c b/src/gallium/drivers/radeonsi/si_compute.c index 7da80da6829..29c2180d010 100644 --- a/src/gallium/drivers/radeonsi/si_compute.c +++ b/src/gallium/drivers/radeonsi/si_compute.c @@ -239,7 +239,7 @@ static void *si_create_compute_state(struct pipe_context *ctx, const struct pipe si_sampler_and_image_descriptors_idx(PIPE_SHADER_COMPUTE); sel->info.base.shared_size = cso->req_local_mem; program->shader.selector = &program->sel; - program->shader.wave_size = si_get_shader_wave_size(&program->shader); + program->shader.wave_size = si_get_shader_wave_size(sscreen, &program->shader); program->ir_type = cso->ir_type; program->private_size = cso->req_private_mem; program->input_size = cso->req_input_mem; diff --git a/src/gallium/drivers/radeonsi/si_compute_blit.c b/src/gallium/drivers/radeonsi/si_compute_blit.c index 42f293fafca..0ab9993a7e0 100644 --- a/src/gallium/drivers/radeonsi/si_compute_blit.c +++ b/src/gallium/drivers/radeonsi/si_compute_blit.c @@ -305,7 +305,8 @@ static void si_compute_do_clear_or_copy(struct si_context *sctx, struct pipe_res src ? SI_COMPUTE_COPY_DW_PER_THREAD : SI_COMPUTE_CLEAR_DW_PER_THREAD; unsigned instructions_per_thread = MAX2(1, dwords_per_thread / 4); unsigned dwords_per_instruction = dwords_per_thread / instructions_per_thread; - unsigned block_size = sctx->screen->compute_wave_size; + /* The shader declares the block size like this: */ + unsigned block_size = si_get_shader_wave_size(sctx->screen, NULL); unsigned dwords_per_wave = dwords_per_thread * block_size; unsigned num_dwords = size / 4; @@ -598,9 +599,10 @@ void si_compute_copy_image(struct si_context *sctx, struct pipe_resource *dst, u info.block[1] = ssrc->surface.u.gfx9.color.dcc_block_height; info.block[2] = ssrc->surface.u.gfx9.color.dcc_block_depth; + unsigned default_wave_size = si_get_shader_wave_size(sctx->screen, NULL);; + /* Make sure the block size is at least the same as wave size. */ - while (info.block[0] * info.block[1] * info.block[2] < - sctx->screen->compute_wave_size) { + while (info.block[0] * info.block[1] * info.block[2] < default_wave_size) { info.block[0] *= 2; } diff --git a/src/gallium/drivers/radeonsi/si_get.c b/src/gallium/drivers/radeonsi/si_get.c index 18726d2cb5d..a931af4b3d4 100644 --- a/src/gallium/drivers/radeonsi/si_get.c +++ b/src/gallium/drivers/radeonsi/si_get.c @@ -881,7 +881,7 @@ static int si_get_compute_param(struct pipe_screen *screen, enum pipe_shader_ir case PIPE_COMPUTE_CAP_SUBGROUP_SIZE: if (ret) { uint32_t *subgroup_size = ret; - *subgroup_size = sscreen->compute_wave_size; + *subgroup_size = si_get_shader_wave_size(sscreen, NULL); } return sizeof(uint32_t); case PIPE_COMPUTE_CAP_MAX_VARIABLE_THREADS_PER_BLOCK: diff --git a/src/gallium/drivers/radeonsi/si_pipe.h b/src/gallium/drivers/radeonsi/si_pipe.h index 793d74c7b2a..f58a48d4b48 100644 --- a/src/gallium/drivers/radeonsi/si_pipe.h +++ b/src/gallium/drivers/radeonsi/si_pipe.h @@ -1995,19 +1995,20 @@ static inline unsigned si_get_wave_size(struct si_screen *sscreen, return sscreen->ge_wave_size; } -static inline unsigned si_get_shader_wave_size(struct si_shader *shader) +static inline unsigned si_get_shader_wave_size(struct si_screen *sscreen, struct si_shader *shader) { - if (shader->is_gs_copy_shader) + /* There are a few uses that pass shader=NULL here, expecting the default compute wave size. */ + struct si_shader_info *info = shader ? &shader->selector->info : NULL; + gl_shader_stage stage = info ? info->stage : MESA_SHADER_COMPUTE; + + if (shader && shader->is_gs_copy_shader) return shader->selector->screen->ge_wave_size; - if (shader->selector->info.stage <= MESA_SHADER_GEOMETRY) { - return si_get_wave_size(shader->selector->screen, shader->selector->info.stage, - shader->key.ge.as_ngg, - shader->key.ge.as_es); + if (stage <= MESA_SHADER_GEOMETRY) { + return si_get_wave_size(sscreen, stage, shader->key.ge.as_ngg, shader->key.ge.as_es); } - return si_get_wave_size(shader->selector->screen, shader->selector->info.stage, - false, false); + return si_get_wave_size(sscreen, stage, false, false); } static inline void si_select_draw_vbo(struct si_context *sctx) diff --git a/src/gallium/drivers/radeonsi/si_shader_llvm_gs.c b/src/gallium/drivers/radeonsi/si_shader_llvm_gs.c index d059f860629..73c3f040698 100644 --- a/src/gallium/drivers/radeonsi/si_shader_llvm_gs.c +++ b/src/gallium/drivers/radeonsi/si_shader_llvm_gs.c @@ -425,7 +425,7 @@ struct si_shader *si_generate_gs_copy_shader(struct si_screen *sscreen, shader->selector = gs_selector; shader->is_gs_copy_shader = true; - shader->wave_size = si_get_shader_wave_size(shader); + shader->wave_size = si_get_shader_wave_size(sscreen, shader); si_llvm_context_init(&ctx, sscreen, compiler, shader->wave_size); ctx.shader = shader; diff --git a/src/gallium/drivers/radeonsi/si_shaderlib_tgsi.c b/src/gallium/drivers/radeonsi/si_shaderlib_tgsi.c index a6827092fe4..68466cd88d6 100644 --- a/src/gallium/drivers/radeonsi/si_shaderlib_tgsi.c +++ b/src/gallium/drivers/radeonsi/si_shaderlib_tgsi.c @@ -134,7 +134,9 @@ void *si_create_dma_compute_shader(struct pipe_context *ctx, unsigned num_dwords if (!ureg) return NULL; - ureg_property(ureg, TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH, sscreen->compute_wave_size); + unsigned default_wave_size = si_get_shader_wave_size(sscreen, NULL); + + ureg_property(ureg, TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH, default_wave_size); ureg_property(ureg, TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT, 1); ureg_property(ureg, TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH, 1); @@ -160,7 +162,7 @@ void *si_create_dma_compute_shader(struct pipe_context *ctx, unsigned num_dwords /* If there are multiple stores, the first store writes into 0*wavesize+tid, * the 2nd store writes into 1*wavesize+tid, the 3rd store writes into 2*wavesize+tid, etc. */ - ureg_UMAD(ureg, store_addr, blk, ureg_imm1u(ureg, sscreen->compute_wave_size * num_mem_ops), + ureg_UMAD(ureg, store_addr, blk, ureg_imm1u(ureg, default_wave_size * num_mem_ops), tid); /* Convert from a "store size unit" into bytes. */ ureg_UMUL(ureg, store_addr, ureg_src(store_addr), ureg_imm1u(ureg, 4 * inst_dwords[0])); @@ -175,7 +177,7 @@ void *si_create_dma_compute_shader(struct pipe_context *ctx, unsigned num_dwords if (is_copy && i < num_mem_ops) { if (i) { ureg_UADD(ureg, load_addr, ureg_src(load_addr), - ureg_imm1u(ureg, 4 * inst_dwords[i] * sscreen->compute_wave_size)); + ureg_imm1u(ureg, 4 * inst_dwords[i] * default_wave_size)); } values[i] = ureg_src(ureg_DECL_temporary(ureg)); @@ -189,7 +191,7 @@ void *si_create_dma_compute_shader(struct pipe_context *ctx, unsigned num_dwords if (d >= 0) { if (d) { ureg_UADD(ureg, store_addr, ureg_src(store_addr), - ureg_imm1u(ureg, 4 * inst_dwords[d] * sscreen->compute_wave_size)); + ureg_imm1u(ureg, 4 * inst_dwords[d] * default_wave_size)); } struct ureg_dst dst = ureg_writemask(dstbuf, u_bit_consecutive(0, inst_dwords[d])); diff --git a/src/gallium/drivers/radeonsi/si_state_shaders.cpp b/src/gallium/drivers/radeonsi/si_state_shaders.cpp index 478e8d5bc85..07bb806f857 100644 --- a/src/gallium/drivers/radeonsi/si_state_shaders.cpp +++ b/src/gallium/drivers/radeonsi/si_state_shaders.cpp @@ -2270,7 +2270,7 @@ static bool si_check_missing_main_part(struct si_screen *sscreen, struct si_shad main_part->key.ge.as_ngg = key->ge.as_ngg; } main_part->is_monolithic = false; - main_part->wave_size = si_get_shader_wave_size(main_part); + main_part->wave_size = si_get_shader_wave_size(sscreen, main_part); if (!si_compile_shader(sscreen, compiler_state->compiler, main_part, &compiler_state->debug)) { @@ -2447,7 +2447,7 @@ current_not_ready: shader->selector = sel; *((SHADER_KEY_TYPE*)&shader->key) = *key; - shader->wave_size = si_get_shader_wave_size(shader); + shader->wave_size = si_get_shader_wave_size(sscreen, shader); shader->compiler_ctx_state.compiler = &sctx->compiler; shader->compiler_ctx_state.debug = sctx->debug; shader->compiler_ctx_state.is_debug_context = sctx->is_debug; @@ -2714,7 +2714,7 @@ static void si_init_shader_selector_async(void *job, void *gdata, int thread_ind sel->info.stage == MESA_SHADER_TESS_EVAL || sel->info.stage == MESA_SHADER_GEOMETRY)) shader->key.ge.as_ngg = 1; - shader->wave_size = si_get_shader_wave_size(shader); + shader->wave_size = si_get_shader_wave_size(sscreen, shader); if (sel->nir) { if (sel->info.stage <= MESA_SHADER_GEOMETRY) {