diff --git a/src/amd/common/ac_gpu_info.c b/src/amd/common/ac_gpu_info.c index b404655b296..2636ebf83c3 100644 --- a/src/amd/common/ac_gpu_info.c +++ b/src/amd/common/ac_gpu_info.c @@ -1210,6 +1210,14 @@ bool ac_query_gpu_info(int fd, void *dev_p, struct radeon_info *info) info->has_export_conflict_bug = info->gfx_level == GFX11; + /* Only dGPUs have SET_*_PAIRS packets for now. + * Register shadowing is only required by SET_SH_REG_PAIRS*, but we require it + * for SET_CONTEXT_REG_PAIRS* as well for simplicity. + */ + info->has_set_pairs_packets = info->gfx_level >= GFX11 && + info->register_shadowing_required && + info->has_dedicated_vram; + /* Get the number of good compute units. */ info->num_cu = 0; for (i = 0; i < info->max_se; i++) { @@ -1680,6 +1688,7 @@ void ac_print_gpu_info(const struct radeon_info *info, FILE *f) fprintf(f, " never_send_perfcounter_stop = %i\n", info->never_send_perfcounter_stop); fprintf(f, " discardable_allows_big_page = %i\n", info->discardable_allows_big_page); fprintf(f, " has_taskmesh_indirect0_bug = %i\n", info->has_taskmesh_indirect0_bug); + fprintf(f, " has_set_pairs_packets = %i\n", info->has_set_pairs_packets); fprintf(f, " conformant_trunc_coord = %i\n", info->conformant_trunc_coord); fprintf(f, "Display features:\n"); diff --git a/src/amd/common/ac_gpu_info.h b/src/amd/common/ac_gpu_info.h index 359e20de78a..8f79698d72b 100644 --- a/src/amd/common/ac_gpu_info.h +++ b/src/amd/common/ac_gpu_info.h @@ -108,6 +108,7 @@ struct radeon_info { bool has_export_conflict_bug; bool has_vrs_ds_export_bug; bool has_taskmesh_indirect0_bug; + bool has_set_pairs_packets; /* conformant_trunc_coord is equal to TA_CNTL2.TRUNCATE_COORD_MODE, which exists since gfx11. * diff --git a/src/gallium/drivers/radeonsi/si_build_pm4.h b/src/gallium/drivers/radeonsi/si_build_pm4.h index 6dd4d3cc005..03c5edb9f7e 100644 --- a/src/gallium/drivers/radeonsi/si_build_pm4.h +++ b/src/gallium/drivers/radeonsi/si_build_pm4.h @@ -133,7 +133,7 @@ } while (0) #define radeon_set_or_push_gfx_sh_reg(reg, value) do { \ - if (GFX_VERSION >= GFX11) { \ + if (HAS_PAIRS) { \ radeon_push_gfx_sh_reg(reg, value); \ } else { \ radeon_set_sh_reg_seq(reg, 1); \ diff --git a/src/gallium/drivers/radeonsi/si_compute.c b/src/gallium/drivers/radeonsi/si_compute.c index 28aaa9de828..4166ae22694 100644 --- a/src/gallium/drivers/radeonsi/si_compute.c +++ b/src/gallium/drivers/radeonsi/si_compute.c @@ -501,7 +501,7 @@ static bool si_switch_compute_shader(struct si_context *sctx, struct si_compute radeon_add_to_buffer_list(sctx, &sctx->gfx_cs, shader->bo, RADEON_USAGE_READ | RADEON_PRIO_SHADER_BINARY); - if (sctx->gfx_level >= GFX11) { + if (sctx->screen->info.has_set_pairs_packets) { radeon_push_compute_sh_reg(R_00B830_COMPUTE_PGM_LO, shader_va >> 8); radeon_opt_push_compute_sh_reg(R_00B848_COMPUTE_PGM_RSRC1, SI_TRACKED_COMPUTE_PGM_RSRC1, config->rsrc1); @@ -529,12 +529,20 @@ static bool si_switch_compute_shader(struct si_context *sctx, struct si_compute radeon_opt_set_sh_reg(sctx, R_00B860_COMPUTE_TMPRING_SIZE, SI_TRACKED_COMPUTE_TMPRING_SIZE, tmpring_size); - if (sctx->family >= CHIP_GFX940 && !sctx->screen->info.has_graphics && shader->scratch_bo) { + if (shader->scratch_bo && + (sctx->gfx_level >= GFX11 || + (sctx->family >= CHIP_GFX940 && !sctx->screen->info.has_graphics))) { radeon_opt_set_sh_reg2(sctx, R_00B840_COMPUTE_DISPATCH_SCRATCH_BASE_LO, SI_TRACKED_COMPUTE_DISPATCH_SCRATCH_BASE_LO, sctx->compute_scratch_buffer->gpu_address >> 8, sctx->compute_scratch_buffer->gpu_address >> 40); } + + if (sctx->gfx_level >= GFX11) { + radeon_opt_set_sh_reg(sctx, R_00B8A0_COMPUTE_PGM_RSRC3, + SI_TRACKED_COMPUTE_PGM_RSRC3, + S_00B8A0_INST_PREF_SIZE(si_get_shader_prefetch_size(shader))); + } radeon_end(); } @@ -733,7 +741,7 @@ static void si_setup_nir_user_data(struct si_context *sctx, const struct pipe_gr } radeon_begin_again(cs); } else { - if (sctx->gfx_level >= GFX11) { + if (sctx->screen->info.has_set_pairs_packets) { radeon_push_compute_sh_reg(grid_size_reg, info->grid[0]); radeon_push_compute_sh_reg(grid_size_reg + 4, info->grid[1]); radeon_push_compute_sh_reg(grid_size_reg + 8, info->grid[2]); @@ -749,7 +757,7 @@ static void si_setup_nir_user_data(struct si_context *sctx, const struct pipe_gr if (sel->info.uses_variable_block_size) { uint32_t value = info->block[0] | (info->block[1] << 10) | (info->block[2] << 20); - if (sctx->gfx_level >= GFX11) { + if (sctx->screen->info.has_set_pairs_packets) { radeon_push_compute_sh_reg(block_size_reg, value); } else { radeon_set_sh_reg(block_size_reg, value); @@ -759,7 +767,7 @@ static void si_setup_nir_user_data(struct si_context *sctx, const struct pipe_gr if (sel->info.base.cs.user_data_components_amd) { unsigned num = sel->info.base.cs.user_data_components_amd; - if (sctx->gfx_level >= GFX11) { + if (sctx->screen->info.has_set_pairs_packets) { for (unsigned i = 0; i < num; i++) radeon_push_compute_sh_reg(cs_user_data_reg + i * 4, sctx->cs_user_data[i]); } else { @@ -795,7 +803,7 @@ static void si_emit_dispatch_packets(struct si_context *sctx, const struct pipe_ sctx->cs_max_waves_per_sh, threadgroups_per_cu); - if (sctx->gfx_level >= GFX11) { + if (sctx->screen->info.has_set_pairs_packets) { radeon_opt_push_compute_sh_reg(R_00B854_COMPUTE_RESOURCE_LIMITS, SI_TRACKED_COMPUTE_RESOURCE_LIMITS, compute_resource_limits); @@ -840,7 +848,7 @@ static void si_emit_dispatch_packets(struct si_context *sctx, const struct pipe_ num_threads[2] = S_00B824_NUM_THREAD_FULL(info->block[2]); } - if (sctx->gfx_level >= GFX11) { + if (sctx->screen->info.has_set_pairs_packets) { radeon_opt_push_compute_sh_reg(R_00B81C_COMPUTE_NUM_THREAD_X, SI_TRACKED_COMPUTE_NUM_THREAD_X, num_threads[0]); radeon_opt_push_compute_sh_reg(R_00B820_COMPUTE_NUM_THREAD_Y, diff --git a/src/gallium/drivers/radeonsi/si_descriptors.c b/src/gallium/drivers/radeonsi/si_descriptors.c index f738401b428..c770162bfe8 100644 --- a/src/gallium/drivers/radeonsi/si_descriptors.c +++ b/src/gallium/drivers/radeonsi/si_descriptors.c @@ -2160,7 +2160,7 @@ void si_shader_change_notify(struct si_context *sctx) if (sh_reg_base) { \ unsigned mask = sctx->shader_pointers_dirty & (pointer_mask); \ \ - if (sctx->gfx_level >= GFX11) { \ + if (sctx->screen->info.has_set_pairs_packets) { \ u_foreach_bit(i, mask) { \ struct si_descriptors *descs = &sctx->descriptors[i]; \ unsigned sh_reg = sh_reg_base + descs->shader_userdata_offset; \ @@ -2187,13 +2187,17 @@ static void si_emit_global_shader_pointers(struct si_context *sctx, struct si_de { radeon_begin(&sctx->gfx_cs); - if (sctx->gfx_level >= GFX11) { + if (sctx->screen->info.has_set_pairs_packets) { radeon_push_gfx_sh_reg(R_00B030_SPI_SHADER_USER_DATA_PS_0 + descs->shader_userdata_offset, descs->gpu_address); radeon_push_gfx_sh_reg(R_00B230_SPI_SHADER_USER_DATA_GS_0 + descs->shader_userdata_offset, descs->gpu_address); radeon_push_gfx_sh_reg(R_00B430_SPI_SHADER_USER_DATA_HS_0 + descs->shader_userdata_offset, descs->gpu_address); + } else if (sctx->gfx_level >= GFX11) { + radeon_emit_one_32bit_pointer(sctx, descs, R_00B030_SPI_SHADER_USER_DATA_PS_0); + radeon_emit_one_32bit_pointer(sctx, descs, R_00B230_SPI_SHADER_USER_DATA_GS_0); + radeon_emit_one_32bit_pointer(sctx, descs, R_00B430_SPI_SHADER_USER_DATA_HS_0); } else if (sctx->gfx_level >= GFX10) { radeon_emit_one_32bit_pointer(sctx, descs, R_00B030_SPI_SHADER_USER_DATA_PS_0); /* HW VS stage only used in non-NGG mode. */ @@ -2242,10 +2246,15 @@ void si_emit_graphics_shader_pointers(struct si_context *sctx) sh_base[PIPE_SHADER_GEOMETRY], gfx); if (sctx->gs_attribute_ring_pointer_dirty) { - assert(sctx->gfx_level >= GFX11); - radeon_push_gfx_sh_reg(R_00B230_SPI_SHADER_USER_DATA_GS_0 + - GFX9_SGPR_ATTRIBUTE_RING_ADDR * 4, - sctx->screen->attribute_ring->gpu_address); + if (sctx->screen->info.has_set_pairs_packets) { + radeon_push_gfx_sh_reg(R_00B230_SPI_SHADER_USER_DATA_GS_0 + + GFX9_SGPR_ATTRIBUTE_RING_ADDR * 4, + sctx->screen->attribute_ring->gpu_address); + } else { + radeon_set_sh_reg(R_00B230_SPI_SHADER_USER_DATA_GS_0 + + GFX9_SGPR_ATTRIBUTE_RING_ADDR * 4, + sctx->screen->attribute_ring->gpu_address); + } sctx->gs_attribute_ring_pointer_dirty = false; } radeon_end(); @@ -2270,7 +2279,7 @@ void si_emit_compute_shader_pointers(struct si_context *sctx) sctx->shader_pointers_dirty &= ~SI_DESCS_SHADER_MASK(COMPUTE); if (sctx->compute_bindless_pointer_dirty) { - if (sctx->gfx_level >= GFX11) { + if (sctx->screen->info.has_set_pairs_packets) { radeon_push_compute_sh_reg(base + sctx->bindless_descriptors.shader_userdata_offset, sctx->bindless_descriptors.gpu_address); } else { diff --git a/src/gallium/drivers/radeonsi/si_pipe.c b/src/gallium/drivers/radeonsi/si_pipe.c index fb0df328580..fb5c02c473b 100644 --- a/src/gallium/drivers/radeonsi/si_pipe.c +++ b/src/gallium/drivers/radeonsi/si_pipe.c @@ -1161,8 +1161,13 @@ static struct pipe_screen *radeonsi_screen_create_impl(struct radeon_winsys *ws, } if (sscreen->debug_flags & DBG(SHADOW_REGS) || - sscreen->info.gfx_level >= GFX11) + sscreen->info.gfx_level >= GFX11) { sscreen->info.register_shadowing_required = true; + /* Recompute has_set_pairs_packets. */ + sscreen->info.has_set_pairs_packets = sscreen->info.gfx_level >= GFX11 && + sscreen->info.register_shadowing_required && + sscreen->info.has_dedicated_vram; + } if (sscreen->debug_flags & DBG(NO_GFX)) sscreen->info.has_graphics = false; diff --git a/src/gallium/drivers/radeonsi/si_pm4.c b/src/gallium/drivers/radeonsi/si_pm4.c index a952dd1469c..a09bc7bf2dc 100644 --- a/src/gallium/drivers/radeonsi/si_pm4.c +++ b/src/gallium/drivers/radeonsi/si_pm4.c @@ -35,16 +35,13 @@ static unsigned packed_opcode_to_unpacked(unsigned opcode) static unsigned unpacked_opcode_to_packed(struct si_pm4_state *state, unsigned opcode) { - switch (opcode) { - case PKT3_SET_CONTEXT_REG: - if (state->screen->info.gfx_level >= GFX11) + if (state->screen->info.has_set_pairs_packets) { + switch (opcode) { + case PKT3_SET_CONTEXT_REG: return PKT3_SET_CONTEXT_REG_PAIRS_PACKED; - break; - case PKT3_SET_SH_REG: - if (state->screen->info.gfx_level >= GFX11 && - state->screen->info.register_shadowing_required) + case PKT3_SET_SH_REG: return PKT3_SET_SH_REG_PAIRS_PACKED; - break; + } } return opcode; diff --git a/src/gallium/drivers/radeonsi/si_state_draw.cpp b/src/gallium/drivers/radeonsi/si_state_draw.cpp index eb951f05318..3ae9d45c9c9 100644 --- a/src/gallium/drivers/radeonsi/si_state_draw.cpp +++ b/src/gallium/drivers/radeonsi/si_state_draw.cpp @@ -856,7 +856,7 @@ static void si_emit_tess_io_layout_state(struct si_context *sctx) if (!sctx->shader.tes.cso || !sctx->shader.tcs.current) return; - if (sctx->gfx_level >= GFX11) { + if (sctx->screen->info.has_set_pairs_packets) { radeon_opt_push_gfx_sh_reg(R_00B42C_SPI_SHADER_PGM_RSRC2_HS, SI_TRACKED_SPI_SHADER_PGM_RSRC2_HS, sctx->ls_hs_rsrc2); @@ -904,7 +904,7 @@ static void si_emit_tess_io_layout_state(struct si_context *sctx) /* These can't be optimized because the user data SGPRs may have different meaning * without tessellation. (they are VS and ES/GS user data SGPRs) */ - if (sctx->gfx_level >= GFX11) { + if (sctx->screen->info.has_set_pairs_packets) { radeon_push_gfx_sh_reg(tes_sh_base + SI_SGPR_TES_OFFCHIP_LAYOUT * 4, sctx->tcs_offchip_layout); radeon_push_gfx_sh_reg(tes_sh_base + SI_SGPR_TES_OFFCHIP_ADDR * 4, @@ -1107,6 +1107,11 @@ enum si_is_draw_vertex_state { DRAW_VERTEX_STATE_ON, }; +enum si_has_pairs { + HAS_PAIRS_OFF, + HAS_PAIRS_ON, +}; + template ALWAYS_INLINE static bool num_instanced_prims_less_than(const struct pipe_draw_indirect_info *indirect, enum mesa_prim prim, @@ -1223,7 +1228,7 @@ static void si_emit_rasterizer_prim_state(struct si_context *sctx) } template ALWAYS_INLINE + si_is_draw_vertex_state IS_DRAW_VERTEX_STATE, si_has_pairs HAS_PAIRS> ALWAYS_INLINE static void si_emit_vs_state(struct si_context *sctx, unsigned index_size) { if (!IS_DRAW_VERTEX_STATE && sctx->num_vs_blit_sgprs) { @@ -1538,8 +1543,8 @@ void gfx11_emit_buffered_compute_sh_regs(struct si_context *sctx) } \ } while (0) -template -ALWAYS_INLINE +template ALWAYS_INLINE static void si_emit_draw_packets(struct si_context *sctx, const struct pipe_draw_info *info, unsigned drawid_base, const struct pipe_draw_indirect_info *indirect, @@ -1679,7 +1684,7 @@ static void si_emit_draw_packets(struct si_context *sctx, const struct pipe_draw assert(indirect_va % 8 == 0); - if (GFX_VERSION >= GFX11) { + if (HAS_PAIRS) { radeon_end(); gfx11_emit_buffered_sh_regs_inline(sctx, &sctx->num_buffered_gfx_sh_regs, sctx->buffered_gfx_sh_regs); @@ -1759,7 +1764,7 @@ static void si_emit_draw_packets(struct si_context *sctx, const struct pipe_draw if (!is_blit) { /* Prefer SET_SH_REG_PAIRS_PACKED* on Gfx11+. */ - if (GFX_VERSION >= GFX11) { + if (HAS_PAIRS) { if (base_vertex != sctx->last_base_vertex || sctx->last_base_vertex == SI_BASE_VERTEX_UNKNOWN) { radeon_push_gfx_sh_reg(sh_base_reg + SI_SGPR_BASE_VERTEX * 4, base_vertex); @@ -1812,7 +1817,7 @@ static void si_emit_draw_packets(struct si_context *sctx, const struct pipe_draw } } - if (GFX_VERSION >= GFX11) { + if (HAS_PAIRS) { radeon_end(); gfx11_emit_buffered_sh_regs_inline(sctx, &sctx->num_buffered_gfx_sh_regs, sctx->buffered_gfx_sh_regs); @@ -2066,7 +2071,8 @@ static unsigned get_vb_descriptor_sgpr_ptr_offset(void) } template ALWAYS_INLINE + si_is_draw_vertex_state IS_DRAW_VERTEX_STATE, si_has_pairs HAS_PAIRS, + util_popcnt POPCNT> ALWAYS_INLINE static bool si_upload_and_prefetch_VB_descriptors(struct si_context *sctx, struct pipe_vertex_state *state, uint32_t partial_velem_mask) @@ -2313,7 +2319,8 @@ static void si_emit_all_states(struct si_context *sctx, unsigned skip_atom_mask) } while (0) template ALWAYS_INLINE + si_is_draw_vertex_state IS_DRAW_VERTEX_STATE, si_has_pairs HAS_PAIRS, + util_popcnt POPCNT> ALWAYS_INLINE static void si_draw(struct pipe_context *ctx, const struct pipe_draw_info *info, unsigned drawid_offset, @@ -2606,7 +2613,8 @@ static void si_draw(struct pipe_context *ctx, si_emit_all_states(sctx, masked_atoms); /* Emit draw states. */ - si_emit_vs_state(sctx, index_size); + si_emit_vs_state + (sctx, index_size); si_emit_draw_registers (sctx, indirect, prim, index_size, instance_count, primitive_restart, info->restart_index, min_direct_count); @@ -2635,13 +2643,13 @@ static void si_draw(struct pipe_context *ctx, * It should done after cache flushing. */ if (unlikely((!si_upload_and_prefetch_VB_descriptors - + (sctx, state, partial_velem_mask)))) { DRAW_CLEANUP; return; } - si_emit_draw_packets + si_emit_draw_packets (sctx, info, drawid_offset, indirect, draws, num_draws, indexbuf, index_size, index_offset, instance_count); /* <-- CUs start to get busy here if we waited. */ @@ -2686,7 +2694,8 @@ static void si_draw(struct pipe_context *ctx, DRAW_CLEANUP; } -template +template static void si_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *info, unsigned drawid_offset, @@ -2694,12 +2703,12 @@ static void si_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_start_count_bias *draws, unsigned num_draws) { - si_draw + si_draw (ctx, info, drawid_offset, indirect, draws, num_draws, NULL, 0); } template + si_has_pairs HAS_PAIRS, util_popcnt POPCNT> static void si_draw_vertex_state(struct pipe_context *ctx, struct pipe_vertex_state *vstate, uint32_t partial_velem_mask, @@ -2715,7 +2724,7 @@ static void si_draw_vertex_state(struct pipe_context *ctx, dinfo.instance_count = 1; dinfo.index.resource = state->b.input.indexbuf; - si_draw + si_draw (ctx, &dinfo, 0, NULL, draws, num_draws, vstate, partial_velem_mask); if (info.take_vertex_state_ownership) @@ -2773,15 +2782,28 @@ static void si_init_draw_vbo(struct si_context *sctx) if (!NGG && GFX_VERSION >= GFX11) return; - sctx->draw_vbo[HAS_TESS][HAS_GS][NGG] = - si_draw_vbo; + if (GFX_VERSION >= GFX11 && sctx->screen->info.has_set_pairs_packets) { + sctx->draw_vbo[HAS_TESS][HAS_GS][NGG] = + si_draw_vbo; - if (util_get_cpu_caps()->has_popcnt) { - sctx->draw_vertex_state[HAS_TESS][HAS_GS][NGG] = - si_draw_vertex_state; + if (util_get_cpu_caps()->has_popcnt) { + sctx->draw_vertex_state[HAS_TESS][HAS_GS][NGG] = + si_draw_vertex_state; + } else { + sctx->draw_vertex_state[HAS_TESS][HAS_GS][NGG] = + si_draw_vertex_state; + } } else { - sctx->draw_vertex_state[HAS_TESS][HAS_GS][NGG] = - si_draw_vertex_state; + sctx->draw_vbo[HAS_TESS][HAS_GS][NGG] = + si_draw_vbo; + + if (util_get_cpu_caps()->has_popcnt) { + sctx->draw_vertex_state[HAS_TESS][HAS_GS][NGG] = + si_draw_vertex_state; + } else { + sctx->draw_vertex_state[HAS_TESS][HAS_GS][NGG] = + si_draw_vertex_state; + } } } diff --git a/src/gallium/drivers/radeonsi/si_state_shaders.cpp b/src/gallium/drivers/radeonsi/si_state_shaders.cpp index 79a5f08508b..78ab64bfdf3 100644 --- a/src/gallium/drivers/radeonsi/si_state_shaders.cpp +++ b/src/gallium/drivers/radeonsi/si_state_shaders.cpp @@ -1187,7 +1187,7 @@ static void gfx10_emit_shader_ngg_tail(struct si_context *sctx, struct si_shader radeon_begin_again(&sctx->gfx_cs); radeon_opt_set_uconfig_reg(sctx, R_030980_GE_PC_ALLOC, SI_TRACKED_GE_PC_ALLOC, shader->ngg.ge_pc_alloc); - if (sctx->gfx_level >= GFX11) { + if (sctx->screen->info.has_set_pairs_packets) { assert(!sctx->screen->info.uses_kernel_cu_mask); radeon_opt_push_gfx_sh_reg(R_00B21C_SPI_SHADER_PGM_RSRC3_GS, SI_TRACKED_SPI_SHADER_PGM_RSRC3_GS, diff --git a/src/gallium/drivers/radeonsi/si_state_viewport.c b/src/gallium/drivers/radeonsi/si_state_viewport.c index d924c69f026..412ee47a3ed 100644 --- a/src/gallium/drivers/radeonsi/si_state_viewport.c +++ b/src/gallium/drivers/radeonsi/si_state_viewport.c @@ -91,7 +91,7 @@ static void si_emit_cull_state(struct si_context *sctx) radeon_add_to_buffer_list(sctx, &sctx->gfx_cs, sctx->small_prim_cull_info_buf, RADEON_USAGE_READ | RADEON_PRIO_CONST_BUFFER); - if (sctx->gfx_level >= GFX11) { + if (sctx->screen->info.has_set_pairs_packets) { radeon_push_gfx_sh_reg(R_00B230_SPI_SHADER_USER_DATA_GS_0 + GFX9_SGPR_SMALL_PRIM_CULL_INFO * 4, sctx->small_prim_cull_info_address);