diff --git a/src/amd/compiler/aco_instruction_selection.cpp b/src/amd/compiler/aco_instruction_selection.cpp index 7385f410ae7..bd70c79fbe3 100644 --- a/src/amd/compiler/aco_instruction_selection.cpp +++ b/src/amd/compiler/aco_instruction_selection.cpp @@ -12292,6 +12292,91 @@ restore_vgprs_from_mem(isel_context* ctx, Operand rsrc) save_or_restore_vgprs(ctx, rsrc, false); } +void +dump_vgprs_to_mem(isel_context* ctx, Builder& bld, Operand rsrc) +{ + const uint32_t ttmp0_idx = ctx->program->gfx_level >= GFX9 ? 108 : 112; + const uint32_t base_offset = offsetof(struct aco_trap_handler_layout, vgprs[0]); + + ac_hw_cache_flags cache_glc; + cache_glc.value = ac_glc; + + PhysReg num_vgprs{ttmp0_idx + 2}; + PhysReg soffset{ttmp0_idx + 3}; + + enable_thread_indexing(ctx, rsrc); + + /* Determine the number of vgprs to dump in a 4-VGPR granularity. */ + const uint32_t vgpr_size_offset = ctx->program->gfx_level >= GFX11 ? 12 : 8; + const uint32_t vgpr_size_width = ctx->program->gfx_level >= GFX10 ? 8 : 6; + + bld.sopk(aco_opcode::s_getreg_b32, Definition(num_vgprs, s1), + ((32 - 1) << 11) | 5 /* GPR_ALLOC */); + bld.sop2(aco_opcode::s_bfe_u32, Definition(num_vgprs, s1), bld.def(s1, scc), + Operand(num_vgprs, s1), Operand::c32((vgpr_size_width << 16) | vgpr_size_offset)); + bld.sop2(aco_opcode::s_add_u32, Definition(num_vgprs, s1), bld.def(s1, scc), + Operand(num_vgprs, s1), Operand::c32(1u)); + bld.sop2(aco_opcode::s_lshl_b32, Definition(num_vgprs, s1), bld.def(s1, scc), + Operand(num_vgprs, s1), Operand::c32(2u)); + bld.sop2(aco_opcode::s_mul_i32, Definition(num_vgprs, s1), Operand::c32(256), + Operand(num_vgprs, s1)); + + /* Initialize m0/soffset to zero. */ + bld.copy(Definition(m0, s1), Operand::c32(0u)); + bld.copy(Definition(soffset, s1), Operand::c32(0u)); + + if (ctx->program->gfx_level < GFX10) { + /* Enable VGPR indexing with m0 as source index. */ + bld.sopc(aco_opcode::s_set_gpr_idx_on, Definition(m0, s1), Operand(m0, s1), + Operand(PhysReg{1}, s1) /* SRC0 mode */); + } + + loop_context lc; + begin_loop(ctx, &lc); + { + bld.reset(ctx->block); + + /* Move from a relative source addr (v0 = v[0 + m0]). */ + if (ctx->program->gfx_level >= GFX10) { + bld.vop1(aco_opcode::v_movrels_b32, Definition(PhysReg{256}, v1), + Operand(PhysReg{256}, v1), Operand(m0, s1)); + } else { + bld.vop1(aco_opcode::v_mov_b32, Definition(PhysReg{256}, v1), Operand(PhysReg{256}, v1)); + } + + bld.mubuf(aco_opcode::buffer_store_dword, Operand(rsrc), Operand(v1), + Operand(PhysReg{soffset}, s1), Operand(PhysReg{256}, v1) /* v0 */, base_offset, + false /* offen */, false /* idxen */, + /* addr64 */ false, /* disable_wqm */ false, cache_glc); + + /* Increase m0 and the offset assuming it's wave64. */ + bld.sop2(aco_opcode::s_add_u32, Definition(m0, s1), bld.def(s1, scc), Operand(m0, s1), + Operand::c32(1u)); + bld.sop2(aco_opcode::s_add_u32, Definition(soffset, s1), bld.def(s1, scc), + Operand(soffset, s1), Operand::c32(256u)); + + const Temp cond = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), Operand(soffset, s1), + Operand(num_vgprs, s1)); + + if_context loop_break; + begin_uniform_if_then(ctx, &loop_break, cond); + { + emit_loop_break(ctx); + } + begin_uniform_if_else(ctx, &loop_break); + end_uniform_if(ctx, &loop_break); + } + end_loop(ctx, &lc); + bld.reset(ctx->block); + + if (ctx->program->gfx_level < GFX10) { + /* Disable VGPR indexing. */ + bld.sopp(aco_opcode::s_set_gpr_idx_off); + } + + disable_thread_indexing(ctx, rsrc); +} + void select_trap_handler_shader(Program* program, ac_shader_config* config, const struct aco_compiler_options* options, @@ -12373,6 +12458,9 @@ select_trap_handler_shader(Program* program, ac_shader_config* config, /* Save VGPRS that needs to be restored. */ save_vgprs_to_mem(&ctx, Operand(tma_rsrc, s4)); + /* Dump VGPRs. */ + dump_vgprs_to_mem(&ctx, bld, Operand(tma_rsrc, s4)); + /* Store TTMP0-TTMP1. */ bld.copy(Definition(PhysReg{256}, v2) /* v[0-1] */, Operand(ttmp0_reg, s2)); @@ -12388,6 +12476,9 @@ select_trap_handler_shader(Program* program, ac_shader_config* config, /* Save VGPRS that needs to be restored. */ save_vgprs_to_mem(&ctx, Operand(tma_rsrc, s4)); + /* Dump VGPRs. */ + dump_vgprs_to_mem(&ctx, bld, Operand(tma_rsrc, s4)); + /* Store TTMP0-TTMP1. */ bld.smem(aco_opcode::s_buffer_store_dwordx2, Operand(tma_rsrc, s4), Operand::c32(offset), Operand(ttmp0_reg, s2), memory_sync_info(), cache_glc); diff --git a/src/amd/compiler/aco_interface.cpp b/src/amd/compiler/aco_interface.cpp index 2551ce92a3f..913a000729f 100644 --- a/src/amd/compiler/aco_interface.cpp +++ b/src/amd/compiler/aco_interface.cpp @@ -420,6 +420,9 @@ aco_compile_trap_handler(const struct aco_compiler_options* options, aco_print_program(program.get(), stderr); validate(program.get()); + insert_exec_mask(program.get()); + validate(program.get()); + lower_to_hw_instr(program.get()); validate(program.get()); diff --git a/src/amd/compiler/aco_shader_info.h b/src/amd/compiler/aco_shader_info.h index 0085aba022e..228547ad128 100644 --- a/src/amd/compiler/aco_shader_info.h +++ b/src/amd/compiler/aco_shader_info.h @@ -197,6 +197,7 @@ struct aco_symbol { }; #define MAX_SGPRS 108 +#define MAX_VGPRS 256 #define NUM_SAVED_VGPRS 2 struct aco_trap_handler_layout { @@ -219,6 +220,7 @@ struct aco_trap_handler_layout { uint32_t exec_lo; uint32_t exec_hi; uint32_t sgprs[MAX_SGPRS]; + uint32_t vgprs[MAX_VGPRS * 64]; }; #ifdef __cplusplus diff --git a/src/amd/vulkan/radv_debug.c b/src/amd/vulkan/radv_debug.c index 90b8934cf32..1bec11b77f4 100644 --- a/src/amd/vulkan/radv_debug.c +++ b/src/amd/vulkan/radv_debug.c @@ -1039,8 +1039,25 @@ radv_dump_sq_hw_regs(struct radv_device *device, const struct aco_trap_handler_l fprintf(f, "\n\n"); } +static uint32_t +radv_get_vgpr_size(const struct radv_device *device, const struct aco_trap_handler_layout *layout) +{ + const struct radv_physical_device *pdev = radv_device_physical(device); + uint32_t vgpr_size; + + if (pdev->info.gfx_level >= GFX11) { + vgpr_size = G_000414_VGPR_SIZE_GFX11(layout->sq_wave_regs.gpr_alloc); + } else if (pdev->info.gfx_level >= GFX10) { + vgpr_size = G_000414_VGPR_SIZE_GFX10(layout->sq_wave_regs.gpr_alloc); + } else { + vgpr_size = G_000054_VGPR_SIZE_GFX6(layout->sq_wave_regs.gpr_alloc); + } + + return vgpr_size; +} + static void -radv_dump_shader_regs(const struct aco_trap_handler_layout *layout, FILE *f) +radv_dump_shader_regs(const struct radv_device *device, const struct aco_trap_handler_layout *layout, FILE *f) { fprintf(f, "\nShader registers:\n"); @@ -1054,6 +1071,28 @@ radv_dump_shader_regs(const struct aco_trap_handler_layout *layout, FILE *f) layout->sgprs[i + 2], layout->sgprs[i + 3]); } fprintf(f, "\n\n"); + + const uint32_t vgpr_size = radv_get_vgpr_size(device, layout); + const uint32_t num_vgprs = (vgpr_size + 1) * 4 /* 4-VGPR granularity */; + + assert(num_vgprs < MAX_VGPRS); + + fprintf(f, "VGPRS:\n"); + fprintf(f, " "); + for (uint32_t i = 0; i < 64; i++) { + fprintf(f, " t%02u ", i); + } + fprintf(f, "\n"); + for (uint32_t i = 0; i < num_vgprs; i++) { + fprintf(f, " [%3u] = {", i); + + for (uint32_t j = 0; j < 64; j++) { + fprintf(f, " %08x", layout->vgprs[i * 64 + j]); + } + fprintf(f, " }\n"); + } + + fprintf(f, "\n\n"); } void @@ -1099,7 +1138,7 @@ radv_check_trap_handler(struct radv_queue *queue) #endif radv_dump_sq_hw_regs(device, layout, f); - radv_dump_shader_regs(layout, f); + radv_dump_shader_regs(device, layout, f); uint32_t ttmp0 = layout->ttmp0; uint32_t ttmp1 = layout->ttmp1;