diff --git a/docs/envvars.rst b/docs/envvars.rst index 7b9f5ea5277..b95484714e6 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -1402,6 +1402,9 @@ RADV driver environment variables dump vertex shader prologs ``psocachestats`` dump PSO cache stats (hits/misses) to verify precompilation of shaders + ``pso_history`` + dump PSO history (pipeline hash + shader VA) to /tmp/radv_pso_history.log. + Useful for debugging GPU hangs with UMR and Fossilize. ``shaders`` dump shaders ``shaderstats`` diff --git a/src/amd/vulkan/radv_debug.h b/src/amd/vulkan/radv_debug.h index c1458272442..2379a1fc364 100644 --- a/src/amd/vulkan/radv_debug.h +++ b/src/amd/vulkan/radv_debug.h @@ -70,6 +70,7 @@ enum { RADV_DEBUG_DUMP_NIR = 1ull << 55, RADV_DEBUG_DUMP_ASM = 1ull << 56, RADV_DEBUG_DUMP_BACKEND_IR = 1ull << 57, + RADV_DEBUG_PSO_HISTORY = 1ull << 58, RADV_DEBUG_DUMP_SHADERS = RADV_DEBUG_DUMP_VS | RADV_DEBUG_DUMP_TCS | RADV_DEBUG_DUMP_TES | RADV_DEBUG_DUMP_GS | RADV_DEBUG_DUMP_PS | RADV_DEBUG_DUMP_TASK | RADV_DEBUG_DUMP_MESH | RADV_DEBUG_DUMP_CS | RADV_DEBUG_DUMP_NIR | RADV_DEBUG_DUMP_ASM | RADV_DEBUG_DUMP_BACKEND_IR, diff --git a/src/amd/vulkan/radv_instance.c b/src/amd/vulkan/radv_instance.c index 05f9bbfe455..8104a18d836 100644 --- a/src/amd/vulkan/radv_instance.c +++ b/src/amd/vulkan/radv_instance.c @@ -85,6 +85,7 @@ static const struct debug_control radv_debug_options[] = {{"nofastclears", RADV_ {"nir", RADV_DEBUG_DUMP_NIR}, {"asm", RADV_DEBUG_DUMP_ASM}, {"ir", RADV_DEBUG_DUMP_BACKEND_IR}, + {"pso_history", RADV_DEBUG_PSO_HISTORY}, {NULL, 0}}; const char * @@ -400,6 +401,14 @@ radv_CreateInstance(const VkInstanceCreateInfo *pCreateInfo, const VkAllocationC instance->debug_flags |= shader_stage_flags; } + if (instance->debug_flags & RADV_DEBUG_PSO_HISTORY) { + const char *filename = "/tmp/radv_pso_history.log"; + + instance->pso_history_logfile = fopen(filename, "w"); + if (!instance->pso_history_logfile) + fprintf(stderr, "radv: Failed to open log file: %s.\n", filename); + } + /* When RADV_FORCE_FAMILY is set, the driver creates a null * device that allows to test the compiler without having an * AMDGPU instance. @@ -433,6 +442,9 @@ radv_DestroyInstance(VkInstance _instance, const VkAllocationCallbacks *pAllocat VG(VALGRIND_DESTROY_MEMPOOL(instance)); + if (instance->pso_history_logfile) + fclose(instance->pso_history_logfile); + simple_mtx_destroy(&instance->shader_dump_mtx); driDestroyOptionCache(&instance->drirc.options); diff --git a/src/amd/vulkan/radv_instance.h b/src/amd/vulkan/radv_instance.h index f8535e050cb..162abf2febd 100644 --- a/src/amd/vulkan/radv_instance.h +++ b/src/amd/vulkan/radv_instance.h @@ -80,6 +80,8 @@ struct radv_instance { int override_vram_size; int override_uniform_offset_alignment; } drirc; + + FILE *pso_history_logfile; }; VK_DEFINE_HANDLE_CASTS(radv_instance, vk.base, VkInstance, VK_OBJECT_TYPE_INSTANCE) diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c index a90100fdb2e..795b5de2a5a 100644 --- a/src/amd/vulkan/radv_pipeline.c +++ b/src/amd/vulkan/radv_pipeline.c @@ -61,8 +61,8 @@ radv_pipeline_capture_shader_stats(const struct radv_device *device, VkPipelineC /* Capture shader statistics when RGP is enabled to correlate shader hashes with Fossilize. */ return (flags & VK_PIPELINE_CREATE_2_CAPTURE_STATISTICS_BIT_KHR) || - (instance->debug_flags & RADV_DEBUG_DUMP_SHADER_STATS) || device->keep_shader_info || - (instance->vk.trace_mode & RADV_TRACE_MODE_RGP); + (instance->debug_flags & (RADV_DEBUG_DUMP_SHADER_STATS | RADV_DEBUG_PSO_HISTORY)) || + device->keep_shader_info || (instance->vk.trace_mode & RADV_TRACE_MODE_RGP); } bool @@ -1238,3 +1238,61 @@ radv_pipeline_hash_shader_stage(VkPipelineCreateFlags2 pipeline_flags, const VkP _mesa_sha1_update(ctx, shader_sha1, sizeof(shader_sha1)); _mesa_sha1_update(ctx, stage_key, sizeof(*stage_key)); } + +static void +radv_print_pso_history(const struct radv_pipeline *pipeline, const struct radv_shader *shader, FILE *output) +{ + const uint64_t start_addr = radv_shader_get_va(shader) & ((1ull << 48) - 1); + const uint64_t end_addr = start_addr + shader->code_size; + + fprintf(output, "pipeline_hash=%.16llx, VA=%.16llx-%.16llx, stage=%s\n", (long long)pipeline->pipeline_hash, + (long long)start_addr, (long long)end_addr, _mesa_shader_stage_to_string(shader->info.stage)); + fflush(output); +} + +void +radv_pipeline_report_pso_history(const struct radv_device *device, struct radv_pipeline *pipeline) +{ + const struct radv_physical_device *pdev = radv_device_physical(device); + const struct radv_instance *instance = radv_physical_device_instance(pdev); + FILE *output = instance->pso_history_logfile ? instance->pso_history_logfile : stderr; + + if (!(instance->debug_flags & RADV_DEBUG_PSO_HISTORY)) + return; + + /* Only report PSO history for application pipelines. */ + if (pipeline->is_internal) + return; + + switch (pipeline->type) { + case RADV_PIPELINE_GRAPHICS: + for (uint32_t i = 0; i < MESA_VULKAN_SHADER_STAGES; i++) { + const struct radv_shader *shader = pipeline->shaders[i]; + + if (shader) + radv_print_pso_history(pipeline, shader, output); + } + + if (pipeline->gs_copy_shader) + radv_print_pso_history(pipeline, pipeline->gs_copy_shader, output); + break; + case RADV_PIPELINE_COMPUTE: + radv_print_pso_history(pipeline, pipeline->shaders[MESA_SHADER_COMPUTE], output); + break; + case RADV_PIPELINE_RAY_TRACING: { + struct radv_ray_tracing_pipeline *rt_pipeline = radv_pipeline_to_ray_tracing(pipeline); + + radv_print_pso_history(pipeline, rt_pipeline->prolog, output); + + for (uint32_t i = 0; i < rt_pipeline->stage_count; i++) { + const struct radv_shader *shader = rt_pipeline->stages[i].shader; + + if (shader) + radv_print_pso_history(pipeline, shader, output); + } + break; + } + default: + break; + } +} diff --git a/src/amd/vulkan/radv_pipeline.h b/src/amd/vulkan/radv_pipeline.h index c0cfc997bcc..f8d96a72cc8 100644 --- a/src/amd/vulkan/radv_pipeline.h +++ b/src/amd/vulkan/radv_pipeline.h @@ -110,4 +110,6 @@ void radv_pipeline_hash_shader_stage(VkPipelineCreateFlags2 pipeline_flags, const VkPipelineShaderStageCreateInfo *sinfo, const struct radv_shader_stage_key *stage_key, struct mesa_sha1 *ctx); +void radv_pipeline_report_pso_history(const struct radv_device *device, struct radv_pipeline *pipeline); + #endif /* RADV_PIPELINE_H */ diff --git a/src/amd/vulkan/radv_pipeline_compute.c b/src/amd/vulkan/radv_pipeline_compute.c index 67630a5fc0d..9863a6c7bb8 100644 --- a/src/amd/vulkan/radv_pipeline_compute.c +++ b/src/amd/vulkan/radv_pipeline_compute.c @@ -307,6 +307,8 @@ radv_compute_pipeline_create(VkDevice _device, VkPipelineCache _cache, const VkC radv_compute_pipeline_init(pipeline, pipeline_layout, pipeline->base.shaders[MESA_SHADER_COMPUTE]); + radv_pipeline_report_pso_history(device, &pipeline->base); + *pPipeline = radv_pipeline_to_handle(&pipeline->base); radv_rmv_log_compute_pipeline_create(device, &pipeline->base, pipeline->base.is_internal); return VK_SUCCESS; diff --git a/src/amd/vulkan/radv_pipeline_graphics.c b/src/amd/vulkan/radv_pipeline_graphics.c index 74f1379a350..0f0444fe924 100644 --- a/src/amd/vulkan/radv_pipeline_graphics.c +++ b/src/amd/vulkan/radv_pipeline_graphics.c @@ -3484,6 +3484,8 @@ radv_graphics_pipeline_create(VkDevice _device, VkPipelineCache _cache, const Vk return result; } + radv_pipeline_report_pso_history(device, &pipeline->base); + *pPipeline = radv_pipeline_to_handle(&pipeline->base); radv_rmv_log_graphics_pipeline_create(device, &pipeline->base, pipeline->base.is_internal); return VK_SUCCESS; @@ -3592,6 +3594,8 @@ radv_graphics_lib_pipeline_create(VkDevice _device, VkPipelineCache _cache, return result; } + radv_pipeline_report_pso_history(device, &pipeline->base.base); + *pPipeline = radv_pipeline_to_handle(&pipeline->base.base); return VK_SUCCESS; diff --git a/src/amd/vulkan/radv_pipeline_rt.c b/src/amd/vulkan/radv_pipeline_rt.c index bcda4aae552..e906494f827 100644 --- a/src/amd/vulkan/radv_pipeline_rt.c +++ b/src/amd/vulkan/radv_pipeline_rt.c @@ -1157,6 +1157,8 @@ radv_rt_pipeline_create(VkDevice _device, VkPipelineCache _cache, const VkRayTra } } + radv_pipeline_report_pso_history(device, &pipeline->base.base); + *pPipeline = radv_pipeline_to_handle(&pipeline->base.base); radv_rmv_log_rt_pipeline_create(device, pipeline);