diff --git a/src/amd/common/ac_rgp.h b/src/amd/common/ac_rgp.h index 694aade80db..51799ce061c 100644 --- a/src/amd/common/ac_rgp.h +++ b/src/amd/common/ac_rgp.h @@ -62,7 +62,7 @@ struct rgp_shader_data { struct rgp_code_object_record { uint32_t shader_stages_mask; - struct rgp_shader_data shader_data[MESA_SHADER_STAGES]; + struct rgp_shader_data shader_data[MESA_VULKAN_SHADER_STAGES]; uint32_t num_shaders_combined; /* count combined shaders as one count */ uint64_t pipeline_hash[2]; struct list_head list; diff --git a/src/amd/vulkan/layers/radv_sqtt_layer.c b/src/amd/vulkan/layers/radv_sqtt_layer.c index 9917d829164..05ea8b831f8 100644 --- a/src/amd/vulkan/layers/radv_sqtt_layer.c +++ b/src/amd/vulkan/layers/radv_sqtt_layer.c @@ -869,7 +869,7 @@ radv_add_code_object(struct radv_device *device, struct radv_pipeline *pipeline) record->pipeline_hash[0] = pipeline->pipeline_hash; record->pipeline_hash[1] = pipeline->pipeline_hash; - for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) { + for (unsigned i = 0; i < MESA_VULKAN_SHADER_STAGES; i++) { struct radv_shader *shader = pipeline->shaders[i]; uint8_t *code; uint64_t va; @@ -922,7 +922,7 @@ radv_register_pipeline(struct radv_device *device, struct radv_pipeline *pipelin return VK_ERROR_OUT_OF_HOST_MEMORY; /* Find the lowest shader BO VA. */ - for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) { + for (unsigned i = 0; i < MESA_VULKAN_SHADER_STAGES; i++) { struct radv_shader *shader = pipeline->shaders[i]; uint64_t va; diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c index fef41c4d745..e6e209f79a0 100644 --- a/src/amd/vulkan/radv_cmd_buffer.c +++ b/src/amd/vulkan/radv_cmd_buffer.c @@ -3141,7 +3141,7 @@ radv_flush_descriptors(struct radv_cmd_buffer *cmd_buffer, VkShaderStageFlags st radv_flush_indirect_descriptor_sets(cmd_buffer, pipeline, bind_point); ASSERTED unsigned cdw_max = - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, MAX_SETS * MESA_SHADER_STAGES * 4); + radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, MAX_SETS * MESA_VULKAN_SHADER_STAGES * 4); if (pipeline) { if (stages & VK_SHADER_STAGE_COMPUTE_BIT) { @@ -3236,7 +3236,7 @@ radv_flush_constants(struct radv_cmd_buffer *cmd_buffer, VkShaderStageFlags stag va += offset; ASSERTED unsigned cdw_max = - radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, MESA_SHADER_STAGES * 4); + radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, MESA_VULKAN_SHADER_STAGES * 4); prev_shader = NULL; radv_foreach_stage(stage, internal_stages) @@ -3457,7 +3457,7 @@ radv_emit_streamout_buffers(struct radv_cmd_buffer *cmd_buffer, uint64_t va) struct radv_userdata_info *loc; uint32_t base_reg; - for (unsigned stage = 0; stage < MESA_SHADER_STAGES; ++stage) { + for (unsigned stage = 0; stage < MESA_VULKAN_SHADER_STAGES; ++stage) { if (!radv_get_shader(pipeline, stage)) continue; @@ -5905,7 +5905,7 @@ static void radv_emit_view_index(struct radv_cmd_buffer *cmd_buffer, unsigned index) { struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; - for (unsigned stage = 0; stage < MESA_SHADER_STAGES; ++stage) { + for (unsigned stage = 0; stage < MESA_VULKAN_SHADER_STAGES; ++stage) { if (!radv_get_shader(pipeline, stage)) continue; diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c index 375a734ba70..e6994cc8539 100644 --- a/src/amd/vulkan/radv_pipeline.c +++ b/src/amd/vulkan/radv_pipeline.c @@ -183,7 +183,7 @@ radv_pipeline_destroy(struct radv_device *device, struct radv_pipeline *pipeline free(pipeline->library.stages); } - for (unsigned i = 0; i < MESA_SHADER_STAGES; ++i) + for (unsigned i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) if (pipeline->shaders[i]) radv_shader_destroy(device, pipeline->shaders[i]); @@ -242,7 +242,7 @@ radv_pipeline_init_scratch(const struct radv_device *device, struct radv_pipelin unsigned scratch_bytes_per_wave = 0; unsigned max_waves = 0; - for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { if (pipeline->shaders[i] && pipeline->shaders[i]->config.scratch_bytes_per_wave) { unsigned max_stage_waves = device->scratch_waves; @@ -2332,7 +2332,7 @@ radv_link_shaders(struct radv_pipeline *pipeline, nir_shader **shaders, bool optimize_conservatively) { - nir_shader *ordered_shaders[MESA_SHADER_STAGES]; + nir_shader *ordered_shaders[MESA_VULKAN_SHADER_STAGES]; int shader_count = 0; if (shaders[MESA_SHADER_FRAGMENT]) { @@ -2503,7 +2503,7 @@ radv_link_shaders(struct radv_pipeline *pipeline, static void radv_set_driver_locations(struct radv_pipeline *pipeline, nir_shader **shaders, - struct radv_shader_info infos[MESA_SHADER_STAGES]) + struct radv_shader_info infos[MESA_VULKAN_SHADER_STAGES]) { if (shaders[MESA_SHADER_FRAGMENT]) { nir_foreach_shader_out_variable(var, shaders[MESA_SHADER_FRAGMENT]) @@ -2833,7 +2833,7 @@ radv_fill_shader_info(struct radv_pipeline *pipeline, unsigned active_stages = 0; unsigned filled_stages = 0; - for (int i = 0; i < MESA_SHADER_STAGES; i++) { + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; i++) { if (nir[i]) active_stages |= (1 << i); } @@ -3007,7 +3007,7 @@ radv_fill_shader_info(struct radv_pipeline *pipeline, infos[MESA_SHADER_COMPUTE].cs.subgroup_size = subgroup_size; } - for (int i = 0; i < MESA_SHADER_STAGES; i++) { + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; i++) { if (nir[i]) { infos[i].wave_size = radv_get_wave_size(pipeline->device, pStages[i], i, &infos[i]); infos[i].ballot_bit_size = @@ -3364,13 +3364,13 @@ radv_create_shaders(struct radv_pipeline *pipeline, struct radv_pipeline_layout VkPipelineCreationFeedbackEXT **stage_feedbacks) { struct vk_shader_module fs_m = {0}; - struct vk_shader_module *modules[MESA_SHADER_STAGES] = { + struct vk_shader_module *modules[MESA_VULKAN_SHADER_STAGES] = { 0, }; - nir_shader *nir[MESA_SHADER_STAGES] = {0}; - struct radv_shader_binary *binaries[MESA_SHADER_STAGES] = {NULL}; + nir_shader *nir[MESA_VULKAN_SHADER_STAGES] = {0}; + struct radv_shader_binary *binaries[MESA_VULKAN_SHADER_STAGES] = {NULL}; struct radv_shader_binary *gs_copy_binary = NULL; - struct radv_shader_info infos[MESA_SHADER_STAGES] = {0}; + struct radv_shader_info infos[MESA_VULKAN_SHADER_STAGES] = {0}; unsigned char hash[20]; bool keep_executable_info = (flags & VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR) || @@ -3384,7 +3384,7 @@ radv_create_shaders(struct radv_pipeline *pipeline, struct radv_pipeline_layout radv_start_feedback(pipeline_feedback); - for (unsigned i = 0; i < MESA_SHADER_STAGES; ++i) { + for (unsigned i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { if (pStages[i]) { modules[i] = vk_shader_module_from_handle(pStages[i]->module); if (modules[i]->nir) @@ -3431,7 +3431,7 @@ radv_create_shaders(struct radv_pipeline *pipeline, struct radv_pipeline_layout modules[MESA_SHADER_FRAGMENT] = &fs_m; } - for (unsigned i = 0; i < MESA_SHADER_STAGES; ++i) { + for (unsigned i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { const VkPipelineShaderStageCreateInfo *stage = pStages[i]; if (!modules[i]) @@ -3458,7 +3458,7 @@ radv_create_shaders(struct radv_pipeline *pipeline, struct radv_pipeline_layout radv_link_shaders(pipeline, pipeline_key, nir, optimize_conservatively); radv_set_driver_locations(pipeline, nir, infos); - for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { if (nir[i]) { radv_start_feedback(stage_feedbacks[i]); radv_optimize_nir(device, nir[i], optimize_conservatively, false); @@ -3505,7 +3505,7 @@ radv_create_shaders(struct radv_pipeline *pipeline, struct radv_pipeline_layout radv_determine_ngg_settings(pipeline, pipeline_key, infos, nir); - for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { if (nir[i]) { radv_start_feedback(stage_feedbacks[i]); @@ -3603,7 +3603,7 @@ radv_create_shaders(struct radv_pipeline *pipeline, struct radv_pipeline_layout } } - for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { if (radv_can_dump_shader(device, modules[i], false)) nir_print_shader(nir[i], stderr); } @@ -3673,7 +3673,7 @@ radv_create_shaders(struct radv_pipeline *pipeline, struct radv_pipeline_layout modules[pre_stage] = NULL; } - for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { if (modules[i] && !pipeline->shaders[i]) { radv_start_feedback(stage_feedbacks[i]); @@ -3704,7 +3704,7 @@ radv_create_shaders(struct radv_pipeline *pipeline, struct radv_pipeline_layout } free(gs_copy_binary); - for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { free(binaries[i]); if (nir[i]) { ralloc_free(nir[i]); @@ -5478,13 +5478,16 @@ radv_pipeline_init_shader_stages_state(struct radv_pipeline *pipeline) { struct radv_device *device = pipeline->device; - for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) { - pipeline->user_data_0[i] = radv_pipeline_stage_to_user_data_0( - pipeline, i, device->physical_device->rad_info.chip_class); + for (unsigned i = 0; i < MESA_VULKAN_SHADER_STAGES; i++) { + bool shader_exists = !!pipeline->shaders[i]; + if (shader_exists || i < MESA_SHADER_COMPUTE) { + /* We need this info for some stages even when the shader doesn't exist. */ + pipeline->user_data_0[i] = radv_pipeline_stage_to_user_data_0( + pipeline, i, device->physical_device->rad_info.chip_class); - if (pipeline->shaders[i]) { - pipeline->need_indirect_descriptor_sets |= - radv_shader_need_indirect_descriptor_sets(pipeline, i); + if (shader_exists) + pipeline->need_indirect_descriptor_sets |= + radv_shader_need_indirect_descriptor_sets(pipeline, i); } } @@ -5522,10 +5525,10 @@ radv_pipeline_init(struct radv_pipeline *pipeline, struct radv_device *device, VkPipelineCreationFeedbackEXT *pipeline_feedback = creation_feedback ? creation_feedback->pPipelineCreationFeedback : NULL; - const VkPipelineShaderStageCreateInfo *pStages[MESA_SHADER_STAGES] = { + const VkPipelineShaderStageCreateInfo *pStages[MESA_VULKAN_SHADER_STAGES] = { 0, }; - VkPipelineCreationFeedbackEXT *stage_feedbacks[MESA_SHADER_STAGES] = {0}; + VkPipelineCreationFeedbackEXT *stage_feedbacks[MESA_VULKAN_SHADER_STAGES] = {0}; for (uint32_t i = 0; i < pCreateInfo->stageCount; i++) { gl_shader_stage stage = ffs(pCreateInfo->pStages[i].stage) - 1; pStages[stage] = &pCreateInfo->pStages[i]; @@ -5773,10 +5776,10 @@ radv_compute_pipeline_create(VkDevice _device, VkPipelineCache _cache, RADV_FROM_HANDLE(radv_device, device, _device); RADV_FROM_HANDLE(radv_pipeline_cache, cache, _cache); RADV_FROM_HANDLE(radv_pipeline_layout, pipeline_layout, pCreateInfo->layout); - const VkPipelineShaderStageCreateInfo *pStages[MESA_SHADER_STAGES] = { + const VkPipelineShaderStageCreateInfo *pStages[MESA_VULKAN_SHADER_STAGES] = { 0, }; - VkPipelineCreationFeedbackEXT *stage_feedbacks[MESA_SHADER_STAGES] = {0}; + VkPipelineCreationFeedbackEXT *stage_feedbacks[MESA_VULKAN_SHADER_STAGES] = {0}; struct radv_pipeline *pipeline; VkResult result; @@ -5862,7 +5865,7 @@ static uint32_t radv_get_executable_count(const struct radv_pipeline *pipeline) { uint32_t ret = 0; - for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { if (!pipeline->shaders[i]) continue; @@ -5879,7 +5882,7 @@ static struct radv_shader * radv_get_shader_from_executable_index(const struct radv_pipeline *pipeline, int index, gl_shader_stage *stage) { - for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { if (!pipeline->shaders[i]) continue; if (!index) { @@ -5927,7 +5930,7 @@ radv_GetPipelineExecutablePropertiesKHR(VkDevice _device, const VkPipelineInfoKH } const uint32_t count = MIN2(total_count, *pExecutableCount); - for (unsigned i = 0, executable_idx = 0; i < MESA_SHADER_STAGES && executable_idx < count; ++i) { + for (unsigned i = 0, executable_idx = 0; i < MESA_VULKAN_SHADER_STAGES && executable_idx < count; ++i) { if (!pipeline->shaders[i]) continue; pProperties[executable_idx].stages = mesa_to_vk_shader_stage(i); diff --git a/src/amd/vulkan/radv_pipeline_cache.c b/src/amd/vulkan/radv_pipeline_cache.c index bfcbe4c3c8b..b6622c0f628 100644 --- a/src/amd/vulkan/radv_pipeline_cache.c +++ b/src/amd/vulkan/radv_pipeline_cache.c @@ -36,9 +36,9 @@ struct cache_entry { unsigned char sha1[20]; uint32_t sha1_dw[5]; }; - uint32_t binary_sizes[MESA_SHADER_STAGES]; + uint32_t binary_sizes[MESA_VULKAN_SHADER_STAGES]; uint32_t num_stack_sizes; - struct radv_shader *shaders[MESA_SHADER_STAGES]; + struct radv_shader *shaders[MESA_VULKAN_SHADER_STAGES]; char code[0]; }; @@ -90,7 +90,7 @@ radv_pipeline_cache_finish(struct radv_pipeline_cache *cache) { for (unsigned i = 0; i < cache->table_size; ++i) if (cache->hash_table[i]) { - for (int j = 0; j < MESA_SHADER_STAGES; ++j) { + for (int j = 0; j < MESA_VULKAN_SHADER_STAGES; ++j) { if (cache->hash_table[i]->shaders[j]) radv_shader_destroy(cache->device, cache->hash_table[i]->shaders[j]); } @@ -106,7 +106,7 @@ static uint32_t entry_size(struct cache_entry *entry) { size_t ret = sizeof(*entry); - for (int i = 0; i < MESA_SHADER_STAGES; ++i) + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) if (entry->binary_sizes[i]) ret += entry->binary_sizes[i]; ret = align(ret, alignof(struct cache_entry)); @@ -126,7 +126,7 @@ radv_hash_shaders(unsigned char *hash, const VkPipelineShaderStageCreateInfo **s if (layout) _mesa_sha1_update(&ctx, layout->sha1, sizeof(layout->sha1)); - for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { if (stages[i]) { RADV_FROM_HANDLE(vk_shader_module, module, stages[i]->module); const VkSpecializationInfo *spec_info = stages[i]->pSpecializationInfo; @@ -347,7 +347,7 @@ radv_create_shaders_from_pipeline_cache( } char *p = entry->code; - for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { if (!entry->shaders[i] && entry->binary_sizes[i]) { struct radv_shader_binary *binary = calloc(1, entry->binary_sizes[i]); memcpy(binary, p, entry->binary_sizes[i]); @@ -373,7 +373,7 @@ radv_create_shaders_from_pipeline_cache( if (device->instance->debug_flags & RADV_DEBUG_NO_MEMORY_CACHE && cache == device->mem_cache) vk_free(&cache->alloc, entry); else { - for (int i = 0; i < MESA_SHADER_STAGES; ++i) + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) if (entry->shaders[i]) p_atomic_inc(&entry->shaders[i]->ref_count); } @@ -395,7 +395,7 @@ radv_pipeline_cache_insert_shaders(struct radv_device *device, struct radv_pipel radv_pipeline_cache_lock(cache); struct cache_entry *entry = radv_pipeline_cache_search_unlocked(cache, sha1); if (entry) { - for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { if (entry->shaders[i]) { radv_shader_destroy(cache->device, shaders[i]); shaders[i] = entry->shaders[i]; @@ -418,7 +418,7 @@ radv_pipeline_cache_insert_shaders(struct radv_device *device, struct radv_pipel } size_t size = sizeof(*entry) + sizeof(*stack_sizes) * num_stack_sizes; - for (int i = 0; i < MESA_SHADER_STAGES; ++i) + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) if (shaders[i]) size += binaries[i]->total_size; const size_t size_without_align = size; @@ -435,7 +435,7 @@ radv_pipeline_cache_insert_shaders(struct radv_device *device, struct radv_pipel char *p = entry->code; - for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { if (!shaders[i]) continue; @@ -479,7 +479,7 @@ radv_pipeline_cache_insert_shaders(struct radv_device *device, struct radv_pipel /* We delay setting the shader so we have reproducible disk cache * items. */ - for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { if (!shaders[i]) continue; @@ -527,7 +527,7 @@ radv_pipeline_cache_load(struct radv_pipeline_cache *cache, const void *data, si dest_entry = vk_alloc(&cache->alloc, size_of_entry, 8, VK_SYSTEM_ALLOCATION_SCOPE_CACHE); if (dest_entry) { memcpy(dest_entry, entry, size_of_entry); - for (int i = 0; i < MESA_SHADER_STAGES; ++i) + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) dest_entry->shaders[i] = NULL; radv_pipeline_cache_add_entry(cache, dest_entry); } @@ -624,7 +624,7 @@ radv_GetPipelineCacheData(VkDevice _device, VkPipelineCache _cache, size_t *pDat } memcpy(p, entry, size_of_entry); - for (int j = 0; j < MESA_SHADER_STAGES; ++j) + for (int j = 0; j < MESA_VULKAN_SHADER_STAGES; ++j) ((struct cache_entry *)p)->shaders[j] = NULL; p = (char *)p + size_of_entry; } diff --git a/src/amd/vulkan/radv_private.h b/src/amd/vulkan/radv_private.h index f85ac5f91c8..7654da11070 100644 --- a/src/amd/vulkan/radv_private.h +++ b/src/amd/vulkan/radv_private.h @@ -1701,7 +1701,7 @@ uint32_t radv_get_hash_flags(const struct radv_device *device, bool stats); bool radv_rt_pipeline_has_dynamic_stack_size(const VkRayTracingPipelineCreateInfoKHR *pCreateInfo); -#define RADV_STAGE_MASK ((1 << MESA_SHADER_STAGES) - 1) +#define RADV_STAGE_MASK ((1 << MESA_VULKAN_SHADER_STAGES) - 1) #define radv_foreach_stage(stage, stage_bits) \ for (gl_shader_stage stage, __tmp = (gl_shader_stage)((stage_bits)&RADV_STAGE_MASK); \ @@ -1768,7 +1768,7 @@ struct radv_pipeline { struct radv_dynamic_state dynamic_state; bool need_indirect_descriptor_sets; - struct radv_shader *shaders[MESA_SHADER_STAGES]; + struct radv_shader *shaders[MESA_VULKAN_SHADER_STAGES]; struct radv_shader *gs_copy_shader; VkShaderStageFlags active_stages; @@ -1789,7 +1789,7 @@ struct radv_pipeline { uint32_t vb_desc_usage_mask; uint32_t vb_desc_alloc_size; - uint32_t user_data_0[MESA_SHADER_STAGES]; + uint32_t user_data_0[MESA_VULKAN_SHADER_STAGES]; union { struct { struct radv_multisample_state ms;