From 9d75795087ce1233e92df2582846ec6f6116d423 Mon Sep 17 00:00:00 2001 From: Vitaliy Triang3l Kuzmin Date: Mon, 3 Apr 2023 22:28:36 +0300 Subject: [PATCH] radv: Move most of DB_SHADER_CONTROL to PS, more precise GFX11 blend WA Move most of the DB_SHADER_CONTROL fields from the pipeline to the pixel shader for preparation for shader objects. Also, the GFX11 export conflict bug workaround doesn't need to be enabled for non-1x sample counts or if blending is not enabled, so make the application of DB_SHADER_CONTROL consider the current sample count and blending state even if they're dynamic. Having access to the exact sample count in DB_SHADER_CONTROL setup is also necessary for good performance in SampleInterlock execution modes of fragment shader interlock, for configuration of POPS_OVERLAP_NUM_SAMPLES (GFX9-10.3) or OVERRIDE_INTRINSIC_RATE (GFX11), as PixelInterlock is massively slower with multisampling due to overlap between adjacent polygons sharing covered pixels among the common edge. The name of the dynamic state controlling DB_SHADER_CONTROL is now unambiguous - previously line rasterization mode had effect on attachment feedback loop state emission. Reviewed-by: Samuel Pitoiset Signed-off-by: Vitaliy Triang3l Kuzmin Part-of: --- src/amd/vulkan/radv_cmd_buffer.c | 116 +++++++++++++----------- src/amd/vulkan/radv_pipeline_graphics.c | 39 -------- src/amd/vulkan/radv_private.h | 4 +- src/amd/vulkan/radv_shader.h | 1 + src/amd/vulkan/radv_shader_info.c | 28 ++++++ 5 files changed, 97 insertions(+), 91 deletions(-) diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c index a1a35762914..cb578ce724a 100644 --- a/src/amd/vulkan/radv_cmd_buffer.c +++ b/src/amd/vulkan/radv_cmd_buffer.c @@ -1850,9 +1850,6 @@ radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer) cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_FRAGMENT_SHADING_RATE; } - if (cmd_buffer->state.emitted_graphics_pipeline->db_shader_control != pipeline->db_shader_control) - cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_ATTACHMENT_FEEDBACK_LOOP_ENABLE; - if (cmd_buffer->state.emitted_graphics_pipeline->db_render_control != pipeline->db_render_control) cmd_buffer->state.dirty |= RADV_CMD_DIRTY_FRAMEBUFFER; } @@ -4370,43 +4367,6 @@ radv_emit_line_rasterization_mode(struct radv_cmd_buffer *cmd_buffer) S_028BDC_PERPENDICULAR_ENDCAP_ENA(d->vk.rs.line.mode == VK_LINE_RASTERIZATION_MODE_RECTANGULAR_EXT)); } -static bool -radv_ps_can_enable_early_z(const struct radv_shader *ps) -{ - return ps->info.ps.early_fragment_test || !ps->info.ps.writes_memory; -} - -static void -radv_emit_attachment_feedback_loop_enable(struct radv_cmd_buffer *cmd_buffer) -{ - const struct radv_graphics_pipeline *pipeline = cmd_buffer->state.graphics_pipeline; - const struct radv_shader *ps = cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT]; - const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; - unsigned db_shader_control = pipeline->db_shader_control; - const bool uses_ds_feedback_loop = - !!(d->feedback_loop_aspects & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)); - unsigned z_order; - - /* When a depth/stencil attachment is used inside feedback loops, use LATE_Z to make sure shader - * invocations read the correct value. - */ - if (!uses_ds_feedback_loop && ps && radv_ps_can_enable_early_z(ps)) { - z_order = V_02880C_EARLY_Z_THEN_LATE_Z; - } else { - z_order = V_02880C_LATE_Z; - } - - /* Bug workaround for smoothing (overrasterization) on GFX6. */ - if (cmd_buffer->device->physical_device->rad_info.gfx_level == GFX6 && - d->vk.rs.line.mode == VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT) { - z_order = V_02880C_LATE_Z; - } - - db_shader_control |= S_02880C_Z_ORDER(z_order); - - radeon_set_context_reg(cmd_buffer->cs, R_02880C_DB_SHADER_CONTROL, db_shader_control); -} - static void radv_cmd_buffer_flush_dynamic_state(struct radv_cmd_buffer *cmd_buffer, const uint64_t states) { @@ -4517,9 +4477,7 @@ radv_cmd_buffer_flush_dynamic_state(struct radv_cmd_buffer *cmd_buffer, const ui RADV_CMD_DIRTY_DYNAMIC_LINE_RASTERIZATION_MODE)) radv_emit_msaa_state(cmd_buffer); - if (states & - (RADV_CMD_DIRTY_DYNAMIC_ATTACHMENT_FEEDBACK_LOOP_ENABLE | RADV_CMD_DIRTY_DYNAMIC_LINE_RASTERIZATION_MODE)) - radv_emit_attachment_feedback_loop_enable(cmd_buffer); + /* RADV_CMD_DIRTY_DYNAMIC_ATTACHMENT_FEEDBACK_LOOP_ENABLE is handled by radv_emit_db_shader_control. */ cmd_buffer->state.dirty &= ~states; } @@ -5723,9 +5681,11 @@ radv_BeginCommandBuffer(VkCommandBuffer commandBuffer, const VkCommandBufferBegi cmd_buffer->state.last_vrs_rates_sgpr_idx = -1; cmd_buffer->state.last_pa_sc_binner_cntl_0 = -1; cmd_buffer->state.last_db_count_control = -1; + cmd_buffer->state.last_db_shader_control = -1; cmd_buffer->usage_flags = pBeginInfo->flags; - cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_ALL | RADV_CMD_DIRTY_GUARDBAND | RADV_CMD_DIRTY_OCCLUSION_QUERY; + cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_ALL | RADV_CMD_DIRTY_GUARDBAND | RADV_CMD_DIRTY_OCCLUSION_QUERY | + RADV_CMD_DIRTY_DB_SHADER_CONTROL; if (cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX7) { uint32_t pred_value = 0; @@ -6393,9 +6353,6 @@ radv_bind_fragment_shader(struct radv_cmd_buffer *cmd_buffer, const struct radv_ cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_RASTERIZATION_SAMPLES | RADV_CMD_DIRTY_DYNAMIC_FRAGMENT_SHADING_RATE; - if (!previous_ps || radv_ps_can_enable_early_z(previous_ps) != radv_ps_can_enable_early_z(ps)) - cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_ATTACHMENT_FEEDBACK_LOOP_ENABLE; - if (cmd_buffer->state.ms.sample_shading_enable != ps->info.ps.uses_sample_shading) { cmd_buffer->state.ms.sample_shading_enable = ps->info.ps.uses_sample_shading; cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_RASTERIZATION_SAMPLES; @@ -6409,6 +6366,9 @@ radv_bind_fragment_shader(struct radv_cmd_buffer *cmd_buffer, const struct radv_ cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_RASTERIZATION_SAMPLES; } + if (!previous_ps || previous_ps->info.ps.db_shader_control != ps->info.ps.db_shader_control) + cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DB_SHADER_CONTROL; + /* Re-emit the PS epilog when a new fragment shader is bound. */ if (ps->info.ps.has_epilog) cmd_buffer->state.emitted_ps_epilog = NULL; @@ -6437,9 +6397,9 @@ radv_bind_shader(struct radv_cmd_buffer *cmd_buffer, struct radv_shader *shader, /* Reset some dynamic states when a shader stage is unbound. */ switch (stage) { case MESA_SHADER_FRAGMENT: - cmd_buffer->state.dirty |= - RADV_CMD_DIRTY_DYNAMIC_CONSERVATIVE_RAST_MODE | RADV_CMD_DIRTY_DYNAMIC_RASTERIZATION_SAMPLES | - RADV_CMD_DIRTY_DYNAMIC_FRAGMENT_SHADING_RATE | RADV_CMD_DIRTY_DYNAMIC_ATTACHMENT_FEEDBACK_LOOP_ENABLE; + cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_CONSERVATIVE_RAST_MODE | + RADV_CMD_DIRTY_DYNAMIC_RASTERIZATION_SAMPLES | + RADV_CMD_DIRTY_DYNAMIC_FRAGMENT_SHADING_RATE | RADV_CMD_DIRTY_DB_SHADER_CONTROL; break; default: break; @@ -7503,6 +7463,8 @@ radv_CmdExecuteCommands(VkCommandBuffer commandBuffer, uint32_t commandBufferCou primary->state.last_pa_sc_binner_cntl_0 = secondary->state.last_pa_sc_binner_cntl_0; + primary->state.last_db_shader_control = secondary->state.last_db_shader_control; + primary->state.rb_noncoherent_dirty |= secondary->state.rb_noncoherent_dirty; } @@ -7510,7 +7472,8 @@ radv_CmdExecuteCommands(VkCommandBuffer commandBuffer, uint32_t commandBufferCou * some states. */ primary->state.dirty |= RADV_CMD_DIRTY_PIPELINE | RADV_CMD_DIRTY_INDEX_BUFFER | RADV_CMD_DIRTY_GUARDBAND | - RADV_CMD_DIRTY_DYNAMIC_ALL | RADV_CMD_DIRTY_NGG_QUERY | RADV_CMD_DIRTY_OCCLUSION_QUERY; + RADV_CMD_DIRTY_DYNAMIC_ALL | RADV_CMD_DIRTY_NGG_QUERY | RADV_CMD_DIRTY_OCCLUSION_QUERY | + RADV_CMD_DIRTY_DB_SHADER_CONTROL; radv_mark_descriptor_sets_dirty(primary, VK_PIPELINE_BIND_POINT_GRAPHICS); radv_mark_descriptor_sets_dirty(primary, VK_PIPELINE_BIND_POINT_COMPUTE); @@ -8732,6 +8695,51 @@ radv_emit_fs_state(struct radv_cmd_buffer *cmd_buffer) radeon_set_sh_reg(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, ps_state); } +static void +radv_emit_db_shader_control(struct radv_cmd_buffer *cmd_buffer) +{ + const struct radeon_info *rad_info = &cmd_buffer->device->physical_device->rad_info; + const struct radv_shader *ps = cmd_buffer->state.shaders[MESA_SHADER_FRAGMENT]; + const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; + const bool uses_ds_feedback_loop = + !!(d->feedback_loop_aspects & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)); + + uint32_t db_shader_control; + + if (ps) { + db_shader_control = ps->info.ps.db_shader_control; + } else { + db_shader_control = S_02880C_CONSERVATIVE_Z_EXPORT(V_02880C_EXPORT_ANY_Z) | + S_02880C_Z_ORDER(V_02880C_EARLY_Z_THEN_LATE_Z) | + S_02880C_DUAL_QUAD_DISABLE(rad_info->has_rbplus && !rad_info->rbplus_allowed); + } + + /* When a depth/stencil attachment is used inside feedback loops, use LATE_Z to make sure shader invocations read the + * correct value. + * Also apply the bug workaround for smoothing (overrasterization) on GFX6. + */ + if (uses_ds_feedback_loop || + (rad_info->gfx_level == GFX6 && d->vk.rs.line.mode == VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT)) + db_shader_control = (db_shader_control & C_02880C_Z_ORDER) | S_02880C_Z_ORDER(V_02880C_LATE_Z); + + if (rad_info->has_export_conflict_bug && radv_get_rasterization_samples(cmd_buffer) == 1) { + for (uint32_t i = 0; i < MAX_RTS; i++) { + if (d->vk.cb.attachments[i].write_mask && d->vk.cb.attachments[i].blend_enable) { + db_shader_control |= S_02880C_OVERRIDE_INTRINSIC_RATE_ENABLE(1) | S_02880C_OVERRIDE_INTRINSIC_RATE(2); + break; + } + } + } + + if (db_shader_control != cmd_buffer->state.last_db_shader_control) { + radeon_set_context_reg(cmd_buffer->cs, R_02880C_DB_SHADER_CONTROL, db_shader_control); + + cmd_buffer->state.last_db_shader_control = db_shader_control; + } + + cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_DB_SHADER_CONTROL; +} + static void radv_emit_all_graphics_states(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *info) { @@ -8807,6 +8815,12 @@ radv_emit_all_graphics_states(struct radv_cmd_buffer *cmd_buffer, const struct r if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_GUARDBAND) radv_emit_guardband_state(cmd_buffer); + if (cmd_buffer->state.dirty & + (RADV_CMD_DIRTY_DB_SHADER_CONTROL | RADV_CMD_DIRTY_DYNAMIC_COLOR_WRITE_MASK | + RADV_CMD_DIRTY_DYNAMIC_COLOR_BLEND_ENABLE | RADV_CMD_DIRTY_DYNAMIC_RASTERIZATION_SAMPLES | + RADV_CMD_DIRTY_DYNAMIC_LINE_RASTERIZATION_MODE | RADV_CMD_DIRTY_DYNAMIC_ATTACHMENT_FEEDBACK_LOOP_ENABLE)) + radv_emit_db_shader_control(cmd_buffer); + if (info->indexed && info->indirect && cmd_buffer->state.dirty & RADV_CMD_DIRTY_INDEX_BUFFER) radv_emit_index_buffer(cmd_buffer); diff --git a/src/amd/vulkan/radv_pipeline_graphics.c b/src/amd/vulkan/radv_pipeline_graphics.c index 95de2b8345d..09c611aed35 100644 --- a/src/amd/vulkan/radv_pipeline_graphics.c +++ b/src/amd/vulkan/radv_pipeline_graphics.c @@ -1162,44 +1162,6 @@ radv_pipeline_init_dynamic_state(const struct radv_device *device, struct radv_g pipeline->dynamic_state.mask = states; } -static uint32_t -radv_compute_db_shader_control(const struct radv_device *device, const struct radv_graphics_pipeline *pipeline, - const struct vk_graphics_pipeline_state *state) -{ - const struct radv_physical_device *pdevice = device->physical_device; - struct radv_shader *ps = pipeline->base.shaders[MESA_SHADER_FRAGMENT]; - unsigned conservative_z_export = V_02880C_EXPORT_ANY_Z; - - if (ps && ps->info.ps.depth_layout == FRAG_DEPTH_LAYOUT_GREATER) - conservative_z_export = V_02880C_EXPORT_GREATER_THAN_Z; - else if (ps && ps->info.ps.depth_layout == FRAG_DEPTH_LAYOUT_LESS) - conservative_z_export = V_02880C_EXPORT_LESS_THAN_Z; - - bool disable_rbplus = pdevice->rad_info.has_rbplus && !pdevice->rad_info.rbplus_allowed; - - /* It shouldn't be needed to export gl_SampleMask when MSAA is disabled - * but this appears to break Project Cars (DXVK). See - * https://bugs.freedesktop.org/show_bug.cgi?id=109401 - */ - bool mask_export_enable = ps && ps->info.ps.writes_sample_mask; - - bool export_conflict_wa = device->physical_device->rad_info.has_export_conflict_bug && - radv_pipeline_is_blend_enabled(pipeline, state->cb) && - (!state->ms || state->ms->rasterization_samples <= 1 || - (pipeline->dynamic_states & RADV_DYNAMIC_RASTERIZATION_SAMPLES)); - - return S_02880C_Z_EXPORT_ENABLE(ps && ps->info.ps.writes_z) | - S_02880C_STENCIL_TEST_VAL_EXPORT_ENABLE(ps && ps->info.ps.writes_stencil) | - S_02880C_KILL_ENABLE(ps && ps->info.ps.can_discard) | S_02880C_MASK_EXPORT_ENABLE(mask_export_enable) | - S_02880C_CONSERVATIVE_Z_EXPORT(conservative_z_export) | - S_02880C_DEPTH_BEFORE_SHADER(ps && ps->info.ps.early_fragment_test) | - S_02880C_PRE_SHADER_DEPTH_COVERAGE_ENABLE(ps && ps->info.ps.post_depth_coverage) | - S_02880C_EXEC_ON_HIER_FAIL(ps && ps->info.ps.writes_memory) | - S_02880C_EXEC_ON_NOOP(ps && ps->info.ps.writes_memory) | S_02880C_DUAL_QUAD_DISABLE(disable_rbplus) | - S_02880C_OVERRIDE_INTRINSIC_RATE_ENABLE(export_conflict_wa) | - S_02880C_OVERRIDE_INTRINSIC_RATE(export_conflict_wa ? 2 : 0); -} - static void gfx10_emit_ge_pc_alloc(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level, uint32_t oversub_pc_lines) { @@ -3985,7 +3947,6 @@ radv_graphics_pipeline_init(struct radv_graphics_pipeline *pipeline, struct radv pipeline->base.push_constant_size = pipeline_layout.push_constant_size; pipeline->base.dynamic_offset_count = pipeline_layout.dynamic_offset_count; - pipeline->db_shader_control = radv_compute_db_shader_control(device, pipeline, &state); for (unsigned i = 0; i < MESA_VULKAN_SHADER_STAGES; i++) { if (pipeline->base.shaders[i]) { diff --git a/src/amd/vulkan/radv_private.h b/src/amd/vulkan/radv_private.h index 3db641be8bb..1a813a7a0dc 100644 --- a/src/amd/vulkan/radv_private.h +++ b/src/amd/vulkan/radv_private.h @@ -1351,6 +1351,7 @@ enum radv_cmd_dirty_bits { RADV_CMD_DIRTY_RBPLUS = 1ull << 56, RADV_CMD_DIRTY_NGG_QUERY = 1ull << 57, RADV_CMD_DIRTY_OCCLUSION_QUERY = 1ull << 58, + RADV_CMD_DIRTY_DB_SHADER_CONTROL = 1ull << 59, }; enum radv_cmd_flush_bits { @@ -1655,6 +1656,8 @@ struct radv_cmd_state { uint32_t last_db_count_control; + uint32_t last_db_shader_control; + /* Whether CP DMA is busy/idle. */ bool dma_is_busy; @@ -2253,7 +2256,6 @@ struct radv_graphics_pipeline { uint32_t attrib_ends[MAX_VERTEX_ATTRIBS]; uint32_t attrib_index_offset[MAX_VERTEX_ATTRIBS]; uint32_t db_render_control; - uint32_t db_shader_control; /* Last pre-PS API stage */ gl_shader_stage last_vgt_api_stage; diff --git a/src/amd/vulkan/radv_shader.h b/src/amd/vulkan/radv_shader.h index 6f97bc3bcf0..2759fda5ea1 100644 --- a/src/amd/vulkan/radv_shader.h +++ b/src/amd/vulkan/radv_shader.h @@ -382,6 +382,7 @@ struct radv_shader_info { uint8_t color0_written; bool load_provoking_vtx; bool load_rasterization_prim; + uint32_t db_shader_control; /* DB_SHADER_CONTROL without intrinsic rate overrides */ } ps; struct { bool uses_grid_size; diff --git a/src/amd/vulkan/radv_shader_info.c b/src/amd/vulkan/radv_shader_info.c index 62b6f76b922..eb7607d9a56 100644 --- a/src/amd/vulkan/radv_shader_info.c +++ b/src/amd/vulkan/radv_shader_info.c @@ -637,6 +637,34 @@ gather_shader_info_fs(const struct radv_device *device, const nir_shader *nir, info->ps.input_mask |= mask << (var->data.location - VARYING_SLOT_VAR0); } } + + /* DB_SHADER_CONTROL based on other fragment shader info fields. */ + + unsigned conservative_z_export = V_02880C_EXPORT_ANY_Z; + if (info->ps.depth_layout == FRAG_DEPTH_LAYOUT_GREATER) + conservative_z_export = V_02880C_EXPORT_GREATER_THAN_Z; + else if (info->ps.depth_layout == FRAG_DEPTH_LAYOUT_LESS) + conservative_z_export = V_02880C_EXPORT_LESS_THAN_Z; + + unsigned z_order = + info->ps.early_fragment_test || !info->ps.writes_memory ? V_02880C_EARLY_Z_THEN_LATE_Z : V_02880C_LATE_Z; + + /* It shouldn't be needed to export gl_SampleMask when MSAA is disabled, but this appears to break Project Cars + * (DXVK). See https://bugs.freedesktop.org/show_bug.cgi?id=109401 + */ + const bool mask_export_enable = info->ps.writes_sample_mask; + + const bool disable_rbplus = + device->physical_device->rad_info.has_rbplus && !device->physical_device->rad_info.rbplus_allowed; + + info->ps.db_shader_control = + S_02880C_Z_EXPORT_ENABLE(info->ps.writes_z) | S_02880C_STENCIL_TEST_VAL_EXPORT_ENABLE(info->ps.writes_stencil) | + S_02880C_KILL_ENABLE(info->ps.can_discard) | S_02880C_MASK_EXPORT_ENABLE(mask_export_enable) | + S_02880C_CONSERVATIVE_Z_EXPORT(conservative_z_export) | S_02880C_Z_ORDER(z_order) | + S_02880C_DEPTH_BEFORE_SHADER(info->ps.early_fragment_test) | + S_02880C_PRE_SHADER_DEPTH_COVERAGE_ENABLE(info->ps.post_depth_coverage) | + S_02880C_EXEC_ON_HIER_FAIL(info->ps.writes_memory) | S_02880C_EXEC_ON_NOOP(info->ps.writes_memory) | + S_02880C_DUAL_QUAD_DISABLE(disable_rbplus); } static void