radeonsi/tests: add a heuristic to pick the baseline

When the baseline for the exact GPU tested isn't available,
try to make a reasonnable guess: look for another baseline
from a GPU in the same class.

Acked-by: Marek Olšák <marek.olsak@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/16347>

src/gallium/drivers/radeonsi/ci/gfx10-navi10-fail.csv

@@ -1,3 +1,4 @@
+# piglit failures
 glx@glx-make-current,Crash
 glx@glx-multi-window-single-context,Fail
 glx@glx-visuals-depth -pixmap,Crash
@@ -168,3 +169,11 @@ spec@oes_shader_io_blocks@compiler@layout-location-aliasing.vert,Fail
 wgl@wgl-multi-context-single-window,Fail
 wgl@wgl-multi-window-single-context,Fail
 wgl@wgl-sanity,Fail
+
+# glcts failures
+KHR-GL46.gl_spirv.spirv_glsl_to_spirv_builtin_functions_test,Fail
+KHR-GL46.pipeline_statistics_query_tests_ARB.functional_geometry_shader_queries,Fail
+KHR-GL46.pipeline_statistics_query_tests_ARB.functional_primitives_vertices_submitted_and_clipping_input_output_primitives,Fail
+KHR-GL46.shader_ballot_tests.ShaderBallotFunctionRead,Fail
+KHR-GL46.shader_ballot_tests.ShaderBallotBitmasks,Fail
+KHR-GL46.sparse_texture_tests.SparseTextureCommitment,Fail

src/gallium/drivers/radeonsi/ci/gfx10_3-sienna_cichlid-fail.csv

@@ -1,3 +1,4 @@
+# piglit failures
 glx@glx-make-current,Crash
 glx@glx-multi-window-single-context,Fail
 glx@glx-swap-event_interval,Fail
@@ -183,3 +184,15 @@ spec@oes_shader_io_blocks@compiler@layout-location-aliasing.vert,Fail
 wgl@wgl-multi-context-single-window,Fail
 wgl@wgl-multi-window-single-context,Fail
 wgl@wgl-sanity,Fail
+
+# glcts failures
+KHR-GL46.pipeline_statistics_query_tests_ARB.functional_geometry_shader_queries,Fail
+KHR-GL46.pipeline_statistics_query_tests_ARB.functional_primitives_vertices_submitted_and_clipping_input_output_primitives,Fail
+KHR-GL46.shader_ballot_tests.ShaderBallotFunctionRead,Fail
+KHR-GL46.sparse_texture2_tests.SparseTexture2Allocation,Fail
+KHR-GL46.sparse_texture2_tests.SparseTexture2Commitment,Fail
+KHR-GL46.sparse_texture2_tests.SparseTexture2Lookup,Fail
+KHR-GL46.sparse_texture2_tests.StandardPageSizesTestCase,Fail
+KHR-GL46.sparse_texture2_tests.UncommittedRegionsAccess,Fail
+KHR-GL46.sparse_texture_clamp_tests.SparseTextureClampLookupColor,Fail
+KHR-GL46.sparse_texture_clamp_tests.SparseTextureClampLookupResidency,Fail

src/gallium/drivers/radeonsi/ci/gfx8-polaris11-fail.csv

@@ -1,3 +1,4 @@
+# piglit failures
 glx@glx-make-current,Crash
 glx@glx-multi-window-single-context,Fail
 glx@glx-swap-event_interval,Fail
@@ -260,3 +261,15 @@ spec@oes_shader_io_blocks@compiler@layout-location-aliasing.vert,Fail
 wgl@wgl-multi-context-single-window,Fail
 wgl@wgl-multi-window-single-context,Fail
 wgl@wgl-sanity,Fail
+
+# glcts failures
+KHR-GL46.geometry_shader.layered_rendering_fbo_no_attachment.layered_rendering_fbo_no_attachment,Fail
+KHR-GL46.shader_ballot_tests.ShaderBallotFunctionRead,Fail
+KHR-GL46.shader_image_load_store.advanced-sso-subroutine,Fail
+
+# deqp failures
+dEQP-GLES31.functional.geometry_shading.instanced.geometry_output_different_2_invocations,Fail
+dEQP-GLES31.functional.geometry_shading.instanced.geometry_output_different_32_invocations,Fail
+dEQP-GLES31.functional.geometry_shading.instanced.geometry_output_different_8_invocations,Fail
+dEQP-GLES31.functional.geometry_shading.instanced.geometry_output_different_max_invocations,Fail
+dEQP-GLES31.functional.srgb_texture_decode.skip_decode.srgba8.texel_fetch,Crash

src/gallium/drivers/radeonsi/ci/gfx9-raven-fail.csv

@@ -1,3 +1,4 @@
+# piglit failures
 glx@glx-make-current,Crash
 glx@glx-multi-window-single-context,Fail
 glx@glx-visuals-depth -pixmap,Crash
@@ -258,3 +259,36 @@ spec@oes_shader_io_blocks@compiler@layout-location-aliasing.vert,Fail
 wgl@wgl-multi-context-single-window,Fail
 wgl@wgl-multi-window-single-context,Fail
 wgl@wgl-sanity,Fail
+
+# glcts failures
+KHR-GL46.gl_spirv.spirv_glsl_to_spirv_builtin_functions_test,Fail
+KHR-GL46.pipeline_statistics_query_tests_ARB.functional_geometry_shader_queries,Fail
+KHR-GL46.pipeline_statistics_query_tests_ARB.functional_primitives_vertices_submitted_and_clipping_input_output_primitives,Fail
+KHR-GL46.packed_pixels.pbo_rectangle.r16_snorm,Fail
+KHR-GL46.packed_pixels.pbo_rectangle.r8_snorm,Fail
+KHR-GL46.packed_pixels.pbo_rectangle.rg16_snorm,Fail
+KHR-GL46.packed_pixels.pbo_rectangle.rg8_snorm,Fail
+KHR-GL46.packed_pixels.pbo_rectangle.rgb16_snorm,Fail
+KHR-GL46.packed_pixels.pbo_rectangle.rgb8_snorm,Fail
+KHR-GL46.packed_pixels.pbo_rectangle.rgba16_snorm,Fail
+KHR-GL46.packed_pixels.pbo_rectangle.rgba8_snorm,Fail
+KHR-GL46.packed_pixels.rectangle.r16_snorm,Fail
+KHR-GL46.packed_pixels.rectangle.r8_snorm,Fail
+KHR-GL46.packed_pixels.rectangle.rg16_snorm,Fail
+KHR-GL46.packed_pixels.rectangle.rg8_snorm,Fail
+KHR-GL46.packed_pixels.rectangle.rgb16_snorm,Fail
+KHR-GL46.packed_pixels.rectangle.rgb8_snorm,Fail
+KHR-GL46.packed_pixels.rectangle.rgba16_snorm,Fail
+KHR-GL46.packed_pixels.rectangle.rgba8_snorm,Fail
+KHR-GL46.packed_pixels.varied_rectangle.r16_snorm,Fail
+KHR-GL46.packed_pixels.varied_rectangle.r8_snorm,Fail
+KHR-GL46.packed_pixels.varied_rectangle.rg16_snorm,Fail
+KHR-GL46.packed_pixels.varied_rectangle.rg8_snorm,Fail
+KHR-GL46.packed_pixels.varied_rectangle.rgb16_snorm,Fail
+KHR-GL46.packed_pixels.varied_rectangle.rgb8_snorm,Fail
+KHR-GL46.packed_pixels.varied_rectangle.rgba16_snorm,Fail
+KHR-GL46.packed_pixels.varied_rectangle.rgba8_snorm,Fail
+KHR-GL46.shader_ballot_tests.ShaderBallotFunctionRead,Fail
+KHR-GL46.shader_ballot_tests.ShaderBallotBitmasks,Fail
+KHR-GL46.sparse_texture_tests.SparseTextureCommitment,Fail
+KHR-GL46.sparse_buffer_tests.BufferStorageTest,Fail

src/gallium/drivers/radeonsi/ci/gfx9-vega20-fail.csv

@@ -1,3 +1,5 @@
+# piglit failures
+
 glx@extension string sanity,Fail
 spec@!opengl 1.1@windowoverlap,Fail
 spec@arb_shader_clock@execution@clock,Fail
@@ -163,3 +165,20 @@ spec@oes_shader_io_blocks@compiler@layout-location-aliasing.vert,Fail
 wgl@wgl-multi-context-single-window,Fail
 wgl@wgl-multi-window-single-context,Fail
 wgl@wgl-sanity,Fail
+
+# glcts failures
+KHR-GL46.direct_state_access.framebuffers_texture_attachment,Fail
+KHR-GL46.direct_state_access.framebuffers_texture_layer_attachment,Fail
+KHR-GL46.geometry_shader.adjacency.adjacency_indiced_triangle_strip,Fail
+KHR-GL46.geometry_shader.adjacency.adjacency_non_indiced_triangle_strip,Fail
+KHR-GL46.geometry_shader.rendering.rendering.triangles_with_adjacency_input_line_strip_output_triangle_strip_adjacency_drawcall,Fail
+KHR-GL46.geometry_shader.rendering.rendering.triangles_with_adjacency_input_triangle_strip_output_triangle_strip_adjacency_drawcall,Fail
+KHR-GL46.gl_spirv.spirv_glsl_to_spirv_builtin_functions_test,Fail
+KHR-GL46.shader_ballot_tests.ShaderBallotFunctionRead,Fail
+KHR-GL46.sparse_buffer_tests.BufferStorageTest,Fail
+KHR-GL46.sparse_texture2_tests.SparseTexture2Lookup,Fail
+KHR-GL46.sparse_texture2_tests.StandardPageSizesTestCase,Fail
+KHR-GL46.sparse_texture2_tests.UncommittedRegionsAccess,Fail
+KHR-GL46.sparse_texture_clamp_tests.SparseTextureClampLookupColor,Fail
+KHR-GL46.sparse_texture_clamp_tests.SparseTextureClampLookupResidency,Fail
+KHR-GL46.sparse_texture_tests.SparseTextureCommitment,Fail

src/gallium/drivers/radeonsi/ci/navi10-glcts-fail.csv

@@ -1,6 +0,0 @@
-KHR-GL46.gl_spirv.spirv_glsl_to_spirv_builtin_functions_test,Fail
-KHR-GL46.pipeline_statistics_query_tests_ARB.functional_geometry_shader_queries,Fail
-KHR-GL46.pipeline_statistics_query_tests_ARB.functional_primitives_vertices_submitted_and_clipping_input_output_primitives,Fail
-KHR-GL46.shader_ballot_tests.ShaderBallotFunctionRead,Fail
-KHR-GL46.shader_ballot_tests.ShaderBallotBitmasks,Fail
-KHR-GL46.sparse_texture_tests.SparseTextureCommitment,Fail

src/gallium/drivers/radeonsi/ci/polaris11-deqp-fail.csv

@@ -1,5 +0,0 @@
-dEQP-GLES31.functional.geometry_shading.instanced.geometry_output_different_2_invocations,Fail
-dEQP-GLES31.functional.geometry_shading.instanced.geometry_output_different_32_invocations,Fail
-dEQP-GLES31.functional.geometry_shading.instanced.geometry_output_different_8_invocations,Fail
-dEQP-GLES31.functional.geometry_shading.instanced.geometry_output_different_max_invocations,Fail
-dEQP-GLES31.functional.srgb_texture_decode.skip_decode.srgba8.texel_fetch,Crash

src/gallium/drivers/radeonsi/ci/polaris11-glcts-fail.csv

@@ -1,3 +0,0 @@
-KHR-GL46.geometry_shader.layered_rendering_fbo_no_attachment.layered_rendering_fbo_no_attachment,Fail
-KHR-GL46.shader_ballot_tests.ShaderBallotFunctionRead,Fail
-KHR-GL46.shader_image_load_store.advanced-sso-subroutine,Fail

src/gallium/drivers/radeonsi/ci/radeonsi-run-tests.py

@@ -199,7 +199,9 @@ env["WAFFLE_GBM_DEVICE"] = available_gpus[args.gpu][0]
 # Use piglit's glinfo to determine the GPU name
 gpu_name = "unknown"
 gpu_name_full = ""
+chip_class = -1
 
+env["AMD_DEBUG"] = "info"
 p = subprocess.run(
     ["./glinfo"],
     capture_output="True",
@@ -207,12 +209,15 @@ p = subprocess.run(
     check=True,
     env=env
 )
+del env["AMD_DEBUG"]
 for line in p.stdout.decode().split("\n"):
     if "GL_RENDER" in line:
         line = line.split("=")[1]
         gpu_name_full = '('.join(line.split("(")[:-1]).strip()
         gpu_name = line.replace("(TM)", "").split("(")[1].split(",")[0].lower()
         break
+    elif "chip_class" in line:
+        chip_class = int(line.split("=")[1])
 
 output_folder = args.output_folder
 print_green("Tested GPU: '{}' ({}) {}".format(gpu_name_full, gpu_name, gpu_device))
@@ -231,6 +236,11 @@ logfile = open(os.path.join(output_folder, "{}-run-tests.log".format(gpu_name)),
 
 spin = itertools.cycle("-\\|/")
 
+def chip_class_to_str(cl):
+    supported = ["gfx6", "gfx7", "gfx8", "gfx9", "gfx10", "gfx10_3", "gfx11"]
+    if 8 <= cl and cl < 8 + len(supported):
+        return supported[cl - 8]
+    return supported[-1]
 
 def run_cmd(args, verbosity):
     if verbosity > 1:
@@ -271,15 +281,11 @@ def run_cmd(args, verbosity):
     )
 
 
-def verify_results(baseline1, baseline2):
-    # We're not using baseline1 because piglit-runner/deqp-runner already are:
-    #  - if no baseline, baseline2 will contain the list of failures
-    #  - if there's a baseline, baseline2 will contain the diff
-    # So in both cases, an empty baseline2 files means a successful run
-    with open(baseline2) as file:
+def verify_results(results):
+    with open(results) as file:
         if len(file.readlines()) == 0:
             return True
-    print_red("New errors. Check {}".format(baseline2))
+    print_red("New results (fails or pass). Check {}".format(results))
     return False
 
 
@@ -297,13 +303,38 @@ def parse_test_filters(include_tests):
             cmd += ["-t", t]
     return cmd
 
+def select_baseline(basepath, chip_class, gpu_name):
+    chip_class_str = chip_class_to_str(chip_class)
+
+    # select the best baseline we can find
+    # 1. exact match
+    exact = os.path.join(base, "{}-{}-fail.csv".format(chip_class_str, gpu_name))
+    if os.path.exists(exact):
+        return exact
+    # 2. any baseline with the same chip_class
+    while chip_class >= 8:
+        for subdir, dirs, files in os.walk(basepath):
+            for file in files:
+                if file.find(chip_class_str) == 0 and file.endswith('-fail.csv'):
+                    return os.path.join(base, file)
+        # No match. Try an earlier class
+        chip_class = chip_class - 1
+        chip_class_str = chip_class_to_str(chip_class)
+
+    return exact
 
 filters_args = parse_test_filters(args.include_tests)
+baseline = select_baseline(base, chip_class, gpu_name)
+flakes = os.path.join(base, "{}-{}-flakes.csv".format(chip_class_to_str(chip_class), gpu_name))
+
+if os.path.exists(baseline):
+    print_yellow("Baseline: {}\n".format(baseline), args.verbose > 0)
+if os.path.exists(flakes):
+    print_yellow("[flakes {}]\n".format(flakes), args.verbose > 0)
 
 # piglit test
 if args.piglit:
     out = os.path.join(output_folder, "piglit")
-    baseline = os.path.join(base, "{}-piglit-quick-fail.csv".format(gpu_name))
     new_baseline = os.path.join(new_baseline_folder, "{}-piglit-quick-fail.csv".format(gpu_name))
     print_yellow("Running piglit tests\n", args.verbose > 0)
     cmd = [
@@ -326,16 +357,13 @@ if args.piglit:
 
     if os.path.exists(baseline):
         cmd += ["--baseline", baseline]
-        print_yellow("[baseline {}]\n".format(baseline), args.verbose > 0)
 
-    flakes = os.path.join(base, "{}-piglit-quick-flakes.csv".format(gpu_name))
     if os.path.exists(flakes):
         cmd += ["--flakes", flakes]
-        print_yellow("[flakes {}]\n".format(flakes), args.verbose > 0)
 
     run_cmd(cmd, args.verbose)
     shutil.copy(os.path.join(out, "failures.csv"), new_baseline)
-    verify_results(baseline, new_baseline)
+    verify_results(new_baseline)
 
 deqp_args = "-- --deqp-surface-width=256 --deqp-surface-height=256 --deqp-gl-config-name=rgba8888d24s8ms0 --deqp-visibility=hidden".split(
     " "
@@ -344,7 +372,6 @@ deqp_args = "-- --deqp-surface-width=256 --deqp-surface-height=256 --deqp-gl-con
 # glcts test
 if args.glcts:
     out = os.path.join(output_folder, "glcts")
-    baseline = os.path.join(base, "{}-glcts-fail.csv".format(gpu_name))
     new_baseline = os.path.join(
         new_baseline_folder, "{}-glcts-fail.csv".format(gpu_name)
     )
@@ -372,11 +399,10 @@ if args.glcts:
 
     if os.path.exists(baseline):
         cmd += ["--baseline", baseline]
-        print_yellow("[baseline {}]".format(baseline), args.verbose > 0)
     cmd += deqp_args
     run_cmd(cmd, args.verbose)
     shutil.copy(os.path.join(out, "failures.csv"), new_baseline)
-    verify_results(baseline, new_baseline)
+    verify_results(new_baseline)
 
 if args.deqp:
     print_yellow("Running   dEQP tests", args.verbose > 0)
@@ -386,14 +412,10 @@ if args.deqp:
     suite_filename = os.path.join(output_folder, "deqp-suite.toml")
     suite = open(suite_filename, "w")
     os.mkdir(out)
-    baseline = os.path.join(base, "{}-deqp-fail.csv".format(gpu_name))
     new_baseline = os.path.join(
         new_baseline_folder, "{}-deqp-fail.csv".format(gpu_name)
     )
 
-    if os.path.exists(baseline):
-        print_yellow("[baseline {}]".format(baseline), args.verbose > 0)
-
     deqp_tests = {
         "egl": args.deqp_egl,
         "gles2": args.deqp_gles2,
@@ -439,4 +461,4 @@ if args.deqp:
     ] + filters_args
     run_cmd(cmd, args.verbose)
     shutil.copy(os.path.join(out, "failures.csv"), new_baseline)
-    verify_results(baseline, new_baseline)
+    verify_results(new_baseline)

src/gallium/drivers/radeonsi/ci/raven-glcts-fail.csv

@@ -1,31 +0,0 @@
-KHR-GL46.gl_spirv.spirv_glsl_to_spirv_builtin_functions_test,Fail
-KHR-GL46.pipeline_statistics_query_tests_ARB.functional_geometry_shader_queries,Fail
-KHR-GL46.pipeline_statistics_query_tests_ARB.functional_primitives_vertices_submitted_and_clipping_input_output_primitives,Fail
-KHR-GL46.packed_pixels.pbo_rectangle.r16_snorm,Fail
-KHR-GL46.packed_pixels.pbo_rectangle.r8_snorm,Fail
-KHR-GL46.packed_pixels.pbo_rectangle.rg16_snorm,Fail
-KHR-GL46.packed_pixels.pbo_rectangle.rg8_snorm,Fail
-KHR-GL46.packed_pixels.pbo_rectangle.rgb16_snorm,Fail
-KHR-GL46.packed_pixels.pbo_rectangle.rgb8_snorm,Fail
-KHR-GL46.packed_pixels.pbo_rectangle.rgba16_snorm,Fail
-KHR-GL46.packed_pixels.pbo_rectangle.rgba8_snorm,Fail
-KHR-GL46.packed_pixels.rectangle.r16_snorm,Fail
-KHR-GL46.packed_pixels.rectangle.r8_snorm,Fail
-KHR-GL46.packed_pixels.rectangle.rg16_snorm,Fail
-KHR-GL46.packed_pixels.rectangle.rg8_snorm,Fail
-KHR-GL46.packed_pixels.rectangle.rgb16_snorm,Fail
-KHR-GL46.packed_pixels.rectangle.rgb8_snorm,Fail
-KHR-GL46.packed_pixels.rectangle.rgba16_snorm,Fail
-KHR-GL46.packed_pixels.rectangle.rgba8_snorm,Fail
-KHR-GL46.packed_pixels.varied_rectangle.r16_snorm,Fail
-KHR-GL46.packed_pixels.varied_rectangle.r8_snorm,Fail
-KHR-GL46.packed_pixels.varied_rectangle.rg16_snorm,Fail
-KHR-GL46.packed_pixels.varied_rectangle.rg8_snorm,Fail
-KHR-GL46.packed_pixels.varied_rectangle.rgb16_snorm,Fail
-KHR-GL46.packed_pixels.varied_rectangle.rgb8_snorm,Fail
-KHR-GL46.packed_pixels.varied_rectangle.rgba16_snorm,Fail
-KHR-GL46.packed_pixels.varied_rectangle.rgba8_snorm,Fail
-KHR-GL46.shader_ballot_tests.ShaderBallotFunctionRead,Fail
-KHR-GL46.shader_ballot_tests.ShaderBallotBitmasks,Fail
-KHR-GL46.sparse_texture_tests.SparseTextureCommitment,Fail
-KHR-GL46.sparse_buffer_tests.BufferStorageTest,Fail

src/gallium/drivers/radeonsi/ci/sienna_cichlid-glcts-fail.csv

@@ -1,10 +0,0 @@
-KHR-GL46.pipeline_statistics_query_tests_ARB.functional_geometry_shader_queries,Fail
-KHR-GL46.pipeline_statistics_query_tests_ARB.functional_primitives_vertices_submitted_and_clipping_input_output_primitives,Fail
-KHR-GL46.shader_ballot_tests.ShaderBallotFunctionRead,Fail
-KHR-GL46.sparse_texture2_tests.SparseTexture2Allocation,Fail
-KHR-GL46.sparse_texture2_tests.SparseTexture2Commitment,Fail
-KHR-GL46.sparse_texture2_tests.SparseTexture2Lookup,Fail
-KHR-GL46.sparse_texture2_tests.StandardPageSizesTestCase,Fail
-KHR-GL46.sparse_texture2_tests.UncommittedRegionsAccess,Fail
-KHR-GL46.sparse_texture_clamp_tests.SparseTextureClampLookupColor,Fail
-KHR-GL46.sparse_texture_clamp_tests.SparseTextureClampLookupResidency,Fail

src/gallium/drivers/radeonsi/ci/vega20-glcts-fail.csv

@@ -1,15 +0,0 @@
-KHR-GL46.direct_state_access.framebuffers_texture_attachment,Fail
-KHR-GL46.direct_state_access.framebuffers_texture_layer_attachment,Fail
-KHR-GL46.geometry_shader.adjacency.adjacency_indiced_triangle_strip,Fail
-KHR-GL46.geometry_shader.adjacency.adjacency_non_indiced_triangle_strip,Fail
-KHR-GL46.geometry_shader.rendering.rendering.triangles_with_adjacency_input_line_strip_output_triangle_strip_adjacency_drawcall,Fail
-KHR-GL46.geometry_shader.rendering.rendering.triangles_with_adjacency_input_triangle_strip_output_triangle_strip_adjacency_drawcall,Fail
-KHR-GL46.gl_spirv.spirv_glsl_to_spirv_builtin_functions_test,Fail
-KHR-GL46.shader_ballot_tests.ShaderBallotFunctionRead,Fail
-KHR-GL46.sparse_buffer_tests.BufferStorageTest,Fail
-KHR-GL46.sparse_texture2_tests.SparseTexture2Lookup,Fail
-KHR-GL46.sparse_texture2_tests.StandardPageSizesTestCase,Fail
-KHR-GL46.sparse_texture2_tests.UncommittedRegionsAccess,Fail
-KHR-GL46.sparse_texture_clamp_tests.SparseTextureClampLookupColor,Fail
-KHR-GL46.sparse_texture_clamp_tests.SparseTextureClampLookupResidency,Fail
-KHR-GL46.sparse_texture_tests.SparseTextureCommitment,Fail