From f406595ec73e195b14b9d67541d834389dfe4390 Mon Sep 17 00:00:00 2001
From: Deborah Brouwer <deborah.brouwer@collabora.com>
Date: Thu, 10 Oct 2024 10:31:31 -0700
Subject: [PATCH] ci/deqp-runner: implement max-fails

If we get more than 40 unexpected fails/crashes in a merge pipeline,
then stop early to preserve pipeline resources.

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/31429>
---
 .gitlab-ci.yml                    | 2 ++
 .gitlab-ci/common/generate-env.sh | 1 +
 .gitlab-ci/deqp-runner.sh         | 4 ++++
 3 files changed, 7 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f4d33f902e9..99fb384be2e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -36,6 +36,8 @@ workflow:
         KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}
         MESA_CI_PERFORMANCE_ENABLED: 1
         VALVE_INFRA_VANGOGH_JOB_PRIORITY: ""  # Empty tags are ignored by gitlab
+        # fast-fail in merge pipelines: stop early if we get this many unexpected fails/crashes
+        DEQP_RUNNER_MAX_FAILS: 40
     # post-merge pipeline
     - if: &is-post-merge $GITLAB_USER_LOGIN == "marge-bot" && $CI_PIPELINE_SOURCE == "push"
     # nightly pipeline
diff --git a/.gitlab-ci/common/generate-env.sh b/.gitlab-ci/common/generate-env.sh
index 2f9cbbb4cbd..ea4d95e4d09 100755
--- a/.gitlab-ci/common/generate-env.sh
+++ b/.gitlab-ci/common/generate-env.sh
@@ -35,6 +35,7 @@ VARS=(
     DEQP_EXPECTED_RENDERER
     DEQP_FRACTION
     DEQP_HEIGHT
+    DEQP_RUNNER_MAX_FAILS
     DEQP_RUNNER_OPTIONS
     DEQP_SUITE
     DEQP_TEMP_DIR
diff --git a/.gitlab-ci/deqp-runner.sh b/.gitlab-ci/deqp-runner.sh
index f626eb36bf4..164002a3ce0 100755
--- a/.gitlab-ci/deqp-runner.sh
+++ b/.gitlab-ci/deqp-runner.sh
@@ -128,6 +128,10 @@ if [ -n "$VK_DRIVER" ] && [ -z "$DEQP_SUITE" ]; then
     DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --tests-per-group ${DEQP_RUNNER_TESTS_PER_GROUP:-5000}"
 fi
 
+if [ -n "${DEQP_RUNNER_MAX_FAILS:-}" ]; then
+    DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --max-fails ${DEQP_RUNNER_MAX_FAILS}"
+fi
+
 # Set the path to VK validation layer settings (in case it ends up getting loaded)
 # Note: If you change the format of this filename, look through the rest of the
 # tree for other places that need to be kept in sync (e.g.