From 2d77c7f9c995811787da3d6cb4bbdfa188997ffd Mon Sep 17 00:00:00 2001
From: Guilherme Gallo <guilherme.gallo@collabora.com>
Date: Tue, 19 Jul 2022 21:41:55 -0300
Subject: [PATCH] ci/skqp: Add an option to run all tests

When the skqp is introduced to a new driver, the best practice is to
run all available tests from skqp and classifying the
failing/crashing/flaking ones.
The default behavior of skqp is to run the tests from the commit where
the skqp built, which may not be adequate for the target driver.

Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17835>
---
 .gitlab-ci/skqp-runner.sh | 65 +++++++++++++++++++++++++++++++++------
 1 file changed, 56 insertions(+), 9 deletions(-)

diff --git a/.gitlab-ci/skqp-runner.sh b/.gitlab-ci/skqp-runner.sh
index 68aca2d3377..5f8199c2283 100755
--- a/.gitlab-ci/skqp-runner.sh
+++ b/.gitlab-ci/skqp-runner.sh
@@ -23,6 +23,10 @@
 # SOFTWARE.
 
 
+run_all_tests() {
+    rm "${SKQP_ASSETS_DIR}"/skqp/*.txt
+}
+
 copy_tests_files() (
     # Copy either unit test or render test files from a specific driver given by
     # GPU VERSION variable.
@@ -49,6 +53,20 @@ copy_tests_files() (
     fi
 )
 
+resolve_tests_files() {
+    if [ -n "${RUN_ALL_TESTS}" ]
+    then
+        run_all_tests
+        return
+    fi
+
+    SKQP_BACKEND=${1}
+    if ! copy_tests_files "${SKQP_BACKEND}"
+    then
+        echo "No override test file found for ${SKQP_BACKEND}. Using the default one."
+    fi
+}
+
 test_vk_backend() {
     if echo "${SKQP_BACKENDS}" | grep -qE 'vk'
     then
@@ -78,8 +96,42 @@ setup_backends() {
     fi
 }
 
+usage() {
+    cat <<EOF
+    Usage: $(basename "$0") [-a]
+
+    Arguments:
+        -a: Run all unit tests and render tests, useful when introducing a new driver to skqp.
+EOF
+}
+
+parse_args() {
+    while getopts ':ah' opt; do
+        case "$opt" in
+            a)
+            echo "Running all skqp tests"
+            export RUN_ALL_TESTS=1
+            shift
+            ;;
+
+            h)
+            usage
+            exit 0
+            ;;
+
+            ?)
+            echo "Invalid command option."
+            usage
+            exit 1
+            ;;
+        esac
+    done
+}
+
 set -ex
 
+parse_args "${@}"
+
 # Needed so configuration files can contain paths to files in /install
 ln -sf "$CI_PROJECT_DIR"/install /install
 INSTALL=${PWD}/install
@@ -102,17 +154,12 @@ mkdir -p "${SKQP_ASSETS_DIR}"/skqp
 SKQP_EXITCODE=0
 for SKQP_BACKEND in ${SKQP_BACKENDS}
 do
-    set -e
-    if !  copy_tests_files "${SKQP_BACKEND}"
-    then
-        echo "No override test file found for ${SKQP_BACKEND}. Using the default one."
-    fi
-
-    set +e
+    resolve_tests_files "${SKQP_BACKEND}"
     SKQP_BACKEND_RESULTS_DIR="${SKQP_RESULTS_DIR}"/"${SKQP_BACKEND}"
     mkdir -p "${SKQP_BACKEND_RESULTS_DIR}"
-    /skqp/skqp "${SKQP_ASSETS_DIR}" "${SKQP_BACKEND_RESULTS_DIR}" "${SKQP_BACKEND}_"
-    BACKEND_EXITCODE=$?
+    BACKEND_EXITCODE=0
+    /skqp/skqp "${SKQP_ASSETS_DIR}" "${SKQP_BACKEND_RESULTS_DIR}" "${SKQP_BACKEND}_" ||
+        BACKEND_EXITCODE=$?
 
     if [ ! $BACKEND_EXITCODE -eq 0 ]
     then