ci/android: Use structured tag for Android CTS version

Structured tagging ensures that we are building and testing the current component version specified in the commit by matching the checksum of the related build script file. In this case, it is worthy to isolate the Android CTS version part, because we don't need to rebuild the entire test-android container when we change the CTS version or the CTS modules filtering. PS: actually the new file `build-android-cts.sh` is not building anything, it is just downloads, filters, compress and reupload the stripped version to S3. The `build-` prefix is to make it work transparently with `bin/ci/update_tag.py` script. Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/35596>
2025-06-18 21:39:13 -03:00
parent b26ad1783d
commit 969bb7bd70
10 changed files with 86 additions and 43 deletions
--- a/.gitlab-ci/android-cts-runner.sh
+++ b/.gitlab-ci/android-cts-runner.sh
@@ -4,6 +4,8 @@

 . "${SCRIPTS_DIR}/setup-test-env.sh"

+ci_tag_test_time_check "ANDROID_CTS_TAG"
+
 export PATH=/android-cts/jdk/bin/:$PATH
 export JAVA_HOME=/android-cts/jdk

--- a/.gitlab-ci/android-runner.sh
+++ b/.gitlab-ci/android-runner.sh
@@ -152,7 +152,7 @@ if [ "$OLD_SF_PID" == "$NEW_SF_PID" ]; then
     exit 1
 fi

-if [ -n "${USE_ANDROID_CTS:-}" ]; then
+if [ -n "${ANDROID_CTS_TAG:-}" ]; then
  # The script sets EXIT_CODE
  . "$(dirname "$0")/android-cts-runner.sh"
 else
--- a/.gitlab-ci/conditional-build-image-tags.yml
+++ b/.gitlab-ci/conditional-build-image-tags.yml
@@ -1,4 +1,5 @@
 variables:
+  CONDITIONAL_BUILD_ANDROID_CTS_TAG: 7dc065d0dbc5af2614ac81d805e5a15f
  CONDITIONAL_BUILD_ANGLE_TAG: f62910e55be46e37cc867d037e4a8121
  CONDITIONAL_BUILD_CROSVM_TAG: 0f59350b1052bdbb28b65a832b494377
  CONDITIONAL_BUILD_FLUSTER_TAG: 3bc3afd7468e106afcbfd569a85f34f9
--- a/.gitlab-ci/container/build-android-cts.sh
+++ b/.gitlab-ci/container/build-android-cts.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+#
+# When changing this file, you need to bump the following
+# .gitlab-ci/image-tags.yml tags:
+# DEBIAN_TEST_ANDROID_TAG
+
+# This script runs in a container to:
+# 1. Download the Android CTS (Compatibility Test Suite)
+# 2. Filter out unneeded test modules
+# 3. Compress and upload the stripped version to S3
+# Note: The 'build-' prefix in the filename is only to make it compatible
+# with the bin/ci/update_tag.py script.
+
+set -euo pipefail
+
+section_start android-cts "Downloading Android CTS"
+
+# xtrace is getting lost with the section switching
+set -x
+
+# Do a very early check to make sure the tag is correct without the need of
+# setting up the environment variables locally
+ci_tag_build_time_check "ANDROID_CTS_TAG"
+
+ANDROID_CTS_VERSION="${ANDROID_VERSION}_r1"
+ANDROID_CTS_DEVICE_ARCH="x86"
+
+# Download the stripped CTS from S3, because the CTS download from Google can take 20 minutes
+CTS_FILENAME="android-cts-${ANDROID_CTS_VERSION}-linux_x86-${ANDROID_CTS_DEVICE_ARCH}"
+ARTIFACT_PATH="${DATA_STORAGE_PATH}/android-cts/${ANDROID_CTS_TAG}.tar.zst"
+
+if FOUND_ARTIFACT_URL="$(find_s3_project_artifact "${ARTIFACT_PATH}")"; then
+    echo "Found Android CTS at: ${FOUND_ARTIFACT_URL}"
+    curl-with-retry "${FOUND_ARTIFACT_URL}" | tar --zstd -x -C /
+else
+    echo "No cached CTS found, downloading from Google and uploading to S3..."
+    curl-with-retry --remote-name "https://dl.google.com/dl/android/cts/${CTS_FILENAME}.zip"
+
+    # Disable zipbomb detection, because the CTS zip file is too big
+    # At least locally, it is detected as a zipbomb
+    UNZIP_DISABLE_ZIPBOMB_DETECTION=true \
+        unzip -q -d / "${CTS_FILENAME}.zip"
+    rm "${CTS_FILENAME}.zip"
+
+    # Keep only the interesting tests to save space
+    # shellcheck disable=SC2086 # we want word splitting
+    ANDROID_CTS_MODULES_KEEP_EXPRESSION=$(printf "%s|" $ANDROID_CTS_MODULES | sed -e 's/|$//g')
+    find /android-cts/testcases/ -mindepth 1 -type d | grep -v -E "$ANDROID_CTS_MODULES_KEEP_EXPRESSION" | xargs rm -rf
+
+    # Using zstd compressed tarball instead of zip, the compression ratio is almost the same, but
+    # the extraction is faster, also LAVA overlays don't support zip compression.
+    tar --zstd -cf "${CTS_FILENAME}.tar.zst" /android-cts
+    ci-fairy s3cp --token-file "${S3_JWT_FILE}" "${CTS_FILENAME}.tar.zst" \
+        "https://${S3_BASE_PATH}/${CI_PROJECT_PATH}/${ARTIFACT_PATH}"
+fi
+
+section_end android-cts
--- a/.gitlab-ci/container/debian/x86_64_test-android.sh
+++ b/.gitlab-ci/container/debian/x86_64_test-android.sh
@@ -160,44 +160,7 @@ section_end cuttlefish

 ############### Downloading Android CTS

-section_start android-cts "Downloading Android CTS"
-
-# xtrace is getting lost with the section switching
-set -x
-
-ANDROID_CTS_VERSION="${ANDROID_VERSION}_r1"
-ANDROID_CTS_DEVICE_ARCH="x86"
-
-# Download the stripped CTS from S3, because the CTS download from Google can take 20 minutes
-CTS_FILENAME="android-cts-${ANDROID_CTS_VERSION}-linux_x86-${ANDROID_CTS_DEVICE_ARCH}"
-ARTIFACT_PATH="${DATA_STORAGE_PATH}/android-cts/${DEBIAN_TEST_ANDROID_TAG}--${CTS_FILENAME}.tar.zst"
-
-if FOUND_ARTIFACT_URL="$(find_s3_project_artifact "${ARTIFACT_PATH}")"; then
-    echo "Found Android CTS at: ${FOUND_ARTIFACT_URL}"
-    curl-with-retry "${FOUND_ARTIFACT_URL}" | tar --zstd -x -C /
-else
-    echo "No cached CTS found, downloading from Google and uploading to S3..."
-    curl-with-retry --remote-name "https://dl.google.com/dl/android/cts/${CTS_FILENAME}.zip"
-
-    # Disable zipbomb detection, because the CTS zip file is too big
-    # At least locally, it is detected as a zipbomb
-    UNZIP_DISABLE_ZIPBOMB_DETECTION=TRUE \
-        unzip -q -d / "${CTS_FILENAME}.zip"
-    rm "${CTS_FILENAME}.zip"
-
-    # Keep only the interesting tests to save space
-    # shellcheck disable=SC2086 # we want word splitting
-    ANDROID_CTS_MODULES_KEEP_EXPRESSION=$(printf "%s|" $ANDROID_CTS_MODULES | sed -e 's/|$//g')
-    find /android-cts/testcases/ -mindepth 1 -type d | grep -v -E "$ANDROID_CTS_MODULES_KEEP_EXPRESSION" | xargs rm -rf
-
-    # Using zstd compressed tarball instead of zip, the compression ratio is almost the same, but
-    # the extraction is faster, also LAVA overlays don't support zip compression.
-    tar --zstd -cf "${CTS_FILENAME}.tar.zst" /android-cts
-    ci-fairy s3cp --token-file "${S3_JWT_FILE}" "${CTS_FILENAME}.tar.zst" \
-        "https://${S3_BASE_PATH}/${CI_PROJECT_PATH}/${ARTIFACT_PATH}"
-fi
-
-section_end android-cts
+. .gitlab-ci/container/build-android-cts.sh

 ############### Uninstall the build software

--- a/.gitlab-ci/container/gitlab-ci.yml
+++ b/.gitlab-ci/container/gitlab-ci.yml
@@ -69,7 +69,7 @@
    FDO_DISTRIBUTION_EXEC: 'bash .gitlab-ci/container/container_job_trampoline.sh "${CI_JOB_NAME}"'
    # no need to pull the whole repo to build the container image
    GIT_STRATEGY: none
-    CI_BUILD_COMPONENTS: "$CI_BUILD_COMPONENTS_ANGLE $CI_BUILD_COMPONENTS_CROSVM $CI_BUILD_COMPONENTS_FLUSTER $CI_BUILD_COMPONENTS_PIGLIT $CI_BUILD_COMPONENTS_VKD3D_PROTON"
+    CI_BUILD_COMPONENTS: "$CI_BUILD_COMPONENTS_ANDROID_CTS $CI_BUILD_COMPONENTS_ANGLE $CI_BUILD_COMPONENTS_CROSVM $CI_BUILD_COMPONENTS_FLUSTER $CI_BUILD_COMPONENTS_PIGLIT $CI_BUILD_COMPONENTS_VKD3D_PROTON"

 .container-builds-angle:
  variables:
@@ -96,8 +96,14 @@
    VKD3D_PROTON_TAG: "${CONDITIONAL_BUILD_VKD3D_PROTON_TAG}"
    CI_BUILD_COMPONENTS_VKD3D_PROTON: vkd3d-proton

+.container-builds-android-cts:
+  variables:
+    ANDROID_CTS_TAG: "${CONDITIONAL_BUILD_ANDROID_CTS_TAG}"
+    CI_BUILD_COMPONENTS_ANDROID_CTS: android-cts
+
 .container-builds-android:
  extends:
+    - .container-builds-android-cts
    - .container-builds-angle

 .container-builds-arm32:
--- a/.gitlab-ci/image-tags.yml
+++ b/.gitlab-ci/image-tags.yml
@@ -23,14 +23,14 @@ variables:

  DEBIAN_BUILD_TAG: "20250611-rust"

-  DEBIAN_TEST_ANDROID_TAG: "20250618-s3-cts"
+  DEBIAN_TEST_ANDROID_TAG: "20250620-sttag"
  DEBIAN_TEST_GL_TAG: "20250616-vkcts-main"
  DEBIAN_TEST_VIDEO_TAG: "20250609-helper"
  DEBIAN_TEST_VK_TAG: "20250619-vkd3d"

  ALPINE_X86_64_BUILD_TAG: "20250611-rust"
  ALPINE_X86_64_LAVA_SSH_TAG: "20250423-rootfs"
-  ALPINE_X86_64_LAVA_TRIGGER_TAG: "20250613-cuttlefish"
+  ALPINE_X86_64_LAVA_TRIGGER_TAG: "20250619-cts-ovl"

  FEDORA_X86_64_BUILD_TAG: "20250611-rust"

--- a/.gitlab-ci/lava/lava-submit.sh
+++ b/.gitlab-ci/lava/lava-submit.sh
@@ -69,6 +69,16 @@ if [ -n "${HWCI_KERNEL_MODULES:-}" ]; then
 		  --format=tar
 	)
 fi
+if [ -n "${ANDROID_CTS_TAG:-}" ]; then
+	LAVA_EXTRA_OVERLAYS+=(
+		- append-overlay
+		  --name=android-cts
+		  --url="$(find_s3_project_artifact "${DATA_STORAGE_PATH}/android-cts/${ANDROID_CTS_TAG}.tar.zst")"
+		  --path="/"
+		  --format=tar
+		  --compression=zstd
+	)
+fi
 if [ -n "${VKD3D_PROTON_TAG:-}" ]; then
 	LAVA_EXTRA_OVERLAYS+=(
 		- append-overlay
--- a/.gitlab-ci/test/gitlab-ci.yml
+++ b/.gitlab-ci/test/gitlab-ci.yml
@@ -187,6 +187,10 @@ yaml-toml-shell-py-test:
    paths:
      - results/

+.test-android-cts:
+  variables:
+    ANDROID_CTS_TAG: ${CONDITIONAL_BUILD_ANDROID_CTS_TAG}
+
 .test-angle:
  variables:
    ANGLE_TAG: ${CONDITIONAL_BUILD_ANGLE_TAG}
--- a/src/gallium/frontends/lavapipe/ci/gitlab-ci.yml
+++ b/src/gallium/frontends/lavapipe/ci/gitlab-ci.yml
@@ -74,12 +74,12 @@ android-angle-lavapipe:

 android-angle-lavapipe-cts:
  variables:
-    USE_ANDROID_CTS: 1
    ANDROID_GPU_MODE: mesa_swrast_guest_angle
    GPU_VERSION: lvp-android-angle
    MESA_VK_IGNORE_CONFORMANCE_WARNING: 1
  timeout: 15m
  extends:
    - .test-android
+    - .test-android-cts
    - .lavapipe-rules
    - .test-angle