diff --git a/.gitlab-ci/android-cts-runner.sh b/.gitlab-ci/android-cts-runner.sh index 72bfd87040a..30f51efe01e 100755 --- a/.gitlab-ci/android-cts-runner.sh +++ b/.gitlab-ci/android-cts-runner.sh @@ -4,6 +4,8 @@ . "${SCRIPTS_DIR}/setup-test-env.sh" +ci_tag_test_time_check "ANDROID_CTS_TAG" + export PATH=/android-cts/jdk/bin/:$PATH export JAVA_HOME=/android-cts/jdk diff --git a/.gitlab-ci/android-runner.sh b/.gitlab-ci/android-runner.sh index 95e4c75cfdd..0b751722042 100755 --- a/.gitlab-ci/android-runner.sh +++ b/.gitlab-ci/android-runner.sh @@ -152,7 +152,7 @@ if [ "$OLD_SF_PID" == "$NEW_SF_PID" ]; then exit 1 fi -if [ -n "${USE_ANDROID_CTS:-}" ]; then +if [ -n "${ANDROID_CTS_TAG:-}" ]; then # The script sets EXIT_CODE . "$(dirname "$0")/android-cts-runner.sh" else diff --git a/.gitlab-ci/conditional-build-image-tags.yml b/.gitlab-ci/conditional-build-image-tags.yml index ad6cde39c38..56b2816c48f 100644 --- a/.gitlab-ci/conditional-build-image-tags.yml +++ b/.gitlab-ci/conditional-build-image-tags.yml @@ -1,4 +1,5 @@ variables: + CONDITIONAL_BUILD_ANDROID_CTS_TAG: 7dc065d0dbc5af2614ac81d805e5a15f CONDITIONAL_BUILD_ANGLE_TAG: f62910e55be46e37cc867d037e4a8121 CONDITIONAL_BUILD_CROSVM_TAG: 0f59350b1052bdbb28b65a832b494377 CONDITIONAL_BUILD_FLUSTER_TAG: 3bc3afd7468e106afcbfd569a85f34f9 diff --git a/.gitlab-ci/container/build-android-cts.sh b/.gitlab-ci/container/build-android-cts.sh new file mode 100644 index 00000000000..111520420ee --- /dev/null +++ b/.gitlab-ci/container/build-android-cts.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# When changing this file, you need to bump the following +# .gitlab-ci/image-tags.yml tags: +# DEBIAN_TEST_ANDROID_TAG + +# This script runs in a container to: +# 1. Download the Android CTS (Compatibility Test Suite) +# 2. Filter out unneeded test modules +# 3. Compress and upload the stripped version to S3 +# Note: The 'build-' prefix in the filename is only to make it compatible +# with the bin/ci/update_tag.py script. + +set -euo pipefail + +section_start android-cts "Downloading Android CTS" + +# xtrace is getting lost with the section switching +set -x + +# Do a very early check to make sure the tag is correct without the need of +# setting up the environment variables locally +ci_tag_build_time_check "ANDROID_CTS_TAG" + +ANDROID_CTS_VERSION="${ANDROID_VERSION}_r1" +ANDROID_CTS_DEVICE_ARCH="x86" + +# Download the stripped CTS from S3, because the CTS download from Google can take 20 minutes +CTS_FILENAME="android-cts-${ANDROID_CTS_VERSION}-linux_x86-${ANDROID_CTS_DEVICE_ARCH}" +ARTIFACT_PATH="${DATA_STORAGE_PATH}/android-cts/${ANDROID_CTS_TAG}.tar.zst" + +if FOUND_ARTIFACT_URL="$(find_s3_project_artifact "${ARTIFACT_PATH}")"; then + echo "Found Android CTS at: ${FOUND_ARTIFACT_URL}" + curl-with-retry "${FOUND_ARTIFACT_URL}" | tar --zstd -x -C / +else + echo "No cached CTS found, downloading from Google and uploading to S3..." + curl-with-retry --remote-name "https://dl.google.com/dl/android/cts/${CTS_FILENAME}.zip" + + # Disable zipbomb detection, because the CTS zip file is too big + # At least locally, it is detected as a zipbomb + UNZIP_DISABLE_ZIPBOMB_DETECTION=true \ + unzip -q -d / "${CTS_FILENAME}.zip" + rm "${CTS_FILENAME}.zip" + + # Keep only the interesting tests to save space + # shellcheck disable=SC2086 # we want word splitting + ANDROID_CTS_MODULES_KEEP_EXPRESSION=$(printf "%s|" $ANDROID_CTS_MODULES | sed -e 's/|$//g') + find /android-cts/testcases/ -mindepth 1 -type d | grep -v -E "$ANDROID_CTS_MODULES_KEEP_EXPRESSION" | xargs rm -rf + + # Using zstd compressed tarball instead of zip, the compression ratio is almost the same, but + # the extraction is faster, also LAVA overlays don't support zip compression. + tar --zstd -cf "${CTS_FILENAME}.tar.zst" /android-cts + ci-fairy s3cp --token-file "${S3_JWT_FILE}" "${CTS_FILENAME}.tar.zst" \ + "https://${S3_BASE_PATH}/${CI_PROJECT_PATH}/${ARTIFACT_PATH}" +fi + +section_end android-cts diff --git a/.gitlab-ci/container/debian/x86_64_test-android.sh b/.gitlab-ci/container/debian/x86_64_test-android.sh index 4cdfea852a1..c45e3542085 100755 --- a/.gitlab-ci/container/debian/x86_64_test-android.sh +++ b/.gitlab-ci/container/debian/x86_64_test-android.sh @@ -160,44 +160,7 @@ section_end cuttlefish ############### Downloading Android CTS -section_start android-cts "Downloading Android CTS" - -# xtrace is getting lost with the section switching -set -x - -ANDROID_CTS_VERSION="${ANDROID_VERSION}_r1" -ANDROID_CTS_DEVICE_ARCH="x86" - -# Download the stripped CTS from S3, because the CTS download from Google can take 20 minutes -CTS_FILENAME="android-cts-${ANDROID_CTS_VERSION}-linux_x86-${ANDROID_CTS_DEVICE_ARCH}" -ARTIFACT_PATH="${DATA_STORAGE_PATH}/android-cts/${DEBIAN_TEST_ANDROID_TAG}--${CTS_FILENAME}.tar.zst" - -if FOUND_ARTIFACT_URL="$(find_s3_project_artifact "${ARTIFACT_PATH}")"; then - echo "Found Android CTS at: ${FOUND_ARTIFACT_URL}" - curl-with-retry "${FOUND_ARTIFACT_URL}" | tar --zstd -x -C / -else - echo "No cached CTS found, downloading from Google and uploading to S3..." - curl-with-retry --remote-name "https://dl.google.com/dl/android/cts/${CTS_FILENAME}.zip" - - # Disable zipbomb detection, because the CTS zip file is too big - # At least locally, it is detected as a zipbomb - UNZIP_DISABLE_ZIPBOMB_DETECTION=TRUE \ - unzip -q -d / "${CTS_FILENAME}.zip" - rm "${CTS_FILENAME}.zip" - - # Keep only the interesting tests to save space - # shellcheck disable=SC2086 # we want word splitting - ANDROID_CTS_MODULES_KEEP_EXPRESSION=$(printf "%s|" $ANDROID_CTS_MODULES | sed -e 's/|$//g') - find /android-cts/testcases/ -mindepth 1 -type d | grep -v -E "$ANDROID_CTS_MODULES_KEEP_EXPRESSION" | xargs rm -rf - - # Using zstd compressed tarball instead of zip, the compression ratio is almost the same, but - # the extraction is faster, also LAVA overlays don't support zip compression. - tar --zstd -cf "${CTS_FILENAME}.tar.zst" /android-cts - ci-fairy s3cp --token-file "${S3_JWT_FILE}" "${CTS_FILENAME}.tar.zst" \ - "https://${S3_BASE_PATH}/${CI_PROJECT_PATH}/${ARTIFACT_PATH}" -fi - -section_end android-cts +. .gitlab-ci/container/build-android-cts.sh ############### Uninstall the build software diff --git a/.gitlab-ci/container/gitlab-ci.yml b/.gitlab-ci/container/gitlab-ci.yml index cc7384833bf..6dd3a83b2fb 100644 --- a/.gitlab-ci/container/gitlab-ci.yml +++ b/.gitlab-ci/container/gitlab-ci.yml @@ -69,7 +69,7 @@ FDO_DISTRIBUTION_EXEC: 'bash .gitlab-ci/container/container_job_trampoline.sh "${CI_JOB_NAME}"' # no need to pull the whole repo to build the container image GIT_STRATEGY: none - CI_BUILD_COMPONENTS: "$CI_BUILD_COMPONENTS_ANGLE $CI_BUILD_COMPONENTS_CROSVM $CI_BUILD_COMPONENTS_FLUSTER $CI_BUILD_COMPONENTS_PIGLIT $CI_BUILD_COMPONENTS_VKD3D_PROTON" + CI_BUILD_COMPONENTS: "$CI_BUILD_COMPONENTS_ANDROID_CTS $CI_BUILD_COMPONENTS_ANGLE $CI_BUILD_COMPONENTS_CROSVM $CI_BUILD_COMPONENTS_FLUSTER $CI_BUILD_COMPONENTS_PIGLIT $CI_BUILD_COMPONENTS_VKD3D_PROTON" .container-builds-angle: variables: @@ -96,8 +96,14 @@ VKD3D_PROTON_TAG: "${CONDITIONAL_BUILD_VKD3D_PROTON_TAG}" CI_BUILD_COMPONENTS_VKD3D_PROTON: vkd3d-proton +.container-builds-android-cts: + variables: + ANDROID_CTS_TAG: "${CONDITIONAL_BUILD_ANDROID_CTS_TAG}" + CI_BUILD_COMPONENTS_ANDROID_CTS: android-cts + .container-builds-android: extends: + - .container-builds-android-cts - .container-builds-angle .container-builds-arm32: diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml index 6a5deb8e7f8..024780812e9 100644 --- a/.gitlab-ci/image-tags.yml +++ b/.gitlab-ci/image-tags.yml @@ -23,14 +23,14 @@ variables: DEBIAN_BUILD_TAG: "20250611-rust" - DEBIAN_TEST_ANDROID_TAG: "20250618-s3-cts" + DEBIAN_TEST_ANDROID_TAG: "20250620-sttag" DEBIAN_TEST_GL_TAG: "20250616-vkcts-main" DEBIAN_TEST_VIDEO_TAG: "20250609-helper" DEBIAN_TEST_VK_TAG: "20250619-vkd3d" ALPINE_X86_64_BUILD_TAG: "20250611-rust" ALPINE_X86_64_LAVA_SSH_TAG: "20250423-rootfs" - ALPINE_X86_64_LAVA_TRIGGER_TAG: "20250613-cuttlefish" + ALPINE_X86_64_LAVA_TRIGGER_TAG: "20250619-cts-ovl" FEDORA_X86_64_BUILD_TAG: "20250611-rust" diff --git a/.gitlab-ci/lava/lava-submit.sh b/.gitlab-ci/lava/lava-submit.sh index cf1aa570388..124ecf1ec2f 100755 --- a/.gitlab-ci/lava/lava-submit.sh +++ b/.gitlab-ci/lava/lava-submit.sh @@ -69,6 +69,16 @@ if [ -n "${HWCI_KERNEL_MODULES:-}" ]; then --format=tar ) fi +if [ -n "${ANDROID_CTS_TAG:-}" ]; then + LAVA_EXTRA_OVERLAYS+=( + - append-overlay + --name=android-cts + --url="$(find_s3_project_artifact "${DATA_STORAGE_PATH}/android-cts/${ANDROID_CTS_TAG}.tar.zst")" + --path="/" + --format=tar + --compression=zstd + ) +fi if [ -n "${VKD3D_PROTON_TAG:-}" ]; then LAVA_EXTRA_OVERLAYS+=( - append-overlay diff --git a/.gitlab-ci/test/gitlab-ci.yml b/.gitlab-ci/test/gitlab-ci.yml index 5f2aa89d25c..4e5896ed863 100644 --- a/.gitlab-ci/test/gitlab-ci.yml +++ b/.gitlab-ci/test/gitlab-ci.yml @@ -187,6 +187,10 @@ yaml-toml-shell-py-test: paths: - results/ +.test-android-cts: + variables: + ANDROID_CTS_TAG: ${CONDITIONAL_BUILD_ANDROID_CTS_TAG} + .test-angle: variables: ANGLE_TAG: ${CONDITIONAL_BUILD_ANGLE_TAG} diff --git a/src/gallium/frontends/lavapipe/ci/gitlab-ci.yml b/src/gallium/frontends/lavapipe/ci/gitlab-ci.yml index db7c74db6f1..6b788a4cf99 100644 --- a/src/gallium/frontends/lavapipe/ci/gitlab-ci.yml +++ b/src/gallium/frontends/lavapipe/ci/gitlab-ci.yml @@ -74,12 +74,12 @@ android-angle-lavapipe: android-angle-lavapipe-cts: variables: - USE_ANDROID_CTS: 1 ANDROID_GPU_MODE: mesa_swrast_guest_angle GPU_VERSION: lvp-android-angle MESA_VK_IGNORE_CONFORMANCE_WARNING: 1 timeout: 15m extends: - .test-android + - .test-android-cts - .lavapipe-rules - .test-angle