ci/lava: Use new pyutils container

Instead of unpacking the x86_64_build container and its billion build
dependencies every time, switch to using only what's in the minimal
pyutils container, and the Python scripts we get as an artifact from the
python-test job. Pulling the artifacts from S3 rather than using GitLab
is also much more efficient.

This should substantially reduce the runtime required to get to testing.

Signed-off-by: Daniel Stone <daniels@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/31151>

.gitlab-ci/container/debian/x86_64_pyutils.sh

@@ -24,6 +24,7 @@ EPHEMERAL=(
     g++
     gcc
     libc6-dev
+    perl
     python3-dev
 )
 
@@ -33,7 +34,6 @@ DEPS=(
     file
     findutils
     git
-    python3-mako
     python3-pil
     python3-pip
     python3-ply

.gitlab-ci/lava/lava-gitlab-ci.yml

@@ -8,8 +8,8 @@ variables:
   variables:
     GIT_STRATEGY: none # testing doesn't build anything from source
     FDO_CI_CONCURRENT: 6 # should be replaced by per-machine definitions
-    # proxy used to cache data locally
-    FDO_HTTP_CACHE_URI: "http://caching-proxy/cache/?uri="
+    # the dispatchers use this to cache data locally
+    LAVA_HTTP_CACHE_URI: "http://caching-proxy/cache/?uri="
     # base system generated by the container build job, shared between many pipelines
     BASE_SYSTEM_HOST_PREFIX: "${S3_HOST}/${S3_KERNEL_BUCKET}"
     BASE_SYSTEM_MAINLINE_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${FDO_UPSTREAM_REPO}/${DISTRIBUTION_TAG}/${DEBIAN_ARCH}"
@@ -17,11 +17,15 @@ variables:
     # per-job build artifacts
     JOB_ROOTFS_OVERLAY_PATH: "${JOB_ARTIFACTS_BASE}/job-rootfs-overlay.tar.gz"
     JOB_RESULTS_PATH: "${JOB_ARTIFACTS_BASE}/results.tar.zst"
-    S3_ARTIFACT_NAME: "mesa-${ARCH}-default-debugoptimized"
+    LAVA_S3_ARTIFACT_NAME: "mesa-${ARCH}-default-debugoptimized"
+    S3_ARTIFACT_NAME: "mesa-python-test"
     S3_RESULTS_UPLOAD: "${JOB_ARTIFACTS_BASE}"
     PIGLIT_NO_WINDOW: 1
     VISIBILITY_GROUP: "Collabora+fdo"
+  before_script:
+    - !reference [.download_s3, before_script]
   script:
+    - . artifacts/setup-test-env.sh
     - ./artifacts/lava/lava-submit.sh
   artifacts:
     name: "${CI_PROJECT_NAME}_${CI_JOB_NAME}"
@@ -37,8 +41,13 @@ variables:
   after_script:
     - curl -L --retry 4 -f --retry-all-errors --retry-delay 60 -s "https://${JOB_RESULTS_PATH}" | tar --zstd -x
   needs:
-    - alpine/x86_64_lava_ssh_client
     - !reference [.required-for-hardware-jobs, needs]
+    - job: alpine/x86_64_lava_ssh_client
+      artifacts: false
+    - job: debian/x86_64_pyutils
+      artifacts: false
+    - job: python-test
+      artifacts: false
 
 .lava-test:arm32:
   variables:
@@ -49,14 +58,15 @@ variables:
     BOOT_METHOD: u-boot
   extends:
     - .use-debian/arm64_build # for same $MESA_ARTIFACTS_TAG as in kernel+rootfs_arm32
-    - .use-debian/x86_64_build
+    - .use-debian/x86_64_pyutils
     - .lava-test
     - .use-kernel+rootfs-arm
   needs:
     - !reference [.lava-test, needs]
-    - kernel+rootfs_arm32
-    - debian/x86_64_build
-    - debian-arm32
+    - job: kernel+rootfs_arm32
+      artifacts: false
+    - job: debian-arm32
+      artifacts: false
 
 .lava-test-deqp:arm32:
   extends:
@@ -73,16 +83,15 @@ variables:
     BOOT_METHOD: u-boot
   extends:
     - .use-debian/arm64_build # for same $MESA_ARTIFACTS_TAG as in kernel+rootfs_arm64
-    - .use-debian/x86_64_build
+    - .use-debian/x86_64_pyutils
     - .lava-test
     - .use-kernel+rootfs-arm
-  dependencies:
-    - debian-arm64
   needs:
     - !reference [.lava-test, needs]
-    - kernel+rootfs_arm64
-    - debian/x86_64_build
-    - debian-arm64
+    - job: kernel+rootfs_arm64
+      artifacts: false
+    - job: debian-arm64
+      artifacts: false
 
 .lava-test-deqp:arm64:
   variables:
@@ -99,13 +108,15 @@ variables:
     BOOT_METHOD: u-boot
   extends:
     - .use-debian/x86_64_build-base # for same $MESA_ARTIFACTS_BASE_TAG as in kernel+rootfs_x86_64
-    - .use-debian/x86_64_build
+    - .use-debian/x86_64_pyutils
     - .lava-test
     - .use-kernel+rootfs-x86_64
   needs:
     - !reference [.lava-test, needs]
-    - kernel+rootfs_x86_64
-    - debian-testing
+    - job: kernel+rootfs_x86_64
+      artifacts: false
+    - job: debian-testing
+      artifacts: false
 
 .lava-test-deqp:x86_64:
   variables:

.gitlab-ci/lava/lava-submit.sh

@@ -32,7 +32,7 @@ section_end variables
 tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ .
 ci-fairy s3cp --token-file "${S3_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}"
 
-ARTIFACT_URL="${FDO_HTTP_CACHE_URI:-}https://${PIPELINE_ARTIFACTS_BASE}/${S3_ARTIFACT_NAME:?}.tar.zst"
+ARTIFACT_URL="${FDO_HTTP_CACHE_URI:-}https://${PIPELINE_ARTIFACTS_BASE}/${LAVA_S3_ARTIFACT_NAME:?}.tar.zst"
 
 touch results/lava.log
 tail -f results/lava.log &
@@ -44,7 +44,7 @@ PYTHONPATH=artifacts/ artifacts/lava/lava_job_submitter.py \
 	--kernel-url-prefix "${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}" \
 	--kernel-external "${FORCE_KERNEL_TAG}" \
 	--build-url "${ARTIFACT_URL}" \
-	--job-rootfs-overlay-url "${FDO_HTTP_CACHE_URI:-}https://${JOB_ROOTFS_OVERLAY_PATH}" \
+	--job-rootfs-overlay-url "${LAVA_HTTP_CACHE_URI:-}https://${JOB_ROOTFS_OVERLAY_PATH}" \
 	--job-timeout-min ${JOB_TIMEOUT:-30} \
 	--first-stage-init artifacts/ci-common/init-stage1.sh \
 	--ci-project-dir "${CI_PROJECT_DIR}" \