ci/fluster/lava: Add fluster in LAVA rootfs

It will be used in next commits to run fluster vaapi tests on AMD
devices on LAVA.

Signed-off-by: Detlev Casanova <detlev.casanova@collabora.com>
Co-authored-by: Sergi Blanch Torne <sergi@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/26113>

.gitlab-ci.yml

@@ -104,6 +104,7 @@ variables:
   # Default priority for non-merge pipelines
   VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
   JOB_PRIORITY: 50
+  DATA_STORAGE_PATH: data_storage
 
 default:
   id_tokens:

.gitlab-ci/container/build-fluster.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC1091 # The relative paths in this file only become valid at runtime.
+# shellcheck disable=SC2034 # Variables are used in scripts called from here
+# shellcheck disable=SC2086 # we want word splitting
+
+# Install fluster in /usr/local.
+
+FLUSTER_REVISION="e997402978f62428fffc8e5a4a709690d9ca9bc5"
+
+git clone https://github.com/fluendo/fluster.git --single-branch --no-checkout
+
+pushd fluster || exit
+git checkout ${FLUSTER_REVISION}
+popd || exit
+
+if [ "${SKIP_UPDATE_FLUSTER_VECTORS}" != 1 ]; then
+    # Download the necessary vectors: H264, H265 and VP9
+    # When updating FLUSTER_REVISION, make sure to update the vectors if necessary or
+    # fluster-runner will report Missing results.
+    fluster/fluster.py download \
+	JVT-AVC_V1 JVT-FR-EXT JVT-MVC JVT-SVC_V1 \
+	JCT-VC-3D-HEVC JCT-VC-HEVC_V1 JCT-VC-MV-HEVC JCT-VC-RExt JCT-VC-SCC JCT-VC-SHVC \
+	VP9-TEST-VECTORS-HIGH VP9-TEST-VECTORS
+
+    # Build fluster vectors archive and upload it
+    tar --zstd -cf "vectors.tar.zst" fluster/resources/
+    ci-fairy s3cp --token-file "${S3_JWT_FILE}" "vectors.tar.zst" \
+          "https://${S3_PATH_FLUSTER}/vectors.tar.zst"
+
+    touch /lava-files/done
+    ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/done "https://${S3_PATH_FLUSTER}/done"
+
+    # Don't include the vectors in the rootfs
+    rm -fr fluster/resources/*
+fi
+
+mkdir -p "${ROOTFS}/usr/local/"
+mv fluster "${ROOTFS}/usr/local/"
+

.gitlab-ci/container/lava_build.sh

@@ -6,6 +6,10 @@
 # When changing this file, you need to bump the following
 # .gitlab-ci/image-tags.yml tags:
 # KERNEL_ROOTFS_TAG
+# If you need to update the fluster vectors cache without updating the fluster revision,
+# you can update the FLUSTER_VECTORS_VERSION tag in .gitlab-ci/image-tags.yml.
+# When changing FLUSTER_REVISION, KERNEL_ROOTFS_TAG needs to be updated as well to rebuild
+# the rootfs.
 
 set -e
 
@@ -16,6 +20,7 @@ set -o xtrace
 export DEBIAN_FRONTEND=noninteractive
 export LLVM_VERSION="${LLVM_VERSION:=15}"
 export FIRMWARE_FILES="${FIRMWARE_FILES}"
+export SKIP_UPDATE_FLUSTER_VECTORS=0
 
 check_minio()
 {
@@ -27,9 +32,22 @@ check_minio()
     fi
 }
 
+check_fluster()
+{
+    S3_PATH_FLUSTER="${S3_HOST}/${S3_KERNEL_BUCKET}/$1/${DATA_STORAGE_PATH}/fluster/${FLUSTER_VECTORS_VERSION}"
+    if curl -L --retry 4 -f --retry-delay 60 -s -X HEAD \
+      "https://${S3_PATH_FLUSTER}/done"; then
+        echo "Fluster vectors are up-to-date, skip downloading them."
+        export SKIP_UPDATE_FLUSTER_VECTORS=1
+    fi
+}
+
 check_minio "${FDO_UPSTREAM_REPO}"
 check_minio "${CI_PROJECT_PATH}"
 
+check_fluster "${FDO_UPSTREAM_REPO}"
+check_fluster "${CI_PROJECT_PATH}"
+
 . .gitlab-ci/container/container_pre_build.sh
 
 # Install rust, which we'll be using for deqp-runner.  It will be cleaned up at the end.
@@ -197,6 +215,7 @@ PKG_MESA_DEP=(
 )
 PKG_DEP=(
   libpng16-16
+  libva-wayland2
   libwaffle-1-0
   libpython3.11 python3 python3-lxml python3-mako python3-numpy python3-packaging python3-pil python3-renderdoc python3-requests python3-simplejson python3-yaml # Python
   sntp
@@ -213,6 +232,7 @@ PKG_DEP=(
 [ "$DEBIAN_ARCH" = "amd64" ] && PKG_ARCH=(
   firmware-amd-graphics
   firmware-misc-nonfree
+  gstreamer1.0-plugins-bad gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly gstreamer1.0-tools gstreamer1.0-vaapi libgstreamer1.0-0 # Fluster
   libgl1 libglu1-mesa
   inetutils-syslogd iptables libcap2
   libfontconfig1
@@ -366,6 +386,13 @@ fi
 . .gitlab-ci/container/build-kdl.sh
 mv /ci-kdl $ROOTFS/
 
+############### Install fluster
+if [[ ${DEBIAN_ARCH} = "amd64" ]]; then
+    section_start fluster "Install fluster"
+    . .gitlab-ci/container/build-fluster.sh
+    section_end fluster
+fi
+
 ############### Build local stuff for use by igt and kernel testing, which
 ############### will reuse most of our container build process from a specific
 ############### hash of the Mesa tree.

.gitlab-ci/image-tags.yml

@@ -43,6 +43,8 @@ variables:
    KERNEL_REPO: "gfx-ci/linux"
    PKG_REPO_REV: "bca9635d"
 
+   FLUSTER_VECTORS_VERSION: "5"
+
    WINDOWS_X64_MSVC_PATH: "windows/x86_64_msvc"
    WINDOWS_X64_MSVC_TAG: "20240827-v143"
 

.gitlab-ci/lava/lava-gitlab-ci.yml

@@ -22,6 +22,8 @@ variables:
     S3_RESULTS_UPLOAD: "${JOB_ARTIFACTS_BASE}"
     PIGLIT_NO_WINDOW: 1
     VISIBILITY_GROUP: "Collabora+fdo"
+    STORAGE_MAINLINE_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${FDO_UPSTREAM_REPO}/${DATA_STORAGE_PATH}"
+    STORAGE_FORK_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${CI_PROJECT_PATH}/${DATA_STORAGE_PATH}"
   before_script:
     - !reference [.download_s3, before_script]
   script: