From aa11f3d6304f12b6073944756e6806474875a269 Mon Sep 17 00:00:00 2001 From: Detlev Casanova Date: Wed, 8 Nov 2023 10:19:23 -0500 Subject: [PATCH] ci/fluster/lava: Add fluster in LAVA rootfs It will be used in next commits to run fluster vaapi tests on AMD devices on LAVA. Signed-off-by: Detlev Casanova Co-authored-by: Sergi Blanch Torne Part-of: --- .gitlab-ci.yml | 1 + .gitlab-ci/container/build-fluster.sh | 40 +++++++++++++++++++++++++++ .gitlab-ci/container/lava_build.sh | 27 ++++++++++++++++++ .gitlab-ci/image-tags.yml | 2 ++ .gitlab-ci/lava/lava-gitlab-ci.yml | 2 ++ 5 files changed, 72 insertions(+) create mode 100644 .gitlab-ci/container/build-fluster.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 462a97ff483..4e66e0ae4a0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -104,6 +104,7 @@ variables: # Default priority for non-merge pipelines VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low JOB_PRIORITY: 50 + DATA_STORAGE_PATH: data_storage default: id_tokens: diff --git a/.gitlab-ci/container/build-fluster.sh b/.gitlab-ci/container/build-fluster.sh new file mode 100644 index 00000000000..18925da6c3a --- /dev/null +++ b/.gitlab-ci/container/build-fluster.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC1091 # The relative paths in this file only become valid at runtime. +# shellcheck disable=SC2034 # Variables are used in scripts called from here +# shellcheck disable=SC2086 # we want word splitting + +# Install fluster in /usr/local. + +FLUSTER_REVISION="e997402978f62428fffc8e5a4a709690d9ca9bc5" + +git clone https://github.com/fluendo/fluster.git --single-branch --no-checkout + +pushd fluster || exit +git checkout ${FLUSTER_REVISION} +popd || exit + +if [ "${SKIP_UPDATE_FLUSTER_VECTORS}" != 1 ]; then + # Download the necessary vectors: H264, H265 and VP9 + # When updating FLUSTER_REVISION, make sure to update the vectors if necessary or + # fluster-runner will report Missing results. + fluster/fluster.py download \ + JVT-AVC_V1 JVT-FR-EXT JVT-MVC JVT-SVC_V1 \ + JCT-VC-3D-HEVC JCT-VC-HEVC_V1 JCT-VC-MV-HEVC JCT-VC-RExt JCT-VC-SCC JCT-VC-SHVC \ + VP9-TEST-VECTORS-HIGH VP9-TEST-VECTORS + + # Build fluster vectors archive and upload it + tar --zstd -cf "vectors.tar.zst" fluster/resources/ + ci-fairy s3cp --token-file "${S3_JWT_FILE}" "vectors.tar.zst" \ + "https://${S3_PATH_FLUSTER}/vectors.tar.zst" + + touch /lava-files/done + ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/done "https://${S3_PATH_FLUSTER}/done" + + # Don't include the vectors in the rootfs + rm -fr fluster/resources/* +fi + +mkdir -p "${ROOTFS}/usr/local/" +mv fluster "${ROOTFS}/usr/local/" + diff --git a/.gitlab-ci/container/lava_build.sh b/.gitlab-ci/container/lava_build.sh index 8917a51812f..95fe6d1ac15 100755 --- a/.gitlab-ci/container/lava_build.sh +++ b/.gitlab-ci/container/lava_build.sh @@ -6,6 +6,10 @@ # When changing this file, you need to bump the following # .gitlab-ci/image-tags.yml tags: # KERNEL_ROOTFS_TAG +# If you need to update the fluster vectors cache without updating the fluster revision, +# you can update the FLUSTER_VECTORS_VERSION tag in .gitlab-ci/image-tags.yml. +# When changing FLUSTER_REVISION, KERNEL_ROOTFS_TAG needs to be updated as well to rebuild +# the rootfs. set -e @@ -16,6 +20,7 @@ set -o xtrace export DEBIAN_FRONTEND=noninteractive export LLVM_VERSION="${LLVM_VERSION:=15}" export FIRMWARE_FILES="${FIRMWARE_FILES}" +export SKIP_UPDATE_FLUSTER_VECTORS=0 check_minio() { @@ -27,9 +32,22 @@ check_minio() fi } +check_fluster() +{ + S3_PATH_FLUSTER="${S3_HOST}/${S3_KERNEL_BUCKET}/$1/${DATA_STORAGE_PATH}/fluster/${FLUSTER_VECTORS_VERSION}" + if curl -L --retry 4 -f --retry-delay 60 -s -X HEAD \ + "https://${S3_PATH_FLUSTER}/done"; then + echo "Fluster vectors are up-to-date, skip downloading them." + export SKIP_UPDATE_FLUSTER_VECTORS=1 + fi +} + check_minio "${FDO_UPSTREAM_REPO}" check_minio "${CI_PROJECT_PATH}" +check_fluster "${FDO_UPSTREAM_REPO}" +check_fluster "${CI_PROJECT_PATH}" + . .gitlab-ci/container/container_pre_build.sh # Install rust, which we'll be using for deqp-runner. It will be cleaned up at the end. @@ -197,6 +215,7 @@ PKG_MESA_DEP=( ) PKG_DEP=( libpng16-16 + libva-wayland2 libwaffle-1-0 libpython3.11 python3 python3-lxml python3-mako python3-numpy python3-packaging python3-pil python3-renderdoc python3-requests python3-simplejson python3-yaml # Python sntp @@ -213,6 +232,7 @@ PKG_DEP=( [ "$DEBIAN_ARCH" = "amd64" ] && PKG_ARCH=( firmware-amd-graphics firmware-misc-nonfree + gstreamer1.0-plugins-bad gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly gstreamer1.0-tools gstreamer1.0-vaapi libgstreamer1.0-0 # Fluster libgl1 libglu1-mesa inetutils-syslogd iptables libcap2 libfontconfig1 @@ -366,6 +386,13 @@ fi . .gitlab-ci/container/build-kdl.sh mv /ci-kdl $ROOTFS/ +############### Install fluster +if [[ ${DEBIAN_ARCH} = "amd64" ]]; then + section_start fluster "Install fluster" + . .gitlab-ci/container/build-fluster.sh + section_end fluster +fi + ############### Build local stuff for use by igt and kernel testing, which ############### will reuse most of our container build process from a specific ############### hash of the Mesa tree. diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml index eb0c91ae690..77db05e48f8 100644 --- a/.gitlab-ci/image-tags.yml +++ b/.gitlab-ci/image-tags.yml @@ -43,6 +43,8 @@ variables: KERNEL_REPO: "gfx-ci/linux" PKG_REPO_REV: "bca9635d" + FLUSTER_VECTORS_VERSION: "5" + WINDOWS_X64_MSVC_PATH: "windows/x86_64_msvc" WINDOWS_X64_MSVC_TAG: "20240827-v143" diff --git a/.gitlab-ci/lava/lava-gitlab-ci.yml b/.gitlab-ci/lava/lava-gitlab-ci.yml index 5022f251d78..7dee19c0a13 100755 --- a/.gitlab-ci/lava/lava-gitlab-ci.yml +++ b/.gitlab-ci/lava/lava-gitlab-ci.yml @@ -22,6 +22,8 @@ variables: S3_RESULTS_UPLOAD: "${JOB_ARTIFACTS_BASE}" PIGLIT_NO_WINDOW: 1 VISIBILITY_GROUP: "Collabora+fdo" + STORAGE_MAINLINE_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${FDO_UPSTREAM_REPO}/${DATA_STORAGE_PATH}" + STORAGE_FORK_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${CI_PROJECT_PATH}/${DATA_STORAGE_PATH}" before_script: - !reference [.download_s3, before_script] script: