a3fb667b1d
Downloading them at runtime proved too slow and unreliable. Keep the logic to upload the vectors to S3 as it's faster, the build job takes around 5-10 minutes when downloading from S3, but 30+ minutes to using Fluster to download the vectors. Move this to the build-fluster.sh script to allow re-using it in the test-video container. Signed-off-by: Valentine Burley <valentine.burley@collabora.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/34451>
65 lines
2.1 KiB
Bash
65 lines
2.1 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# shellcheck disable=SC1091 # The relative paths in this file only become valid at runtime.
|
|
# shellcheck disable=SC2034 # Variables are used in scripts called from here
|
|
# shellcheck disable=SC2086 # we want word splitting
|
|
|
|
# When changing this file, you need to bump the following
|
|
# .gitlab-ci/image-tags.yml tags:
|
|
# DEBIAN_TEST_VIDEO_TAG
|
|
|
|
# Install fluster in /usr/local.
|
|
|
|
set -uex
|
|
|
|
section_start fluster "Install fluster"
|
|
|
|
FLUSTER_REVISION="e997402978f62428fffc8e5a4a709690d9ca9bc5"
|
|
|
|
git clone https://github.com/fluendo/fluster.git --single-branch --no-checkout
|
|
|
|
export SKIP_UPDATE_FLUSTER_VECTORS=false
|
|
|
|
check_fluster()
|
|
{
|
|
S3_PATH_FLUSTER="${S3_HOST}/${S3_KERNEL_BUCKET}/$1/${DATA_STORAGE_PATH}/fluster/${FLUSTER_VECTORS_VERSION}/vectors.tar.zst"
|
|
if curl -L --retry 4 -f --retry-connrefused --retry-delay 30 -s --head \
|
|
"https://${S3_PATH_FLUSTER}"; then
|
|
echo "Fluster vectors are up-to-date, skip rebuilding them."
|
|
export SKIP_UPDATE_FLUSTER_VECTORS=true
|
|
fi
|
|
}
|
|
|
|
check_fluster "${FDO_UPSTREAM_REPO}"
|
|
if ! $SKIP_UPDATE_FLUSTER_VECTORS; then
|
|
check_fluster "${CI_PROJECT_PATH}"
|
|
fi
|
|
|
|
pushd fluster || exit
|
|
git checkout "${FLUSTER_REVISION}"
|
|
popd || exit
|
|
|
|
if ! $SKIP_UPDATE_FLUSTER_VECTORS; then
|
|
# Download the necessary vectors: H264, H265 and VP9
|
|
# When updating FLUSTER_REVISION, make sure to update the vectors if necessary or
|
|
# fluster-runner will report Missing results.
|
|
fluster/fluster.py download -j ${FDO_CI_CONCURRENT:-4} \
|
|
JVT-AVC_V1 JVT-FR-EXT JVT-MVC JVT-SVC_V1 \
|
|
JCT-VC-3D-HEVC JCT-VC-HEVC_V1 JCT-VC-MV-HEVC JCT-VC-RExt JCT-VC-SCC JCT-VC-SHVC \
|
|
VP9-TEST-VECTORS-HIGH VP9-TEST-VECTORS
|
|
|
|
# Build fluster vectors archive and upload it
|
|
tar --zstd -cf "vectors.tar.zst" fluster/resources/
|
|
ci-fairy s3cp --token-file "${S3_JWT_FILE}" "vectors.tar.zst" \
|
|
"https://${S3_PATH_FLUSTER}/vectors.tar.zst"
|
|
fi
|
|
|
|
mv fluster/ /usr/local/
|
|
|
|
if $SKIP_UPDATE_FLUSTER_VECTORS; then
|
|
curl -L --retry 4 -f --retry-connrefused --retry-delay 30 \
|
|
"${FDO_HTTP_CACHE_URI:-}https://${S3_PATH_FLUSTER}" | tar --zstd -x -C /usr/local/
|
|
fi
|
|
|
|
section_end fluster
|