AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
37ace710fa71b8ad8ff2ada60146d304492bc353
mesa/.gitlab-ci.yml
Martin Roukala (né Peres) 37ace710fa ci/ci-tron: switch to the upstream ci-tron template 
Up until now, every project using CI-Tron had to write their own job
submission flow because CI-Tron itself was not providing any official
way of interacting with it via GitLab.

This however changed, and the solution is vastly superior to what we
have been using in Mesa:

 * Ability to pass all the environment variables of the job to the DUT,
   so no need to remember to add variables in
   `export-gitlab-job-env-for-dut.sh` anymore

 * No dependency on Mesa code, which means no need to wait on
   python-artifacts and the ability to replicate a run by just copying
   the job description outputted by the job \o/

 * Ability to have as many initrd, HTTP, and TFTP artifacts as wanted

 * Ability to expose a variable through a TFTP/HTTP endpoint or as an
   initrd

 * Ability to overwrite the platform environment (machine-specific FW)

 * Ability to have as many kernel cmdline variables, all merged when
   generating the final cmdline. This makes it easy to share some
   snippets of cmdline between jobs

Transitioning from the custom to the generic template is however pretty
involved. This commit does the minimum changes needed to switch to the
new model, often simply replacing the B2C_ prefix with CI_TRON_.

Further renaming of "b2c" prefixes into "ci-tron" is left for future
commits.

Co-authored-by: Eric Engestrom <eric@igalia.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/34795>
2025-05-07 21:02:15 +00:00

439 lines
16 KiB
YAML
Raw Blame History

# Types of CI pipelines:
# | pipeline name | context | description |
# |----------------------|-----------|-------------------------------------------------------------|
# | merge pipeline | mesa/mesa | pipeline running for an MR; if it passes the MR gets merged |
# | pre-merge pipeline | mesa/mesa | same as above, except its status doesn't affect the MR |
# | post-merge pipeline | mesa/mesa | pipeline immediately after merging |
# | fork pipeline | fork | pipeline running in a user fork |
# | scheduled pipeline | mesa/mesa | nightly pipelines, running every morning at 4am UTC |
# | direct-push pipeline | mesa/mesa | when commits are pushed directly to mesa/mesa, bypassing Marge and its gating pipeline |
#
# Note that the release branches maintained by the release manager fall under
# the "direct push" category.
#
# "context" indicates the permissions that the jobs get; notably, any
# container created in mesa/mesa gets pushed immediately for everyone to use
# as soon as the image tag change is merged.
#
# Merge pipelines contain all jobs that must pass before the MR can be merged.
# Pre-merge pipelines contain the exact same jobs as merge pipelines.
# Post-merge pipelines contain *only* the `pages` job that deploys the new
# version of the website.
# Fork pipelines contain everything.
# Scheduled pipelines only contain the container+build jobs, and some extra
# test jobs (typically "full" variants of pre-merge jobs that only run 1/X
# test cases), but not a repeat of the merge pipeline jobs.
# Direct-push pipelines contain the same jobs as merge pipelines.
workflow:
rules:
# do not duplicate pipelines on merge pipelines
- if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"
when: never
# tag pipelines are disabled as it's too late to run all the tests by
# then, the release has been made based on the staging pipelines results
- if: $CI_COMMIT_TAG
when: never
# merge pipeline
- if: &is-merge-attempt $GITLAB_USER_LOGIN == "marge-bot" && $CI_PIPELINE_SOURCE == "merge_request_event"
variables:
MESA_CI_PERFORMANCE_ENABLED: 1
FDO_RUNNER_JOB_PRIORITY_TAG_X86_64: priority:high
FDO_RUNNER_JOB_PRIORITY_TAG_X86_64_KVM: priority:high-kvm
FDO_RUNNER_JOB_PRIORITY_TAG_AARCH64: priority:high-aarch64
CI_TRON_JOB_PRIORITY_TAG: "" # Empty tags are ignored by gitlab
JOB_PRIORITY: 75
# fast-fail in merge pipelines: stop early if we get this many unexpected fails/crashes
DEQP_RUNNER_MAX_FAILS: 40
# post-merge pipeline
- if: &is-post-merge $GITLAB_USER_LOGIN == "marge-bot" && $CI_PIPELINE_SOURCE == "push"
variables:
FDO_RUNNER_JOB_PRIORITY_TAG_X86_64: priority:high
FDO_RUNNER_JOB_PRIORITY_TAG_X86_64_KVM: priority:high-kvm
FDO_RUNNER_JOB_PRIORITY_TAG_AARCH64: priority:high-aarch64
# Pre-merge pipeline
- if: &is-pre-merge $CI_PIPELINE_SOURCE == "merge_request_event"
# Push to a branch on a fork
- if: &is-fork-push $CI_PROJECT_NAMESPACE != "mesa" && $CI_PIPELINE_SOURCE == "push"
# nightly pipeline
- if: &is-scheduled-pipeline $CI_PIPELINE_SOURCE == "schedule"
variables:
FDO_RUNNER_JOB_PRIORITY_TAG_X86_64: priority:low
FDO_RUNNER_JOB_PRIORITY_TAG_X86_64_KVM: priority:low-kvm
FDO_RUNNER_JOB_PRIORITY_TAG_AARCH64: priority:low-aarch64
JOB_PRIORITY: 45
# (some) nightly builds perform LTO, so they take much longer than the
# short timeout allowed in other pipelines.
# Note: 0 = infinity = gitlab's job `timeout:` applies, which is 1h
BUILD_JOB_TIMEOUT_OVERRIDE: 0
# pipeline for direct pushes that bypassed the CI
- if: &is-direct-push $CI_PROJECT_NAMESPACE == "mesa" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
variables:
JOB_PRIORITY: 70
# pipeline for direct pushes from release maintainer
- if: &is-staging-push $CI_PROJECT_NAMESPACE == "mesa" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_NAME =~ /^staging\//
variables:
JOB_PRIORITY: 70
variables:
FDO_UPSTREAM_REPO: mesa/mesa
MESA_TEMPLATES_COMMIT: &ci-templates-commit c6aeb16f86e32525fa630fb99c66c4f3e62fc3cb
CI_PRE_CLONE_SCRIPT: |-
set -o xtrace
curl --silent --location --fail --retry-connrefused --retry 3 --retry-delay 10 \
${CI_PROJECT_URL}/-/raw/${CI_COMMIT_SHA}/.gitlab-ci/download-git-cache.sh | bash
set +o xtrace
S3_JWT_FILE: /s3_jwt
S3_JWT_FILE_SCRIPT: |-
echo -n '${S3_JWT}' > '${S3_JWT_FILE}' &&
unset CI_JOB_JWT S3_JWT # Unsetting vulnerable env variables
S3_HOST: s3.freedesktop.org
# This bucket is used to fetch ANDROID prebuilts and images
S3_ANDROID_BUCKET: mesa-rootfs
# This bucket is used to fetch the kernel image
S3_KERNEL_BUCKET: mesa-rootfs
# Bucket for git cache
S3_GITCACHE_BUCKET: git-cache
# Bucket for the pipeline artifacts pushed to S3
S3_ARTIFACTS_BUCKET: artifacts
# Buckets for traces
S3_TRACIE_RESULTS_BUCKET: mesa-tracie-results
S3_TRACIE_PUBLIC_BUCKET: mesa-tracie-public
S3_TRACIE_PRIVATE_BUCKET: mesa-tracie-private
# per-pipeline artifact storage on MinIO
PIPELINE_ARTIFACTS_BASE: ${S3_HOST}/${S3_ARTIFACTS_BUCKET}/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
# per-job artifact storage on MinIO
JOB_ARTIFACTS_BASE: ${PIPELINE_ARTIFACTS_BASE}/${CI_JOB_ID}
# reference images stored for traces
PIGLIT_REPLAY_REFERENCE_IMAGES_BASE: "${S3_HOST}/${S3_TRACIE_RESULTS_BUCKET}/$FDO_UPSTREAM_REPO"
# For individual CI farm status see .ci-farms folder
# Disable farm with `git mv .ci-farms{,-disabled}/$farm_name`
# Re-enable farm with `git mv .ci-farms{-disabled,}/$farm_name`
# NEVER MIX FARM MAINTENANCE WITH ANY OTHER CHANGE IN THE SAME MERGE REQUEST!
ARTIFACTS_BASE_URL: https://${CI_PROJECT_ROOT_NAMESPACE}.${CI_PAGES_DOMAIN}/-/${CI_PROJECT_NAME}/-/jobs/${CI_JOB_ID}/artifacts
# Python scripts for structured logger
PYTHONPATH: "$PYTHONPATH:$CI_PROJECT_DIR/install"
# No point in continuing once the device is lost
MESA_VK_ABORT_ON_DEVICE_LOSS: 1
# Avoid the wall of "Unsupported SPIR-V capability" warnings in CI job log, hiding away useful output
MESA_SPIRV_LOG_LEVEL: error
# Default priority for non-merge pipelines
FDO_RUNNER_JOB_PRIORITY_TAG_X86_64: "" # Empty tags are ignored by gitlab
FDO_RUNNER_JOB_PRIORITY_TAG_X86_64_KVM: kvm
FDO_RUNNER_JOB_PRIORITY_TAG_AARCH64: aarch64
CI_TRON_JOB_PRIORITY_TAG: ci-tron:priority:low
JOB_PRIORITY: 50
DATA_STORAGE_PATH: data_storage
CI_TRON_JOB_TEMPLATE_PROJECT: &ci-tron-template-project gfx-ci/ci-tron
CI_TRON_JOB_TEMPLATE_COMMIT: &ci-tron-template-commit e4ab9c5ecc941c7893d991cf723cde877b5dca8d
CI_TRON_JOB_TEMPLATE_PROJECT_URL: "https://gitlab.freedesktop.org/$CI_TRON_JOB_TEMPLATE_PROJECT"
default:
timeout: 1m # catch any jobs which don't specify a timeout
id_tokens:
S3_JWT:
aud: https://s3.freedesktop.org
before_script:
- |
if [ -z "${KERNEL_IMAGE_BASE:-}" ]; then
export KERNEL_IMAGE_BASE="https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${EXTERNAL_KERNEL_TAG:-$KERNEL_TAG}"
fi
- >
export SCRIPTS_DIR=$(mktemp -d) &&
curl -L -s --retry 4 -f --retry-all-errors --retry-delay 60 -O --output-dir "${SCRIPTS_DIR}" "${CI_PROJECT_URL}/-/raw/${CI_COMMIT_SHA}/.gitlab-ci/setup-test-env.sh" &&
. ${SCRIPTS_DIR}/setup-test-env.sh
- eval "$S3_JWT_FILE_SCRIPT"
after_script:
# Work around https://gitlab.com/gitlab-org/gitlab/-/issues/20338
- find -name '*.log' -exec mv {} {}.txt \;
# Retry when job fails. Failed jobs can be found in the Mesa CI Daily Reports:
# https://gitlab.freedesktop.org/mesa/mesa/-/issues/?sort=created_date&state=opened&label_name%5B%5D=CI%20daily
retry:
max: 1
# Ignore runner_unsupported, stale_schedule, archived_failure, or
# unmet_prerequisites
when:
- api_failure
- runner_system_failure
- script_failure
- job_execution_timeout
- scheduler_failure
- data_integrity_failure
- unknown_failure
stages:
- sanity
- container
- git-archive
- build-for-tests
- build-only
- code-validation
- amd
- amd-nightly
- intel
- intel-nightly
- nouveau
- nouveau-nightly
- arm
- arm-nightly
- broadcom
- broadcom-nightly
- freedreno
- freedreno-nightly
- etnaviv
- etnaviv-nightly
- software-renderer
- software-renderer-nightly
- layered-backends
- layered-backends-nightly
- performance
- deploy
include:
- project: 'freedesktop/ci-templates'
ref: *ci-templates-commit
file:
- '/templates/alpine.yml'
- '/templates/debian.yml'
- '/templates/fedora.yml'
- '/templates/ci-fairy.yml'
- project: *ci-tron-template-project
ref: *ci-tron-template-commit
file: '/.gitlab-ci/dut.yml'
- local: '.gitlab-ci/image-tags.yml'
- local: '.gitlab-ci/lava/lava-gitlab-ci.yml'
- local: '.gitlab-ci/container/gitlab-ci.yml'
- local: '.gitlab-ci/build/gitlab-ci.yml'
- local: '.gitlab-ci/test/gitlab-ci.yml'
- local: '.gitlab-ci/farm-rules.yml'
- local: '.gitlab-ci/test-source-dep.yml'
- local: 'docs/gitlab-ci.yml'
- local: 'src/**/ci/gitlab-ci.yml'
# Rules applied to every job in the pipeline
.common-rules:
rules:
- if: *is-fork-push
when: manual
.never-post-merge-rules:
rules:
- if: *is-post-merge
when: never
# Note: make sure the branches in this list are the same as in
# `.build-only-delayed-rules` below.
.container+build-rules:
rules:
- !reference [.common-rules, rules]
# Run when re-enabling a disabled farm, but not when disabling it
- !reference [.disable-farm-mr-rules, rules]
# Never run immediately after merging, as we just ran everything
- !reference [.never-post-merge-rules, rules]
# Build everything in merge pipelines, if any files affecting the pipeline
# were changed
- if: *is-merge-attempt
changes: &all_paths
- VERSION
- bin/git_sha1_gen.py
- bin/install_megadrivers.py
- bin/symbols-check.py
- bin/ci/**/*
# GitLab CI
- .gitlab-ci.yml
- .gitlab-ci/**/*
- .ci-farms/*
# Meson
- meson*
- build-support/**/*
- subprojects/**/*
# clang format
- .clang-format
- .clang-format-include
- .clang-format-ignore
# Source code
- include/**/*
- src/**/*
when: on_success
# Same as above, but for pre-merge pipelines
- if: *is-pre-merge
changes:
*all_paths
when: manual
# Skip everything for pre-merge and merge pipelines which don't change
# anything in the build
- if: *is-merge-attempt
when: never
- if: *is-pre-merge
when: never
# Build everything after someone bypassed the CI
- if: *is-direct-push
when: on_success
# Build everything when pushing to staging branches
- if: *is-staging-push
when: on_success
# Build everything in scheduled pipelines
- if: *is-scheduled-pipeline
when: on_success
# Allow building everything in fork pipelines, but build nothing unless
# manually triggered
- when: manual
# Repeat of the above but with `when: on_success` replaced with
# `when: delayed` + `start_in:`, for build-only jobs.
# Note: make sure the branches in this list are the same as in
# `.container+build-rules` above.
.build-only-delayed-rules:
rules:
- !reference [.common-rules, rules]
# Run when re-enabling a disabled farm, but not when disabling it
- !reference [.disable-farm-mr-rules, rules]
# Never run immediately after merging, as we just ran everything
- !reference [.never-post-merge-rules, rules]
# Build everything in merge pipelines, if any files affecting the pipeline
# were changed
- if: *is-merge-attempt
changes: *all_paths
when: delayed
start_in: &build-delay 5 minutes
# Same as above, but for pre-merge pipelines
- if: *is-pre-merge
changes: *all_paths
when: manual
# Skip everything for pre-merge and merge pipelines which don't change
# anything in the build
- if: *is-merge-attempt
when: never
- if: *is-pre-merge
when: never
# Build everything after someone bypassed the CI
- if: *is-direct-push
when: delayed
start_in: *build-delay
# Build everything when pushing to staging branches
- if: *is-staging-push
when: delayed
start_in: *build-delay
# Build everything in scheduled pipelines
- if: *is-scheduled-pipeline
when: delayed
start_in: *build-delay
# Allow building everything in fork pipelines, but build nothing unless
# manually triggered
- when: manual
.ci-deqp-artifacts:
artifacts:
name: "${CI_PROJECT_NAME}_${CI_JOB_NAME}"
when: always
untracked: false
paths:
# Watch out! Artifacts are relative to the build dir.
# https://gitlab.com/gitlab-org/gitlab-ce/commit/8788fb925706cad594adf6917a6c5f6587dd1521
- artifacts
- _build/meson-logs/*.txt
- _build/meson-logs/strace
- _build/.ninja_log
# Git archive
make-git-archive:
extends:
- .fdo.ci-fairy
stage: git-archive
rules:
- !reference [.scheduled_pipeline-rules, rules]
script:
# Compactify the .git directory
- git gc --aggressive
# Download & cache the perfetto subproject as well.
- rm -rf subprojects/perfetto ; mkdir -p subprojects/perfetto && curl --fail https://android.googlesource.com/platform/external/perfetto/+archive/$(grep 'revision =' subprojects/perfetto.wrap | cut -d ' ' -f3).tar.gz | tar zxf - -C subprojects/perfetto
# compress the current folder
- tar -cvzf ../$CI_PROJECT_NAME.tar.gz .
- ci-fairy s3cp --token-file "${S3_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$S3_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz
# Sanity checks of MR settings and commit logs
sanity:
extends:
- .fdo.ci-fairy
stage: sanity
tags:
- $FDO_RUNNER_JOB_PRIORITY_TAG_X86_64
rules:
- if: *is-pre-merge
when: on_success
- when: never
variables:
GIT_STRATEGY: none
script:
# ci-fairy check-commits --junit-xml=check-commits.xml
- ci-fairy check-merge-request --require-allow-collaboration --junit-xml=check-merge-request.xml
- |
set -eu
image_tags=(
ALPINE_X86_64_BUILD_TAG
ALPINE_X86_64_LAVA_SSH_TAG
DEBIAN_BASE_TAG
DEBIAN_BUILD_TAG
DEBIAN_PYUTILS_TAG
DEBIAN_TEST_ANDROID_TAG
DEBIAN_TEST_GL_TAG
DEBIAN_TEST_VK_TAG
FEDORA_X86_64_BUILD_TAG
KERNEL_TAG
PKG_REPO_REV
WINDOWS_X64_BUILD_TAG
WINDOWS_X64_MSVC_TAG
WINDOWS_X64_TEST_TAG
)
for var in "${image_tags[@]}"
do
if [ "$(echo -n "${!var}" | wc -c)" -gt 20 ]
then
echo "$var is too long; please make sure it is at most 20 chars."
exit 1
fi
done
artifacts:
when: on_failure
reports:
junit: check-*.xml
mr-label-maker-test:
extends:
- .fdo.ci-fairy
stage: sanity
tags:
- $FDO_RUNNER_JOB_PRIORITY_TAG_X86_64
rules:
- !reference [.mr-label-maker-rules, rules]
variables:
GIT_STRATEGY: fetch
timeout: 10m
script:
- set -eu
- python3 -m venv .venv
- source .venv/bin/activate
- pip install git+https://gitlab.freedesktop.org/freedesktop/mr-label-maker
- mr-label-maker --dry-run --mr $CI_MERGE_REQUEST_IID
# Jobs that need to pass before spending hardware resources on further testing
.required-for-hardware-jobs:
needs:
- job: rustfmt
optional: true
artifacts: false
- job: yaml-toml-shell-py-test
optional: true
artifacts: false
Reference in New Issue View Git Blame Copy Permalink