ci/ci-tron: switch to the upstream ci-tron template

Up until now, every project using CI-Tron had to write their own job
submission flow because CI-Tron itself was not providing any official
way of interacting with it via GitLab.

This however changed, and the solution is vastly superior to what we
have been using in Mesa:

 * Ability to pass all the environment variables of the job to the DUT,
   so no need to remember to add variables in
   `export-gitlab-job-env-for-dut.sh` anymore

 * No dependency on Mesa code, which means no need to wait on
   python-artifacts and the ability to replicate a run by just copying
   the job description outputted by the job \o/

 * Ability to have as many initrd, HTTP, and TFTP artifacts as wanted

 * Ability to expose a variable through a TFTP/HTTP endpoint or as an
   initrd

 * Ability to overwrite the platform environment (machine-specific FW)

 * Ability to have as many kernel cmdline variables, all merged when
   generating the final cmdline. This makes it easy to share some
   snippets of cmdline between jobs

Transitioning from the custom to the generic template is however pretty
involved. This commit does the minimum changes needed to switch to the
new model, often simply replacing the B2C_ prefix with CI_TRON_.

Further renaming of "b2c" prefixes into "ci-tron" is left for future
commits.

Co-authored-by: Eric Engestrom <eric@igalia.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/34795>
This commit is contained in:
Martin Roukala (né Peres)
2025-03-15 14:49:11 +02:00
committed by Marge Bot
parent a062f0dd3d
commit 37ace710fa
17 changed files with 160 additions and 415 deletions

View File

@@ -1,155 +0,0 @@
version: 1
# Rules to match for a machine to qualify
target:
id: '{{ CI_RUNNER_DESCRIPTION }}'
timeouts:
first_console_activity: # This limits the time it can take to receive the first console log
minutes: {{ B2C_TIMEOUT_FIRST_CONSOLE_ACTIVITY_MINUTES | default(0, true) }}
seconds: {{ B2C_TIMEOUT_FIRST_CONSOLE_ACTIVITY_SECONDS | default(0, true) }}
retries: {{ B2C_TIMEOUT_FIRST_CONSOLE_ACTIVITY_RETRIES }}
console_activity: # Reset every time we receive a message from the logs
minutes: {{ B2C_TIMEOUT_CONSOLE_ACTIVITY_MINUTES | default(0, true) }}
seconds: {{ B2C_TIMEOUT_CONSOLE_ACTIVITY_SECONDS | default(0, true) }}
retries: {{ B2C_TIMEOUT_CONSOLE_ACTIVITY_RETRIES }}
boot_cycle:
minutes: {{ B2C_TIMEOUT_BOOT_MINUTES | default(0, true) }}
seconds: {{ B2C_TIMEOUT_BOOT_SECONDS | default(0, true) }}
retries: {{ B2C_TIMEOUT_BOOT_RETRIES }}
overall: # Maximum time the job can take, not overrideable by the "continue" deployment
minutes: {{ B2C_TIMEOUT_OVERALL_MINUTES | default(0, true) }}
seconds: {{ B2C_TIMEOUT_OVERALL_SECONDS | default(0, true) }}
retries: 0
# no retries possible here
watchdogs:
boot:
minutes: {{ B2C_TIMEOUT_BOOT_WD_MINUTES | default(0, true) }}
seconds: {{ B2C_TIMEOUT_BOOT_WD_SECONDS | default(0, true) }}
retries: {{ B2C_TIMEOUT_BOOT_WD_RETRIES | default(0, true) }}
console_patterns:
session_end:
regex: >-
{{ B2C_SESSION_END_REGEX }}
{% if B2C_SESSION_REBOOT_REGEX %}
session_reboot:
regex: >-
{{ B2C_SESSION_REBOOT_REGEX }}
{% endif %}
job_success:
regex: >-
{{ B2C_JOB_SUCCESS_REGEX }}
{% if B2C_JOB_WARN_REGEX %}
job_warn:
regex: >-
{{ B2C_JOB_WARN_REGEX }}
{% endif %}
{% if B2C_BOOT_WD_START_REGEX and B2C_BOOT_WD_STOP_REGEX %}
watchdogs:
boot:
start:
regex: >-
{{ B2C_BOOT_WD_START_REGEX }}
reset:
regex: >-
{{ B2C_BOOT_WD_RESET_REGEX | default(B2C_BOOT_WD_START_REGEX, true) }}
stop:
regex: >-
{{ B2C_BOOT_WD_STOP_REGEX }}
{% endif %}
# Environment to deploy
deployment:
# Initial boot
start:
storage:
{% if B2C_IMAGESTORE_PLATFORM %}
imagestore:
public:
# List of images that should be pulled into the image store ahead of execution
images:
mars:
name: "{{ B2C_MACHINE_REGISTRATION_IMAGE }}"
platform: "{{ B2C_IMAGESTORE_PLATFORM }}"
tls_verify: false
{% set machine_registration_image="{% raw %}{{ job.imagestore.public.mars.image_id }}{% endraw %}" %}
telegraf:
name: "{{ B2C_TELEGRAF_IMAGE }}"
platform: "{{ B2C_IMAGESTORE_PLATFORM }}"
tls_verify: false
{% set telegraf_image="{% raw %}{{ job.imagestore.public.telegraf.image_id }}{% endraw %}" %}
image_under_test:
name: "{{ B2C_IMAGE_UNDER_TEST }}"
platform: "{{ B2C_IMAGESTORE_PLATFORM }}"
tls_verify: false
{% set image_under_test="{% raw %}{{ job.imagestore.public.image_under_test.image_id }}{% endraw %}" %}
nbd:
storage:
max_connections: 5
size: 10G
{% endif %}
http:
- path: "/install.tar.zst"
url: "{{ B2C_INSTALL_TARBALL_URL }}"
- path: "/b2c-extra-args"
data: >
b2c.pipefail b2c.poweroff_delay={{ B2C_POWEROFF_DELAY }}
b2c.minio="gateway,{{ '{{' }} minio_url }},{{ '{{' }} job_bucket_access_key }},{{ '{{' }} job_bucket_secret_key }}"
b2c.volume="{{ '{{' }} job_bucket }}-results,mirror=gateway/{{ '{{' }} job_bucket }},pull_on=pipeline_start,push_on=changes,overwrite{% for excl in B2C_JOB_VOLUME_EXCLUSIONS.split(',') %},exclude={{ excl }}{% endfor %},remove,expiration=pipeline_end,preserve"
{% for volume in B2C_VOLUMES %}
b2c.volume={{ volume }}
{% endfor %}
b2c.run_service="--privileged --tls-verify=false --pid=host {{ B2C_TELEGRAF_IMAGE }}" b2c.hostname=dut-{{ '{{' }} machine.full_name }}
b2c.run="-ti --tls-verify=false {{ B2C_MACHINE_REGISTRATION_IMAGE }} {% if B2C_MARS_SETUP_TAGS %}setup --tags {{ B2C_MARS_SETUP_TAGS }}{% else %}check{% endif %}"
b2c.run="-v {{ '{{' }} job_bucket }}-results:{{ CI_PROJECT_DIR }} -w {{ CI_PROJECT_DIR }} {% for mount_volume in B2C_MOUNT_VOLUMES %} -v {{ mount_volume }}{% endfor %} --tls-verify=false --entrypoint bash {{ B2C_IMAGE_UNDER_TEST }} -euc 'curl --fail -q {{ '{{' }} job.http.url }}/install.tar.zst | tar --zstd -x; {{ B2C_CONTAINER_CMD }}'"
kernel:
{% if B2C_KERNEL_URL %}
url: '{{ B2C_KERNEL_URL }}'
{% endif %}
# NOTE: b2c.cache_device should not be here, but this works around
# a limitation of b2c which will be removed in the next release
cmdline: >
SALAD.machine_id={{ '{{' }} machine_id }}
console={{ '{{' }} local_tty_device }},115200
b2c.ntp_peer=10.42.0.1
b2c.extra_args_url={{ '{{' }} job.http.url }}/b2c-extra-args
{% if B2C_IMAGESTORE_PLATFORM is defined %}
{{ '{{' }} imagestore.mount("public").nfs.to_b2c_filesystem("publicimgstore") }}
b2c.storage="additionalimagestores=publicimgstore"
b2c.nbd=/dev/nbd0,host=ci-gateway,port={% raw %}{{ '{{' }} job.nbd.storage.tcp_port }}{% endraw %},connections=5
b2c.cache_device=/dev/nbd0
{% else %}
b2c.cache_device=auto
{% endif %}
{% if B2C_KERNEL_CMDLINE_EXTRAS is defined %}
{{ B2C_KERNEL_CMDLINE_EXTRAS }}
{% endif %}
{% if B2C_INITRAMFS_URL or B2C_FIRMWARE_URL %}
initramfs:
{% if B2C_FIRMWARE_URL %}
- url: '{{ B2C_FIRMWARE_URL }}'
{% endif %}
{% if B2C_INITRAMFS_URL %}
- url: '{{ B2C_INITRAMFS_URL }}'
{% endif %}
{% endif %}
{% if B2C_DTB_URL %}
dtb:
url: '{{ B2C_DTB_URL }}'
{% if B2C_DTB_MATCH %}
format:
archive:
match: "{{ B2C_DTB_MATCH }}"
{% endif %}
{% endif %}

View File

@@ -1,40 +0,0 @@
#!/usr/bin/env python3
# Copyright © 2022 Valve Corporation
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice (including the next
# paragraph) shall be included in all copies or substantial portions of the
# Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from jinja2 import Environment, FileSystemLoader
from os import environ, path
# Pass through all the CI and B2C environment variables
values = {
key: environ[key]
for key in environ if key.startswith("B2C_") or key.startswith("CI_")
}
env = Environment(loader=FileSystemLoader(path.dirname(environ['B2C_JOB_TEMPLATE'])),
trim_blocks=True, lstrip_blocks=True)
template = env.get_template(path.basename(environ['B2C_JOB_TEMPLATE']))
with open(path.splitext(path.basename(environ['B2C_JOB_TEMPLATE']))[0], "w") as f:
f.write(template.render(values))

View File

@@ -1,7 +1,6 @@
#!/bin/bash
VARS=(
ACO_DEBUG
ANGLE_TAG
ANGLE_TRACE_FILES_TAG
ANV_DEBUG
@@ -114,9 +113,6 @@ VARS=(
PIGLIT_TRACES_FILE
PIPELINE_ARTIFACTS_BASE
RADEON_DEBUG
RADV_DEBUG
radv_enable_float16_gfx8
RADV_PERFTEST
S3_HOST
S3_JWT_FILE
S3_RESULTS_UPLOAD
@@ -126,7 +122,6 @@ VARS=(
VIRGL_HOST_API
VIRGL_RENDER_SERVER
VK_DRIVER
vk_require_etc2
WAFFLE_PLATFORM
ZINK_DEBUG
ZINK_DESCRIPTORS

View File

@@ -15,7 +15,6 @@ mkdir -p artifacts/
cp -Rp .gitlab-ci/report-flakes.py artifacts/
cp -Rp .gitlab-ci/setup-test-env.sh artifacts/
cp -Rp .gitlab-ci/common artifacts/ci-common
cp -Rp .gitlab-ci/b2c artifacts/
cp -Rp .gitlab-ci/bare-metal artifacts/
cp -Rp .gitlab-ci/lava artifacts/
cp -Rp .gitlab-ci/bin/*_logger.py artifacts/

View File

@@ -379,123 +379,53 @@ yaml-toml-shell-py-test:
HWCI_TEST_SCRIPT: "/install/deqp-runner.sh"
FDO_CI_CONCURRENT: 0 # Default to number of CPUs
# For Valve's bare-metal testing farm jobs.
# For CI-tron based testing farm jobs.
.b2c-test:
# It would be nice to use ci-templates within Mesa CI for this job's
# image:, but the integration is not possible for the current
# use-case. Within this job, two containers are managed. 1) the
# gitlab runner container from which the job is submitted to the
# DUT, and 2) the test container (e.g. debian/x86_64_test-vk) within
# which the test cases will run on the DUT. Since ci-templates and
# the associated image setting macros in this file rely on variables
# like FDO_DISTRIBUTION_TAG for *the* image, there is no way to
# depend on more than one image per job. So, the job container is
# built as part of the CI in the boot2container project.
image: registry.freedesktop.org/gfx-ci/ci-tron/mesa-trigger:2024-01-05.1
extends:
- .ci-tron-b2c-job-v1
timeout: 1h 40m
variables:
# No need to pull the whole git repo, the artifacts from python-artifacts
# contain everything we need.
GIT_STRATEGY: none
# boot2container initrd configuration parameters.
B2C_VERSION: v0.9.14
B2C_JOB_SUCCESS_REGEX: 'hwci: mesa: pass, exit_code: 0\r$'
B2C_LOG_LEVEL: 6
B2C_POWEROFF_DELAY: 15
B2C_SESSION_END_REGEX: '^.*It''s now safe to turn off your computer\r$'
B2C_SESSION_REBOOT_REGEX: ''
B2C_TIMEOUT_BOOT_MINUTES: 45
B2C_TIMEOUT_BOOT_RETRIES: 0
B2C_TIMEOUT_FIRST_CONSOLE_ACTIVITY_MINUTES: 2
B2C_TIMEOUT_FIRST_CONSOLE_ACTIVITY_RETRIES: 3
B2C_TIMEOUT_CONSOLE_ACTIVITY_MINUTES: 5
B2C_TIMEOUT_OVERALL_MINUTES: 90
B2C_TIMEOUT_CONSOLE_ACTIVITY_RETRIES: 0
B2C_JOB_VOLUME_EXCLUSIONS: "*.shader_cache,install/*,*/install/*,*/vkd3d-proton.cache*,vkd3d-proton.cache*,*.qpa"
B2C_MACHINE_REGISTRATION_IMAGE: "registry.freedesktop.org/gfx-ci/ci-tron/machine-registration:latest"
B2C_TELEGRAF_IMAGE: "registry.freedesktop.org/gfx-ci/ci-tron/telegraf:latest"
B2C_KERNEL_CMDLINE_EXTRAS: ""
# As noted in the top description, we make a distinction between the
# container used by gitlab-runner to queue the work, and the container
# used by the DUTs/test machines. To make this distinction quite clear,
# we rename the variable.
B2C_IMAGE_UNDER_TEST: "$MESA_IMAGE"
SCRIPTS_DIR: install
B2C_INSTALL_TARBALL_URL: "https://${PIPELINE_ARTIFACTS_BASE}/${S3_ARTIFACT_NAME}.tar.zst"
B2C_CONTAINER_CMD: "./install/common/init-stage2.sh"
CI_TRON_PATTERN__JOB_SUCCESS__REGEX: 'hwci: mesa: pass, exit_code: 0\r$'
CI_TRON_PATTERN__SESSION_END__REGEX: '^.*It''s now safe to turn off your computer\r$'
CI_B2C_ARTIFACTS: "./artifacts/b2c"
CI_COMMON_SCRIPTS: "./artifacts/ci-common"
B2C_JOB_TEMPLATE: "${CI_B2C_ARTIFACTS}/b2c.yml.jinja2.jinja2"
JOB_FOLDER: "job_folder"
CI_TRON_TIMEOUT__BOOT_CYCLE__MINUTES: 45
CI_TRON_TIMEOUT__FIRST_CONSOLE_ACTIVITY__MINUTES: 2
CI_TRON_TIMEOUT__FIRST_CONSOLE_ACTIVITY__RETRIES: 3
CI_TRON_TIMEOUT__CONSOLE_ACTIVITY__MINUTES: 5
CI_TRON_TIMEOUT__OVERALL__MINUTES: 90
CI_TRON__B2C_ARTIFACT_EXCLUSION: "*.shader_cache,install/*,*/install/*,*/vkd3d-proton.cache*,vkd3d-proton.cache*,*.qpa"
CI_TRON_HTTP_ARTIFACT__INSTALL__PATH: "/install.tar.zst"
CI_TRON_HTTP_ARTIFACT__INSTALL__URL: "https://$PIPELINE_ARTIFACTS_BASE/$S3_ARTIFACT_NAME.tar.zst"
CI_TRON__B2C_MACHINE_REGISTRATION_CMD: "setup --tags $CI_TRON_DUT_SETUP_TAGS"
CI_TRON__B2C_IMAGE_UNDER_TEST: $MESA_IMAGE
CI_TRON__B2C_EXEC_CMD: "curl --silent --fail-with-body {{ job.http.url }}$CI_TRON_HTTP_ARTIFACT__INSTALL__PATH | tar --zstd --extract && $SCRIPTS_DIR/common/init-stage2.sh"
# Assume by default this is running deqp, as that's almost always true
HWCI_TEST_SCRIPT: install/deqp-runner.sh
needs:
- job: python-artifacts
artifacts: true
- !reference [.required-for-hardware-jobs, needs]
tags:
- farm:$RUNNER_FARM_LOCATION
- $B2C_MARS_SETUP_TAGS
- $CI_TRON_DUT_SETUP_TAGS
# Override the default before_script, as it is not compatible with the CI-tron environment. We just keep the clearing
# of the JWT token for security reasons
before_script:
# We don't want the tarball unpacking of .test, but will take the JWT bits.
- !reference [default, before_script]
- |
set -eux
set -eu
section_start b2c_kernel_boot "Booting hardware device"
eval "$S3_JWT_FILE_SCRIPT"
[ -d "$CI_B2C_ARTIFACTS" ] || exit 1
[ -d "$CI_COMMON_SCRIPTS" ] || exit 1
# Pull all our images through our proxy registry
B2C_IMAGE_UNDER_TEST=${B2C_IMAGE_UNDER_TEST//registry.freedesktop.org/{{ fdo_proxy_registry \}\}}
B2C_MACHINE_REGISTRATION_IMAGE=${B2C_MACHINE_REGISTRATION_IMAGE//registry.freedesktop.org/{{ fdo_proxy_registry \}\}}
B2C_TELEGRAF_IMAGE=${B2C_TELEGRAF_IMAGE//registry.freedesktop.org/{{ fdo_proxy_registry \}\}}
# The Valve CI gateway receives jobs in a YAML format. Create a
# job description from the CI environment.
python3 "$CI_B2C_ARTIFACTS"/generate_b2c.py
cat b2c.yml.jinja2
rm -rf ${JOB_FOLDER} || true
mkdir -v ${JOB_FOLDER}
# Create a script to regenerate the CI environment when this job
# begins running on the remote DUT.
set +x
"$CI_COMMON_SCRIPTS"/export-gitlab-job-env-for-dut.sh > ${JOB_FOLDER}/set-job-env-vars.sh
echo "export SCRIPTS_DIR=./install" >> ${JOB_FOLDER}/set-job-env-vars.sh
echo "Variables passed through:"
cat ${JOB_FOLDER}/set-job-env-vars.sh
set -x
script: |
slugify () {
echo "$1" | sed -r s/[~\^]+//g | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | tr A-Z a-z
}
# Submit the job to Valve's CI gateway service with the CI
# provisioned job_folder.
PYTHONUNBUFFERED=1 \
executorctl run \
--machine-id "$CI_RUNNER_DESCRIPTION" \
--job-id $(slugify "$CI_JOB_NAME") \
--share-directory "$JOB_FOLDER" \
--wait \
b2c.yml.jinja2
after_script:
# Keep the results path the same as baremetal and LAVA
- mkdir -p "${JOB_FOLDER}"/results
- mv "${JOB_FOLDER}"/results ./
- !reference [default, after_script]
# Open a section that will be closed by b2c
echo -e "\n\e[0Ksection_start:`date +%s`:b2c_kernel_boot[collapsed=true]\r\e[0K\e[0;36m[$(cut -d ' ' -f1 /proc/uptime)]: Submitting the CI-tron job and booting the DUT\e[0m\n"
# Anything our job places in results/ will be collected by the
# Gitlab coordinator for status presentation. results/junit.xml
@@ -513,8 +443,14 @@ yaml-toml-shell-py-test:
extends:
- .b2c-test
variables:
B2C_KERNEL_URL: 'https://gitlab.freedesktop.org/gfx-ci/ci-tron/-/package_files/519/download' # Linux 6.1
B2C_INITRAMFS_URL: 'https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/${B2C_VERSION}/downloads/initramfs.linux_amd64.cpio.xz'
CI_TRON_INITRAMFS__B2C__URL: 'https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/$B2C_VERSION/downloads/initramfs.linux_amd64.cpio.xz'
CI_TRON_KERNEL__URL: 'https://gitlab.freedesktop.org/gfx-ci/ci-tron/-/package_files/519/download' # Linux 6.1
# Set the following variables if you need AMD, Intel, or NVIDIA support
# CI_TRON_INITRAMFS__DEPMOD__URL: "https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/$B2C_VERSION/downloads/linux-x86_64.depmod.cpio.xz"
# CI_TRON_INITRAMFS__GPU__URL: "https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/$B2C_VERSION/downloads/linux-x86_64.gpu.cpio"
# CI_TRON_INITRAMFS__GPU__FORMAT__0__ARCHIVE__KEEP__0__PATH: "(lib/(modules|firmware/amdgpu)/.*)"
S3_ARTIFACT_NAME: "mesa-x86_64-default-debugoptimized"
.b2c-x86_64-test-vk:
@@ -569,8 +505,8 @@ yaml-toml-shell-py-test:
extends:
- .b2c-test
variables:
B2C_INITRAMFS_URL: 'https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/${B2C_VERSION}/downloads/initramfs.linux_arm64.cpio.xz'
B2C_KERNEL_URL: 'https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/${B2C_VERSION}/downloads/linux-arm64'
CI_TRON_INITRAMFS__B2C__URL: 'https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/$B2C_VERSION/downloads/initramfs.linux_arm64.cpio.xz'
CI_TRON_KERNEL__URL: 'https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/$B2C_VERSION/downloads/linux-arm64'
S3_ARTIFACT_NAME: "mesa-arm64-default-debugoptimized"
.b2c-arm64-test-vk:
@@ -599,8 +535,8 @@ yaml-toml-shell-py-test:
extends:
- .b2c-test
variables:
B2C_INITRAMFS_URL: 'https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/${B2C_VERSION}/downloads/initramfs.linux_arm.cpio.xz'
B2C_KERNEL_URL: 'https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/${B2C_VERSION}/downloads/linux-arm'
CI_TRON_INITRAMFS__B2C__URL: 'https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/$B2C_VERSION/downloads/initramfs.linux_arm.cpio.xz'
CI_TRON_KERNEL__URL: 'https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/$B2C_VERSION/downloads/linux-arm'
S3_ARTIFACT_NAME: "mesa-arm32-default-debugoptimized"
.b2c-arm32-test-vk: