diff --git a/.gitlab-ci/conditional-build-image-tags.yml b/.gitlab-ci/conditional-build-image-tags.yml new file mode 100644 index 00000000000..0b0cbb2935c --- /dev/null +++ b/.gitlab-ci/conditional-build-image-tags.yml @@ -0,0 +1 @@ +variables: diff --git a/.gitlab-ci/container/container_pre_build.sh b/.gitlab-ci/container/container_pre_build.sh index 86cdc6e36eb..b4b36a93acd 100755 --- a/.gitlab-ci/container/container_pre_build.sh +++ b/.gitlab-ci/container/container_pre_build.sh @@ -1,4 +1,7 @@ #!/bin/sh +# When changing this file, you need to bump the following +# .gitlab-ci/image-tags.yml tags: +# DEBIAN_BUILD_TAG if test -x /usr/bin/ccache; then if test -f /etc/debian_version; then @@ -49,3 +52,23 @@ if [ -f "$CARGO_ENV_FILE" ]; then # shellcheck disable=SC1090 source "$CARGO_ENV_FILE" fi + +ci_tag_early_checks() { + # Runs the first part of the build script to perform the tag check only + uncollapsed_section_switch "ci_tag_early_checks" "Ensuring component versions match declared tags in CI builds" + echo "[Structured Tagging] Checking components: ${CI_BUILD_COMPONENTS}" + # shellcheck disable=SC2086 + for component in ${CI_BUILD_COMPONENTS}; do + bin/ci/update_tag.py --check ${component} || exit 1 + done + echo "[Structured Tagging] Components check done" + section_end "ci_tag_early_checks" +} + +# Check if each declared tag component is up to date before building +if [ -n "${CI_BUILD_COMPONENTS:-}" ]; then + # Remove any duplicates by splitting on whitespace, sorting, then joining back + CI_BUILD_COMPONENTS="$(echo "${CI_BUILD_COMPONENTS}" | xargs -n1 | sort -u | xargs)" + + ci_tag_early_checks +fi diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml index eeeb71de0da..c0153dbcd1d 100644 --- a/.gitlab-ci/image-tags.yml +++ b/.gitlab-ci/image-tags.yml @@ -11,6 +11,13 @@ # cannot parse input: "$image:$tag": invalid reference format # check the length of $tag; if it's > 128 chars you need to shorten your tag. +include: + # Include the conditional build image tags used for structured tagging runtime checks that happens + # during both build and test jobs + # It can be auto-generated by bin/ci/update_tag.py script, so to keep the script's serde simple, + # let's keep the component tags in a separate file + - .gitlab-ci/conditional-build-image-tags.yml + variables: DEBIAN_X86_64_BUILD_BASE_IMAGE: "debian/x86_64_build-base" DEBIAN_BASE_TAG: "20250223-way-prot" diff --git a/.gitlab-ci/setup-test-env.sh b/.gitlab-ci/setup-test-env.sh index ba2b5a689c2..9354d5b4ecf 100644 --- a/.gitlab-ci/setup-test-env.sh +++ b/.gitlab-ci/setup-test-env.sh @@ -187,6 +187,147 @@ function trap_err { error ${CURRENT_SECTION:-'unknown-section'}: ret code: $* } +# ------ Structured tagging +export _CI_TAG_CHECK_DIR="/mesa-ci-build-tag" + +_ci_tag_from_name_to_var() { + # Transforms MY_COMPONENT_TAG to my-component + echo "${1%_TAG}" | tr '[:upper:]' '[:lower:]' | tr '_' '-' +} + +_ci_tag_check() ( + x_off + _declared_name="${1}" + declare -n _declared="${_declared_name}" + _calculated="${2}" + local component_lower=$(_ci_tag_from_name_to_var "${_declared_name}") + + if [ -z "${_declared:-}" ]; then + # Close the section + error "Fatal error" + _error_msg "Structured tag is not set: ${_declared_name}" + _error_msg "" + echo "If you are adding a new component, please run:" + echo "bin/ci/update_tag.py --include ${component_lower}" + echo "This will automatically update the YAML file for you." + echo "Or manually edit .gitlab-ci/conditional-build-image-tags.yml to add the new" + echo "component." + error "" + exit 2 + fi + + if [ "${_declared}" != "${_calculated}" ]; then + # Close the section + error "Fatal error" + _error_msg "Mismatch in declared and calculated tags:" + _error_msg " ${_declared_name} from the YAML is \"${_declared}\"" + _error_msg " ... but I think it should be \"${_calculated}\"" + _error_msg "" + echo "Usually this happens when you change what you want to be built without also" + echo "changing the YAML declaration. For example, you've bumped SKQP to version" + echo "1.2.3, but you still have 'SKQP_VER: 1.2.2' in" + echo ".gitlab-ci/conditional-build-image-tags.yml." + echo "" + echo "If you meant to change the component I'm talking about, please change the" + echo "tag and resubmit. You can also run:" + echo "bin/ci/update_tag.py --include ${component_lower}" + echo "to update the tag automatically." + echo "" + echo "If you didn't mean to change the component, please ping @mesa/ci-helpers and we" + echo "can help you figure out what's gone wrong." + echo "" + echo "But for now, I've got to fail this build. Sorry." + exit 2 + fi + x_restore +) + +_ci_tag_check_build() { + x_off + if [ -n "${NEW_TAG_DRY_RUN:-}" ]; then + echo "${2}" + exit 0 + fi + + _ci_tag_check "${1}" "${2}" + + if [ -n "${CI_NOT_BUILDING_ANYTHING:-}" ]; then + exit 0 + fi + x_restore +} + +get_tag_file() { + x_off + # If no tag name is provided, return the directory + echo "${_CI_TAG_CHECK_DIR}/${1:-}" + x_restore +} + +_ci_tag_write() ( + set +x + local tag_name="${1}" + local tag_value="${2}" + + mkdir -p "${_CI_TAG_CHECK_DIR}" + echo -n "${tag_value}" > "$(get_tag_file "${tag_name}")" +) + +_ci_calculate_tag() { + x_off + # the args are files that can affect the build output + mapfile -t extra_files < <(printf '%s\n' "$@") + ( + for extra_file in "${extra_files[@]}"; do + if [ ! -f "${extra_file}" ]; then + error "File '${extra_file}' does not exist" + exit 1 + fi + cat "${extra_file}" + done + ) | md5sum | cut -d' ' -f1 + x_restore +} + +ci_tag_build_time_check() { + # Get the caller script and hash its contents plus the extra files + x_off + local tag_name="${1}" + local build_script_file="build-$(_ci_tag_from_name_to_var "${tag_name}").sh" + local build_script=".gitlab-ci/container/${build_script_file}" + shift + # now $@ has the extra files + local calculated_tag=$(_ci_calculate_tag "${build_script}" "$@") + + _ci_tag_check_build "${tag_name}" "${calculated_tag}" + _ci_tag_write "${tag_name}" "${calculated_tag}" + x_restore +} + +ci_tag_test_time_check() { + x_off + local tag_file=$(get_tag_file "${1}") + if [ ! -f "${tag_file}" ]; then + _error_msg "Structured tag file ${tag_file} does not exist" + _error_msg "Please run the ci_tag_build_time_check first and rebuild the image/rootfs" + exit 2 + fi + _ci_tag_check "${1}" "$(cat "${tag_file}")" + x_restore +} + +# Export all functions +export -f _ci_calculate_tag +export -f _ci_tag_check +export -f _ci_tag_check_build +export -f _ci_tag_from_name_to_var +export -f _ci_tag_write +export -f ci_tag_build_time_check +export -f ci_tag_test_time_check +export -f get_tag_file + +# Structured tagging ------ + export -f error export -f trap_err diff --git a/bin/ci/test/test_update_tag.py b/bin/ci/test/test_update_tag.py new file mode 100644 index 00000000000..1584ae1e2fd --- /dev/null +++ b/bin/ci/test/test_update_tag.py @@ -0,0 +1,368 @@ +import os +import sys +import subprocess +import yaml +import pytest + +from textwrap import dedent +from unittest.mock import patch + +from bin.ci.update_tag import ( + from_component_to_build_tag, + filter_components, + from_component_to_tag_var, + from_script_name_to_component, + from_script_name_to_tag_var, + load_container_yaml, + load_image_tags_yaml, + update_image_tag_in_yaml, + run_build_script, + main, +) + + +@pytest.fixture +def temp_image_tags_file(tmp_path, monkeypatch): + """ + Fixture that creates a temporary YAML file (to simulate CONDITIONAL_TAGS_FILE) + and updates the global variable accordingly. + """ + temp_file = tmp_path / "conditional-build-image-tags.yml" + # Write initial dummy content. + temp_file.write_text(yaml.dump({"variables": {}})) + monkeypatch.setattr("bin.ci.update_tag.CONDITIONAL_TAGS_FILE", str(temp_file)) + return temp_file + + +@pytest.fixture +def temp_container_ci_file(tmp_path, monkeypatch): + """ + Fixture that creates a temporary container CI file (to simulate CONTAINER_CI_FILE) + and updates the global variable accordingly. + """ + temp_file = tmp_path / "container-ci.yml" + temp_file.touch() + monkeypatch.setattr("bin.ci.update_tag.CONTAINER_CI_FILE", temp_file) + return temp_file + + +@pytest.fixture +def mock_container_dir(tmp_path, monkeypatch): + """ + Fixture that creates a dummy container directory and build script. + """ + # Create a dummy container directory and build script. + container_dir = tmp_path / "container" + container_dir.mkdir() + + # Patch CONTAINER_DIR so that the build script is found. + monkeypatch.setattr("bin.ci.update_tag.CONTAINER_DIR", container_dir) + + # Create a dummy setup-test-env.sh file and patch set_dummy_env_vars. + dummy_setup_path = tmp_path / "setup-test-env.sh" + dummy_setup_path.write_text("echo Setup") + monkeypatch.setattr( + "bin.ci.update_tag.prepare_setup_env_script", lambda: dummy_setup_path + ) + + return container_dir + + +@pytest.fixture +def mock_build_script(mock_container_dir): + """ + Fixture that creates a dummy build script in the container directory. + """ + build_script = mock_container_dir / "build-foo.sh" + return build_script + + +############################################################################### +# Tests for argument parsing and helper functions +############################################################################### + + +@pytest.mark.parametrize( + "component, expected_tag_var", + [ + ("foo", "FOO_TAG"), + ("my-component", "MY_COMPONENT_TAG"), + ("foo-bar-baz", "FOO_BAR_BAZ_TAG"), + ], +) +def test_from_component_to_tag_var(component, expected_tag_var): + """ + Test that from_component_to_tag_var returns the correct tag variable name. + """ + assert from_component_to_tag_var(component) == expected_tag_var + + +@pytest.mark.parametrize( + "component, expected_build_tag", + [ + ("foo", "CONDITIONAL_BUILD_FOO_TAG"), + ("my-component", "CONDITIONAL_BUILD_MY_COMPONENT_TAG"), + ("foo-bar-baz", "CONDITIONAL_BUILD_FOO_BAR_BAZ_TAG"), + ], +) +def test_from_component_to_build_tag(component, expected_build_tag): + """ + Test that from_component_to_build_tag returns the correct build tag name. + """ + assert from_component_to_build_tag(component) == expected_build_tag + + +@pytest.mark.parametrize( + "script_name, expected_component", + [ + ("build-foo.sh", "foo"), + ("build-my-component.sh", "my-component"), + ("build-foo-bar-baz.sh", "foo-bar-baz"), + ], +) +def test_from_script_name_to_component(script_name, expected_component): + """ + Test that from_script_name_to_component returns the correct component name. + """ + assert from_script_name_to_component(script_name) == expected_component + + +@pytest.mark.parametrize( + "script_name, expected_tag_var", + [ + ("build-foo.sh", "FOO_TAG"), + ("build-my-component.sh", "MY_COMPONENT_TAG"), + ("build-foo-bar-baz.sh", "FOO_BAR_BAZ_TAG"), + ], +) +def test_from_script_name_to_tag_var(script_name, expected_tag_var): + """ + Test that from_script_name_to_tag_var returns the correct tag variable name. + """ + assert from_script_name_to_tag_var(script_name) == expected_tag_var + + +def test_filter_components(): + """ + Test that filter_components returns only components that match an include regex + and do not match any exclude regex. If includes is empty, an empty list is returned. + """ + components = ["alpha", "beta", "gamma", "delta"] + + # When includes is empty, should return an empty list. + assert filter_components(components, [], []) == [] + + # Test includes only. + # Components that start with 'a' or 'b' + expected = ["alpha", "beta"] + result = filter_components(components, ["^[a-b].*$"], []) + assert result == expected + + # Test with an exclude that filters out "alpha" (which matches "lph"). + result = filter_components(components, ["^[a-b].*$"], ["^.*lph.*$"]) + # "alpha" is removed. + assert result == ["beta"] + + +@pytest.mark.parametrize( + "component, tag", + [ + # Basic case + ("test", "new_tag"), + # Component with hyphen + ("my-component", "v1.2.3"), + # Uppercase component name + ("UpperCase", "123"), + # Numbers in component name + ("123service", "build-456"), + ], +) +def test_update_image_tag_in_yaml(component, tag, temp_image_tags_file): + """ + Test multiple updates with different component names and tags, verifying: + 1. Correct variable name generation + 2. Proper tag value storage + 3. Maintained sorting of variables + """ + # Initial update + update_image_tag_in_yaml(component, tag) + + data = load_image_tags_yaml() + expected_var = from_component_to_build_tag(component) + assert data["variables"][expected_var] == tag + + # Add second component and verify both exist + update_image_tag_in_yaml("another_component", "secondary_tag") + updated_data = load_image_tags_yaml() + + assert from_component_to_build_tag("another_component") in updated_data["variables"] + assert updated_data["variables"][expected_var] == tag # Original value remains + + # Verify sorting + variables = list(data["variables"].keys()) + assert len(updated_data["variables"]) == len(variables) + 1 + assert variables == sorted(variables), "Variables are not alphabetically sorted" + + +def test_if_run_extracts_the_tag_from_stdout(monkeypatch, mock_build_script): + """ + Test that run_build_script returns the tag (last stdout line) when the build + script executes successfully. + """ + mock_build_script.write_text("#!/bin/bash\necho Build script\necho new_tag") + # Create a fake subprocess.CompletedProcess to simulate a successful build. + fake_result = subprocess.CompletedProcess( + args=[], returncode=0, stdout="line1\nnew_tag", stderr="" + ) + monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: fake_result) + + tag = run_build_script("foo", check_only=False) + assert tag == "new_tag" + + +def test_running_real_process_works(monkeypatch, mock_build_script): + """ + Test that run_build_script returns the tag (last stdout line) when the build + script executes successfully. + """ + mock_build_script.write_text("echo Build script\necho new_tag") + tag = run_build_script("foo", check_only=False) + assert tag == "new_tag" + + +############################################################################### +# Tests for main() argument features +############################################################################### + + +def setup_build_script_and_container_ci_file(container_dir, container_ci_file, comp): + tag_var = from_component_to_build_tag(comp) + build_check_var = from_component_to_tag_var(comp) + build_script_path = container_dir / f"build-{comp}.sh" + build_script_path.write_text( + dedent( + f""" + #!/bin/bash + ci_tag_build_time_check {tag_var} + echo "new_tag" + """ + ) + ) + container_ci_file.write_text( + container_ci_file.read_text() + + dedent( + f""" + .container-builds-{comp}: + variables: + {tag_var}: "${{{build_check_var}}}" + """ + ) + ) + + +def test_main_list(monkeypatch, capsys, mock_container_dir, temp_container_ci_file): + """ + Test that when the --list argument is provided, main() prints all detected + components + """ + # Monkeypatch find_components to return a known list. + for comp in ["comp-a", "comp-b"]: + setup_build_script_and_container_ci_file( + mock_container_dir, temp_container_ci_file, comp + ) + # Set sys.argv to simulate passing --list. + monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--list"]) + + main() + + captured = capsys.readouterr().out + assert "Detected components:" in captured + assert "comp-a" in captured + assert "comp-b" in captured + + +def test_main_check(monkeypatch, temp_image_tags_file, temp_container_ci_file): + """ + Test that when --check is provided, main() calls run_build_script in check mode. + """ + + with patch( + "bin.ci.update_tag.run_build_script", return_value="dummy_tag" + ) as mock_run_build_script: + # Simulate command line: --check compX + monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--check", "comp-x"]) + + main() + # Verify run_build_script was called with check_only=True. + assert mock_run_build_script.call_count == 1 + assert "comp-x" in mock_run_build_script.call_args.args[0] + assert mock_run_build_script.call_args.kwargs["check_only"] is True + + +EXIT_CODE_SCENARIOS = { + "unbound_variable": ( + "line 2: UNDEFINED_VARIABLE: unbound variable", + 127, + 3, + "Please set the variable UNDEFINED_VARIABLE", + ), + "build script error": ( + "", + 50, + 50, + "", + ), + "tag_mismatch": ( + "Tag mismatch for foo.", + 2, + 2, + "Tag mismatch for foo.", + ), +} + + +@pytest.mark.parametrize( + "stderr_content, script_returncode, expected_exit_code, expected_error_message", + EXIT_CODE_SCENARIOS.values(), + ids=list(EXIT_CODE_SCENARIOS.keys()), +) +def test_build_script_error_exit_codes( + monkeypatch, + mock_build_script, + capsys, + stderr_content, + script_returncode, + expected_exit_code, + expected_error_message, +): + """ + Test that build script errors generate appropriate exit codes: + 1 - Unhandled error in this script + 2 - Tag mismatch when using --check + 3 - Unbound variable error in build script + x - Build script failed with return code x + """ + # Create a build script that will fail + mock_build_script.write_text("#!/bin/bash\necho 'This will fail'") + + # Create a fake subprocess.CompletedProcess to simulate a failed build + fake_result = subprocess.CompletedProcess( + args=[], returncode=script_returncode, stdout="", stderr=stderr_content + ) + monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: fake_result) + monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--check", "foo"]) + monkeypatch.setattr( + "bin.ci.update_tag.get_current_tag_value", lambda *args: "current_tag" + ) + + # Mock sys.exit to capture the exit code instead of exiting the test + with pytest.raises(SystemExit) as e: + main() + + # Check for expected error message if one is specified + if expected_error_message: + captured = capsys.readouterr() + assert expected_error_message in captured.err + + # Verify correct exit code + assert e.value.code == expected_exit_code diff --git a/bin/ci/update_tag.py b/bin/ci/update_tag.py new file mode 100755 index 00000000000..5a315085000 --- /dev/null +++ b/bin/ci/update_tag.py @@ -0,0 +1,419 @@ +#!/usr/bin/env python3 +import logging +import os +import re +import sys +import argparse +import subprocess +import yaml +from typing import Optional, Set, Any + +from datetime import datetime, timezone +from pathlib import Path + +CI_PROJECT_DIR: str | None = os.environ.get("CI_PROJECT_DIR", None) +GIT_REPO_ROOT: str = CI_PROJECT_DIR or str(Path(__file__).resolve().parent.parent.parent) +SETUP_TEST_ENV_PATH: Path = Path(GIT_REPO_ROOT) / ".gitlab-ci" / "setup-test-env.sh" +CONDITIONAL_TAGS_FILE: Path = ( + Path(GIT_REPO_ROOT) / ".gitlab-ci" / "conditional-build-image-tags.yml" +) +CONTAINER_DIR: Path = Path(GIT_REPO_ROOT) / ".gitlab-ci" / "container" +CONTAINER_CI_FILE: Path = CONTAINER_DIR / "gitlab-ci.yml" + +# Very simple type alias for GitLab YAML data structure +# It is composed by a dictionary of job names, each with a dictionary of fields +# (e.g., script, stage, rules, etc.) +YamlData = dict[str, dict[str, Any]] + +# Dummy environment vars to make build scripts happy +# To avoid set -u errors in build scripts +DUMMY_ENV_VARS: dict[str, str] = { + # setup-test-env.sh + "CI_JOB_STARTED_AT": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S%z"), + # build-deqp.sh + "DEQP_API": "dummy", + "DEQP_TARGET": "dummy", +} + + +def from_component_to_build_tag(component: str) -> str: + # e.g., "angle" -> "CONDITIONAL_BUILD_ANGLE_TAG" + return "CONDITIONAL_BUILD_" + re.sub(r"-", "_", component.upper()) + "_TAG" + + +def from_component_to_tag_var(component: str) -> str: + # e.g., "angle" -> "ANGLE_TAG" + return re.sub(r"-", "_", component.upper()) + "_TAG" + + +def from_script_name_to_component(script_name: str) -> str: + # e.g., "build-angle.sh" -> "angle" + return re.sub(r"^build-([a-z0-9_-]+)\.sh$", r"\1", script_name) + + +def from_script_name_to_tag_var(script_name: str) -> str: + # e.g., "build-angle.sh" -> "ANGLE_TAG" + return ( + re.sub(r"^build-([a-z0-9_-]+)\.sh$", r"\1_TAG", script_name) + .replace("-", "_") + .upper() + ) + + +def prepare_setup_env_script() -> Path: + """ + Sets up dummy environment variables to mimic the script in the CI repo. + Returns the path to the setup-test-env.sh script. + """ + if not SETUP_TEST_ENV_PATH.exists(): + sys.exit(".gitlab-ci/setup-test-env.sh not found. Exiting.") + + # Dummy environment vars to mimic the script + for key, value in DUMMY_ENV_VARS.items(): + os.environ[key] = value + + os.environ["CI_PROJECT_DIR"] = GIT_REPO_ROOT + + return SETUP_TEST_ENV_PATH + + +def validate_build_script(script_filename: str) -> bool: + """ + Returns True if the build script for the given component uses the structured tag variable. + """ + build_script = CONTAINER_DIR / script_filename + with open(build_script, "r", encoding="utf-8") as f: + script_content = f.read() + + tag_var = from_script_name_to_tag_var(script_filename) + if not re.search(tag_var, script_content, re.IGNORECASE): + logging.debug( + f"Skipping {build_script} because it doesn't use {tag_var}", + ) + return False + return True + + +def load_container_yaml() -> YamlData: + if not os.path.isfile(CONTAINER_CI_FILE): + sys.exit(f"File not found: {CONTAINER_CI_FILE}") + + # Ignore !reference and other custom GitLab tags, we just want to know the + # job names and fields + yaml.SafeLoader.add_multi_constructor('', lambda loader, suffix, node: None) + with open(CONTAINER_CI_FILE, "r", encoding="utf-8") as f: + + data = yaml.load(f, Loader=yaml.SafeLoader) + if not isinstance(data, dict): + return {"variables": {}} + return data + + +def find_candidate_components() -> list[str]: + """ + 1) Reads .gitlab-ci/container/gitlab-ci.yml to find component links: + lines matching '.*.container-builds-' + 2) Looks for matching build-.sh in .gitlab-ci/container/ + 3) Returns a sorted list of components in the intersection of these sets. + """ + container_yaml = load_container_yaml() + # Extract patterns like `container-builds-foo` from job names + candidates: Set[str] = set() + for job_name in container_yaml: + if match := re.search(r"\.container-builds-([a-z0-9_-]+)$", str(job_name)): + candidates.add(match.group(1)) + + if not candidates: + logging.error( + f"No viable build components found in {CONTAINER_CI_FILE}. " + "Please check the file for valid component names. " + "They should be named like '.container-builds-'." + ) + return [] + + # Find build scripts named build-.sh + build_scripts: list[str] = [] + for path in CONTAINER_DIR.glob("build-*.sh"): + if validate_build_script(path.name): + logging.info(f"Found build script: {path.name}") + component = from_script_name_to_component(path.name) + build_scripts.append(component) + + # Return sorted intersection of components found in build scripts and candidates + return sorted(candidates.intersection(build_scripts)) + + +def filter_components( + components: list[str], includes: list[str], excludes: list[str] +) -> list[str]: + """ + Returns components that match at least one `includes` regex and none of the `excludes` regex. + If includes is empty, returns an empty list (unless user explicitly does --all or --include). + """ + if not includes: + return [] + + filtered = [] + for comp in components: + # Must match at least one "include" + if not any(re.fullmatch(inc, comp) for inc in includes): + logging.debug(f"Excluding {comp}, no matches in includes.") + continue + # Must not match any "exclude" + if any(re.fullmatch(exc, comp) for exc in excludes): + logging.debug(f"Excluding {comp}, matched exclude pattern.") + continue + filtered.append(comp) + return filtered + + +def run_build_script(component: str, check_only: bool = False) -> Optional[str]: + """ + Runs .gitlab-ci/container/build-.sh to produce a new tag (last line of stdout). + If check_only=True, we skip updates to the YAML (but do the build to see if it passes). + Returns the extracted tag (string) on success, or None on failure. + """ + # 1) Set up environment + setup_env_script = prepare_setup_env_script() + + build_script = os.path.join(CONTAINER_DIR, f"build-{component}.sh") + if not os.path.isfile(build_script): + logging.error(f"Build script not found for {component}: {build_script}") + return None + + # Tag var should appear in the script, e.g., ANGLE_TAG for 'angle' + tag_var = from_component_to_tag_var(component) + + # We set up environment for the child process + child_env: dict[str, str] = {} + child_env["NEW_TAG_DRY_RUN"] = "1" + if check_only: + # For checking only + child_env.pop("NEW_TAG_DRY_RUN", None) + child_env["CI_NOT_BUILDING_ANYTHING"] = "1" + if tag_value := get_current_tag_value(component): + child_env[tag_var] = tag_value + else: + logging.error(f"No current tag value for {component}") + return None + + logging.debug( + f"Running build for {component} with " + f"{tag_var}={child_env.get(tag_var)} " + f"(check_only={check_only})" + ) + + # Run the build script + result = subprocess.run( + ["bash", "-c", f"source {setup_env_script} && bash -x {build_script}"], + env=os.environ | child_env, + capture_output=True, + text=True, + ) + logging.debug(f"{' '.join(result.args)}") + + # Tag check succeeded + if result.returncode == 0: + # Tag is assumed to be the last line of stdout + lines = result.stdout.strip().splitlines() + return lines[-1] if lines else "" + + # Tag check failed, let's dissect the error + + if result.returncode == 2: + logging.error( + f"Tag mismatch for {component}." + ) + logging.error(result.stdout) + return None + + # Check if there's an unbound variable error + err_output = result.stderr + unbound_match = re.search(r"([A-Z_]+)(?=: unbound variable)", err_output) + if unbound_match: + var_name = unbound_match.group(1) + logging.fatal(f"Please set the variable {var_name} in {build_script}.") + sys.exit(3) + + # Unexpected error in the build script, propagate the exit code + logging.fatal( + f"Build script for {component} failed with return code {result.returncode}" + ) + logging.error(result.stdout) + sys.exit(result.returncode) + + +def load_image_tags_yaml() -> YamlData: + try: + with open(CONDITIONAL_TAGS_FILE, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + if not isinstance(data, dict): + return {"variables": {}} + if "variables" not in data: + data["variables"] = {} + return data + except FileNotFoundError: + return {"variables": {}} + + +def get_current_tag_value(component: str) -> Optional[str]: + full_tag_var = from_component_to_build_tag(component) + data = load_image_tags_yaml() + variables = data.get("variables", {}) + if not isinstance(variables, dict): + return None + return variables.get(full_tag_var) + + +def update_image_tag_in_yaml(component: str, tag_value: str) -> None: + """ + Uses PyYAML to edit the YAML file at IMAGE_TAGS_FILE, setting the environment variable + for the given component. Maintains sorted keys. + """ + full_tag_var = from_component_to_build_tag(component) + data = load_image_tags_yaml() + + # Ensure we have a variables dictionary + if "variables" not in data: + data["variables"] = {} + elif not isinstance(data["variables"], dict): + data["variables"] = {} + + # Update the tag + data["variables"][full_tag_var] = tag_value + + # Sort the variables + data["variables"] = dict(sorted(data["variables"].items())) + + # Write back to file + with open(CONDITIONAL_TAGS_FILE, "w", encoding="utf-8") as f: + yaml.dump(data, f, sort_keys=False) # Don't sort top-level keys + + +def parse_args(): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="Manage container image tags for CI builds with regex-based includes/excludes.", + epilog=""" +Exit codes: + 0 - Success + 1 - Unhandled error in this script + 2 - Tag mismatch when using --check + 3 - Unbound variable error in build script + x - Build script failed with return code x + """, + ) + + parser.add_argument( + "--include", + "-i", + action="append", + default=[], + help="Full match regex pattern for components to include.", + ) + parser.add_argument( + "--exclude", + "-x", + action="append", + default=[], + help="Full match regex pattern for components to exclude.", + ) + parser.add_argument( + "--all", action="store_true", help="Equivalent to --include '.*'" + ) + parser.add_argument( + "--check", + "-c", + action="append", + default=[], + help="Check matching components instead of updating YAML. " + "If any component fails, exit with a non-zero exit code.", + ) + parser.add_argument( + "--list", + "-l", + action="store_true", + help="List all available components and exit.", + ) + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Increase verbosity level (-v for info, -vv for debug)", + ) + + if len(sys.argv) == 1: + parser.print_help() + sys.exit(0) + + return parser.parse_args() + + +def main(): + args = parse_args() + + # Configure logging based on verbosity level + if args.verbose == 1: + log_level = logging.INFO + elif args.verbose == 2: + log_level = logging.DEBUG + else: + log_level = logging.WARNING + + logging.basicConfig(level=log_level, format="%(levelname)s: %(message)s") + + # 0) Check if the YAML file exists + if not os.path.isfile(CONDITIONAL_TAGS_FILE): + logging.fatal( + f"Conditional build image tags file not found: {CONDITIONAL_TAGS_FILE}" + ) + return + + # 1) If checking, just run build scripts in check mode and propagate errors + if args.check: + tag_mismatch = False + for comp in args.check: + try: + if run_build_script(comp, check_only=True) is None: + # The tag is invalid + tag_mismatch = True + except SystemExit as e: + # Let custom exit codes propagate + raise e + except Exception as e: + logging.error(f"Internal error: {e}") + sys.exit(3) + # If any component failed, exit with code 1 + if tag_mismatch: + sys.exit(2) + return + + # Convert --all into a wildcard include + if args.all: + args.include.append(".*") + + # 2) If --list, just show all discovered components + all_components = find_candidate_components() + if args.list: + print("Detected components:", ", ".join(all_components)) + return + + # 3) Filter components + final_components = filter_components(all_components, args.include, args.exclude) + + if args.verbose: + logging.debug(f"Found components: {all_components}") + logging.debug(f"Filtered components: {final_components}") + + for comp in final_components: + logging.info(f"Updating {comp}...") + new_tag = run_build_script(comp, check_only=False) + if new_tag is not None: + update_image_tag_in_yaml(comp, new_tag) + if args.verbose: + logging.debug(f"Updated {comp} with tag: {new_tag}") + + +if __name__ == "__main__": + main()