AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ci/bin: update_tag: improve tag load

Browse Source 
Replace global path variables with ProjectPaths dataclass
- Add explicit file existence check before loading YAML
- Enhance tag retrieval by checking environment variables first
- Add logging for better debugging of tag selection process
- Remove redundant file existence check in main function
- Improve error handling for missing conditional tags file

Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/33863>
This commit is contained in:
Guilherme Gallo
2025-03-04 01:23:49 -03:00
committed by Marge Bot
parent 5798f5d05f
commit 82073f7be3
3 changed files with 126 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitlab-ci/image-tags.yml
+1 -1
View File
@@ -20,7 +20,7 @@ include:
variables:
DEBIAN_X86_64_BUILD_BASE_IMAGE: "debian/x86_64_build-base"
DEBIAN_BASE_TAG: "20250304-virglcrosvm"
DEBIAN_BASE_TAG: "20250310-update-tag"
DEBIAN_X86_64_BUILD_IMAGE_PATH: "debian/x86_64_build"
DEBIAN_BUILD_TAG: "20250303-setup-env"
bin/ci/test/test_update_tag.py
+45 -17
View File
@@ -13,37 +13,58 @@ from bin.ci.update_tag import (
from_component_to_tag_var,
from_script_name_to_component,
from_script_name_to_tag_var,
load_container_yaml,
load_image_tags_yaml,
update_image_tag_in_yaml,
run_build_script,
main,
PATHS,
)
@pytest.fixture
def temp_image_tags_file(tmp_path, monkeypatch):
"""
Fixture that creates a temporary YAML file (to simulate CONDITIONAL_TAGS_FILE)
Fixture that creates a temporary YAML file (to simulate conditional_tags)
and updates the global variable accordingly.
"""
# Create a temporary test file
temp_file = tmp_path / "conditional-build-image-tags.yml"
# Write initial dummy content.
temp_file.write_text(yaml.dump({"variables": {}}))
monkeypatch.setattr("bin.ci.update_tag.CONDITIONAL_TAGS_FILE", str(temp_file))
return temp_file
# Create a copy of the original PATHS values
original_conditional_tags = PATHS.conditional_tags
# Patch the PATHS instance with our test file
PATHS.conditional_tags = temp_file
# Yield the test file for the test to use
yield temp_file
# Restore the original value after the test
PATHS.conditional_tags = original_conditional_tags
@pytest.fixture
def temp_container_ci_file(tmp_path, monkeypatch):
"""
Fixture that creates a temporary container CI file (to simulate CONTAINER_CI_FILE)
Fixture that creates a temporary container CI file (to simulate container_ci)
and updates the global variable accordingly.
"""
# Create a temporary test file
temp_file = tmp_path / "container-ci.yml"
temp_file.touch()
monkeypatch.setattr("bin.ci.update_tag.CONTAINER_CI_FILE", temp_file)
return temp_file
# Create a copy of the original PATHS values
original_container_ci = PATHS.container_ci
# Patch the PATHS instance with our test file
PATHS.container_ci = temp_file
# Yield the test file for the test to use
yield temp_file
# Restore the original value after the test
PATHS.container_ci = original_container_ci
@pytest.fixture
@@ -51,21 +72,28 @@ def mock_container_dir(tmp_path, monkeypatch):
"""
Fixture that creates a dummy container directory and build script.
"""
# Create a dummy container directory and build script.
# Create a dummy container directory
container_dir = tmp_path / "container"
container_dir.mkdir()
# Patch CONTAINER_DIR so that the build script is found.
monkeypatch.setattr("bin.ci.update_tag.CONTAINER_DIR", container_dir)
# Create a dummy setup-test-env.sh file and patch set_dummy_env_vars.
# Create a dummy setup-test-env.sh file
dummy_setup_path = tmp_path / "setup-test-env.sh"
dummy_setup_path.write_text("echo Setup")
monkeypatch.setattr(
"bin.ci.update_tag.prepare_setup_env_script", lambda: dummy_setup_path
)
return container_dir
# Save original values
original_container_dir = PATHS.container_dir
original_setup_test_env = PATHS.setup_test_env
# Patch the PATHS instance
PATHS.container_dir = container_dir
PATHS.setup_test_env = dummy_setup_path
# Yield the directory for the test to use
yield container_dir
# Restore original values
PATHS.container_dir = original_container_dir
PATHS.setup_test_env = original_setup_test_env
@pytest.fixture
bin/ci/update_tag.py
+80 -41
View File
@@ -7,18 +7,11 @@ import argparse
import subprocess
import yaml
from typing import Optional, Set, Any
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
CI_PROJECT_DIR: str | None = os.environ.get("CI_PROJECT_DIR", None)
GIT_REPO_ROOT: str = CI_PROJECT_DIR or str(Path(__file__).resolve().parent.parent.parent)
SETUP_TEST_ENV_PATH: Path = Path(GIT_REPO_ROOT) / ".gitlab-ci" / "setup-test-env.sh"
CONDITIONAL_TAGS_FILE: Path = (
Path(GIT_REPO_ROOT) / ".gitlab-ci" / "conditional-build-image-tags.yml"
)
CONTAINER_DIR: Path = Path(GIT_REPO_ROOT) / ".gitlab-ci" / "container"
CONTAINER_CI_FILE: Path = CONTAINER_DIR / "gitlab-ci.yml"
# Very simple type alias for GitLab YAML data structure
# It is composed by a dictionary of job names, each with a dictionary of fields
@@ -36,6 +29,49 @@ DUMMY_ENV_VARS: dict[str, str] = {
}
@dataclass
class ProjectPaths:
"""Manages all project-related paths"""
root: Path = field(default_factory=lambda: Path(), init=False)
setup_test_env: Path = field(default_factory=lambda: Path(), init=False)
conditional_tags: Path = field(default_factory=lambda: Path(), init=False)
container_dir: Path = field(default_factory=lambda: Path(), init=False)
container_ci: Path = field(default_factory=lambda: Path(), init=False)
def __post_init__(self):
self.root = self.find_root()
self.setup_test_env = self.root / ".gitlab-ci" / "setup-test-env.sh"
self.conditional_tags = self.root / ".gitlab-ci" / "conditional-build-image-tags.yml"
self.container_dir = self.root / ".gitlab-ci" / "container"
self.container_ci = self.container_dir / "gitlab-ci.yml"
def find_root(self) -> Path:
"""Find git repository root or fallback to other methods"""
try:
logging.debug("Getting git repo root using git rev-parse")
return Path(
subprocess.check_output(
["git", "rev-parse", "--show-toplevel"],
text=True,
stderr=subprocess.PIPE,
).strip()
)
# FileNotFoundError is raised if git is not installed
except (subprocess.CalledProcessError, FileNotFoundError):
# Fallback to CI_PROJECT_DIR or path-based method
if ci_dir := os.environ.get("CI_PROJECT_DIR"):
logging.debug(f"Getting git repo root using CI_PROJECT_DIR: {ci_dir}")
return Path(ci_dir)
# Last resort, use the path to this script
logging.debug("Getting git repo root using path to this script")
return Path(__file__).resolve().parent.parent.parent
PATHS: ProjectPaths = ProjectPaths()
def from_component_to_build_tag(component: str) -> str:
# e.g., "angle" -> "CONDITIONAL_BUILD_ANGLE_TAG"
return "CONDITIONAL_BUILD_" + re.sub(r"-", "_", component.upper()) + "_TAG"
@@ -65,23 +101,24 @@ def prepare_setup_env_script() -> Path:
Sets up dummy environment variables to mimic the script in the CI repo.
Returns the path to the setup-test-env.sh script.
"""
if not SETUP_TEST_ENV_PATH.exists():
sys.exit(".gitlab-ci/setup-test-env.sh not found. Exiting.")
if not PATHS.setup_test_env.exists():
logging.error(".gitlab-ci/setup-test-env.sh not found. Exiting.")
sys.exit(1)
# Dummy environment vars to mimic the script
for key, value in DUMMY_ENV_VARS.items():
os.environ[key] = value
os.environ["CI_PROJECT_DIR"] = GIT_REPO_ROOT
os.environ["CI_PROJECT_DIR"] = str(PATHS.root)
return SETUP_TEST_ENV_PATH
return PATHS.setup_test_env
def validate_build_script(script_filename: str) -> bool:
"""
Returns True if the build script for the given component uses the structured tag variable.
"""
build_script = CONTAINER_DIR / script_filename
build_script = PATHS.container_dir / script_filename
with open(build_script, "r", encoding="utf-8") as f:
script_content = f.read()
@@ -95,14 +132,14 @@ def validate_build_script(script_filename: str) -> bool:
def load_container_yaml() -> YamlData:
if not os.path.isfile(CONTAINER_CI_FILE):
sys.exit(f"File not found: {CONTAINER_CI_FILE}")
if not PATHS.container_ci.is_file():
logging.error(f"File not found: {PATHS.container_ci}")
sys.exit(1)
# Ignore !reference and other custom GitLab tags, we just want to know the
# job names and fields
yaml.SafeLoader.add_multi_constructor('', lambda loader, suffix, node: None)
with open(CONTAINER_CI_FILE, "r", encoding="utf-8") as f:
yaml.SafeLoader.add_multi_constructor("", lambda loader, suffix, node: None)
with open(str(PATHS.container_ci), "r", encoding="utf-8") as f:
data = yaml.load(f, Loader=yaml.SafeLoader)
if not isinstance(data, dict):
return {"variables": {}}
@@ -125,7 +162,7 @@ def find_candidate_components() -> list[str]:
if not candidates:
logging.error(
f"No viable build components found in {CONTAINER_CI_FILE}. "
f"No viable build components found in {PATHS.container_ci}. "
"Please check the file for valid component names. "
"They should be named like '.container-builds-<component>'."
)
@@ -133,7 +170,7 @@ def find_candidate_components() -> list[str]:
# Find build scripts named build-<component>.sh
build_scripts: list[str] = []
for path in CONTAINER_DIR.glob("build-*.sh"):
for path in PATHS.container_dir.glob("build-*.sh"):
if validate_build_script(path.name):
logging.info(f"Found build script: {path.name}")
component = from_script_name_to_component(path.name)
@@ -176,8 +213,8 @@ def run_build_script(component: str, check_only: bool = False) -> Optional[str]:
# 1) Set up environment
setup_env_script = prepare_setup_env_script()
build_script = os.path.join(CONTAINER_DIR, f"build-{component}.sh")
if not os.path.isfile(build_script):
build_script = PATHS.container_dir / f"build-{component}.sh"
if not build_script.is_file():
logging.error(f"Build script not found for {component}: {build_script}")
return None
@@ -244,24 +281,33 @@ def run_build_script(component: str, check_only: bool = False) -> Optional[str]:
def load_image_tags_yaml() -> YamlData:
try:
with open(CONDITIONAL_TAGS_FILE, "r", encoding="utf-8") as f:
data = yaml.safe_load(f)
if not isinstance(data, dict):
return {"variables": {}}
if "variables" not in data:
data["variables"] = {}
return data
except FileNotFoundError:
return {"variables": {}}
# 0) Check if the YAML file exists
if not os.path.isfile(CONDITIONAL_TAGS_FILE):
raise FileNotFoundError(f"Conditional build image tags file not found: {CONDITIONAL_TAGS_FILE}")
with open(str(PATHS.conditional_tags), "r", encoding="utf-8") as f:
data = yaml.safe_load(f)
if not isinstance(data, dict):
return {"variables": {}}
if "variables" not in data:
data["variables"] = {}
return data
def get_current_tag_value(component: str) -> Optional[str]:
# If the CI job already set the tag, use it
tag_var = from_component_to_tag_var(component)
if os.getenv(tag_var):
logging.debug(f"Using tag from environment variable {tag_var}")
return os.getenv(tag_var)
# Otherwise, look in the YAML file, which normally occurs when updating tags locally
full_tag_var = from_component_to_build_tag(component)
data = load_image_tags_yaml()
variables = data.get("variables", {})
if not isinstance(variables, dict):
return None
logging.debug(f"Using tag from YAML file {full_tag_var}")
return variables.get(full_tag_var)
@@ -286,7 +332,7 @@ def update_image_tag_in_yaml(component: str, tag_value: str) -> None:
data["variables"] = dict(sorted(data["variables"].items()))
# Write back to file
with open(CONDITIONAL_TAGS_FILE, "w", encoding="utf-8") as f:
with open(str(PATHS.conditional_tags), "w", encoding="utf-8") as f:
yaml.dump(data, f, sort_keys=False) # Don't sort top-level keys
@@ -363,13 +409,6 @@ def main():
logging.basicConfig(level=log_level, format="%(levelname)s: %(message)s")
# 0) Check if the YAML file exists
if not os.path.isfile(CONDITIONAL_TAGS_FILE):
logging.fatal(
f"Conditional build image tags file not found: {CONDITIONAL_TAGS_FILE}"
)
return
# 1) If checking, just run build scripts in check mode and propagate errors
if args.check:
tag_mismatch = False
@@ -383,7 +422,7 @@ def main():
raise e
except Exception as e:
logging.error(f"Internal error: {e}")
sys.exit(3)
sys.exit(1)
# If any component failed, exit with code 1
if tag_mismatch:
sys.exit(2)