From 4173e4b18f255886aafc689c2e0010a52d4babba Mon Sep 17 00:00:00 2001 From: Guilherme Gallo Date: Wed, 17 May 2023 01:12:28 -0300 Subject: [PATCH] ci/lava: Hide JWT block during YAML dump Make hide_sensitive_data work in a block fashion, not only hiding the JWT line, since these tokens are huge, it may break the line when it extrapolates the YAML dump width. Signed-off-by: Guilherme Gallo Part-of: --- .gitlab-ci/lava/utils/lava_job_definition.py | 6 +++--- .gitlab-ci/lava/utils/log_follower.py | 17 +++++++++++++++-- .gitlab-ci/tests/utils/test_lava_log.py | 16 ++++++++-------- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci/lava/utils/lava_job_definition.py b/.gitlab-ci/lava/utils/lava_job_definition.py index b05961dfb53..55dbb902dac 100644 --- a/.gitlab-ci/lava/utils/lava_job_definition.py +++ b/.gitlab-ci/lava/utils/lava_job_definition.py @@ -127,9 +127,9 @@ def artifact_download_steps(args): if args.jwt_file: with open(args.jwt_file) as jwt_file: download_steps += [ - "set +x", - f'echo -n "{jwt_file.read()}" > "{args.jwt_file}" # HIDEME', - "set -x", + "set +x # HIDE_START", + f'echo -n "{jwt_file.read()}" > "{args.jwt_file}"', + "set -x # HIDE_END", f'echo "export CI_JOB_JWT_FILE={args.jwt_file}" >> /set-job-env-vars.sh', ] else: diff --git a/.gitlab-ci/lava/utils/log_follower.py b/.gitlab-ci/lava/utils/log_follower.py index 19837543833..1fdf490bcb8 100644 --- a/.gitlab-ci/lava/utils/log_follower.py +++ b/.gitlab-ci/lava/utils/log_follower.py @@ -293,5 +293,18 @@ def fatal_err(msg, exception=None): sys.exit(1) -def hide_sensitive_data(yaml_data: str, hide_tag: str ="HIDEME"): - return "".join(line for line in yaml_data.splitlines(True) if hide_tag not in line) +def hide_sensitive_data(yaml_data: str, start_hide: str = "HIDE_START", end_hide: str = "HIDE_END") -> str: + skip_line = False + dump_data: list[str] = [] + for line in yaml_data.splitlines(True): + if start_hide in line: + skip_line = True + elif end_hide in line: + skip_line = False + + if skip_line: + continue + + dump_data.append(line) + + return "".join(dump_data) diff --git a/.gitlab-ci/tests/utils/test_lava_log.py b/.gitlab-ci/tests/utils/test_lava_log.py index 269253b0336..e74aaf2fead 100644 --- a/.gitlab-ci/tests/utils/test_lava_log.py +++ b/.gitlab-ci/tests/utils/test_lava_log.py @@ -153,29 +153,29 @@ SENSITIVE_DATA_SCENARIOS = { "no sensitive data tagged": ( ["bla bla", "mytoken: asdkfjsde1341=="], ["bla bla", "mytoken: asdkfjsde1341=="], - "HIDEME", + ["HIDEME"], ), "sensitive data tagged": ( ["bla bla", "mytoken: asdkfjsde1341== # HIDEME"], ["bla bla"], - "HIDEME", + ["HIDEME"], ), "sensitive data tagged with custom word": ( - ["bla bla", "mytoken: asdkfjsde1341== # DELETETHISLINE", "third line"], - ["bla bla", "third line"], - "DELETETHISLINE", + ["bla bla", "mytoken: asdkfjsde1341== # DELETETHISLINE", "third line # NOTANYMORE"], + ["bla bla", "third line # NOTANYMORE"], + ["DELETETHISLINE", "NOTANYMORE"], ), } @pytest.mark.parametrize( - "input, expectation, tag", + "input, expectation, tags", SENSITIVE_DATA_SCENARIOS.values(), ids=SENSITIVE_DATA_SCENARIOS.keys(), ) -def test_hide_sensitive_data(input, expectation, tag): +def test_hide_sensitive_data(input, expectation, tags): yaml_data = yaml_dump(input) - yaml_result = hide_sensitive_data(yaml_data, tag) + yaml_result = hide_sensitive_data(yaml_data, *tags) result = lava_yaml.load(yaml_result) assert result == expectation