From 422e65557d76c4a87431f4dd3765190067c0ec39 Mon Sep 17 00:00:00 2001 From: Guilherme Gallo Date: Wed, 5 Mar 2025 17:19:50 -0300 Subject: [PATCH] ci/lava: Tweak timeouts LAVA actions follow a hierarchical structure, where most subactions have their timeouts overridden if the parent action supports a retry mechanism, such as the `depthcharge-retry` action. The timeout is calculated as: [1] ``` parent action timeout / failure_retry value ``` To adjust a subaction's timeout, we need to modify the nearest parent action. [1] https://gitlab.collabora.com/lava/lava/-/blob/collabora/production/lava_dispatcher/action.py#L149 Signed-off-by: Guilherme Gallo Part-of: --- .gitlab-ci/lava/utils/constants.py | 5 ++++- .gitlab-ci/lava/utils/lava_job_definition.py | 20 +++++++++++++------ ...TBOOT_force_uart=False_job_definition.yaml | 12 +++++------ ...STBOOT_force_uart=True_job_definition.yaml | 10 ++++------ ...UBOOT_force_uart=False_job_definition.yaml | 12 +++++------ .../UBOOT_force_uart=True_job_definition.yaml | 10 ++++------ 6 files changed, 36 insertions(+), 33 deletions(-) diff --git a/.gitlab-ci/lava/utils/constants.py b/.gitlab-ci/lava/utils/constants.py index 82f0b66fc7e..23ddfd27a49 100644 --- a/.gitlab-ci/lava/utils/constants.py +++ b/.gitlab-ci/lava/utils/constants.py @@ -1,7 +1,10 @@ from os import getenv # How many attempts should be made when a timeout happen during LAVA device boot. -NUMBER_OF_ATTEMPTS_LAVA_BOOT = int(getenv("LAVA_NUMBER_OF_ATTEMPTS_LAVA_BOOT", 3)) +NUMBER_OF_ATTEMPTS_LAVA_BOOT = int(getenv("LAVA_NUMBER_OF_ATTEMPTS_LAVA_BOOT", 2)) +# The deploy involves downloading and decompressing the kernel, modules, dtb and the overlays. +# We should retry, to overcome network issues. +NUMBER_OF_ATTEMPTS_LAVA_DEPLOY = int(getenv("LAVA_NUMBER_OF_ATTEMPTS_LAVA_DEPLOY", 2)) # Supports any integers in [0, 100]. diff --git a/.gitlab-ci/lava/utils/lava_job_definition.py b/.gitlab-ci/lava/utils/lava_job_definition.py index c8d61d22423..d67b59d0ed3 100644 --- a/.gitlab-ci/lava/utils/lava_job_definition.py +++ b/.gitlab-ci/lava/utils/lava_job_definition.py @@ -26,7 +26,12 @@ from lava.utils.uart_job_definition import ( if TYPE_CHECKING: from lava.lava_job_submitter import LAVAJobSubmitter -from .constants import FORCE_UART, JOB_PRIORITY +from .constants import ( + FORCE_UART, + JOB_PRIORITY, + NUMBER_OF_ATTEMPTS_LAVA_BOOT, + NUMBER_OF_ATTEMPTS_LAVA_DEPLOY, +) class LAVAJobDefinition: @@ -155,8 +160,10 @@ class LAVAJobDefinition: "job": {"minutes": self.job_submitter.job_timeout_min}, "actions": { "depthcharge-retry": { - # Could take between 1 and 1.5 min in slower boots - "minutes": 4 + # Setting higher values here, to affect the subactions, namely + # `bootloader-commands` and `login-action` + # So this value can be higher than `depthcharge-action` timeout. + "minutes": 3 * NUMBER_OF_ATTEMPTS_LAVA_DEPLOY }, "depthcharge-action": { # This timeout englobes the entire depthcharge timing, @@ -169,9 +176,10 @@ class LAVAJobDefinition: # The LAVA action that wraps it is `uboot-commands`, but we can't set a # timeout for it directly, it is overridden by one third of `uboot-action` # timeout. - # So actually, this timeout is here to enforce that `uboot-commands` - # timeout to be 100 seconds (300 sec / 3), which is more than enough. - "minutes": 5 + # So actually, this timeout is here to enforce that `uboot-action` + # timeout to be 100 seconds (uboot-action timeout / + # NUMBER_OF_ATTEMPTS_LAVA_BOOT), which is more than enough. + "seconds": 100 * NUMBER_OF_ATTEMPTS_LAVA_BOOT }, }, }, diff --git a/.gitlab-ci/tests/data/FASTBOOT_force_uart=False_job_definition.yaml b/.gitlab-ci/tests/data/FASTBOOT_force_uart=False_job_definition.yaml index 78042b575a6..5b732871739 100644 --- a/.gitlab-ci/tests/data/FASTBOOT_force_uart=False_job_definition.yaml +++ b/.gitlab-ci/tests/data/FASTBOOT_force_uart=False_job_definition.yaml @@ -11,13 +11,11 @@ timeouts: minutes: 10 actions: depthcharge-retry: - minutes: 4 - depthcharge-start: - minutes: 1 + minutes: 6 depthcharge-action: - minutes: 15 - uboot-action: minutes: 5 + uboot-action: + seconds: 200 actions: - deploy: timeout: @@ -58,13 +56,13 @@ actions: boot: url: downloads://boot.img namespace: dut - failure_retry: 3 + failure_retry: 2 - boot: timeout: minutes: 2 docker: image: registry.gitlab.collabora.com/lava/health-check-docker - failure_retry: 3 + failure_retry: 2 method: fastboot prompts: - 'lava-shell:' diff --git a/.gitlab-ci/tests/data/FASTBOOT_force_uart=True_job_definition.yaml b/.gitlab-ci/tests/data/FASTBOOT_force_uart=True_job_definition.yaml index e9a36be899c..f517222eaf5 100644 --- a/.gitlab-ci/tests/data/FASTBOOT_force_uart=True_job_definition.yaml +++ b/.gitlab-ci/tests/data/FASTBOOT_force_uart=True_job_definition.yaml @@ -11,13 +11,11 @@ timeouts: minutes: 10 actions: depthcharge-retry: - minutes: 4 - depthcharge-start: - minutes: 1 + minutes: 6 depthcharge-action: - minutes: 15 - uboot-action: minutes: 5 + uboot-action: + seconds: 200 actions: - deploy: timeout: @@ -60,7 +58,7 @@ actions: minutes: 2 docker: image: registry.gitlab.collabora.com/lava/health-check-docker - failure_retry: 3 + failure_retry: 2 method: fastboot prompts: - 'lava-shell:' diff --git a/.gitlab-ci/tests/data/UBOOT_force_uart=False_job_definition.yaml b/.gitlab-ci/tests/data/UBOOT_force_uart=False_job_definition.yaml index e0246ea5586..2881ae8830a 100644 --- a/.gitlab-ci/tests/data/UBOOT_force_uart=False_job_definition.yaml +++ b/.gitlab-ci/tests/data/UBOOT_force_uart=False_job_definition.yaml @@ -11,13 +11,11 @@ timeouts: minutes: 10 actions: depthcharge-retry: - minutes: 4 - depthcharge-start: - minutes: 1 + minutes: 6 depthcharge-action: - minutes: 15 - uboot-action: minutes: 5 + uboot-action: + seconds: 200 actions: - deploy: timeout: @@ -34,9 +32,9 @@ actions: dtb: url: None/my_dtb_filename.dtb namespace: dut - failure_retry: 3 + failure_retry: 2 - boot: - failure_retry: 3 + failure_retry: 2 method: u-boot prompts: - 'lava-shell:' diff --git a/.gitlab-ci/tests/data/UBOOT_force_uart=True_job_definition.yaml b/.gitlab-ci/tests/data/UBOOT_force_uart=True_job_definition.yaml index ea52a6d3db9..67ae05b1cfb 100644 --- a/.gitlab-ci/tests/data/UBOOT_force_uart=True_job_definition.yaml +++ b/.gitlab-ci/tests/data/UBOOT_force_uart=True_job_definition.yaml @@ -11,13 +11,11 @@ timeouts: minutes: 10 actions: depthcharge-retry: - minutes: 4 - depthcharge-start: - minutes: 1 + minutes: 6 depthcharge-action: - minutes: 15 - uboot-action: minutes: 5 + uboot-action: + seconds: 200 actions: - deploy: timeout: @@ -34,7 +32,7 @@ actions: dtb: url: None/my_dtb_filename.dtb - boot: - failure_retry: 3 + failure_retry: 2 method: u-boot prompts: - 'lava-shell:'