ci/lava: Tweak timeouts

LAVA actions follow a hierarchical structure, where most subactions have
their timeouts overridden if the parent action supports a retry
mechanism, such as the `depthcharge-retry` action.

The timeout is calculated as: [1]

```
parent action timeout / failure_retry value
```

To adjust a subaction's timeout, we need to modify the nearest parent
action.

[1]
https://gitlab.collabora.com/lava/lava/-/blob/collabora/production/lava_dispatcher/action.py#L149

Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/33906>

.gitlab-ci/lava/utils/constants.py

@@ -1,7 +1,10 @@
 from os import getenv
 
 # How many attempts should be made when a timeout happen during LAVA device boot.
-NUMBER_OF_ATTEMPTS_LAVA_BOOT = int(getenv("LAVA_NUMBER_OF_ATTEMPTS_LAVA_BOOT", 3))
+NUMBER_OF_ATTEMPTS_LAVA_BOOT = int(getenv("LAVA_NUMBER_OF_ATTEMPTS_LAVA_BOOT", 2))
+# The deploy involves downloading and decompressing the kernel, modules, dtb and the overlays.
+# We should retry, to overcome network issues.
+NUMBER_OF_ATTEMPTS_LAVA_DEPLOY = int(getenv("LAVA_NUMBER_OF_ATTEMPTS_LAVA_DEPLOY", 2))
 
 
 # Supports any integers in [0, 100].

.gitlab-ci/lava/utils/lava_job_definition.py

@@ -26,7 +26,12 @@ from lava.utils.uart_job_definition import (
 if TYPE_CHECKING:
     from lava.lava_job_submitter import LAVAJobSubmitter
 
-from .constants import FORCE_UART, JOB_PRIORITY
+from .constants import (
+    FORCE_UART,
+    JOB_PRIORITY,
+    NUMBER_OF_ATTEMPTS_LAVA_BOOT,
+    NUMBER_OF_ATTEMPTS_LAVA_DEPLOY,
+)
 
 
 class LAVAJobDefinition:
@@ -155,8 +160,10 @@ class LAVAJobDefinition:
                 "job": {"minutes": self.job_submitter.job_timeout_min},
                 "actions": {
                     "depthcharge-retry": {
-                        # Could take between 1 and 1.5 min in slower boots
-                        "minutes": 4
+                        # Setting higher values here, to affect the subactions, namely
+                        # `bootloader-commands` and `login-action`
+                        # So this value can be higher than `depthcharge-action` timeout.
+                        "minutes": 3 * NUMBER_OF_ATTEMPTS_LAVA_DEPLOY
                     },
                     "depthcharge-action": {
                         # This timeout englobes the entire depthcharge timing,
@@ -169,9 +176,10 @@ class LAVAJobDefinition:
                         # The LAVA action that wraps it is `uboot-commands`, but we can't set a
                         # timeout for it directly, it is overridden by one third of `uboot-action`
                         # timeout.
-                        # So actually, this timeout is here to enforce that `uboot-commands`
-                        # timeout to be 100 seconds (300 sec / 3), which is more than enough.
-                        "minutes": 5
+                        # So actually, this timeout is here to enforce that `uboot-action`
+                        # timeout to be 100 seconds (uboot-action timeout /
+                        # NUMBER_OF_ATTEMPTS_LAVA_BOOT), which is more than enough.
+                        "seconds": 100 * NUMBER_OF_ATTEMPTS_LAVA_BOOT
                     },
                 },
             },

.gitlab-ci/tests/data/FASTBOOT_force_uart=False_job_definition.yaml

@@ -11,13 +11,11 @@ timeouts:
     minutes: 10
   actions:
     depthcharge-retry:
-      minutes: 4
-    depthcharge-start:
-      minutes: 1
+      minutes: 6
     depthcharge-action:
-      minutes: 15
-    uboot-action:
       minutes: 5
+    uboot-action:
+      seconds: 200
 actions:
 - deploy:
     timeout:
@@ -58,13 +56,13 @@ actions:
       boot:
         url: downloads://boot.img
     namespace: dut
-    failure_retry: 3
+    failure_retry: 2
 - boot:
     timeout:
       minutes: 2
     docker:
       image: registry.gitlab.collabora.com/lava/health-check-docker
-    failure_retry: 3
+    failure_retry: 2
     method: fastboot
     prompts:
     - 'lava-shell:'

.gitlab-ci/tests/data/FASTBOOT_force_uart=True_job_definition.yaml

@@ -11,13 +11,11 @@ timeouts:
     minutes: 10
   actions:
     depthcharge-retry:
-      minutes: 4
-    depthcharge-start:
-      minutes: 1
+      minutes: 6
     depthcharge-action:
-      minutes: 15
-    uboot-action:
       minutes: 5
+    uboot-action:
+      seconds: 200
 actions:
 - deploy:
     timeout:
@@ -60,7 +58,7 @@ actions:
       minutes: 2
     docker:
       image: registry.gitlab.collabora.com/lava/health-check-docker
-    failure_retry: 3
+    failure_retry: 2
     method: fastboot
     prompts:
     - 'lava-shell:'

.gitlab-ci/tests/data/UBOOT_force_uart=False_job_definition.yaml

@@ -11,13 +11,11 @@ timeouts:
     minutes: 10
   actions:
     depthcharge-retry:
-      minutes: 4
-    depthcharge-start:
-      minutes: 1
+      minutes: 6
     depthcharge-action:
-      minutes: 15
-    uboot-action:
       minutes: 5
+    uboot-action:
+      seconds: 200
 actions:
 - deploy:
     timeout:
@@ -34,9 +32,9 @@ actions:
     dtb:
       url: None/my_dtb_filename.dtb
     namespace: dut
-    failure_retry: 3
+    failure_retry: 2
 - boot:
-    failure_retry: 3
+    failure_retry: 2
     method: u-boot
     prompts:
     - 'lava-shell:'

.gitlab-ci/tests/data/UBOOT_force_uart=True_job_definition.yaml

@@ -11,13 +11,11 @@ timeouts:
     minutes: 10
   actions:
     depthcharge-retry:
-      minutes: 4
-    depthcharge-start:
-      minutes: 1
+      minutes: 6
     depthcharge-action:
-      minutes: 15
-    uboot-action:
       minutes: 5
+    uboot-action:
+      seconds: 200
 actions:
 - deploy:
     timeout:
@@ -34,7 +32,7 @@ actions:
     dtb:
       url: None/my_dtb_filename.dtb
 - boot:
-    failure_retry: 3
+    failure_retry: 2
     method: u-boot
     prompts:
     - 'lava-shell:'