ci: Remove EXTERNAL_KERNEL_TAG variable

The EXTERNAL_KERNEL_TAG variable is no longer needed.
For LAVA and bare-metal, we can override the KERNEL_TAG variable to fetch
both the kernel image and modules from a different tag than the default
mainline gfx-ci/linux kernel.

For LAVA, this also avoids the issue where jobs using EXTERNAL_KERNEL_TAG
would still have mainline kernel modules downloaded by the LAVA overlay.

Signed-off-by: Valentine Burley <valentine.burley@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/34873>

.gitlab-ci.yml

@@ -125,6 +125,7 @@ variables:
   CI_TRON_JOB_PRIORITY_TAG: ci-tron:priority:low
   JOB_PRIORITY: 50
   DATA_STORAGE_PATH: data_storage
+  KERNEL_IMAGE_BASE: "https://$S3_HOST/$S3_KERNEL_BUCKET/$KERNEL_REPO/$KERNEL_TAG"
 
   CI_TRON_JOB_TEMPLATE_PROJECT: &ci-tron-template-project gfx-ci/ci-tron
   CI_TRON_JOB_TEMPLATE_COMMIT: &ci-tron-template-commit e4ab9c5ecc941c7893d991cf723cde877b5dca8d
@@ -137,10 +138,6 @@ default:
     S3_JWT:
       aud: https://s3.freedesktop.org
   before_script:
-    - |
-      if [ -z "${KERNEL_IMAGE_BASE:-}" ]; then
-        export KERNEL_IMAGE_BASE="https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${EXTERNAL_KERNEL_TAG:-$KERNEL_TAG}"
-      fi
     - >
       export SCRIPTS_DIR=$(mktemp -d) &&
       curl -L -s --retry 4 -f --retry-all-errors --retry-delay 60 -O --output-dir "${SCRIPTS_DIR}" "${CI_PROJECT_URL}/-/raw/${CI_COMMIT_SHA}/.gitlab-ci/setup-test-env.sh" &&

.gitlab-ci/bare-metal/cros-servo.sh

@@ -89,14 +89,7 @@ mkdir -p /nfs/results
 rm -rf /tftp/*
 if echo "$BM_KERNEL" | grep -q http; then
   curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-      $BM_KERNEL -o /tftp/vmlinuz
+      "$BM_KERNEL" -o /tftp/vmlinuz
-elif [ -n "${EXTERNAL_KERNEL_TAG}" ]; then
-  curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-    "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${BM_KERNEL}" -o /tftp/vmlinuz
-  curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-    "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/modules.tar.zst" -o modules.tar.zst
-  tar --keep-directory-symlink --zstd -xf modules.tar.zst -C "/nfs/"
-  rm modules.tar.zst &
 else
   cp /baremetal-files/"$BM_KERNEL" /tftp/vmlinuz
 fi

.gitlab-ci/bare-metal/fastboot.sh

@@ -102,22 +102,6 @@ if echo "$BM_KERNEL $BM_DTB" | grep -q http; then
       "$BM_DTB" -o dtb
 
   cat kernel dtb > Image.gz-dtb
-
-elif [ -n "${EXTERNAL_KERNEL_TAG}" ]; then
-  curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-      "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${BM_KERNEL}" -o kernel
-  curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-      "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/modules.tar.zst" -o modules.tar.zst
-
-  if [ -n "$BM_DTB" ]; then
-    curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-	"${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${BM_DTB}.dtb" -o dtb
-  fi
-
-  cat kernel dtb > Image.gz-dtb || echo "No DTB available, using pure kernel."
-  rm kernel
-  tar --keep-directory-symlink --zstd -xf modules.tar.zst -C "$BM_ROOTFS/"
-  rm modules.tar.zst &
 else
   cat /baremetal-files/"$BM_KERNEL" /baremetal-files/"$BM_DTB".dtb > Image.gz-dtb
   cp /baremetal-files/"$BM_DTB".dtb dtb

.gitlab-ci/bare-metal/poe-powered.sh

@@ -103,29 +103,11 @@ if [ -f "${BM_BOOTFS}" ]; then
   BM_BOOTFS=/tmp/bootfs
 fi
 
-# If BM_KERNEL and BM_DTS is present
-if [ -n "${EXTERNAL_KERNEL_TAG}" ]; then
-  if [ -z "${BM_KERNEL}" ] || [ -z "${BM_DTB}" ]; then
-    echo "This machine cannot be tested with external kernel since BM_KERNEL or BM_DTB missing!"
-    exit 1
-  fi
-
-  curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-      "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${BM_KERNEL}" -o "${BM_KERNEL}"
-  curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-      "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${BM_DTB}.dtb" -o "${BM_DTB}.dtb"
-  curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-      "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/modules.tar.zst" -o modules.tar.zst
-fi
-
 date +'%F %T'
 
 # Install kernel modules (it could be either in /lib/modules or
 # /usr/lib/modules, but we want to install in the latter)
-if [ -n "${EXTERNAL_KERNEL_TAG}" ]; then
+if [ -n "${BM_BOOTFS}" ]; then
-  tar --keep-directory-symlink --zstd -xf modules.tar.zst -C /nfs/
-  rm modules.tar.zst &
-elif [ -n "${BM_BOOTFS}" ]; then
   [ -d $BM_BOOTFS/usr/lib/modules ] && rsync -a $BM_BOOTFS/usr/lib/modules/ /nfs/usr/lib/modules/
   [ -d $BM_BOOTFS/lib/modules ] && rsync -a $BM_BOOTFS/lib/modules/ /nfs/lib/modules/
 else
@@ -136,7 +118,7 @@ fi
 date +'%F %T'
 
 # Install kernel image + bootloader files
-if [ -n "${EXTERNAL_KERNEL_TAG}" ] || [ -z "$BM_BOOTFS" ]; then
+if [ -z "$BM_BOOTFS" ]; then
   mv "${BM_KERNEL}" "${BM_DTB}.dtb" /tftp/
 else  # BM_BOOTFS
   rsync -aL --delete $BM_BOOTFS/boot/ /tftp/

.gitlab-ci/crosvm-runner.sh

@@ -122,7 +122,7 @@ fi
 
 if [ ! -f "/kernel/${KERNEL_IMAGE_NAME:-bzImage}" ]; then
   mkdir -p /kernel
-  # shellcheck disable=SC2153 # KERNEL_IMAGE_BASE is set in the default before_script
+  # shellcheck disable=SC2153
   curl -L --retry 4 -f --retry-connrefused --retry-delay 30 \
     -o "/kernel/${KERNEL_IMAGE_NAME:-bzImage}" "${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH:-amd64}/${KERNEL_IMAGE_NAME:-bzImage}"
 fi

.gitlab-ci/lava/lava-submit.sh

@@ -73,7 +73,6 @@ PYTHONPATH=artifacts/ artifacts/lava/lava_job_submitter.py \
 	--pipeline-info "$CI_JOB_NAME: $CI_PIPELINE_URL on $CI_COMMIT_REF_NAME ${CI_NODE_INDEX}/${CI_NODE_TOTAL}" \
 	--rootfs-url "${ROOTFS_URL}" \
 	--kernel-url-prefix "${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}" \
-	--kernel-external "${EXTERNAL_KERNEL_TAG}" \
 	--first-stage-init artifacts/ci-common/init-stage1.sh \
 	--dtb-filename "${DTB}" \
 	--jwt-file "${S3_JWT_FILE}" \
@@ -100,7 +99,7 @@ PYTHONPATH=artifacts/ artifacts/lava/lava_job_submitter.py \
 		--path="/" \
 		--format=tar \
 	- append-overlay \
-		--name=extra-modules \
+		--name=kernel-modules \
 		--url="${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/modules.tar.zst" \
 		--compression=zstd \
 		--path="/" \

.gitlab-ci/lava/lava_job_submitter.py

@@ -407,7 +407,6 @@ class LAVAJobSubmitter(PathResolver):
     kernel_image_name: str = None
     kernel_image_type: str = ""
     kernel_url_prefix: str = None
-    kernel_external: str = None
     lava_tags: str | tuple[str, ...] = ()  # Comma-separated LAVA tags for the job
     mesa_job_name: str = "mesa_ci_job"
     pipeline_info: str = ""

.gitlab-ci/lava/utils/lava_job_definition.py

@@ -202,13 +202,6 @@ class LAVAJobDefinition:
                 f"{self.job_submitter.dtb_filename}.dtb"
             }
 
-    def attach_external_modules(self, deploy_field):
-        if self.job_submitter.kernel_external:
-            deploy_field["modules"] = {
-                "url": f"{self.job_submitter.kernel_url_prefix}/modules.tar.zst",
-                "compression": "zstd"
-            }
-
     def jwt_steps(self):
         """
         This function is responsible for setting up the SSH server in the DUT and to

.gitlab-ci/lava/utils/uart_job_definition.py

@@ -61,7 +61,6 @@ def fastboot_deploy_actions(
     # URLs to our kernel rootfs to boot from, both generated by the base
     # container build
     job_definition.attach_kernel_and_dtb(fastboot_deploy_prepare["images"])
-    job_definition.attach_external_modules(fastboot_deploy_nfs)
 
     return (fastboot_deploy_nfs, fastboot_deploy_prepare, fastboot_deploy)
 
@@ -78,7 +77,6 @@ def tftp_deploy_actions(job_definition: "LAVAJobDefinition", nfsrootfs) -> tuple
         "nfsrootfs": nfsrootfs,
     }
     job_definition.attach_kernel_and_dtb(tftp_deploy)
-    job_definition.attach_external_modules(tftp_deploy)
 
     return (tftp_deploy,)
 
@@ -96,7 +94,6 @@ def qemu_deploy_actions(job_definition: "LAVAJobDefinition", nfsrootfs) -> tuple
             "nfsrootfs": nfsrootfs,
         },
     }
-    job_definition.attach_external_modules(qemu_deploy)
 
     return (qemu_deploy,)
 

src/amd/ci/gitlab-ci-inc.yml

@@ -202,7 +202,7 @@
     RUNNER_TAG: mesa-ci-x86-64-lava-asus-CM1400CXA-dalboz
     # Force fixed 6.6 kernel, amdgpu doesn't revcover from GPU resets on 6.13
     # https://gitlab.freedesktop.org/drm/amd/-/issues/3861
-    EXTERNAL_KERNEL_TAG: "v6.6.21-mesa-f8ea"
+    KERNEL_TAG: "v6.6.21-mesa-f8ea"
 
 # Status: https://lava.collabora.dev/scheduler/device_type/lenovo-TPad-C13-Yoga-zork
 .lava-lenovo-TPad-C13-Yoga-zork:x86_64:
@@ -218,7 +218,7 @@
     RUNNER_TAG: mesa-ci-x86-64-lava-lenovo-TPad-C13-Yoga-zork
     # Force fixed 6.6 kernel, amdgpu doesn't revcover from GPU resets on 6.13
     # https://gitlab.freedesktop.org/drm/amd/-/issues/3861
-    EXTERNAL_KERNEL_TAG: "v6.6.21-mesa-f8ea"
+    KERNEL_TAG: "v6.6.21-mesa-f8ea"
 
 # Status: https://lava.collabora.dev/scheduler/device_type/hp-x360-14a-cb0001xx-zork
 .lava-hp-x360-14a-cb0001xx-zork:x86_64:
@@ -234,7 +234,7 @@
     RUNNER_TAG: mesa-ci-x86-64-lava-hp-x360-14a-cb0001xx-zork
     # Force fixed 6.6 kernel, amdgpu doesn't revcover from GPU resets on 6.13
     # https://gitlab.freedesktop.org/drm/amd/-/issues/3861
-    EXTERNAL_KERNEL_TAG: "v6.6.21-mesa-f8ea"
+    KERNEL_TAG: "v6.6.21-mesa-f8ea"
 
 
 ############### LAVA

src/broadcom/ci/gitlab-ci-inc.yml

@@ -29,8 +29,6 @@
 .vc4-rules:
   stage: broadcom
   rules:
-    - if: $EXTERNAL_KERNEL_TAG != null
-      when: never
     - !reference [.test, rules]
     - !reference [.igalia-farm-rules, rules]
     - !reference [.gl-rules, rules]
@@ -56,8 +54,6 @@
 .v3d-rules:
   stage: broadcom
   rules:
-    - if: $EXTERNAL_KERNEL_TAG != null
-      when: never
     - !reference [.test, rules]
     - !reference [.igalia-farm-rules, rules]
     - !reference [.gl-rules, rules]
@@ -97,8 +93,6 @@
 .v3dv-rules:
   stage: broadcom
   rules:
-    - if: $EXTERNAL_KERNEL_TAG != null
-      when: never
     - !reference [.test, rules]
     - !reference [.igalia-farm-rules, rules]
     - !reference [.vulkan-rules, rules]

src/gallium/drivers/llvmpipe/ci/gitlab-ci-inc.yml

@@ -3,8 +3,6 @@
   timeout: 12m
   rules:
     - !reference [.test, rules]
-    - if: $EXTERNAL_KERNEL_TAG != null
-      when: never
     - !reference [.gl-rules, rules]
     - changes: &llvmpipe_file_list
         - src/gallium/drivers/llvmpipe/*

src/gallium/drivers/softpipe/ci/gitlab-ci-inc.yml

@@ -2,8 +2,6 @@
   stage: software-renderer
   rules:
     - !reference [.test, rules]
-    - if: $EXTERNAL_KERNEL_TAG != null
-      when: never
     - !reference [.gl-rules, rules]
     - changes: &softpipe_file_list
       - src/gallium/drivers/softpipe/**/*

src/gallium/drivers/svga/ci/gitlab-ci.yml

@@ -39,5 +39,5 @@ vmware-vmx-piglit:x86_64:
     DEVICE_TYPE: x86
     BOOT_METHOD: ipxe
     KERNEL_REPO: "blu/linux"
-    EXTERNAL_KERNEL_TAG: "1106ba88b"
+    KERNEL_TAG: "1106ba88b"
     FDO_CI_CONCURRENT: 1

src/gallium/drivers/virgl/ci/gitlab-ci-inc.yml

@@ -2,8 +2,6 @@
   stage: layered-backends
   rules:
     - !reference [.test, rules]
-    - if: $EXTERNAL_KERNEL_TAG != null
-      when: never
     - !reference [.gl-rules, rules]
     - !reference [.llvmpipe-rules, rules]
     - changes: &virgl_file_list

src/gallium/frontends/lavapipe/ci/gitlab-ci-inc.yml

@@ -3,8 +3,6 @@
   timeout: 15m
   rules:
     - !reference [.test, rules]
-    - if: $EXTERNAL_KERNEL_TAG != null
-      when: never
     - !reference [.vulkan-rules, rules]
     - !reference [.gallium-core-rules, rules]
     - changes: &lavapipe_file_list

src/panfrost/ci/gitlab-ci-inc.yml

@@ -204,7 +204,7 @@
     KERNEL_IMAGE_TYPE: ""
     RUNNER_TAG: mesa-ci-x86-64-lava-mt8192-asurada-spherion-r0
     # Force fixed 6.6 kernel, we're not reliably getting a GPU on 6.13
-    EXTERNAL_KERNEL_TAG: "v6.6.21-mesa-f8ea"
+    KERNEL_TAG: "v6.6.21-mesa-f8ea"
 
 # Status: https://lava.collabora.dev/scheduler/device_type/mt8195-cherry-tomato-r2
 .lava-mt8195-cherry-tomato-r2:arm64: