From a5a5a50ae82b98fee0caa82a280e67d775be1c5d Mon Sep 17 00:00:00 2001 From: Daniel Stone Date: Tue, 15 Oct 2024 16:02:02 +0100 Subject: [PATCH] ci/angle: Update ANGLE, reduce build times ANGLE currently pulls absolutely loads of stuff that we don't need. Fix it up so we don't need to do that anymore, so it's much faster to build. Signed-off-by: Daniel Stone Part-of: --- .gitlab-ci/container/build-angle.sh | 85 ++++++++--- ...e_deps_Make-more-sources-conditional.patch | 144 ++++++++++++++++++ .gitlab-ci/image-tags.yml | 2 +- src/amd/ci/angle-radv-stoney-aco-fails.txt | 4 +- src/intel/ci/angle-anv-adl-fails.txt | 4 +- src/intel/ci/angle-anv-jsl-fails.txt | 4 +- src/intel/ci/angle-anv-tgl-fails.txt | 4 +- 7 files changed, 218 insertions(+), 29 deletions(-) create mode 100644 .gitlab-ci/container/patches/build-angle_deps_Make-more-sources-conditional.patch diff --git a/.gitlab-ci/container/build-angle.sh b/.gitlab-ci/container/build-angle.sh index 281358b9c0f..bdf11fa0781 100644 --- a/.gitlab-ci/container/build-angle.sh +++ b/.gitlab-ci/container/build-angle.sh @@ -6,40 +6,76 @@ set -uex -ANGLE_REV="1409a05a81e3ccb279142433a2b987bc330f555b" +ANGLE_REV="76025caa1a059f464a2b0e8f879dbd4746f092b9" +SCRIPTS_DIR="$(realpath "$(dirname "$0")/..")" +ANGLE_PATCH_DIR="${SCRIPTS_DIR}/container/patches" # DEPOT tools -git clone --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git -PWD=$(pwd) -export PATH=$PWD/depot_tools:$PATH +git clone --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git /depot-tools +export PATH=/depot-tools:$PATH export DEPOT_TOOLS_UPDATE=0 mkdir /angle-build +mkdir /angle pushd /angle-build git init git remote add origin https://chromium.googlesource.com/angle/angle.git git fetch --depth 1 origin "$ANGLE_REV" git checkout FETCH_HEAD -# source preparation -python3 scripts/bootstrap.py -mkdir -p build/config -gclient sync +angle_patch_files=( + build-angle_deps_Make-more-sources-conditional.patch +) +for patch in "${angle_patch_files[@]}"; do + echo "Apply patch to ANGLE from ${patch}" + GIT_COMMITTER_DATE=$(date -d@0) git am < "${ANGLE_PATCH_DIR}/${patch}" +done -sed -i "/catapult/d" testing/BUILD.gn +{ + echo "ANGLE base version $ANGLE_REV" + echo "The following local patches are applied on top:" + git log --reverse --oneline $ANGLE_REV.. --format='- %s' +} > /angle/version + +# source preparation +gclient config --name REPLACE-WITH-A-DOT --unmanaged \ + --custom-var='angle_enable_cl=False' \ + --custom-var='angle_enable_cl_testing=False' \ + --custom-var='angle_enable_vulkan_validation_layers=False' \ + --custom-var='angle_enable_wgpu=False' \ + --custom-var='build_allow_regenerate=False' \ + --custom-var='build_angle_deqp_tests=False' \ + --custom-var='build_angle_perftests=False' \ + --custom-var='build_with_catapult=False' \ + --custom-var='build_with_swiftshader=False' \ + https://chromium.googlesource.com/angle/angle.git +sed -e 's/REPLACE-WITH-A-DOT/./;' -i .gclient +gclient sync -j"${FDO_CI_CONCURRENT:-4}" mkdir -p out/Release echo ' -is_debug = false -angle_enable_swiftshader = false -angle_enable_null = false -angle_enable_gl = false -angle_enable_vulkan = true -angle_has_histograms = false -build_angle_trace_perf_tests = false -build_angle_deqp_tests = false -angle_use_custom_libvulkan = false +angle_build_all=false +angle_build_tests=false +angle_enable_cl=false +angle_enable_cl_testing=false +angle_enable_gl=false +angle_enable_gl_desktop_backend=false +angle_enable_null=false +angle_enable_swiftshader=false +angle_enable_trace=false +angle_enable_wgpu=false +angle_enable_vulkan=true +angle_enable_vulkan_api_dump_layer=false +angle_enable_vulkan_validation_layers=false +angle_has_frame_capture=false +angle_has_histograms=false +angle_use_custom_libvulkan=false +angle_egl_extension="so.1" +angle_glesv2_extension="so.2" +build_angle_deqp_tests=false dcheck_always_on=true +enable_expensive_dchecks=false +is_debug=false ' > out/Release/args.gn if [[ "$DEBIAN_ARCH" = "arm64" ]]; then @@ -49,14 +85,15 @@ fi gn gen out/Release # depot_tools overrides ninja with a version that doesn't work. We want # ninja with FDO_CI_CONCURRENT anyway. -/usr/local/bin/ninja -C out/Release/ +/usr/local/bin/ninja -C out/Release/ libEGL libGLESv2 -mkdir /angle -cp out/Release/lib*GL*.so /angle/ -ln -s libEGL.so /angle/libEGL.so.1 -ln -s libGLESv2.so /angle/libGLESv2.so.2 +rm -f out/Release/libvulkan.so* out/Release/*.so.TOC +cp out/Release/lib*.so* /angle/ +ln -s libEGL.so.1 /angle/libEGL.so +ln -s libGLESv2.so.2 /angle/libGLESv2.so rm -rf out popd -rm -rf ./depot_tools +rm -rf /depot-tools +rm -rf /angle-build diff --git a/.gitlab-ci/container/patches/build-angle_deps_Make-more-sources-conditional.patch b/.gitlab-ci/container/patches/build-angle_deps_Make-more-sources-conditional.patch new file mode 100644 index 00000000000..50a5532c4f7 --- /dev/null +++ b/.gitlab-ci/container/patches/build-angle_deps_Make-more-sources-conditional.patch @@ -0,0 +1,144 @@ +From 2f4a38ecfde470abcd5d3c0ae7337bf780343469 Mon Sep 17 00:00:00 2001 +From: Daniel Stone +Date: Tue, 15 Oct 2024 16:02:26 +0100 +Subject: [PATCH] deps: Make more sources conditional + +Fetching all the dependent sources - including at least one copy of LLVM +- can take a surprising amount of time. Mesa needs to build ANGLE as +part of CI, and the cost of downloading all the sources all of the time +is not OK for the number of dependencies we don't need during the build. +--- + DEPS | 33 +++++++++++++++++++++++---------- + 1 file changed, 23 insertions(+), 10 deletions(-) + +Submitted upstream at: +https://chromium-review.googlesource.com/c/angle/angle/+/5937820 + +diff --git a/DEPS b/DEPS +index 61263fb7af..0cff8c3126 100644 +--- a/DEPS ++++ b/DEPS +@@ -17,6 +17,17 @@ gclient_gn_args = [ + ] + + vars = { ++ 'angle_enable_cl': True, ++ 'angle_enable_cl_testing': False, ++ 'angle_enable_vulkan': True, ++ 'angle_enable_vulkan_validation_layers': True, ++ 'angle_enable_wgpu': True, ++ 'build_angle_deqp_tests': True, ++ 'build_angle_perftests': True, ++ 'build_with_swiftshader': True, ++ 'use_custom_libcxx': True, ++ 'export_libcxxapi_from_executables': True, ++ + 'android_git': 'https://android.googlesource.com', + 'chromium_git': 'https://chromium.googlesource.com', + 'chrome_internal_git': 'https://chrome-internal.googlesource.com', +@@ -673,7 +684,7 @@ deps = { + + 'third_party/catapult': { + 'url': Var('chromium_git') + '/catapult.git' + '@' + Var('catapult_revision'), +- 'condition': 'not build_with_chromium', ++ 'condition': 'build_with_catapult and not build_with_chromium', + }, + + # Cherry is a dEQP/VK-GL-CTS management GUI written in Go. We use it for viewing test results. +@@ -689,7 +700,7 @@ deps = { + + 'third_party/clspv/src': { + 'url': Var('chromium_git') + '/external/github.com/google/clspv@a173c052455434a422bcfe5c12ffe44d574fd6e1', +- 'condition': 'not build_with_chromium', ++ 'condition': 'angle_enable_cl and angle_enable_vulkan and not build_with_chromium', + }, + + 'third_party/cpu_features/src': { +@@ -700,7 +711,7 @@ deps = { + + 'third_party/dawn': { + 'url': Var('dawn_git') + '/dawn.git' + '@' + Var('dawn_revision'), +- 'condition': 'not build_with_chromium' ++ 'condition': 'angle_enable_wgpu and not build_with_chromium' + }, + + 'third_party/depot_tools': { +@@ -745,6 +756,7 @@ deps = { + # glmark2 is a GPL3-licensed OpenGL ES 2.0 benchmark. We use it for testing. + 'third_party/glmark2/src': { + 'url': Var('chromium_git') + '/external/github.com/glmark2/glmark2@ca8de51fedb70bace5351c6b002eb952c747e889', ++ 'condition': 'build_angle_perftests', + }, + + 'third_party/googletest': { +@@ -777,7 +789,7 @@ deps = { + # libjpeg_turbo is used by glmark2. + 'third_party/libjpeg_turbo': { + 'url': Var('chromium_git') + '/chromium/deps/libjpeg_turbo.git@927aabfcd26897abb9776ecf2a6c38ea5bb52ab6', +- 'condition': 'not build_with_chromium', ++ 'condition': 'build_angle_perftests and not build_with_chromium', + }, + + 'third_party/libpng/src': { +@@ -787,7 +799,7 @@ deps = { + + 'third_party/llvm/src': { + 'url': Var('chromium_git') + '/external/github.com/llvm/llvm-project@d222fa4521531cc4ac14b8e157d231c108c003be', +- 'condition': 'not build_with_chromium', ++ 'condition': '(build_with_swiftshader or (angle_enable_cl and angle_enable_vulkan)) and not build_with_chromium', + }, + + 'third_party/jdk': { +@@ -824,12 +836,12 @@ deps = { + + 'third_party/libc++/src': { + 'url': Var('chromium_git') + '/external/github.com/llvm/llvm-project/libcxx.git@6a68fd412b9aecd515a20a7cf84d11b598bfaf96', +- 'condition': 'not build_with_chromium', ++ 'condition': 'use_custom_libcxx and not build_with_chromium', + }, + + 'third_party/libc++abi/src': { + 'url': Var('chromium_git') + '/external/github.com/llvm/llvm-project/libcxxabi.git@9a1d90c3b412d5ebeb97a6e33d98e1d0dd923221', +- 'condition': 'not build_with_chromium', ++ 'condition': 'export_libcxxapi_from_executables and not build_with_chromium', + }, + + 'third_party/libunwind/src': { +@@ -872,7 +884,7 @@ deps = { + + 'third_party/OpenCL-CTS/src': { + 'url': Var('chromium_git') + '/external/github.com/KhronosGroup/OpenCL-CTS@e0a31a03fc8f816d59fd8b3051ac6a61d3fa50c6', +- 'condition': 'not build_with_chromium', ++ 'condition': 'angle_enable_cl_testing and not build_with_chromium', + }, + + 'third_party/OpenCL-Docs/src': { +@@ -968,7 +980,7 @@ deps = { + + 'third_party/SwiftShader': { + 'url': Var('swiftshader_git') + '/SwiftShader@7a9a492a38b7c701f7c96a15a76046aed8f8c0c3', +- 'condition': 'not build_with_chromium', ++ 'condition': 'build_with_swiftshader and not build_with_chromium', + }, + + 'third_party/turbine/cipd': { +@@ -984,6 +996,7 @@ deps = { + + 'third_party/VK-GL-CTS/src': { + 'url': Var('chromium_git') + '/external/github.com/KhronosGroup/VK-GL-CTS' + '@' + Var('vk_gl_cts_revision'), ++ 'condition': 'build_angle_deqp_tests', + }, + + 'third_party/vulkan-deps': { +@@ -1038,7 +1051,7 @@ deps = { + + 'third_party/vulkan-validation-layers/src': { + 'url': '{chromium_git}/external/github.com/KhronosGroup/Vulkan-ValidationLayers@b63e9bd51fbd7bf8fea161a4f7c06994abc24b75', +- 'condition': 'not build_with_chromium', ++ 'condition': 'angle_enable_vulkan_validation_layers and not build_with_chromium', + }, + + 'third_party/vulkan_memory_allocator': { +-- +2.46.2 + diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml index 9f6b13a50c3..8bad6400813 100644 --- a/.gitlab-ci/image-tags.yml +++ b/.gitlab-ci/image-tags.yml @@ -30,7 +30,7 @@ variables: DEBIAN_TEST_ANDROID_TAG: "20241016-zz-top" DEBIAN_TEST_GL_TAG: "20241017-piglit-79" DEBIAN_TEST_VK_TAG: "20241017-piglit-79" - KERNEL_ROOTFS_TAG: "20241017-piglit-79" + KERNEL_ROOTFS_TAG: "20241018-oblique" DEBIAN_PYUTILS_IMAGE: "debian/x86_64_pyutils" DEBIAN_PYUTILS_TAG: "20241002-pyutils" diff --git a/src/amd/ci/angle-radv-stoney-aco-fails.txt b/src/amd/ci/angle-radv-stoney-aco-fails.txt index 36bfe822db7..84672fa693a 100644 --- a/src/amd/ci/angle-radv-stoney-aco-fails.txt +++ b/src/amd/ci/angle-radv-stoney-aco-fails.txt @@ -63,7 +63,6 @@ dEQP-GLES2.functional.fbo.render.color_clear.rbo_bgra_depth_component16_stencil_ dEQP-GLES2.functional.fbo.render.stencil_clear.rbo_bgra_stencil_index8,Fail dEQP-GLES2.functional.fbo.render.recreate_stencilbuffer.rebind_rbo_bgra_stencil_index8,Fail dEQP-GLES2.functional.fbo.render.color_clear.rbo_bgra_depth_component16,Fail -KHR-GLES3.framebuffer_blit.framebuffer_blit_functionality_multisampled_to_singlesampled_blit,Fail dEQP-GLES2.functional.fbo.render.recreate_stencilbuffer.no_rebind_rbo_bgra_depth_component16_stencil_index8,Fail dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.rebind_rbo_bgra,Fail dEQP-GLES2.functional.fbo.render.shared_depthbuffer.rbo_bgra_depth_component16,Fail @@ -90,3 +89,6 @@ dEQP-GLES2.functional.fbo.render.stencil_clear.rbo_bgra_depth_component16_stenci dEQP-GLES2.functional.fbo.render.color.blend_rbo_bgra,Fail dEQP-GLES2.functional.fbo.render.color_clear.rbo_bgra,Fail dEQP-GLES2.functional.fbo.render.recreate_depthbuffer.no_rebind_rbo_bgra_depth_component16,Fail + +# new with ANGLE 2024-10-17 +KHR-GLES31.core.texture_buffer.texture_buffer_max_size,Fail diff --git a/src/intel/ci/angle-anv-adl-fails.txt b/src/intel/ci/angle-anv-adl-fails.txt index f9d2770b13a..c88599e64f8 100644 --- a/src/intel/ci/angle-anv-adl-fails.txt +++ b/src/intel/ci/angle-anv-adl-fails.txt @@ -308,7 +308,6 @@ dEQP-GLES2.functional.fbo.render.stencil_clear.rbo_bgra_stencil_index8,Fail dEQP-GLES2.functional.fbo.render.shared_colorbuffer.rbo_bgra_depth_component16,Fail dEQP-GLES2.functional.fbo.render.stencil.rbo_bgra_stencil_index8,Fail dEQP-GLES2.functional.fbo.render.stencil_clear.rbo_bgra_depth_component16_stencil_index8,Fail -KHR-GLES3.framebuffer_blit.framebuffer_blit_functionality_multisampled_to_singlesampled_blit,Fail KHR-GLES3.clip_distance.coverage,Fail dEQP-GLES2.functional.fbo.render.stencil.npot_rbo_bgra_depth_component16_stencil_index8,Fail dEQP-GLES2.functional.fbo.render.recreate_depthbuffer.rebind_rbo_bgra_depth_component16_stencil_index8,Fail @@ -337,3 +336,6 @@ dEQP-GLES2.functional.fbo.render.color.blend_npot_rbo_bgra,Fail dEQP-GLES3.functional.fbo.completeness.renderable.renderbuffer.color0.bgra,Fail dEQP-GLES3.functional.fbo.completeness.renderable.renderbuffer.depth.bgra,Fail dEQP-GLES3.functional.fbo.completeness.renderable.renderbuffer.stencil.bgra,Fail + +# new failures with ANGLE 2024-10-17 +KHR-GLES31.core.shader_image_load_store.basic-allTargets-loadStoreVS,Fail diff --git a/src/intel/ci/angle-anv-jsl-fails.txt b/src/intel/ci/angle-anv-jsl-fails.txt index daa400a378e..8a1a23e7d87 100644 --- a/src/intel/ci/angle-anv-jsl-fails.txt +++ b/src/intel/ci/angle-anv-jsl-fails.txt @@ -34,7 +34,6 @@ dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.no_rebind_rbo_bgra,Fail dEQP-GLES2.functional.fbo.render.recreate_depthbuffer.no_rebind_rbo_bgra_depth_component16,Fail dEQP-GLES2.functional.fbo.render.depth.rbo_bgra_depth_component16,Fail dEQP-GLES2.functional.fbo.render.shared_colorbuffer.rbo_bgra_stencil_index8,Fail -KHR-GLES3.framebuffer_blit.framebuffer_blit_functionality_multisampled_to_singlesampled_blit,Fail dEQP-GLES2.functional.fbo.completeness.renderable.renderbuffer.stencil.bgra,Fail dEQP-GLES2.functional.fbo.render.color.blend_rbo_bgra_depth_component16,Fail dEQP-GLES2.functional.fbo.render.color_clear.rbo_bgra_depth_component16,Fail @@ -64,3 +63,6 @@ dEQP-GLES2.functional.fbo.render.stencil.npot_rbo_bgra_stencil_index8,Fail dEQP-GLES2.functional.fbo.render.stencil.rbo_bgra_depth_component16_stencil_index8,Fail dEQP-GLES2.functional.fbo.render.stencil_clear.rbo_bgra_depth_component16_stencil_index8,Fail dEQP-GLES3.functional.fbo.completeness.renderable.renderbuffer.color0.bgra,Fail + +# new with ANGLE 2024-10-17 +KHR-GLES31.core.shader_image_load_store.basic-allTargets-loadStoreVS,Fail diff --git a/src/intel/ci/angle-anv-tgl-fails.txt b/src/intel/ci/angle-anv-tgl-fails.txt index fce5c9e991a..e50e8a12981 100644 --- a/src/intel/ci/angle-anv-tgl-fails.txt +++ b/src/intel/ci/angle-anv-tgl-fails.txt @@ -22,7 +22,6 @@ dEQP-GLES2.functional.fbo.render.stencil_clear.rbo_bgra_stencil_index8,Fail dEQP-GLES2.functional.fbo.render.shared_colorbuffer.rbo_bgra_depth_component16,Fail dEQP-GLES2.functional.fbo.render.stencil.rbo_bgra_stencil_index8,Fail dEQP-GLES2.functional.fbo.render.stencil_clear.rbo_bgra_depth_component16_stencil_index8,Fail -KHR-GLES3.framebuffer_blit.framebuffer_blit_functionality_multisampled_to_singlesampled_blit,Fail KHR-GLES3.clip_distance.coverage,Fail dEQP-GLES2.functional.fbo.render.stencil.npot_rbo_bgra_depth_component16_stencil_index8,Fail dEQP-GLES2.functional.fbo.render.recreate_depthbuffer.rebind_rbo_bgra_depth_component16_stencil_index8,Fail @@ -51,3 +50,6 @@ dEQP-GLES2.functional.fbo.render.color.blend_npot_rbo_bgra,Fail dEQP-GLES3.functional.fbo.completeness.renderable.renderbuffer.color0.bgra,Fail dEQP-GLES3.functional.fbo.completeness.renderable.renderbuffer.depth.bgra,Fail dEQP-GLES3.functional.fbo.completeness.renderable.renderbuffer.stencil.bgra,Fail + +# new with ANGLE 2024-10-17 +KHR-GLES31.core.shader_image_load_store.basic-allTargets-loadStoreVS,Fail