From 4658a9a4798c97e39aa65a94751d75fc52e8030c Mon Sep 17 00:00:00 2001 From: Emma Anholt Date: Wed, 21 Jul 2021 13:12:29 -0700 Subject: [PATCH] ci: Uprev deqp-runner and use "suite" support to merge softpipe runs. The deqp-runner "suite" support lets us specify all the deqp variants at once and shard them across gitlab jobs, so we reduce docker overhead and get fewer jobs to review the output of. Acked-by: Alyssa Rosenzweig Reviewed-by: Tomeu Vizoso Part-of: --- .gitlab-ci.yml | 4 +- .gitlab-ci/container/build-deqp-runner.sh | 2 +- .gitlab-ci/deqp-runner.sh | 101 ++++++++++-------- .gitlab-ci/prepare-artifacts.sh | 1 + .../drivers/softpipe/ci/deqp-softpipe.toml | 51 +++++++++ src/gallium/drivers/softpipe/ci/gitlab-ci.yml | 44 ++------ 6 files changed, 121 insertions(+), 82 deletions(-) create mode 100644 src/gallium/drivers/softpipe/ci/deqp-softpipe.toml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b150c44dff2..b59c3e78a5f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -407,7 +407,7 @@ debian/android_build: debian/x86_test-base: extends: debian/x86_build-base variables: - MESA_IMAGE_TAG: &debian-x86_test-base "2021-07-02-bump-libdrm" + MESA_IMAGE_TAG: &debian-x86_test-base "2021-07-30-deqp-runner" .use-debian/x86_test-base: extends: @@ -480,7 +480,7 @@ fedora/x86_build: variables: GIT_STRATEGY: fetch KERNEL_URL: *kernel-rootfs-url - MESA_ROOTFS_TAG: &kernel-rootfs "2021-08-02-rust-bump" + MESA_ROOTFS_TAG: &kernel-rootfs "2021-08-03-deqp-runner" DISTRIBUTION_TAG: &distribution-tag-arm "${MESA_ROOTFS_TAG}--${MESA_ARTIFACTS_TAG}--${MESA_TEMPLATES_COMMIT}" script: - .gitlab-ci/container/lava_build.sh diff --git a/.gitlab-ci/container/build-deqp-runner.sh b/.gitlab-ci/container/build-deqp-runner.sh index 532e71cf5d4..3c12f433739 100644 --- a/.gitlab-ci/container/build-deqp-runner.sh +++ b/.gitlab-ci/container/build-deqp-runner.sh @@ -4,6 +4,6 @@ set -ex cargo install --locked deqp-runner \ -j ${FDO_CI_CONCURRENT:-4} \ - --version 0.7.2 \ + --version 0.8.2 \ --root /usr/local \ $EXTRA_CARGO_ARGS diff --git a/.gitlab-ci/deqp-runner.sh b/.gitlab-ci/deqp-runner.sh index e9c02f4a26a..cc4a539e2b3 100755 --- a/.gitlab-ci/deqp-runner.sh +++ b/.gitlab-ci/deqp-runner.sh @@ -47,47 +47,51 @@ HANG_DETECTION_CMD="" # Generate test case list file. if [ "$DEQP_VER" = "vk" ]; then - cp /deqp/mustpass/vk-$DEQP_VARIANT.txt /tmp/case-list.txt + MUSTPASS=/deqp/mustpass/vk-$DEQP_VARIANT.txt DEQP=/deqp/external/vulkancts/modules/vulkan/deqp-vk HANG_DETECTION_CMD="/parallel-deqp-runner/build/bin/hang-detection" elif [ "$DEQP_VER" = "gles2" -o "$DEQP_VER" = "gles3" -o "$DEQP_VER" = "gles31" -o "$DEQP_VER" = "egl" ]; then - cp /deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt /tmp/case-list.txt + MUSTPASS=/deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt DEQP=/deqp/modules/$DEQP_VER/deqp-$DEQP_VER SUITE=dEQP elif [ "$DEQP_VER" = "gles2-khr" -o "$DEQP_VER" = "gles3-khr" -o "$DEQP_VER" = "gles31-khr" -o "$DEQP_VER" = "gles32-khr" ]; then - cp /deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt /tmp/case-list.txt + MUSTPASS=/deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt DEQP=/deqp/external/openglcts/modules/glcts SUITE=dEQP else - cp /deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt /tmp/case-list.txt + MUSTPASS=/deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt DEQP=/deqp/external/openglcts/modules/glcts SUITE=KHR fi -# If the caselist is too long to run in a reasonable amount of time, let the job -# specify what fraction (1/n) of the caselist we should run. Note: N~M is a gnu -# sed extension to match every nth line (first line is #1). -if [ -n "$DEQP_FRACTION" ]; then - sed -ni 1~$DEQP_FRACTION"p" /tmp/case-list.txt -fi +if [ -z "$DEQP_SUITE" ]; then + cp $MUSTPASS /tmp/case-list.txt -# If the job is parallel at the gitab job level, take the corresponding fraction -# of the caselist. -if [ -n "$CI_NODE_INDEX" ]; then - sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt -fi + # If the caselist is too long to run in a reasonable amount of time, let the job + # specify what fraction (1/n) of the caselist we should run. Note: N~M is a gnu + # sed extension to match every nth line (first line is #1). + if [ -n "$DEQP_FRACTION" ]; then + sed -ni 1~$DEQP_FRACTION"p" /tmp/case-list.txt + fi -if [ -n "$DEQP_CASELIST_FILTER" ]; then - sed -ni "/$DEQP_CASELIST_FILTER/p" /tmp/case-list.txt -fi + # If the job is parallel at the gitab job level, take the corresponding fraction + # of the caselist. + if [ -n "$CI_NODE_INDEX" ]; then + sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt + fi -if [ -n "$DEQP_CASELIST_INV_FILTER" ]; then - sed -ni "/$DEQP_CASELIST_INV_FILTER/!p" /tmp/case-list.txt -fi + if [ -n "$DEQP_CASELIST_FILTER" ]; then + sed -ni "/$DEQP_CASELIST_FILTER/p" /tmp/case-list.txt + fi -if [ ! -s /tmp/case-list.txt ]; then - echo "Caselist generation failed" - exit 1 + if [ -n "$DEQP_CASELIST_INV_FILTER" ]; then + sed -ni "/$DEQP_CASELIST_INV_FILTER/!p" /tmp/case-list.txt + fi + + if [ ! -s /tmp/case-list.txt ]; then + echo "Caselist generation failed" + exit 1 + fi fi if [ -e "$INSTALL/deqp-$GPU_VERSION-fails.txt" ]; then @@ -116,25 +120,6 @@ if [ -n "$DEQP_NO_SAVE_RESULTS" ]; then SUMMARY_LIMIT="--summary-limit 0" fi -run_cts() { - deqp=$1 - caselist=$2 - output=$3 - deqp-runner \ - run \ - --deqp $deqp \ - --output $RESULTS \ - --caselist $caselist \ - --skips $INSTALL/deqp-all-skips.txt $DEQP_SKIPS \ - --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt \ - --testlog-to-xml /deqp/executor/testlog-to-xml \ - $JOB \ - $SUMMARY_LIMIT \ - $DEQP_RUNNER_OPTIONS \ - -- \ - $DEQP_OPTIONS -} - parse_renderer() { RENDERER=`grep -A1 TestCaseResult.\*info.renderer $RESULTS/deqp-info.qpa | grep '||g' | sed 's|||g'` VERSION=`grep -A1 TestCaseResult.\*info.version $RESULTS/deqp-info.qpa | grep '||g' | sed 's|||g'` @@ -214,7 +199,35 @@ FAILURES_CSV=$RESULTS/failures.csv export LD_PRELOAD=$TEST_LD_PRELOAD -run_cts $DEQP /tmp/case-list.txt $RESULTS_CSV +if [ -z "$DEQP_SUITE" ]; then + deqp-runner \ + run \ + --deqp $DEQP \ + --output $RESULTS \ + --caselist /tmp/case-list.txt \ + --skips $INSTALL/deqp-all-skips.txt $DEQP_SKIPS \ + --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt \ + --testlog-to-xml /deqp/executor/testlog-to-xml \ + $JOB \ + $SUMMARY_LIMIT \ + $DEQP_RUNNER_OPTIONS \ + -- \ + $DEQP_OPTIONS +else + deqp-runner \ + suite \ + --suite $INSTALL/deqp-$DEQP_SUITE.toml \ + --output $RESULTS \ + --skips $INSTALL/deqp-all-skips.txt $DEQP_SKIPS \ + --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt \ + --testlog-to-xml /deqp/executor/testlog-to-xml \ + --fraction-start $CI_NODE_INDEX \ + --fraction $CI_NODE_TOTAL \ + $JOB \ + $SUMMARY_LIMIT \ + $DEQP_RUNNER_OPTIONS +fi + DEQP_EXITCODE=$? export LD_PRELOAD= diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh index a9a47daedb0..393329effc4 100755 --- a/.gitlab-ci/prepare-artifacts.sh +++ b/.gitlab-ci/prepare-artifacts.sh @@ -38,6 +38,7 @@ cp -Rp .gitlab-ci/deqp-*.txt install/ cp -Rp .gitlab-ci/report-flakes.py install/ cp -Rp .gitlab-ci/vkd3d-proton install/ find . -path \*/ci/\*.txt \ + -o -path \*/ci/\*.toml \ -o -path \*/ci/\*traces\*.yml \ | xargs -I '{}' cp -p '{}' install/ diff --git a/src/gallium/drivers/softpipe/ci/deqp-softpipe.toml b/src/gallium/drivers/softpipe/ci/deqp-softpipe.toml new file mode 100644 index 00000000000..a5995e6fe69 --- /dev/null +++ b/src/gallium/drivers/softpipe/ci/deqp-softpipe.toml @@ -0,0 +1,51 @@ +[[deqp]] +deqp = "/deqp/modules/gles2/deqp-gles2" +caselists = ["/deqp/mustpass/gles2-master.txt"] +deqp_args = [ + "--deqp-surface-width=256", + "--deqp-surface-height=256", + "--deqp-surface-type=pbuffer", + "--deqp-gl-config-name=rgba8888d24s8ms0", + "--deqp-visibility=hidden" +] + +[[deqp]] +deqp = "/deqp/modules/gles3/deqp-gles3" +caselists = ["/deqp/mustpass/gles3-master.txt"] +deqp_args = [ + "--deqp-surface-width=256", + "--deqp-surface-height=256", + "--deqp-surface-type=pbuffer", + "--deqp-gl-config-name=rgba8888d24s8ms0", + "--deqp-visibility=hidden" +] + +[[deqp]] +deqp = "/deqp/modules/gles31/deqp-gles31" +caselists = ["/deqp/mustpass/gles31-master.txt"] +deqp_args = [ + "--deqp-surface-width=256", + "--deqp-surface-height=256", + "--deqp-surface-type=pbuffer", + "--deqp-gl-config-name=rgba8888d24s8ms0", + "--deqp-visibility=hidden" +] + +# Note that KHR-GL3* test sets include all tests from the previous +# version, so we only need to run one test list (unlike dEQP-GLES, +# where the test sets are separate). +[[deqp]] +deqp = "/deqp/external/openglcts/modules/glcts" +caselists = [ + "/deqp/mustpass/gles2-khr-master.txt", + "/deqp/mustpass/gles3-khr-master.txt", + "/deqp/mustpass/gles31-khr-master.txt", + "/deqp/mustpass/gl33-master.txt", +] +deqp_args = [ + "--deqp-surface-width=256", + "--deqp-surface-height=256", + "--deqp-surface-type=pbuffer", + "--deqp-gl-config-name=rgba8888d24s8ms0", + "--deqp-visibility=hidden" +] diff --git a/src/gallium/drivers/softpipe/ci/gitlab-ci.yml b/src/gallium/drivers/softpipe/ci/gitlab-ci.yml index 7f21bde882e..fdb925e1b1c 100644 --- a/src/gallium/drivers/softpipe/ci/gitlab-ci.yml +++ b/src/gallium/drivers/softpipe/ci/gitlab-ci.yml @@ -1,38 +1,21 @@ -softpipe-gles2: +.softpipe-deqp-test: extends: - - llvmpipe-gles2 - .softpipe-rules + - .test-gl + - .deqp-test variables: GPU_VERSION: softpipe + LIBGL_ALWAYS_SOFTWARE: "true" GALLIUM_DRIVER: "softpipe" DEQP_EXPECTED_RENDERER: softpipe FLAKES_CHANNEL: "#mesa-swrast-ci" -softpipe-gles3: +softpipe-deqp: variables: - DEQP_VER: gles3 - extends: softpipe-gles2 - -softpipe-gles31: + DEQP_VER: gles2 # for the renderer check + DEQP_SUITE: softpipe parallel: 2 - variables: - DEQP_VER: gles31 - extends: softpipe-gles2 - -softpipe-khr-gles2: - variables: - DEQP_VER: gles2-khr - extends: softpipe-gles2 - -softpipe-khr-gles3: - variables: - DEQP_VER: gles3-khr - extends: softpipe-gles2 - -softpipe-khr-gles31: - variables: - DEQP_VER: gles31-khr - extends: softpipe-gles2 + extends: .softpipe-deqp-test softpipe-asan-gles31: variables: @@ -40,7 +23,7 @@ softpipe-asan-gles31: DEQP_FRACTION: 10 DEQP_VER: gles31 TEST_LD_PRELOAD: libasan.so.6 - extends: softpipe-gles2 + extends: .softpipe-deqp-test needs: - debian/x86_test-gl - debian-testing-asan @@ -86,12 +69,3 @@ softpipe-piglit-quick: -x glsl-uniform-interstage-limits PIGLIT_PROFILES: quick_gl quick_shader PIGLIT_RESULTS: softpipe-quick - -# Note that KHR-GL3* test sets include all tests from the previous -# version, so we only need to run one test list (unlike dEQP-GLES, -# where the test sets are separate). -softpipe-gl: - variables: - DEQP_VER: gl33 - extends: - - softpipe-gles2