diff --git a/.gitlab-ci/vkd3d-runner.sh b/.gitlab-ci/vkd3d-runner.sh index 3f2bfae2a46..175dcd65890 100755 --- a/.gitlab-ci/vkd3d-runner.sh +++ b/.gitlab-ci/vkd3d-runner.sh @@ -57,6 +57,35 @@ else touch "$EXPECTATIONFILE" fi +if [ -f "$INSTALL/$GPU_VERSION-vkd3d-flakes.txt" ]; then + mapfile -t flakes < <(grep -vE '^#|^$' "$INSTALL/$GPU_VERSION-vkd3d-flakes.txt") +else + flakes=() +fi + +# Some sanity checks before we start +mapfile -t flakes_dups < <( + [ ${#flakes[@]} -eq 0 ] || + printf '%s\n' "${flakes[@]}" | sort | uniq -d +) +if [ ${#flakes_dups[@]} -gt 0 ]; then + echo >&2 'Duplicate flakes lines:' + printf >&2 ' %s\n' "${flakes_dups[@]}" + exit 1 +fi + +flakes_in_baseline=() +for flake in "${flakes[@]}"; do + if grep -qF "$flake" "$EXPECTATIONFILE"; then + flakes_in_baseline+=("$flake") + fi +done +if [ ${#flakes_in_baseline[@]} -gt 0 ]; then + echo >&2 "Flakes found in $EXPECTATIONFILE:" + printf >&2 ' %s\n' "${flakes_in_baseline[@]}" + exit 1 +fi + printf "%s\n" "Running vkd3d-proton testsuite..." if ! /vkd3d-proton-tests/x64/bin/d3d12 &> "$RESULTS/vkd3d-proton-log.txt"; then @@ -78,11 +107,32 @@ if ! /vkd3d-proton-tests/x64/bin/d3d12 &> "$RESULTS/vkd3d-proton-log.txt"; then exit 1 fi + # Ignore flakes when comparing + STABLERESULTSFILE="$RESULTS/$GPU_VERSION-results-minus-flakes.txt" + cp "$RESULTSFILE" "$STABLERESULTSFILE" + for flake in "${flakes[@]}"; do + grep -vF "$flake" "$STABLERESULTSFILE" > tmp && mv tmp "$STABLERESULTSFILE" + done + # Make sure that the failures found in this run match the current expectation - if ! diff --color=always -u "$EXPECTATIONFILE" "$RESULTSFILE"; then + if ! diff --color=always -u "$EXPECTATIONFILE" "$STABLERESULTSFILE"; then error "Changes found, see ${ARTIFACTS_BASE_URL}/results/vkd3d-proton-log.txt" exit 1 fi + + # Print list of flakes seen this time + flakes_seen=() + for flake in "${flakes[@]}"; do + if grep -qF "$flake" "$RESULTSFILE"; then + flakes_seen+=("$flake") + fi + done + if [ ${#flakes_seen[@]} -gt 0 ]; then + # Keep this string and output format in line with the corresponding + # deqp-runner message + echo >&2 'Some known flakes found:' + printf >&2 ' %s\n' "${flakes_seen[@]}" + fi fi exit 0 diff --git a/src/amd/ci/gitlab-ci-inc.yml b/src/amd/ci/gitlab-ci-inc.yml index 2f3c98c54aa..10a9c86ae09 100644 --- a/src/amd/ci/gitlab-ci-inc.yml +++ b/src/amd/ci/gitlab-ci-inc.yml @@ -15,6 +15,7 @@ - src/amd/ci/$GPU_VERSION-skips.txt - src/amd/ci/$PIGLIT_TRACES_FILE - src/amd/ci/$GPU_VERSION-vkd3d-fails.txt + - src/amd/ci/$GPU_VERSION-vkd3d-flakes.txt - src/amd/ci/$GPU_VERSION-vkd3d-skips.txt - src/amd/common/**/* - src/amd/llvm/**/*