mirror of
https://gitlab.gnome.org/GNOME/gtk.git
synced 2025-01-14 14:20:21 +00:00
957dd49ef7
There are two possible interpretations of "expected failure": either
the test *must* fail (exactly the inverse of an ordinary test, with
success becoming failure and failure becoming success), or the test
*may* fail (with success intended, but failure possible in some
environments). Autotools had the second interpretation, which seems
more useful in practice, but Meson has the first.
Instead of using should_fail, we can put the tests in one of two new
suites: "flaky" is intended for tests that succeed or fail unpredictably
according to the test environment or chance, while "failing" is for
tests that ought to succeed but currently never do as a result of a
bug or missing functionality. With a sufficiently new version of Meson,
the flaky and failing tests are not run by default, but can be requested
by running a setup that does not exclude them, with a command like:
meson test --setup=x11_unstable --suite=flaky --suite=failing
As a bonus, now that we're setting up setups and their excluded suites
programmatically, the gsk-compare-broadway tests are also excluded by
default when running the test setup for a non-broadway backend.
When running the tests in CI, --suite=gtk overrides the default
exclude_suites, so we have to specify --no-suite=flaky and
--no-suite=failing explicitly.
This arrangement is inspired by GNOME/glib!2987, which was contributed
by Marco Trevisan.
Signed-off-by: Simon McVittie <smcv@debian.org>
148 lines
4.3 KiB
Bash
Executable File
148 lines
4.3 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set +x
|
|
set +e
|
|
|
|
srcdir=$( pwd )
|
|
builddir=$1
|
|
backend=$2
|
|
|
|
# Ignore memory leaks lower in dependencies
|
|
export LSAN_OPTIONS=suppressions=$srcdir/lsan.supp:print_suppressions=0
|
|
export G_SLICE=always-malloc
|
|
|
|
case "${backend}" in
|
|
x11)
|
|
xvfb-run -a -s "-screen 0 1024x768x24 -noreset" \
|
|
meson test -C ${builddir} \
|
|
--timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \
|
|
--print-errorlogs \
|
|
--setup=${backend} \
|
|
--suite=gtk \
|
|
--no-suite=failing \
|
|
--no-suite=flaky \
|
|
--no-suite=gsk-compare-broadway
|
|
|
|
# Store the exit code for the CI run, but always
|
|
# generate the reports
|
|
exit_code=$?
|
|
|
|
xvfb-run -a -s "-screen 0 1024x768x24 -noreset" \
|
|
meson test -C ${builddir} \
|
|
--timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \
|
|
--print-errorlogs \
|
|
--setup=${backend}_unstable \
|
|
--suite=flaky \
|
|
--suite=failing || true
|
|
;;
|
|
|
|
wayland)
|
|
export XDG_RUNTIME_DIR="$(mktemp -p $(pwd) -d xdg-runtime-XXXXXX)"
|
|
|
|
weston --backend=headless-backend.so --socket=wayland-5 --idle-time=0 &
|
|
compositor=$!
|
|
export WAYLAND_DISPLAY=wayland-5
|
|
|
|
meson test -C ${builddir} \
|
|
--timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \
|
|
--print-errorlogs \
|
|
--setup=${backend} \
|
|
--suite=gtk \
|
|
--no-suite=failing \
|
|
--no-suite=flaky \
|
|
--no-suite=gsk-compare-broadway
|
|
exit_code=$?
|
|
|
|
meson test -C ${builddir} \
|
|
--timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \
|
|
--print-errorlogs \
|
|
--setup=${backend}_unstable \
|
|
--suite=flaky \
|
|
--suite=failing || true
|
|
|
|
kill ${compositor}
|
|
;;
|
|
|
|
waylandgles)
|
|
export XDG_RUNTIME_DIR="$(mktemp -p $(pwd) -d xdg-runtime-XXXXXX)"
|
|
|
|
weston --backend=headless-backend.so --socket=wayland-6 --idle-time=0 &
|
|
compositor=$!
|
|
export WAYLAND_DISPLAY=wayland-6
|
|
|
|
meson test -C ${builddir} \
|
|
--timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \
|
|
--print-errorlogs \
|
|
--setup=${backend} \
|
|
--suite=gtk \
|
|
--no-suite=failing \
|
|
--no-suite=flaky \
|
|
--no-suite=gsk-compare-broadway
|
|
exit_code=$?
|
|
|
|
meson test -C ${builddir} \
|
|
--timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \
|
|
--print-errorlogs \
|
|
--setup=${backend}_unstable \
|
|
--suite=flaky \
|
|
--suite=failing || true
|
|
|
|
kill ${compositor}
|
|
;;
|
|
|
|
broadway)
|
|
export XDG_RUNTIME_DIR="$(mktemp -p $(pwd) -d xdg-runtime-XXXXXX)"
|
|
|
|
${builddir}/gdk/broadway/gtk4-broadwayd :5 &
|
|
server=$!
|
|
export BROADWAY_DISPLAY=:5
|
|
|
|
meson test -C ${builddir} \
|
|
--timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \
|
|
--print-errorlogs \
|
|
--setup=${backend} \
|
|
--suite=gtk \
|
|
--no-suite=failing \
|
|
--no-suite=flaky \
|
|
--no-suite=gsk-compare-opengl
|
|
|
|
# don't let Broadway failures fail the run, for now
|
|
exit_code=0
|
|
|
|
meson test -C ${builddir} \
|
|
--timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \
|
|
--print-errorlogs \
|
|
--setup=${backend}_unstable \
|
|
--suite=flaky \
|
|
--suite=failing || true
|
|
|
|
kill ${server}
|
|
;;
|
|
|
|
*)
|
|
echo "Failed to add ${backend} to .gitlab-ci/run-tests.sh"
|
|
exit 1
|
|
;;
|
|
|
|
esac
|
|
|
|
cd ${builddir}
|
|
|
|
for suffix in "" "_unstable"; do
|
|
$srcdir/.gitlab-ci/meson-junit-report.py \
|
|
--project-name=gtk \
|
|
--backend="${backend}${suffix}" \
|
|
--job-id="${CI_JOB_NAME}" \
|
|
--output="report-${backend}${suffix}.xml" \
|
|
"meson-logs/testlog-${backend}${suffix}.json"
|
|
$srcdir/.gitlab-ci/meson-html-report.py \
|
|
--project-name=gtk \
|
|
--backend="${backend}${suffix}" \
|
|
--job-id="${CI_JOB_NAME}" \
|
|
--reftest-output-dir="testsuite/reftests/output/${backend}${suffix}" \
|
|
--output="report-${backend}${suffix}.html" \
|
|
"meson-logs/testlog-${backend}${suffix}.json"
|
|
done
|
|
|
|
exit $exit_code
|