From 3dc6a07f5d8f73316600fc1fcbb7060ec9368db8 Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Thu, 13 Feb 2020 14:04:41 +0000 Subject: [PATCH 1/8] ci: Re-enable ccache --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6993d5ae1a..30f2750751 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,7 +18,6 @@ variables: COMMON_MESON_FLAGS: "--fatal-meson-warnings --werror" MESON_TEST_TIMEOUT_MULTIPLIER: 2 G_MESSAGES_DEBUG: "all" - CCACHE_DISABLE: "true" .only-default: only: From f4a8edf441efef49e009fa220e633229a9079bd2 Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Thu, 13 Feb 2020 13:52:57 +0000 Subject: [PATCH 2/8] ci: Move HTML report meta from header to article Makes the bar smaller. --- .gitlab-ci/meson-html-report.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci/meson-html-report.py b/.gitlab-ci/meson-html-report.py index ef895a948a..437a6cb664 100755 --- a/.gitlab-ci/meson-html-report.py +++ b/.gitlab-ci/meson-html-report.py @@ -139,14 +139,17 @@ ul.images li {

{{ report.project_name }} :: Test Reports

-
-

Branch: {{ report.branch_name }}

-

Date:

- {% if report.job_id %}

Job ID: {{ report.job_id }}

{% endif %} -
+
+
+

Branch: {{ report.branch_name }}

+

Date:

+ {% if report.job_id %}

Job ID: {{ report.job_id }}

{% endif %} +
+
+

Summary

From adc7bf9f256bc51033740d9a44b184bcd5f396bb Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Thu, 13 Feb 2020 13:54:02 +0000 Subject: [PATCH 3/8] ci: Put the branch name in the HTML header --- .gitlab-ci/meson-html-report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci/meson-html-report.py b/.gitlab-ci/meson-html-report.py index 437a6cb664..4ba5fceadf 100755 --- a/.gitlab-ci/meson-html-report.py +++ b/.gitlab-ci/meson-html-report.py @@ -138,7 +138,7 @@ ul.images li {
-

{{ report.project_name }} :: Test Reports

+

{{ report.project_name }}/{{ report.branch_name }} :: Test Reports

From 1a4d1f899404c0e82269bef279c2b17e2330fd9c Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Thu, 13 Feb 2020 13:54:22 +0000 Subject: [PATCH 4/8] ci: Unexpected passes are failures List them as such in our reports. --- .gitlab-ci/meson-html-report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci/meson-html-report.py b/.gitlab-ci/meson-html-report.py index 4ba5fceadf..a7bb16a89b 100755 --- a/.gitlab-ci/meson-html-report.py +++ b/.gitlab-ci/meson-html-report.py @@ -214,7 +214,7 @@ ul.images li {

Failed

    - {% for failure in suite_result.failures if failure.result == 'FAIL' %} + {% for failure in suite_result.failures if failure.result in [ 'FAIL', 'UNEXPECTEDPASS' ] %}
  • {{ failure.name }} - result: {{ failure.result }}
    {% if failure.stdout %} Output: 
    {{ failure.stdout }}
    @@ -331,7 +331,7 @@ for name, units in suites.items(): print('Processing {} suite {}:'.format(project_name, suite_name)) def if_failed(unit): - if unit['result'] in ['FAIL', 'TIMEOUT']: + if unit['result'] in ['FAIL', 'UNEXPECTEDPASS', 'TIMEOUT']: return True return False From 2b1e273c53ffe0701df6876225d11297d0e9987f Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Thu, 13 Feb 2020 13:56:33 +0000 Subject: [PATCH 5/8] ci: Use result instead of exit code in the JUnit report We have a result code coming from Meson which is more accurate than just looking at the exit code of the unit. --- .gitlab-ci/meson-junit-report.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci/meson-junit-report.py b/.gitlab-ci/meson-junit-report.py index 248ef6e2b1..f63c82eb36 100755 --- a/.gitlab-ci/meson-junit-report.py +++ b/.gitlab-ci/meson-junit-report.py @@ -51,6 +51,7 @@ for line in args.infile: duration = data['duration'] return_code = data['returncode'] + result = data['result'] log = data['stdout'] unit = { @@ -58,6 +59,7 @@ for line in args.infile: 'name': unit_name, 'duration': duration, 'returncode': return_code, + 'result': result, 'stdout': log, } @@ -68,12 +70,12 @@ for name, units in suites.items(): print('Processing suite {} (units: {})'.format(name, len(units))) def if_failed(unit): - if unit['returncode'] != 0: + if unit['result'] in ['FAIL', 'UNEXPECTEDPASS', 'TIMEOUT']: return True return False def if_succeded(unit): - if unit['returncode'] == 0: + if unit['result'] in ['OK', 'EXPECTEDFAIL', 'SKIP']: return True return False From affc7b58fa7e8a862fbb96814371d2d5fae04658 Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Thu, 13 Feb 2020 14:23:59 +0000 Subject: [PATCH 6/8] ci: Fix the HTML anchor in the report --- .gitlab-ci/meson-html-report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci/meson-html-report.py b/.gitlab-ci/meson-html-report.py index a7bb16a89b..faa5e68264 100755 --- a/.gitlab-ci/meson-html-report.py +++ b/.gitlab-ci/meson-html-report.py @@ -212,7 +212,7 @@ ul.images li {
-

Failed

+

Failures

    {% for failure in suite_result.failures if failure.result in [ 'FAIL', 'UNEXPECTEDPASS' ] %}
  • {{ failure.name }} - result: {{ failure.result }}
    From 5ed3f8d98b6783117d186dcedf42d6818df53a28 Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Thu, 13 Feb 2020 14:26:40 +0000 Subject: [PATCH 7/8] ci: Rearrange the results in the HTML report Move the failures up top, so they stand out. --- .gitlab-ci/meson-html-report.py | 82 ++++++++++++++++----------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/.gitlab-ci/meson-html-report.py b/.gitlab-ci/meson-html-report.py index faa5e68264..10c6d135e3 100755 --- a/.gitlab-ci/meson-html-report.py +++ b/.gitlab-ci/meson-html-report.py @@ -167,50 +167,10 @@ ul.images li {

    Suite: {{ suite_result.suite_name }}

    • Units: {{ suite_result.n_units }}
      • -
    • Passed: {{ suite_result.n_successes }}
    • Failed: {{ suite_result.n_failures }}
      • +
    • Passed: {{ suite_result.n_successes }}
    -
    -

    Passed

    -
      - {% for success in suite_result.successes if success.result == 'OK' %} -
    • {{ success.name }} - result: {{ success.result }}
      • - {% else %} -
    • None
      • - {% endfor %} -
    - -

    Skipped

    -
      - {% for success in suite_result.successes if success.result == 'SKIP' %} -
    • {{ success.name }} - result: {{ success.result }}
      • - {% else %} -
    • None
      • - {% endfor %} -
    - -

    Expected failures

    -
      - {% for success in suite_result.successes if success.result == 'EXPECTEDFAIL' %} -
    • {{ success.name }} - result: {{ success.result }}
      - {% if success.stdout %} - Output: 
      {{ success.stdout }}
      - {% endif %} - {% if success.image_data is defined %} -
        -
      • ref
        • -
      • out
        • -
      • diff
        • -
      - {% endif %} -
      • - {% else %} -
    • None
      • - {% endfor %} -
    -
    -

    Failures

      @@ -246,6 +206,46 @@ ul.images li {
    +
    +

    Skipped

    +
      + {% for success in suite_result.successes if success.result == 'SKIP' %} +
    • {{ success.name }} - result: {{ success.result }}
      • + {% else %} +
    • None
      • + {% endfor %} +
    + +

    Passed

    +
      + {% for success in suite_result.successes if success.result == 'OK' %} +
    • {{ success.name }} - result: {{ success.result }}
      • + {% else %} +
    • None
      • + {% endfor %} +
    + +

    Expected failures

    +
      + {% for success in suite_result.successes if success.result == 'EXPECTEDFAIL' %} +
    • {{ success.name }} - result: {{ success.result }}
      + {% if success.stdout %} + Output: 
      {{ success.stdout }}
      + {% endif %} + {% if success.image_data is defined %} +
        +
      • ref
        • +
      • out
        • +
      • diff
        • +
      + {% endif %} +
      • + {% else %} +
    • None
      • + {% endfor %} +
    +
    +
{% endfor %} From de1543d4fa580a041abd1e58ef7c2dee067c7d92 Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Thu, 13 Feb 2020 14:45:08 +0000 Subject: [PATCH 8/8] ci: Use per-suite anchors Anchors are global, so they need to be namespaced. --- .gitlab-ci/meson-html-report.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci/meson-html-report.py b/.gitlab-ci/meson-html-report.py index 10c6d135e3..f0bb35728b 100755 --- a/.gitlab-ci/meson-html-report.py +++ b/.gitlab-ci/meson-html-report.py @@ -155,8 +155,8 @@ ul.images li {

Summary

@@ -167,12 +167,12 @@ ul.images li {

Suite: {{ suite_result.suite_name }}

-

Failures

+

Failures

    {% for failure in suite_result.failures if failure.result in [ 'FAIL', 'UNEXPECTEDPASS' ] %}
  • {{ failure.name }} - result: {{ failure.result }}
    @@ -192,7 +192,7 @@ ul.images li { {% endfor %}
-

Timed out

+

Timed out

    {% for failure in suite_result.failures if failure.result == 'TIMEOUT' %}
  • {{ failure.name }} - result: {{ failure.result }}
    @@ -207,7 +207,7 @@ ul.images li {
-

Skipped

+

Skipped

    {% for success in suite_result.successes if success.result == 'SKIP' %}
  • {{ success.name }} - result: {{ success.result }}
    • @@ -216,7 +216,7 @@ ul.images li { {% endfor %}
-

Passed

+

Passed

    {% for success in suite_result.successes if success.result == 'OK' %}
  • {{ success.name }} - result: {{ success.result }}
    • @@ -225,7 +225,7 @@ ul.images li { {% endfor %}
-

Expected failures

+

Expected failures

    {% for success in suite_result.successes if success.result == 'EXPECTEDFAIL' %}
  • {{ success.name }} - result: {{ success.result }}