forked from AuroraMiddleware/gtk
ci: Add an HTML report generator
The JUnit cover report is useful, but only up to a point; for instance, it's not used unless it's part of a merge request. This means you don't get a report if you're pushing to a branch that does not have an MR open. With a simple Python script and some minimal templating, we can generate an HTML report from the "I Can't Believe it's not JSON™" log that Meson produces, and keep it as a CI artifact.
This commit is contained in:
Emmanuele Bassi
parent
f9d5869774
commit
3bc8ab91a2
@ -26,6 +26,7 @@ fedora-x86_64:
|
||||
paths:
|
||||
- "${CI_PROJECT_DIR}/_build/meson-logs"
|
||||
- "${CI_PROJECT_DIR}/_build/report.xml"
|
||||
- "${CI_PROJECT_DIR}/_build/report.html"
|
||||
- "${CI_PROJECT_DIR}/_build/testsuite/reftests/output/*.png"
|
||||
cache:
|
||||
key: "$CI_JOB_NAME"
|
||||
|
||||
@ -73,6 +73,8 @@ RUN dnf -y install \
|
||||
|
||||
RUN pip3 install meson==0.50.0
|
||||
|
||||
RUN pip3 install jinja2
|
||||
|
||||
ARG HOST_USER_ID=5555
|
||||
ENV HOST_USER_ID ${HOST_USER_ID}
|
||||
RUN useradd -u $HOST_USER_ID -ms /bin/bash user
|
||||
|
||||
164
.gitlab-ci/meson-html-report.py
Executable file
164
.gitlab-ci/meson-html-report.py
Executable file
@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright 2019 GNOME Foundation
|
||||
|
||||
# Turns a test log generated by Meson into an HTML report
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from jinja2 import Template
|
||||
|
||||
REPORT_TEMPLATE = '''
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<title>{{ report.project_name }} Test Report</title>
|
||||
<meta charset="utf-8" />
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>{{ report.project_name }} :: Test Reports</h1>
|
||||
<div class="report-meta">
|
||||
<p><strong>Branch:</strong> {{ report.branch_name }}</p>
|
||||
<p><strong>Date:</strong> <time datetime="{{ report.date.isoformat() }}">{{ report.locale_date }}</time></p>
|
||||
{% if report.job_id %}<p><strong>Job ID:</strong> {{ report.job_id }}</p>{% endif %}
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<article>
|
||||
<section>
|
||||
<div class="summary">
|
||||
<h3>Summary</h3>
|
||||
<ul>
|
||||
<li><strong>Total units:</strong> {{ report.total_units }}</li>
|
||||
<li><strong>Passed:</strong> {{ report.total_successes }}</li>
|
||||
<li><strong>Failed:</strong> {{ report.total_failures }}</li>
|
||||
</u>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{% for suite_result in report.results_list %}
|
||||
<section>
|
||||
<div class="result">
|
||||
<h3>Suite: {{ suite_result.suite_name }}</h3>
|
||||
<ul>
|
||||
<li><strong>Units:</strong> {{ suite_result.n_units }}</li>
|
||||
<li><strong>Passed:</strong> {{ suite_result.n_successes }}</li>
|
||||
<li><strong>Failed:</strong> {{ suite_result.n_failures }}</li>
|
||||
</ul>
|
||||
{% for failure in suite_result.failures %}
|
||||
{% if loop.first %}
|
||||
<div>
|
||||
<h4>Failures</h4>
|
||||
<ul>
|
||||
{% endif %}
|
||||
<li>{{ failure.name }} - result: <span class="failure">{{ failure.result }}</span><br/>
|
||||
<pre>{{ failure.stdout }}</pre>
|
||||
</li>
|
||||
{% if loop.last %}
|
||||
</ul>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</section>
|
||||
{% endfor %}
|
||||
|
||||
</article>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
|
||||
aparser = argparse.ArgumentParser(description='Turns a Meson test log into an HTML report')
|
||||
aparser.add_argument('--project-name', metavar='NAME',
|
||||
help='The project name',
|
||||
default='Unknown')
|
||||
aparser.add_argument('--job-id', metavar='ID',
|
||||
help='The job ID for the report',
|
||||
default=None)
|
||||
aparser.add_argument('--branch', metavar='NAME',
|
||||
help='Branch of the project being tested',
|
||||
default='master')
|
||||
aparser.add_argument('--output', metavar='FILE',
|
||||
help='The output HTML file, stdout by default',
|
||||
type=argparse.FileType('w', encoding='UTF-8'),
|
||||
default=sys.stdout)
|
||||
aparser.add_argument('infile', metavar='FILE',
|
||||
help='The input testlog.json, stdin by default',
|
||||
type=argparse.FileType('r', encoding='UTF-8'),
|
||||
default=sys.stdin)
|
||||
|
||||
args = aparser.parse_args()
|
||||
|
||||
outfile = args.output
|
||||
|
||||
suites = {}
|
||||
for line in args.infile:
|
||||
data = json.loads(line)
|
||||
(full_suite, unit_name) = data['name'].split(' / ')
|
||||
(project_name, suite_name) = full_suite.split(':')
|
||||
|
||||
unit = {
|
||||
'project-name': project_name,
|
||||
'suite': suite_name,
|
||||
'name': unit_name,
|
||||
'duration': data['duration'],
|
||||
'returncode': data['returncode'],
|
||||
'result': data['result'],
|
||||
'stdout': data['stdout'],
|
||||
}
|
||||
|
||||
units = suites.setdefault(full_suite, [])
|
||||
units.append(unit)
|
||||
|
||||
report = {}
|
||||
report['date'] = datetime.datetime.utcnow()
|
||||
report['locale_date'] = report['date'].strftime("%c")
|
||||
report['project_name'] = args.project_name
|
||||
report['job_id'] = args.job_id
|
||||
report['branch_name'] = args.branch
|
||||
report['total_successes'] = 0
|
||||
report['total_failures'] = 0
|
||||
report['total_units'] = 0
|
||||
report['results_list'] = []
|
||||
|
||||
for name, units in suites.items():
|
||||
(project_name, suite_name) = name.split(':')
|
||||
print('Processing {} suite {}:'.format(project_name, suite_name))
|
||||
|
||||
def if_failed(unit):
|
||||
if unit['result'] in ['FAIL', 'TIMEOUT']:
|
||||
return True
|
||||
return False
|
||||
|
||||
def if_succeded(unit):
|
||||
if unit['result'] in ['OK', 'EXPECTEDFAIL', 'SKIP']:
|
||||
return True
|
||||
return False
|
||||
|
||||
successes = list(filter(if_succeded, units))
|
||||
failures = list(filter(if_failed, units))
|
||||
|
||||
n_units = len(units)
|
||||
n_successes = len(successes)
|
||||
n_failures = len(failures)
|
||||
|
||||
report['total_units'] += n_units
|
||||
report['total_successes'] += n_successes
|
||||
report['total_failures'] += n_failures
|
||||
print(' - {}: {} total, {} pass, {} fail'.format(suite_name, n_units, n_successes, n_failures))
|
||||
|
||||
suite_report = {
|
||||
'suite_name': suite_name,
|
||||
'n_units': n_units,
|
||||
'successes': successes,
|
||||
'n_successes': n_successes,
|
||||
'failures': failures,
|
||||
'n_failures': n_failures,
|
||||
}
|
||||
report['results_list'].append(suite_report)
|
||||
|
||||
template = Template(REPORT_TEMPLATE)
|
||||
outfile.write(template.render({'report': report}))
|
||||
@ -37,11 +37,17 @@ xvfb-run -a -s "-screen 0 1024x768x24" \
|
||||
# Save the exit code
|
||||
exit_code=$?
|
||||
|
||||
# We always want to run the report generator
|
||||
# We always want to run the report generators
|
||||
$srcdir/.gitlab-ci/meson-junit-report.py \
|
||||
--project-name=gtk \
|
||||
--job-id="${CI_JOB_NAME}" \
|
||||
--output=report.xml \
|
||||
meson-logs/testlog.json
|
||||
|
||||
$srcdir/.gitlab-ci/meson-html-report.py \
|
||||
--project-name=GTK \
|
||||
--job-id="${CI_JOB_NAME}" \
|
||||
--output=report.html \
|
||||
meson-logs/testlog.json
|
||||
|
||||
exit $exit_code
|
||||
|
||||
Loading…
Reference in New Issue
Block a user