#!/usr/bin/env python3 # Turns a Mason testlog.json file into an HTML report # # Copyright 2019 GNOME Foundation # # SPDX-License-Identifier: LGPL-2.1-or-later # # Original author: Emmanuele Bassi import argparse import datetime import json import os import sys from jinja2 import Template REPORT_TEMPLATE = ''' {{ report.project_name }} Test Report

{{ report.project_name }}/{{ report.backend }}/{{ report.branch_name }} :: Test Reports

Backend: {{ report.backend }}

Branch: {{ report.branch_name }}

Date:

{% if report.job_id %}

Job ID: {{ report.job_id }}

{% endif %}

Summary

  • Total units: {{ report.total_units }}
  • Failed: {{ report.total_failures }}
  • Passed: {{ report.total_successes }}
{% for suite_result in report.results_list %}

Suite: {{ suite_result.suite_name }}

Failures

    {% for failure in suite_result.failures if failure.result in [ 'ERROR', 'FAIL', 'UNEXPECTEDPASS' ] %}
  • {{ failure.name }} - result: {{ failure.result }}
    {% if failure.stdout %} Output: 
    {{ failure.stdout }}
    {% endif %} {% if failure.image_data is defined %} {% endif %}
    • {% else %}
  • None
    • {% endfor %}

Timed out

    {% for failure in suite_result.failures if failure.result == 'TIMEOUT' %}
  • {{ failure.name }} - result: {{ failure.result }}
    {% if failure.stdout %} Output: 
    {{ failure.stdout }}
    {% endif %}
    • {% else %}
  • None
    • {% endfor %}

Expected failures

    {% for success in suite_result.successes if success.result == 'EXPECTEDFAIL' %}
  • {{ success.name }} - result: {{ success.result }}
    {% if success.stdout %} Output: 
    {{ success.stdout }}
    {% endif %} {% if success.image_data is defined %}
    • ref
    • out
    • diff
    {% endif %}
    • {% else %}
  • None
    • {% endfor %}

Skipped

    {% for success in suite_result.successes if success.result == 'SKIP' %}
  • {{ success.name }} - result: {{ success.result }}
    • {% else %}
  • None
    • {% endfor %}

Passed

    {% for success in suite_result.successes if success.result == 'OK' %}
  • {{ success.name }} - result: {{ success.result }}
    • {% else %}
  • None
    • {% endfor %}
{% endfor %}
''' aparser = argparse.ArgumentParser(description='Turns a Meson test log into an HTML report') aparser.add_argument('--project-name', metavar='NAME', help='The project name', default='Unknown') aparser.add_argument('--backend', metavar='NAME', help='The used backend', default='unknown') aparser.add_argument('--job-id', metavar='ID', help='The job ID for the report', default=None) aparser.add_argument('--branch', metavar='NAME', help='Branch of the project being tested', default='main') aparser.add_argument('--output', metavar='FILE', help='The output HTML file, stdout by default', type=argparse.FileType('w', encoding='UTF-8'), default=sys.stdout) aparser.add_argument('--reftest-suite', metavar='NAME', help='The name of the reftests suite', default='reftest') aparser.add_argument('--reftest-output-dir', metavar='DIR', help='The output directory for reftests data', default=None) aparser.add_argument('infile', metavar='FILE', help='The input testlog.json, stdin by default', type=argparse.FileType('r', encoding='UTF-8'), default=sys.stdin) args = aparser.parse_args() outfile = args.output suites = {} for line in args.infile: data = json.loads(line) (full_suite, unit_name) = data['name'].split(' / ') (project_name, suite_name) = full_suite.split(':') unit = { 'project-name': project_name, 'suite': suite_name, 'name': unit_name, 'duration': data['duration'], 'returncode': data['returncode'], 'result': data['result'], 'stdout': data['stdout'], } if args.reftest_output_dir is not None and suite_name == args.reftest_suite: filename = unit_name.split(' ')[1] basename = os.path.splitext(filename)[0] image_data = { 'ref': os.path.join(args.reftest_output_dir, '{}.ref.png'.format(basename)), 'out': os.path.join(args.reftest_output_dir, '{}.out.png'.format(basename)), 'refnode': os.path.join(args.reftest_output_dir, '{}.ref.node'.format(basename)), 'outnode': os.path.join(args.reftest_output_dir, '{}.out.node'.format(basename)), 'diff': os.path.join(args.reftest_output_dir, '{}.diff.png'.format(basename)), } unit['image_data'] = image_data units = suites.setdefault(full_suite, []) units.append(unit) report = {} report['date'] = datetime.datetime.utcnow() report['locale_date'] = report['date'].strftime("%c") report['project_name'] = args.project_name report['backend'] = args.backend report['job_id'] = args.job_id report['branch_name'] = args.branch report['total_successes'] = 0 report['total_failures'] = 0 report['total_units'] = 0 report['results_list'] = [] for name, units in suites.items(): (project_name, suite_name) = name.split(':') print('Processing {} suite {}:'.format(project_name, suite_name)) def if_failed(unit): if unit['result'] in ['FAIL', 'UNEXPECTEDPASS', 'TIMEOUT', 'ERROR',]: return True return False def if_succeded(unit): if unit['result'] in ['OK', 'EXPECTEDFAIL', 'SKIP']: return True return False successes = list(filter(if_succeded, units)) failures = list(filter(if_failed, units)) n_units = len(units) n_successes = len(successes) n_failures = len(failures) report['total_units'] += n_units report['total_successes'] += n_successes report['total_failures'] += n_failures print(' - {}: {} total, {} pass, {} fail'.format(suite_name, n_units, n_successes, n_failures)) suite_report = { 'suite_name': suite_name, 'n_units': n_units, 'successes': successes, 'n_successes': n_successes, 'failures': failures, 'n_failures': n_failures, } report['results_list'].append(suite_report) template = Template(REPORT_TEMPLATE) outfile.write(template.render({'report': report}))