From 42b1161e8c45df468d1696b43a7820c3ffe29da1 Mon Sep 17 00:00:00 2001 From: Siddhesh Poyarekar Date: Wed, 11 Jun 2014 14:16:29 +0530 Subject: [PATCH] Validate bench.out against a JSON schema This patch adds a JSON schema for the benchmark output file and also adds a script that validates the generated output against the schema. --- ChangeLog | 6 ++ benchtests/Makefile | 2 + benchtests/scripts/benchout.schema.json | 42 ++++++++++++ benchtests/scripts/validate_benchout.py | 85 +++++++++++++++++++++++++ 4 files changed, 135 insertions(+) create mode 100644 benchtests/scripts/benchout.schema.json create mode 100755 benchtests/scripts/validate_benchout.py diff --git a/ChangeLog b/ChangeLog index 1f7b6d0c91..85e631e1f1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +2014-06-11 Siddhesh Poyarekar + + * benchtests/scripts/validate_benchout.py: New script. + * benchtests/Makefile (bench-func): Call it. + * benchtests/scripts/benchout.schema.json: New file. + 2014-06-10 Chris Metcalf * sysdeps/unix/sysv/linux/tile/nptl/bits/pthreadtypes.h: Moved ... diff --git a/benchtests/Makefile b/benchtests/Makefile index fbcee1328b..dc9ee043d6 100644 --- a/benchtests/Makefile +++ b/benchtests/Makefile @@ -131,6 +131,8 @@ bench-func: $(binaries-bench) mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \ fi; \ mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out + scripts/validate_benchout.py $(objpfx)bench.out \ + scripts/benchout.schema.json $(timing-type) $(binaries-bench) $(binaries-benchset): %: %.o $(objpfx)json-lib.o \ $(sort $(filter $(common-objpfx)lib%,$(link-libc))) \ diff --git a/benchtests/scripts/benchout.schema.json b/benchtests/scripts/benchout.schema.json new file mode 100644 index 0000000000..affb7c11f4 --- /dev/null +++ b/benchtests/scripts/benchout.schema.json @@ -0,0 +1,42 @@ +{ + "title": "benchmark", + "type": "object", + "properties": { + "timing_type": { + "type": "string" + }, + "functions": { + "title": "Associative array of functions", + "type": "object", + "patternProperties": { + "^[_a-zA-Z][_a-zA-Z0-9]+$": { + "title": "Function names", + "type": "object", + "patternProperties": { + "^[_a-zA-Z0-9]*$": { + "title": "Function variants", + "type": "object", + "properties": { + "duration": {"type": "number"}, + "iterations": {"type": "number"}, + "max": {"type": "number"}, + "min": {"type": "number"}, + "mean": {"type": "number"}, + "timings": { + "type": "array", + "items": {"type": "number"} + } + }, + "required": ["duration", "iterations", "max", "min", "mean"], + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "minProperties": 1 + } + }, + "required": ["timing_type", "functions"], + "additionalProperties": false +} diff --git a/benchtests/scripts/validate_benchout.py b/benchtests/scripts/validate_benchout.py new file mode 100755 index 0000000000..61a8cbd5d3 --- /dev/null +++ b/benchtests/scripts/validate_benchout.py @@ -0,0 +1,85 @@ +#!/usr/bin/python +# Copyright (C) 2014 Free Software Foundation, Inc. +# This file is part of the GNU C Library. +# +# The GNU C Library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# The GNU C Library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with the GNU C Library; if not, see +# . +"""Benchmark output validator + +Given a benchmark output file in json format and a benchmark schema file, +validate the output against the schema. +""" + +from __future__ import print_function +import json +import sys +import os + +try: + import jsonschema +except ImportError: + print('Could not find jsonschema module. Output not validated.') + # Return success because we don't want the bench target to fail just + # because the jsonschema module was not found. + sys.exit(os.EX_OK) + + +def validate_bench(benchfile, schemafile): + """Validate benchmark file + + Validate a benchmark output file against a JSON schema. + + Args: + benchfile: The file name of the bench.out file. + schemafile: The file name of the JSON schema file to validate + bench.out against. + + Exceptions: + jsonschema.ValidationError: When bench.out is not valid + jsonschema.SchemaError: When the JSON schema is not valid + IOError: If any of the files are not found. + """ + with open(benchfile, 'r') as bfile: + with open(schemafile, 'r') as sfile: + bench = json.load(bfile) + schema = json.load(sfile) + jsonschema.validate(bench, schema) + + # If we reach here, we're all good. + print("Benchmark output in %s is valid." % benchfile) + + +def main(args): + """Main entry point + + Args: + args: The command line arguments to the program + + Returns: + 0 on success or a non-zero failure code + + Exceptions: + Exceptions thrown by validate_bench + """ + if len(args) != 2: + print("Usage: %s " % sys.argv[0], + file=sys.stderr) + return os.EX_USAGE + + validate_bench(args[0], args[1]) + return os.EX_OK + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:]))