diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 9d011db55..96599bd53 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -7,9 +7,12 @@ less likely to be useful. """ import argparse +import re import sys import traceback +import check_test_cases + class Results: """Process analysis results.""" @@ -44,9 +47,40 @@ class TestCaseOutcomes: """ return len(self.successes) + len(self.failures) +class TestDescriptions(check_test_cases.TestDescriptionExplorer): + """Collect the available test cases.""" + + def __init__(self): + super().__init__() + self.descriptions = set() + + def process_test_case(self, _per_file_state, + file_name, _line_number, description): + """Record an available test case.""" + base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name)) + key = ';'.join([base_name, description.decode('utf-8')]) + self.descriptions.add(key) + +def collect_available_test_cases(): + """Collect the available test cases.""" + explorer = TestDescriptions() + explorer.walk_all() + return sorted(explorer.descriptions) + +def analyze_coverage(results, outcomes): + """Check that all available test cases are executed at least once.""" + available = collect_available_test_cases() + for key in available: + hits = outcomes[key].hits() if key in outcomes else 0 + if hits == 0: + # Make this a warning, not an error, as long as we haven't + # fixed this branch to have full coverage of test cases. + results.warning('Test case not executed: {}', key) + def analyze_outcomes(outcomes): """Run all analyses on the given outcome collection.""" results = Results() + analyze_coverage(results, outcomes) return results def read_outcome_file(outcome_file):