AuroraMiddleware/v8
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

[test] Change wildcards to prefix rules.

Browse Source 
Test suite contract changes:
- support * only at the end of the rule.
- loading status file is mandatory before filtering by status file.

Bug: v8:6917
Change-Id: Ia345ebfa7827c50f13f20e5cb7489e62c53f3357
Reviewed-on: https://chromium-review.googlesource.com/779185
Commit-Queue: Michał Majewski <majeski@google.com>
Reviewed-by: Sergiy Byelozyorov <sergiyb@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#49496}
This commit is contained in:
Michal Majewski 2017-11-20 14:48:53 +01:00 committed by Commit Bot
parent 657e726a0d
commit 5bd4e2d893
4 changed files with 54 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
tools/testrunner/local/statusfile.py
View File

@ -129,7 +129,7 @@ def _EvalExpression(exp, variables):
return VARIANT_EXPRESSION
def _EvalVariantExpression(section, rules, wildcards, variant, variables):
def _EvalVariantExpression(section, rules, prefix_rules, variant, variables):
variables_with_variant = {}
variables_with_variant.update(variables)
variables_with_variant["variant"] = variant
@ -139,7 +139,7 @@ def _EvalVariantExpression(section, rules, wildcards, variant, variables):
_ReadSection(
section[1],
rules[variant],
wildcards[variant],
prefix_rules[variant],
variables_with_variant,
)
else:
@ -191,12 +191,12 @@ def ReadContent(content):
def ReadStatusFile(content, variables):
# Empty defaults for rules and wildcards. Variant-independent
# Empty defaults for rules and prefix_rules. Variant-independent
# rules are mapped by "", others by the variant name.
rules = {variant: {} for variant in ALL_VARIANTS}
rules[""] = {}
wildcards = {variant: {} for variant in ALL_VARIANTS}
wildcards[""] = {}
prefix_rules = {variant: {} for variant in ALL_VARIANTS}
prefix_rules[""] = {}
variables.update(VARIABLES)
for section in ReadContent(content):
@ -210,25 +210,30 @@ def ReadStatusFile(content, variables):
# If the expression contains one or more "variant" keywords, we evaluate
# it for all possible variants and create rules for those that apply.
for variant in ALL_VARIANTS:
_EvalVariantExpression(section, rules, wildcards, variant, variables)
_EvalVariantExpression(section, rules, prefix_rules, variant, variables)
else:
# The expression is variant-independent and evaluates to True.
assert exp is True, "Make sure expressions evaluate to boolean values"
_ReadSection(
section[1],
rules[""],
wildcards[""],
prefix_rules[""],
variables,
)
return Freeze(rules), Freeze(wildcards)
return Freeze(rules), Freeze(prefix_rules)
def _ReadSection(section, rules, wildcards, variables):
def _ReadSection(section, rules, prefix_rules, variables):
assert type(section) == dict
for rule in section:
assert type(rule) == str
# Wildcards are allowed only as the last character.
wildcards_count = rule.count('*')
assert wildcards_count == 0 or (wildcards_count == 1 and rule[-1] == '*')
if rule[-1] == '*':
_ParseOutcomeList(rule, section[rule], wildcards, variables)
_ParseOutcomeList(rule[:-1], section[rule], prefix_rules, variables)
else:
_ParseOutcomeList(rule, section[rule], rules, variables)

26
tools/testrunner/local/statusfile_unittest.py
View File

@ -87,7 +87,7 @@ class StatusFileTest(unittest.TestCase):
)
def test_read_statusfile_section_true(self):
rules, wildcards = statusfile.ReadStatusFile(
rules, prefix_rules = statusfile.ReadStatusFile(
TEST_STATUS_FILE % 'system==linux', make_variables())
self.assertEquals(
@ -99,15 +99,15 @@ class StatusFileTest(unittest.TestCase):
)
self.assertEquals(
{
'foo/*': set(['SLOW', 'FAIL']),
'foo/': set(['SLOW', 'FAIL']),
},
wildcards[''],
prefix_rules[''],
)
self.assertEquals({}, rules['default'])
self.assertEquals({}, wildcards['default'])
self.assertEquals({}, prefix_rules['default'])
def test_read_statusfile_section_false(self):
rules, wildcards = statusfile.ReadStatusFile(
rules, prefix_rules = statusfile.ReadStatusFile(
TEST_STATUS_FILE % 'system==windows', make_variables())
self.assertEquals(
@ -119,15 +119,15 @@ class StatusFileTest(unittest.TestCase):
)
self.assertEquals(
{
'foo/*': set(['PASS', 'SLOW']),
'foo/': set(['PASS', 'SLOW']),
},
wildcards[''],
prefix_rules[''],
)
self.assertEquals({}, rules['default'])
self.assertEquals({}, wildcards['default'])
self.assertEquals({}, prefix_rules['default'])
def test_read_statusfile_section_variant(self):
rules, wildcards = statusfile.ReadStatusFile(
rules, prefix_rules = statusfile.ReadStatusFile(
TEST_STATUS_FILE % 'system==linux and variant==default',
make_variables(),
)
@ -141,9 +141,9 @@ class StatusFileTest(unittest.TestCase):
)
self.assertEquals(
{
'foo/*': set(['PASS', 'SLOW']),
'foo/': set(['PASS', 'SLOW']),
},
wildcards[''],
prefix_rules[''],
)
self.assertEquals(
{
@ -153,9 +153,9 @@ class StatusFileTest(unittest.TestCase):
)
self.assertEquals(
{
'foo/*': set(['FAIL']),
'foo/': set(['FAIL']),
},
wildcards['default'],
prefix_rules['default'],
)

40
tools/testrunner/local/testsuite.py
View File

@ -86,8 +86,8 @@ class TestSuite(object):
self.name = name # string
self.root = root # string containing path
self.tests = None # list of TestCase objects
self.rules = None # dictionary mapping test path to list of outcomes
self.wildcards = None # dictionary mapping test paths to list of outcomes
self.rules = None # {variant: {test name: [rule]}}
self.prefix_rules = None # {variant: {test name prefix: [rule]}}
self.total_duration = None # float, assigned on demand
def suffix(self):
@ -130,7 +130,7 @@ class TestSuite(object):
def ReadStatusFile(self, variables):
with open(self.status_file()) as f:
self.rules, self.wildcards = (
self.rules, self.prefix_rules = (
statusfile.ReadStatusFile(f.read(), variables))
def ReadTestCases(self, context):
@ -148,18 +148,21 @@ class TestSuite(object):
slow_tests="dontcare",
pass_fail_tests="dontcare",
variants=False):
# Load statusfile before.
assert(self.rules is not None)
assert(self.prefix_rules is not None)
# Use only variants-dependent rules and wildcards when filtering
# Use only variants-dependent rules and prefix_rules when filtering
# respective test cases and generic rules when filtering generic test
# cases.
if not variants:
rules = self.rules[""]
wildcards = self.wildcards[""]
prefix_rules = self.prefix_rules[""]
else:
# We set rules and wildcards to a variant-specific version for each test
# below.
# We set rules and prefix_rules to a variant-specific version for each
# test below.
rules = {}
wildcards = {}
prefix_rules = {}
filtered = []
@ -174,7 +177,7 @@ class TestSuite(object):
variant = t.variant or ""
if variants:
rules = self.rules[variant]
wildcards = self.wildcards[variant]
prefix_rules = self.prefix_rules[variant]
if testname in rules:
used_rules.add((testname, variant))
# Even for skipped tests, as the TestCase object stays around and
@ -188,14 +191,13 @@ class TestSuite(object):
slow = statusfile.IsSlow(t.outcomes)
pass_fail = statusfile.IsPassOrFail(t.outcomes)
skip = False
for rule in wildcards:
assert rule[-1] == '*'
if testname.startswith(rule[:-1]):
used_rules.add((rule, variant))
t.outcomes = t.outcomes | wildcards[rule]
for prefix in prefix_rules:
if testname.startswith(prefix):
used_rules.add((prefix, variant))
t.outcomes = t.outcomes | prefix_rules[prefix]
if statusfile.DoSkip(t.outcomes):
skip = True
break # "for rule in wildcards"
break # "for rule in prefix_rules"
slow = slow or statusfile.IsSlow(t.outcomes)
pass_fail = pass_fail or statusfile.IsPassOrFail(t.outcomes)
if (skip
@ -213,20 +215,20 @@ class TestSuite(object):
if (rule, "") not in used_rules:
print("Unused rule: %s -> %s (variant independent)" % (
rule, self.rules[""][rule]))
for rule in self.wildcards[""]:
for rule in self.prefix_rules[""]:
if (rule, "") not in used_rules:
print("Unused rule: %s -> %s (variant independent)" % (
rule, self.wildcards[""][rule]))
rule, self.prefix_rules[""][rule]))
else:
for variant in ALL_VARIANTS:
for rule in self.rules[variant]:
if (rule, variant) not in used_rules:
print("Unused rule: %s -> %s (variant: %s)" % (
rule, self.rules[variant][rule], variant))
for rule in self.wildcards[variant]:
for rule in self.prefix_rules[variant]:
if (rule, variant) not in used_rules:
print("Unused rule: %s -> %s (variant: %s)" % (
rule, self.wildcards[variant][rule], variant))
rule, self.prefix_rules[variant][rule], variant))
def FilterTestCasesByArgs(self, args):

10
tools/testrunner/local/testsuite_unittest.py
View File

@ -29,9 +29,9 @@ class TestSuiteTest(unittest.TestCase):
'baz/bar': set(['PASS', 'FAIL']),
},
}
suite.wildcards = {
suite.prefix_rules = {
'': {
'baz/*': set(['PASS', 'SLOW']),
'baz/': set(['PASS', 'SLOW']),
},
}
suite.FilterTestCasesByStatus(warn_unused_rules=False)
@ -67,12 +67,12 @@ class TestSuiteTest(unittest.TestCase):
'baz/bar': set(['SKIP']),
},
}
suite.wildcards = {
suite.prefix_rules = {
'default': {
'baz/*': set(['PASS', 'SLOW']),
'baz/': set(['PASS', 'SLOW']),
},
'stress': {
'foo/*': set(['PASS', 'SLOW']),
'foo/': set(['PASS', 'SLOW']),
},
}
suite.FilterTestCasesByStatus(warn_unused_rules=False, variants=True)