v8/tools/testrunner/local/statusfile.py

# Copyright 2012 the V8 project authors. All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#     * Neither the name of Google Inc. nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# for py2/py3 compatibility
from __future__ import print_function

import os
import re

from variants import ALL_VARIANTS
from utils import Freeze

# Possible outcomes
FAIL = "FAIL"
PASS = "PASS"
TIMEOUT = "TIMEOUT"
CRASH = "CRASH"

# Outcomes only for status file, need special handling
FAIL_OK = "FAIL_OK"
FAIL_SLOPPY = "FAIL_SLOPPY"

# Modifiers
SKIP = "SKIP"
SLOW = "SLOW"
NO_VARIANTS = "NO_VARIANTS"
FAIL_PHASE_ONLY = "FAIL_PHASE_ONLY"

ALWAYS = "ALWAYS"

KEYWORDS = {}
for key in [SKIP, FAIL, PASS, CRASH, SLOW, FAIL_OK, NO_VARIANTS, FAIL_SLOPPY,
            ALWAYS, FAIL_PHASE_ONLY]:
  KEYWORDS[key] = key

# Support arches, modes to be written as keywords instead of strings.
VARIABLES = {ALWAYS: True}
for var in ["debug", "release", "big", "little", "android",
            "android_arm", "android_arm64", "android_ia32", "android_x64",
            "arm", "arm64", "ia32", "mips", "mipsel", "mips64", "mips64el",
            "x64", "ppc", "ppc64", "s390", "s390x", "macos", "windows",
            "linux", "aix", "r1", "r2", "r3", "r5", "r6"]:
  VARIABLES[var] = var

# Allow using variants as keywords.
for var in ALL_VARIANTS:
  VARIABLES[var] = var

class StatusFile(object):
  def __init__(self, path, variables):
    """
    _rules:        {variant: {test name: [rule]}}
    _prefix_rules: {variant: {test name prefix: [rule]}}
    """
    with open(path) as f:
      self._rules, self._prefix_rules = ReadStatusFile(f.read(), variables)

  def get_outcomes(self, testname, variant=None):
    """Merges variant dependent and independent rules."""
    outcomes = frozenset()

    for key in set([variant or '', '']):
      rules = self._rules.get(key, {})
      prefix_rules = self._prefix_rules.get(key, {})

      if testname in rules:
        outcomes |= rules[testname]

      for prefix in prefix_rules:
        if testname.startswith(prefix):
          outcomes |= prefix_rules[prefix]

    return outcomes

  def warn_unused_rules(self, tests, check_variant_rules=False):
    """Finds and prints unused rules in status file.

    Rule X is unused when it doesn't apply to any tests, which can also mean
    that all matching tests were skipped by another rule before evaluating X.

    Args:
      tests: list of pairs (testname, variant)
      check_variant_rules: if set variant dependent rules are checked
    """

    if check_variant_rules:
      variants = list(ALL_VARIANTS)
    else:
      variants = ['']
    used_rules = set()

    for testname, variant in tests:
      variant = variant or ''

      if testname in self._rules.get(variant, {}):
        used_rules.add((testname, variant))
        if SKIP in self._rules[variant][testname]:
          continue

      for prefix in self._prefix_rules.get(variant, {}):
        if testname.startswith(prefix):
          used_rules.add((prefix, variant))
          if SKIP in self._prefix_rules[variant][prefix]:
            break

    for variant in variants:
      for rule, value in (
          list(self._rules.get(variant, {}).iteritems()) +
          list(self._prefix_rules.get(variant, {}).iteritems())):
        if (rule, variant) not in used_rules:
          if variant == '':
            variant_desc = 'variant independent'
          else:
            variant_desc = 'variant: %s' % variant
          print('Unused rule: %s -> %s (%s)' % (rule, value, variant_desc))


def _JoinsPassAndFail(outcomes1, outcomes2):
  """Indicates if we join PASS and FAIL from two different outcome sets and
  the first doesn't already contain both.
  """
  return (
      PASS in outcomes1 and
      not (FAIL in outcomes1 or FAIL_OK in outcomes1) and
      (FAIL in outcomes2 or FAIL_OK in outcomes2)
  )

VARIANT_EXPRESSION = object()

def _EvalExpression(exp, variables):
  """Evaluates expression and returns its result. In case of NameError caused by
  undefined "variant" identifier returns VARIANT_EXPRESSION marker.
  """

  try:
    return eval(exp, variables)
  except NameError as e:
    identifier = re.match("name '(.*)' is not defined", e.message).group(1)
    assert identifier == "variant", "Unknown identifier: %s" % identifier
    return VARIANT_EXPRESSION


def _EvalVariantExpression(
  condition, section, variables, variant, rules, prefix_rules):
  variables_with_variant = dict(variables)
  variables_with_variant["variant"] = variant
  result = _EvalExpression(condition, variables_with_variant)
  assert result != VARIANT_EXPRESSION
  if result is True:
    _ReadSection(
        section,
        variables_with_variant,
        rules[variant],
        prefix_rules[variant],
    )
  else:
    assert result is False, "Make sure expressions evaluate to boolean values"


def _ParseOutcomeList(rule, outcomes, variables, target_dict):
  """Outcome list format: [condition, outcome, outcome, ...]"""

  result = set([])
  if type(outcomes) == str:
    outcomes = [outcomes]
  for item in outcomes:
    if type(item) == str:
      result.add(item)
    elif type(item) == list:
      condition = item[0]
      exp = _EvalExpression(condition, variables)
      assert exp != VARIANT_EXPRESSION, (
        "Nested variant expressions are not supported")
      if exp is False:
        continue

      # Ensure nobody uses an identifier by mistake, like "default",
      # which would evaluate to true here otherwise.
      assert exp is True, "Make sure expressions evaluate to boolean values"

      for outcome in item[1:]:
        assert type(outcome) == str
        result.add(outcome)
    else:
      assert False
  if len(result) == 0:
    return
  if rule in target_dict:
    # A FAIL without PASS in one rule has always precedence over a single
    # PASS (without FAIL) in another. Otherwise the default PASS expectation
    # in a rule with a modifier (e.g. PASS, SLOW) would be joined to a FAIL
    # from another rule (which intended to mark a test as FAIL and not as
    # PASS and FAIL).
    if _JoinsPassAndFail(target_dict[rule], result):
      target_dict[rule] -= set([PASS])
    if _JoinsPassAndFail(result, target_dict[rule]):
      result -= set([PASS])
    target_dict[rule] |= result
  else:
    target_dict[rule] = result


def ReadContent(content):
  return eval(content, KEYWORDS)


def ReadStatusFile(content, variables):
  """Status file format
  Status file := [section]
  section = [CONDITION, section_rules]
  section_rules := {path: outcomes}
  outcomes := outcome | [outcome, ...]
  outcome := SINGLE_OUTCOME | [CONDITION, SINGLE_OUTCOME, SINGLE_OUTCOME, ...]
  """

  # Empty defaults for rules and prefix_rules. Variant-independent
  # rules are mapped by "", others by the variant name.
  rules = {variant: {} for variant in ALL_VARIANTS}
  rules[""] = {}
  prefix_rules = {variant: {} for variant in ALL_VARIANTS}
  prefix_rules[""] = {}

  variables.update(VARIABLES)
  for conditional_section in ReadContent(content):
    assert type(conditional_section) == list
    assert len(conditional_section) == 2
    condition, section = conditional_section
    exp = _EvalExpression(condition, variables)

    # The expression is variant-independent and evaluates to False.
    if exp is False:
      continue

    # The expression is variant-independent and evaluates to True.
    if exp is True:
      _ReadSection(
          section,
          variables,
          rules[''],
          prefix_rules[''],
      )
      continue

    # The expression is variant-dependent (contains "variant" keyword)
    if exp == VARIANT_EXPRESSION:
      # If the expression contains one or more "variant" keywords, we evaluate
      # it for all possible variants and create rules for those that apply.
      for variant in ALL_VARIANTS:
        _EvalVariantExpression(
            condition, section, variables, variant, rules, prefix_rules)
      continue

    assert False, "Make sure expressions evaluate to boolean values"

  return Freeze(rules), Freeze(prefix_rules)


def _ReadSection(section, variables, rules, prefix_rules):
  assert type(section) == dict
  for rule, outcome_list in section.iteritems():
    assert type(rule) == str

    if rule[-1] == '*':
      _ParseOutcomeList(rule[:-1], outcome_list, variables, prefix_rules)
    else:
      _ParseOutcomeList(rule, outcome_list, variables, rules)

JS_TEST_PATHS = {
  'debugger': [[]],
  'inspector': [[]],
  'intl': [[]],
  'message': [[]],
  'mjsunit': [[]],
  'mozilla': [['data']],
  'test262': [['data', 'test'], ['local-tests', 'test']],
  'webkit': [[]],
}

def PresubmitCheck(path):
  with open(path) as f:
    contents = ReadContent(f.read())
  basename = os.path.basename(os.path.dirname(path))
  root_prefix = basename + "/"
  status = {"success": True}
  def _assert(check, message):  # Like "assert", but doesn't throw.
    if not check:
      print("%s: Error: %s" % (path, message))
      status["success"] = False
  try:
    for section in contents:
      _assert(type(section) == list, "Section must be a list")
      _assert(len(section) == 2, "Section list must have exactly 2 entries")
      section = section[1]
      _assert(type(section) == dict,
              "Second entry of section must be a dictionary")
      for rule in section:
        _assert(type(rule) == str, "Rule key must be a string")
        _assert(not rule.startswith(root_prefix),
                "Suite name prefix must not be used in rule keys")
        _assert(not rule.endswith('.js'),
                ".js extension must not be used in rule keys.")
        _assert('*' not in rule or (rule.count('*') == 1 and rule[-1] == '*'),
                "Only the last character of a rule key can be a wildcard")
        if basename in JS_TEST_PATHS  and '*' not in rule:
          _assert(any(os.path.exists(os.path.join(os.path.dirname(path),
                                                  *(paths + [rule + ".js"])))
                      for paths in JS_TEST_PATHS[basename]),
                  "missing file for %s test %s" % (basename, rule))
    return status["success"]
  except Exception as e:
    print(e)
    return False