v8/tools/testrunner/local/testsuite.py

# Copyright 2012 the V8 project authors. All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#     * Neither the name of Google Inc. nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


import imp
import os

from . import statusfile
from . import utils

class TestSuite(object):

  @staticmethod
  def LoadTestSuite(root):
    name = root.split(os.path.sep)[-1]
    f = None
    try:
      (f, pathname, description) = imp.find_module("testcfg", [root])
      module = imp.load_module("testcfg", f, pathname, description)
      suite = module.GetSuite(name, root)
    finally:
      if f:
        f.close()
    return suite

  def __init__(self, name, root):
    self.name = name  # string
    self.root = root  # string containing path
    self.tests = None  # list of TestCase objects
    self.rules = None  # dictionary mapping test path to list of outcomes
    self.wildcards = None  # dictionary mapping test paths to list of outcomes
    self.total_duration = None  # float, assigned on demand

  def shell(self):
    return "d8"

  def suffix(self):
    return ".js"

  def status_file(self):
    return "%s/%s.status" % (self.root, self.name)

  # Used in the status file and for stdout printing.
  def CommonTestName(self, testcase):
    return testcase.path

  def ListTests(self, context):
    raise NotImplementedError

  def VariantFlags(self):
    return None

  def DownloadData(self):
    pass

  def ReadStatusFile(self, variables):
    (self.rules, self.wildcards) = \
        statusfile.ReadStatusFile(self.status_file(), variables)

  def ReadTestCases(self, context):
    self.tests = self.ListTests(context)

  def FilterTestCasesByStatus(self, warn_unused_rules):
    filtered = []
    used_rules = set()
    for t in self.tests:
      testname = self.CommonTestName(t)
      if utils.IsWindows():
        testname = testname.replace("\\", "/")
      if testname in self.rules:
        used_rules.add(testname)
        outcomes = self.rules[testname]
        t.outcomes = outcomes  # Even for skipped tests, as the TestCase
        # object stays around and PrintReport() uses it.
        if statusfile.DoSkip(outcomes):
          continue  # Don't add skipped tests to |filtered|.
      if len(self.wildcards) != 0:
        skip = False
        for rule in self.wildcards:
          assert rule[-1] == '*'
          if testname.startswith(rule[:-1]):
            used_rules.add(rule)
            outcomes = self.wildcards[rule]
            t.outcomes = outcomes
            if statusfile.DoSkip(outcomes):
              skip = True
              break  # "for rule in self.wildcards"
        if skip: continue  # "for t in self.tests"
      filtered.append(t)
    self.tests = filtered

    if not warn_unused_rules:
      return

    for rule in self.rules:
      if rule not in used_rules:
        print("Unused rule: %s -> %s" % (rule, self.rules[rule]))
    for rule in self.wildcards:
      if rule not in used_rules:
        print("Unused rule: %s -> %s" % (rule, self.wildcards[rule]))

  def FilterTestCasesByArgs(self, args):
    filtered = []
    filtered_args = []
    for a in args:
      argpath = a.split(os.path.sep)
      if argpath[0] != self.name:
        continue
      if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'):
        return  # Don't filter, run all tests in this suite.
      path = os.path.sep.join(argpath[1:])
      if path[-1] == '*':
        path = path[:-1]
      filtered_args.append(path)
    for t in self.tests:
      for a in filtered_args:
        if t.path.startswith(a):
          filtered.append(t)
          break
    self.tests = filtered

  def GetFlagsForTestCase(self, testcase, context):
    raise NotImplementedError

  def GetSourceForTest(self, testcase):
    return "(no source available)"

  def IsFailureOutput(self, output, testpath):
    return output.exit_code != 0

  def IsNegativeTest(self, testcase):
    return False

  def HasFailed(self, testcase):
    execution_failed = self.IsFailureOutput(testcase.output, testcase.path)
    if self.IsNegativeTest(testcase):
      return not execution_failed
    else:
      return execution_failed

  def HasUnexpectedOutput(self, testcase):
    if testcase.output.HasCrashed():
      outcome = statusfile.CRASH
    elif testcase.output.HasTimedOut():
      outcome = statusfile.TIMEOUT
    elif self.HasFailed(testcase):
      outcome = statusfile.FAIL
    else:
      outcome = statusfile.PASS
    if not testcase.outcomes:
      return outcome != statusfile.PASS
    return not outcome in testcase.outcomes

  def StripOutputForTransmit(self, testcase):
    if not self.HasUnexpectedOutput(testcase):
      testcase.output.stdout = ""
      testcase.output.stderr = ""

  def CalculateTotalDuration(self):
    self.total_duration = 0.0
    for t in self.tests:
      self.total_duration += t.duration
    return self.total_duration