diff --git a/gm/rebaseline_server/compare_configs.py b/gm/rebaseline_server/compare_configs.py index 8f92551559..ba256cab60 100755 --- a/gm/rebaseline_server/compare_configs.py +++ b/gm/rebaseline_server/compare_configs.py @@ -47,12 +47,12 @@ class ConfigComparisons(results.BaseComparisons): """Loads results from two different configurations into an ImagePairSet. Loads actual and expected results from all builders, except for those skipped - by BaseComparisons._ignore_builder(). + by _ignore_builder(). """ def __init__(self, configs, actuals_root=results.DEFAULT_ACTUALS_DIR, generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT, - diff_base_url=None): + diff_base_url=None, builder_regex_list=None): """ Args: configs: (string, string) tuple; pair of configs to compare @@ -62,8 +62,12 @@ class ConfigComparisons(results.BaseComparisons): diff_base_url: base URL within which the client should look for diff images; if not specified, defaults to a "file:///" URL representation of generated_images_root + builder_regex_list: List of regular expressions specifying which builders + we will process. If None, process all builders. """ time_start = int(time.time()) + if builder_regex_list != None: + self.set_match_builders_pattern_list(builder_regex_list) self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) self._diff_base_url = ( diff_base_url or @@ -84,8 +88,7 @@ class ConfigComparisons(results.BaseComparisons): """ logging.info('Reading actual-results JSON files from %s...' % self._actuals_root) - actual_builder_dicts = ConfigComparisons._read_dicts_from_root( - self._actuals_root) + actual_builder_dicts = self._read_dicts_from_root(self._actuals_root) configA, configB = configs logging.info('Comparing configs %s and %s...' % (configA, configB)) diff --git a/gm/rebaseline_server/compare_rendered_pictures.py b/gm/rebaseline_server/compare_rendered_pictures.py index 14b1fb1d80..80a42e51f9 100755 --- a/gm/rebaseline_server/compare_rendered_pictures.py +++ b/gm/rebaseline_server/compare_rendered_pictures.py @@ -97,9 +97,9 @@ class RenderedPicturesComparisons(results.BaseComparisons): 'Reading actual-results JSON files from %s subdirs within %s...' % ( subdirs, actuals_root)) subdirA, subdirB = subdirs - subdirA_builder_dicts = results.BaseComparisons._read_dicts_from_root( + subdirA_builder_dicts = self._read_dicts_from_root( os.path.join(actuals_root, subdirA)) - subdirB_builder_dicts = results.BaseComparisons._read_dicts_from_root( + subdirB_builder_dicts = self._read_dicts_from_root( os.path.join(actuals_root, subdirB)) logging.info('Comparing subdirs %s and %s...' % (subdirA, subdirB)) diff --git a/gm/rebaseline_server/compare_to_expectations.py b/gm/rebaseline_server/compare_to_expectations.py index c8510d6595..2389b61dad 100755 --- a/gm/rebaseline_server/compare_to_expectations.py +++ b/gm/rebaseline_server/compare_to_expectations.py @@ -65,7 +65,7 @@ class ExpectationComparisons(results.BaseComparisons): def __init__(self, actuals_root=results.DEFAULT_ACTUALS_DIR, expected_root=DEFAULT_EXPECTATIONS_DIR, generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT, - diff_base_url=None): + diff_base_url=None, builder_regex_list=None): """ Args: actuals_root: root directory containing all actual-results.json files @@ -75,8 +75,12 @@ class ExpectationComparisons(results.BaseComparisons): diff_base_url: base URL within which the client should look for diff images; if not specified, defaults to a "file:///" URL representation of generated_images_root + builder_regex_list: List of regular expressions specifying which builders + we will process. If None, process all builders. """ time_start = int(time.time()) + if builder_regex_list != None: + self.set_match_builders_pattern_list(builder_regex_list) self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) self._diff_base_url = ( diff_base_url or @@ -117,8 +121,7 @@ class ExpectationComparisons(results.BaseComparisons): ] """ - expected_builder_dicts = ExpectationComparisons._read_dicts_from_root( - self._expected_root) + expected_builder_dicts = self._read_dicts_from_root(self._expected_root) for mod in modifications: image_name = results.IMAGE_FILENAME_FORMATTER % ( mod[imagepair.KEY__EXTRA_COLUMN_VALUES] @@ -174,8 +177,6 @@ class ExpectationComparisons(results.BaseComparisons): for dirpath, dirnames, filenames in os.walk(root): for matching_filename in fnmatch.filter(filenames, pattern): builder = os.path.basename(dirpath) - if ExpectationComparisons._ignore_builder(builder): - continue per_builder_dict = meta_dict.get(builder) if per_builder_dict is not None: fullpath = os.path.join(dirpath, matching_filename) @@ -199,12 +200,10 @@ class ExpectationComparisons(results.BaseComparisons): """ logging.info('Reading actual-results JSON files from %s...' % self._actuals_root) - actual_builder_dicts = ExpectationComparisons._read_dicts_from_root( - self._actuals_root) + actual_builder_dicts = self._read_dicts_from_root(self._actuals_root) logging.info('Reading expected-results JSON files from %s...' % self._expected_root) - expected_builder_dicts = ExpectationComparisons._read_dicts_from_root( - self._expected_root) + expected_builder_dicts = self._read_dicts_from_root(self._expected_root) all_image_pairs = imagepairset.ImagePairSet( descriptions=IMAGEPAIR_SET_DESCRIPTIONS, diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/results.py index 49e32519a7..255dfa31e2 100755 --- a/gm/rebaseline_server/results.py +++ b/gm/rebaseline_server/results.py @@ -59,26 +59,24 @@ KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config) -# Ignore expectations/actuals for builders matching any of these patterns. +DEFAULT_ACTUALS_DIR = '.gm-actuals' +DEFAULT_GENERATED_IMAGES_ROOT = os.path.join( + PARENT_DIRECTORY, '.generated-images') + +# Define the default set of builders we will process expectations/actuals for. # This allows us to ignore builders for which we don't maintain expectations # (trybots, Valgrind, ASAN, TSAN), and avoid problems like # https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server # produces error when trying to add baselines for ASAN/TSAN builders') -SKIP_BUILDERS_PATTERN_LIST = [re.compile(p) for p in [ - '.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*']] - -DEFAULT_ACTUALS_DIR = '.gm-actuals' -DEFAULT_GENERATED_IMAGES_ROOT = os.path.join( - PARENT_DIRECTORY, '.generated-images') +DEFAULT_MATCH_BUILDERS_PATTERN_LIST = ['.*'] +DEFAULT_SKIP_BUILDERS_PATTERN_LIST = [ + '.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*'] class BaseComparisons(object): """Base class for generating summary of comparisons between two image sets. """ - def __init__(self): - raise NotImplementedError('cannot instantiate the abstract base class') - def get_results_of_type(self, results_type): """Return results of some/all tests (depending on 'results_type' parameter). @@ -138,9 +136,45 @@ class BaseComparisons(object): """ return self._timestamp - @staticmethod - def _ignore_builder(builder): - """Returns True if this builder matches any of SKIP_BUILDERS_PATTERN_LIST. + _match_builders_pattern_list = [ + re.compile(p) for p in DEFAULT_MATCH_BUILDERS_PATTERN_LIST] + _skip_builders_pattern_list = [ + re.compile(p) for p in DEFAULT_SKIP_BUILDERS_PATTERN_LIST] + + def set_match_builders_pattern_list(self, pattern_list): + """Override the default set of builders we should process. + + The default is DEFAULT_MATCH_BUILDERS_PATTERN_LIST . + + Note that skip_builders_pattern_list overrides this; regardless of whether a + builder is in the "match" list, if it's in the "skip" list, we will skip it. + + Args: + pattern_list: list of regex patterns; process builders that match any + entry within this list + """ + if pattern_list == None: + pattern_list = [] + self._match_builders_pattern_list = [re.compile(p) for p in pattern_list] + + def set_skip_builders_pattern_list(self, pattern_list): + """Override the default set of builders we should skip while processing. + + The default is DEFAULT_SKIP_BUILDERS_PATTERN_LIST . + + This overrides match_builders_pattern_list; regardless of whether a + builder is in the "match" list, if it's in the "skip" list, we will skip it. + + Args: + pattern_list: list of regex patterns; skip builders that match any + entry within this list + """ + if pattern_list == None: + pattern_list = [] + self._skip_builders_pattern_list = [re.compile(p) for p in pattern_list] + + def _ignore_builder(self, builder): + """Returns True if we should skip processing this builder. Args: builder: name of this builder, as a string @@ -148,13 +182,15 @@ class BaseComparisons(object): Returns: True if we should ignore expectations and actuals for this builder. """ - for pattern in SKIP_BUILDERS_PATTERN_LIST: + for pattern in self._skip_builders_pattern_list: if pattern.match(builder): return True - return False + for pattern in self._match_builders_pattern_list: + if pattern.match(builder): + return False + return True - @staticmethod - def _read_dicts_from_root(root, pattern='*.json'): + def _read_dicts_from_root(self, root, pattern='*.json'): """Read all JSON dictionaries within a directory tree. Args: @@ -174,7 +210,7 @@ class BaseComparisons(object): for dirpath, dirnames, filenames in os.walk(root): for matching_filename in fnmatch.filter(filenames, pattern): builder = os.path.basename(dirpath) - if BaseComparisons._ignore_builder(builder): + if self._ignore_builder(builder): continue fullpath = os.path.join(dirpath, matching_filename) meta_dict[builder] = gm_json.LoadFromFile(fullpath) diff --git a/gm/rebaseline_server/results_test.py b/gm/rebaseline_server/results_test.py index a2f4073dcf..f22e833fe3 100755 --- a/gm/rebaseline_server/results_test.py +++ b/gm/rebaseline_server/results_test.py @@ -17,6 +17,29 @@ import results class ResultsTest(base_unittest.TestCase): + def test_ignore_builder(self): + """Test _ignore_builder().""" + results_obj = results.BaseComparisons() + self.assertEqual(results_obj._ignore_builder('SomethingTSAN'), True) + self.assertEqual(results_obj._ignore_builder('Something-Trybot'), True) + self.assertEqual(results_obj._ignore_builder( + 'Test-Ubuntu12-ShuttleA-GTX660-x86-Release'), False) + results_obj.set_skip_builders_pattern_list(['.*TSAN.*', '.*GTX660.*']) + self.assertEqual(results_obj._ignore_builder('SomethingTSAN'), True) + self.assertEqual(results_obj._ignore_builder('Something-Trybot'), False) + self.assertEqual(results_obj._ignore_builder( + 'Test-Ubuntu12-ShuttleA-GTX660-x86-Release'), True) + results_obj.set_skip_builders_pattern_list(None) + self.assertEqual(results_obj._ignore_builder('SomethingTSAN'), False) + self.assertEqual(results_obj._ignore_builder('Something-Trybot'), False) + self.assertEqual(results_obj._ignore_builder( + 'Test-Ubuntu12-ShuttleA-GTX660-x86-Release'), False) + results_obj.set_match_builders_pattern_list(['.*TSAN']) + self.assertEqual(results_obj._ignore_builder('SomethingTSAN'), False) + self.assertEqual(results_obj._ignore_builder('Something-Trybot'), True) + self.assertEqual(results_obj._ignore_builder( + 'Test-Ubuntu12-ShuttleA-GTX660-x86-Release'), True) + def test_combine_subdicts_typical(self): """Test combine_subdicts() with no merge conflicts. """ input_dict = { diff --git a/gm/rebaseline_server/server.py b/gm/rebaseline_server/server.py index 04620f51ca..73cfbef21e 100755 --- a/gm/rebaseline_server/server.py +++ b/gm/rebaseline_server/server.py @@ -216,7 +216,7 @@ class Server(object): actuals_repo_revision=DEFAULT_ACTUALS_REPO_REVISION, actuals_repo_url=DEFAULT_ACTUALS_REPO_URL, port=DEFAULT_PORT, export=False, editable=True, - reload_seconds=0, config_pairs=None): + reload_seconds=0, config_pairs=None, builder_regex_list=None): """ Args: actuals_dir: directory under which we will check out the latest actual @@ -233,6 +233,8 @@ class Server(object): config_pairs: List of (string, string) tuples; for each tuple, compare actual results of these two configs. If None or empty, don't compare configs at all. + builder_regex_list: List of regular expressions specifying which builders + we will process. If None, process all builders. """ self._actuals_dir = actuals_dir self._actuals_repo_revision = actuals_repo_revision @@ -242,6 +244,7 @@ class Server(object): self._editable = editable self._reload_seconds = reload_seconds self._config_pairs = config_pairs or [] + self._builder_regex_list = builder_regex_list _create_index( file_path=os.path.join( PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_HTML_SUBDIR, @@ -329,7 +332,8 @@ class Server(object): PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_IMAGES_SUBDIR), diff_base_url=posixpath.join( - os.pardir, STATIC_CONTENTS_SUBDIR, GENERATED_IMAGES_SUBDIR)) + os.pardir, STATIC_CONTENTS_SUBDIR, GENERATED_IMAGES_SUBDIR), + builder_regex_list=self._builder_regex_list) json_dir = os.path.join( PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_JSON_SUBDIR) @@ -344,7 +348,8 @@ class Server(object): PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_IMAGES_SUBDIR), diff_base_url=posixpath.join( - os.pardir, GENERATED_IMAGES_SUBDIR)) + os.pardir, GENERATED_IMAGES_SUBDIR), + builder_regex_list=self._builder_regex_list) for summary_type in SUMMARY_TYPES: gm_json.WriteToFile( config_comparisons.get_packaged_results_of_type( @@ -627,6 +632,10 @@ def main(): 'argument in conjunction with --editable; you ' 'probably only want to edit results at HEAD.'), default=DEFAULT_ACTUALS_REPO_REVISION) + parser.add_argument('--builders', metavar='BUILDER_REGEX', nargs='+', + help=('Only process builders matching these regular ' + 'expressions. If unspecified, process all ' + 'builders.')) parser.add_argument('--compare-configs', action='store_true', help=('In addition to generating differences between ' 'expectations and actuals, also generate ' @@ -663,7 +672,8 @@ def main(): actuals_repo_revision=args.actuals_revision, actuals_repo_url=args.actuals_repo, port=args.port, export=args.export, editable=args.editable, - reload_seconds=args.reload, config_pairs=config_pairs) + reload_seconds=args.reload, config_pairs=config_pairs, + builder_regex_list=args.builders) _SERVER.run()