skia2/infra/bots/gen_compile_isolate.py

234 lines
6.7 KiB
Python
Raw Normal View History

#!/usr/bin/env python
#
# Copyright 2019 Google LLC
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import difflib
import os
import re
import subprocess
import sys
# Any files in Git which match these patterns will be included, either directly
# or indirectly via a parent dir.
PATH_PATTERNS = [
r'.*\.c$',
r'.*\.cc$',
r'.*\.cpp$',
r'.*\.gn$',
r'.*\.gni$',
r'.*\.h$',
]
# These paths are always added to the inclusion list. Note that they may not
# appear in the isolate if they are included indirectly via a parent dir.
EXPLICIT_PATHS = [
'../.gclient',
'.clang-format',
'.clang-tidy',
'bin/fetch-clang-format',
'bin/fetch-gn',
'buildtools',
'infra/bots/assets/android_ndk_darwin/VERSION',
'infra/bots/assets/android_ndk_linux/VERSION',
'infra/bots/assets/android_ndk_windows/VERSION',
'infra/bots/assets/cast_toolchain/VERSION',
'infra/bots/assets/clang_linux/VERSION',
'infra/bots/assets/clang_win/VERSION',
'infra/bots/assets/mips64el_toolchain_linux/VERSION',
'infra/canvaskit',
'infra/pathkit',
'resources',
'third_party/externals',
]
# If a parent path contains more than this many immediate child paths (ie. files
# and dirs which are directly inside it as opposed to indirect descendants), we
# will include the parent in the isolate file instead of the children. This
# results in a simpler isolate file which should need to be changed less often.
COMBINE_PATHS_THRESHOLD = 3
# Template for the isolate file content.
ISOLATE_TMPL = '''{
'includes': [
'run_recipe.isolate',
],
'variables': {
'files': [
%s
],
},
}
'''
# Absolute path to the infra/bots dir.
INFRABOTS_DIR = os.path.realpath(os.path.dirname(os.path.abspath(__file__)))
# Absolute path to the compile.isolate file.
ISOLATE_FILE = os.path.join(INFRABOTS_DIR, 'compile.isolate')
def all_paths():
"""Return all paths which are checked in to git."""
repo_root = os.path.abspath(os.path.join(INFRABOTS_DIR, os.pardir, os.pardir))
output = subprocess.check_output(['git', 'ls-files'], cwd=repo_root).rstrip()
return output.splitlines()
def get_relevant_paths():
"""Return all checked-in paths in PATH_PATTERNS or EXPLICIT_PATHS."""
paths = []
for f in all_paths():
for regexp in PATH_PATTERNS:
if re.match(regexp, f):
paths.append(f)
break
paths.extend(EXPLICIT_PATHS)
return paths
class Tree(object):
"""Tree helps with deduplicating and collapsing paths."""
class Node(object):
"""Node represents an individual node in a Tree."""
def __init__(self, name):
self._children = {}
self._name = name
self._is_leaf = False
@property
def is_root(self):
"""Return True iff this is the root node."""
return self._name is None
def add(self, entry):
"""Add the given entry (given as a list of strings) to the Node."""
# Remove the first element if we're not the root node.
if not self.is_root:
if entry[0] != self._name:
raise ValueError('Cannot add a non-matching entry to a Node!')
entry = entry[1:]
# If the entry is now empty, this node is a leaf.
if not entry:
self._is_leaf = True
return
# Add a child node.
if not self._is_leaf:
child = self._children.get(entry[0])
if not child:
child = Tree.Node(entry[0])
self._children[entry[0]] = child
child.add(entry)
# If we have more than COMBINE_PATHS_THRESHOLD immediate children,
# combine them into this node.
immediate_children = 0
for child in self._children.itervalues():
if child._is_leaf:
immediate_children += 1
if not self.is_root and immediate_children >= COMBINE_PATHS_THRESHOLD:
self._is_leaf = True
self._children = {}
def entries(self):
"""Return the entries represented by this node and its children.
Will not return children in the following cases:
- This Node is a leaf, ie. it represents an entry which was explicitly
inserted into the Tree, as opposed to only part of a path to other
entries.
- This Node has immediate children exceeding COMBINE_PATHS_THRESHOLD and
thus has been upgraded to a leaf node.
"""
if self._is_leaf:
return [self._name]
rv = []
for child in self._children.itervalues():
for entry in child.entries():
if not self.is_root:
entry = self._name + '/' + entry
rv.append(entry)
return rv
def __init__(self):
self._root = Tree.Node(None)
def add(self, entry):
"""Add the given entry to the tree."""
split = entry.split('/')
if split[-1] == '':
split = split[:-1]
self._root.add(split)
def entries(self):
"""Return the list of entries in the tree.
Entries will be de-duplicated as follows:
- Any entry which is a sub-path of another entry will not be returned.
- Any entry which was not explicitly inserted but has children exceeding
the COMBINE_PATHS_THRESHOLD will be returned while its children will not
be returned.
"""
return self._root.entries()
def relpath(repo_path):
"""Return a relative path to the given path within the repo.
The path is relative to the infra/bots dir, where the compile.isolate file
lives.
"""
repo_path = '../../' + repo_path
repo_path = repo_path.replace('../../infra/', '../')
repo_path = repo_path.replace('../bots/', '')
return repo_path
def get_isolate_content(paths):
"""Construct the new content of the isolate file based on the given paths."""
lines = [' \'%s\',' % relpath(p) for p in paths]
lines.sort()
return ISOLATE_TMPL % '\n'.join(lines)
def main():
"""Regenerate the compile.isolate file, or verify that it hasn't changed."""
testing = False
if len(sys.argv) == 2 and sys.argv[1] == 'test':
testing = True
elif len(sys.argv) != 1:
print >> sys.stderr, 'Usage: %s [test]' % sys.argv[0]
sys.exit(1)
tree = Tree()
for p in get_relevant_paths():
tree.add(p)
content = get_isolate_content(tree.entries())
if testing:
with open(ISOLATE_FILE, 'rb') as f:
expect_content = f.read()
if content != expect_content:
print >> sys.stderr, 'Found diff in %s:' % ISOLATE_FILE
a = expect_content.splitlines()
b = content.splitlines()
diff = difflib.context_diff(a, b, lineterm='')
for line in diff:
sys.stderr.write(line + '\n')
print >> sys.stderr, 'You may need to run:\n\n\tpython %s' % sys.argv[0]
sys.exit(1)
else:
with open(ISOLATE_FILE, 'wb') as f:
f.write(content)
if __name__ == '__main__':
main()