#!/usr/bin/env python3 # Copyright 2014 Google Inc. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Parse a DEPS file and git checkout all of the dependencies. """ EXTRA_HELP = """ Environment Variables: GIT_EXECUTABLE: path to "git" binary; if unset, will look for one of ['git', 'git.exe', 'git.bat'] in your default path. GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset, will use the file ../DEPS relative to this script's directory. GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages. Git Config: To disable syncing of a single repository: cd path/to/repository git config sync-deps.disable true To re-enable sync: cd path/to/repository git config --unset sync-deps.disable """ import argparse import os import re import subprocess import sys import threading from builtins import bytes def git_executable(): """Find the git executable. Returns: A triple: A string suitable for passing to subprocess functions, or None. The major version number The minor version number """ envgit = os.environ.get('GIT_EXECUTABLE') searchlist = ['git', 'git.exe', 'git.bat'] if envgit: searchlist.insert(0, envgit) with open(os.devnull, 'w') as devnull: for git in searchlist: major=None minor=None try: version_info = subprocess.check_output([git, '--version']).decode('utf-8') match = re.search("^git version (\d+)\.(\d+)",version_info) print("Using {}".format(version_info)) if match: major = int(match.group(1)) minor = int(match.group(2)) else: continue except (OSError,): continue return (git,major,minor) return (None,0,0) DEFAULT_DEPS_PATH = os.path.normpath( os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS')) def get_deps_os_str(deps_file): parsed_deps = parse_file_to_dict(deps_file) parts = [] if 'deps_os' in parsed_deps: for deps_os in parsed_deps['deps_os']: parts.append(' [{}]]'.format(deps_os)) return "\n".join(parts) def looks_like_raw_commit(commit): return re.match('^[a-f0-9]{40}$', commit) is not None def git_repository_sync_is_disabled(git, directory): try: disable = subprocess.check_output( [git, 'config', 'sync-deps.disable'], cwd=directory) return disable.lower().strip() in ['true', '1', 'yes', 'on'] except subprocess.CalledProcessError: return False def is_git_toplevel(git, directory): """Return true iff the directory is the top level of a Git repository. Args: git (string) the git executable directory (string) the path into which the repository is expected to be checked out. """ try: toplevel = subprocess.check_output( [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip() return os.path.realpath(bytes(directory, 'utf8')) == os.path.realpath(toplevel) except subprocess.CalledProcessError: return False def status(directory, checkoutable): def truncate(s, length): return s if len(s) <= length else '...' + s[-(length - 3):] dlen = 36 directory = truncate(directory, dlen) checkoutable = truncate(checkoutable, 40) sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable)) def git_checkout_to_directory(git, repo, checkoutable, directory, verbose, treeless): """Checkout (and clone if needed) a Git repository. Args: git (string) the git executable repo (string) the location of the repository, suitable for passing to `git clone`. checkoutable (string) a tag, branch, or commit, suitable for passing to `git checkout` directory (string) the path into which the repository should be checked out. verbose (boolean): emit status info to stdout treeless (boolean): when true, clone without any trees. Raises an exception if any calls to git fail. """ if not os.path.isdir(directory): # Use blobless or treeless checkouts for faster downloads. # This defers some work to checkout time. # https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ filter = ['--filter=tree:0'] if treeless else ['--filter=blob:none'] # If the thing to check out looks like a tag (and not like a commit), # then limit the checkout to that branch. branch = [] if looks_like_raw_commit(checkoutable) else ['--branch={}'.format(checkoutable)] subprocess.check_call( [git, 'clone', '--quiet', '--single-branch'] + filter + branch + [repo, directory]) if not is_git_toplevel(git, directory): # if the directory exists, but isn't a git repo, you will modify # the parent repostory, which isn't what you want. sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory) return # Check to see if this repo is disabled. Quick return. if git_repository_sync_is_disabled(git, directory): sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory) return with open(os.devnull, 'w') as devnull: # If this fails, we will fetch before trying again. Don't spam user # with error information. if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable], cwd=directory, stderr=devnull): # if this succeeds, skip slow `git fetch`. if verbose: status(directory, checkoutable) # Success. return # If the repo has changed, always force use of the correct repo. # If origin already points to repo, this is a quick no-op. subprocess.check_call( [git, 'remote', 'set-url', 'origin', repo], cwd=directory) subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory) subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory) if verbose: status(directory, checkoutable) # Success. def parse_file_to_dict(path): dictionary = {} contents = open(path).read() # Need to convert Var() to vars[], so that the DEPS is actually Python. Var() # comes from Autoroller using gclient which has a slightly different DEPS # format. contents = re.sub(r"Var\((.*?)\)", r"vars[\1]", contents) exec(contents, dictionary) return dictionary def git_sync_deps(deps_file_path, command_line_os_requests, verbose, treeless): """Grab dependencies, with optional platform support. Args: deps_file_path (string) Path to the DEPS file. command_line_os_requests (list of strings) Can be empty list. List of strings that should each be a key in the deps_os dictionary in the DEPS file. verbose (boolean): emit status info to stdout treeless (boolean): when true, clone as treeless instead of blobless Raises git Exceptions. """ (git,git_major,git_minor) = git_executable() assert git # --filter=tree:0 is available in git 2.20 and later if (git_major,git_minor) < (2,20): print("disabling --treeless: git is older than v2.20") treeless = False deps_file_directory = os.path.dirname(deps_file_path) deps_file = parse_file_to_dict(deps_file_path) dependencies = deps_file['deps'].copy() os_specific_dependencies = deps_file.get('deps_os', dict()) if 'all' in command_line_os_requests: for value in list(os_specific_dependencies.values()): dependencies.update(value) else: for os_name in command_line_os_requests: # Add OS-specific dependencies if os_name in os_specific_dependencies: dependencies.update(os_specific_dependencies[os_name]) for directory in dependencies: for other_dir in dependencies: if directory.startswith(other_dir + '/'): raise Exception('%r is parent of %r' % (other_dir, directory)) list_of_arg_lists = [] for directory in sorted(dependencies): if '@' in dependencies[directory]: repo, checkoutable = dependencies[directory].split('@', 1) else: raise Exception("please specify commit or tag") relative_directory = os.path.join(deps_file_directory, directory) list_of_arg_lists.append( (git, repo, checkoutable, relative_directory, verbose, treeless)) multithread(git_checkout_to_directory, list_of_arg_lists) for directory in deps_file.get('recursedeps', []): recursive_path = os.path.join(deps_file_directory, directory, 'DEPS') git_sync_deps(recursive_path, command_line_os_requests, verbose) def multithread(function, list_of_arg_lists): # for args in list_of_arg_lists: # function(*args) # return threads = [] for args in list_of_arg_lists: thread = threading.Thread(None, function, None, args) thread.start() threads.append(thread) for thread in threads: thread.join() def main(argv): argparser = argparse.ArgumentParser( prog = "git-sync-deps", description = "Checkout git-based dependencies as specified by the DEPS file", add_help=False # Because we want to print deps_os with -h option ) argparser.add_argument("--help", "-h", action='store_true', help="show this help message and exit") argparser.add_argument("--deps", default = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH), help="location of the the DEPS file") argparser.add_argument("--verbose", default=not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False)), action='store_true', help="be verbose: print status messages") argparser.add_argument("--treeless", default=False, action='store_true', help=""" Clone repos without trees (--filter=tree:0). This is the fastest option for a build machine, when you only need a single commit. Defers getting objects until checking out a commit. The default is to clone with trees but without blobs. Only takes effect if using git 2.20 or later. See https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ """) argparser.add_argument("os_requests",nargs="*", help="OS requests, as keys in the deps_os dictionariy in the DEPS file") args = argparser.parse_args() if args.help: print(argparser.format_help()) print(EXTRA_HELP) print(get_deps_os_str(args.deps)) return 0 git_sync_deps(args.deps, args.os_requests, args.verbose, args.treeless) return 0 if __name__ == '__main__': exit(main(sys.argv[1:]))