diff --git a/DEPS b/DEPS index db24e9b04..f4557e971 100644 --- a/DEPS +++ b/DEPS @@ -1,174 +1,26 @@ use_relative_paths = True vars = { - 'chromium_git': 'https://chromium.googlesource.com', 'github': 'https://github.com', - 'build_revision': '037f38ae0fe5e11b4f7c33b750fd7a1e9634a606', - 'buildtools_revision': 'ab7b6a7b350dd15804c87c20ce78982811fdd76f', - 'clang_revision': 'abe5e4f9dc0f1df848c7a0efa05256253e77a7b7', - 'effcee_revision': '04b624799f5a9dbaf3fa1dbed2ba9dce2fc8dcf2', - 'googletest_revision': '98a0d007d7092b72eea0e501bb9ad17908a1a036', - 'testing_revision': '340252637e2e7c72c0901dcbeeacfff419e19b59', - 're2_revision': '6cf8ccd82dbaab2668e9b13596c68183c9ecd13f', + 'effcee_revision': 'b83b58d177b797edd1f94c5f10837f2cc2863f0a', + 'googletest_revision': '2f42d769ad1b08742f7ccb5ad4dd357fc5ff248c', + 're2_revision': '848dfb7e1d7ba641d598cb66f81590f3999a555a', 'spirv_headers_revision': 'de99d4d834aeb51dd9f099baa285bd44fd04bb3d', } deps = { - "build": - Var('chromium_git') + "/chromium/src/build.git@" + Var('build_revision'), - - 'buildtools': - Var('chromium_git') + '/chromium/buildtools.git@' + - Var('buildtools_revision'), - - 'external/spirv-headers': - Var('github') + '/KhronosGroup/SPIRV-Headers.git@' + - Var('spirv_headers_revision'), + 'external/effcee': + Var('github') + '/google/effcee.git@' + Var('effcee_revision'), 'external/googletest': Var('github') + '/google/googletest.git@' + Var('googletest_revision'), - 'external/effcee': - Var('github') + '/google/effcee.git@' + Var('effcee_revision'), - 'external/re2': Var('github') + '/google/re2.git@' + Var('re2_revision'), - 'testing': - Var('chromium_git') + '/chromium/src/testing@' + - Var('testing_revision'), - - 'tools/clang': - Var('chromium_git') + '/chromium/src/tools/clang@' + Var('clang_revision') + 'external/spirv-headers': + Var('github') + '/KhronosGroup/SPIRV-Headers.git@' + + Var('spirv_headers_revision'), } -recursedeps = [ - # buildtools provides clang_format, libc++, and libc++api - 'buildtools', -] - -hooks = [ - { - 'name': 'gn_win', - 'action': [ 'download_from_google_storage', - '--no_resume', - '--platform=win32', - '--no_auth', - '--bucket', 'chromium-gn', - '-s', 'SPIRV-Tools/buildtools/win/gn.exe.sha1', - ], - }, - { - 'name': 'gn_mac', - 'pattern': '.', - 'action': [ 'download_from_google_storage', - '--no_resume', - '--platform=darwin', - '--no_auth', - '--bucket', 'chromium-gn', - '-s', 'SPIRV-Tools/buildtools/mac/gn.sha1', - ], - }, - { - 'name': 'gn_linux64', - 'pattern': '.', - 'action': [ 'download_from_google_storage', - '--no_resume', - '--platform=linux*', - '--no_auth', - '--bucket', 'chromium-gn', - '-s', 'SPIRV-Tools/buildtools/linux64/gn.sha1', - ], - }, - # Pull clang-format binaries using checked-in hashes. - { - 'name': 'clang_format_win', - 'pattern': '.', - 'action': [ 'download_from_google_storage', - '--no_resume', - '--platform=win32', - '--no_auth', - '--bucket', 'chromium-clang-format', - '-s', 'SPIRV-Tools/buildtools/win/clang-format.exe.sha1', - ], - }, - { - 'name': 'clang_format_mac', - 'pattern': '.', - 'action': [ 'download_from_google_storage', - '--no_resume', - '--platform=darwin', - '--no_auth', - '--bucket', 'chromium-clang-format', - '-s', 'SPIRV-Tools/buildtools/mac/clang-format.sha1', - ], - }, - { - 'name': 'clang_format_linux', - 'pattern': '.', - 'action': [ 'download_from_google_storage', - '--no_resume', - '--platform=linux*', - '--no_auth', - '--bucket', 'chromium-clang-format', - '-s', 'SPIRV-Tools/buildtools/linux64/clang-format.sha1', - ], - }, - { - # Pull clang - 'name': 'clang', - 'pattern': '.', - 'action': ['python', - 'SPIRV-Tools/tools/clang/scripts/update.py' - ], - }, - { - 'name': 'sysroot_arm', - 'pattern': '.', - 'condition': 'checkout_linux and checkout_arm', - 'action': ['python', 'SPIRV-Tools/build/linux/sysroot_scripts/install-sysroot.py', - '--arch=arm'], - }, - { - 'name': 'sysroot_arm64', - 'pattern': '.', - 'condition': 'checkout_linux and checkout_arm64', - 'action': ['python', 'SPIRV-Tools/build/linux/sysroot_scripts/install-sysroot.py', - '--arch=arm64'], - }, - { - 'name': 'sysroot_x86', - 'pattern': '.', - 'condition': 'checkout_linux and (checkout_x86 or checkout_x64)', - 'action': ['python', 'SPIRV-Tools/build/linux/sysroot_scripts/install-sysroot.py', - '--arch=x86'], - }, - { - 'name': 'sysroot_mips', - 'pattern': '.', - 'condition': 'checkout_linux and checkout_mips', - 'action': ['python', 'SPIRV-Tools/build/linux/sysroot_scripts/install-sysroot.py', - '--arch=mips'], - }, - { - 'name': 'sysroot_x64', - 'pattern': '.', - 'condition': 'checkout_linux and checkout_x64', - 'action': ['python', 'SPIRV-Tools/build/linux/sysroot_scripts/install-sysroot.py', - '--arch=x64'], - }, - { - # Update the Windows toolchain if necessary. - 'name': 'win_toolchain', - 'pattern': '.', - 'condition': 'checkout_win', - 'action': ['python', 'SPIRV-Tools/build/vs_toolchain.py', 'update', '--force'], - }, - { - # Update the Mac toolchain if necessary. - 'name': 'mac_toolchain', - 'pattern': '.', - 'action': ['python', 'SPIRV-Tools/build/mac_toolchain.py'], - }, -] diff --git a/README.md b/README.md index 9737f7e78..8a5c85e11 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,10 @@ mkdir build && cd build cmake [-G ] ``` +*Note*: +The script `utils/git-sync-deps` can be used to checkout and/or update the +contents of the repos under `external/` instead of manually maintaining them. + Once the build files have been generated, build using your preferred development environment. @@ -344,6 +348,13 @@ $ANDROID_NDK/ndk-build -C ../android_test \ NDK_APP_OUT=`pwd`/app ``` +### Updating DEPS +Occasionally the entries in DEPS will need to be updated. This is done on demand +when there is a request to do this, often due to downstream breakages. There is +a script `utils/roll_deps.sh` provided, which will generate a patch with the +updated DEPS values. This will still need to be tested in your checkout to +confirm that there are no integration issues that need to be resolved. + ## Library ### Usage diff --git a/utils/git-sync-deps b/utils/git-sync-deps new file mode 100755 index 000000000..057564132 --- /dev/null +++ b/utils/git-sync-deps @@ -0,0 +1,282 @@ +#!/usr/bin/env python +# Copyright 2014 Google Inc. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Parse a DEPS file and git checkout all of the dependencies. + +Args: + An optional list of deps_os values. + +Environment Variables: + GIT_EXECUTABLE: path to "git" binary; if unset, will look for one of + ['git', 'git.exe', 'git.bat'] in your default path. + + GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset, + will use the file ../DEPS relative to this script's directory. + + GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages. + +Git Config: + To disable syncing of a single repository: + cd path/to/repository + git config sync-deps.disable true + + To re-enable sync: + cd path/to/repository + git config --unset sync-deps.disable +""" + + +import os +import re +import subprocess +import sys +import threading +from builtins import bytes + + +def git_executable(): + """Find the git executable. + + Returns: + A string suitable for passing to subprocess functions, or None. + """ + envgit = os.environ.get('GIT_EXECUTABLE') + searchlist = ['git', 'git.exe', 'git.bat'] + if envgit: + searchlist.insert(0, envgit) + with open(os.devnull, 'w') as devnull: + for git in searchlist: + try: + subprocess.call([git, '--version'], stdout=devnull) + except (OSError,): + continue + return git + return None + + +DEFAULT_DEPS_PATH = os.path.normpath( + os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS')) + + +def usage(deps_file_path = None): + sys.stderr.write( + 'Usage: run to grab dependencies, with optional platform support:\n') + sys.stderr.write(' %s %s' % (sys.executable, __file__)) + if deps_file_path: + parsed_deps = parse_file_to_dict(deps_file_path) + if 'deps_os' in parsed_deps: + for deps_os in parsed_deps['deps_os']: + sys.stderr.write(' [%s]' % deps_os) + sys.stderr.write('\n\n') + sys.stderr.write(__doc__) + + +def git_repository_sync_is_disabled(git, directory): + try: + disable = subprocess.check_output( + [git, 'config', 'sync-deps.disable'], cwd=directory) + return disable.lower().strip() in ['true', '1', 'yes', 'on'] + except subprocess.CalledProcessError: + return False + + +def is_git_toplevel(git, directory): + """Return true iff the directory is the top level of a Git repository. + + Args: + git (string) the git executable + + directory (string) the path into which the repository + is expected to be checked out. + """ + try: + toplevel = subprocess.check_output( + [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip() + return os.path.realpath(bytes(directory, 'utf8')) == os.path.realpath(toplevel) + except subprocess.CalledProcessError: + return False + + +def status(directory, checkoutable): + def truncate(s, length): + return s if len(s) <= length else s[:(length - 3)] + '...' + dlen = 36 + directory = truncate(directory, dlen) + checkoutable = truncate(checkoutable, 40) + sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable)) + + +def git_checkout_to_directory(git, repo, checkoutable, directory, verbose): + """Checkout (and clone if needed) a Git repository. + + Args: + git (string) the git executable + + repo (string) the location of the repository, suitable + for passing to `git clone`. + + checkoutable (string) a tag, branch, or commit, suitable for + passing to `git checkout` + + directory (string) the path into which the repository + should be checked out. + + verbose (boolean) + + Raises an exception if any calls to git fail. + """ + if not os.path.isdir(directory): + subprocess.check_call( + [git, 'clone', '--quiet', repo, directory]) + + if not is_git_toplevel(git, directory): + # if the directory exists, but isn't a git repo, you will modify + # the parent repostory, which isn't what you want. + sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory) + return + + # Check to see if this repo is disabled. Quick return. + if git_repository_sync_is_disabled(git, directory): + sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory) + return + + with open(os.devnull, 'w') as devnull: + # If this fails, we will fetch before trying again. Don't spam user + # with error infomation. + if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable], + cwd=directory, stderr=devnull): + # if this succeeds, skip slow `git fetch`. + if verbose: + status(directory, checkoutable) # Success. + return + + # If the repo has changed, always force use of the correct repo. + # If origin already points to repo, this is a quick no-op. + subprocess.check_call( + [git, 'remote', 'set-url', 'origin', repo], cwd=directory) + + subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory) + + subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory) + + if verbose: + status(directory, checkoutable) # Success. + + +def parse_file_to_dict(path): + dictionary = {} + contents = open(path).read() + # Need to convert Var() to vars[], so that the DEPS is actually Python. Var() + # comes from Autoroller using gclient which has a slightly different DEPS + # format. + contents = re.sub(r"Var\((.*?)\)", r"vars[\1]", contents) + exec(contents, dictionary) + return dictionary + + +def git_sync_deps(deps_file_path, command_line_os_requests, verbose): + """Grab dependencies, with optional platform support. + + Args: + deps_file_path (string) Path to the DEPS file. + + command_line_os_requests (list of strings) Can be empty list. + List of strings that should each be a key in the deps_os + dictionary in the DEPS file. + + Raises git Exceptions. + """ + git = git_executable() + assert git + + deps_file_directory = os.path.dirname(deps_file_path) + deps_file = parse_file_to_dict(deps_file_path) + dependencies = deps_file['deps'].copy() + os_specific_dependencies = deps_file.get('deps_os', dict()) + if 'all' in command_line_os_requests: + for value in list(os_specific_dependencies.values()): + dependencies.update(value) + else: + for os_name in command_line_os_requests: + # Add OS-specific dependencies + if os_name in os_specific_dependencies: + dependencies.update(os_specific_dependencies[os_name]) + for directory in dependencies: + for other_dir in dependencies: + if directory.startswith(other_dir + '/'): + raise Exception('%r is parent of %r' % (other_dir, directory)) + list_of_arg_lists = [] + for directory in sorted(dependencies): + if '@' in dependencies[directory]: + repo, checkoutable = dependencies[directory].split('@', 1) + else: + raise Exception("please specify commit or tag") + + relative_directory = os.path.join(deps_file_directory, directory) + + list_of_arg_lists.append( + (git, repo, checkoutable, relative_directory, verbose)) + + multithread(git_checkout_to_directory, list_of_arg_lists) + + for directory in deps_file.get('recursedeps', []): + recursive_path = os.path.join(deps_file_directory, directory, 'DEPS') + git_sync_deps(recursive_path, command_line_os_requests, verbose) + + +def multithread(function, list_of_arg_lists): + # for args in list_of_arg_lists: + # function(*args) + # return + threads = [] + for args in list_of_arg_lists: + thread = threading.Thread(None, function, None, args) + thread.start() + threads.append(thread) + for thread in threads: + thread.join() + + +def main(argv): + deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH) + verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False)) + + if '--help' in argv or '-h' in argv: + usage(deps_file_path) + return 1 + + git_sync_deps(deps_file_path, argv, verbose) + # subprocess.check_call( + # [sys.executable, + # os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')]) + return 0 + + +if __name__ == '__main__': + exit(main(sys.argv[1:])) diff --git a/utils/roll_deps.sh b/utils/roll_deps.sh new file mode 100755 index 000000000..622afc9ed --- /dev/null +++ b/utils/roll_deps.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Copyright (c) 2019 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Attempts to roll all entries in DEPS to origin/master and creates a +# commit. +# +# Depends on roll-dep from depot_path being in PATH. + +# This script assumes it's parent directory is the repo root. +repo_path=$(dirname "$0")/.. + +effcee_dir="external/effcee/" +googletest_dir="external/googletest/" +re2_dir="external/re2/" +spirv_headers_dir="external/spirv-headers/" + +cd "$repo_path" + +roll-dep "$@" "${effcee_dir}" "${googletest_dir}" "${re2_dir}" "${spirv_headers_dir}"