AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Convert Win toolchain to CIPD package

Browse Source 
BUG=skia:5427
GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2111713003

Review-Url: https://codereview.chromium.org/2111713003
This commit is contained in:
borenet 2016-07-15 08:34:08 -07:00 committed by Commit bot
parent 1b5f968103
commit f1120ea72c
9 changed files with 236 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
infra/bots/assets/win_toolchain/VERSION Normal file
View File

@ -0,0 +1 @@
1

26
infra/bots/assets/win_toolchain/common.py Executable file
View File

@ -0,0 +1,26 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Common vars used by scripts in this directory."""
import os
import sys
FILE_DIR = os.path.dirname(os.path.abspath(__file__))
INFRA_BOTS_DIR = os.path.realpath(os.path.join(FILE_DIR, os.pardir, os.pardir))
sys.path.insert(0, INFRA_BOTS_DIR)
from assets import assets
ASSET_NAME = os.path.basename(FILE_DIR)
def run(cmd):
"""Run a command, eg. "upload" or "download". """
assets.main([cmd, ASSET_NAME] + sys.argv[1:])

128
infra/bots/assets/win_toolchain/create.py Executable file
View File

@ -0,0 +1,128 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Download an updated VS toolchain"""
import argparse
import common
import json
import os
import shlex
import shutil
import subprocess
import sys
import utils
import win_toolchain_utils
# By default the toolchain includes a bunch of unnecessary stuff with long path
# names. Trim out directories with these names.
IGNORE_LIST = [
'WindowsMobile',
'App Certification Kit',
'Debuggers',
'Extension SDKs',
'winrt',
'DesignTime',
'AccChecker',
]
REPO_CHROME = 'https://chromium.googlesource.com/chromium/src.git'
def filter_toolchain_files(dirname, files):
"""Callback for shutil.copytree. Return lists of files to skip."""
split = dirname.split(os.path.sep)
for ign in IGNORE_LIST:
if ign in split:
print 'Ignoring dir %s' % dirname
return files
return []
def get_toolchain_dir(toolchain_dir_output):
"""Find the toolchain directory."""
prefix = 'vs_path = '
for line in toolchain_dir_output.splitlines():
if line.startswith(prefix):
return line[len(prefix):].strip('"')
raise Exception('Unable to find toolchain dir in output:\n%s' % (
toolchain_dir_output))
def gen_toolchain(chrome_path, msvs_version, target_dir):
"""Update the VS toolchain and copy it to the target_dir."""
with utils.chdir(os.path.join(chrome_path, 'src')):
subprocess.check_call([utils.GCLIENT, 'sync'])
depot_tools = subprocess.check_output([
'python', os.path.join('build', 'find_depot_tools.py')]).rstrip()
with utils.git_branch():
vs_toolchain_py = os.path.join('build', 'vs_toolchain.py')
env = os.environ.copy()
env['GYP_MSVS_VERSION'] = msvs_version
subprocess.check_call(['python', vs_toolchain_py, 'update'], env=env)
output = subprocess.check_output(['python', vs_toolchain_py,
'get_toolchain_dir'], env=env).rstrip()
src_dir = get_toolchain_dir(output)
# Mock out absolute paths in win_toolchain.json.
win_toolchain_utils.abstract(os.path.join('build', 'win_toolchain.json'),
os.path.dirname(depot_tools))
# Copy the toolchain files to the target_dir.
build = os.path.join(os.getcwd(), 'build')
dst_build = os.path.join(target_dir, 'src', 'build')
os.makedirs(dst_build)
for f in ('find_depot_tools.py', 'vs_toolchain.py', 'win_toolchain.json'):
shutil.copyfile(os.path.join(build, f), os.path.join(dst_build, f))
shutil.copytree(os.path.join(os.getcwd(), 'tools', 'gyp', 'pylib'),
os.path.join(target_dir, 'src', 'tools', 'gyp', 'pylib'))
dst_depot_tools = os.path.join(target_dir, 'depot_tools')
os.makedirs(dst_depot_tools)
for f in ('gclient.py', 'breakpad.py'):
shutil.copyfile(os.path.join(depot_tools, f),
os.path.join(dst_depot_tools, f))
toolchain_dst = os.path.join(
target_dir, 'depot_tools', os.path.relpath(src_dir, depot_tools))
shutil.copytree(src_dir, toolchain_dst, ignore=filter_toolchain_files)
def create_asset(target_dir, msvs_version, chrome_path=None):
"""Create the asset."""
if not os.path.isdir(target_dir):
os.makedirs(target_dir)
with utils.tmp_dir() as tmp_dir:
if not chrome_path:
print ('Syncing Chrome from scratch. If you already have a checkout, '
'specify --chrome_path to save time.')
chrome_path = os.path.join(tmp_dir.name, 'src')
if not os.path.isdir(chrome_path):
subprocess.check_call([utils.GCLIENT, 'config', REPO_CHROME, '--managed'])
subprocess.check_call([utils.GCLIENT, 'sync'])
gen_toolchain(chrome_path, msvs_version, target_dir)
def main():
if sys.platform != 'win32':
print >> sys.stderr, 'This script only runs on Windows.'
sys.exit(1)
parser = argparse.ArgumentParser()
parser.add_argument('--msvs_version', required=True)
parser.add_argument('--chrome_path')
parser.add_argument('--target_dir', '-t', required=True)
args = parser.parse_args()
target_dir = os.path.abspath(args.target_dir)
create_asset(target_dir, args.msvs_version, args.chrome_path)
if __name__ == '__main__':
main()

49
infra/bots/assets/win_toolchain/create_and_upload.py Executable file
View File

@ -0,0 +1,49 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Create the asset and upload it."""
import argparse
import common
import os
import subprocess
import sys
import utils
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--gsutil')
parser.add_argument('--chrome_path')
parser.add_argument('--msvs_version', required=True)
args = parser.parse_args()
with utils.tmp_dir():
cwd = os.getcwd()
create_script = os.path.join(common.FILE_DIR, 'create.py')
upload_script = os.path.join(common.FILE_DIR, 'upload.py')
try:
cmd = ['python', create_script,
'-t', cwd,
'--msvs_version', args.msvs_version]
if args.chrome_path:
cmd.extend(['--chrome_path', args.chrome_path])
subprocess.check_call(cmd)
cmd = ['python', upload_script, '-t', cwd]
if args.gsutil:
cmd.extend(['--gsutil', args.gsutil])
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
# Trap exceptions to avoid printing two stacktraces.
sys.exit(1)
if __name__ == '__main__':
main()

16
infra/bots/assets/win_toolchain/download.py Executable file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Download the current version of the asset."""
import common
if __name__ == '__main__':
common.run('download')

16
infra/bots/assets/win_toolchain/upload.py Executable file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Upload a new version of the asset."""
import common
if __name__ == '__main__':
common.run('upload')

124
infra/bots/isolate_win_toolchain.py
View File

@ -1,124 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Download an updated VS toolchain, isolate it, upload a CL to update Skia."""
import argparse
import json
import os
import shlex
import shutil
import subprocess
import sys
import utils
import win_toolchain_utils
REPO_CHROME = 'https://chromium.googlesource.com/chromium/src.git'
REPO_SKIA = 'https://skia.googlesource.com/skia.git'
def get_toolchain_dir(toolchain_dir_output):
"""Find the toolchain directory."""
prefix = 'vs_path = '
for line in toolchain_dir_output.splitlines():
if line.startswith(prefix):
return line[len(prefix):].strip('"')
raise Exception('Unable to find toolchain dir in output:\n%s' % (
toolchain_dir_output))
def gen_toolchain(chrome_path, msvs_version, isolate_file):
"""Update the VS toolchain, isolate it, and return the isolated hash."""
with utils.chdir(chrome_path):
subprocess.check_call([utils.GCLIENT, 'sync'])
depot_tools = subprocess.check_output([
'python', os.path.join('build', 'find_depot_tools.py')]).rstrip()
with utils.git_branch():
vs_toolchain_py = os.path.join('build', 'vs_toolchain.py')
env = os.environ.copy()
env['GYP_MSVS_VERSION'] = msvs_version
subprocess.check_call(['python', vs_toolchain_py, 'update'], env=env)
output = subprocess.check_output(['python', vs_toolchain_py,
'get_toolchain_dir'], env=env).rstrip()
src_dir = get_toolchain_dir(output)
# Mock out absolute paths in win_toolchain.json.
win_toolchain_utils.abstract(os.path.join('build', 'win_toolchain.json'),
os.path.dirname(depot_tools))
# Isolate the toolchain. Assumes we're running on Windows, since the above
# would fail otherwise.
isolate_file_dirname = os.path.dirname(isolate_file)
toolchain_relpath = os.path.relpath(src_dir, isolate_file_dirname)
chrome_relpath = os.path.relpath(os.getcwd(), isolate_file_dirname)
depot_tools_relpath = os.path.relpath(depot_tools, isolate_file_dirname)
isolate = os.path.join(
os.curdir, 'tools', 'luci-go', 'win64', 'isolate.exe')
isolate_cmd = [isolate, 'archive', '--quiet',
'--isolate-server', 'https://isolateserver.appspot.com',
'-i', isolate_file,
'-s', 'win_toolchain_%s.isolated' % msvs_version,
'--extra-variable', 'WIN_TOOLCHAIN_DIR=%s' % toolchain_relpath,
'--extra-variable', 'DEPOT_TOOLS_DIR=%s' % depot_tools_relpath,
'--extra-variable', 'CHROME_DIR=%s' % chrome_relpath]
isolate_out = subprocess.check_output(isolate_cmd).rstrip()
return shlex.split(isolate_out)[0]
def update_toolchain_file(skia_path, msvs_version, isolated_hash):
"""Edit the win_toolchain_hash file, upload a CL."""
with utils.chdir(skia_path):
with utils.git_branch():
hash_file = os.path.join('infra', 'bots', 'win_toolchain_hash.json')
with open(hash_file) as f:
hashes = json.load(f)
hashes[msvs_version] = isolated_hash
with open(hash_file, 'w') as f:
json.dump(hashes, f, indent=4, sort_keys=True)
subprocess.check_call([utils.GIT, 'add', hash_file])
subprocess.check_call([utils.GIT, 'commit', '-m', 'Update Win toolchain'])
subprocess.check_call([utils.GIT, 'cl', 'upload', '--bypass-hooks'])
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--msvs_version', required=True)
parser.add_argument('--chrome_path')
parser.add_argument('--skia_path')
args = parser.parse_args()
isolate_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'win_toolchain.isolate')
with utils.print_timings():
with utils.tmp_dir() as tmp_dir:
chrome_path = args.chrome_path
if not chrome_path:
print ('Syncing Chrome from scratch. If you already have a checkout, '
'specify --chrome_path to save time.')
chrome_path = os.path.join(tmp_dir.name, 'src')
if not os.path.isdir(chrome_path):
utils.git_clone(REPO_CHROME, chrome_path)
skia_path = args.skia_path
if not skia_path:
print ('Syncing Skia from scratch. If you already have a checkout, '
'specify --chrome_path to save time.')
skia_path = os.path.join(tmp_dir.name, 'skia')
if not os.path.isdir(skia_path):
utils.git_clone(REPO_SKIA, skia_path)
isolated_hash = gen_toolchain(chrome_path, args.msvs_version,
isolate_file)
update_toolchain_file(skia_path, args.msvs_version, isolated_hash)
if __name__ == '__main__':
main()

13
infra/bots/win_toolchain.isolate
View File

@ -1,13 +0,0 @@
{
'variables': {
'files': [
'<(CHROME_DIR)/build/find_depot_tools.py',
'<(CHROME_DIR)/build/vs_toolchain.py',
'<(CHROME_DIR)/build/win_toolchain.json',
'<(CHROME_DIR)/tools/gyp/pylib/',
'<(DEPOT_TOOLS_DIR)/gclient.py',
'<(DEPOT_TOOLS_DIR)/breakpad.py',
'<(WIN_TOOLCHAIN_DIR)/',
],
},
}

4
infra/bots/win_toolchain_hash.json
View File

@ -1,4 +0,0 @@
{
"2013": "705384d88f80da637eb367e5acc6f315c0e1db2f",
"2015": "a59d15e51c043be61a559315f2110ac15969aa86"
}