Port chromium landmines script.
This runs the landmines script as a gclient hook. It can as such be used to clobber local checkouts when hooks are run locally. It is a softer version than chromium's landmines script, as it only deletes directories in the output directory due to compatibility with MSVS which has "build" hardcoded as output directory in several places. BUG=chromium:403263 LOG=n Review URL: https://codereview.chromium.org/955463002 Cr-Commit-Position: refs/heads/master@{#26831}
This commit is contained in:
machenbach
Commit bot
parent
7fdcd4f705
commit
89731cfbf8
1
.gitignore
vendored
1
.gitignore
vendored
@ -24,6 +24,7 @@
|
||||
.cproject
|
||||
.d8_history
|
||||
.gclient_entries
|
||||
.landmines
|
||||
.project
|
||||
.pydevproject
|
||||
.settings
|
||||
|
||||
11
DEPS
11
DEPS
@ -46,6 +46,17 @@ skip_child_includes = [
|
||||
]
|
||||
|
||||
hooks = [
|
||||
{
|
||||
# This clobbers when necessary (based on get_landmines.py). It must be the
|
||||
# first hook so that other things that get/generate into the output
|
||||
# directory will not subsequently be clobbered.
|
||||
'name': 'landmines',
|
||||
'pattern': '.',
|
||||
'action': [
|
||||
'python',
|
||||
'v8/build/landmines.py',
|
||||
],
|
||||
},
|
||||
# Pull clang-format binaries using checked-in hashes.
|
||||
{
|
||||
"name": "clang_format_win",
|
||||
|
||||
52
build/gyp_environment.py
Normal file
52
build/gyp_environment.py
Normal file
@ -0,0 +1,52 @@
|
||||
# Copyright 2015 the V8 project authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""
|
||||
Sets up various automatic gyp environment variables. These are used by
|
||||
gyp_v8 and landmines.py which run at different stages of runhooks. To
|
||||
make sure settings are consistent between them, all setup should happen here.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
V8_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir))
|
||||
|
||||
|
||||
def apply_gyp_environment(file_path=None):
|
||||
"""
|
||||
Reads in a *.gyp_env file and applies the valid keys to os.environ.
|
||||
"""
|
||||
if not file_path or not os.path.exists(file_path):
|
||||
return
|
||||
file_contents = open(file_path).read()
|
||||
try:
|
||||
file_data = eval(file_contents, {'__builtins__': None}, None)
|
||||
except SyntaxError, e:
|
||||
e.filename = os.path.abspath(file_path)
|
||||
raise
|
||||
supported_vars = ( 'V8_GYP_FILE',
|
||||
'V8_GYP_SYNTAX_CHECK',
|
||||
'GYP_DEFINES',
|
||||
'GYP_GENERATOR_FLAGS',
|
||||
'GYP_GENERATOR_OUTPUT', )
|
||||
for var in supported_vars:
|
||||
val = file_data.get(var)
|
||||
if val:
|
||||
if var in os.environ:
|
||||
print 'INFO: Environment value for "%s" overrides value in %s.' % (
|
||||
var, os.path.abspath(file_path)
|
||||
)
|
||||
else:
|
||||
os.environ[var] = val
|
||||
|
||||
|
||||
def set_environment():
|
||||
"""Sets defaults for GYP_* variables."""
|
||||
|
||||
if 'SKIP_V8_GYP_ENV' not in os.environ:
|
||||
# Update the environment based on v8.gyp_env
|
||||
gyp_env_path = os.path.join(os.path.dirname(V8_ROOT), 'v8.gyp_env')
|
||||
apply_gyp_environment(gyp_env_path)
|
||||
41
build/gyp_v8
41
build/gyp_v8
@ -31,6 +31,7 @@
|
||||
# is invoked by V8 beyond what can be done in the gclient hooks.
|
||||
|
||||
import glob
|
||||
import gyp_environment
|
||||
import os
|
||||
import platform
|
||||
import shlex
|
||||
@ -48,34 +49,6 @@ sys.path.insert(
|
||||
1, os.path.abspath(os.path.join(v8_root, 'tools', 'generate_shim_headers')))
|
||||
|
||||
|
||||
def apply_gyp_environment(file_path=None):
|
||||
"""
|
||||
Reads in a *.gyp_env file and applies the valid keys to os.environ.
|
||||
"""
|
||||
if not file_path or not os.path.exists(file_path):
|
||||
return
|
||||
file_contents = open(file_path).read()
|
||||
try:
|
||||
file_data = eval(file_contents, {'__builtins__': None}, None)
|
||||
except SyntaxError, e:
|
||||
e.filename = os.path.abspath(file_path)
|
||||
raise
|
||||
supported_vars = ( 'V8_GYP_FILE',
|
||||
'V8_GYP_SYNTAX_CHECK',
|
||||
'GYP_DEFINES',
|
||||
'GYP_GENERATOR_FLAGS',
|
||||
'GYP_GENERATOR_OUTPUT', )
|
||||
for var in supported_vars:
|
||||
val = file_data.get(var)
|
||||
if val:
|
||||
if var in os.environ:
|
||||
print 'INFO: Environment value for "%s" overrides value in %s.' % (
|
||||
var, os.path.abspath(file_path)
|
||||
)
|
||||
else:
|
||||
os.environ[var] = val
|
||||
|
||||
|
||||
def additional_include_files(args=[]):
|
||||
"""
|
||||
Returns a list of additional (.gypi) files to include, without
|
||||
@ -109,13 +82,6 @@ def additional_include_files(args=[]):
|
||||
def run_gyp(args):
|
||||
rc = gyp.main(args)
|
||||
|
||||
# Check for landmines (reasons to clobber the build). This must be run here,
|
||||
# rather than a separate runhooks step so that any environment modifications
|
||||
# from above are picked up.
|
||||
print 'Running build/landmines.py...'
|
||||
subprocess.check_call(
|
||||
[sys.executable, os.path.join(script_dir, 'landmines.py')])
|
||||
|
||||
if rc != 0:
|
||||
print 'Error running GYP'
|
||||
sys.exit(rc)
|
||||
@ -124,10 +90,7 @@ def run_gyp(args):
|
||||
if __name__ == '__main__':
|
||||
args = sys.argv[1:]
|
||||
|
||||
if 'SKIP_V8_GYP_ENV' not in os.environ:
|
||||
# Update the environment based on v8.gyp_env
|
||||
gyp_env_path = os.path.join(os.path.dirname(v8_root), 'v8.gyp_env')
|
||||
apply_gyp_environment(gyp_env_path)
|
||||
gyp_environment.set_environment()
|
||||
|
||||
# This could give false positives since it doesn't actually do real option
|
||||
# parsing. Oh well.
|
||||
|
||||
@ -47,10 +47,19 @@ def gyp_defines():
|
||||
return dict(arg.split('=', 1)
|
||||
for arg in shlex.split(os.environ.get('GYP_DEFINES', '')))
|
||||
|
||||
|
||||
@memoize()
|
||||
def gyp_generator_flags():
|
||||
"""Parses and returns GYP_GENERATOR_FLAGS env var as a dictionary."""
|
||||
return dict(arg.split('=', 1)
|
||||
for arg in shlex.split(os.environ.get('GYP_GENERATOR_FLAGS', '')))
|
||||
|
||||
|
||||
@memoize()
|
||||
def gyp_msvs_version():
|
||||
return os.environ.get('GYP_MSVS_VERSION', '')
|
||||
|
||||
|
||||
@memoize()
|
||||
def distributor():
|
||||
"""
|
||||
|
||||
@ -4,10 +4,9 @@
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""
|
||||
This script runs every build as a hook. If it detects that the build should
|
||||
be clobbered, it will touch the file <build_dir>/.landmine_triggered. The
|
||||
various build scripts will then check for the presence of this file and clobber
|
||||
accordingly. The script will also emit the reasons for the clobber to stdout.
|
||||
This script runs every build as the first hook (See DEPS). If it detects that
|
||||
the build should be clobbered, it will delete the contents of the build
|
||||
directory.
|
||||
|
||||
A landmine is tripped when a builder checks out a different revision, and the
|
||||
diff between the new landmines and the old ones is non-null. At this point, the
|
||||
@ -15,9 +14,13 @@ build is clobbered.
|
||||
"""
|
||||
|
||||
import difflib
|
||||
import errno
|
||||
import gyp_environment
|
||||
import logging
|
||||
import optparse
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import subprocess
|
||||
import time
|
||||
@ -28,46 +31,118 @@ import landmine_utils
|
||||
SRC_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
|
||||
|
||||
def get_target_build_dir(build_tool, target):
|
||||
def get_build_dir(build_tool, is_iphone=False):
|
||||
"""
|
||||
Returns output directory absolute path dependent on build and targets.
|
||||
Examples:
|
||||
r'c:\b\build\slave\win\build\src\out\Release'
|
||||
'/mnt/data/b/build/slave/linux/build/src/out/Debug'
|
||||
'/b/build/slave/ios_rel_device/build/src/xcodebuild/Release-iphoneos'
|
||||
r'c:\b\build\slave\win\build\src\out'
|
||||
'/mnt/data/b/build/slave/linux/build/src/out'
|
||||
'/b/build/slave/ios_rel_device/build/src/xcodebuild'
|
||||
|
||||
Keep this function in sync with tools/build/scripts/slave/compile.py
|
||||
"""
|
||||
ret = None
|
||||
if build_tool == 'xcode':
|
||||
ret = os.path.join(SRC_DIR, 'xcodebuild', target)
|
||||
ret = os.path.join(SRC_DIR, 'xcodebuild')
|
||||
elif build_tool in ['make', 'ninja', 'ninja-ios']: # TODO: Remove ninja-ios.
|
||||
ret = os.path.join(SRC_DIR, 'out', target)
|
||||
if 'CHROMIUM_OUT_DIR' in os.environ:
|
||||
output_dir = os.environ.get('CHROMIUM_OUT_DIR').strip()
|
||||
if not output_dir:
|
||||
raise Error('CHROMIUM_OUT_DIR environment variable is set but blank!')
|
||||
else:
|
||||
output_dir = landmine_utils.gyp_generator_flags().get('output_dir', 'out')
|
||||
ret = os.path.join(SRC_DIR, output_dir)
|
||||
elif build_tool in ['msvs', 'vs', 'ib']:
|
||||
ret = os.path.join(SRC_DIR, 'build', target)
|
||||
ret = os.path.join(SRC_DIR, 'build')
|
||||
else:
|
||||
raise NotImplementedError('Unexpected GYP_GENERATORS (%s)' % build_tool)
|
||||
return os.path.abspath(ret)
|
||||
|
||||
|
||||
def set_up_landmines(target, new_landmines):
|
||||
"""Does the work of setting, planting, and triggering landmines."""
|
||||
out_dir = get_target_build_dir(landmine_utils.builder(), target)
|
||||
def extract_gn_build_commands(build_ninja_file):
|
||||
"""Extracts from a build.ninja the commands to run GN.
|
||||
|
||||
landmines_path = os.path.join(out_dir, '.landmines')
|
||||
if not os.path.exists(out_dir):
|
||||
The commands to run GN are the gn rule and build.ninja build step at the
|
||||
top of the build.ninja file. We want to keep these when deleting GN builds
|
||||
since we want to preserve the command-line flags to GN.
|
||||
|
||||
On error, returns the empty string."""
|
||||
result = ""
|
||||
with open(build_ninja_file, 'r') as f:
|
||||
# Read until the second blank line. The first thing GN writes to the file
|
||||
# is the "rule gn" and the second is the section for "build build.ninja",
|
||||
# separated by blank lines.
|
||||
num_blank_lines = 0
|
||||
while num_blank_lines < 2:
|
||||
line = f.readline()
|
||||
if len(line) == 0:
|
||||
return '' # Unexpected EOF.
|
||||
result += line
|
||||
if line[0] == '\n':
|
||||
num_blank_lines = num_blank_lines + 1
|
||||
return result
|
||||
|
||||
def delete_build_dir(build_dir):
|
||||
# GN writes a build.ninja.d file. Note that not all GN builds have args.gn.
|
||||
build_ninja_d_file = os.path.join(build_dir, 'build.ninja.d')
|
||||
if not os.path.exists(build_ninja_d_file):
|
||||
shutil.rmtree(build_dir)
|
||||
return
|
||||
|
||||
if not os.path.exists(landmines_path):
|
||||
print "Landmines tracker didn't exists."
|
||||
# GN builds aren't automatically regenerated when you sync. To avoid
|
||||
# messing with the GN workflow, erase everything but the args file, and
|
||||
# write a dummy build.ninja file that will automatically rerun GN the next
|
||||
# time Ninja is run.
|
||||
build_ninja_file = os.path.join(build_dir, 'build.ninja')
|
||||
build_commands = extract_gn_build_commands(build_ninja_file)
|
||||
|
||||
try:
|
||||
gn_args_file = os.path.join(build_dir, 'args.gn')
|
||||
with open(gn_args_file, 'r') as f:
|
||||
args_contents = f.read()
|
||||
except IOError:
|
||||
args_contents = ''
|
||||
|
||||
shutil.rmtree(build_dir)
|
||||
|
||||
# Put back the args file (if any).
|
||||
os.mkdir(build_dir)
|
||||
if args_contents != '':
|
||||
with open(gn_args_file, 'w') as f:
|
||||
f.write(args_contents)
|
||||
|
||||
# Write the build.ninja file sufficiently to regenerate itself.
|
||||
with open(os.path.join(build_dir, 'build.ninja'), 'w') as f:
|
||||
if build_commands != '':
|
||||
f.write(build_commands)
|
||||
else:
|
||||
# Couldn't parse the build.ninja file, write a default thing.
|
||||
f.write('''rule gn
|
||||
command = gn -q gen //out/%s/
|
||||
description = Regenerating ninja files
|
||||
|
||||
build build.ninja: gn
|
||||
generator = 1
|
||||
depfile = build.ninja.d
|
||||
''' % (os.path.split(build_dir)[1]))
|
||||
|
||||
# Write a .d file for the build which references a nonexistant file. This
|
||||
# will make Ninja always mark the build as dirty.
|
||||
with open(build_ninja_d_file, 'w') as f:
|
||||
f.write('build.ninja: nonexistant_file.gn\n')
|
||||
|
||||
|
||||
def clobber_if_necessary(new_landmines):
|
||||
"""Does the work of setting, planting, and triggering landmines."""
|
||||
out_dir = get_build_dir(landmine_utils.builder())
|
||||
landmines_path = os.path.normpath(os.path.join(out_dir, '..', '.landmines'))
|
||||
try:
|
||||
os.makedirs(out_dir)
|
||||
except OSError as e:
|
||||
if e.errno == errno.EEXIST:
|
||||
pass
|
||||
|
||||
# FIXME(machenbach): Clobber deletes the .landmines tracker. Difficult
|
||||
# to know if we are right after a clobber or if it is first-time landmines
|
||||
# deployment. Also, a landmine-triggered clobber right after a clobber is
|
||||
# not possible. Different clobber methods for msvs, xcode and make all
|
||||
# have different blacklists of files that are not deleted.
|
||||
if os.path.exists(landmines_path):
|
||||
triggered = os.path.join(out_dir, '.landmines_triggered')
|
||||
with open(landmines_path, 'r') as f:
|
||||
old_landmines = f.readlines()
|
||||
if old_landmines != new_landmines:
|
||||
@ -75,14 +150,20 @@ def set_up_landmines(target, new_landmines):
|
||||
diff = difflib.unified_diff(old_landmines, new_landmines,
|
||||
fromfile='old_landmines', tofile='new_landmines',
|
||||
fromfiledate=old_date, tofiledate=time.ctime(), n=0)
|
||||
sys.stdout.write('Clobbering due to:\n')
|
||||
sys.stdout.writelines(diff)
|
||||
|
||||
with open(triggered, 'w') as f:
|
||||
f.writelines(diff)
|
||||
print "Setting landmine: %s" % triggered
|
||||
elif os.path.exists(triggered):
|
||||
# Remove false triggered landmines.
|
||||
os.remove(triggered)
|
||||
print "Removing landmine: %s" % triggered
|
||||
# Clobber contents of build directory but not directory itself: some
|
||||
# checkouts have the build directory mounted.
|
||||
for f in os.listdir(out_dir):
|
||||
path = os.path.join(out_dir, f)
|
||||
# Soft version of chromium's clobber. Only delete directories not files
|
||||
# as e.g. on windows the output dir is the build dir that shares some
|
||||
# checked out files.
|
||||
if os.path.isdir(path) and re.search(r"(?:[Rr]elease)|(?:[Dd]ebug)", f):
|
||||
delete_build_dir(path)
|
||||
|
||||
# Save current set of landmines for next time.
|
||||
with open(landmines_path, 'w') as f:
|
||||
f.writelines(new_landmines)
|
||||
|
||||
@ -123,14 +204,14 @@ def main():
|
||||
if landmine_utils.builder() in ('dump_dependency_json', 'eclipse'):
|
||||
return 0
|
||||
|
||||
gyp_environment.set_environment()
|
||||
|
||||
landmines = []
|
||||
for s in landmine_scripts:
|
||||
proc = subprocess.Popen([sys.executable, s], stdout=subprocess.PIPE)
|
||||
output, _ = proc.communicate()
|
||||
landmines.extend([('%s\n' % l.strip()) for l in output.splitlines()])
|
||||
|
||||
for target in ('Debug', 'Release'):
|
||||
set_up_landmines(target, landmines)
|
||||
clobber_if_necessary(landmines)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user