Port chromium landmines script.

This runs the landmines script as a gclient hook. It can
as such be used to clobber local checkouts when hooks are
run locally.

It is a softer version than chromium's landmines script, as
it only deletes directories in the output directory due
to compatibility with MSVS which has "build" hardcoded as
output directory in several places.

BUG=chromium:403263
LOG=n

Review URL: https://codereview.chromium.org/955463002

Cr-Commit-Position: refs/heads/master@{#26831}

.gitignore

@@ -24,6 +24,7 @@
 .cproject
 .d8_history
 .gclient_entries
+.landmines
 .project
 .pydevproject
 .settings

DEPS

@@ -46,6 +46,17 @@ skip_child_includes = [
 ]
 
 hooks = [
+  {
+    # This clobbers when necessary (based on get_landmines.py). It must be the
+    # first hook so that other things that get/generate into the output
+    # directory will not subsequently be clobbered.
+    'name': 'landmines',
+    'pattern': '.',
+    'action': [
+        'python',
+        'v8/build/landmines.py',
+    ],
+  },
   # Pull clang-format binaries using checked-in hashes.
   {
     "name": "clang_format_win",

build/gyp_environment.py

@@ -0,0 +1,52 @@
+# Copyright 2015 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Sets up various automatic gyp environment variables. These are used by
+gyp_v8 and landmines.py which run at different stages of runhooks. To
+make sure settings are consistent between them, all setup should happen here.
+"""
+
+import os
+import sys
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+V8_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir))
+
+
+def apply_gyp_environment(file_path=None):
+  """
+  Reads in a *.gyp_env file and applies the valid keys to os.environ.
+  """
+  if not file_path or not os.path.exists(file_path):
+    return
+  file_contents = open(file_path).read()
+  try:
+    file_data = eval(file_contents, {'__builtins__': None}, None)
+  except SyntaxError, e:
+    e.filename = os.path.abspath(file_path)
+    raise
+  supported_vars = ( 'V8_GYP_FILE',
+                     'V8_GYP_SYNTAX_CHECK',
+                     'GYP_DEFINES',
+                     'GYP_GENERATOR_FLAGS',
+                     'GYP_GENERATOR_OUTPUT', )
+  for var in supported_vars:
+    val = file_data.get(var)
+    if val:
+      if var in os.environ:
+        print 'INFO: Environment value for "%s" overrides value in %s.' % (
+            var, os.path.abspath(file_path)
+        )
+      else:
+        os.environ[var] = val
+
+
+def set_environment():
+  """Sets defaults for GYP_* variables."""
+
+  if 'SKIP_V8_GYP_ENV' not in os.environ:
+    # Update the environment based on v8.gyp_env
+    gyp_env_path = os.path.join(os.path.dirname(V8_ROOT), 'v8.gyp_env')
+    apply_gyp_environment(gyp_env_path)

build/gyp_v8

@@ -31,6 +31,7 @@
 # is invoked by V8 beyond what can be done in the gclient hooks.
 
 import glob
+import gyp_environment
 import os
 import platform
 import shlex
@@ -48,34 +49,6 @@ sys.path.insert(
     1, os.path.abspath(os.path.join(v8_root, 'tools', 'generate_shim_headers')))
 
 
-def apply_gyp_environment(file_path=None):
-  """
-  Reads in a *.gyp_env file and applies the valid keys to os.environ.
-  """
-  if not file_path or not os.path.exists(file_path):
-    return
-  file_contents = open(file_path).read()
-  try:
-    file_data = eval(file_contents, {'__builtins__': None}, None)
-  except SyntaxError, e:
-    e.filename = os.path.abspath(file_path)
-    raise
-  supported_vars = ( 'V8_GYP_FILE',
-                     'V8_GYP_SYNTAX_CHECK',
-                     'GYP_DEFINES',
-                     'GYP_GENERATOR_FLAGS',
-                     'GYP_GENERATOR_OUTPUT', )
-  for var in supported_vars:
-    val = file_data.get(var)
-    if val:
-      if var in os.environ:
-        print 'INFO: Environment value for "%s" overrides value in %s.' % (
-            var, os.path.abspath(file_path)
-        )
-      else:
-        os.environ[var] = val
-
-
 def additional_include_files(args=[]):
   """
   Returns a list of additional (.gypi) files to include, without
@@ -109,13 +82,6 @@ def additional_include_files(args=[]):
 def run_gyp(args):
   rc = gyp.main(args)
 
-  # Check for landmines (reasons to clobber the build). This must be run here,
-  # rather than a separate runhooks step so that any environment modifications
-  # from above are picked up.
-  print 'Running build/landmines.py...'
-  subprocess.check_call(
-      [sys.executable, os.path.join(script_dir, 'landmines.py')])
-
   if rc != 0:
     print 'Error running GYP'
     sys.exit(rc)
@@ -124,10 +90,7 @@ def run_gyp(args):
 if __name__ == '__main__':
   args = sys.argv[1:]
 
-  if 'SKIP_V8_GYP_ENV' not in os.environ:
+  gyp_environment.set_environment()
-    # Update the environment based on v8.gyp_env
-    gyp_env_path = os.path.join(os.path.dirname(v8_root), 'v8.gyp_env')
-    apply_gyp_environment(gyp_env_path)
 
   # This could give false positives since it doesn't actually do real option
   # parsing.  Oh well.

build/landmine_utils.py

@@ -47,10 +47,19 @@ def gyp_defines():
   return dict(arg.split('=', 1)
       for arg in shlex.split(os.environ.get('GYP_DEFINES', '')))
 
+
+@memoize()
+def gyp_generator_flags():
+  """Parses and returns GYP_GENERATOR_FLAGS env var as a dictionary."""
+  return dict(arg.split('=', 1)
+      for arg in shlex.split(os.environ.get('GYP_GENERATOR_FLAGS', '')))
+
+
 @memoize()
 def gyp_msvs_version():
   return os.environ.get('GYP_MSVS_VERSION', '')
 
+
 @memoize()
 def distributor():
   """

build/landmines.py

@@ -4,10 +4,9 @@
 # found in the LICENSE file.
 
 """
-This script runs every build as a hook. If it detects that the build should
+This script runs every build as the first hook (See DEPS). If it detects that
-be clobbered, it will touch the file <build_dir>/.landmine_triggered. The
+the build should be clobbered, it will delete the contents of the build
-various build scripts will then check for the presence of this file and clobber
+directory.
-accordingly. The script will also emit the reasons for the clobber to stdout.
 
 A landmine is tripped when a builder checks out a different revision, and the
 diff between the new landmines and the old ones is non-null. At this point, the
@@ -15,9 +14,13 @@ build is clobbered.
 """
 
 import difflib
+import errno
+import gyp_environment
 import logging
 import optparse
 import os
+import re
+import shutil
 import sys
 import subprocess
 import time
@@ -28,46 +31,118 @@ import landmine_utils
 SRC_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
 
-def get_target_build_dir(build_tool, target):
+def get_build_dir(build_tool, is_iphone=False):
   """
   Returns output directory absolute path dependent on build and targets.
   Examples:
-    r'c:\b\build\slave\win\build\src\out\Release'
+    r'c:\b\build\slave\win\build\src\out'
-    '/mnt/data/b/build/slave/linux/build/src/out/Debug'
+    '/mnt/data/b/build/slave/linux/build/src/out'
-    '/b/build/slave/ios_rel_device/build/src/xcodebuild/Release-iphoneos'
+    '/b/build/slave/ios_rel_device/build/src/xcodebuild'
 
   Keep this function in sync with tools/build/scripts/slave/compile.py
   """
   ret = None
   if build_tool == 'xcode':
-    ret = os.path.join(SRC_DIR, 'xcodebuild', target)
+    ret = os.path.join(SRC_DIR, 'xcodebuild')
   elif build_tool in ['make', 'ninja', 'ninja-ios']:  # TODO: Remove ninja-ios.
-    ret = os.path.join(SRC_DIR, 'out', target)
+    if 'CHROMIUM_OUT_DIR' in os.environ:
+      output_dir = os.environ.get('CHROMIUM_OUT_DIR').strip()
+      if not output_dir:
+        raise Error('CHROMIUM_OUT_DIR environment variable is set but blank!')
+    else:
+      output_dir = landmine_utils.gyp_generator_flags().get('output_dir', 'out')
+    ret = os.path.join(SRC_DIR, output_dir)
   elif build_tool in ['msvs', 'vs', 'ib']:
-    ret = os.path.join(SRC_DIR, 'build', target)
+    ret = os.path.join(SRC_DIR, 'build')
   else:
     raise NotImplementedError('Unexpected GYP_GENERATORS (%s)' % build_tool)
   return os.path.abspath(ret)
 
 
-def set_up_landmines(target, new_landmines):
+def extract_gn_build_commands(build_ninja_file):
-  """Does the work of setting, planting, and triggering landmines."""
+  """Extracts from a build.ninja the commands to run GN.
-  out_dir = get_target_build_dir(landmine_utils.builder(), target)
 
-  landmines_path = os.path.join(out_dir, '.landmines')
+  The commands to run GN are the gn rule and build.ninja build step at the
-  if not os.path.exists(out_dir):
+  top of the build.ninja file. We want to keep these when deleting GN builds
+  since we want to preserve the command-line flags to GN.
+
+  On error, returns the empty string."""
+  result = ""
+  with open(build_ninja_file, 'r') as f:
+    # Read until the second blank line. The first thing GN writes to the file
+    # is the "rule gn" and the second is the section for "build build.ninja",
+    # separated by blank lines.
+    num_blank_lines = 0
+    while num_blank_lines < 2:
+      line = f.readline()
+      if len(line) == 0:
+        return ''  # Unexpected EOF.
+      result += line
+      if line[0] == '\n':
+        num_blank_lines = num_blank_lines + 1
+  return result
+
+def delete_build_dir(build_dir):
+  # GN writes a build.ninja.d file. Note that not all GN builds have args.gn.
+  build_ninja_d_file = os.path.join(build_dir, 'build.ninja.d')
+  if not os.path.exists(build_ninja_d_file):
+    shutil.rmtree(build_dir)
     return
 
-  if not os.path.exists(landmines_path):
+  # GN builds aren't automatically regenerated when you sync. To avoid
-    print "Landmines tracker didn't exists."
+  # messing with the GN workflow, erase everything but the args file, and
+  # write a dummy build.ninja file that will automatically rerun GN the next
+  # time Ninja is run.
+  build_ninja_file = os.path.join(build_dir, 'build.ninja')
+  build_commands = extract_gn_build_commands(build_ninja_file)
+
+  try:
+    gn_args_file = os.path.join(build_dir, 'args.gn')
+    with open(gn_args_file, 'r') as f:
+      args_contents = f.read()
+  except IOError:
+    args_contents = ''
+
+  shutil.rmtree(build_dir)
+
+  # Put back the args file (if any).
+  os.mkdir(build_dir)
+  if args_contents != '':
+    with open(gn_args_file, 'w') as f:
+      f.write(args_contents)
+
+  # Write the build.ninja file sufficiently to regenerate itself.
+  with open(os.path.join(build_dir, 'build.ninja'), 'w') as f:
+    if build_commands != '':
+      f.write(build_commands)
+    else:
+      # Couldn't parse the build.ninja file, write a default thing.
+      f.write('''rule gn
+command = gn -q gen //out/%s/
+description = Regenerating ninja files
+
+build build.ninja: gn
+generator = 1
+depfile = build.ninja.d
+''' % (os.path.split(build_dir)[1]))
+
+  # Write a .d file for the build which references a nonexistant file. This
+  # will make Ninja always mark the build as dirty.
+  with open(build_ninja_d_file, 'w') as f:
+    f.write('build.ninja: nonexistant_file.gn\n')
+
+
+def clobber_if_necessary(new_landmines):
+  """Does the work of setting, planting, and triggering landmines."""
+  out_dir = get_build_dir(landmine_utils.builder())
+  landmines_path = os.path.normpath(os.path.join(out_dir, '..', '.landmines'))
+  try:
+    os.makedirs(out_dir)
+  except OSError as e:
+    if e.errno == errno.EEXIST:
+      pass
 
-  # FIXME(machenbach): Clobber deletes the .landmines tracker. Difficult
-  # to know if we are right after a clobber or if it is first-time landmines
-  # deployment. Also, a landmine-triggered clobber right after a clobber is
-  # not possible. Different clobber methods for msvs, xcode and make all
-  # have different blacklists of files that are not deleted.
   if os.path.exists(landmines_path):
-    triggered = os.path.join(out_dir, '.landmines_triggered')
     with open(landmines_path, 'r') as f:
       old_landmines = f.readlines()
     if old_landmines != new_landmines:
@@ -75,14 +150,20 @@ def set_up_landmines(target, new_landmines):
       diff = difflib.unified_diff(old_landmines, new_landmines,
           fromfile='old_landmines', tofile='new_landmines',
           fromfiledate=old_date, tofiledate=time.ctime(), n=0)
+      sys.stdout.write('Clobbering due to:\n')
+      sys.stdout.writelines(diff)
 
-      with open(triggered, 'w') as f:
+      # Clobber contents of build directory but not directory itself: some
-        f.writelines(diff)
+      # checkouts have the build directory mounted.
-      print "Setting landmine: %s" % triggered
+      for f in os.listdir(out_dir):
-    elif os.path.exists(triggered):
+        path = os.path.join(out_dir, f)
-      # Remove false triggered landmines.
+        # Soft version of chromium's clobber. Only delete directories not files
-      os.remove(triggered)
+        # as e.g. on windows the output dir is the build dir that shares some
-      print "Removing landmine: %s" % triggered
+        # checked out files.
+        if os.path.isdir(path) and re.search(r"(?:[Rr]elease)|(?:[Dd]ebug)", f):
+          delete_build_dir(path)
+
+  # Save current set of landmines for next time.
   with open(landmines_path, 'w') as f:
     f.writelines(new_landmines)
 
@@ -123,14 +204,14 @@ def main():
   if landmine_utils.builder() in ('dump_dependency_json', 'eclipse'):
     return 0
 
+  gyp_environment.set_environment()
+
   landmines = []
   for s in landmine_scripts:
     proc = subprocess.Popen([sys.executable, s], stdout=subprocess.PIPE)
     output, _ = proc.communicate()
     landmines.extend([('%s\n' % l.strip()) for l in output.splitlines()])
-
+  clobber_if_necessary(landmines)
-  for target in ('Debug', 'Release'):
-    set_up_landmines(target, landmines)
 
   return 0