[node] scripts to fetch deps and build monolithic .a for node.

Once this CL lands and is included in Node.js, we can
- run `tools/node/fetch_deps.py <v8_path>` to fetch necessary deps.
- run `tools/node/build_gn.py <build mode> <v8_path> <out_dir>` to build.
- use new v8_monolith target in v8.gyp to call build_gn.py.

R=machenbach@chromium.org

Bug: v8:6105
Change-Id: I482bfddb40f77df62394a913335bd43627cc0c43
Reviewed-on: https://chromium-review.googlesource.com/792944
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Commit-Queue: Yang Guo <yangguo@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50152}

DEPS

@@ -5,6 +5,7 @@
 vars = {
   'checkout_instrumented_libraries': False,
   'chromium_url': 'https://chromium.googlesource.com',
+  'build_for_node': False,
 }
 
 deps = {
@@ -93,7 +94,7 @@ hooks = [
   {
     'name': 'clang_format_win',
     'pattern': '.',
-    'condition': 'host_os == "win"',
+    'condition': 'host_os == "win" and build_for_node != True',
     'action': [ 'download_from_google_storage',
                 '--no_resume',
                 '--platform=win32',
@@ -105,7 +106,7 @@ hooks = [
   {
     'name': 'clang_format_mac',
     'pattern': '.',
-    'condition': 'host_os == "mac"',
+    'condition': 'host_os == "mac" and build_for_node != True',
     'action': [ 'download_from_google_storage',
                 '--no_resume',
                 '--platform=darwin',
@@ -117,7 +118,7 @@ hooks = [
   {
     'name': 'clang_format_linux',
     'pattern': '.',
-    'condition': 'host_os == "linux"',
+    'condition': 'host_os == "linux" and build_for_node != True',
     'action': [ 'download_from_google_storage',
                 '--no_resume',
                 '--platform=linux*',
@@ -129,6 +130,7 @@ hooks = [
   {
     'name': 'gcmole',
     'pattern': '.',
+    'condition': 'build_for_node != True',
     # TODO(machenbach): Insert condition and remove GYP_DEFINES dependency.
     'action': [
         'python',
@@ -138,6 +140,7 @@ hooks = [
   {
     'name': 'jsfunfuzz',
     'pattern': '.',
+    'condition': 'build_for_node != True',
     # TODO(machenbach): Insert condition and remove GYP_DEFINES dependency.
     'action': [
         'python',
@@ -148,7 +151,7 @@ hooks = [
   {
     'name': 'luci-go_win',
     'pattern': '.',
-    'condition': 'host_os == "win"',
+    'condition': 'host_os == "win" and build_for_node != True',
     'action': [ 'download_from_google_storage',
                 '--no_resume',
                 '--platform=win32',
@@ -160,7 +163,7 @@ hooks = [
   {
     'name': 'luci-go_mac',
     'pattern': '.',
-    'condition': 'host_os == "mac"',
+    'condition': 'host_os == "mac" and build_for_node != True',
     'action': [ 'download_from_google_storage',
                 '--no_resume',
                 '--platform=darwin',
@@ -172,7 +175,7 @@ hooks = [
   {
     'name': 'luci-go_linux',
     'pattern': '.',
-    'condition': 'host_os == "linux"',
+    'condition': 'host_os == "linux" and build_for_node != True',
     'action': [ 'download_from_google_storage',
                 '--no_resume',
                 '--platform=linux*',
@@ -221,6 +224,7 @@ hooks = [
   {
     'name': 'wasm_spec_tests',
     'pattern': '.',
+    'condition': 'build_for_node != True',
     'action': [ 'download_from_google_storage',
                 '--no_resume',
                 '--no_auth',
@@ -232,6 +236,7 @@ hooks = [
   {
     'name': 'closure_compiler',
     'pattern': '.',
+    'condition': 'build_for_node != True',
     'action': [ 'download_from_google_storage',
                 '--no_resume',
                 '--no_auth',
@@ -246,6 +251,7 @@ hooks = [
     # change.
     'name': 'sysroot',
     'pattern': '.',
+    'condition': 'build_for_node != True',
     'action': [
         'python',
         'v8/build/linux/sysroot_scripts/install-sysroot.py',
@@ -287,7 +293,7 @@ hooks = [
   {
     'name': 'binutils',
     'pattern': 'v8/third_party/binutils',
-    'condition': 'host_os == "linux"',
+    'condition': 'host_os == "linux" and build_for_node != True',
     'action': [
         'python',
         'v8/third_party/binutils/download.py',

src/v8.gyp

@@ -2587,5 +2587,40 @@
         },
       ],
     },
+    {
+      'target_name': 'v8_monolith',
+      'type': 'static_library',
+      'direct_dependent_settings': {
+        'include_dirs': [
+          '../include',
+        ],
+      },
+      'actions': [
+        {
+          'action_name': 'build_with_gn',
+          'inputs': [
+            '../tools/node/build_gn.py',
+          ],
+          'outputs': [
+            '<(INTERMEDIATE_DIR)/obj/libv8_monolith.a',
+            '<(INTERMEDIATE_DIR)/args.gn',
+          ],
+          'action': [
+            '../tools/node/build_gn.py',
+            '<(CONFIGURATION_NAME)',
+            '../',
+            '<(INTERMEDIATE_DIR)',
+            'v8_promise_internal_field_count=<(v8_promise_internal_field_count)',
+            'target_cpu="<(target_arch)"',
+            'target_os="<(OS)"',
+            'v8_target_cpu="<(v8_target_arch)"',
+            'v8_embedder_string="<(v8_embedder_string)"',
+            'v8_use_snapshot=<(v8_use_snapshot)',
+            'v8_optimized_debug=<(v8_optimized_debug)',
+            'v8_enable_disassembler=<(v8_enable_disassembler)',
+          ],
+        },
+      ],
+    },
   ],
 }

tools/node/backport_node.py

@@ -27,12 +27,19 @@ import subprocess
 import re
 import sys
 
-from common_includes import *
-
 TARGET_SUBDIR = os.path.join("deps", "v8")
 VERSION_FILE = os.path.join("include", "v8-version.h")
 VERSION_PATTERN = r'(?<=#define V8_PATCH_LEVEL )\d+'
 
+def FileToText(file_name):
+  with open(file_name) as f:
+    return f.read()
+
+def TextToFile(text, file_name):
+  with open(file_name, "w") as f:
+    f.write(text)
+
+
 def Clean(options):
   print ">> Cleaning target directory."
   subprocess.check_call(["git", "clean", "-fd"],

tools/node/build_gn.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Use this script to build libv8_monolith.a as dependency for Node.js
+Required dependencies can be fetched with fetch_deps.py.
+
+Usage: build_gn.py <Debug/Release> <v8-path> <build-path> [<build-flags>]...
+
+Build flags are passed either as "strings" or numeric value. True/false
+are represented as 1/0. E.g.
+
+  v8_promise_internal_field_count=2
+  target_cpu="x64"
+  v8_enable_disassembler=0
+"""
+
+import os
+import subprocess
+import sys
+
+import node_common
+
+GN_ARGS = [
+  "v8_monolithic = true",
+  "is_component_build = false",
+  "v8_use_external_startup_data = false",
+  "use_custom_libcxx = false",
+  "use_sysroot = false",
+]
+
+BUILD_SUBDIR = "gn"
+
+# TODO: make this cross-platform.
+GN_SUBDIR = ["buildtools", "linux64", "gn"]
+
+def Build(v8_path, build_path, depot_tools, is_debug, build_flags):
+  print "Setting GN args."
+  lines = []
+  lines.extend(GN_ARGS)
+  for flag in build_flags:
+    flag = flag.replace("=1", "=true")
+    flag = flag.replace("=0", "=false")
+    flag = flag.replace("target_cpu=ia32", "target_cpu=\"x86\"")
+    lines.append(flag)
+  lines.append("is_debug = %s" % ("true" if is_debug else "false"))
+  with open(os.path.join(build_path, "args.gn"), "w") as args_file:
+    args_file.write("\n".join(lines))
+  gn = os.path.join(v8_path, *GN_SUBDIR)
+  subprocess.check_call([gn, "gen", "-C", build_path], cwd=v8_path)
+  ninja = os.path.join(depot_tools, "ninja")
+  print "Building."
+  subprocess.check_call([ninja, "-v", "-C", build_path, "v8_monolith"],
+                        cwd=v8_path)
+
+def Main(v8_path, build_path, is_debug, build_flags):
+  # Verify paths.
+  v8_path = os.path.abspath(v8_path)
+  assert os.path.isdir(v8_path)
+  build_path = os.path.abspath(build_path)
+  build_path = os.path.join(build_path, BUILD_SUBDIR)
+  if not os.path.isdir(build_path):
+    os.makedirs(build_path)
+
+  # Check that we have depot tools.
+  depot_tools = node_common.EnsureDepotTools(v8_path, False)
+
+  # Build with GN.
+  Build(v8_path, build_path, depot_tools, is_debug, build_flags)
+
+if __name__ == "__main__":
+  # TODO: use argparse to parse arguments.
+  build_mode = sys.argv[1]
+  v8_path = sys.argv[2]
+  build_path = sys.argv[3]
+  assert build_mode == "Debug" or build_mode == "Release"
+  is_debug = build_mode == "Debug"
+  # TODO: introduce "--" flag for pass-through flags.
+  build_flags = sys.argv[4:]
+  Main(v8_path, build_path, is_debug, build_flags)

tools/node/fetch_deps.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Use this script to fetch all dependencies for V8 to run build_gn.py.
+
+Usage: fetch_deps.py <v8-path>
+"""
+
+import os
+import subprocess
+import sys
+
+import node_common
+
+GCLIENT_SOLUTION = [
+  { "name"        : "v8",
+    "url"         : "https://chromium.googlesource.com/v8/v8.git",
+    "deps_file"   : "DEPS",
+    "managed"     : False,
+    "custom_deps" : {
+      # These deps are already part of Node.js.
+      "v8/base/trace_event/common" : None,
+      "v8/testing/gtest"           : None,
+      "v8/third_party/jinja2"      : None,
+      "v8/third_party/markupsafe"  : None,
+      # These deps are unnecessary for building.
+      "v8/test/benchmarks/data"               : None,
+      "v8/testing/gmock"                      : None,
+      "v8/test/mozilla/data"                  : None,
+      "v8/test/test262/data"                  : None,
+      "v8/test/test262/harness"               : None,
+      "v8/test/wasm-js"                       : None,
+      "v8/third_party/android_tools"          : None,
+      "v8/third_party/catapult"               : None,
+      "v8/third_party/colorama/src"           : None,
+      "v8/third_party/instrumented_libraries" : None,
+      "v8/tools/gyp"                          : None,
+      "v8/tools/luci-go"                      : None,
+      "v8/tools/swarming_client"              : None,
+    },
+    "custom_vars": {
+      "build_for_node" : True,
+    },
+  },
+]
+
+def EnsureGit(v8_path):
+  expected_git_dir = os.path.join(v8_path, ".git")
+  actual_git_dir = subprocess.check_output(
+      ["git", "rev-parse", "--absolute-git-dir"], cwd=v8_path).strip()
+  if expected_git_dir == actual_git_dir:
+    print "V8 is tracked stand-alone by git."
+    return False
+  print "Initializing temporary git repository in v8."
+  subprocess.check_call(["git", "init"], cwd=v8_path)
+  subprocess.check_call(["git", "commit", "--allow-empty", "-m", "init"],
+                        cwd=v8_path)
+  return True
+
+def FetchDeps(v8_path, depot_tools):
+  temporary_git = EnsureGit(v8_path)
+  try:
+    print "Fetching dependencies."
+    gclient = os.path.join(depot_tools, "gclient")
+    spec = "solutions = %s" % GCLIENT_SOLUTION
+    subprocess.check_call([gclient, "sync", "--spec", spec],
+                          cwd=os.path.join(v8_path, os.path.pardir))
+  except:
+    raise
+  finally:
+    if temporary_git:
+      node_common.UninitGit(v8_path)
+
+def Main(v8_path):
+  # Verify paths.
+  v8_path = os.path.abspath(v8_path)
+  assert os.path.isdir(v8_path)
+
+  # Check out depot_tools if necessary.
+  depot_tools = node_common.EnsureDepotTools(v8_path, True)
+
+  # Fetch dependencies with gclient.
+  FetchDeps(v8_path, depot_tools)
+
+if __name__ == "__main__":
+  # Disabled for now.
+  pass
+  # Main(sys.argv[1])

tools/node/node_common.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import shutil
+import stat
+import subprocess
+
+DEPOT_TOOLS_URL = \
+  "https://chromium.googlesource.com/chromium/tools/depot_tools.git"
+
+def EnsureDepotTools(v8_path, fetch_if_not_exist):
+  def _Get(v8_path):
+    depot_tools = os.path.join(v8_path, "_depot_tools")
+    try:
+      gclient_path = os.path.join(depot_tools, "gclient")
+      gclient_check = subprocess.check_output([gclient_path, "--version"])
+      if "gclient.py" in gclient_check:
+        return depot_tools
+    except:
+      pass
+    if fetch_if_not_exist:
+      print "Checking out depot_tools."
+      subprocess.check_call(["git", "clone", DEPOT_TOOLS_URL, depot_tools])
+      return depot_tools
+    return None
+  depot_tools = _Get(v8_path)
+  assert depot_tools is not None
+  print "Using depot tools in %s" % depot_tools
+  return depot_tools
+
+def UninitGit(v8_path):
+  print "Uninitializing temporary git repository"
+  target = os.path.join(v8_path, ".git")
+  if os.path.isdir(target):
+    print ">> Cleaning up %s" % target
+    def OnRmError(func, path, exec_info):
+      # This might happen on Windows
+      os.chmod(path, stat.S_IWRITE)
+      os.unlink(path)
+    shutil.rmtree(target, onerror=OnRmError)

tools/node/test_backport_node.py

@@ -10,7 +10,6 @@ import sys
 import tempfile
 import unittest
 
-from common_includes import FileToText
 import backport_node
 
 # Base paths.
@@ -65,7 +64,7 @@ class TestUpdateNode(unittest.TestCase):
 
     # Check version.
     version_file = os.path.join(node_cwd, "deps", "v8", "include", "v8-version.h")
-    self.assertIn('#define V8_PATCH_LEVEL 4322', FileToText(version_file))
+    self.assertIn('#define V8_PATCH_LEVEL 4322', backport_node.FileToText(version_file))
 
 if __name__ == "__main__":
   unittest.main()

tools/node/test_update_node.py

@@ -32,6 +32,7 @@ EXPECTED_GITIGNORE = """
 EXPECTED_GIT_DIFF = """
  create mode 100644 deps/v8/base/trace_event/common/common
  rename deps/v8/baz/{delete_me => v8_new} (100%)
+ delete mode 100644 deps/v8/include/v8-version.h
  rename deps/v8/{delete_me => new/v8_new} (100%)
  create mode 100644 deps/v8/third_party/jinja2/jinja2
  create mode 100644 deps/v8/third_party/markupsafe/markupsafe

tools/node/update_node.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Use this script to update V8 in a Node.js checkout.
+
+Requirements:
+  - Node.js checkout in which V8 should be updated.
+  - V8 checkout at the commit to which Node.js should be updated.
+
+Usage:
+  $ update_node.py <path_to_v8> <path_to_node>
+
+  This will synchronize the content of <path_to_node>/deps/v8 with <path_to_v8>,
+  and a few V8 dependencies require in Node.js. It will also update .gitignore
+  appropriately.
+
+Optional flags:
+  --gclient     Run `gclient sync` on the V8 checkout before updating.
+  --commit      Create commit with the updated V8 in the Node.js checkout.
+  --with-patch  Also include currently staged files in the V8 checkout.
+"""
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+import stat
+import node_common
+
+TARGET_SUBDIR = os.path.join("deps", "v8")
+
+SUB_REPOSITORIES = [ ["base", "trace_event", "common"],
+                     ["testing", "gtest"],
+                     ["third_party", "jinja2"],
+                     ["third_party", "markupsafe"] ]
+
+DELETE_FROM_GITIGNORE = [ "/base",
+                          "/testing/gtest",
+                          "/third_party/jinja2",
+                          "/third_party/markupsafe" ]
+
+# Node.js requires only a single header file from gtest to build V8.
+# Both jinja2 and markupsafe are required to generate part of the inspector.
+ADD_TO_GITIGNORE = [ "/testing/gtest/*",
+                     "!/testing/gtest/include",
+                     "/testing/gtest/include/*",
+                     "!/testing/gtest/include/gtest",
+                     "/testing/gtest/include/gtest/*",
+                     "!/testing/gtest/include/gtest/gtest_prod.h",
+                     "!/third_party/jinja2",
+                     "!/third_party/markupsafe" ]
+
+def RunGclient(path):
+  assert os.path.isdir(path)
+  print ">> Running gclient sync"
+  subprocess.check_call(["gclient", "sync", "--nohooks"], cwd=path)
+
+def CommitPatch(options):
+  """Makes a dummy commit for the changes in the index.
+
+  On trybots, bot_updated applies the patch to the index. We commit it to make
+  the fake git clone fetch it into node.js. We can leave the commit, as
+  bot_update will ensure a clean state on each run.
+  """
+  print ">> Committing patch"
+  subprocess.check_call(
+      ["git", "-c", "user.name=fake", "-c", "user.email=fake@chromium.org",
+       "commit", "--allow-empty", "-m", "placeholder-commit"],
+      cwd=options.v8_path,
+  )
+
+def UpdateTarget(repository, options):
+  source = os.path.join(options.v8_path, *repository)
+  target = os.path.join(options.node_path, TARGET_SUBDIR, *repository)
+  print ">> Updating target directory %s" % target
+  print ">>     from active branch at %s" % source
+  if not os.path.exists(target):
+    os.makedirs(target)
+  # Remove possible remnants of previous incomplete runs.
+  node_common.UninitGit(target)
+
+  git_commands = [
+    ["git", "init"],                             # initialize target repo
+    ["git", "remote", "add", "origin", source],  # point to the source repo
+    ["git", "fetch", "origin", "HEAD"],          # sync to the current branch
+    ["git", "reset", "--hard", "FETCH_HEAD"],    # reset to the current branch
+    ["git", "clean", "-fd"],                     # delete removed files
+  ]
+  try:
+    for command in git_commands:
+      subprocess.check_call(command, cwd=target)
+  except:
+    raise
+  finally:
+    node_common.UninitGit(target)
+
+def UpdateGitIgnore(options):
+  file_name = os.path.join(options.node_path, TARGET_SUBDIR, ".gitignore")
+  assert os.path.isfile(file_name)
+  print ">> Updating .gitignore with lines"
+  with open(file_name) as gitignore:
+    content = gitignore.readlines()
+  content = [x.strip() for x in content]
+  for x in DELETE_FROM_GITIGNORE:
+    if x in content:
+      print "- %s" % x
+      content.remove(x)
+  for x in ADD_TO_GITIGNORE:
+    if x not in content:
+      print "+ %s" % x
+      content.append(x)
+  content.sort(key=lambda x: x[1:] if x.startswith("!") else x)
+  with open(file_name, "w") as gitignore:
+    for x in content:
+      gitignore.write("%s\n" % x)
+
+def CreateCommit(options):
+  print ">> Creating commit."
+  # Find git hash from source.
+  githash = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"],
+                                    cwd=options.v8_path).strip()
+  # Create commit at target.
+  git_commands = [
+    ["git", "checkout", "-b", "update_v8_to_%s" % githash],  # new branch
+    ["git", "add", "."],                                     # add files
+    ["git", "commit", "-m", "Update V8 to %s" % githash]     # new commit
+  ]
+  for command in git_commands:
+    subprocess.check_call(command, cwd=options.node_path)
+
+def ParseOptions(args):
+  parser = argparse.ArgumentParser(description="Update V8 in Node.js")
+  parser.add_argument("v8_path", help="Path to V8 checkout")
+  parser.add_argument("node_path", help="Path to Node.js checkout")
+  parser.add_argument("--gclient", action="store_true", help="Run gclient sync")
+  parser.add_argument("--commit", action="store_true", help="Create commit")
+  parser.add_argument("--with-patch", action="store_true",
+                      help="Apply also staged files")
+  options = parser.parse_args(args)
+  assert os.path.isdir(options.v8_path)
+  options.v8_path = os.path.abspath(options.v8_path)
+  assert os.path.isdir(options.node_path)
+  options.node_path = os.path.abspath(options.node_path)
+  return options
+
+def Main(args):
+  options = ParseOptions(args)
+  if options.gclient:
+    RunGclient(options.v8_path)
+  # Commit patch on trybots to main V8 repository.
+  if options.with_patch:
+    CommitPatch(options)
+  # Update main V8 repository.
+  UpdateTarget([""], options)
+  # Patch .gitignore before updating sub-repositories.
+  UpdateGitIgnore(options)
+  for repo in SUB_REPOSITORIES:
+    UpdateTarget(repo, options)
+  if options.commit:
+    CreateCommit(options)
+
+if __name__ == "__main__":
+  Main(sys.argv[1:])