Revert "[infra] Make most builds idempotent"

This reverts commit 57aa178dea.

Reason for revert: Causes Build-Debian9-Clang-x86-devrel-Android_SKQP to fail

Original change's description:
> [infra] Make most builds idempotent
> 
> Change-Id: I084645a46feda183747e9d7fc3b40b1aa56c5f78
> Reviewed-on: https://skia-review.googlesource.com/c/skia/+/211103
> Commit-Queue: Eric Boren <borenet@google.com>
> Reviewed-by: Ben Wagner aka dogben <benjaminwagner@google.com>

TBR=borenet@google.com,benjaminwagner@google.com

Change-Id: I886483a01c53da2cb7816651a02b35b780ffcf67
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/228441
Reviewed-by: Ben Wagner aka dogben <benjaminwagner@google.com>
Commit-Queue: Ben Wagner aka dogben <benjaminwagner@google.com>

PRESUBMIT.py

@@ -192,28 +192,6 @@ def _CheckGNFormatted(input_api, output_api):
   return results
 
 
-def _CheckCompileIsolate(input_api, output_api):
-  """Ensure that gen_compile_isolate.py does not change compile.isolate."""
-  # Only run the check if files were added or removed.
-  results = []
-  script = os.path.join('infra', 'bots', 'gen_compile_isolate.py')
-  isolate = os.path.join('infra', 'bots', 'compile.isolated')
-  for f in input_api.AffectedFiles():
-    if f.Action() in ('A', 'D', 'R'):
-      break
-    if f.LocalPath() in (script, isolate):
-      break
-  else:
-    return results
-
-  cmd = ['python', script, 'test']
-  try:
-    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-  except subprocess.CalledProcessError as e:
-    results.append(output_api.PresubmitError(e.output))
-  return results
-
-
 class _WarningsAsErrors():
   def __init__(self, output_api):
     self.output_api = output_api
@@ -250,7 +228,6 @@ def _CommonChecks(input_api, output_api):
   results.extend(_CopyrightChecks(input_api, output_api,
                                   source_file_filter=sources))
   results.extend(_ToolFlags(input_api, output_api))
-  results.extend(_CheckCompileIsolate(input_api, output_api))
   return results
 
 

infra/bots/compile.isolate

@@ -1,47 +0,0 @@
-{
-  'includes': [
-    'run_recipe.isolate',
-  ],
-  'variables': {
-    'files': [
-      '../../../.gclient',
-      '../../.clang-format',
-      '../../.clang-tidy',
-      '../../.gn',
-      '../../BUILD.gn',
-      '../../bench',
-      '../../bin/fetch-clang-format',
-      '../../bin/fetch-gn',
-      '../../buildtools',
-      '../../dm',
-      '../../docs/examples',
-      '../../example',
-      '../../experimental',
-      '../../fuzz',
-      '../../gm',
-      '../../gn',
-      '../../include',
-      '../../modules',
-      '../../platform_tools/android/apps/arcore/src/main/cpp',
-      '../../platform_tools/android/apps/skottie/src/main/cpp',
-      '../../platform_tools/android/launcher/skia_launcher.cpp',
-      '../../platform_tools/android/vulkan/Skia_Vulkan_Android.h',
-      '../../platform_tools/libraries/include/arcore_c_api.h',
-      '../../resources',
-      '../../samplecode',
-      '../../src',
-      '../../tests',
-      '../../third_party',
-      '../../tools',
-      '../canvaskit',
-      '../pathkit',
-      'assets/android_ndk_darwin/VERSION',
-      'assets/android_ndk_linux/VERSION',
-      'assets/android_ndk_windows/VERSION',
-      'assets/cast_toolchain/VERSION',
-      'assets/clang_linux/VERSION',
-      'assets/clang_win/VERSION',
-      'assets/mips64el_toolchain_linux/VERSION',
-    ],
-  },
-}

infra/bots/gen_compile_isolate.py

@@ -1,233 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2019 Google LLC
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import difflib
-import os
-import re
-import subprocess
-import sys
-
-
-# Any files in Git which match these patterns will be included, either directly
-# or indirectly via a parent dir.
-PATH_PATTERNS = [
-  r'.*\.c$',
-  r'.*\.cc$',
-  r'.*\.cpp$',
-  r'.*\.gn$',
-  r'.*\.gni$',
-  r'.*\.h$',
-]
-
-# These paths are always added to the inclusion list. Note that they may not
-# appear in the isolate if they are included indirectly via a parent dir.
-EXPLICIT_PATHS = [
-  '../.gclient',
-  '.clang-format',
-  '.clang-tidy',
-  'bin/fetch-clang-format',
-  'bin/fetch-gn',
-  'buildtools',
-  'infra/bots/assets/android_ndk_darwin/VERSION',
-  'infra/bots/assets/android_ndk_linux/VERSION',
-  'infra/bots/assets/android_ndk_windows/VERSION',
-  'infra/bots/assets/cast_toolchain/VERSION',
-  'infra/bots/assets/clang_linux/VERSION',
-  'infra/bots/assets/clang_win/VERSION',
-  'infra/bots/assets/mips64el_toolchain_linux/VERSION',
-  'infra/canvaskit',
-  'infra/pathkit',
-  'resources',
-  'third_party/externals',
-]
-
-# If a parent path contains more than this many immediate child paths (ie. files
-# and dirs which are directly inside it as opposed to indirect descendants), we
-# will include the parent in the isolate file instead of the children. This
-# results in a simpler isolate file which should need to be changed less often.
-COMBINE_PATHS_THRESHOLD = 3
-
-# Template for the isolate file content.
-ISOLATE_TMPL = '''{
-  'includes': [
-    'run_recipe.isolate',
-  ],
-  'variables': {
-    'files': [
-%s
-    ],
-  },
-}
-'''
-
-# Absolute path to the infra/bots dir.
-INFRABOTS_DIR = os.path.realpath(os.path.dirname(os.path.abspath(__file__)))
-
-# Absolute path to the compile.isolate file.
-ISOLATE_FILE = os.path.join(INFRABOTS_DIR, 'compile.isolate')
-
-
-def all_paths():
-  """Return all paths which are checked in to git."""
-  repo_root = os.path.abspath(os.path.join(INFRABOTS_DIR, os.pardir, os.pardir))
-  output = subprocess.check_output(['git', 'ls-files'], cwd=repo_root).rstrip()
-  return output.splitlines()
-
-
-def get_relevant_paths():
-  """Return all checked-in paths in PATH_PATTERNS or EXPLICIT_PATHS."""
-  paths = []
-  for f in all_paths():
-    for regexp in PATH_PATTERNS:
-      if re.match(regexp, f):
-        paths.append(f)
-        break
-
-  paths.extend(EXPLICIT_PATHS)
-  return paths
-
-
-class Tree(object):
-  """Tree helps with deduplicating and collapsing paths."""
-  class Node(object):
-    """Node represents an individual node in a Tree."""
-    def __init__(self, name):
-      self._children = {}
-      self._name = name
-      self._is_leaf = False
-
-    @property
-    def is_root(self):
-      """Return True iff this is the root node."""
-      return self._name is None
-
-    def add(self, entry):
-      """Add the given entry (given as a list of strings) to the Node."""
-      # Remove the first element if we're not the root node.
-      if not self.is_root:
-        if entry[0] != self._name:
-          raise ValueError('Cannot add a non-matching entry to a Node!')
-        entry = entry[1:]
-
-        # If the entry is now empty, this node is a leaf.
-        if not entry:
-          self._is_leaf = True
-          return
-
-      # Add a child node.
-      if not self._is_leaf:
-        child = self._children.get(entry[0])
-        if not child:
-          child = Tree.Node(entry[0])
-          self._children[entry[0]] = child
-        child.add(entry)
-
-        # If we have more than COMBINE_PATHS_THRESHOLD immediate children,
-        # combine them into this node.
-        immediate_children = 0
-        for child in self._children.itervalues():
-          if child._is_leaf:
-            immediate_children += 1
-        if not self.is_root and immediate_children >= COMBINE_PATHS_THRESHOLD:
-          self._is_leaf = True
-          self._children = {}
-
-    def entries(self):
-      """Return the entries represented by this node and its children.
-
-      Will not return children in the following cases:
-        - This Node is a leaf, ie. it represents an entry which was explicitly
-          inserted into the Tree, as opposed to only part of a path to other
-          entries.
-        - This Node has immediate children exceeding COMBINE_PATHS_THRESHOLD and
-          thus has been upgraded to a leaf node.
-      """
-      if self._is_leaf:
-        return [self._name]
-      rv = []
-      for child in self._children.itervalues():
-        for entry in child.entries():
-          if not self.is_root:
-            entry = self._name + '/' + entry
-          rv.append(entry)
-      return rv
-
-  def __init__(self):
-    self._root = Tree.Node(None)
-
-  def add(self, entry):
-    """Add the given entry to the tree."""
-    split = entry.split('/')
-    if split[-1] == '':
-      split = split[:-1]
-    self._root.add(split)
-
-  def entries(self):
-    """Return the list of entries in the tree.
-
-    Entries will be de-duplicated as follows:
-      - Any entry which is a sub-path of another entry will not be returned.
-      - Any entry which was not explicitly inserted but has children exceeding
-        the COMBINE_PATHS_THRESHOLD will be returned while its children will not
-        be returned.
-    """
-    return self._root.entries()
-
-
-def relpath(repo_path):
-  """Return a relative path to the given path within the repo.
-
-  The path is relative to the infra/bots dir, where the compile.isolate file
-  lives.
-  """
-  repo_path = '../../' + repo_path
-  repo_path = repo_path.replace('../../infra/', '../')
-  repo_path = repo_path.replace('../bots/', '')
-  return repo_path
-
-
-def get_isolate_content(paths):
-  """Construct the new content of the isolate file based on the given paths."""
-  lines = ['      \'%s\',' % relpath(p) for p in paths]
-  lines.sort()
-  return ISOLATE_TMPL % '\n'.join(lines)
-
-
-def main():
-  """Regenerate the compile.isolate file, or verify that it hasn't changed."""
-  testing = False
-  if len(sys.argv) == 2 and sys.argv[1] == 'test':
-    testing = True
-  elif len(sys.argv) != 1:
-    print >> sys.stderr, 'Usage: %s [test]' % sys.argv[0]
-    sys.exit(1)
-
-  tree = Tree()
-  for p in get_relevant_paths():
-    tree.add(p)
-  content = get_isolate_content(tree.entries())
-
-  if testing:
-    with open(ISOLATE_FILE, 'rb') as f:
-      expect_content = f.read()
-    if content != expect_content:
-      print >> sys.stderr, 'Found diff in %s:' % ISOLATE_FILE
-      a = expect_content.splitlines()
-      b = content.splitlines()
-      diff = difflib.context_diff(a, b, lineterm='')
-      for line in diff:
-        sys.stderr.write(line + '\n')
-      print >> sys.stderr, 'You may need to run:\n\n\tpython %s' % sys.argv[0]
-      sys.exit(1)
-  else:
-    with open(ISOLATE_FILE, 'wb') as f:
-      f.write(content)
-
-
-if __name__ == '__main__':
-  main()

infra/bots/gen_tasks.go

@@ -12,7 +12,6 @@ import (
 	"go.skia.org/skia/infra/bots/gen_tasks_logic"
 )
 
-// Regenerate the tasks.json file.
 func main() {
 	gen_tasks_logic.GenTasks(nil)
 }

infra/bots/gen_tasks_logic/gen_tasks_logic.go

@@ -997,26 +997,8 @@ func attempts(name string) int {
 // compile generates a compile task. Returns the name of the last task in the
 // generated chain of tasks, which the Job should add as a dependency.
 func (b *builder) compile(name string, parts map[string]string) string {
-	recipe := "compile"
+	task := b.kitchenTask(name, "compile", "swarm_recipe.isolate", b.cfg.ServiceAccountCompile, b.swarmDimensions(parts), EXTRA_PROPS, OUTPUT_BUILD)
-	isolate := "compile.isolate"
+	b.usesGit(task, name)
-	var props map[string]string
-	needSync := false
-	if strings.Contains(name, "NoDEPS") ||
-		strings.Contains(name, "CMake") ||
-		strings.Contains(name, "CommandBuffer") ||
-		strings.Contains(name, "Flutter") ||
-		strings.Contains(name, "ParentRevision") {
-		recipe = "sync_and_compile"
-		isolate = "swarm_recipe.isolate"
-		props = EXTRA_PROPS
-		needSync = true
-	}
-	task := b.kitchenTask(name, recipe, isolate, b.cfg.ServiceAccountCompile, b.swarmDimensions(parts), props, OUTPUT_BUILD)
-	if needSync {
-		b.usesGit(task, name)
-	} else {
-		task.Idempotent = true
-	}
 	usesDocker(task, name)
 
 	// Android bots require a toolchain.

infra/bots/recipes/compile.expected/Build-Debian9-Clang-universal-devrel-Android_SKQP.json

@@ -1,4 +1,113 @@
 [
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "ensure-directory",
+      "--mode",
+      "0777",
+      "[START_DIR]/cache/work"
+    ],
+    "infra_step": true,
+    "name": "makedirs checkout_path"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "remove",
+      "[START_DIR]/cache/work/.gclient_entries"
+    ],
+    "infra_step": true,
+    "name": "remove [START_DIR]/cache/work/.gclient_entries"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
+      "--spec-path",
+      "cache_dir = '[START_DIR]/cache/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]",
+      "--patch_root",
+      "skia",
+      "--revision_mapping_file",
+      "{\"got_revision\": \"skia\"}",
+      "--git-cache-dir",
+      "[START_DIR]/cache/git",
+      "--cleanup-dir",
+      "[CLEANUP]/bot_update",
+      "--output_json",
+      "/path/to/tmp/json",
+      "--revision",
+      "skia@abc123"
+    ],
+    "cwd": "[START_DIR]/cache/work",
+    "env_prefixes": {
+      "PATH": [
+        "RECIPE_REPO[depot_tools]"
+      ]
+    },
+    "infra_step": true,
+    "name": "bot_update",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@Some step text@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"did_run\": true, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"fixed_revisions\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"skia\": \"abc123\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"manifest\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"skia\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"repository\": \"https://fake.org/skia.git\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"patch_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"patch_root\": \"skia\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"properties\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"root\": \"skia\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"source_manifest\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"directories\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"skia\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"git_checkout\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@          \"repo_url\": \"https://fake.org/skia.git\", @@@",
+      "@@@STEP_LOG_LINE@json.output@          \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
+      "@@@STEP_LOG_LINE@json.output@        }@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"version\": 0@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"step_text\": \"Some step text\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
+      "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "ensure-directory",
+      "--mode",
+      "0777",
+      "[START_DIR]/tmp"
+    ],
+    "infra_step": true,
+    "name": "makedirs tmp_dir"
+  },
   {
     "cmd": [
       "python",
@@ -22,7 +131,7 @@
       "--workdir",
       "/SRC/skia/infra/skqp",
       "--volume",
-      "[START_DIR]:/SRC",
+      "[START_DIR]/cache/work:/SRC",
       "--volume",
       "[START_DIR]/cache/docker/skqp:/OUT",
       "gcr.io/skia-public/android-skqp:r20_v1",
@@ -122,7 +231,7 @@
       "--json-output",
       "/path/to/tmp/json",
       "copy",
-      "[START_DIR]/skia/infra/cts/whitelist_devices.json",
+      "[START_DIR]/cache/work/skia/infra/cts/whitelist_devices.json",
       "[START_DIR]/[SWARM_OUT_DIR]"
     ],
     "infra_step": true,

infra/bots/recipes/compile.expected/Build-Debian9-Clang-x86_64-Release-NoDEPS.json

@@ -3,9 +3,9 @@
     "cmd": [
       "python",
       "-u",
-      "RECIPE_MODULE[depot_tools::git]\\resources\\git_setup.py",
+      "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py",
       "--path",
-      "[START_DIR]\\skia",
+      "[START_DIR]/skia",
       "--url",
       "https://skia.googlesource.com/skia.git"
     ],
@@ -20,9 +20,9 @@
       "abc123",
       "--progress"
     ],
-    "cwd": "[START_DIR]\\skia",
+    "cwd": "[START_DIR]/skia",
     "env": {
-      "PATH": "RECIPE_REPO[depot_tools];<PATH>"
+      "PATH": "RECIPE_REPO[depot_tools]:<PATH>"
     },
     "infra_step": true,
     "name": "git fetch"
@@ -34,7 +34,7 @@
       "-f",
       "FETCH_HEAD"
     ],
-    "cwd": "[START_DIR]\\skia",
+    "cwd": "[START_DIR]/skia",
     "infra_step": true,
     "name": "git checkout"
   },
@@ -44,7 +44,7 @@
       "rev-parse",
       "HEAD"
     ],
-    "cwd": "[START_DIR]\\skia",
+    "cwd": "[START_DIR]/skia",
     "infra_step": true,
     "name": "read revision",
     "~followup_annotations": [
@@ -59,7 +59,7 @@
       "-d",
       "-x"
     ],
-    "cwd": "[START_DIR]\\skia",
+    "cwd": "[START_DIR]/skia",
     "infra_step": true,
     "name": "git clean"
   },
@@ -67,13 +67,13 @@
     "cmd": [
       "python",
       "-u",
-      "RECIPE_MODULE[recipe_engine::file]\\resources\\fileutil.py",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
       "--json-output",
       "/path/to/tmp/json",
       "ensure-directory",
       "--mode",
       "0777",
-      "[START_DIR]\\tmp"
+      "[START_DIR]/tmp"
     ],
     "infra_step": true,
     "name": "makedirs tmp_dir"
@@ -82,41 +82,41 @@
     "cmd": [
       "python",
       "-u",
-      "RECIPE_MODULE[recipe_engine::file]\\resources\\fileutil.py",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
       "--json-output",
       "/path/to/tmp/json",
       "copy",
-      "[START_DIR]\\skia\\infra\\bots\\assets\\clang_win\\VERSION",
+      "[START_DIR]/skia/infra/bots/assets/clang_linux/VERSION",
       "/path/to/tmp/"
     ],
     "infra_step": true,
-    "name": "Get clang_win VERSION"
+    "name": "Get clang_linux VERSION"
   },
   {
     "cmd": [
       "python",
       "-u",
-      "[START_DIR]\\skia\\bin\\fetch-gn"
+      "[START_DIR]/skia/bin/fetch-gn"
     ],
-    "cwd": "[START_DIR]\\skia",
+    "cwd": "[START_DIR]/skia",
     "env": {
       "CHROME_HEADLESS": "1",
-      "PATH": "<PATH>;RECIPE_REPO[depot_tools]"
+      "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
     },
     "infra_step": true,
     "name": "fetch-gn"
   },
   {
     "cmd": [
-      "[START_DIR]\\skia\\bin\\gn",
+      "[START_DIR]/skia/bin/gn",
       "gen",
-      "[START_DIR]\\skia\\out\\Build-Win10-Clang-x86_64-Release-NoDEPS\\Release_x64",
+      "[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS/Release",
-      "--args=cc=\"clang\" clang_win=\"[START_DIR]\\clang_win\" cxx=\"clang++\" extra_cflags=[\"-DDUMMY_clang_win_version=42\"] is_debug=false is_official_build=true skia_enable_fontmgr_empty=true skia_enable_gpu=true skia_enable_pdf=false skia_use_expat=false skia_use_freetype=false skia_use_harfbuzz=false skia_use_libjpeg_turbo=false skia_use_libpng=false skia_use_libwebp=false skia_use_vulkan=false skia_use_zlib=false target_cpu=\"x86_64\" werror=true win_sdk=\"[START_DIR]\\win_toolchain/win_sdk\" win_vc=\"[START_DIR]\\win_toolchain/VC\""
+      "--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\", \"-DDUMMY_clang_linux_version=42\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\"] is_debug=false is_official_build=true skia_enable_fontmgr_empty=true skia_enable_gpu=true skia_enable_pdf=false skia_use_expat=false skia_use_freetype=false skia_use_harfbuzz=false skia_use_libjpeg_turbo=false skia_use_libpng=false skia_use_libwebp=false skia_use_vulkan=false skia_use_zlib=false target_cpu=\"x86_64\" werror=true"
     ],
-    "cwd": "[START_DIR]\\skia",
+    "cwd": "[START_DIR]/skia",
     "env": {
       "CHROME_HEADLESS": "1",
-      "PATH": "<PATH>;RECIPE_REPO[depot_tools]"
+      "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
     },
     "name": "gn gen"
   },
@@ -124,12 +124,12 @@
     "cmd": [
       "ninja",
       "-C",
-      "[START_DIR]\\skia\\out\\Build-Win10-Clang-x86_64-Release-NoDEPS\\Release_x64"
+      "[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS/Release"
     ],
-    "cwd": "[START_DIR]\\skia",
+    "cwd": "[START_DIR]/skia",
     "env": {
       "CHROME_HEADLESS": "1",
-      "PATH": "<PATH>;RECIPE_REPO[depot_tools]"
+      "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
     },
     "name": "ninja"
   },
@@ -138,8 +138,8 @@
       "python",
       "-u",
       "import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'hello-opencl', 'hello-opencl.exe', 'nanobench', 'nanobench.exe', 'skpbench', 'skpbench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'skottie_tool', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n  os.makedirs(dst)\nexcept OSError as e:\n  if e.errno != errno.EEXIST:\n    raise\n\nfor pattern in build_products_whitelist:\n  path = os.path.join(src, pattern)\n  for f in glob.glob(path):\n    dst_path = os.path.join(dst, os.path.relpath(f, src))\n    if not os.path.isdir(os.path.dirname(dst_path)):\n      os.makedirs(os.path.dirname(dst_path))\n    print 'Copying build product %s to %s' % (f, dst_path)\n    shutil.move(f, dst_path)\n",
-      "[START_DIR]\\skia\\out\\Build-Win10-Clang-x86_64-Release-NoDEPS\\Release_x64",
+      "[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS/Release",
-      "[START_DIR]\\[SWARM_OUT_DIR]"
+      "[START_DIR]/[SWARM_OUT_DIR]"
     ],
     "infra_step": true,
     "name": "copy build products",
@@ -171,25 +171,6 @@
       "@@@STEP_LOG_END@python.inline@@@"
     ]
   },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "import psutil\nfor p in psutil.process_iter():\n  try:\n    if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):\n      p.kill()\n  except psutil._error.AccessDenied:\n    pass\n"
-    ],
-    "infra_step": true,
-    "name": "cleanup",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@import psutil@@@",
-      "@@@STEP_LOG_LINE@python.inline@for p in psutil.process_iter():@@@",
-      "@@@STEP_LOG_LINE@python.inline@  try:@@@",
-      "@@@STEP_LOG_LINE@python.inline@    if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):@@@",
-      "@@@STEP_LOG_LINE@python.inline@      p.kill()@@@",
-      "@@@STEP_LOG_LINE@python.inline@  except psutil._error.AccessDenied:@@@",
-      "@@@STEP_LOG_LINE@python.inline@    pass@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
   {
     "name": "$result"
   }

infra/bots/recipes/compile.expected/Build-Win-Clang-x86-Debug.json

@@ -1,4 +1,113 @@
 [
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]\\resources\\fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "ensure-directory",
+      "--mode",
+      "0777",
+      "[START_DIR]\\cache\\work"
+    ],
+    "infra_step": true,
+    "name": "makedirs checkout_path"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]\\resources\\fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "remove",
+      "[START_DIR]\\cache\\work\\.gclient_entries"
+    ],
+    "infra_step": true,
+    "name": "remove [START_DIR]\\cache\\work\\.gclient_entries"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[depot_tools::bot_update]\\resources\\bot_update.py",
+      "--spec-path",
+      "cache_dir = '[START_DIR]\\\\cache\\\\git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]",
+      "--patch_root",
+      "skia",
+      "--revision_mapping_file",
+      "{\"got_revision\": \"skia\"}",
+      "--git-cache-dir",
+      "[START_DIR]\\cache\\git",
+      "--cleanup-dir",
+      "[CLEANUP]\\bot_update",
+      "--output_json",
+      "/path/to/tmp/json",
+      "--revision",
+      "skia@abc123"
+    ],
+    "cwd": "[START_DIR]\\cache\\work",
+    "env_prefixes": {
+      "PATH": [
+        "RECIPE_REPO[depot_tools]"
+      ]
+    },
+    "infra_step": true,
+    "name": "bot_update",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@Some step text@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"did_run\": true, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"fixed_revisions\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"skia\": \"abc123\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"manifest\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"skia\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"repository\": \"https://fake.org/skia.git\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"patch_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"patch_root\": \"skia\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"properties\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"root\": \"skia\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"source_manifest\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"directories\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"skia\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"git_checkout\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@          \"repo_url\": \"https://fake.org/skia.git\", @@@",
+      "@@@STEP_LOG_LINE@json.output@          \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
+      "@@@STEP_LOG_LINE@json.output@        }@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"version\": 0@@@",
+      "@@@STEP_LOG_LINE@json.output@  }, @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"step_text\": \"Some step text\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
+      "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]\\resources\\fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "ensure-directory",
+      "--mode",
+      "0777",
+      "[START_DIR]\\tmp"
+    ],
+    "infra_step": true,
+    "name": "makedirs tmp_dir"
+  },
   {
     "cmd": [
       "python",
@@ -7,7 +116,7 @@
       "--json-output",
       "/path/to/tmp/json",
       "copy",
-      "[START_DIR]\\skia\\infra\\bots\\assets\\clang_win\\VERSION",
+      "[START_DIR]\\cache\\work\\skia\\infra\\bots\\assets\\clang_win\\VERSION",
       "/path/to/tmp/"
     ],
     "infra_step": true,
@@ -17,9 +126,9 @@
     "cmd": [
       "python",
       "-u",
-      "[START_DIR]\\skia\\bin\\fetch-gn"
+      "[START_DIR]\\cache\\work\\skia\\bin\\fetch-gn"
     ],
-    "cwd": "[START_DIR]\\skia",
+    "cwd": "[START_DIR]\\cache\\work\\skia",
     "env": {
       "CHROME_HEADLESS": "1",
       "PATH": "<PATH>;RECIPE_REPO[depot_tools]"
@@ -29,12 +138,12 @@
   },
   {
     "cmd": [
-      "[START_DIR]\\skia\\bin\\gn",
+      "[START_DIR]\\cache\\work\\skia\\bin\\gn",
       "gen",
       "[START_DIR]\\cache\\work\\skia\\out\\Build-Win-Clang-x86-Debug\\Debug",
       "--args=cc=\"clang\" clang_win=\"[START_DIR]\\clang_win\" cxx=\"clang++\" extra_cflags=[\"-O1\", \"-DDUMMY_clang_win_version=42\"] target_cpu=\"x86\" werror=true win_sdk=\"[START_DIR]\\win_toolchain/win_sdk\" win_vc=\"[START_DIR]\\win_toolchain/VC\""
     ],
-    "cwd": "[START_DIR]\\skia",
+    "cwd": "[START_DIR]\\cache\\work\\skia",
     "env": {
       "CHROME_HEADLESS": "1",
       "PATH": "<PATH>;RECIPE_REPO[depot_tools]"
@@ -47,7 +156,7 @@
       "-C",
       "[START_DIR]\\cache\\work\\skia\\out\\Build-Win-Clang-x86-Debug\\Debug"
     ],
-    "cwd": "[START_DIR]\\skia",
+    "cwd": "[START_DIR]\\cache\\work\\skia",
     "env": {
       "CHROME_HEADLESS": "1",
       "PATH": "<PATH>;RECIPE_REPO[depot_tools]"

infra/bots/recipes/compile.py

@@ -25,15 +25,53 @@ DEPS = [
 def RunSteps(api):
   api.vars.setup()
 
-  checkout_root = api.path['start_dir']
+  # Check out code.
-  out_dir = api.vars.cache_dir.join(
+  bot_update = True
-      'work', 'skia', 'out', api.vars.builder_name, api.vars.configuration)
+  checkout_root = api.checkout.default_checkout_root
+  checkout_chromium = False
+  checkout_flutter = False
+  extra_gclient_env = {}
+  flutter_android = False
+  parent_rev = False
+
+  if 'NoDEPS' in api.properties['buildername']:
+    bot_update = False
+    checkout_root = api.path['start_dir']
+  if 'CommandBuffer' in api.vars.builder_name:
+    checkout_chromium = True
+  if 'Flutter' in api.vars.builder_name:
+    checkout_root = checkout_root.join('flutter')
+    checkout_flutter = True
+    if 'Android' in api.vars.builder_name:
+      flutter_android = True
+  if 'ParentRevision' in api.vars.builder_name:
+    parent_rev = True
+
+  if bot_update:
+    api.checkout.bot_update(
+        checkout_root=checkout_root,
+        checkout_chromium=checkout_chromium,
+        checkout_flutter=checkout_flutter,
+        extra_gclient_env=extra_gclient_env,
+        flutter_android=flutter_android,
+        parent_rev=parent_rev)
+  else:
+    api.checkout.git(checkout_root=checkout_root)
+
+  api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir)
+
+  out_dir = checkout_root.join(
+      'skia', 'out', api.vars.builder_name, api.vars.configuration)
+  if 'Flutter' in api.vars.builder_name:
+    out_dir = checkout_root.join('src', 'out', 'android_release')
 
   try:
     api.build(checkout_root=checkout_root, out_dir=out_dir)
 
     # TODO(borenet): Move this out of the try/finally.
     dst = api.vars.swarming_out_dir
+    if 'ParentRevision' in api.vars.builder_name:
+      dst = api.vars.swarming_out_dir.join('ParentRevision')
     api.build.copy_build_products(out_dir=out_dir, dst=dst)
     if 'SKQP' in api.vars.extra_tokens:
       wlist = checkout_root.join(
@@ -58,6 +96,10 @@ for p in psutil.process_iter():
 
 TEST_BUILDERS = [
   'Build-Debian9-Clang-universal-devrel-Android_SKQP',
+  'Build-Debian9-Clang-x86_64-Release-NoDEPS',
+  'Build-Debian9-Clang-x86_64-Release-ParentRevision',
+  'Build-Debian9-GCC-x86_64-Release-Flutter_Android',
+  'Build-Mac-Clang-x86_64-Debug-CommandBuffer',
   'Build-Win-Clang-x86-Debug',
 ]
 

infra/bots/recipes/sync_and_compile.py

@@ -1,114 +0,0 @@
-# Copyright 2016 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-# Recipe module for Skia Swarming compile.
-
-
-DEPS = [
-  'build',
-  'checkout',
-  'recipe_engine/context',
-  'recipe_engine/file',
-  'recipe_engine/json',
-  'recipe_engine/path',
-  'recipe_engine/platform',
-  'recipe_engine/properties',
-  'recipe_engine/python',
-  'recipe_engine/step',
-  'run',
-  'vars',
-]
-
-
-def RunSteps(api):
-  api.vars.setup()
-
-  # Check out code.
-  bot_update = True
-  checkout_root = api.checkout.default_checkout_root
-  checkout_chromium = False
-  checkout_flutter = False
-  flutter_android = False
-  parent_rev = False
-
-  if 'NoDEPS' in api.properties['buildername']:
-    bot_update = False
-    checkout_root = api.path['start_dir']
-  if 'CommandBuffer' in api.vars.builder_name:
-    checkout_chromium = True
-  if 'Flutter' in api.vars.builder_name:
-    checkout_root = checkout_root.join('flutter')
-    checkout_flutter = True
-    if 'Android' in api.vars.builder_name:
-      flutter_android = True
-  if 'ParentRevision' in api.vars.builder_name:
-    parent_rev = True
-
-  if bot_update:
-    api.checkout.bot_update(
-        checkout_root=checkout_root,
-        checkout_chromium=checkout_chromium,
-        checkout_flutter=checkout_flutter,
-        flutter_android=flutter_android,
-        parent_rev=parent_rev)
-  else:
-    api.checkout.git(checkout_root=checkout_root)
-
-  api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir)
-
-  out_dir = checkout_root.join(
-      'skia', 'out', api.vars.builder_name, api.vars.configuration)
-  if 'Flutter' in api.vars.builder_name:
-    out_dir = checkout_root.join('src', 'out', 'android_release')
-
-  try:
-    api.build(checkout_root=checkout_root, out_dir=out_dir)
-
-    # TODO(borenet): Move this out of the try/finally.
-    dst = api.vars.swarming_out_dir
-    if 'ParentRevision' in api.vars.builder_name:
-      dst = api.vars.swarming_out_dir.join('ParentRevision')
-    api.build.copy_build_products(out_dir=out_dir, dst=dst)
-  finally:
-    if 'Win' in api.vars.builder_cfg.get('os', ''):
-      api.python.inline(
-          name='cleanup',
-          program='''import psutil
-for p in psutil.process_iter():
-  try:
-    if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):
-      p.kill()
-  except psutil._error.AccessDenied:
-    pass
-''',
-          infra_step=True)
-
-  api.run.check_failure()
-
-
-TEST_BUILDERS = [
-  'Build-Debian9-Clang-x86_64-Release-ParentRevision',
-  'Build-Debian9-GCC-x86_64-Release-Flutter_Android',
-  'Build-Mac-Clang-x86_64-Debug-CommandBuffer',
-  'Build-Win10-Clang-x86_64-Release-NoDEPS',
-]
-
-
-def GenTests(api):
-  for builder in TEST_BUILDERS:
-    test = (
-      api.test(builder) +
-      api.properties(buildername=builder,
-                     repository='https://skia.googlesource.com/skia.git',
-                     revision='abc123',
-                     path_config='kitchen',
-                     swarm_out_dir='[SWARM_OUT_DIR]') +
-      api.path.exists(
-          api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt')
-      )
-    )
-    if 'Win' in builder:
-      test += api.platform('win', 64)
-    yield test