9087d5f865
TEST=python -m unittest test_scripts.ScriptTest.testPrepareChangeLog R=ulan@chromium.org Review URL: https://codereview.chromium.org/98173003 git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@18232 ce2b1a6d-e550-0410-aec6-3dcde31c8c00
494 lines
16 KiB
Python
494 lines
16 KiB
Python
#!/usr/bin/env python
|
|
# Copyright 2013 the V8 project authors. All rights reserved.
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions are
|
|
# met:
|
|
#
|
|
# * Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# * Redistributions in binary form must reproduce the above
|
|
# copyright notice, this list of conditions and the following
|
|
# disclaimer in the documentation and/or other materials provided
|
|
# with the distribution.
|
|
# * Neither the name of Google Inc. nor the names of its
|
|
# contributors may be used to endorse or promote products derived
|
|
# from this software without specific prior written permission.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
import datetime
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import textwrap
|
|
import time
|
|
import urllib2
|
|
|
|
PERSISTFILE_BASENAME = "PERSISTFILE_BASENAME"
|
|
TEMP_BRANCH = "TEMP_BRANCH"
|
|
BRANCHNAME = "BRANCHNAME"
|
|
DOT_GIT_LOCATION = "DOT_GIT_LOCATION"
|
|
VERSION_FILE = "VERSION_FILE"
|
|
CHANGELOG_FILE = "CHANGELOG_FILE"
|
|
CHANGELOG_ENTRY_FILE = "CHANGELOG_ENTRY_FILE"
|
|
COMMITMSG_FILE = "COMMITMSG_FILE"
|
|
PATCH_FILE = "PATCH_FILE"
|
|
|
|
|
|
def TextToFile(text, file_name):
|
|
with open(file_name, "w") as f:
|
|
f.write(text)
|
|
|
|
|
|
def AppendToFile(text, file_name):
|
|
with open(file_name, "a") as f:
|
|
f.write(text)
|
|
|
|
|
|
def LinesInFile(file_name):
|
|
with open(file_name) as f:
|
|
for line in f:
|
|
yield line
|
|
|
|
|
|
def FileToText(file_name):
|
|
with open(file_name) as f:
|
|
return f.read()
|
|
|
|
|
|
def MSub(rexp, replacement, text):
|
|
return re.sub(rexp, replacement, text, flags=re.MULTILINE)
|
|
|
|
|
|
def Fill80(line):
|
|
# Replace tabs and remove surrounding space.
|
|
line = re.sub(r"\t", r" ", line.strip())
|
|
|
|
# Format with 8 characters indentation and line width 80.
|
|
return textwrap.fill(line, width=80, initial_indent=" ",
|
|
subsequent_indent=" ")
|
|
|
|
|
|
def GetLastChangeLogEntries(change_log_file):
|
|
result = []
|
|
for line in LinesInFile(change_log_file):
|
|
if re.search(r"^\d{4}-\d{2}-\d{2}:", line) and result: break
|
|
result.append(line)
|
|
return "".join(result)
|
|
|
|
|
|
def MakeComment(text):
|
|
return MSub(r"^( ?)", "#", text)
|
|
|
|
|
|
def StripComments(text):
|
|
# Use split not splitlines to keep terminal newlines.
|
|
return "\n".join(filter(lambda x: not x.startswith("#"), text.split("\n")))
|
|
|
|
|
|
def MakeChangeLogBody(commit_messages, auto_format=False):
|
|
result = ""
|
|
added_titles = set()
|
|
for (title, body, author) in commit_messages:
|
|
# TODO(machenbach): Better check for reverts. A revert should remove the
|
|
# original CL from the actual log entry.
|
|
title = title.strip()
|
|
if auto_format:
|
|
# Only add commits that set the LOG flag correctly.
|
|
log_exp = r"^[ \t]*LOG[ \t]*=[ \t]*(?:(?:Y(?:ES)?)|TRUE)"
|
|
if not re.search(log_exp, body, flags=re.I | re.M):
|
|
continue
|
|
# Never include reverts.
|
|
if title.startswith("Revert "):
|
|
continue
|
|
# Don't include duplicates.
|
|
if title in added_titles:
|
|
continue
|
|
|
|
# Add and format the commit's title and bug reference. Move dot to the end.
|
|
added_titles.add(title)
|
|
raw_title = re.sub(r"(\.|\?|!)$", "", title)
|
|
bug_reference = MakeChangeLogBugReference(body)
|
|
space = " " if bug_reference else ""
|
|
result += "%s\n" % Fill80("%s%s%s." % (raw_title, space, bug_reference))
|
|
|
|
# Append the commit's author for reference if not in auto-format mode.
|
|
if not auto_format:
|
|
result += "%s\n" % Fill80("(%s)" % author.strip())
|
|
|
|
result += "\n"
|
|
return result
|
|
|
|
|
|
def MakeChangeLogBugReference(body):
|
|
"""Grep for "BUG=xxxx" lines in the commit message and convert them to
|
|
"(issue xxxx)".
|
|
"""
|
|
crbugs = []
|
|
v8bugs = []
|
|
|
|
def AddIssues(text):
|
|
ref = re.match(r"^BUG[ \t]*=[ \t]*(.+)$", text.strip())
|
|
if not ref:
|
|
return
|
|
for bug in ref.group(1).split(","):
|
|
bug = bug.strip()
|
|
match = re.match(r"^v8:(\d+)$", bug)
|
|
if match: v8bugs.append(int(match.group(1)))
|
|
else:
|
|
match = re.match(r"^(?:chromium:)?(\d+)$", bug)
|
|
if match: crbugs.append(int(match.group(1)))
|
|
|
|
# Add issues to crbugs and v8bugs.
|
|
map(AddIssues, body.splitlines())
|
|
|
|
# Filter duplicates, sort, stringify.
|
|
crbugs = map(str, sorted(set(crbugs)))
|
|
v8bugs = map(str, sorted(set(v8bugs)))
|
|
|
|
bug_groups = []
|
|
def FormatIssues(prefix, bugs):
|
|
if len(bugs) > 0:
|
|
plural = "s" if len(bugs) > 1 else ""
|
|
bug_groups.append("%sissue%s %s" % (prefix, plural, ", ".join(bugs)))
|
|
|
|
FormatIssues("", v8bugs)
|
|
FormatIssues("Chromium ", crbugs)
|
|
|
|
if len(bug_groups) > 0:
|
|
return "(%s)" % ", ".join(bug_groups)
|
|
else:
|
|
return ""
|
|
|
|
|
|
# Some commands don't like the pipe, e.g. calling vi from within the script or
|
|
# from subscripts like git cl upload.
|
|
def Command(cmd, args="", prefix="", pipe=True):
|
|
# TODO(machenbach): Use timeout.
|
|
cmd_line = "%s %s %s" % (prefix, cmd, args)
|
|
print "Command: %s" % cmd_line
|
|
try:
|
|
if pipe:
|
|
return subprocess.check_output(cmd_line, shell=True)
|
|
else:
|
|
return subprocess.check_call(cmd_line, shell=True)
|
|
except subprocess.CalledProcessError:
|
|
return None
|
|
|
|
|
|
# Wrapper for side effects.
|
|
class SideEffectHandler(object):
|
|
def Command(self, cmd, args="", prefix="", pipe=True):
|
|
return Command(cmd, args, prefix, pipe)
|
|
|
|
def ReadLine(self):
|
|
return sys.stdin.readline().strip()
|
|
|
|
def ReadURL(self, url):
|
|
# pylint: disable=E1121
|
|
url_fh = urllib2.urlopen(url, None, 60)
|
|
try:
|
|
return url_fh.read()
|
|
finally:
|
|
url_fh.close()
|
|
|
|
def Sleep(self, seconds):
|
|
time.sleep(seconds)
|
|
|
|
def GetDate(self):
|
|
return datetime.date.today().strftime("%Y-%m-%d")
|
|
|
|
DEFAULT_SIDE_EFFECT_HANDLER = SideEffectHandler()
|
|
|
|
|
|
class Step(object):
|
|
def __init__(self, text, requires, number, config, state, options, handler):
|
|
self._text = text
|
|
self._requires = requires
|
|
self._number = number
|
|
self._config = config
|
|
self._state = state
|
|
self._options = options
|
|
self._side_effect_handler = handler
|
|
assert self._number >= 0
|
|
assert self._config is not None
|
|
assert self._state is not None
|
|
assert self._side_effect_handler is not None
|
|
|
|
def Config(self, key):
|
|
return self._config[key]
|
|
|
|
def Run(self):
|
|
if self._requires:
|
|
self.RestoreIfUnset(self._requires)
|
|
if not self._state[self._requires]:
|
|
return
|
|
print ">>> Step %d: %s" % (self._number, self._text)
|
|
self.RunStep()
|
|
|
|
def RunStep(self):
|
|
raise NotImplementedError
|
|
|
|
def Retry(self, cb, retry_on=None, wait_plan=None):
|
|
""" Retry a function.
|
|
Params:
|
|
cb: The function to retry.
|
|
retry_on: A callback that takes the result of the function and returns
|
|
True if the function should be retried. A function throwing an
|
|
exception is always retried.
|
|
wait_plan: A list of waiting delays between retries in seconds. The
|
|
maximum number of retries is len(wait_plan).
|
|
"""
|
|
retry_on = retry_on or (lambda x: False)
|
|
wait_plan = list(wait_plan or [])
|
|
wait_plan.reverse()
|
|
while True:
|
|
got_exception = False
|
|
try:
|
|
result = cb()
|
|
except Exception:
|
|
got_exception = True
|
|
if got_exception or retry_on(result):
|
|
if not wait_plan:
|
|
raise Exception("Retried too often. Giving up.")
|
|
wait_time = wait_plan.pop()
|
|
print "Waiting for %f seconds." % wait_time
|
|
self._side_effect_handler.Sleep(wait_time)
|
|
print "Retrying..."
|
|
else:
|
|
return result
|
|
|
|
def ReadLine(self, default=None):
|
|
# Don't prompt in forced mode.
|
|
if self._options and self._options.f and default is not None:
|
|
print "%s (forced)" % default
|
|
return default
|
|
else:
|
|
return self._side_effect_handler.ReadLine()
|
|
|
|
def Git(self, args="", prefix="", pipe=True, retry_on=None):
|
|
cmd = lambda: self._side_effect_handler.Command("git", args, prefix, pipe)
|
|
return self.Retry(cmd, retry_on, [5, 30])
|
|
|
|
def Editor(self, args):
|
|
return self._side_effect_handler.Command(os.environ["EDITOR"], args,
|
|
pipe=False)
|
|
|
|
def ReadURL(self, url, retry_on=None, wait_plan=None):
|
|
wait_plan = wait_plan or [3, 60, 600]
|
|
cmd = lambda: self._side_effect_handler.ReadURL(url)
|
|
return self.Retry(cmd, retry_on, wait_plan)
|
|
|
|
def GetDate(self):
|
|
return self._side_effect_handler.GetDate()
|
|
|
|
def Die(self, msg=""):
|
|
if msg != "":
|
|
print "Error: %s" % msg
|
|
print "Exiting"
|
|
raise Exception(msg)
|
|
|
|
def DieInForcedMode(self, msg=""):
|
|
if self._options and self._options.f:
|
|
msg = msg or "Not implemented in forced mode."
|
|
self.Die(msg)
|
|
|
|
def Confirm(self, msg):
|
|
print "%s [Y/n] " % msg,
|
|
answer = self.ReadLine(default="Y")
|
|
return answer == "" or answer == "Y" or answer == "y"
|
|
|
|
def DeleteBranch(self, name):
|
|
git_result = self.Git("branch").strip()
|
|
for line in git_result.splitlines():
|
|
if re.match(r".*\s+%s$" % name, line):
|
|
msg = "Branch %s exists, do you want to delete it?" % name
|
|
if self.Confirm(msg):
|
|
if self.Git("branch -D %s" % name) is None:
|
|
self.Die("Deleting branch '%s' failed." % name)
|
|
print "Branch %s deleted." % name
|
|
else:
|
|
msg = "Can't continue. Please delete branch %s and try again." % name
|
|
self.Die(msg)
|
|
|
|
def Persist(self, var, value):
|
|
value = value or "__EMPTY__"
|
|
TextToFile(value, "%s-%s" % (self._config[PERSISTFILE_BASENAME], var))
|
|
|
|
def Restore(self, var):
|
|
value = FileToText("%s-%s" % (self._config[PERSISTFILE_BASENAME], var))
|
|
value = value or self.Die("Variable '%s' could not be restored." % var)
|
|
return "" if value == "__EMPTY__" else value
|
|
|
|
def RestoreIfUnset(self, var_name):
|
|
if self._state.get(var_name) is None:
|
|
self._state[var_name] = self.Restore(var_name)
|
|
|
|
def InitialEnvironmentChecks(self):
|
|
# Cancel if this is not a git checkout.
|
|
if not os.path.exists(self._config[DOT_GIT_LOCATION]):
|
|
self.Die("This is not a git checkout, this script won't work for you.")
|
|
|
|
# TODO(machenbach): Don't use EDITOR in forced mode as soon as script is
|
|
# well tested.
|
|
# Cancel if EDITOR is unset or not executable.
|
|
if (not os.environ.get("EDITOR") or
|
|
Command("which", os.environ["EDITOR"]) is None):
|
|
self.Die("Please set your EDITOR environment variable, you'll need it.")
|
|
|
|
def CommonPrepare(self):
|
|
# Check for a clean workdir.
|
|
if self.Git("status -s -uno").strip() != "":
|
|
self.Die("Workspace is not clean. Please commit or undo your changes.")
|
|
|
|
# Persist current branch.
|
|
current_branch = ""
|
|
git_result = self.Git("status -s -b -uno").strip()
|
|
for line in git_result.splitlines():
|
|
match = re.match(r"^## (.+)", line)
|
|
if match:
|
|
current_branch = match.group(1)
|
|
break
|
|
self.Persist("current_branch", current_branch)
|
|
|
|
# Fetch unfetched revisions.
|
|
if self.Git("svn fetch") is None:
|
|
self.Die("'git svn fetch' failed.")
|
|
|
|
def PrepareBranch(self):
|
|
# Get ahold of a safe temporary branch and check it out.
|
|
self.RestoreIfUnset("current_branch")
|
|
if self._state["current_branch"] != self._config[TEMP_BRANCH]:
|
|
self.DeleteBranch(self._config[TEMP_BRANCH])
|
|
self.Git("checkout -b %s" % self._config[TEMP_BRANCH])
|
|
|
|
# Delete the branch that will be created later if it exists already.
|
|
self.DeleteBranch(self._config[BRANCHNAME])
|
|
|
|
def CommonCleanup(self):
|
|
self.RestoreIfUnset("current_branch")
|
|
self.Git("checkout -f %s" % self._state["current_branch"])
|
|
if self._config[TEMP_BRANCH] != self._state["current_branch"]:
|
|
self.Git("branch -D %s" % self._config[TEMP_BRANCH])
|
|
if self._config[BRANCHNAME] != self._state["current_branch"]:
|
|
self.Git("branch -D %s" % self._config[BRANCHNAME])
|
|
|
|
# Clean up all temporary files.
|
|
Command("rm", "-f %s*" % self._config[PERSISTFILE_BASENAME])
|
|
|
|
def ReadAndPersistVersion(self, prefix=""):
|
|
def ReadAndPersist(var_name, def_name):
|
|
match = re.match(r"^#define %s\s+(\d*)" % def_name, line)
|
|
if match:
|
|
value = match.group(1)
|
|
self.Persist("%s%s" % (prefix, var_name), value)
|
|
self._state["%s%s" % (prefix, var_name)] = value
|
|
for line in LinesInFile(self._config[VERSION_FILE]):
|
|
for (var_name, def_name) in [("major", "MAJOR_VERSION"),
|
|
("minor", "MINOR_VERSION"),
|
|
("build", "BUILD_NUMBER"),
|
|
("patch", "PATCH_LEVEL")]:
|
|
ReadAndPersist(var_name, def_name)
|
|
|
|
def RestoreVersionIfUnset(self, prefix=""):
|
|
for v in ["major", "minor", "build", "patch"]:
|
|
self.RestoreIfUnset("%s%s" % (prefix, v))
|
|
|
|
def WaitForLGTM(self):
|
|
print ("Please wait for an LGTM, then type \"LGTM<Return>\" to commit "
|
|
"your change. (If you need to iterate on the patch or double check "
|
|
"that it's sane, do so in another shell, but remember to not "
|
|
"change the headline of the uploaded CL.")
|
|
answer = ""
|
|
while answer != "LGTM":
|
|
print "> ",
|
|
# TODO(machenbach): Add default="LGTM" to avoid prompt when script is
|
|
# well tested and when prepare push cl has TBR flag.
|
|
answer = self.ReadLine()
|
|
if answer != "LGTM":
|
|
print "That was not 'LGTM'."
|
|
|
|
def WaitForResolvingConflicts(self, patch_file):
|
|
print("Applying the patch \"%s\" failed. Either type \"ABORT<Return>\", "
|
|
"or resolve the conflicts, stage *all* touched files with "
|
|
"'git add', and type \"RESOLVED<Return>\"")
|
|
self.DieInForcedMode()
|
|
answer = ""
|
|
while answer != "RESOLVED":
|
|
if answer == "ABORT":
|
|
self.Die("Applying the patch failed.")
|
|
if answer != "":
|
|
print "That was not 'RESOLVED' or 'ABORT'."
|
|
print "> ",
|
|
answer = self.ReadLine()
|
|
|
|
# Takes a file containing the patch to apply as first argument.
|
|
def ApplyPatch(self, patch_file, reverse_patch=""):
|
|
args = "apply --index --reject %s \"%s\"" % (reverse_patch, patch_file)
|
|
if self.Git(args) is None:
|
|
self.WaitForResolvingConflicts(patch_file)
|
|
|
|
|
|
class UploadStep(Step):
|
|
MESSAGE = "Upload for code review."
|
|
|
|
def RunStep(self):
|
|
if self._options.r:
|
|
print "Using account %s for review." % self._options.r
|
|
reviewer = self._options.r
|
|
else:
|
|
print "Please enter the email address of a V8 reviewer for your patch: ",
|
|
self.DieInForcedMode("A reviewer must be specified in forced mode.")
|
|
reviewer = self.ReadLine()
|
|
force_flag = " -f" if self._options.f else ""
|
|
args = "cl upload -r \"%s\" --send-mail%s" % (reviewer, force_flag)
|
|
# TODO(machenbach): Check output in forced mode. Verify that all required
|
|
# base files were uploaded, if not retry.
|
|
if self.Git(args, pipe=False) is None:
|
|
self.Die("'git cl upload' failed, please try again.")
|
|
|
|
|
|
def MakeStep(step_class=Step, number=0, state=None, config=None,
|
|
options=None, side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER):
|
|
# Allow to pass in empty dictionaries.
|
|
state = state if state is not None else {}
|
|
config = config if config is not None else {}
|
|
|
|
try:
|
|
message = step_class.MESSAGE
|
|
except AttributeError:
|
|
message = step_class.__name__
|
|
try:
|
|
requires = step_class.REQUIRES
|
|
except AttributeError:
|
|
requires = None
|
|
|
|
return step_class(message, requires, number=number, config=config,
|
|
state=state, options=options,
|
|
handler=side_effect_handler)
|
|
|
|
|
|
def RunScript(step_classes,
|
|
config,
|
|
options,
|
|
side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER):
|
|
state = {}
|
|
steps = []
|
|
for (number, step_class) in enumerate(step_classes):
|
|
steps.append(MakeStep(step_class, number, state, config,
|
|
options, side_effect_handler))
|
|
|
|
for step in steps[options.s:]:
|
|
step.Run()
|