diff --git a/tools/gcmole/gcmole.lua b/tools/gcmole/gcmole.lua index 68dc4919ee..9739684629 100644 --- a/tools/gcmole/gcmole.lua +++ b/tools/gcmole/gcmole.lua @@ -34,6 +34,9 @@ local FLAGS = { -- Do not build gcsuspects file and reuse previously generated one. reuse_gcsuspects = false; + -- Don't use parallel python runner. + sequential = false; + -- Print commands to console before executing them. verbose = false; @@ -113,20 +116,66 @@ local function MakeClangCommandLine( .. " " .. arch_options end +local function IterTable(t) + return coroutine.wrap(function () + for i, v in ipairs(t) do + coroutine.yield(v) + end + end) +end + +local function SplitResults(lines, func) + -- Splits the output of parallel.py and calls func on each result. + -- Bails out in case of an error in one of the executions. + local current = {} + local filename = "" + for line in lines do + local new_file = line:match "^______________ (.*)$" + local code = line:match "^______________ finish (%d+) ______________$" + if code then + if tonumber(code) > 0 then + log(table.concat(current, "\n")) + log("Failed to examine " .. filename) + return false + end + log("-- %s", filename) + func(filename, IterTable(current)) + elseif new_file then + filename = new_file + current = {} + else + table.insert(current, line) + end + end + return true +end + function InvokeClangPluginForEachFile(filenames, cfg, func) local cmd_line = MakeClangCommandLine(cfg.plugin, cfg.plugin_args, cfg.triple, cfg.arch_define, cfg.arch_options) - for _, filename in ipairs(filenames) do - log("-- %s", filename) - local action = cmd_line .. " " .. filename .. " 2>&1" + if FLAGS.sequential then + log("** Sequential execution.") + for _, filename in ipairs(filenames) do + log("-- %s", filename) + local action = cmd_line .. " " .. filename .. " 2>&1" + if FLAGS.verbose then print('popen ', action) end + local pipe = io.popen(action) + func(filename, pipe:lines()) + local success = pipe:close() + if not success then error("Failed to run: " .. action) end + end + else + log("** Parallel execution.") + local action = "python tools/gcmole/parallel.py \"" + .. cmd_line .. "\" " .. table.concat(filenames, " ") if FLAGS.verbose then print('popen ', action) end local pipe = io.popen(action) - func(filename, pipe:lines()) - local success = pipe:close() - if not success then error("Failed to run: " .. action) end + local success = SplitResults(pipe:lines(), func) + local closed = pipe:close() + if not (success and closed) then error("Failed to run: " .. action) end end end diff --git a/tools/gcmole/parallel.py b/tools/gcmole/parallel.py new file mode 100755 index 0000000000..0c045f423c --- /dev/null +++ b/tools/gcmole/parallel.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# Copyright 2015 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +This script calls the first argument for each of the following arguments in +parallel. E.g. +parallel.py "clang --opt" file1 file2 +calls +clang --opt file1 +clang --opt file2 + +The output (stdout and stderr) is concatenated sequentially in the form: +______________ file1 + +______________ finish ______________ +______________ file2 + +______________ finish ______________ +""" + +import itertools +import multiprocessing +import subprocess +import sys + +def invoke(cmdline): + try: + return (subprocess.check_output( + cmdline, shell=True, stderr=subprocess.STDOUT), 0) + except subprocess.CalledProcessError as e: + return (e.output, e.returncode) + +if __name__ == '__main__': + assert len(sys.argv) > 2 + processes = multiprocessing.cpu_count() + pool = multiprocessing.Pool(processes=processes) + cmdlines = ["%s %s" % (sys.argv[1], filename) for filename in sys.argv[2:]] + for filename, result in itertools.izip( + sys.argv[2:], pool.imap(invoke, cmdlines)): + print "______________ %s" % filename + print result[0] + print "______________ finish %d ______________" % result[1]