2018-01-05 13:34:17 +00:00
|
|
|
# Copyright 2018 the V8 project authors. All rights reserved.
|
|
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
|
|
# found in the LICENSE file.
|
|
|
|
|
|
|
|
import collections
|
|
|
|
import traceback
|
|
|
|
|
|
|
|
from . import base
|
|
|
|
from ..local import pool
|
|
|
|
|
|
|
|
|
|
|
|
# Global function for multiprocessing, because pickling a static method doesn't
|
|
|
|
# work on Windows.
|
2018-01-17 10:59:24 +00:00
|
|
|
def run_job(job, process_context):
|
|
|
|
return job.run(process_context)
|
|
|
|
|
|
|
|
|
2018-01-23 08:20:56 +00:00
|
|
|
def create_process_context(result_reduction):
|
|
|
|
return ProcessContext(result_reduction)
|
2018-01-05 13:34:17 +00:00
|
|
|
|
|
|
|
|
|
|
|
JobResult = collections.namedtuple('JobResult', ['id', 'result'])
|
2018-01-23 08:20:56 +00:00
|
|
|
ProcessContext = collections.namedtuple('ProcessContext', ['result_reduction'])
|
2018-01-05 13:34:17 +00:00
|
|
|
|
|
|
|
|
|
|
|
class Job(object):
|
2018-01-17 10:59:24 +00:00
|
|
|
def __init__(self, test_id, cmd, outproc, keep_output):
|
2018-01-05 13:34:17 +00:00
|
|
|
self.test_id = test_id
|
|
|
|
self.cmd = cmd
|
|
|
|
self.outproc = outproc
|
2018-01-17 10:59:24 +00:00
|
|
|
self.keep_output = keep_output
|
2018-01-05 13:34:17 +00:00
|
|
|
|
2018-01-17 10:59:24 +00:00
|
|
|
def run(self, process_ctx):
|
2018-01-05 13:34:17 +00:00
|
|
|
output = self.cmd.execute()
|
2018-01-23 08:20:56 +00:00
|
|
|
reduction = process_ctx.result_reduction if not self.keep_output else None
|
|
|
|
result = self.outproc.process(output, reduction)
|
2018-01-17 10:59:24 +00:00
|
|
|
return JobResult(self.test_id, result)
|
2018-01-05 13:34:17 +00:00
|
|
|
|
|
|
|
|
|
|
|
class ExecutionProc(base.TestProc):
|
2018-01-09 13:56:18 +00:00
|
|
|
"""Last processor in the chain. Instead of passing tests further it creates
|
|
|
|
commands and output processors, executes them in multiple worker processes and
|
|
|
|
sends results to the previous processor.
|
|
|
|
"""
|
|
|
|
|
2018-02-01 10:01:21 +00:00
|
|
|
def __init__(self, jobs, outproc_factory=None):
|
2018-01-05 13:34:17 +00:00
|
|
|
super(ExecutionProc, self).__init__()
|
2019-08-05 10:52:38 +00:00
|
|
|
self._pool = pool.Pool(jobs)
|
2018-01-25 15:38:11 +00:00
|
|
|
self._outproc_factory = outproc_factory or (lambda t: t.output_proc)
|
2018-01-05 13:34:17 +00:00
|
|
|
self._tests = {}
|
|
|
|
|
|
|
|
def connect_to(self, next_proc):
|
|
|
|
assert False, 'ExecutionProc cannot be connected to anything'
|
|
|
|
|
2018-02-20 17:23:37 +00:00
|
|
|
def run(self):
|
|
|
|
it = self._pool.imap_unordered(
|
2018-01-05 13:34:17 +00:00
|
|
|
fn=run_job,
|
|
|
|
gen=[],
|
2018-01-17 10:59:24 +00:00
|
|
|
process_context_fn=create_process_context,
|
|
|
|
process_context_args=[self._prev_requirement],
|
2018-02-20 17:23:37 +00:00
|
|
|
)
|
|
|
|
for pool_result in it:
|
|
|
|
self._unpack_result(pool_result)
|
2018-01-05 13:34:17 +00:00
|
|
|
|
|
|
|
def next_test(self, test):
|
2018-01-31 11:04:33 +00:00
|
|
|
if self.is_stopped:
|
2019-01-30 12:20:17 +00:00
|
|
|
return False
|
2018-01-31 11:04:33 +00:00
|
|
|
|
2018-01-05 13:34:17 +00:00
|
|
|
test_id = test.procid
|
2018-01-31 12:01:49 +00:00
|
|
|
cmd = test.get_command()
|
2018-01-17 12:29:53 +00:00
|
|
|
self._tests[test_id] = test, cmd
|
2018-01-05 13:34:17 +00:00
|
|
|
|
2018-01-25 15:38:11 +00:00
|
|
|
outproc = self._outproc_factory(test)
|
2018-01-17 12:29:53 +00:00
|
|
|
self._pool.add([Job(test_id, cmd, outproc, test.keep_output)])
|
2018-01-05 13:34:17 +00:00
|
|
|
|
2019-01-30 12:20:17 +00:00
|
|
|
return True
|
|
|
|
|
2018-01-15 08:03:48 +00:00
|
|
|
def result_for(self, test, result):
|
2018-01-05 13:34:17 +00:00
|
|
|
assert False, 'ExecutionProc cannot receive results'
|
2018-01-31 10:21:44 +00:00
|
|
|
|
|
|
|
def stop(self):
|
2018-01-31 11:04:33 +00:00
|
|
|
super(ExecutionProc, self).stop()
|
2018-02-20 17:23:37 +00:00
|
|
|
self._pool.abort()
|
2018-01-31 10:21:44 +00:00
|
|
|
|
|
|
|
def _unpack_result(self, pool_result):
|
|
|
|
if pool_result.heartbeat:
|
|
|
|
self.heartbeat()
|
|
|
|
return
|
|
|
|
|
|
|
|
job_result = pool_result.value
|
|
|
|
test_id, result = job_result
|
|
|
|
|
|
|
|
test, result.cmd = self._tests[test_id]
|
|
|
|
del self._tests[test_id]
|
|
|
|
self._send_result(test, result)
|