AuroraMiddleware/scuffed-code
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ICU-20593 Buildtool: adding some options and improving exec mode.

Browse Source
This commit is contained in:
Shane Carr 2019-05-03 19:08:14 -07:00 committed by Shane F. Carr
parent e27c075b92
commit b603285082
4 changed files with 95 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
icu4c/source/data/buildtool/__main__.py
View File

@ -14,7 +14,7 @@ import sys
from . import *
from .comment_stripper import CommentStripper
from .request_types import CopyRequest
from .renderers import makefile, unix_exec, windows_exec
from .renderers import makefile, common_exec
from . import filtration, utils
import BUILDRULES
@ -63,7 +63,7 @@ arg_group_required = flag_parser.add_argument_group("required arguments")
arg_group_required.add_argument(
"--mode",
help = "What to do with the generated rules.",
choices = ["gnumake", "unix-exec", "windows-exec"],
choices = ["gnumake", "unix-exec", "windows-exec", "bazel-exec"],
required = True
)
@ -84,12 +84,24 @@ flag_parser.add_argument(
default = False,
action = "store_true"
)
flag_parser.add_argument(
"--ignore_xml_deprecates",
help = "Whether to ignore XML deprecates files for building res_index.",
default = False,
action = "store_true"
)
flag_parser.add_argument(
"--seqmode",
help = "Whether to optimize rules to be run sequentially (fewer threads) or in parallel (many threads). Defaults to 'sequential', which is better for unix-exec and windows-exec modes. 'parallel' is often better for massively parallel build systems.",
choices = ["sequential", "parallel"],
default = "sequential"
)
flag_parser.add_argument(
"--verbose",
help = "Print more verbose output (default false).",
default = False,
action = "store_true"
)
arg_group_exec = flag_parser.add_argument_group("arguments for unix-exec and windows-exec modes")
arg_group_exec.add_argument(
@ -123,6 +135,9 @@ class Config(object):
# Boolean: Whether to include core Unicode data files in the .dat file
self.include_uni_core_data = args.include_uni_core_data
# Boolean: Whether to ignore the XML files
self.ignore_xml_deprecates = args.ignore_xml_deprecates
# Default fields before processing filter file
self.filters_json_data = {}
@ -209,8 +224,8 @@ def add_copy_input_requests(requests, config, common_vars):
return result
def main():
args = flag_parser.parse_args()
def main(argv):
args = flag_parser.parse_args(argv)
config = Config(args)
if args.mode == "gnumake":
@ -269,19 +284,32 @@ def main():
common_vars = common
))
elif args.mode == "windows-exec":
return windows_exec.run(
return common_exec.run(
platform = "windows",
build_dirs = build_dirs,
requests = requests,
common_vars = common,
tool_dir = args.tool_dir,
tool_cfg = args.tool_cfg
tool_cfg = args.tool_cfg,
verbose = args.verbose,
)
elif args.mode == "unix-exec":
return unix_exec.run(
return common_exec.run(
platform = "unix",
build_dirs = build_dirs,
requests = requests,
common_vars = common,
tool_dir = args.tool_dir
tool_dir = args.tool_dir,
verbose = args.verbose,
)
elif args.mode == "bazel-exec":
return common_exec.run(
platform = "bazel",
build_dirs = build_dirs,
requests = requests,
common_vars = common,
tool_dir = args.tool_dir,
verbose = args.verbose,
)
else:
print("Mode not supported: %s" % args.mode)
@ -289,4 +317,4 @@ def main():
return 0
if __name__ == "__main__":
exit(main())
exit(main(sys.argv[1:]))

70
icu4c/source/data/buildtool/renderers/common_exec.py
View File

@ -9,24 +9,40 @@ from ..request_types import *
import os
import shutil
import subprocess
import sys
def run(build_dirs, requests, common_vars, **kwargs):
def run(build_dirs, requests, common_vars, verbose=True, **kwargs):
for bd in build_dirs:
os.makedirs(bd.format(**common_vars), exist_ok=True)
makedirs(bd.format(**common_vars))
for request in requests:
status = run_helper(request, common_vars, **kwargs)
status = run_helper(request, common_vars, verbose=verbose, **kwargs)
if status != 0:
print("!!! ERROR executing above command line: exit code %d" % status)
return 1
if verbose:
print("All data build commands executed")
return 0
def run_helper(request, common_vars, is_windows, tool_dir, tool_cfg=None, **kwargs):
def makedirs(dirs):
"""makedirs compatible between Python 2 and 3"""
try:
# Python 3 version
os.makedirs(dirs, exist_ok=True)
except TypeError as e:
# Python 2 version
try:
os.makedirs(dirs)
except OSError as e:
if e.errno != errno.EEXIST:
raise e
def run_helper(request, common_vars, platform, tool_dir, verbose, tool_cfg=None, **kwargs):
if isinstance(request, PrintFileRequest):
output_path = "{DIRNAME}/{FILENAME}".format(
DIRNAME = utils.dir_for(request.output_file).format(**common_vars),
FILENAME = request.output_file.filename,
)
if verbose:
print("Printing to file: %s" % output_path)
with open(output_path, "w") as f:
f.write(request.content)
@ -40,6 +56,7 @@ def run_helper(request, common_vars, is_windows, tool_dir, tool_cfg=None, **kwar
DIRNAME = utils.dir_for(request.output_file).format(**common_vars),
FILENAME = request.output_file.filename,
)
if verbose:
print("Copying file to: %s" % output_path)
shutil.copyfile(input_path, output_path)
return 0
@ -48,19 +65,27 @@ def run_helper(request, common_vars, is_windows, tool_dir, tool_cfg=None, **kwar
return 0
assert isinstance(request.tool, IcuTool)
if is_windows:
if platform == "windows":
cmd_template = "{TOOL_DIR}/{TOOL}/{TOOL_CFG}/{TOOL}.exe {{ARGS}}".format(
TOOL_DIR = tool_dir,
TOOL_CFG = tool_cfg,
TOOL = request.tool.name,
**common_vars
)
else:
elif platform == "unix":
cmd_template = "{TOOL_DIR}/{TOOL} {{ARGS}}".format(
TOOL_DIR = tool_dir,
TOOL = request.tool.name,
**common_vars
)
elif platform == "bazel":
cmd_template = "{TOOL_DIR}/{TOOL}/{TOOL} {{ARGS}}".format(
TOOL_DIR = tool_dir,
TOOL = request.tool.name,
**common_vars
)
else:
raise ValueError("Unknown platform: %s" % platform)
if isinstance(request, RepeatedExecutionRequest):
for loop_vars in utils.repeated_execution_request_looper(request):
@ -70,13 +95,12 @@ def run_helper(request, common_vars, is_windows, tool_dir, tool_cfg=None, **kwar
loop_vars,
common_vars
)
if is_windows:
if platform == "windows":
# Note: this / to \ substitution may be too aggressive?
command_line = command_line.replace("/", "\\")
print("Running: %s" % command_line)
res = subprocess.run(command_line, shell=True)
if res.returncode != 0:
return res.returncode
returncode = run_shell_command(command_line, verbose)
if returncode != 0:
return returncode
return 0
if isinstance(request, SingleExecutionRequest):
command_line = utils.format_single_request_command(
@ -84,10 +108,26 @@ def run_helper(request, common_vars, is_windows, tool_dir, tool_cfg=None, **kwar
cmd_template,
common_vars
)
if is_windows:
if platform == "windows":
# Note: this / to \ substitution may be too aggressive?
command_line = command_line.replace("/", "\\")
print("Running: %s" % command_line)
res = subprocess.run(command_line, shell=True)
return res.returncode
returncode = run_shell_command(command_line, verbose)
return returncode
assert False
def run_shell_command(command_line, verbose):
if verbose:
print("Running: %s" % command_line)
return subprocess.call(
command_line,
shell = True
)
else:
# Pipe output to /dev/null in quiet mode
with open(os.devnull, "w") as devnull:
return subprocess.call(
command_line,
shell = True,
stdout = devnull,
stderr = devnull
)

7
icu4c/source/data/buildtool/renderers/unix_exec.py
View File

@ -1,7 +0,0 @@
# Copyright (C) 2018 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
from . import common_exec
def run(**kwargs):
return common_exec.run(is_windows=False, **kwargs)

7
icu4c/source/data/buildtool/renderers/windows_exec.py
View File

@ -1,7 +0,0 @@
# Copyright (C) 2018 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
from . import common_exec
def run(**kwargs):
return common_exec.run(is_windows=True, **kwargs)