Improve run_pro2cmake.py
Add two new options: If you pass --only-existing, the pro2cmake script will only be executed on .pro files that already have a CMakeLists.txt next to them. This is useful if you modify pro2cmake, and only want to regenerate existing files. If you pass --only-qtbase-main-modules, the script will be executed on the main modules in qtbase/src. This is useful if you want to check if your pro2cmake modification works correctly on the more complicated projects. Change-Id: I5228411a252dbef6d77f01ca742a7b98583c5a75 Reviewed-by: Simon Hausmann <simon.hausmann@qt.io> Reviewed-by: Qt CMake Build Bot
This commit is contained in:
Alexandru Croitor
parent
891c897b80
commit
d0eb985f74
@ -31,18 +31,24 @@ import glob
|
||||
import os
|
||||
import subprocess
|
||||
import concurrent.futures
|
||||
import sys
|
||||
import typing
|
||||
|
||||
script_path = os.path.dirname(os.path.abspath(__file__))
|
||||
base_path = os.path.dirname(script_path)
|
||||
pro2cmake = os.path.join(script_path, 'pro2cmake.py')
|
||||
|
||||
if len(sys.argv) > 1:
|
||||
base_path = os.path.abspath(sys.argv[1])
|
||||
import argparse
|
||||
from argparse import ArgumentParser
|
||||
|
||||
|
||||
def find_all_pro_files():
|
||||
def parse_command_line():
|
||||
parser = ArgumentParser(description='Run pro2cmake on all .pro files recursively in given path.')
|
||||
parser.add_argument('--only-existing', dest='only_existing', action='store_true',
|
||||
help='Run pro2cmake only on .pro files that already have a CMakeLists.txt.')
|
||||
parser.add_argument('--only-qtbase-main-modules', dest='only_qtbase_main_modules', action='store_true',
|
||||
help='Run pro2cmake only on the main modules in qtbase.')
|
||||
parser.add_argument('path', metavar='<path>', type=str,
|
||||
help='The path where to look for .pro files.')
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def find_all_pro_files(base_path: str, args: argparse.Namespace):
|
||||
|
||||
def sorter(pro_file: str) -> str:
|
||||
""" Sorter that tries to prioritize main pro files in a directory. """
|
||||
@ -54,9 +60,49 @@ def find_all_pro_files():
|
||||
|
||||
all_files = []
|
||||
previous_dir_name: str = None
|
||||
for pro_file in sorted(glob.glob(os.path.join(base_path, '**/*.pro'),
|
||||
recursive=True),
|
||||
key=sorter):
|
||||
|
||||
print('Finding .pro files.')
|
||||
glob_result = glob.glob(os.path.join(base_path, '**/*.pro'), recursive=True)
|
||||
|
||||
def cmake_lists_exists_filter(path):
|
||||
path_dir_name = os.path.dirname(path)
|
||||
if os.path.exists(os.path.join(path_dir_name, 'CMakeLists.txt')):
|
||||
return True
|
||||
return False
|
||||
|
||||
def qtbase_main_modules_filter(path):
|
||||
main_modules = [
|
||||
'corelib',
|
||||
'network',
|
||||
'gui',
|
||||
'widgets',
|
||||
'testlib',
|
||||
'printsupport',
|
||||
'opengl',
|
||||
'sql',
|
||||
'dbus',
|
||||
'concurrent',
|
||||
'xml',
|
||||
]
|
||||
path_suffixes = ['src/{}/{}.pro'.format(m, m, '.pro') for m in main_modules]
|
||||
|
||||
for path_suffix in path_suffixes:
|
||||
if path.endswith(path_suffix):
|
||||
return True
|
||||
return False
|
||||
|
||||
filter_result = glob_result
|
||||
filter_func = None
|
||||
if args.only_existing:
|
||||
filter_func = cmake_lists_exists_filter
|
||||
elif args.only_qtbase_main_modules:
|
||||
filter_func = qtbase_main_modules_filter
|
||||
|
||||
if filter_func:
|
||||
print('Filtering.')
|
||||
filter_result = [p for p in filter_result if filter_func(p)]
|
||||
|
||||
for pro_file in sorted(filter_result, key=sorter):
|
||||
dir_name = os.path.dirname(pro_file)
|
||||
if dir_name == previous_dir_name:
|
||||
print("Skipping:", pro_file)
|
||||
@ -66,31 +112,56 @@ def find_all_pro_files():
|
||||
return all_files
|
||||
|
||||
|
||||
failed_files = []
|
||||
all_files = find_all_pro_files()
|
||||
files_count = len(all_files)
|
||||
def run(all_files: typing.List[str], pro2cmake: str, args: argparse.Namespace) -> typing.List[str]:
|
||||
failed_files = []
|
||||
files_count = len(all_files)
|
||||
workers = (os.cpu_count() or 1)
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(initializer=os.nice, initargs=(10,)) as pool:
|
||||
if args.only_qtbase_main_modules:
|
||||
# qtbase main modules take longer than usual to process.
|
||||
workers = 2
|
||||
|
||||
def _process_a_file(data: typing.Tuple[str, int, int]) -> typing.Tuple[int, str, str]:
|
||||
filename, index, total = data
|
||||
result = subprocess.run((pro2cmake, os.path.basename(filename)),
|
||||
cwd=os.path.dirname(filename),
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
stdout = 'Converted[{}/{}]: {}\n'.format(index, total, filename)
|
||||
return result.returncode, filename, stdout + result.stdout.decode()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=workers, initializer=os.nice, initargs=(10,)) as pool:
|
||||
print('Firing up thread pool executor.')
|
||||
|
||||
for return_code, filename, stdout in pool.map(_process_a_file,
|
||||
zip(all_files,
|
||||
range(1, files_count + 1),
|
||||
(files_count for _ in all_files))):
|
||||
if return_code:
|
||||
failed_files.append(filename)
|
||||
print(stdout)
|
||||
def _process_a_file(data: typing.Tuple[str, int, int]) -> typing.Tuple[int, str, str]:
|
||||
filename, index, total = data
|
||||
result = subprocess.run((pro2cmake, os.path.basename(filename)),
|
||||
cwd=os.path.dirname(filename),
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
stdout = 'Converted[{}/{}]: {}\n'.format(index, total, filename)
|
||||
return result.returncode, filename, stdout + result.stdout.decode()
|
||||
|
||||
if failed_files:
|
||||
print('The following files were not successfully '
|
||||
'converted ({} of {}):'.format(len(failed_files), files_count))
|
||||
for f in failed_files:
|
||||
print(' "{}"'.format(f))
|
||||
for return_code, filename, stdout in pool.map(_process_a_file,
|
||||
zip(all_files,
|
||||
range(1, files_count + 1),
|
||||
(files_count for _ in all_files))):
|
||||
if return_code:
|
||||
failed_files.append(filename)
|
||||
print(stdout)
|
||||
|
||||
return failed_files
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_command_line()
|
||||
|
||||
script_path = os.path.dirname(os.path.abspath(__file__))
|
||||
pro2cmake = os.path.join(script_path, 'pro2cmake.py')
|
||||
base_path = args.path
|
||||
|
||||
all_files = find_all_pro_files(base_path, args)
|
||||
files_count = len(all_files)
|
||||
failed_files = run(all_files, pro2cmake, args)
|
||||
if len(all_files) == 0:
|
||||
print('No files found.')
|
||||
|
||||
if failed_files:
|
||||
print('The following files were not successfully '
|
||||
'converted ({} of {}):'.format(len(failed_files), files_count))
|
||||
for f in failed_files:
|
||||
print(' "{}"'.format(f))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user