Improve run_pro2cmake.py

Add two new options:

If you pass --only-existing, the pro2cmake script will only be
executed on .pro files that already have a CMakeLists.txt next
to them. This is useful if you modify pro2cmake, and only want
to regenerate existing files.

If you pass --only-qtbase-main-modules, the script will be
executed on the main modules in qtbase/src. This is useful
if you want to check if your pro2cmake modification
works correctly on the more complicated projects.

Change-Id: I5228411a252dbef6d77f01ca742a7b98583c5a75
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>
Reviewed-by: Qt CMake Build Bot

util/cmake/run_pro2cmake.py

@@ -31,18 +31,24 @@ import glob
 import os
 import subprocess
 import concurrent.futures
-import sys
 import typing
-
+import argparse
-script_path = os.path.dirname(os.path.abspath(__file__))
+from argparse import ArgumentParser
-base_path = os.path.dirname(script_path)
-pro2cmake = os.path.join(script_path, 'pro2cmake.py')
-
-if len(sys.argv) > 1:
-    base_path = os.path.abspath(sys.argv[1])
 
 
-def find_all_pro_files():
+def parse_command_line():
+    parser = ArgumentParser(description='Run pro2cmake on all .pro files recursively in given path.')
+    parser.add_argument('--only-existing', dest='only_existing', action='store_true',
+                        help='Run pro2cmake only on .pro files that already have a CMakeLists.txt.')
+    parser.add_argument('--only-qtbase-main-modules', dest='only_qtbase_main_modules', action='store_true',
+                        help='Run pro2cmake only on the main modules in qtbase.')
+    parser.add_argument('path', metavar='<path>', type=str,
+                        help='The path where to look for .pro files.')
+
+    return parser.parse_args()
+
+
+def find_all_pro_files(base_path: str, args: argparse.Namespace):
 
     def sorter(pro_file: str) -> str:
         """ Sorter that tries to prioritize main pro files in a directory. """
@@ -54,9 +60,49 @@ def find_all_pro_files():
 
     all_files = []
     previous_dir_name: str = None
-    for pro_file in sorted(glob.glob(os.path.join(base_path, '**/*.pro'),
+
-                                     recursive=True),
+    print('Finding .pro files.')
-                           key=sorter):
+    glob_result = glob.glob(os.path.join(base_path, '**/*.pro'), recursive=True)
+
+    def cmake_lists_exists_filter(path):
+        path_dir_name = os.path.dirname(path)
+        if os.path.exists(os.path.join(path_dir_name, 'CMakeLists.txt')):
+            return True
+        return False
+
+    def qtbase_main_modules_filter(path):
+        main_modules = [
+            'corelib',
+            'network',
+            'gui',
+            'widgets',
+            'testlib',
+            'printsupport',
+            'opengl',
+            'sql',
+            'dbus',
+            'concurrent',
+            'xml',
+        ]
+        path_suffixes = ['src/{}/{}.pro'.format(m, m, '.pro') for m in main_modules]
+
+        for path_suffix in path_suffixes:
+            if path.endswith(path_suffix):
+                return True
+        return False
+
+    filter_result = glob_result
+    filter_func = None
+    if args.only_existing:
+        filter_func = cmake_lists_exists_filter
+    elif args.only_qtbase_main_modules:
+        filter_func = qtbase_main_modules_filter
+
+    if filter_func:
+        print('Filtering.')
+        filter_result = [p for p in filter_result if filter_func(p)]
+
+    for pro_file in sorted(filter_result, key=sorter):
         dir_name = os.path.dirname(pro_file)
         if dir_name == previous_dir_name:
             print("Skipping:", pro_file)
@@ -66,31 +112,56 @@ def find_all_pro_files():
     return all_files
 
 
-failed_files = []
+def run(all_files: typing.List[str], pro2cmake: str, args: argparse.Namespace) -> typing.List[str]:
-all_files = find_all_pro_files()
+    failed_files = []
-files_count = len(all_files)
+    files_count = len(all_files)
+    workers = (os.cpu_count() or 1)
 
-with concurrent.futures.ThreadPoolExecutor(initializer=os.nice, initargs=(10,)) as pool:
+    if args.only_qtbase_main_modules:
+        # qtbase main modules take longer than usual to process.
+        workers = 2
 
-    def _process_a_file(data: typing.Tuple[str, int, int]) -> typing.Tuple[int, str, str]:
+    with concurrent.futures.ThreadPoolExecutor(max_workers=workers, initializer=os.nice, initargs=(10,)) as pool:
-        filename, index, total = data
+        print('Firing up thread pool executor.')
-        result = subprocess.run((pro2cmake, os.path.basename(filename)),
-                                cwd=os.path.dirname(filename),
-                                stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        stdout = 'Converted[{}/{}]: {}\n'.format(index, total, filename)
-        return result.returncode, filename, stdout + result.stdout.decode()
 
-    for return_code, filename, stdout in pool.map(_process_a_file,
+        def _process_a_file(data: typing.Tuple[str, int, int]) -> typing.Tuple[int, str, str]:
-                                                  zip(all_files,
+            filename, index, total = data
-                                                      range(1, files_count + 1),
+            result = subprocess.run((pro2cmake, os.path.basename(filename)),
-                                                      (files_count for _ in all_files))):
+                                    cwd=os.path.dirname(filename),
-        if return_code:
+                                    stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-            failed_files.append(filename)
+            stdout = 'Converted[{}/{}]: {}\n'.format(index, total, filename)
-        print(stdout)
+            return result.returncode, filename, stdout + result.stdout.decode()
 
-if failed_files:
+        for return_code, filename, stdout in pool.map(_process_a_file,
-    print('The following files were not successfully '
+                                                      zip(all_files,
-          'converted ({} of {}):'.format(len(failed_files), files_count))
+                                                          range(1, files_count + 1),
-    for f in failed_files:
+                                                          (files_count for _ in all_files))):
-        print('    "{}"'.format(f))
+            if return_code:
+                failed_files.append(filename)
+            print(stdout)
 
+    return failed_files
+
+
+def main() -> None:
+    args = parse_command_line()
+
+    script_path = os.path.dirname(os.path.abspath(__file__))
+    pro2cmake = os.path.join(script_path, 'pro2cmake.py')
+    base_path = args.path
+
+    all_files = find_all_pro_files(base_path, args)
+    files_count = len(all_files)
+    failed_files = run(all_files, pro2cmake, args)
+    if len(all_files) == 0:
+        print('No files found.')
+
+    if failed_files:
+        print('The following files were not successfully '
+              'converted ({} of {}):'.format(len(failed_files), files_count))
+        for f in failed_files:
+            print('    "{}"'.format(f))
+
+
+if __name__ == '__main__':
+    main()