9c85078dc2
Change-Id: Ib899a5446a81e2604ba3642b9ad37aedc18ca650 Reviewed-by: Alexandru Croitor <alexandru.croitor@qt.io>
433 lines
16 KiB
Python
433 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
#############################################################################
|
|
##
|
|
## Copyright (C) 2019 The Qt Company Ltd.
|
|
## Contact: https://www.qt.io/licensing/
|
|
##
|
|
## This file is part of the plugins of the Qt Toolkit.
|
|
##
|
|
## $QT_BEGIN_LICENSE:GPL-EXCEPT$
|
|
## Commercial License Usage
|
|
## Licensees holding valid commercial Qt licenses may use this file in
|
|
## accordance with the commercial license agreement provided with the
|
|
## Software or, alternatively, in accordance with the terms contained in
|
|
## a written agreement between you and The Qt Company. For licensing terms
|
|
## and conditions see https://www.qt.io/terms-conditions. For further
|
|
## information use the contact form at https://www.qt.io/contact-us.
|
|
##
|
|
## GNU General Public License Usage
|
|
## Alternatively, this file may be used under the terms of the GNU
|
|
## General Public License version 3 as published by the Free Software
|
|
## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
|
|
## included in the packaging of this file. Please review the following
|
|
## information to ensure the GNU General Public License requirements will
|
|
## be met: https://www.gnu.org/licenses/gpl-3.0.html.
|
|
##
|
|
## $QT_END_LICENSE$
|
|
##
|
|
#############################################################################
|
|
|
|
"""
|
|
This is a helper script that takes care of reapplying special case
|
|
modifications when regenerating a CMakeLists.txt file using
|
|
pro2cmake.py or configure.cmake with configurejson2cmake.py.
|
|
|
|
It has two modes of operation:
|
|
1) Dumb "special case" block removal and re-application.
|
|
2) Smart "special case" diff application, using a previously generated
|
|
"clean" CMakeLists.txt/configure.cmake as a source. "clean" in this
|
|
case means a generated file which has no "special case" modifications.
|
|
|
|
Both modes use a temporary git repository to compute and reapply
|
|
"special case" diffs.
|
|
|
|
For the first mode to work, the developer has to mark changes
|
|
with "# special case" markers on every line they want to keep. Or
|
|
enclose blocks of code they want to keep between "# special case begin"
|
|
and "# special case end" markers.
|
|
|
|
For example:
|
|
|
|
SOURCES
|
|
foo.cpp
|
|
bar.cpp # special case
|
|
|
|
SOURCES
|
|
foo1.cpp
|
|
foo2.cpp
|
|
# special case begin
|
|
foo3.cpp
|
|
foo4.cpp
|
|
# special case end
|
|
|
|
The second mode, as mentioned, requires a previous "clean"
|
|
CMakeLists.txt/configure.cmake file.
|
|
|
|
The script can then compute the exact diff between
|
|
a "clean" and "modified" (with special cases) file, and reapply that
|
|
diff to a newly generated "CMakeLists.txt"/"configure.cmake" file.
|
|
|
|
This implies that we always have to keep a "clean" file alongside the
|
|
"modified" project file for each project (corelib, gui, etc.) So we
|
|
have to commit both files to the repository.
|
|
|
|
If there is no such "clean" file, we can use the first operation mode
|
|
to generate one. After that, we only have to use the second operation
|
|
mode for the project file in question.
|
|
|
|
When the script is used, the developer only has to take care of fixing
|
|
the newly generated "modified" file. The "clean" file is automatically
|
|
handled and git add'ed by the script, and will be committed together
|
|
with the "modified" file.
|
|
|
|
|
|
"""
|
|
|
|
import re
|
|
import os
|
|
import subprocess
|
|
import filecmp
|
|
import time
|
|
import typing
|
|
import stat
|
|
|
|
from shutil import copyfile
|
|
from shutil import rmtree
|
|
from textwrap import dedent
|
|
|
|
|
|
def remove_special_cases(original: str) -> str:
|
|
# Remove content between the following markers
|
|
# '# special case begin' and '# special case end'.
|
|
# This also remove the markers.
|
|
replaced = re.sub(
|
|
r"\n[^#\n]*?#[^\n]*?special case begin.*?#[^\n]*special case end[^\n]*?\n",
|
|
"\n",
|
|
original,
|
|
0,
|
|
re.DOTALL,
|
|
)
|
|
|
|
# Remove individual lines that have the "# special case" marker.
|
|
replaced = re.sub(r"\n.*#.*special case[^\n]*\n", "\n", replaced)
|
|
return replaced
|
|
|
|
|
|
def read_content_from_file(file_path: str) -> str:
|
|
with open(file_path, "r") as file_fd:
|
|
content = file_fd.read()
|
|
return content
|
|
|
|
|
|
def write_content_to_file(file_path: str, content: str) -> None:
|
|
with open(file_path, "w") as file_fd:
|
|
file_fd.write(content)
|
|
|
|
|
|
def resolve_simple_git_conflicts(file_path: str, debug=False) -> None:
|
|
content = read_content_from_file(file_path)
|
|
# If the conflict represents the addition of a new content hunk,
|
|
# keep the content and remove the conflict markers.
|
|
if debug:
|
|
print("Resolving simple conflicts automatically.")
|
|
replaced = re.sub(r"\n<<<<<<< HEAD\n=======(.+?)>>>>>>> master\n", r"\1", content, 0, re.DOTALL)
|
|
write_content_to_file(file_path, replaced)
|
|
|
|
|
|
def copyfile_log(src: str, dst: str, debug=False):
|
|
if debug:
|
|
print(f"Copying {src} to {dst}.")
|
|
copyfile(src, dst)
|
|
|
|
|
|
def check_if_git_in_path() -> bool:
|
|
is_win = os.name == "nt"
|
|
for path in os.environ["PATH"].split(os.pathsep):
|
|
git_path = os.path.join(path, "git")
|
|
if is_win:
|
|
git_path += ".exe"
|
|
if os.path.isfile(git_path) and os.access(git_path, os.X_OK):
|
|
return True
|
|
return False
|
|
|
|
|
|
def run_process_quiet(args_string: str, debug=False) -> bool:
|
|
if debug:
|
|
print(f'Running command: "{args_string}"')
|
|
args_list = args_string.split()
|
|
try:
|
|
subprocess.run(args_list, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
except subprocess.CalledProcessError as e:
|
|
# git merge with conflicts returns with exit code 1, but that's not
|
|
# an error for us.
|
|
if "git merge" not in args_string:
|
|
if debug:
|
|
print(
|
|
dedent(
|
|
f"""\
|
|
Error while running: "{args_string}"
|
|
{e.stdout}"""
|
|
)
|
|
)
|
|
return False
|
|
return True
|
|
|
|
|
|
def does_file_have_conflict_markers(file_path: str, debug=False) -> bool:
|
|
if debug:
|
|
print(f"Checking if {file_path} has no leftover conflict markers.")
|
|
content_actual = read_content_from_file(file_path)
|
|
if "<<<<<<< HEAD" in content_actual:
|
|
print(f"Conflict markers found in {file_path}. " "Please remove or solve them first.")
|
|
return True
|
|
return False
|
|
|
|
|
|
def create_file_with_no_special_cases(
|
|
original_file_path: str, no_special_cases_file_path: str, debug=False
|
|
):
|
|
"""
|
|
Reads content of original CMakeLists.txt/configure.cmake, removes all content
|
|
between "# special case" markers or lines, saves the result into a
|
|
new file.
|
|
"""
|
|
content_actual = read_content_from_file(original_file_path)
|
|
if debug:
|
|
print(f"Removing special case blocks from {original_file_path}.")
|
|
content_no_special_cases = remove_special_cases(content_actual)
|
|
|
|
if debug:
|
|
print(
|
|
f"Saving original contents of {original_file_path} "
|
|
f"with removed special case blocks to {no_special_cases_file_path}"
|
|
)
|
|
write_content_to_file(no_special_cases_file_path, content_no_special_cases)
|
|
|
|
|
|
def rm_tree_on_error_handler(func: typing.Callable[..., None], path: str, exception_info: tuple):
|
|
# If the path is read only, try to make it writable, and try
|
|
# to remove the path again.
|
|
if not os.access(path, os.W_OK):
|
|
os.chmod(path, stat.S_IWRITE)
|
|
func(path)
|
|
else:
|
|
print(f"Error while trying to remove path: {path}. Exception: {exception_info}")
|
|
|
|
|
|
class SpecialCaseHandler(object):
|
|
def __init__(
|
|
self,
|
|
original_file_path: str,
|
|
generated_file_path: str,
|
|
base_dir: str,
|
|
keep_temporary_files=False,
|
|
debug=False,
|
|
convertingProFiles=True,
|
|
) -> None:
|
|
self.base_dir = base_dir
|
|
self.original_file_path = original_file_path
|
|
self.generated_file_path = generated_file_path
|
|
self.keep_temporary_files = keep_temporary_files
|
|
self.use_heuristic = False
|
|
self.debug = debug
|
|
self.convertingProFiles = convertingProFiles
|
|
|
|
@property
|
|
def prev_file_path(self) -> str:
|
|
if self.convertingProFiles:
|
|
filename = ".prev_CMakeLists.txt"
|
|
else:
|
|
filename = ".prev_configure.cmake"
|
|
return os.path.join(self.base_dir, filename)
|
|
|
|
@property
|
|
def post_merge_file_path(self) -> str:
|
|
if self.convertingProFiles:
|
|
filename = "CMakeLists-post-merge.txt"
|
|
else:
|
|
filename = "configure-post-merge.cmake"
|
|
return os.path.join(self.base_dir, filename)
|
|
|
|
@property
|
|
def no_special_file_path(self) -> str:
|
|
if self.convertingProFiles:
|
|
filename = "CMakeLists.no-special.txt"
|
|
else:
|
|
filename = "configure.no-special.cmake"
|
|
return os.path.join(self.base_dir, filename)
|
|
|
|
def apply_git_merge_magic(self, no_special_cases_file_path: str) -> None:
|
|
# Create new folder for temporary repo, and ch dir into it.
|
|
repo = os.path.join(self.base_dir, "tmp_repo")
|
|
repo_absolute_path = os.path.abspath(repo)
|
|
if self.convertingProFiles:
|
|
txt = "CMakeLists.txt"
|
|
else:
|
|
txt = "configure.cmake"
|
|
|
|
try:
|
|
os.mkdir(repo)
|
|
current_dir = os.getcwd()
|
|
os.chdir(repo)
|
|
except Exception as e:
|
|
print(f"Failed to create temporary directory for temporary git repo. Exception: {e}")
|
|
raise e
|
|
|
|
generated_file_path = os.path.join("..", self.generated_file_path)
|
|
original_file_path = os.path.join("..", self.original_file_path)
|
|
no_special_cases_file_path = os.path.join("..", no_special_cases_file_path)
|
|
post_merge_file_path = os.path.join("..", self.post_merge_file_path)
|
|
|
|
try:
|
|
# Create new repo with the "clean" CMakeLists.txt/configure.cmake file.
|
|
run_process_quiet("git init .", debug=self.debug)
|
|
run_process_quiet("git config user.name fake", debug=self.debug)
|
|
run_process_quiet("git config user.email fake@fake", debug=self.debug)
|
|
copyfile_log(no_special_cases_file_path, txt, debug=self.debug)
|
|
run_process_quiet(f"git add {txt}", debug=self.debug)
|
|
run_process_quiet("git commit -m no_special", debug=self.debug)
|
|
run_process_quiet("git checkout -b no_special", debug=self.debug)
|
|
|
|
# Copy the original "modified" file (with the special cases)
|
|
# and make a new commit.
|
|
run_process_quiet("git checkout -b original", debug=self.debug)
|
|
copyfile_log(original_file_path, txt, debug=self.debug)
|
|
run_process_quiet(f"git add {txt}", debug=self.debug)
|
|
run_process_quiet("git commit -m original", debug=self.debug)
|
|
|
|
# Checkout the commit with "clean" file again, and create a
|
|
# new branch.
|
|
run_process_quiet("git checkout no_special", debug=self.debug)
|
|
run_process_quiet("git checkout -b newly_generated", debug=self.debug)
|
|
|
|
# Copy the new "modified" file and make a commit.
|
|
copyfile_log(generated_file_path, txt, debug=self.debug)
|
|
run_process_quiet(f"git add {txt}", debug=self.debug)
|
|
run_process_quiet("git commit -m newly_generated", debug=self.debug)
|
|
|
|
# Merge the "old" branch with modifications into the "new"
|
|
# branch with the newly generated file.
|
|
run_process_quiet("git merge original", debug=self.debug)
|
|
|
|
# Resolve some simple conflicts (just remove the markers)
|
|
# for cases that don't need intervention.
|
|
resolve_simple_git_conflicts(txt, debug=self.debug)
|
|
|
|
# Copy the resulting file from the merge.
|
|
copyfile_log(txt, post_merge_file_path)
|
|
except Exception as e:
|
|
print(f"Git merge conflict resolution process failed. Exception: {e}")
|
|
raise e
|
|
finally:
|
|
os.chdir(current_dir)
|
|
|
|
# Remove the temporary repo.
|
|
try:
|
|
if not self.keep_temporary_files:
|
|
rmtree(repo_absolute_path, onerror=rm_tree_on_error_handler)
|
|
except Exception as e:
|
|
print(f"Error removing temporary repo. Exception: {e}")
|
|
|
|
def save_next_clean_file(self):
|
|
files_are_equivalent = filecmp.cmp(self.generated_file_path, self.post_merge_file_path)
|
|
|
|
if not files_are_equivalent:
|
|
# Before overriding the generated file with the post
|
|
# merge result, save the new "clean" file for future
|
|
# regenerations.
|
|
copyfile_log(self.generated_file_path, self.prev_file_path, debug=self.debug)
|
|
|
|
# Attempt to git add until we succeed. It can fail when
|
|
# run_pro2cmake executes pro2cmake in multiple threads, and git
|
|
# has acquired the index lock.
|
|
success = False
|
|
failed_once = False
|
|
i = 0
|
|
while not success and i < 20:
|
|
success = run_process_quiet(f"git add {self.prev_file_path}", debug=self.debug)
|
|
if not success:
|
|
failed_once = True
|
|
i += 1
|
|
time.sleep(0.1)
|
|
|
|
if failed_once and not success:
|
|
if self.debug:
|
|
print("Retrying git add, the index.lock was probably acquired.")
|
|
if failed_once and success:
|
|
if self.debug:
|
|
print("git add succeeded.")
|
|
elif failed_once and not success:
|
|
print(f"git add failed. Make sure to git add {self.prev_file_path} yourself.")
|
|
|
|
def handle_special_cases_helper(self) -> bool:
|
|
"""
|
|
Uses git to reapply special case modifications to the "new"
|
|
generated CMakeLists.gen.txt/configure.cmake.gen file.
|
|
|
|
If use_heuristic is True, a new file is created from the
|
|
original file, with special cases removed.
|
|
|
|
If use_heuristic is False, an existing "clean" file with no
|
|
special cases is used from a previous conversion. The "clean"
|
|
file is expected to be in the same folder as the original one.
|
|
"""
|
|
try:
|
|
if does_file_have_conflict_markers(self.original_file_path):
|
|
return False
|
|
|
|
if self.use_heuristic:
|
|
create_file_with_no_special_cases(
|
|
self.original_file_path, self.no_special_file_path
|
|
)
|
|
no_special_cases_file_path = self.no_special_file_path
|
|
else:
|
|
no_special_cases_file_path = self.prev_file_path
|
|
|
|
if self.debug:
|
|
print(
|
|
f"Using git to reapply special case modifications to newly "
|
|
f"generated {self.generated_file_path} file"
|
|
)
|
|
|
|
self.apply_git_merge_magic(no_special_cases_file_path)
|
|
self.save_next_clean_file()
|
|
|
|
copyfile_log(self.post_merge_file_path, self.generated_file_path)
|
|
if not self.keep_temporary_files:
|
|
os.remove(self.post_merge_file_path)
|
|
if self.debug:
|
|
print(
|
|
"Special case reapplication using git is complete. "
|
|
"Make sure to fix remaining conflict markers."
|
|
)
|
|
|
|
except Exception as e:
|
|
print(f"Error occurred while trying to reapply special case modifications: {e}")
|
|
return False
|
|
finally:
|
|
if not self.keep_temporary_files and self.use_heuristic:
|
|
os.remove(self.no_special_file_path)
|
|
|
|
return True
|
|
|
|
def handle_special_cases(self) -> bool:
|
|
original_file_exists = os.path.isfile(self.original_file_path)
|
|
prev_file_exists = os.path.isfile(self.prev_file_path)
|
|
self.use_heuristic = not prev_file_exists
|
|
|
|
git_available = check_if_git_in_path()
|
|
keep_special_cases = original_file_exists and git_available
|
|
|
|
if not git_available:
|
|
print(
|
|
"You need to have git in PATH in order to reapply the special "
|
|
"case modifications."
|
|
)
|
|
|
|
copy_generated_file = True
|
|
|
|
if keep_special_cases:
|
|
copy_generated_file = self.handle_special_cases_helper()
|
|
|
|
return copy_generated_file
|