From dcb2df4a4e3e1cc79265bf3d85a19d125eac6fb4 Mon Sep 17 00:00:00 2001 From: Ryan Prichard Date: Sun, 14 Oct 2018 20:17:52 -0700 Subject: [PATCH] ship.py: build packages using cygwin/msys2 prebuilts --- .gitignore | 1 + ship/common_ship.py | 33 ++++++++-- ship/make_msvc_package.py | 8 +-- ship/ship.py | 133 +++++++++++++++++++++++++++----------- 4 files changed, 127 insertions(+), 48 deletions(-) diff --git a/.gitignore b/.gitignore index 89e60bd..68c6b47 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ winpty.opensdf /build-gyp /build-libpty /ship/packages +/ship/tmp /src/Default /src/Release /src/gen diff --git a/ship/common_ship.py b/ship/common_ship.py index b46cd5b..181a064 100644 --- a/ship/common_ship.py +++ b/ship/common_ship.py @@ -1,6 +1,10 @@ import os import sys +# These scripts need to continue using Python 2 rather than 3, because +# make_msvc_package.py puts the current Python interpreter on the PATH for the +# sake of gyp, and gyp doesn't work with Python 3 yet. +# https://bugs.chromium.org/p/gyp/issues/detail?id=36 if os.name != "nt": sys.exit("Error: ship scripts require native Python 2.7. (wrong os.name)") if sys.version_info[0:2] != (2,7): @@ -20,11 +24,11 @@ def rmrf(patterns): for pattern in patterns: for path in glob.glob(pattern): if os.path.isdir(path) and not os.path.islink(path): - print "+ rm -r " + path + print("+ rm -r " + path) sys.stdout.flush() shutil.rmtree(path) elif os.path.isfile(path): - print "+ rm " + path + print("+ rm " + path) sys.stdout.flush() os.remove(path) @@ -37,17 +41,38 @@ def requireExe(name, guesses): for guess in guesses: if os.path.exists(guess): newDir = os.path.dirname(guess) - print "Adding " + newDir + " to Path to provide " + name + print("Adding " + newDir + " to Path to provide " + name) os.environ["Path"] = newDir + ";" + os.environ["Path"] ret = find_executable(name) if ret is None: sys.exit("Error: required EXE is missing from Path: " + name) return ret +class ModifyEnv: + def __init__(self, **kwargs): + self._changes = dict(kwargs) + self._original = dict() + + def __enter__(self): + for var, val in self._changes.items(): + self._original[var] = os.environ[var] + os.environ[var] = val + + def __exit__(self, type, value, traceback): + for var, val in self._original.items(): + os.environ[var] = val + requireExe("git.exe", [ "C:\\Program Files\\Git\\cmd\\git.exe", "C:\\Program Files (x86)\\Git\\cmd\\git.exe" ]) -commitHash = subprocess.check_output(["git.exe", "rev-parse", "HEAD"]).decode().strip() +commitHash = subprocess.check_output(["git.exe", "rev-parse", "HEAD"]).strip() defaultPathEnviron = "C:\\Windows\\System32;C:\\Windows" + +ZIP_TOOL = requireExe("7z.exe", [ + "C:\\Program Files\\7-Zip\\7z.exe", + "C:\\Program Files (x86)\\7-Zip\\7z.exe", +]) + +requireExe("curl.exe", []) diff --git a/ship/make_msvc_package.py b/ship/make_msvc_package.py index 220f02b..fc49c15 100755 --- a/ship/make_msvc_package.py +++ b/ship/make_msvc_package.py @@ -36,10 +36,6 @@ import subprocess import sys os.chdir(common_ship.topDir) -ZIP_TOOL = common_ship.requireExe("7z.exe", [ - "C:\\Program Files\\7-Zip\\7z.exe", - "C:\\Program Files (x86)\\7-Zip\\7z.exe", -]) MSVC_VERSION_TABLE = { "2015" : { @@ -103,7 +99,7 @@ def build(arch, packageDir, xp=False): newEnv = os.environ.copy() newEnv["PATH"] = os.path.dirname(sys.executable) + ";" + common_ship.defaultPathEnviron commandLine = ( - '"' + devCmdPath + '" && ' + '"' + devCmdPath + '" && ' + " vcbuild.bat" + " --gyp-msvs-version " + versionInfo["gyp_version"] + " --msvc-platform " + archInfo["msvc_platform"] + @@ -159,7 +155,7 @@ def buildPackage(): shutil.copy(topDir + "/README.md", packageDir) shutil.copy(topDir + "/RELEASES.md", packageDir) - subprocess.check_call([ZIP_TOOL, "a", packageFile, "."], cwd=packageDir) + subprocess.check_call([common_ship.ZIP_TOOL, "a", packageFile, "."], cwd=packageDir) if __name__ == "__main__": buildPackage() diff --git a/ship/ship.py b/ship/ship.py index de63d0a..e6378fc 100755 --- a/ship/ship.py +++ b/ship/ship.py @@ -23,10 +23,6 @@ # # Run with native CPython 2.7 on a 64-bit computer. # -# Each of the targets in BUILD_TARGETS must be installed to the default -# location. Each target must have the appropriate MinGW and non-MinGW -# compilers installed, as well as make and tar. -# import common_ship @@ -36,7 +32,6 @@ import shutil import subprocess import sys -os.chdir(common_ship.topDir) def dllVersion(path): version = subprocess.check_output( @@ -44,58 +39,120 @@ def dllVersion(path): "[System.Diagnostics.FileVersionInfo]::GetVersionInfo(\"" + path + "\").FileVersion"]) return version.strip() + +os.chdir(common_ship.topDir) + +URL_BASE = "file://c:/rprichard/proj/winpty-cygwin-prebuilts/out/artifact/" + # Determine other build parameters. -print "Determining Cygwin/MSYS2 DLL versions..." +print("Determining Cygwin/MSYS2 DLL versions...") sys.stdout.flush() BUILD_TARGETS = [ { - "name": "msys2-" + dllVersion("C:\\msys32\\usr\\bin\\msys-2.0.dll") + "-ia32", - "path": "C:\\msys32\\mingw32\\bin;C:\\msys32\\usr\\bin", + "systemName": "msys32", + "systemArchive": "msys32-20181014-dll2.11.1-msysgcc7.3.0-wingcc7.3.0.7z", + "systemArchiveUrl": URL_BASE + "msys32-20181014-dll2.11.1-msysgcc7.3.0-wingcc7.3.0.7z", + "packageName": "msys2-{dllver}-ia32", + "rebase": ["usr\\bin\\ash.exe", "/usr/bin/rebaseall", "-v"], + "dll": "usr\\bin\\msys-2.0.dll", + "path": ["opt\\bin", "usr\\bin"], }, { - "name": "msys2-" + dllVersion("C:\\msys64\\usr\\bin\\msys-2.0.dll") + "-x64", - "path": "C:\\msys64\\mingw64\\bin;C:\\msys64\\usr\\bin", + "systemName": "msys64", + "systemArchive": "msys64-20181014-dll2.11.1-msysgcc7.3.0-wingcc7.3.0.7z", + "systemArchiveUrl": URL_BASE + "msys64-20181014-dll2.11.1-msysgcc7.3.0-wingcc7.3.0.7z", + "packageName": "msys2-{dllver}-x64", + "rebase": ["usr\\bin\\ash.exe", "/usr/bin/rebaseall", "-v"], + "dll": "usr\\bin\\msys-2.0.dll", + "path": ["opt\\bin", "usr\\bin"], }, { - "name": "cygwin-" + dllVersion("C:\\cygwin\\bin\\cygwin1.dll") + "-ia32", - "path": "C:\\cygwin\\bin", + "systemName": "cygwin32", + "systemArchive": "cygwin32-20181014-dll2.11.1-cyggcc7.3.0-wingcc6.4.0.7z", + "systemArchiveUrl": URL_BASE + "cygwin32-20181014-dll2.11.1-cyggcc7.3.0-wingcc6.4.0.7z", + "packageName": "cygwin-{dllver}-ia32", + "rebase": ["bin\\ash.exe", "/bin/rebaseall", "-v"], + "dll": "bin\\cygwin1.dll", + "path": ["bin"], }, { - "name": "cygwin-" + dllVersion("C:\\cygwin64\\bin\\cygwin1.dll") + "-x64", - "path": "C:\\cygwin64\\bin", + "systemName": "cygwin64", + "systemArchive": "cygwin64-20181014-dll2.11.1-cyggcc7.3.0-wingcc6.4.0.7z", + "systemArchiveUrl": URL_BASE + "cygwin64-20181014-dll2.11.1-cyggcc7.3.0-wingcc6.4.0.7z", + "packageName": "cygwin-{dllver}-x64", + "rebase": ["bin\\ash.exe", "/bin/rebaseall", "-v"], + "dll": "bin\\cygwin1.dll", + "path": ["bin"], }, ] + +def targetSystemDir(target): + return os.path.abspath(os.path.join("ship", "tmp", target["systemName"])) + + +def setup_cyg_system(target): + common_ship.mkdir("ship/tmp") + + systemDir = targetSystemDir(target) + systemArchivePath = os.path.abspath(os.path.join("ship", "tmp", target["systemArchive"])) + binPaths = [os.path.join(systemDir, p) for p in target["path"]] + binPaths += common_ship.defaultPathEnviron.split(";") + newPath = ";".join(binPaths) + + if not os.path.exists(systemDir): + if not os.path.exists(systemArchivePath): + subprocess.check_call(["curl.exe", "-fL", "-o", systemArchivePath, target["systemArchiveUrl"]]) + assert os.path.exists(systemArchivePath) + subprocess.check_call(["7z.exe", "x", systemArchivePath], cwd=os.path.dirname(systemDir)) + with common_ship.ModifyEnv(PATH=newPath): + rebaseCmd = target["rebase"] + rebaseCmd[0] = os.path.join(systemDir, rebaseCmd[0]) + subprocess.check_call(rebaseCmd) + assert os.path.exists(systemDir) + + return newPath + + def buildTarget(target): - packageName = "winpty-" + common_ship.winptyVersion + "-" + target["name"] + newPath = setup_cyg_system(target) + + dllver = dllVersion(os.path.join(targetSystemDir(target), target["dll"])) + packageName = "winpty-" + common_ship.winptyVersion + "-" + target["packageName"].format(dllver=dllver) if os.path.exists("ship\\packages\\" + packageName): shutil.rmtree("ship\\packages\\" + packageName) - oldPath = os.environ["PATH"] - os.environ["PATH"] = target["path"] + ";" + common_ship.defaultPathEnviron - subprocess.check_call(["sh.exe", "configure"]) - subprocess.check_call(["make.exe", "clean"]) - makeBaseCmd = [ - "make.exe", - "USE_PCH=0", - "COMMIT_HASH=" + common_ship.commitHash, - "PREFIX=ship/packages/" + packageName - ] - subprocess.check_call(makeBaseCmd + ["all", "tests", "-j%d" % multiprocessing.cpu_count()]) - subprocess.check_call(["build\\trivial_test.exe"]) - subprocess.check_call(makeBaseCmd + ["install"]) - subprocess.check_call(["tar.exe", "cvfz", - packageName + ".tar.gz", - packageName], cwd=os.path.join(os.getcwd(), "ship", "packages")) - os.environ["PATH"] = oldPath + + print("+ Setting PATH to: {}".format(newPath)) + with common_ship.ModifyEnv(PATH=newPath): + subprocess.check_call(["sh.exe", "configure"]) + subprocess.check_call(["make.exe", "clean"]) + makeBaseCmd = [ + "make.exe", + "COMMIT_HASH=" + common_ship.commitHash, + "PREFIX=ship/packages/" + packageName + ] + subprocess.check_call(makeBaseCmd + ["all", "tests", "-j%d" % multiprocessing.cpu_count()]) + subprocess.check_call(["build\\trivial_test.exe"]) + subprocess.check_call(makeBaseCmd + ["install"]) + subprocess.check_call(["tar.exe", "cvfz", + packageName + ".tar.gz", + packageName], cwd=os.path.join(os.getcwd(), "ship", "packages")) + def main(): - oldPath = os.environ["PATH"] - for t in BUILD_TARGETS: - os.environ["PATH"] = t["path"] + ";" + common_ship.defaultPathEnviron - subprocess.check_output(["tar.exe", "--help"]) - subprocess.check_output(["make.exe", "--help"]) - for t in BUILD_TARGETS: + targets = list(BUILD_TARGETS) + if len(sys.argv) != 1: + targets = [t for t in targets if t["systemName"] in sys.argv[1:]] + + for t in targets: + newPath = setup_cyg_system(t) + with common_ship.ModifyEnv(PATH=newPath): + subprocess.check_output(["tar.exe", "--help"]) + subprocess.check_output(["make.exe", "--help"]) + + for t in targets: buildTarget(t) + if __name__ == "__main__": main()