ship.py: build packages using cygwin/msys2 prebuilts

.gitignore

@@ -10,6 +10,7 @@ winpty.opensdf
 /build-gyp
 /build-libpty
 /ship/packages
+/ship/tmp
 /src/Default
 /src/Release
 /src/gen

ship/common_ship.py

@@ -1,6 +1,10 @@
 import os
 import sys
 
+# These scripts need to continue using Python 2 rather than 3, because
+# make_msvc_package.py puts the current Python interpreter on the PATH for the
+# sake of gyp, and gyp doesn't work with Python 3 yet.
+# https://bugs.chromium.org/p/gyp/issues/detail?id=36
 if os.name != "nt":
     sys.exit("Error: ship scripts require native Python 2.7. (wrong os.name)")
 if sys.version_info[0:2] != (2,7):
@@ -20,11 +24,11 @@ def rmrf(patterns):
     for pattern in patterns:
         for path in glob.glob(pattern):
             if os.path.isdir(path) and not os.path.islink(path):
-                print "+ rm -r " + path
+                print("+ rm -r " + path)
                 sys.stdout.flush()
                 shutil.rmtree(path)
             elif os.path.isfile(path):
-                print "+ rm " + path
+                print("+ rm " + path)
                 sys.stdout.flush()
                 os.remove(path)
 
@@ -37,17 +41,38 @@ def requireExe(name, guesses):
         for guess in guesses:
             if os.path.exists(guess):
                 newDir = os.path.dirname(guess)
-                print "Adding " + newDir + " to Path to provide " + name
+                print("Adding " + newDir + " to Path to provide " + name)
                 os.environ["Path"] = newDir + ";" + os.environ["Path"]
     ret = find_executable(name)
     if ret is None:
         sys.exit("Error: required EXE is missing from Path: " + name)
     return ret
 
+class ModifyEnv:
+    def __init__(self, **kwargs):
+        self._changes = dict(kwargs)
+        self._original = dict()
+
+    def __enter__(self):
+        for var, val in self._changes.items():
+            self._original[var] = os.environ[var]
+            os.environ[var] = val
+
+    def __exit__(self, type, value, traceback):
+        for var, val in self._original.items():
+            os.environ[var] = val
+
 requireExe("git.exe", [
     "C:\\Program Files\\Git\\cmd\\git.exe",
     "C:\\Program Files (x86)\\Git\\cmd\\git.exe"
 ])
 
-commitHash = subprocess.check_output(["git.exe", "rev-parse", "HEAD"]).decode().strip()
+commitHash = subprocess.check_output(["git.exe", "rev-parse", "HEAD"]).strip()
 defaultPathEnviron = "C:\\Windows\\System32;C:\\Windows"
+
+ZIP_TOOL = requireExe("7z.exe", [
+    "C:\\Program Files\\7-Zip\\7z.exe",
+    "C:\\Program Files (x86)\\7-Zip\\7z.exe",
+])
+
+requireExe("curl.exe", [])

ship/make_msvc_package.py

@@ -36,10 +36,6 @@ import subprocess
 import sys
 
 os.chdir(common_ship.topDir)
-ZIP_TOOL = common_ship.requireExe("7z.exe", [
-    "C:\\Program Files\\7-Zip\\7z.exe",
-    "C:\\Program Files (x86)\\7-Zip\\7z.exe",
-])
 
 MSVC_VERSION_TABLE = {
     "2015" : {
@@ -103,7 +99,7 @@ def build(arch, packageDir, xp=False):
     newEnv = os.environ.copy()
     newEnv["PATH"] = os.path.dirname(sys.executable) + ";" + common_ship.defaultPathEnviron
     commandLine = (
-        '"' + devCmdPath + '" && '
+        '"' + devCmdPath + '" && ' +
         " vcbuild.bat" +
         " --gyp-msvs-version " + versionInfo["gyp_version"] +
         " --msvc-platform " + archInfo["msvc_platform"] +
@@ -159,7 +155,7 @@ def buildPackage():
     shutil.copy(topDir + "/README.md",                          packageDir)
     shutil.copy(topDir + "/RELEASES.md",                        packageDir)
 
-    subprocess.check_call([ZIP_TOOL, "a", packageFile, "."], cwd=packageDir)
+    subprocess.check_call([common_ship.ZIP_TOOL, "a", packageFile, "."], cwd=packageDir)
 
 if __name__ == "__main__":
     buildPackage()

ship/ship.py

@@ -23,10 +23,6 @@
 #
 # Run with native CPython 2.7 on a 64-bit computer.
 #
-# Each of the targets in BUILD_TARGETS must be installed to the default
-# location.  Each target must have the appropriate MinGW and non-MinGW
-# compilers installed, as well as make and tar.
-#
 
 import common_ship
 
@@ -36,7 +32,6 @@ import shutil
 import subprocess
 import sys
 
-os.chdir(common_ship.topDir)
 
 def dllVersion(path):
     version = subprocess.check_output(
@@ -44,58 +39,120 @@ def dllVersion(path):
         "[System.Diagnostics.FileVersionInfo]::GetVersionInfo(\"" + path + "\").FileVersion"])
     return version.strip()
 
+
+os.chdir(common_ship.topDir)
+
+URL_BASE = "file://c:/rprichard/proj/winpty-cygwin-prebuilts/out/artifact/"
+
 # Determine other build parameters.
-print "Determining Cygwin/MSYS2 DLL versions..."
+print("Determining Cygwin/MSYS2 DLL versions...")
 sys.stdout.flush()
 BUILD_TARGETS = [
     {
-        "name": "msys2-" + dllVersion("C:\\msys32\\usr\\bin\\msys-2.0.dll") + "-ia32",
+        "systemName": "msys32",
-        "path": "C:\\msys32\\mingw32\\bin;C:\\msys32\\usr\\bin",
+        "systemArchive":               "msys32-20181014-dll2.11.1-msysgcc7.3.0-wingcc7.3.0.7z",
+        "systemArchiveUrl": URL_BASE + "msys32-20181014-dll2.11.1-msysgcc7.3.0-wingcc7.3.0.7z",
+        "packageName": "msys2-{dllver}-ia32",
+        "rebase": ["usr\\bin\\ash.exe", "/usr/bin/rebaseall", "-v"],
+        "dll": "usr\\bin\\msys-2.0.dll",
+        "path": ["opt\\bin", "usr\\bin"],
     },
     {
-        "name": "msys2-" + dllVersion("C:\\msys64\\usr\\bin\\msys-2.0.dll") + "-x64",
+        "systemName": "msys64",
-        "path": "C:\\msys64\\mingw64\\bin;C:\\msys64\\usr\\bin",
+        "systemArchive":               "msys64-20181014-dll2.11.1-msysgcc7.3.0-wingcc7.3.0.7z",
+        "systemArchiveUrl": URL_BASE + "msys64-20181014-dll2.11.1-msysgcc7.3.0-wingcc7.3.0.7z",
+        "packageName": "msys2-{dllver}-x64",
+        "rebase": ["usr\\bin\\ash.exe", "/usr/bin/rebaseall", "-v"],
+        "dll": "usr\\bin\\msys-2.0.dll",
+        "path": ["opt\\bin", "usr\\bin"],
     },
     {
-        "name": "cygwin-" + dllVersion("C:\\cygwin\\bin\\cygwin1.dll") + "-ia32",
+        "systemName": "cygwin32",
-        "path": "C:\\cygwin\\bin",
+        "systemArchive":               "cygwin32-20181014-dll2.11.1-cyggcc7.3.0-wingcc6.4.0.7z",
+        "systemArchiveUrl": URL_BASE + "cygwin32-20181014-dll2.11.1-cyggcc7.3.0-wingcc6.4.0.7z",
+        "packageName": "cygwin-{dllver}-ia32",
+        "rebase": ["bin\\ash.exe", "/bin/rebaseall", "-v"],
+        "dll": "bin\\cygwin1.dll",
+        "path": ["bin"],
     },
     {
-        "name": "cygwin-" + dllVersion("C:\\cygwin64\\bin\\cygwin1.dll") + "-x64",
+        "systemName": "cygwin64",
-        "path": "C:\\cygwin64\\bin",
+        "systemArchive":               "cygwin64-20181014-dll2.11.1-cyggcc7.3.0-wingcc6.4.0.7z",
+        "systemArchiveUrl": URL_BASE + "cygwin64-20181014-dll2.11.1-cyggcc7.3.0-wingcc6.4.0.7z",
+        "packageName": "cygwin-{dllver}-x64",
+        "rebase": ["bin\\ash.exe", "/bin/rebaseall", "-v"],
+        "dll": "bin\\cygwin1.dll",
+        "path": ["bin"],
     },
 ]
 
+
+def targetSystemDir(target):
+    return os.path.abspath(os.path.join("ship", "tmp", target["systemName"]))
+
+
+def setup_cyg_system(target):
+    common_ship.mkdir("ship/tmp")
+
+    systemDir = targetSystemDir(target)
+    systemArchivePath = os.path.abspath(os.path.join("ship", "tmp", target["systemArchive"]))
+    binPaths = [os.path.join(systemDir, p) for p in target["path"]]
+    binPaths += common_ship.defaultPathEnviron.split(";")
+    newPath = ";".join(binPaths)
+
+    if not os.path.exists(systemDir):
+        if not os.path.exists(systemArchivePath):
+            subprocess.check_call(["curl.exe", "-fL", "-o", systemArchivePath, target["systemArchiveUrl"]])
+            assert os.path.exists(systemArchivePath)
+        subprocess.check_call(["7z.exe", "x", systemArchivePath], cwd=os.path.dirname(systemDir))
+        with common_ship.ModifyEnv(PATH=newPath):
+            rebaseCmd = target["rebase"]
+            rebaseCmd[0] = os.path.join(systemDir, rebaseCmd[0])
+            subprocess.check_call(rebaseCmd)
+    assert os.path.exists(systemDir)
+
+    return newPath
+
+
 def buildTarget(target):
-    packageName = "winpty-" + common_ship.winptyVersion + "-" + target["name"]
+    newPath = setup_cyg_system(target)
+
+    dllver = dllVersion(os.path.join(targetSystemDir(target), target["dll"]))
+    packageName = "winpty-" + common_ship.winptyVersion + "-" + target["packageName"].format(dllver=dllver)
     if os.path.exists("ship\\packages\\" + packageName):
         shutil.rmtree("ship\\packages\\" + packageName)
-    oldPath = os.environ["PATH"]
+
-    os.environ["PATH"] = target["path"] + ";" + common_ship.defaultPathEnviron
+    print("+ Setting PATH to: {}".format(newPath))
-    subprocess.check_call(["sh.exe", "configure"])
+    with common_ship.ModifyEnv(PATH=newPath):
-    subprocess.check_call(["make.exe", "clean"])
+        subprocess.check_call(["sh.exe", "configure"])
-    makeBaseCmd = [
+        subprocess.check_call(["make.exe", "clean"])
-        "make.exe",
+        makeBaseCmd = [
-        "USE_PCH=0",
+            "make.exe",
-        "COMMIT_HASH=" + common_ship.commitHash,
+            "COMMIT_HASH=" + common_ship.commitHash,
-        "PREFIX=ship/packages/" + packageName
+            "PREFIX=ship/packages/" + packageName
-    ]
+        ]
-    subprocess.check_call(makeBaseCmd + ["all", "tests", "-j%d" % multiprocessing.cpu_count()])
+        subprocess.check_call(makeBaseCmd + ["all", "tests", "-j%d" % multiprocessing.cpu_count()])
-    subprocess.check_call(["build\\trivial_test.exe"])
+        subprocess.check_call(["build\\trivial_test.exe"])
-    subprocess.check_call(makeBaseCmd + ["install"])
+        subprocess.check_call(makeBaseCmd + ["install"])
-    subprocess.check_call(["tar.exe", "cvfz",
+        subprocess.check_call(["tar.exe", "cvfz",
-        packageName + ".tar.gz",
+            packageName + ".tar.gz",
-        packageName], cwd=os.path.join(os.getcwd(), "ship", "packages"))
+            packageName], cwd=os.path.join(os.getcwd(), "ship", "packages"))
-    os.environ["PATH"] = oldPath
+
 
 def main():
-    oldPath = os.environ["PATH"]
+    targets = list(BUILD_TARGETS)
-    for t in BUILD_TARGETS:
+    if len(sys.argv) != 1:
-        os.environ["PATH"] = t["path"] + ";" + common_ship.defaultPathEnviron
+        targets = [t for t in targets if t["systemName"] in sys.argv[1:]]
-        subprocess.check_output(["tar.exe", "--help"])
+
-        subprocess.check_output(["make.exe", "--help"])
+    for t in targets:
-    for t in BUILD_TARGETS:
+        newPath = setup_cyg_system(t)
+        with common_ship.ModifyEnv(PATH=newPath):
+            subprocess.check_output(["tar.exe", "--help"])
+            subprocess.check_output(["make.exe", "--help"])
+
+    for t in targets:
         buildTarget(t)
 
+
 if __name__ == "__main__":
     main()