Build/package winpty for embedding using gyp and MSVC

.gitignore

@@ -2,10 +2,12 @@
 *.suo
 *.vcxproj
 *.vcxproj.filters
+*.pyc
 winpty.sdf
 winpty.opensdf
 /config.mk
 /build
+/build-gyp
 /ship/packages
 /src/Default
 /src/Release

Makefile

@@ -145,8 +145,8 @@ install : \
 clean :
 	rm -fr build
 
-.PHONY : clean-msvs
+.PHONY : clean-msvc
-clean-msvs :
+clean-msvc :
 	rm -fr src/Default src/Release src/.vs
 	rm -f src/*.vcxproj src/*.vcxproj.filters src/*.sln src/*.sdf
 

ship/common_ship.py

@@ -0,0 +1,48 @@
+import os
+import sys
+
+if os.name != "nt":
+    sys.exit("Error: ship scripts require native Python 2.7. (wrong os.name)")
+if sys.version_info[0:2] != (2,7):
+    sys.exit("Error: ship scripts require native Python 2.7. (wrong version)")
+
+import glob
+import shutil
+import subprocess
+from distutils.spawn import find_executable
+
+topDir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+
+with open(topDir + "/VERSION.txt", "rt") as f:
+    winptyVersion = f.read().strip()
+
+def writeBuildInfo():
+    with open(topDir + "/BUILD_INFO.txt", "w") as f:
+        f.write("VERSION_SUFFIX=\n")
+        f.write("COMMIT_HASH=" + commitHash + "\n")
+
+def rmrf(patterns):
+    for pattern in patterns:
+        for path in glob.glob(pattern):
+            if os.path.isdir(path) and not os.path.islink(path):
+                print "+ rm -r " + path
+                sys.stdout.flush()
+                shutil.rmtree(path)
+            elif os.path.isfile(path):
+                print "+ rm " + path
+                sys.stdout.flush()
+                os.remove(path)
+
+def mkdir(path):
+    if not os.path.isdir(path):
+        os.makedirs(path)
+
+def requireExe(name):
+    ret = find_executable(name)
+    if ret is None:
+        sys.exit("Error: required EXE is missing from Path: " + name)
+    return ret
+
+requireExe("git.exe")
+commitHash = subprocess.check_output(["git.exe", "rev-parse", "HEAD"]).decode().strip()
+defaultPathEnviron = "C:\\Windows\\System32;C:\\Windows"

ship/make_msvc_package.py

@@ -0,0 +1,168 @@
+#!python
+
+# Copyright (c) 2016 Ryan Prichard
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+#
+# Run with native CPython 2.7.
+#
+# These programs must be in your Path:
+#  - 7z.exe
+#  - git.exe
+#
+# This script looks for MSVC using a version-specific environment variable,
+# such as VS140COMNTOOLS for MSVC 2015.
+#
+
+import common_ship
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+
+os.chdir(common_ship.topDir)
+ZIP_TOOL = common_ship.requireExe("7z.exe")
+
+MSVC_VERSION_TABLE = {
+    "2015" : {
+        "package_name" : "msvc2015",
+        "gyp_version" : "2015",
+        "common_tools_env" : "VS140COMNTOOLS",
+        "xp_toolset" : "v140_xp",
+    },
+    "2013" : {
+        "package_name" : "msvc2013",
+        "gyp_version" : "2013",
+        "common_tools_env" : "VS120COMNTOOLS",
+        "xp_toolset" : "v120_xp",
+    },
+}
+
+ARCH_TABLE = {
+    "x64" : {
+        "msvc_platform" : "x64",
+    },
+    "ia32" : {
+        "msvc_platform" : "Win32",
+    },
+}
+
+def readArguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--msvc-version", default="2015")
+    ret = parser.parse_args()
+    if ret.msvc_version not in MSVC_VERSION_TABLE:
+        sys.exit("Error: unrecognized version: " + ret.msvc_version + ". " +
+            "Versions: " + " ".join(sorted(MSVC_VERSION_TABLE.keys())))
+    return ret
+
+ARGS = readArguments()
+
+def checkoutGyp():
+    if os.path.isdir("build-gyp"):
+        return
+    subprocess.check_call([
+        "git.exe",
+        "clone",
+        "https://chromium.googlesource.com/external/gyp",
+        "build-gyp"
+    ])
+
+def cleanMsvc():
+    common_ship.rmrf("""
+        src/Release src/.vs
+        src/*.vcxproj src/*.vcxproj.filters src/*.sln src/*.sdf
+    """.split())
+
+def build(arch, packageDir, xp=False):
+    archInfo = ARCH_TABLE[arch]
+    versionInfo = MSVC_VERSION_TABLE[ARGS.msvc_version]
+
+    subprocess.check_call([
+        sys.executable,
+        "../build-gyp/gyp_main.py",
+        "winpty.gyp",
+        "-I", "configurations.gypi",
+        "-G", "msvs_version=" + versionInfo["gyp_version"]] +
+        (["-D", "WINPTY_MSBUILD_TOOLSET=" + versionInfo["xp_toolset"]] if xp else []),
+        cwd="src")
+    devCmdPath = os.path.join(os.environ[versionInfo["common_tools_env"]], "VsDevCmd.bat")
+    if not os.path.isfile(devCmdPath):
+        sys.exit("Error: MSVC environment script missing: " + devCmdPath)
+    subprocess.check_call(
+        '"' + devCmdPath + '" && ' +
+        "msbuild winpty.sln /m /p:Platform=" + ARCH_TABLE[arch]["msvc_platform"],
+        shell=True,
+        cwd="src")
+
+    archPackageDir = os.path.join(packageDir, arch)
+    if xp:
+        archPackageDir += "_xp"
+
+    common_ship.mkdir(archPackageDir + "/bin")
+    common_ship.mkdir(archPackageDir + "/lib")
+
+    binSrc = os.path.join(common_ship.topDir, "src/Release", archInfo["msvc_platform"])
+
+    shutil.copy(binSrc + "/winpty.dll",                 archPackageDir + "/bin")
+    shutil.copy(binSrc + "/winpty-agent.exe",           archPackageDir + "/bin")
+    shutil.copy(binSrc + "/winpty-debugserver.exe",     archPackageDir + "/bin")
+    shutil.copy(binSrc + "/winpty.lib",                 archPackageDir + "/lib")
+
+def buildPackage():
+    versionInfo = MSVC_VERSION_TABLE[ARGS.msvc_version]
+
+    packageName = "winpty-%s-%s" % (
+        common_ship.winptyVersion,
+        versionInfo["package_name"],
+    )
+
+    packageRoot = os.path.join(common_ship.topDir, "ship/packages")
+    packageDir = os.path.join(packageRoot, packageName)
+    packageFile = packageDir + ".zip"
+
+    common_ship.rmrf([packageDir])
+    common_ship.rmrf([packageFile])
+    common_ship.mkdir(packageDir)
+
+    checkoutGyp()
+    cleanMsvc()
+    build("ia32", packageDir, True)
+    cleanMsvc()
+    build("ia32", packageDir)
+    build("x64", packageDir)
+
+    topDir = common_ship.topDir
+
+    common_ship.mkdir(packageDir + "/include")
+    shutil.copy(topDir + "/src/include/winpty.h",               packageDir + "/include")
+    shutil.copy(topDir + "/src/include/winpty_constants.h",     packageDir + "/include")
+    shutil.copy(topDir + "/LICENSE",                            packageDir)
+    shutil.copy(topDir + "/README.md",                          packageDir)
+    shutil.copy(topDir + "/RELEASES.md",                        packageDir)
+
+    subprocess.check_call([ZIP_TOOL, "a", packageFile, "."], cwd=packageDir)
+
+    common_ship.rmrf([packageDir])
+
+if __name__ == "__main__":
+    buildPackage()

ship/ship.py

@@ -21,29 +21,22 @@
 # IN THE SOFTWARE.
 
 #
-# Run with native CPython 2 on a 64-bit computer.  The pip package, "pefile",
+# Run with native CPython 2.7 on a 64-bit computer.
-# must be installed.
 #
 # Each of the targets in BUILD_TARGETS must be installed to the default
 # location.  Each target must have the appropriate MinGW and non-MinGW
 # compilers installed, as well as make and tar.
 #
 
+import common_ship
+
+import multiprocessing
 import os
 import shutil
 import subprocess
+import sys
 
-# Ensure that we're in the root directory.
+os.chdir(common_ship.topDir)
-if not os.path.exists("VERSION.txt"):
-    os.chdir("..")
-with open("VERSION.txt", "rt") as f:
-    VERSION = f.read().strip()
-
-# Check other environment considerations
-if os.name != "nt":
-    sys.exit("Error: ship.py should run in a native CPython.")
-if os.environ.get("SHELL") is not None:
-    sys.exit("Error: ship.py should run outside a Cygwin environment.")
 
 def dllVersion(path):
     version = subprocess.check_output(
@@ -53,7 +46,7 @@ def dllVersion(path):
 
 # Determine other build parameters.
 print "Determining Cygwin/MSYS2 DLL versions..."
-COMMIT_HASH = subprocess.check_output(["git.exe", "rev-parse", "HEAD"]).decode().strip()
+sys.stdout.flush()
 BUILD_TARGETS = [
     {
         "name": "msys",
@@ -79,20 +72,15 @@ BUILD_TARGETS = [
     },
 ]
 
-def writeBuildInfo():
-    with open("BUILD_INFO.txt", "w") as f:
-        f.write("VERSION_SUFFIX=\n")
-        f.write("COMMIT_HASH=" + COMMIT_HASH + "\n")
-
 def buildTarget(target):
-    packageName = "winpty-" + VERSION + "-" + target["name"]
+    packageName = "winpty-" + common_ship.winptyVersion + "-" + target["name"]
     oldPath = os.environ["PATH"]
-    os.environ["PATH"] = target["path"] + ";" + oldPath
+    os.environ["PATH"] = target["path"] + ";" + common_ship.defaultPathEnviron
     subprocess.check_call(["sh.exe", "configure"])
     subprocess.check_call(["make.exe", "clean"])
     makeBinary = target.get("make_binary", "make.exe")
     buildArgs = [makeBinary, "USE_PCH=0", "all", "tests"]
-    buildArgs += ["-j8"]
+    buildArgs += ["-j%d" % multiprocessing.cpu_count()]
     subprocess.check_call(buildArgs)
     subprocess.check_call(["build\\trivial_test.exe"])
     subprocess.check_call([makeBinary, "USE_PCH=0", "PREFIX=ship/packages/" + packageName, "install"])
@@ -104,12 +92,12 @@ def buildTarget(target):
 
 def main():
     try:
-        writeBuildInfo()
+        common_ship.writeBuildInfo()
         if os.path.exists("ship\\packages"):
             shutil.rmtree("ship\\packages")
         oldPath = os.environ["PATH"]
         for t in BUILD_TARGETS:
-            os.environ["PATH"] = t["path"] + ";" + oldPath
+            os.environ["PATH"] = t["path"] + ";" + common_ship.defaultPathEnviron
             subprocess.check_output(["tar.exe", "--help"])
             subprocess.check_output(["make.exe", "--help"])
         for t in BUILD_TARGETS: