diff --git a/.gitmodules b/.gitmodules index 1e3e8a3..af7df38 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "terryfy"] - path = scripts/terryfy - url = https://github.com/MacPython/terryfy.git [submodule "research/esaxx"] path = research/esaxx url = https://github.com/hillbig/esaxx diff --git a/.travis.yml b/.travis.yml index 73a7615..2f26eaa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -125,9 +125,22 @@ matrix: env: BUILD_SYSTEM=cmake C_COMPILER=pgcc CXX_COMPILER=pgc++ ### - ## Python build on Linux + ## Python 2.7 and 3.6 builds on Linux ### - os: linux + language: python + python: 2.7 + env: BUILD_SYSTEM=python C_COMPILER=gcc-6 CXX_COMPILER=g++-6 + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-6 + - g++-6 + - os: linux + language: python + python: 3.6 env: BUILD_SYSTEM=python C_COMPILER=gcc-6 CXX_COMPILER=g++-6 addons: apt: @@ -163,14 +176,10 @@ matrix: env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.4 CXX_COMPILER=g++-4.4 ### - ## Python OS X builds + ## Python 2.7 OS X build (using the system /usr/bin/python) ### - os: osx - env: BUILD_SYSTEM=python INSTALL_TYPE=macpython PYTHON_VERSION=2.7.12 C_COMPILER=gcc CXX_COMPILER=g++ - - os: osx - env: BUILD_SYSTEM=python INSTALL_TYPE=macpython PYTHON_VERSION=3.4.4 C_COMPILER=gcc CXX_COMPILER=g++ - - os: osx - env: BUILD_SYSTEM=python INSTALL_TYPE=macpython PYTHON_VERSION=3.5.2 C_COMPILER=gcc CXX_COMPILER=g++ + env: BUILD_SYSTEM=python C_COMPILER=gcc CXX_COMPILER=g++ ### ## Sanitizers @@ -262,19 +271,9 @@ after_success: - scripts/.travis.sh after_success before_deploy: -- if [ "${BUILD_SYSTEM}" = "python" ] && [ "${TRAVIS_OS_NAME}" = "osx" ]; then export WHEELS=$(ls ./dist/*.whl); fi - scripts/.travis.sh before_deploy deploy: -- provider: releases - api_key: - secure: YcCBi6W/w4dtKCa59Wfm8L5lGWvK7KxaFNDr3yh1Hz5aStXXf758pEMHGewnlbfbwuj5a3SjBb1nLp1M69OQJfxm442uXBaBKo52PM9PPbD7NjvbNIso73pqcSODXQXKuZxDFpEhfuDTVq3hUkUqiwhChWhrFucJsSL51i7qSss= - file: "${WHEELS}" - skip_cleanup: true - on: - repo: "google/brotli" - tags: true - condition: "${BUILD_SYSTEM} = python && ${TRAVIS_OS_NAME} = osx" - provider: bintray file: "scripts/.bintray.json" user: "eustas" diff --git a/python/Makefile b/python/Makefile index 646f330..d5d88f8 100644 --- a/python/Makefile +++ b/python/Makefile @@ -8,10 +8,12 @@ .PHONY: all # Build .PHONY: build +# Develop +.PHONY: develop # Install .PHONY: install # Test -.PHONY: test tests +.PHONY: test # Clean .PHONY: clean # Format @@ -31,16 +33,17 @@ all: build build: $(EXTENSIONS) $(EXTENSIONS): $(EXT_SOURCES) + @cd .. && $(PYTHON) setup.py build_ext --inplace + +develop: @cd .. && $(PYTHON) setup.py develop install: @cd .. && $(PYTHON) setup.py install -test: tests - -tests: build +test: build @echo 'running tests' - @$(PYTHON) -m unittest discover -p '*_test.py' + @$(PYTHON) -m unittest discover -v -p '*_test.py' clean: @cd .. && $(PYTHON) setup.py clean diff --git a/python/README.md b/python/README.md index 33cba56..3b0d8b2 100644 --- a/python/README.md +++ b/python/README.md @@ -17,19 +17,20 @@ following command from this directory: ### Development -For development, reinstalling the module with every change is time -consuming. Instead, we recommend using the `setuptools` -"[development mode][]" to make the module available while still being -able to edit the source files. +You may run the following commands from this directory: -For convenience, you may run the following commands from this directory: - - $ make # Deploy the module in "development mode" + $ make # Build the module in-place - $ make tests # Test the module + $ make test # Test the module $ make clean # Remove all temporary files and build output +If you wish to make the module available while still being +able to edit the source files, you can use the `setuptools` +"[development mode][]": + + $ make develop # Install the module in "development mode" + ### Code Style Brotli's code follows the [Google Python Style Guide][]. To diff --git a/scripts/.travis.sh b/scripts/.travis.sh index be9313e..63939a6 100755 --- a/scripts/.travis.sh +++ b/scripts/.travis.sh @@ -16,11 +16,6 @@ case "$1" in esac case "${BUILD_SYSTEM}" in - "python") - source scripts/terryfy/travis_tools.sh - get_python_environment $INSTALL_TYPE $PYTHON_VERSION venv - pip install --upgrade wheel - ;; "bazel") brew install bazel ;; @@ -50,10 +45,7 @@ case "$1" in ctest -V ;; "python") - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then - source venv/bin/activate - fi - python setup.py build test + python setup.py test ;; "maven") cd java/org/brotli @@ -65,16 +57,6 @@ case "$1" in esac ;; "after_success") - case "${BUILD_SYSTEM}" in - "python") - case "${TRAVIS_OS_NAME}" in - "osx") - source venv/bin/activate - pip wheel -w dist . - ;; - esac - ;; - esac ;; "before_deploy") case "${BUILD_SYSTEM}" in diff --git a/scripts/appveyor.yml b/scripts/appveyor.yml index b5b1294..c342001 100644 --- a/scripts/appveyor.yml +++ b/scripts/appveyor.yml @@ -1,11 +1,4 @@ environment: - - global: - # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the - # /E:ON and /V:ON options are not enabled in the batch script intepreter - # See: http://stackoverflow.com/a/13751649/163740 - WITH_COMPILER: "cmd /E:ON /V:ON /C .\\scripts\\run_with_compiler.cmd" - matrix: - BUILD_SYSTEM: CMake GENERATOR: Visual Studio 14 2015 Win64 @@ -25,13 +18,8 @@ environment: PYTHON_ARCH: "32" - BUILD_SYSTEM: Python - PYTHON: "C:\\Python34" - PYTHON_VERSION: "3.4.x" - PYTHON_ARCH: "32" - - - BUILD_SYSTEM: Python - PYTHON: "C:\\Python35" - PYTHON_VERSION: "3.5.0" + PYTHON: "C:\\Python36" + PYTHON_VERSION: "3.6.x" PYTHON_ARCH: "32" - BUILD_SYSTEM: Python @@ -40,29 +28,21 @@ environment: PYTHON_ARCH: "64" - BUILD_SYSTEM: Python - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4.x" - PYTHON_ARCH: "64" - - - BUILD_SYSTEM: Python - PYTHON: "C:\\Python35-x64" - PYTHON_VERSION: "3.5.0" + PYTHON: "C:\\Python36-x64" + PYTHON_VERSION: "3.6.x" PYTHON_ARCH: "64" install: - ps: | if ($env:BUILD_SYSTEM -eq "Python") { - # install Python and pip when not already installed - if (-not(Test-Path($env:PYTHON))) { & scripts\install_py.ps1 } - - # prepend newly installed Python to the PATH + # prepend Python to the PATH $env:Path = $env:PYTHON + ';' + $env:PYTHON + '\\Scripts;' + $env:PATH # upgrade pip to avoid out-of-date warnings pip install --disable-pip-version-check --user --upgrade pip - # install/upgrade setuptools and wheel to build packages - pip install --upgrade setuptools six wheel + # install/upgrade setuptools to build packages + pip install --upgrade setuptools } before_build: @@ -82,26 +62,4 @@ build_script: test_script: - if "%BUILD_SYSTEM%" == "CMake" ( ctest --output-on-failure --interactive-debug-mode 0 -C Debug ) - - if "%BUILD_SYSTEM%" == "Python" ( python setup.py build test ) - -after_test: - - if "%BUILD_SYSTEM%" == "Python" ( pip wheel -w dist . ) - - if "%BUILD_SYSTEM%" == "Python" ( python setup.py sdist --formats=gztar,zip ) - -artifacts: - # archive the generated packages in the ci.appveyor.com build report - - path: dist\*.whl - - path: dist\*.zip - - path: dist\*.tar.gz - -# For info, see: http://www.appveyor.com/docs/deployment/github - -deploy: - - provider: GitHub - auth_token: - secure: dfL56DgbwuGJNNE5GzKi/pAgBQnJ37Du+AnCtnsTnIYxpis8ah3fPmA/G+bn4NJ3 - artifact: - draft: false - prerelease: false - on: - appveyor_repo_tag: true + - if "%BUILD_SYSTEM%" == "Python" ( python setup.py test ) diff --git a/scripts/install_py.ps1 b/scripts/install_py.ps1 deleted file mode 100644 index bdd0cd3..0000000 --- a/scripts/install_py.ps1 +++ /dev/null @@ -1,177 +0,0 @@ -# Sample script to install Python and pip under Windows -# Authors: Olivier Grisel, Jonathan Helmus, Kyle Kastner, and Alex Willmer -# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ -# Source: https://github.com/ogrisel/python-appveyor-demo/blob/master/appveyor/install.ps1 - -$BASE_URL = "https://www.python.org/ftp/python/" -$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" -$GET_PIP_PATH = "C:\get-pip.py" - -$PYTHON_PRERELEASE_REGEX = @" -(?x) -(?\d+) -\. -(?\d+) -\. -(?\d+) -(?[a-z]{1,2}\d+) -"@ - - -function Download ($filename, $url) { - $webclient = New-Object System.Net.WebClient - - $basedir = $pwd.Path + "\" - $filepath = $basedir + $filename - if (Test-Path $filename) { - Write-Host "Reusing" $filepath - return $filepath - } - - # Download and retry up to 3 times in case of network transient errors. - Write-Host "Downloading" $filename "from" $url - $retry_attempts = 2 - for ($i = 0; $i -lt $retry_attempts; $i++) { - try { - $webclient.DownloadFile($url, $filepath) - break - } - Catch [Exception]{ - Start-Sleep 1 - } - } - if (Test-Path $filepath) { - Write-Host "File saved at" $filepath - } else { - # Retry once to get the error message if any at the last try - $webclient.DownloadFile($url, $filepath) - } - return $filepath -} - - -function ParsePythonVersion ($python_version) { - if ($python_version -match $PYTHON_PRERELEASE_REGEX) { - return ([int]$matches.major, [int]$matches.minor, [int]$matches.micro, - $matches.prerelease) - } - $version_obj = [version]$python_version - return ($version_obj.major, $version_obj.minor, $version_obj.build, "") -} - - -function DownloadPython ($python_version, $platform_suffix) { - $major, $minor, $micro, $prerelease = ParsePythonVersion $python_version - - if (($major -le 2 -and $micro -eq 0) ` - -or ($major -eq 3 -and $minor -le 2 -and $micro -eq 0) ` - ) { - $dir = "$major.$minor" - $python_version = "$major.$minor$prerelease" - } else { - $dir = "$major.$minor.$micro" - } - - if ($prerelease) { - if (($major -le 2) ` - -or ($major -eq 3 -and $minor -eq 1) ` - -or ($major -eq 3 -and $minor -eq 2) ` - -or ($major -eq 3 -and $minor -eq 3) ` - ) { - $dir = "$dir/prev" - } - } - - if (($major -le 2) -or ($major -le 3 -and $minor -le 4)) { - $ext = "msi" - if ($platform_suffix) { - $platform_suffix = ".$platform_suffix" - } - } else { - $ext = "exe" - if ($platform_suffix) { - $platform_suffix = "-$platform_suffix" - } - } - - $filename = "python-$python_version$platform_suffix.$ext" - $url = "$BASE_URL$dir/$filename" - $filepath = Download $filename $url - return $filepath -} - - -function InstallPython ($python_version, $architecture, $python_home) { - Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home - if (Test-Path $python_home) { - Write-Host $python_home "already exists, skipping." - return $false - } - if ($architecture -eq "32") { - $platform_suffix = "" - } else { - $platform_suffix = "amd64" - } - $installer_path = DownloadPython $python_version $platform_suffix - $installer_ext = [System.IO.Path]::GetExtension($installer_path) - Write-Host "Installing $installer_path to $python_home" - $install_log = $python_home + ".log" - if ($installer_ext -eq '.msi') { - InstallPythonMSI $installer_path $python_home $install_log - } else { - InstallPythonEXE $installer_path $python_home $install_log - } - if (Test-Path $python_home) { - Write-Host "Python $python_version ($architecture) installation complete" - } else { - Write-Host "Failed to install Python in $python_home" - Get-Content -Path $install_log - Exit 1 - } -} - - -function InstallPythonEXE ($exepath, $python_home, $install_log) { - $install_args = "/quiet InstallAllUsers=1 TargetDir=$python_home" - RunCommand $exepath $install_args -} - - -function InstallPythonMSI ($msipath, $python_home, $install_log) { - $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home" - $uninstall_args = "/qn /x $msipath" - RunCommand "msiexec.exe" $install_args - if (-not(Test-Path $python_home)) { - Write-Host "Python seems to be installed else-where, reinstalling." - RunCommand "msiexec.exe" $uninstall_args - RunCommand "msiexec.exe" $install_args - } -} - -function RunCommand ($command, $command_args) { - Write-Host $command $command_args - Start-Process -FilePath $command -ArgumentList $command_args -Wait -Passthru -} - - -function InstallPip ($python_home) { - $pip_path = $python_home + "\Scripts\pip.exe" - $python_path = $python_home + "\python.exe" - if (-not(Test-Path $pip_path)) { - Write-Host "Installing pip..." - $webclient = New-Object System.Net.WebClient - $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH) - Write-Host "Executing:" $python_path $GET_PIP_PATH - & $python_path $GET_PIP_PATH - } else { - Write-Host "pip already installed." - } -} - - -function main () { - InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON - InstallPip $env:PYTHON -} - -main \ No newline at end of file diff --git a/scripts/run_with_compiler.cmd b/scripts/run_with_compiler.cmd deleted file mode 100644 index ad125ec..0000000 --- a/scripts/run_with_compiler.cmd +++ /dev/null @@ -1,80 +0,0 @@ -:: To build extensions for 64 bit Python 3, we need to configure environment -:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) -:: -:: To build extensions for 64 bit Python 2, we need to configure environment -:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) -:: -:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific -:: environment configurations. -:: -:: Note: this script needs to be run with the /E:ON and /V:ON flags for the -:: cmd interpreter, at least for (SDK v7.0) -:: -:: More details at: -:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows -:: http://stackoverflow.com/a/13751649/163740 -:: -:: Original source: -:: https://github.com/ogrisel/python-appveyor-demo/blob/master/appveyor/run_with_env.cmd -:: -:: Author: Olivier Grisel -:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ -@ECHO OFF - -SET COMMAND_TO_RUN=%* -SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows -SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf - -:: Extract the major and minor versions, and allow for the minor version to be -:: more than 9. This requires the version number to have two dots in it. -SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1% -IF "%PYTHON_VERSION:~3,1%" == "." ( - SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1% -) ELSE ( - SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2% -) - -:: Based on the Python version, determine what SDK version to use, and whether -:: to set the SDK for 64-bit. -IF %MAJOR_PYTHON_VERSION% == 2 ( - SET WINDOWS_SDK_VERSION="v7.0" - SET SET_SDK_64=Y -) ELSE ( - IF %MAJOR_PYTHON_VERSION% == 3 ( - SET WINDOWS_SDK_VERSION="v7.1" - IF %MINOR_PYTHON_VERSION% LEQ 4 ( - SET SET_SDK_64=Y - ) ELSE ( - SET SET_SDK_64=N - IF EXIST "%WIN_WDK%" ( - :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/ - REN "%WIN_WDK%" 0wdf - ) - ) - ) ELSE ( - ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" - EXIT 1 - ) -) - -IF %PYTHON_ARCH% == 64 ( - IF %SET_SDK_64% == Y ( - ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture - SET DISTUTILS_USE_SDK=1 - SET MSSdk=1 - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 - ) ELSE ( - ECHO Using default MSVC build environment for 64 bit architecture - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 - ) -) ELSE ( - ECHO Using default MSVC build environment for 32 bit architecture - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 -) diff --git a/scripts/terryfy b/scripts/terryfy deleted file mode 160000 index 8bb673f..0000000 --- a/scripts/terryfy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8bb673f4410819df06920fdcfd24e18d235d84f7 diff --git a/setup.py b/setup.py index 6dd513f..a8a2ebe 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,9 @@ except: from distutils.core import Extension from distutils.core import setup from distutils.command.build_ext import build_ext +from distutils import errors +from distutils import dep_util +from distutils import log CURR_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) @@ -53,6 +56,20 @@ class BuildExt(build_ext): return filenames def build_extension(self, ext): + if ext.sources is None or not isinstance(ext.sources, (list, tuple)): + raise errors.DistutilsSetupError( + "in 'ext_modules' option (extension '%s'), " + "'sources' must be present and must be " + "a list of source filenames" % ext.name) + + ext_path = self.get_ext_fullpath(ext.name) + depends = ext.sources + ext.depends + if not (self.force or dep_util.newer_group(depends, ext_path, 'newer')): + log.debug("skipping '%s' extension (up-to-date)", ext.name) + return + else: + log.info("building '%s' extension", ext.name) + c_sources = [] cxx_sources = [] for source in ext.sources: @@ -211,7 +228,7 @@ EXT_MODULES = [ 'c/enc/cluster_inc.h', 'c/enc/command.h', 'c/enc/compress_fragment.h', - 'c/enc/compress_fragment_two_pass.h' + 'c/enc/compress_fragment_two_pass.h', 'c/enc/context.h', 'c/enc/dictionary_hash.h', 'c/enc/entropy_encode.h',