diff --git a/.travis.yml b/.travis.yml index 39ae19c19..cdb68ac42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ -language: c -compiler: gcc +# Declare python as our language. This way we get our chosen Python version, +# and pip is available. Gcc and clang are available anyway. +language: python +python: 3.5 sudo: false cache: ccache @@ -16,10 +18,6 @@ jobs: - libnewlib-arm-none-eabi - gcc-arm-linux-gnueabi - libc6-dev-armel-cross - language: python # Needed to get pip for Python 3 - python: 3.5 # version from Ubuntu 16.04 - install: - - pip install mypy==0.780 pylint==2.4.4 script: - tests/scripts/all.sh -k 'check_*' - tests/scripts/all.sh -k test_default_out_of_box @@ -32,11 +30,16 @@ jobs: - name: Windows os: windows + # The language 'python' is currently unsupported on the + # Windows Build Environment. And 'generic' causes the job to get stuck + # on "Booting virtual machine". + language: c before_install: - choco install python --version=3.5.4 env: # Add the directory where the Choco packages go - PATH=/c/Python35:/c/Python35/Scripts:$PATH + - PYTHON=python.exe script: - type perl; perl --version - type python; python --version @@ -53,6 +56,9 @@ env: - SEED=1 - secure: "FrI5d2s+ckckC17T66c8jm2jV6i2DkBPU5nyWzwbedjmEBeocREfQLd/x8yKpPzLDz7ghOvr+/GQvsPPn0dVkGlNzm3Q+hGHc/ujnASuUtGrcuMM+0ALnJ3k4rFr9xEvjJeWb4SmhJO5UCAZYvTItW4k7+bj9L+R6lt3TzQbXzg=" +install: + - $PYTHON scripts/min_requirements.py + addons: apt: packages: diff --git a/README.md b/README.md index c8d94500e..ea1d7a37b 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,10 @@ The source code of Mbed TLS includes some files that are automatically generated The following tools are required: * Perl, for some library source files and for Visual Studio build files. -* Python 3, for some sample programs and test data. +* Python 3 and some Python packages, for some library source files, sample programs and test data. To install the necessary packages, run + ``` + python -m pip install -r scripts/basic.requirements.txt + ``` * A C compiler for the host platform, for some test data. If you are cross-compiling, you must set the `CC` environment variable to a C compiler for the host platform when generating the configuration-independent files. diff --git a/scripts/basic.requirements.txt b/scripts/basic.requirements.txt new file mode 100644 index 000000000..1be3d0c23 --- /dev/null +++ b/scripts/basic.requirements.txt @@ -0,0 +1,5 @@ +# Python modules required to build Mbed TLS in ordinary conditions. + +# Required to (re-)generate source files. Not needed if the generated source +# files are already present and up-to-date. +-r driver.requirements.txt diff --git a/scripts/ci.requirements.txt b/scripts/ci.requirements.txt new file mode 100644 index 000000000..209ae3d8f --- /dev/null +++ b/scripts/ci.requirements.txt @@ -0,0 +1,12 @@ +# Python package requirements for Mbed TLS testing. + +-r driver.requirements.txt + +# Use a known version of Pylint, because new versions tend to add warnings +# that could start rejecting our code. +# 2.4.4 is the version in Ubuntu 20.04. It supports Python >=3.5. +pylint == 2.4.4 + +# Use the earliest version of mypy that works with our code base. +# See https://github.com/ARMmbed/mbedtls/pull/3953 . +mypy >= 0.780 diff --git a/scripts/driver.requirements.txt b/scripts/driver.requirements.txt new file mode 100644 index 000000000..17569bb17 --- /dev/null +++ b/scripts/driver.requirements.txt @@ -0,0 +1,10 @@ +# Python package requirements for driver implementers. + +# Use the version of Jinja that's in Ubuntu 20.04. +# See https://github.com/ARMmbed/mbedtls/pull/5067#discussion_r738794607 . +# Note that Jinja 3.0 drops support for Python 3.5, so we need to support +# Jinja 2.x as long as we're still using Python 3.5 anywhere. +Jinja2 >= 2.10.1 +# Jinja2 >=2.10, <<3.0 needs a separate package for type annotations +types-Jinja2 + diff --git a/scripts/maintainer.requirements.txt b/scripts/maintainer.requirements.txt new file mode 100644 index 000000000..b149921a2 --- /dev/null +++ b/scripts/maintainer.requirements.txt @@ -0,0 +1,10 @@ +# Python packages that are only useful to Mbed TLS maintainers. + +-r ci.requirements.txt + +# For source code analyses +clang + +# For building some test vectors +pycryptodomex +pycryptodome-test-vectors diff --git a/scripts/min_requirements.py b/scripts/min_requirements.py new file mode 100755 index 000000000..eecab1c1e --- /dev/null +++ b/scripts/min_requirements.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Install all the required Python packages, with the minimum Python version. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import re +import subprocess +import sys +import tempfile +import typing + +from typing import List, Optional +from mbedtls_dev import typing_util + +def pylint_doesn_t_notice_that_certain_types_are_used_in_annotations( + _list: List[typing.Any], +) -> None: + pass + + +class Requirements: + """Collect and massage Python requirements.""" + + def __init__(self) -> None: + self.requirements = [] #type: List[str] + + def adjust_requirement(self, req: str) -> str: + """Adjust a requirement to the minimum specified version.""" + # allow inheritance #pylint: disable=no-self-use + # If a requirement specifies a minimum version, impose that version. + req = re.sub(r'>=|~=', r'==', req) + return req + + def add_file(self, filename: str) -> None: + """Add requirements from the specified file. + + This method supports a subset of pip's requirement file syntax: + * One requirement specifier per line, which is passed to + `adjust_requirement`. + * Comments (``#`` at the beginning of the line or after whitespace). + * ``-r FILENAME`` to include another file. + """ + for line in open(filename): + line = line.strip() + line = re.sub(r'(\A|\s+)#.*', r'', line) + if not line: + continue + m = re.match(r'-r\s+', line) + if m: + nested_file = os.path.join(os.path.dirname(filename), + line[m.end(0):]) + self.add_file(nested_file) + continue + self.requirements.append(self.adjust_requirement(line)) + + def write(self, out: typing_util.Writable) -> None: + """List the gathered requirements.""" + for req in self.requirements: + out.write(req + '\n') + + def install( + self, + pip_general_options: Optional[List[str]] = None, + pip_install_options: Optional[List[str]] = None, + ) -> None: + """Call pip to install the requirements.""" + if pip_general_options is None: + pip_general_options = [] + if pip_install_options is None: + pip_install_options = [] + with tempfile.TemporaryDirectory() as temp_dir: + # This is more complicated than it needs to be for the sake + # of Windows. Use a temporary file rather than the command line + # to avoid quoting issues. Use a temporary directory rather + # than NamedTemporaryFile because with a NamedTemporaryFile on + # Windows, the subprocess can't open the file because this process + # has an exclusive lock on it. + req_file_name = os.path.join(temp_dir, 'requirements.txt') + with open(req_file_name, 'w') as req_file: + self.write(req_file) + subprocess.check_call([sys.executable, '-m', 'pip'] + + pip_general_options + + ['install'] + pip_install_options + + ['-r', req_file_name]) + +DEFAULT_REQUIREMENTS_FILE = 'ci.requirements.txt' + +def main() -> None: + """Command line entry point.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--no-act', '-n', + action='store_true', + help="Don't act, just print what will be done") + parser.add_argument('--pip-install-option', + action='append', dest='pip_install_options', + help="Pass this option to pip install") + parser.add_argument('--pip-option', + action='append', dest='pip_general_options', + help="Pass this general option to pip") + parser.add_argument('--user', + action='append_const', dest='pip_install_options', + const='--user', + help="Install to the Python user install directory" + " (short for --pip-install-option --user)") + parser.add_argument('files', nargs='*', metavar='FILE', + help="Requirement files" + " (default: {} in the script's directory)" \ + .format(DEFAULT_REQUIREMENTS_FILE)) + options = parser.parse_args() + if not options.files: + options.files = [os.path.join(os.path.dirname(__file__), + DEFAULT_REQUIREMENTS_FILE)] + reqs = Requirements() + for filename in options.files: + reqs.add_file(filename) + reqs.write(sys.stdout) + if not options.no_act: + reqs.install(pip_general_options=options.pip_general_options, + pip_install_options=options.pip_install_options) + +if __name__ == '__main__': + main() diff --git a/tests/docker/bionic/Dockerfile b/tests/docker/bionic/Dockerfile index 41789c677..50f5a7fba 100644 --- a/tests/docker/bionic/Dockerfile +++ b/tests/docker/bionic/Dockerfile @@ -160,7 +160,3 @@ RUN cd /tmp \ ENV GNUTLS_NEXT_CLI=/usr/local/gnutls-3.7.2/bin/gnutls-cli ENV GNUTLS_NEXT_SERV=/usr/local/gnutls-3.7.2/bin/gnutls-serv - -RUN pip3 install --no-cache-dir \ - mbed-host-tests \ - mock