Merge pull request #5197 from gilles-peskine-arm/pip-requirements

In-tree Python package requirements

.travis.yml

@@ -1,5 +1,7 @@
-language: c
-compiler: gcc
+# Declare python as our language. This way we get our chosen Python version,
+# and pip is available. Gcc and clang are available anyway.
+language: python
+python: 3.5
 sudo: false
 cache: ccache
 
@@ -16,10 +18,6 @@ jobs:
           - libnewlib-arm-none-eabi
           - gcc-arm-linux-gnueabi
           - libc6-dev-armel-cross
-      language: python # Needed to get pip for Python 3
-      python: 3.5 # version from Ubuntu 16.04
-      install:
-        - pip install mypy==0.780 pylint==2.4.4
       script:
         - tests/scripts/all.sh -k 'check_*'
         - tests/scripts/all.sh -k test_default_out_of_box
@@ -32,11 +30,16 @@ jobs:
 
     - name: Windows
       os: windows
+      # The language 'python' is currently unsupported on the
+      # Windows Build Environment. And 'generic' causes the job to get stuck
+      # on "Booting virtual machine".
+      language: c
       before_install:
         - choco install python --version=3.5.4
       env:
         # Add the directory where the Choco packages go
         - PATH=/c/Python35:/c/Python35/Scripts:$PATH
+        - PYTHON=python.exe
       script:
         - type perl; perl --version
         - type python; python --version
@@ -53,6 +56,9 @@ env:
     - SEED=1
     - secure: "FrI5d2s+ckckC17T66c8jm2jV6i2DkBPU5nyWzwbedjmEBeocREfQLd/x8yKpPzLDz7ghOvr+/GQvsPPn0dVkGlNzm3Q+hGHc/ujnASuUtGrcuMM+0ALnJ3k4rFr9xEvjJeWb4SmhJO5UCAZYvTItW4k7+bj9L+R6lt3TzQbXzg="
 
+install:
+  - $PYTHON scripts/min_requirements.py
+
 addons:
   apt:
     packages:

README.md

@@ -59,7 +59,10 @@ The source code of Mbed TLS includes some files that are automatically generated
 The following tools are required:
 
 * Perl, for some library source files and for Visual Studio build files.
-* Python 3, for some sample programs and test data.
+* Python 3 and some Python packages, for some library source files, sample programs and test data. To install the necessary packages, run
+    ```
+    python -m pip install -r scripts/basic.requirements.txt
+    ```
 * A C compiler for the host platform, for some test data.
 
 If you are cross-compiling, you must set the `CC` environment variable to a C compiler for the host platform when generating the configuration-independent files.

scripts/basic.requirements.txt

@@ -0,0 +1,5 @@
+# Python modules required to build Mbed TLS in ordinary conditions.
+
+# Required to (re-)generate source files. Not needed if the generated source
+# files are already present and up-to-date.
+-r driver.requirements.txt

scripts/ci.requirements.txt

@@ -0,0 +1,12 @@
+# Python package requirements for Mbed TLS testing.
+
+-r driver.requirements.txt
+
+# Use a known version of Pylint, because new versions tend to add warnings
+# that could start rejecting our code.
+# 2.4.4 is the version in Ubuntu 20.04. It supports Python >=3.5.
+pylint == 2.4.4
+
+# Use the earliest version of mypy that works with our code base.
+# See https://github.com/ARMmbed/mbedtls/pull/3953 .
+mypy >= 0.780

scripts/driver.requirements.txt

@@ -0,0 +1,10 @@
+# Python package requirements for driver implementers.
+
+# Use the version of Jinja that's in Ubuntu 20.04.
+# See https://github.com/ARMmbed/mbedtls/pull/5067#discussion_r738794607 .
+# Note that Jinja 3.0 drops support for Python 3.5, so we need to support
+# Jinja 2.x as long as we're still using Python 3.5 anywhere.
+Jinja2 >= 2.10.1
+# Jinja2 >=2.10, <<3.0 needs a separate package for type annotations
+types-Jinja2
+

scripts/maintainer.requirements.txt

@@ -0,0 +1,10 @@
+# Python packages that are only useful to Mbed TLS maintainers.
+
+-r ci.requirements.txt
+
+# For source code analyses
+clang
+
+# For building some test vectors
+pycryptodomex
+pycryptodome-test-vectors

scripts/min_requirements.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Install all the required Python packages, with the minimum Python version.
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import typing
+
+from typing import List, Optional
+from mbedtls_dev import typing_util
+
+def pylint_doesn_t_notice_that_certain_types_are_used_in_annotations(
+        _list: List[typing.Any],
+) -> None:
+    pass
+
+
+class Requirements:
+    """Collect and massage Python requirements."""
+
+    def __init__(self) -> None:
+        self.requirements = [] #type: List[str]
+
+    def adjust_requirement(self, req: str) -> str:
+        """Adjust a requirement to the minimum specified version."""
+        # allow inheritance #pylint: disable=no-self-use
+        # If a requirement specifies a minimum version, impose that version.
+        req = re.sub(r'>=|~=', r'==', req)
+        return req
+
+    def add_file(self, filename: str) -> None:
+        """Add requirements from the specified file.
+
+        This method supports a subset of pip's requirement file syntax:
+        * One requirement specifier per line, which is passed to
+          `adjust_requirement`.
+        * Comments (``#`` at the beginning of the line or after whitespace).
+        * ``-r FILENAME`` to include another file.
+        """
+        for line in open(filename):
+            line = line.strip()
+            line = re.sub(r'(\A|\s+)#.*', r'', line)
+            if not line:
+                continue
+            m = re.match(r'-r\s+', line)
+            if m:
+                nested_file = os.path.join(os.path.dirname(filename),
+                                           line[m.end(0):])
+                self.add_file(nested_file)
+                continue
+            self.requirements.append(self.adjust_requirement(line))
+
+    def write(self, out: typing_util.Writable) -> None:
+        """List the gathered requirements."""
+        for req in self.requirements:
+            out.write(req + '\n')
+
+    def install(
+            self,
+            pip_general_options: Optional[List[str]] = None,
+            pip_install_options: Optional[List[str]] = None,
+    ) -> None:
+        """Call pip to install the requirements."""
+        if pip_general_options is None:
+            pip_general_options = []
+        if pip_install_options is None:
+            pip_install_options = []
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # This is more complicated than it needs to be for the sake
+            # of Windows. Use a temporary file rather than the command line
+            # to avoid quoting issues. Use a temporary directory rather
+            # than NamedTemporaryFile because with a NamedTemporaryFile on
+            # Windows, the subprocess can't open the file because this process
+            # has an exclusive lock on it.
+            req_file_name = os.path.join(temp_dir, 'requirements.txt')
+            with open(req_file_name, 'w') as req_file:
+                self.write(req_file)
+            subprocess.check_call([sys.executable, '-m', 'pip'] +
+                                  pip_general_options +
+                                  ['install'] + pip_install_options +
+                                  ['-r', req_file_name])
+
+DEFAULT_REQUIREMENTS_FILE = 'ci.requirements.txt'
+
+def main() -> None:
+    """Command line entry point."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--no-act', '-n',
+                        action='store_true',
+                        help="Don't act, just print what will be done")
+    parser.add_argument('--pip-install-option',
+                        action='append', dest='pip_install_options',
+                        help="Pass this option to pip install")
+    parser.add_argument('--pip-option',
+                        action='append', dest='pip_general_options',
+                        help="Pass this general option to pip")
+    parser.add_argument('--user',
+                        action='append_const', dest='pip_install_options',
+                        const='--user',
+                        help="Install to the Python user install directory"
+                             " (short for --pip-install-option --user)")
+    parser.add_argument('files', nargs='*', metavar='FILE',
+                        help="Requirement files"
+                             " (default: {} in the script's directory)" \
+                             .format(DEFAULT_REQUIREMENTS_FILE))
+    options = parser.parse_args()
+    if not options.files:
+        options.files = [os.path.join(os.path.dirname(__file__),
+                                      DEFAULT_REQUIREMENTS_FILE)]
+    reqs = Requirements()
+    for filename in options.files:
+        reqs.add_file(filename)
+    reqs.write(sys.stdout)
+    if not options.no_act:
+        reqs.install(pip_general_options=options.pip_general_options,
+                     pip_install_options=options.pip_install_options)
+
+if __name__ == '__main__':
+    main()

tests/docker/bionic/Dockerfile

@@ -160,7 +160,3 @@ RUN cd /tmp \
 
 ENV GNUTLS_NEXT_CLI=/usr/local/gnutls-3.7.2/bin/gnutls-cli
 ENV GNUTLS_NEXT_SERV=/usr/local/gnutls-3.7.2/bin/gnutls-serv
-
-RUN pip3 install --no-cache-dir \
-    mbed-host-tests \
-    mock