Merge pull request #5197 from gilles-peskine-arm/pip-requirements
In-tree Python package requirements
diff --git a/.travis.yml b/.travis.yml
index 39ae19c..cdb68ac 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,7 @@
-language: c
-compiler: gcc
+# Declare python as our language. This way we get our chosen Python version,
+# and pip is available. Gcc and clang are available anyway.
+language: python
+python: 3.5
sudo: false
cache: ccache
@@ -16,10 +18,6 @@
- libnewlib-arm-none-eabi
- gcc-arm-linux-gnueabi
- libc6-dev-armel-cross
- language: python # Needed to get pip for Python 3
- python: 3.5 # version from Ubuntu 16.04
- install:
- - pip install mypy==0.780 pylint==2.4.4
script:
- tests/scripts/all.sh -k 'check_*'
- tests/scripts/all.sh -k test_default_out_of_box
@@ -32,11 +30,16 @@
- name: Windows
os: windows
+ # The language 'python' is currently unsupported on the
+ # Windows Build Environment. And 'generic' causes the job to get stuck
+ # on "Booting virtual machine".
+ language: c
before_install:
- choco install python --version=3.5.4
env:
# Add the directory where the Choco packages go
- PATH=/c/Python35:/c/Python35/Scripts:$PATH
+ - PYTHON=python.exe
script:
- type perl; perl --version
- type python; python --version
@@ -53,6 +56,9 @@
- SEED=1
- secure: "FrI5d2s+ckckC17T66c8jm2jV6i2DkBPU5nyWzwbedjmEBeocREfQLd/x8yKpPzLDz7ghOvr+/GQvsPPn0dVkGlNzm3Q+hGHc/ujnASuUtGrcuMM+0ALnJ3k4rFr9xEvjJeWb4SmhJO5UCAZYvTItW4k7+bj9L+R6lt3TzQbXzg="
+install:
+ - $PYTHON scripts/min_requirements.py
+
addons:
apt:
packages:
diff --git a/README.md b/README.md
index c8d9450..ea1d7a3 100644
--- a/README.md
+++ b/README.md
@@ -59,7 +59,10 @@
The following tools are required:
* Perl, for some library source files and for Visual Studio build files.
-* Python 3, for some sample programs and test data.
+* Python 3 and some Python packages, for some library source files, sample programs and test data. To install the necessary packages, run
+ ```
+ python -m pip install -r scripts/basic.requirements.txt
+ ```
* A C compiler for the host platform, for some test data.
If you are cross-compiling, you must set the `CC` environment variable to a C compiler for the host platform when generating the configuration-independent files.
diff --git a/scripts/basic.requirements.txt b/scripts/basic.requirements.txt
new file mode 100644
index 0000000..1be3d0c
--- /dev/null
+++ b/scripts/basic.requirements.txt
@@ -0,0 +1,5 @@
+# Python modules required to build Mbed TLS in ordinary conditions.
+
+# Required to (re-)generate source files. Not needed if the generated source
+# files are already present and up-to-date.
+-r driver.requirements.txt
diff --git a/scripts/ci.requirements.txt b/scripts/ci.requirements.txt
new file mode 100644
index 0000000..209ae3d
--- /dev/null
+++ b/scripts/ci.requirements.txt
@@ -0,0 +1,12 @@
+# Python package requirements for Mbed TLS testing.
+
+-r driver.requirements.txt
+
+# Use a known version of Pylint, because new versions tend to add warnings
+# that could start rejecting our code.
+# 2.4.4 is the version in Ubuntu 20.04. It supports Python >=3.5.
+pylint == 2.4.4
+
+# Use the earliest version of mypy that works with our code base.
+# See https://github.com/ARMmbed/mbedtls/pull/3953 .
+mypy >= 0.780
diff --git a/scripts/driver.requirements.txt b/scripts/driver.requirements.txt
new file mode 100644
index 0000000..17569bb
--- /dev/null
+++ b/scripts/driver.requirements.txt
@@ -0,0 +1,10 @@
+# Python package requirements for driver implementers.
+
+# Use the version of Jinja that's in Ubuntu 20.04.
+# See https://github.com/ARMmbed/mbedtls/pull/5067#discussion_r738794607 .
+# Note that Jinja 3.0 drops support for Python 3.5, so we need to support
+# Jinja 2.x as long as we're still using Python 3.5 anywhere.
+Jinja2 >= 2.10.1
+# Jinja2 >=2.10, <<3.0 needs a separate package for type annotations
+types-Jinja2
+
diff --git a/scripts/maintainer.requirements.txt b/scripts/maintainer.requirements.txt
new file mode 100644
index 0000000..b149921
--- /dev/null
+++ b/scripts/maintainer.requirements.txt
@@ -0,0 +1,10 @@
+# Python packages that are only useful to Mbed TLS maintainers.
+
+-r ci.requirements.txt
+
+# For source code analyses
+clang
+
+# For building some test vectors
+pycryptodomex
+pycryptodome-test-vectors
diff --git a/scripts/min_requirements.py b/scripts/min_requirements.py
new file mode 100755
index 0000000..eecab1c
--- /dev/null
+++ b/scripts/min_requirements.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Install all the required Python packages, with the minimum Python version.
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import typing
+
+from typing import List, Optional
+from mbedtls_dev import typing_util
+
+def pylint_doesn_t_notice_that_certain_types_are_used_in_annotations(
+ _list: List[typing.Any],
+) -> None:
+ pass
+
+
+class Requirements:
+ """Collect and massage Python requirements."""
+
+ def __init__(self) -> None:
+ self.requirements = [] #type: List[str]
+
+ def adjust_requirement(self, req: str) -> str:
+ """Adjust a requirement to the minimum specified version."""
+ # allow inheritance #pylint: disable=no-self-use
+ # If a requirement specifies a minimum version, impose that version.
+ req = re.sub(r'>=|~=', r'==', req)
+ return req
+
+ def add_file(self, filename: str) -> None:
+ """Add requirements from the specified file.
+
+ This method supports a subset of pip's requirement file syntax:
+ * One requirement specifier per line, which is passed to
+ `adjust_requirement`.
+ * Comments (``#`` at the beginning of the line or after whitespace).
+ * ``-r FILENAME`` to include another file.
+ """
+ for line in open(filename):
+ line = line.strip()
+ line = re.sub(r'(\A|\s+)#.*', r'', line)
+ if not line:
+ continue
+ m = re.match(r'-r\s+', line)
+ if m:
+ nested_file = os.path.join(os.path.dirname(filename),
+ line[m.end(0):])
+ self.add_file(nested_file)
+ continue
+ self.requirements.append(self.adjust_requirement(line))
+
+ def write(self, out: typing_util.Writable) -> None:
+ """List the gathered requirements."""
+ for req in self.requirements:
+ out.write(req + '\n')
+
+ def install(
+ self,
+ pip_general_options: Optional[List[str]] = None,
+ pip_install_options: Optional[List[str]] = None,
+ ) -> None:
+ """Call pip to install the requirements."""
+ if pip_general_options is None:
+ pip_general_options = []
+ if pip_install_options is None:
+ pip_install_options = []
+ with tempfile.TemporaryDirectory() as temp_dir:
+ # This is more complicated than it needs to be for the sake
+ # of Windows. Use a temporary file rather than the command line
+ # to avoid quoting issues. Use a temporary directory rather
+ # than NamedTemporaryFile because with a NamedTemporaryFile on
+ # Windows, the subprocess can't open the file because this process
+ # has an exclusive lock on it.
+ req_file_name = os.path.join(temp_dir, 'requirements.txt')
+ with open(req_file_name, 'w') as req_file:
+ self.write(req_file)
+ subprocess.check_call([sys.executable, '-m', 'pip'] +
+ pip_general_options +
+ ['install'] + pip_install_options +
+ ['-r', req_file_name])
+
+DEFAULT_REQUIREMENTS_FILE = 'ci.requirements.txt'
+
+def main() -> None:
+ """Command line entry point."""
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--no-act', '-n',
+ action='store_true',
+ help="Don't act, just print what will be done")
+ parser.add_argument('--pip-install-option',
+ action='append', dest='pip_install_options',
+ help="Pass this option to pip install")
+ parser.add_argument('--pip-option',
+ action='append', dest='pip_general_options',
+ help="Pass this general option to pip")
+ parser.add_argument('--user',
+ action='append_const', dest='pip_install_options',
+ const='--user',
+ help="Install to the Python user install directory"
+ " (short for --pip-install-option --user)")
+ parser.add_argument('files', nargs='*', metavar='FILE',
+ help="Requirement files"
+ " (default: {} in the script's directory)" \
+ .format(DEFAULT_REQUIREMENTS_FILE))
+ options = parser.parse_args()
+ if not options.files:
+ options.files = [os.path.join(os.path.dirname(__file__),
+ DEFAULT_REQUIREMENTS_FILE)]
+ reqs = Requirements()
+ for filename in options.files:
+ reqs.add_file(filename)
+ reqs.write(sys.stdout)
+ if not options.no_act:
+ reqs.install(pip_general_options=options.pip_general_options,
+ pip_install_options=options.pip_install_options)
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/docker/bionic/Dockerfile b/tests/docker/bionic/Dockerfile
index 41789c6..50f5a7f 100644
--- a/tests/docker/bionic/Dockerfile
+++ b/tests/docker/bionic/Dockerfile
@@ -160,7 +160,3 @@
ENV GNUTLS_NEXT_CLI=/usr/local/gnutls-3.7.2/bin/gnutls-cli
ENV GNUTLS_NEXT_SERV=/usr/local/gnutls-3.7.2/bin/gnutls-serv
-
-RUN pip3 install --no-cache-dir \
- mbed-host-tests \
- mock