From 12fe74e3609d20a85556ddf236562c501c1082c4 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 17 Nov 2021 19:13:34 +0100 Subject: [PATCH 1/9] Declare which Python packages we use Add pip requirements files. We'll have separate requirements files for different target audiences. Each file can use `-r` lines to include other files. This commit adds two requirement files: one with everything that's needed to pass the CI, and one with additional tools that are suggested for Mbed TLS maintainers to install locally. Signed-off-by: Gilles Peskine --- scripts/ci.requirements.txt | 10 ++++++++++ scripts/maintainer.requirements.txt | 6 ++++++ 2 files changed, 16 insertions(+) create mode 100644 scripts/ci.requirements.txt create mode 100644 scripts/maintainer.requirements.txt diff --git a/scripts/ci.requirements.txt b/scripts/ci.requirements.txt new file mode 100644 index 000000000..18b40ec17 --- /dev/null +++ b/scripts/ci.requirements.txt @@ -0,0 +1,10 @@ +# Python package requirements for Mbed TLS testing. + +# Use a known version of Pylint, because new versions tend to add warnings +# that could start rejecting our code. +# 2.4.4 is the version in Ubuntu 20.04. It supports Python >=3.5. +pylint == 2.4.4 + +# Use the earliest version of mypy that works with our code base. +# See https://github.com/ARMmbed/mbedtls/pull/3953 . +mypy >= 0.780 diff --git a/scripts/maintainer.requirements.txt b/scripts/maintainer.requirements.txt new file mode 100644 index 000000000..670617ba7 --- /dev/null +++ b/scripts/maintainer.requirements.txt @@ -0,0 +1,6 @@ +# Python packages that are only useful to Mbed TLS maintainers. + +-r ci.requirements.txt + +# For source code analyses +clang From e4d142f1e7aab9fab6bd18687e3137d1d222abfe Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 17 Nov 2021 19:25:43 +0100 Subject: [PATCH 2/9] Script to install minimum versions of the requirements Wherever we have a requirement on foo>=N, install foo==N. This is for testing, to ensure that we don't accidentally depend on features that are not present in the minimum version we declare support for. Signed-off-by: Gilles Peskine --- scripts/min_requirements.py | 106 ++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100755 scripts/min_requirements.py diff --git a/scripts/min_requirements.py b/scripts/min_requirements.py new file mode 100755 index 000000000..3b156a36a --- /dev/null +++ b/scripts/min_requirements.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Install all the required Python packages, with the minimum Python version. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import re +import sys +import typing + +from typing import List +from mbedtls_dev import typing_util + +def pylint_doesn_t_notice_that_certain_types_are_used_in_annotations( + _list: List[typing.Any], +) -> None: + pass + + +class Requirements: + """Collect and massage Python requirements.""" + + def __init__(self) -> None: + self.requirements = [] #type: List[str] + + def adjust_requirement(self, req: str) -> str: + """Adjust a requirement to the minimum specified version.""" + # allow inheritance #pylint: disable=no-self-use + # If a requirement specifies a minimum version, impose that version. + req = re.sub(r'>=|~=', r'==', req) + return req + + def add_file(self, filename: str) -> None: + """Add requirements from the specified file. + + This method supports a subset of pip's requirement file syntax: + * One requirement specifier per line, which is passed to + `adjust_requirement`. + * Comments (``#`` at the beginning of the line or after whitespace). + * ``-r FILENAME`` to include another file. + """ + for line in open(filename): + line = line.strip() + line = re.sub(r'(\A|\s+)#.*', r'', line) + if not line: + continue + m = re.match(r'-r\s+', line) + if m: + nested_file = os.path.join(os.path.dirname(filename), + line[m.end(0):]) + self.add_file(nested_file) + continue + self.requirements.append(self.adjust_requirement(line)) + + def write(self, out: typing_util.Writable) -> None: + """List the gathered requirements.""" + for req in self.requirements: + out.write(req + '\n') + + def install(self) -> None: + """Call pip to install the requirements.""" + if not self.requirements: + return + ret = os.spawnl(os.P_WAIT, sys.executable, 'python', '-m', 'pip', + 'install', *self.requirements) + if ret != 0: + sys.exit(ret) + + +def main() -> None: + """Command line entry point.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--no-act', '-n', + action='store_true', + help="Don't act, just print what will be done") + parser.add_argument('files', nargs='*', metavar='FILE', + help="Requirement files" + "(default: requirements.txt in the script's directory)") + options = parser.parse_args() + if not options.files: + options.files = [os.path.join(os.path.dirname(__file__), + 'ci.requirements.txt')] + reqs = Requirements() + for filename in options.files: + reqs.add_file(filename) + reqs.write(sys.stdout) + if not options.no_act: + reqs.install() + +if __name__ == '__main__': + main() From ce8ccaf55bc2ffeb84252addf5e9637580291bdd Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 17 Nov 2021 19:48:21 +0100 Subject: [PATCH 3/9] Docker: Python requirements are now managed in-tree Neither mbed-host-tests nor mock are currently used. Signed-off-by: Gilles Peskine --- tests/docker/bionic/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/docker/bionic/Dockerfile b/tests/docker/bionic/Dockerfile index 1d24aa326..69662f40f 100644 --- a/tests/docker/bionic/Dockerfile +++ b/tests/docker/bionic/Dockerfile @@ -161,6 +161,4 @@ RUN cd /tmp \ ENV GNUTLS_NEXT_CLI=/usr/local/gnutls-3.6.5/bin/gnutls-cli ENV GNUTLS_NEXT_SERV=/usr/local/gnutls-3.6.5/bin/gnutls-serv -RUN pip3 install --no-cache-dir \ - mbed-host-tests \ - mock +RUN scripts/min_requirements.py From 6d253cc4fc1c8467076a12173cfa4bd7d632dae5 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 17 Nov 2021 19:29:38 +0100 Subject: [PATCH 4/9] Travis: use the in-tree Python package requirements Signed-off-by: Gilles Peskine --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 56e145649..33546073b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ jobs: language: python # Needed to get pip for Python 3 python: 3.5 # version from Ubuntu 16.04 install: - - pip install mypy==0.780 pylint==2.4.4 + - scripts/min_requirements.py script: - tests/scripts/all.sh -k 'check_*' - tests/scripts/all.sh -k test_default_out_of_box From c31780f62f8d0369dc4294a5ff1d815c24f8aa82 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 18 Nov 2021 18:18:35 +0100 Subject: [PATCH 5/9] Use a method to invoke pip that works on Windows Passing arguments on the command line apparently didn't work due to quoting issues. Use a temporary file instead. Signed-off-by: Gilles Peskine --- scripts/min_requirements.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/scripts/min_requirements.py b/scripts/min_requirements.py index 3b156a36a..ef6c42bf6 100755 --- a/scripts/min_requirements.py +++ b/scripts/min_requirements.py @@ -20,7 +20,9 @@ import argparse import os import re +import subprocess import sys +import tempfile import typing from typing import List @@ -74,12 +76,18 @@ class Requirements: def install(self) -> None: """Call pip to install the requirements.""" - if not self.requirements: - return - ret = os.spawnl(os.P_WAIT, sys.executable, 'python', '-m', 'pip', - 'install', *self.requirements) - if ret != 0: - sys.exit(ret) + with tempfile.TemporaryDirectory() as temp_dir: + # This is more complicated than it needs to be for the sake + # of Windows. Use a temporary file rather than the command line + # to avoid quoting issues. Use a temporary directory rather + # than NamedTemporaryFile because with a NamedTemporaryFile on + # Windows, the subprocess can't open the file because this process + # has an exclusive lock on it. + req_file_name = os.path.join(temp_dir, 'requirements.txt') + with open(req_file_name, 'w') as req_file: + self.write(req_file) + subprocess.check_call([sys.executable, '-m', 'pip', + 'install', '-r', req_file_name]) def main() -> None: From ca07ea08020eb59ab3a9e8495aabd48349352aae Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 22 Nov 2021 12:52:24 +0000 Subject: [PATCH 6/9] Allow passing options to pip Signed-off-by: Gilles Peskine --- scripts/min_requirements.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/scripts/min_requirements.py b/scripts/min_requirements.py index ef6c42bf6..db301e891 100755 --- a/scripts/min_requirements.py +++ b/scripts/min_requirements.py @@ -25,7 +25,7 @@ import sys import tempfile import typing -from typing import List +from typing import List, Optional from mbedtls_dev import typing_util def pylint_doesn_t_notice_that_certain_types_are_used_in_annotations( @@ -74,8 +74,16 @@ class Requirements: for req in self.requirements: out.write(req + '\n') - def install(self) -> None: + def install( + self, + pip_general_options: Optional[List[str]] = None, + pip_install_options: Optional[List[str]] = None, + ) -> None: """Call pip to install the requirements.""" + if pip_general_options is None: + pip_general_options = [] + if pip_install_options is None: + pip_install_options = [] with tempfile.TemporaryDirectory() as temp_dir: # This is more complicated than it needs to be for the sake # of Windows. Use a temporary file rather than the command line @@ -86,8 +94,10 @@ class Requirements: req_file_name = os.path.join(temp_dir, 'requirements.txt') with open(req_file_name, 'w') as req_file: self.write(req_file) - subprocess.check_call([sys.executable, '-m', 'pip', - 'install', '-r', req_file_name]) + subprocess.check_call([sys.executable, '-m', 'pip'] + + pip_general_options + + ['install'] + pip_install_options + + ['-r', req_file_name]) def main() -> None: @@ -96,9 +106,20 @@ def main() -> None: parser.add_argument('--no-act', '-n', action='store_true', help="Don't act, just print what will be done") + parser.add_argument('--pip-install-option', + action='append', dest='pip_install_options', + help="Pass this option to pip install") + parser.add_argument('--pip-option', + action='append', dest='pip_general_options', + help="Pass this general option to pip") + parser.add_argument('--user', + action='append_const', dest='pip_install_options', + const='--user', + help="Install to the Python user install directory" + " (short for --pip-install-option --user)") parser.add_argument('files', nargs='*', metavar='FILE', help="Requirement files" - "(default: requirements.txt in the script's directory)") + " (default: requirements.txt in the script's directory)") options = parser.parse_args() if not options.files: options.files = [os.path.join(os.path.dirname(__file__), @@ -108,7 +129,8 @@ def main() -> None: reqs.add_file(filename) reqs.write(sys.stdout) if not options.no_act: - reqs.install() + reqs.install(pip_general_options=options.pip_general_options, + pip_install_options=options.pip_install_options) if __name__ == '__main__': main() From 3f5f7df75ba3e224f1ff2595d94fe094dac7615d Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 2 Dec 2021 12:44:50 +0100 Subject: [PATCH 7/9] Remove accidental requirement on the worktree content This made the build impossible since mbedtls isn't available when building the container. Signed-off-by: Gilles Peskine --- tests/docker/bionic/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/docker/bionic/Dockerfile b/tests/docker/bionic/Dockerfile index 69662f40f..3132be9ca 100644 --- a/tests/docker/bionic/Dockerfile +++ b/tests/docker/bionic/Dockerfile @@ -160,5 +160,3 @@ RUN cd /tmp \ ENV GNUTLS_NEXT_CLI=/usr/local/gnutls-3.6.5/bin/gnutls-cli ENV GNUTLS_NEXT_SERV=/usr/local/gnutls-3.6.5/bin/gnutls-serv - -RUN scripts/min_requirements.py From f3564bfe990b5f2934fab9c61ba9106b7e4453dd Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 2 Dec 2021 12:50:06 +0100 Subject: [PATCH 8/9] Add Cryptodome to maintainer requirements See e.g. https://github.com/ARMmbed/mbedtls/pull/5218 Signed-off-by: Gilles Peskine --- scripts/maintainer.requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/maintainer.requirements.txt b/scripts/maintainer.requirements.txt index 670617ba7..b149921a2 100644 --- a/scripts/maintainer.requirements.txt +++ b/scripts/maintainer.requirements.txt @@ -4,3 +4,7 @@ # For source code analyses clang + +# For building some test vectors +pycryptodomex +pycryptodome-test-vectors From 4b71e9b96a498f2e1effaecf3b7cbfbc7f512ce7 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 3 Dec 2021 13:32:10 +0100 Subject: [PATCH 9/9] Correct default requirements file name in help Signed-off-by: Gilles Peskine --- scripts/min_requirements.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/min_requirements.py b/scripts/min_requirements.py index db301e891..eecab1c1e 100755 --- a/scripts/min_requirements.py +++ b/scripts/min_requirements.py @@ -99,6 +99,7 @@ class Requirements: ['install'] + pip_install_options + ['-r', req_file_name]) +DEFAULT_REQUIREMENTS_FILE = 'ci.requirements.txt' def main() -> None: """Command line entry point.""" @@ -119,11 +120,12 @@ def main() -> None: " (short for --pip-install-option --user)") parser.add_argument('files', nargs='*', metavar='FILE', help="Requirement files" - " (default: requirements.txt in the script's directory)") + " (default: {} in the script's directory)" \ + .format(DEFAULT_REQUIREMENTS_FILE)) options = parser.parse_args() if not options.files: options.files = [os.path.join(os.path.dirname(__file__), - 'ci.requirements.txt')] + DEFAULT_REQUIREMENTS_FILE)] reqs = Requirements() for filename in options.files: reqs.add_file(filename)