From 0abf04d6c280904812f60c9b74c5171849dcd92f Mon Sep 17 00:00:00 2001 From: Shaun Verch Date: Mon, 10 Jan 2022 16:43:56 -0500 Subject: [PATCH] Remove requirements.txt as a dependency (#1111) * Remove requirements.txt as a dependency This converts both docker and tox to use poetry, eliminating usage of requirements.txt in both flows. - In tox, uses the tox-poetry package which installs dependencies from the lockfile. - In docker, uses https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker as a reference. * Don't copy pyproject.toml * Remove obsoleted docs about requirements.txt * Add --full-trace option to pytest * Fix liccheck liccheck works with requirements.txt, not with poetry, so there needs to be an extra translation step. * TEMP: Add WIP fix for pandas issue This is just to see if the github actions would pass once this fix gets merged, but it's being reviewed separately. * Revert "TEMP: Add WIP fix for pandas issue" This reverts commit 06e38e8cc77f5f3105c6e7a9449901db67aa1c82. --- data/data-pipeline/Dockerfile | 26 +++-- data/data-pipeline/README.md | 4 - data/data-pipeline/poetry.lock | 22 ++++- data/data-pipeline/pyproject.toml | 1 + data/data-pipeline/requirements.txt | 109 --------------------- data/data-pipeline/scripts/run-liccheck.sh | 15 +++ data/data-pipeline/tox.ini | 23 +---- 7 files changed, 60 insertions(+), 140 deletions(-) delete mode 100644 data/data-pipeline/requirements.txt create mode 100644 data/data-pipeline/scripts/run-liccheck.sh diff --git a/data/data-pipeline/Dockerfile b/data/data-pipeline/Dockerfile index 980c03b0..e01c6330 100644 --- a/data/data-pipeline/Dockerfile +++ b/data/data-pipeline/Dockerfile @@ -24,13 +24,25 @@ RUN /bin/sh -c make && make install RUN add-apt-repository ppa:ubuntugis/ppa RUN apt-get -y install gdal-bin -# Prepare python packages -WORKDIR /data-pipeline -RUN pip3 install --upgrade pip setuptools wheel -COPY . . +# Python package installation using poetry. See: +# https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker +ENV PYTHONFAULTHANDLER=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONHASHSEED=random \ + PIP_NO_CACHE_DIR=off \ + PIP_DISABLE_PIP_VERSION_CHECK=on \ + PIP_DEFAULT_TIMEOUT=100 \ + POETRY_VERSION=1.1.12 -COPY requirements.txt . -RUN pip3 install -r requirements.txt -RUN pip3 install . +WORKDIR /data-pipeline +COPY poetry.lock /data-pipeline/ + +RUN pip install "poetry==$POETRY_VERSION" +RUN poetry config virtualenvs.create false \ + && poetry config virtualenvs.in-project false \ + && poetry install --no-dev --no-interaction --no-ansi + +# Copy all project files into the container +COPY . /data-pipeline CMD python3 -m data_pipeline.application data-full-run --check -s aws diff --git a/data/data-pipeline/README.md b/data/data-pipeline/README.md index adf90745..4f730f51 100644 --- a/data/data-pipeline/README.md +++ b/data/data-pipeline/README.md @@ -279,10 +279,6 @@ After installing the poetry dependencies, you can see a list of commands with th For more information, see [nbextensions docs](https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/install.html) and see [python-markdown docs](https://github.com/ipython-contrib/jupyter_contrib_nbextensions/tree/master/src/jupyter_contrib_nbextensions/nbextensions/python-markdown). -## Miscellaneous - -- To export packages from Poetry to `requirements.txt` run `poetry export --without-hashes > requirements.txt` - ## Testing ### Background diff --git a/data/data-pipeline/poetry.lock b/data/data-pipeline/poetry.lock index 1e9902f9..5b5f0988 100644 --- a/data/data-pipeline/poetry.lock +++ b/data/data-pipeline/poetry.lock @@ -1531,6 +1531,22 @@ virtualenv = ">=16.0.0,<20.0.0 || >20.0.0,<20.0.1 || >20.0.1,<20.0.2 || >20.0.2, docs = ["pygments-github-lexers (>=0.0.5)", "sphinx (>=2.0.0)", "sphinxcontrib-autoprogram (>=0.1.5)", "towncrier (>=18.5.0)"] testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "psutil (>=5.6.1)", "pytest (>=4.0.0)", "pytest-cov (>=2.5.1)", "pytest-mock (>=1.10.0)", "pytest-randomly (>=1.0.0)", "pytest-xdist (>=1.22.2)", "pathlib2 (>=2.3.3)"] +[[package]] +name = "tox-poetry" +version = "0.4.1" +description = "Tox poetry plugin" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +pluggy = "*" +toml = "*" +tox = {version = ">=3.7.0", markers = "python_version >= \"3\""} + +[package.extras] +test = ["coverage", "pytest", "pycodestyle", "pylint"] + [[package]] name = "tqdm" version = "4.62.0" @@ -1684,7 +1700,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "1823efb9fae610a00eb714ec61fd165ec5cda3d38fa83c12b00e9d38f4358fa3" +content-hash = "f832db8cb99fd9f868d6e03c7536b7679bfdc0228693ba796ae2a4028b641a28" [metadata.files] appnope = [ @@ -2782,6 +2798,10 @@ tox = [ {file = "tox-3.24.4-py2.py3-none-any.whl", hash = "sha256:5e274227a53dc9ef856767c21867377ba395992549f02ce55eb549f9fb9a8d10"}, {file = "tox-3.24.4.tar.gz", hash = "sha256:c30b57fa2477f1fb7c36aa1d83292d5c2336cd0018119e1b1c17340e2c2708ca"}, ] +tox-poetry = [ + {file = "tox-poetry-0.4.1.tar.gz", hash = "sha256:2395808e1ce487b5894c10f2202e14702bfa6d6909c0d1e525170d14809ac7ef"}, + {file = "tox_poetry-0.4.1-py2.py3-none-any.whl", hash = "sha256:11d9cd4e51d4cd9484b3ba63f2650ab4cfb4096e5f0682ecf561ddfc3c8e8c92"}, +] tqdm = [ {file = "tqdm-4.62.0-py2.py3-none-any.whl", hash = "sha256:706dea48ee05ba16e936ee91cb3791cd2ea6da348a0e50b46863ff4363ff4340"}, {file = "tqdm-4.62.0.tar.gz", hash = "sha256:3642d483b558eec80d3c831e23953582c34d7e4540db86d9e5ed9dad238dabc6"}, diff --git a/data/data-pipeline/pyproject.toml b/data/data-pipeline/pyproject.toml index 18a45ede..490ab806 100644 --- a/data/data-pipeline/pyproject.toml +++ b/data/data-pipeline/pyproject.toml @@ -36,6 +36,7 @@ pytest = "^6.2.4" safety = "^1.10.3" tox = "^3.24.0" pytest-mock = "^3.6.1" +tox-poetry = "^0.4.1" [build-system] build-backend = "poetry.core.masonry.api" diff --git a/data/data-pipeline/requirements.txt b/data/data-pipeline/requirements.txt deleted file mode 100644 index 3eacb5cf..00000000 --- a/data/data-pipeline/requirements.txt +++ /dev/null @@ -1,109 +0,0 @@ -appnope==0.1.2; sys_platform == "darwin" and python_version >= "3.7" and platform_system == "Darwin" -argcomplete==1.12.3; python_version < "3.8.0" and python_version >= "3.7" -argon2-cffi==21.1.0; python_version >= "3.6" -astroid==2.8.3; python_version >= "3.6" and python_version < "4.0" -attrs==21.2.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7" -backcall==0.2.0; python_version >= "3.7" -bleach==4.1.0; python_version >= "3.7" -censusdata==1.15; python_version >= "2.7" -certifi==2021.10.8; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7" -cffi==1.15.0; implementation_name == "pypy" and python_version >= "3.7" and python_full_version >= "3.6.1" -charset-normalizer==2.0.7; python_full_version >= "3.6.0" and python_version >= "3" -click-plugins==1.1.1; python_version >= "3.6" -click==8.0.3; python_version >= "3.6" -cligj==0.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version < "4" and python_version >= "3.6" -colorama==0.4.4; python_version >= "3.7" and python_full_version < "3.0.0" and platform_system == "Windows" and sys_platform == "win32" and python_version < "4.0" or platform_system == "Windows" and python_version >= "3.7" and python_full_version >= "3.5.0" and sys_platform == "win32" and python_version < "4.0" -cycler==0.10.0; python_version >= "3.7" -debugpy==1.5.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7" -decorator==5.1.0; python_version >= "3.7" -defusedxml==0.7.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7" -dynaconf==3.1.7; python_version >= "3.7" -entrypoints==0.3; python_full_version >= "3.6.1" and python_version >= "3.7" -fiona==1.8.20; python_version >= "3.6" -geopandas==0.9.0; python_version >= "3.6" -idna==3.3; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5" -importlib-metadata==4.8.1; python_version == "3.7" -ipdb==0.13.9; python_version >= "2.7" -ipykernel==6.4.2; python_version >= "3.7" -ipython-genutils==0.2.0; python_version >= "3.7" -ipython==7.28.0; python_version >= "3.7" -ipywidgets==7.6.5 -isort==5.9.3; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6" -jedi==0.18.0; python_version >= "3.7" -jellyfish==0.6.1 -jinja2==3.0.2; python_version >= "3.7" -jsonschema==4.1.2; python_version >= "3.7" -jupyter-client==7.0.6; python_full_version >= "3.6.1" and python_version >= "3.7" -jupyter-console==6.4.0; python_version >= "3.6" -jupyter-contrib-core==0.3.3 -jupyter-contrib-nbextensions==0.5.1 -jupyter-core==4.8.1; python_full_version >= "3.6.1" and python_version >= "3.7" -jupyter-highlight-selected-word==0.2.0 -jupyter-latex-envs==1.4.6 -jupyter-nbextensions-configurator==0.4.1 -jupyter==1.0.0 -jupyterlab-pygments==0.1.2; python_version >= "3.7" -jupyterlab-widgets==1.0.2; python_version >= "3.6" -kiwisolver==1.3.2; python_version >= "3.7" -lazy-object-proxy==1.6.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0" -lxml==4.6.5; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" -markupsafe==2.0.1; python_version >= "3.7" -matplotlib-inline==0.1.3; python_version >= "3.7" -matplotlib==3.4.3; python_version >= "3.7" -mccabe==0.6.1; python_version >= "3.6" and python_version < "4.0" -mistune==0.8.4; python_version >= "3.7" -munch==2.5.0; python_version >= "3.6" -nbclient==0.5.4; python_full_version >= "3.6.1" and python_version >= "3.7" -nbconvert==6.2.0; python_version >= "3.7" -nbformat==5.1.3; python_full_version >= "3.6.1" and python_version >= "3.7" -nest-asyncio==1.5.1; python_full_version >= "3.6.1" and python_version >= "3.7" -notebook==6.4.5; python_version >= "3.6" -numpy==1.21.1; python_version >= "3.7" -packaging==21.0; python_version >= "3.7" -pandas==1.3.4; python_full_version >= "3.7.1" -pandocfilters==1.5.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.7" -parso==0.8.2; python_version >= "3.7" -pexpect==4.8.0; sys_platform != "win32" and python_version >= "3.7" -pickleshare==0.7.5; python_version >= "3.7" -pillow==8.4.0; python_version >= "3.7" -platformdirs==2.4.0; python_version >= "3.6" and python_version < "4.0" -prometheus-client==0.11.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -prompt-toolkit==3.0.21; python_full_version >= "3.6.2" and python_version >= "3.7" -ptyprocess==0.7.0; sys_platform != "win32" and python_version >= "3.7" and os_name != "nt" -py==1.10.0; implementation_name == "pypy" and python_version >= "3.7" and python_full_version >= "3.6.1" -pycparser==2.20; implementation_name == "pypy" and python_version >= "3.7" and python_full_version >= "3.6.1" -pygments==2.10.0; python_version >= "3.7" -pylint==2.11.1; python_version >= "3.6" and python_version < "4.0" -pypandoc==1.6.4 -pyparsing==2.4.7; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7" -pyproj==3.2.1; python_version >= "3.7" -pyrsistent==0.18.0; python_version >= "3.7" -python-dateutil==2.8.2; python_full_version >= "3.7.1" and python_version >= "3.7" -pytz==2021.3; python_full_version >= "3.7.1" and python_version >= "3.6" -pywin32==302; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_version >= "3.7" -pywinpty==1.1.4; os_name == "nt" and python_version >= "3.6" -pyyaml==6.0; python_version >= "3.6" -pyzmq==22.3.0; python_full_version >= "3.6.1" and python_version >= "3.7" -qtconsole==5.1.1; python_version >= "3.6" -qtpy==1.11.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -requests==2.26.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") -send2trash==1.8.0; python_version >= "3.6" -shapely==1.7.1; python_version >= "3.6" -six==1.16.0; python_full_version >= "3.7.1" and python_version >= "3.7" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7") -terminado==0.12.1; python_version >= "3.6" -testpath==0.5.0; python_version >= "3.7" -toml==0.10.2; python_version > "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.3.0" and python_version > "3.6" and python_version < "4.0" -tornado==6.1; python_full_version >= "3.6.1" and python_version >= "3.7" -tqdm==4.62.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") -traitlets==5.1.0; python_full_version >= "3.6.1" and python_version >= "3.7" -typed-ast==1.4.3; implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6" -types-requests==2.25.11 -typing-extensions==3.10.0.2; python_version >= "3.6" and python_version < "3.8" -urllib3==1.26.7; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "2.7" -us==2.0.2 -wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.7" -webencodings==0.5.1; python_version >= "3.7" -widgetsnbextension==3.5.1 -wrapt==1.13.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0" -xlsxwriter==2.0.0 -zipp==3.6.0; python_version < "3.8" and python_version >= "3.6" diff --git a/data/data-pipeline/scripts/run-liccheck.sh b/data/data-pipeline/scripts/run-liccheck.sh new file mode 100644 index 00000000..d154fe8a --- /dev/null +++ b/data/data-pipeline/scripts/run-liccheck.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Script to run liccheck using poetry, because we use poetry but liccheck uses +# requirements.txt. + +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail + +if ! command -v which; then + echo "You must have poetry installed to run this script." +fi + +poetry export --without-hashes --output requirements.txt +liccheck +rm -f requirements.txt diff --git a/data/data-pipeline/tox.ini b/data/data-pipeline/tox.ini index 059520a4..6e2dee51 100644 --- a/data/data-pipeline/tox.ini +++ b/data/data-pipeline/tox.ini @@ -7,32 +7,17 @@ skip_missing_interpreters = true [testenv:lint] # lints python code in src and tests -deps = -rrequirements.txt -# These are "external" because they are dev dependencies that not in -# "requirements.txt" and therefore not installed in this env, but they will be -# available where this is being run. See: -# https://stackoverflow.com/questions/47642747/tox-warningtest-command-found-but-not-installed-in-testenv -allowlist_externals = black - flake8 - pylint commands = black data_pipeline flake8 data_pipeline -# Ignore tests this lint check because test dependencies are not installed here. - pylint data_pipeline --ignore tests + pylint data_pipeline [testenv:checkdeps] # checks the dependencies for security vulnerabilities and open source licenses -deps = -rrequirements.txt -# These are "external" because they are dev dependencies that not in -# "requirements.txt" and therefore not installed in this env, but they will be -# available where this is being run. See: -# https://stackoverflow.com/questions/47642747/tox-warningtest-command-found-but-not-installed-in-testenv -allowlist_externals = safety - liccheck +allowlist_externals = bash commands = safety check - liccheck + bash scripts/run-liccheck.sh [testenv:pytest] # Run tests deps = pytest -commands = pytest +commands = pytest --full-trace