Remove requirements.txt as a dependency (#1111)

* Remove requirements.txt as a dependency

This converts both docker and tox to use poetry, eliminating usage of
requirements.txt in both flows.

- In tox, uses the tox-poetry package which installs dependencies from
  the lockfile.
- In docker, uses
  https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker
  as a reference.

* Don't copy pyproject.toml

* Remove obsoleted docs about requirements.txt

* Add --full-trace option to pytest

* Fix liccheck

liccheck works with requirements.txt, not with poetry, so there needs to
be an extra translation step.

* TEMP: Add WIP fix for pandas issue

This is just to see if the github actions would pass once this fix gets
merged, but it's being reviewed separately.

* Revert "TEMP: Add WIP fix for pandas issue"

This reverts commit 06e38e8cc77f5f3105c6e7a9449901db67aa1c82.

data/data-pipeline/Dockerfile

@@ -24,13 +24,25 @@ RUN /bin/sh -c make && make install
 RUN add-apt-repository ppa:ubuntugis/ppa
 RUN apt-get -y install gdal-bin
 
-# Prepare python packages
+# Python package installation using poetry. See:
-WORKDIR /data-pipeline
+# https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker
-RUN pip3 install --upgrade pip setuptools wheel
+ENV PYTHONFAULTHANDLER=1 \
-COPY . .
+  PYTHONUNBUFFERED=1 \
+  PYTHONHASHSEED=random \
+  PIP_NO_CACHE_DIR=off \
+  PIP_DISABLE_PIP_VERSION_CHECK=on \
+  PIP_DEFAULT_TIMEOUT=100 \
+  POETRY_VERSION=1.1.12
 
-COPY requirements.txt .
+WORKDIR /data-pipeline
-RUN pip3 install -r requirements.txt
+COPY poetry.lock /data-pipeline/
-RUN pip3 install .
+
+RUN pip install "poetry==$POETRY_VERSION"
+RUN poetry config virtualenvs.create false \
+  && poetry config virtualenvs.in-project false \
+  && poetry install --no-dev --no-interaction --no-ansi
+
+# Copy all project files into the container
+COPY . /data-pipeline
 
 CMD python3 -m data_pipeline.application data-full-run --check -s aws

data/data-pipeline/README.md

@@ -279,10 +279,6 @@ After installing the poetry dependencies, you can see a list of commands with th
 For more information, see [nbextensions docs](https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/install.html) and
 see [python-markdown docs](https://github.com/ipython-contrib/jupyter_contrib_nbextensions/tree/master/src/jupyter_contrib_nbextensions/nbextensions/python-markdown).
 
-## Miscellaneous
-
-- To export packages from Poetry to `requirements.txt` run `poetry export --without-hashes > requirements.txt`
-
 ## Testing
 
 ### Background

data/data-pipeline/poetry.lock

@@ -1531,6 +1531,22 @@ virtualenv = ">=16.0.0,<20.0.0 || >20.0.0,<20.0.1 || >20.0.1,<20.0.2 || >20.0.2,
 docs = ["pygments-github-lexers (>=0.0.5)", "sphinx (>=2.0.0)", "sphinxcontrib-autoprogram (>=0.1.5)", "towncrier (>=18.5.0)"]
 testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "psutil (>=5.6.1)", "pytest (>=4.0.0)", "pytest-cov (>=2.5.1)", "pytest-mock (>=1.10.0)", "pytest-randomly (>=1.0.0)", "pytest-xdist (>=1.22.2)", "pathlib2 (>=2.3.3)"]
 
+[[package]]
+name = "tox-poetry"
+version = "0.4.1"
+description = "Tox poetry plugin"
+category = "dev"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+pluggy = "*"
+toml = "*"
+tox = {version = ">=3.7.0", markers = "python_version >= \"3\""}
+
+[package.extras]
+test = ["coverage", "pytest", "pycodestyle", "pylint"]
+
 [[package]]
 name = "tqdm"
 version = "4.62.0"
@@ -1684,7 +1700,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7.1"
-content-hash = "1823efb9fae610a00eb714ec61fd165ec5cda3d38fa83c12b00e9d38f4358fa3"
+content-hash = "f832db8cb99fd9f868d6e03c7536b7679bfdc0228693ba796ae2a4028b641a28"
 
 [metadata.files]
 appnope = [
@@ -2782,6 +2798,10 @@ tox = [
     {file = "tox-3.24.4-py2.py3-none-any.whl", hash = "sha256:5e274227a53dc9ef856767c21867377ba395992549f02ce55eb549f9fb9a8d10"},
     {file = "tox-3.24.4.tar.gz", hash = "sha256:c30b57fa2477f1fb7c36aa1d83292d5c2336cd0018119e1b1c17340e2c2708ca"},
 ]
+tox-poetry = [
+    {file = "tox-poetry-0.4.1.tar.gz", hash = "sha256:2395808e1ce487b5894c10f2202e14702bfa6d6909c0d1e525170d14809ac7ef"},
+    {file = "tox_poetry-0.4.1-py2.py3-none-any.whl", hash = "sha256:11d9cd4e51d4cd9484b3ba63f2650ab4cfb4096e5f0682ecf561ddfc3c8e8c92"},
+]
 tqdm = [
     {file = "tqdm-4.62.0-py2.py3-none-any.whl", hash = "sha256:706dea48ee05ba16e936ee91cb3791cd2ea6da348a0e50b46863ff4363ff4340"},
     {file = "tqdm-4.62.0.tar.gz", hash = "sha256:3642d483b558eec80d3c831e23953582c34d7e4540db86d9e5ed9dad238dabc6"},

data/data-pipeline/pyproject.toml

@@ -36,6 +36,7 @@ pytest = "^6.2.4"
 safety = "^1.10.3"
 tox = "^3.24.0"
 pytest-mock = "^3.6.1"
+tox-poetry = "^0.4.1"
 
 [build-system]
 build-backend = "poetry.core.masonry.api"

data/data-pipeline/requirements.txt

@@ -1,109 +0,0 @@
-appnope==0.1.2; sys_platform == "darwin" and python_version >= "3.7" and platform_system == "Darwin"
-argcomplete==1.12.3; python_version < "3.8.0" and python_version >= "3.7"
-argon2-cffi==21.1.0; python_version >= "3.6"
-astroid==2.8.3; python_version >= "3.6" and python_version < "4.0"
-attrs==21.2.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
-backcall==0.2.0; python_version >= "3.7"
-bleach==4.1.0; python_version >= "3.7"
-censusdata==1.15; python_version >= "2.7"
-certifi==2021.10.8; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7"
-cffi==1.15.0; implementation_name == "pypy" and python_version >= "3.7" and python_full_version >= "3.6.1"
-charset-normalizer==2.0.7; python_full_version >= "3.6.0" and python_version >= "3"
-click-plugins==1.1.1; python_version >= "3.6"
-click==8.0.3; python_version >= "3.6"
-cligj==0.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version < "4" and python_version >= "3.6"
-colorama==0.4.4; python_version >= "3.7" and python_full_version < "3.0.0" and platform_system == "Windows" and sys_platform == "win32" and python_version < "4.0" or platform_system == "Windows" and python_version >= "3.7" and python_full_version >= "3.5.0" and sys_platform == "win32" and python_version < "4.0"
-cycler==0.10.0; python_version >= "3.7"
-debugpy==1.5.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
-decorator==5.1.0; python_version >= "3.7"
-defusedxml==0.7.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
-dynaconf==3.1.7; python_version >= "3.7"
-entrypoints==0.3; python_full_version >= "3.6.1" and python_version >= "3.7"
-fiona==1.8.20; python_version >= "3.6"
-geopandas==0.9.0; python_version >= "3.6"
-idna==3.3; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5"
-importlib-metadata==4.8.1; python_version == "3.7"
-ipdb==0.13.9; python_version >= "2.7"
-ipykernel==6.4.2; python_version >= "3.7"
-ipython-genutils==0.2.0; python_version >= "3.7"
-ipython==7.28.0; python_version >= "3.7"
-ipywidgets==7.6.5
-isort==5.9.3; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
-jedi==0.18.0; python_version >= "3.7"
-jellyfish==0.6.1
-jinja2==3.0.2; python_version >= "3.7"
-jsonschema==4.1.2; python_version >= "3.7"
-jupyter-client==7.0.6; python_full_version >= "3.6.1" and python_version >= "3.7"
-jupyter-console==6.4.0; python_version >= "3.6"
-jupyter-contrib-core==0.3.3
-jupyter-contrib-nbextensions==0.5.1
-jupyter-core==4.8.1; python_full_version >= "3.6.1" and python_version >= "3.7"
-jupyter-highlight-selected-word==0.2.0
-jupyter-latex-envs==1.4.6
-jupyter-nbextensions-configurator==0.4.1
-jupyter==1.0.0
-jupyterlab-pygments==0.1.2; python_version >= "3.7"
-jupyterlab-widgets==1.0.2; python_version >= "3.6"
-kiwisolver==1.3.2; python_version >= "3.7"
-lazy-object-proxy==1.6.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0"
-lxml==4.6.5; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
-markupsafe==2.0.1; python_version >= "3.7"
-matplotlib-inline==0.1.3; python_version >= "3.7"
-matplotlib==3.4.3; python_version >= "3.7"
-mccabe==0.6.1; python_version >= "3.6" and python_version < "4.0"
-mistune==0.8.4; python_version >= "3.7"
-munch==2.5.0; python_version >= "3.6"
-nbclient==0.5.4; python_full_version >= "3.6.1" and python_version >= "3.7"
-nbconvert==6.2.0; python_version >= "3.7"
-nbformat==5.1.3; python_full_version >= "3.6.1" and python_version >= "3.7"
-nest-asyncio==1.5.1; python_full_version >= "3.6.1" and python_version >= "3.7"
-notebook==6.4.5; python_version >= "3.6"
-numpy==1.21.1; python_version >= "3.7"
-packaging==21.0; python_version >= "3.7"
-pandas==1.3.4; python_full_version >= "3.7.1"
-pandocfilters==1.5.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.7"
-parso==0.8.2; python_version >= "3.7"
-pexpect==4.8.0; sys_platform != "win32" and python_version >= "3.7"
-pickleshare==0.7.5; python_version >= "3.7"
-pillow==8.4.0; python_version >= "3.7"
-platformdirs==2.4.0; python_version >= "3.6" and python_version < "4.0"
-prometheus-client==0.11.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-prompt-toolkit==3.0.21; python_full_version >= "3.6.2" and python_version >= "3.7"
-ptyprocess==0.7.0; sys_platform != "win32" and python_version >= "3.7" and os_name != "nt"
-py==1.10.0; implementation_name == "pypy" and python_version >= "3.7" and python_full_version >= "3.6.1"
-pycparser==2.20; implementation_name == "pypy" and python_version >= "3.7" and python_full_version >= "3.6.1"
-pygments==2.10.0; python_version >= "3.7"
-pylint==2.11.1; python_version >= "3.6" and python_version < "4.0"
-pypandoc==1.6.4
-pyparsing==2.4.7; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7"
-pyproj==3.2.1; python_version >= "3.7"
-pyrsistent==0.18.0; python_version >= "3.7"
-python-dateutil==2.8.2; python_full_version >= "3.7.1" and python_version >= "3.7"
-pytz==2021.3; python_full_version >= "3.7.1" and python_version >= "3.6"
-pywin32==302; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_version >= "3.7"
-pywinpty==1.1.4; os_name == "nt" and python_version >= "3.6"
-pyyaml==6.0; python_version >= "3.6"
-pyzmq==22.3.0; python_full_version >= "3.6.1" and python_version >= "3.7"
-qtconsole==5.1.1; python_version >= "3.6"
-qtpy==1.11.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-requests==2.26.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
-send2trash==1.8.0; python_version >= "3.6"
-shapely==1.7.1; python_version >= "3.6"
-six==1.16.0; python_full_version >= "3.7.1" and python_version >= "3.7" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7")
-terminado==0.12.1; python_version >= "3.6"
-testpath==0.5.0; python_version >= "3.7"
-toml==0.10.2; python_version > "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.3.0" and python_version > "3.6" and python_version < "4.0"
-tornado==6.1; python_full_version >= "3.6.1" and python_version >= "3.7"
-tqdm==4.62.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-traitlets==5.1.0; python_full_version >= "3.6.1" and python_version >= "3.7"
-typed-ast==1.4.3; implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6"
-types-requests==2.25.11
-typing-extensions==3.10.0.2; python_version >= "3.6" and python_version < "3.8"
-urllib3==1.26.7; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "2.7"
-us==2.0.2
-wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.7"
-webencodings==0.5.1; python_version >= "3.7"
-widgetsnbextension==3.5.1
-wrapt==1.13.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0"
-xlsxwriter==2.0.0
-zipp==3.6.0; python_version < "3.8" and python_version >= "3.6"

data/data-pipeline/scripts/run-liccheck.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Script to run liccheck using poetry, because we use poetry but liccheck uses
+# requirements.txt.
+
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -euo pipefail
+
+if ! command -v which; then
+	echo "You must have poetry installed to run this script."
+fi
+
+poetry export --without-hashes --output requirements.txt
+liccheck
+rm -f requirements.txt

data/data-pipeline/tox.ini

@@ -7,32 +7,17 @@ skip_missing_interpreters = true
 
 [testenv:lint]
 # lints python code in src and tests
-deps = -rrequirements.txt
-# These are "external" because they are dev dependencies that not in
-# "requirements.txt" and therefore not installed in this env, but they will be
-# available where this is being run. See:
-# https://stackoverflow.com/questions/47642747/tox-warningtest-command-found-but-not-installed-in-testenv
-allowlist_externals = black
-                      flake8
-                      pylint
 commands = black data_pipeline
            flake8 data_pipeline
-# Ignore tests this lint check because test dependencies are not installed here.
+           pylint data_pipeline
-           pylint data_pipeline --ignore tests
 
 [testenv:checkdeps]
 # checks the dependencies for security vulnerabilities and open source licenses
-deps = -rrequirements.txt
+allowlist_externals = bash
-# These are "external" because they are dev dependencies that not in
-# "requirements.txt" and therefore not installed in this env, but they will be
-# available where this is being run. See:
-# https://stackoverflow.com/questions/47642747/tox-warningtest-command-found-but-not-installed-in-testenv
-allowlist_externals = safety
-                      liccheck
 commands = safety check
-           liccheck
+           bash scripts/run-liccheck.sh
 
 [testenv:pytest]
 # Run tests
 deps = pytest
-commands = pytest
+commands = pytest --full-trace