mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-02-22 01:31:25 -08:00
Remove requirements.txt as a dependency (#1111)
* Remove requirements.txt as a dependency This converts both docker and tox to use poetry, eliminating usage of requirements.txt in both flows. - In tox, uses the tox-poetry package which installs dependencies from the lockfile. - In docker, uses https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker as a reference. * Don't copy pyproject.toml * Remove obsoleted docs about requirements.txt * Add --full-trace option to pytest * Fix liccheck liccheck works with requirements.txt, not with poetry, so there needs to be an extra translation step. * TEMP: Add WIP fix for pandas issue This is just to see if the github actions would pass once this fix gets merged, but it's being reviewed separately. * Revert "TEMP: Add WIP fix for pandas issue" This reverts commit 06e38e8cc77f5f3105c6e7a9449901db67aa1c82.
This commit is contained in:
Shaun Verch
GitHub
parent
f90ec76295
commit
0abf04d6c2
7 changed files with 60 additions and 140 deletions
|
|
@ -24,13 +24,25 @@ RUN /bin/sh -c make && make install
|
|||
RUN add-apt-repository ppa:ubuntugis/ppa
|
||||
RUN apt-get -y install gdal-bin
|
||||
|
||||
# Prepare python packages
|
||||
WORKDIR /data-pipeline
|
||||
RUN pip3 install --upgrade pip setuptools wheel
|
||||
COPY . .
|
||||
# Python package installation using poetry. See:
|
||||
# https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker
|
||||
ENV PYTHONFAULTHANDLER=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONHASHSEED=random \
|
||||
PIP_NO_CACHE_DIR=off \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=on \
|
||||
PIP_DEFAULT_TIMEOUT=100 \
|
||||
POETRY_VERSION=1.1.12
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip3 install -r requirements.txt
|
||||
RUN pip3 install .
|
||||
WORKDIR /data-pipeline
|
||||
COPY poetry.lock /data-pipeline/
|
||||
|
||||
RUN pip install "poetry==$POETRY_VERSION"
|
||||
RUN poetry config virtualenvs.create false \
|
||||
&& poetry config virtualenvs.in-project false \
|
||||
&& poetry install --no-dev --no-interaction --no-ansi
|
||||
|
||||
# Copy all project files into the container
|
||||
COPY . /data-pipeline
|
||||
|
||||
CMD python3 -m data_pipeline.application data-full-run --check -s aws
|
||||
|
|
|
|||
|
|
@ -279,10 +279,6 @@ After installing the poetry dependencies, you can see a list of commands with th
|
|||
For more information, see [nbextensions docs](https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/install.html) and
|
||||
see [python-markdown docs](https://github.com/ipython-contrib/jupyter_contrib_nbextensions/tree/master/src/jupyter_contrib_nbextensions/nbextensions/python-markdown).
|
||||
|
||||
## Miscellaneous
|
||||
|
||||
- To export packages from Poetry to `requirements.txt` run `poetry export --without-hashes > requirements.txt`
|
||||
|
||||
## Testing
|
||||
|
||||
### Background
|
||||
|
|
|
|||
22
data/data-pipeline/poetry.lock
generated
22
data/data-pipeline/poetry.lock
generated
|
|
@ -1531,6 +1531,22 @@ virtualenv = ">=16.0.0,<20.0.0 || >20.0.0,<20.0.1 || >20.0.1,<20.0.2 || >20.0.2,
|
|||
docs = ["pygments-github-lexers (>=0.0.5)", "sphinx (>=2.0.0)", "sphinxcontrib-autoprogram (>=0.1.5)", "towncrier (>=18.5.0)"]
|
||||
testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "psutil (>=5.6.1)", "pytest (>=4.0.0)", "pytest-cov (>=2.5.1)", "pytest-mock (>=1.10.0)", "pytest-randomly (>=1.0.0)", "pytest-xdist (>=1.22.2)", "pathlib2 (>=2.3.3)"]
|
||||
|
||||
[[package]]
|
||||
name = "tox-poetry"
|
||||
version = "0.4.1"
|
||||
description = "Tox poetry plugin"
|
||||
category = "dev"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[package.dependencies]
|
||||
pluggy = "*"
|
||||
toml = "*"
|
||||
tox = {version = ">=3.7.0", markers = "python_version >= \"3\""}
|
||||
|
||||
[package.extras]
|
||||
test = ["coverage", "pytest", "pycodestyle", "pylint"]
|
||||
|
||||
[[package]]
|
||||
name = "tqdm"
|
||||
version = "4.62.0"
|
||||
|
|
@ -1684,7 +1700,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes
|
|||
[metadata]
|
||||
lock-version = "1.1"
|
||||
python-versions = "^3.7.1"
|
||||
content-hash = "1823efb9fae610a00eb714ec61fd165ec5cda3d38fa83c12b00e9d38f4358fa3"
|
||||
content-hash = "f832db8cb99fd9f868d6e03c7536b7679bfdc0228693ba796ae2a4028b641a28"
|
||||
|
||||
[metadata.files]
|
||||
appnope = [
|
||||
|
|
@ -2782,6 +2798,10 @@ tox = [
|
|||
{file = "tox-3.24.4-py2.py3-none-any.whl", hash = "sha256:5e274227a53dc9ef856767c21867377ba395992549f02ce55eb549f9fb9a8d10"},
|
||||
{file = "tox-3.24.4.tar.gz", hash = "sha256:c30b57fa2477f1fb7c36aa1d83292d5c2336cd0018119e1b1c17340e2c2708ca"},
|
||||
]
|
||||
tox-poetry = [
|
||||
{file = "tox-poetry-0.4.1.tar.gz", hash = "sha256:2395808e1ce487b5894c10f2202e14702bfa6d6909c0d1e525170d14809ac7ef"},
|
||||
{file = "tox_poetry-0.4.1-py2.py3-none-any.whl", hash = "sha256:11d9cd4e51d4cd9484b3ba63f2650ab4cfb4096e5f0682ecf561ddfc3c8e8c92"},
|
||||
]
|
||||
tqdm = [
|
||||
{file = "tqdm-4.62.0-py2.py3-none-any.whl", hash = "sha256:706dea48ee05ba16e936ee91cb3791cd2ea6da348a0e50b46863ff4363ff4340"},
|
||||
{file = "tqdm-4.62.0.tar.gz", hash = "sha256:3642d483b558eec80d3c831e23953582c34d7e4540db86d9e5ed9dad238dabc6"},
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ pytest = "^6.2.4"
|
|||
safety = "^1.10.3"
|
||||
tox = "^3.24.0"
|
||||
pytest-mock = "^3.6.1"
|
||||
tox-poetry = "^0.4.1"
|
||||
|
||||
[build-system]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
|
|
|
|||
|
|
@ -1,109 +0,0 @@
|
|||
appnope==0.1.2; sys_platform == "darwin" and python_version >= "3.7" and platform_system == "Darwin"
|
||||
argcomplete==1.12.3; python_version < "3.8.0" and python_version >= "3.7"
|
||||
argon2-cffi==21.1.0; python_version >= "3.6"
|
||||
astroid==2.8.3; python_version >= "3.6" and python_version < "4.0"
|
||||
attrs==21.2.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
|
||||
backcall==0.2.0; python_version >= "3.7"
|
||||
bleach==4.1.0; python_version >= "3.7"
|
||||
censusdata==1.15; python_version >= "2.7"
|
||||
certifi==2021.10.8; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7"
|
||||
cffi==1.15.0; implementation_name == "pypy" and python_version >= "3.7" and python_full_version >= "3.6.1"
|
||||
charset-normalizer==2.0.7; python_full_version >= "3.6.0" and python_version >= "3"
|
||||
click-plugins==1.1.1; python_version >= "3.6"
|
||||
click==8.0.3; python_version >= "3.6"
|
||||
cligj==0.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version < "4" and python_version >= "3.6"
|
||||
colorama==0.4.4; python_version >= "3.7" and python_full_version < "3.0.0" and platform_system == "Windows" and sys_platform == "win32" and python_version < "4.0" or platform_system == "Windows" and python_version >= "3.7" and python_full_version >= "3.5.0" and sys_platform == "win32" and python_version < "4.0"
|
||||
cycler==0.10.0; python_version >= "3.7"
|
||||
debugpy==1.5.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
|
||||
decorator==5.1.0; python_version >= "3.7"
|
||||
defusedxml==0.7.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
|
||||
dynaconf==3.1.7; python_version >= "3.7"
|
||||
entrypoints==0.3; python_full_version >= "3.6.1" and python_version >= "3.7"
|
||||
fiona==1.8.20; python_version >= "3.6"
|
||||
geopandas==0.9.0; python_version >= "3.6"
|
||||
idna==3.3; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5"
|
||||
importlib-metadata==4.8.1; python_version == "3.7"
|
||||
ipdb==0.13.9; python_version >= "2.7"
|
||||
ipykernel==6.4.2; python_version >= "3.7"
|
||||
ipython-genutils==0.2.0; python_version >= "3.7"
|
||||
ipython==7.28.0; python_version >= "3.7"
|
||||
ipywidgets==7.6.5
|
||||
isort==5.9.3; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
|
||||
jedi==0.18.0; python_version >= "3.7"
|
||||
jellyfish==0.6.1
|
||||
jinja2==3.0.2; python_version >= "3.7"
|
||||
jsonschema==4.1.2; python_version >= "3.7"
|
||||
jupyter-client==7.0.6; python_full_version >= "3.6.1" and python_version >= "3.7"
|
||||
jupyter-console==6.4.0; python_version >= "3.6"
|
||||
jupyter-contrib-core==0.3.3
|
||||
jupyter-contrib-nbextensions==0.5.1
|
||||
jupyter-core==4.8.1; python_full_version >= "3.6.1" and python_version >= "3.7"
|
||||
jupyter-highlight-selected-word==0.2.0
|
||||
jupyter-latex-envs==1.4.6
|
||||
jupyter-nbextensions-configurator==0.4.1
|
||||
jupyter==1.0.0
|
||||
jupyterlab-pygments==0.1.2; python_version >= "3.7"
|
||||
jupyterlab-widgets==1.0.2; python_version >= "3.6"
|
||||
kiwisolver==1.3.2; python_version >= "3.7"
|
||||
lazy-object-proxy==1.6.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0"
|
||||
lxml==4.6.5; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
|
||||
markupsafe==2.0.1; python_version >= "3.7"
|
||||
matplotlib-inline==0.1.3; python_version >= "3.7"
|
||||
matplotlib==3.4.3; python_version >= "3.7"
|
||||
mccabe==0.6.1; python_version >= "3.6" and python_version < "4.0"
|
||||
mistune==0.8.4; python_version >= "3.7"
|
||||
munch==2.5.0; python_version >= "3.6"
|
||||
nbclient==0.5.4; python_full_version >= "3.6.1" and python_version >= "3.7"
|
||||
nbconvert==6.2.0; python_version >= "3.7"
|
||||
nbformat==5.1.3; python_full_version >= "3.6.1" and python_version >= "3.7"
|
||||
nest-asyncio==1.5.1; python_full_version >= "3.6.1" and python_version >= "3.7"
|
||||
notebook==6.4.5; python_version >= "3.6"
|
||||
numpy==1.21.1; python_version >= "3.7"
|
||||
packaging==21.0; python_version >= "3.7"
|
||||
pandas==1.3.4; python_full_version >= "3.7.1"
|
||||
pandocfilters==1.5.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.7"
|
||||
parso==0.8.2; python_version >= "3.7"
|
||||
pexpect==4.8.0; sys_platform != "win32" and python_version >= "3.7"
|
||||
pickleshare==0.7.5; python_version >= "3.7"
|
||||
pillow==8.4.0; python_version >= "3.7"
|
||||
platformdirs==2.4.0; python_version >= "3.6" and python_version < "4.0"
|
||||
prometheus-client==0.11.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
|
||||
prompt-toolkit==3.0.21; python_full_version >= "3.6.2" and python_version >= "3.7"
|
||||
ptyprocess==0.7.0; sys_platform != "win32" and python_version >= "3.7" and os_name != "nt"
|
||||
py==1.10.0; implementation_name == "pypy" and python_version >= "3.7" and python_full_version >= "3.6.1"
|
||||
pycparser==2.20; implementation_name == "pypy" and python_version >= "3.7" and python_full_version >= "3.6.1"
|
||||
pygments==2.10.0; python_version >= "3.7"
|
||||
pylint==2.11.1; python_version >= "3.6" and python_version < "4.0"
|
||||
pypandoc==1.6.4
|
||||
pyparsing==2.4.7; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7"
|
||||
pyproj==3.2.1; python_version >= "3.7"
|
||||
pyrsistent==0.18.0; python_version >= "3.7"
|
||||
python-dateutil==2.8.2; python_full_version >= "3.7.1" and python_version >= "3.7"
|
||||
pytz==2021.3; python_full_version >= "3.7.1" and python_version >= "3.6"
|
||||
pywin32==302; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_version >= "3.7"
|
||||
pywinpty==1.1.4; os_name == "nt" and python_version >= "3.6"
|
||||
pyyaml==6.0; python_version >= "3.6"
|
||||
pyzmq==22.3.0; python_full_version >= "3.6.1" and python_version >= "3.7"
|
||||
qtconsole==5.1.1; python_version >= "3.6"
|
||||
qtpy==1.11.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
|
||||
requests==2.26.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
|
||||
send2trash==1.8.0; python_version >= "3.6"
|
||||
shapely==1.7.1; python_version >= "3.6"
|
||||
six==1.16.0; python_full_version >= "3.7.1" and python_version >= "3.7" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7")
|
||||
terminado==0.12.1; python_version >= "3.6"
|
||||
testpath==0.5.0; python_version >= "3.7"
|
||||
toml==0.10.2; python_version > "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.3.0" and python_version > "3.6" and python_version < "4.0"
|
||||
tornado==6.1; python_full_version >= "3.6.1" and python_version >= "3.7"
|
||||
tqdm==4.62.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
|
||||
traitlets==5.1.0; python_full_version >= "3.6.1" and python_version >= "3.7"
|
||||
typed-ast==1.4.3; implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6"
|
||||
types-requests==2.25.11
|
||||
typing-extensions==3.10.0.2; python_version >= "3.6" and python_version < "3.8"
|
||||
urllib3==1.26.7; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "2.7"
|
||||
us==2.0.2
|
||||
wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.7"
|
||||
webencodings==0.5.1; python_version >= "3.7"
|
||||
widgetsnbextension==3.5.1
|
||||
wrapt==1.13.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0"
|
||||
xlsxwriter==2.0.0
|
||||
zipp==3.6.0; python_version < "3.8" and python_version >= "3.6"
|
||||
15
data/data-pipeline/scripts/run-liccheck.sh
Normal file
15
data/data-pipeline/scripts/run-liccheck.sh
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Script to run liccheck using poetry, because we use poetry but liccheck uses
|
||||
# requirements.txt.
|
||||
|
||||
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
|
||||
set -euo pipefail
|
||||
|
||||
if ! command -v which; then
|
||||
echo "You must have poetry installed to run this script."
|
||||
fi
|
||||
|
||||
poetry export --without-hashes --output requirements.txt
|
||||
liccheck
|
||||
rm -f requirements.txt
|
||||
|
|
@ -7,32 +7,17 @@ skip_missing_interpreters = true
|
|||
|
||||
[testenv:lint]
|
||||
# lints python code in src and tests
|
||||
deps = -rrequirements.txt
|
||||
# These are "external" because they are dev dependencies that not in
|
||||
# "requirements.txt" and therefore not installed in this env, but they will be
|
||||
# available where this is being run. See:
|
||||
# https://stackoverflow.com/questions/47642747/tox-warningtest-command-found-but-not-installed-in-testenv
|
||||
allowlist_externals = black
|
||||
flake8
|
||||
pylint
|
||||
commands = black data_pipeline
|
||||
flake8 data_pipeline
|
||||
# Ignore tests this lint check because test dependencies are not installed here.
|
||||
pylint data_pipeline --ignore tests
|
||||
pylint data_pipeline
|
||||
|
||||
[testenv:checkdeps]
|
||||
# checks the dependencies for security vulnerabilities and open source licenses
|
||||
deps = -rrequirements.txt
|
||||
# These are "external" because they are dev dependencies that not in
|
||||
# "requirements.txt" and therefore not installed in this env, but they will be
|
||||
# available where this is being run. See:
|
||||
# https://stackoverflow.com/questions/47642747/tox-warningtest-command-found-but-not-installed-in-testenv
|
||||
allowlist_externals = safety
|
||||
liccheck
|
||||
allowlist_externals = bash
|
||||
commands = safety check
|
||||
liccheck
|
||||
bash scripts/run-liccheck.sh
|
||||
|
||||
[testenv:pytest]
|
||||
# Run tests
|
||||
deps = pytest
|
||||
commands = pytest
|
||||
commands = pytest --full-trace
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue