Re-export requirements.txt to fix version errors (#1099)

* Re-export requirements.txt to fix version errors

The version of lxml in this file had a known vulnerability that got
caught by the "safety" checker, but it is updated in the poetry files.

Regenerated using:
https://github.com/usds/justice40-tool/tree/main/data/data-pipeline#miscellaneous

* Fix lint error

* Run lint on all envs and add comments

* Ignore testst that fail lint because of dev deps

* Ignore medium.com in link checker

It's returning 403s to github actions...

data/data-pipeline/data_pipeline/etl/sources/energy_definition_alternative_draft/etl.py

@@ -95,9 +95,13 @@ class EnergyDefinitionAlternativeDraft(ExtractTransformLoad):
         )
 
         # Convert to boolean:
-        self.df[field_names.ENERGY_RELATED_COMMUNITIES_DEFINITION_ALTERNATIVE] = \
+        self.df[
-            self.df[field_names.ENERGY_RELATED_COMMUNITIES_DEFINITION_ALTERNATIVE
+            field_names.ENERGY_RELATED_COMMUNITIES_DEFINITION_ALTERNATIVE
-            ].astype('bool')
+        ] = self.df[
+            field_names.ENERGY_RELATED_COMMUNITIES_DEFINITION_ALTERNATIVE
+        ].astype(
+            "bool"
+        )
 
     def validate(self) -> None:
         logger.info("Validating data")

data/data-pipeline/requirements.txt

@@ -1,48 +1,39 @@
 appnope==0.1.2; sys_platform == "darwin" and python_version >= "3.7" and platform_system == "Darwin"
 argcomplete==1.12.3; python_version < "3.8.0" and python_version >= "3.7"
 argon2-cffi==21.1.0; python_version >= "3.6"
-astroid==2.8.0; python_version >= "3.6" and python_version < "4.0"
+astroid==2.8.3; python_version >= "3.6" and python_version < "4.0"
-atomicwrites==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.4.0"
+attrs==21.2.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
-attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 backcall==0.2.0; python_version >= "3.7"
-backports.entry-points-selectable==1.1.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "2.7"
-black==21.9b0; python_full_version >= "3.6.2"
 bleach==4.1.0; python_version >= "3.7"
 censusdata==1.15; python_version >= "2.7"
-certifi==2021.5.30; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7"
+certifi==2021.10.8; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7"
-cffi==1.14.6; implementation_name == "pypy" and python_version >= "3.6"
+cffi==1.15.0; implementation_name == "pypy" and python_version >= "3.7" and python_full_version >= "3.6.1"
-charset-normalizer==2.0.6; python_full_version >= "3.6.0" and python_version >= "3"
+charset-normalizer==2.0.7; python_full_version >= "3.6.0" and python_version >= "3"
 click-plugins==1.1.1; python_version >= "3.6"
-click==8.0.1; python_version >= "3.6"
+click==8.0.3; python_version >= "3.6"
 cligj==0.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version < "4" and python_version >= "3.6"
-colorama==0.4.4; platform_system == "Windows" and python_version >= "3.7" and python_full_version >= "3.6.2" and sys_platform == "win32" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.5.0") and (python_version >= "3.7" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.7" and python_full_version >= "3.5.0")
+colorama==0.4.4; python_version >= "3.7" and python_full_version < "3.0.0" and platform_system == "Windows" and sys_platform == "win32" and python_version < "4.0" or platform_system == "Windows" and python_version >= "3.7" and python_full_version >= "3.5.0" and sys_platform == "win32" and python_version < "4.0"
-configparser==5.0.2; python_version >= "3.6"
 cycler==0.10.0; python_version >= "3.7"
-debugpy==1.4.3; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
+debugpy==1.5.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
 decorator==5.1.0; python_version >= "3.7"
 defusedxml==0.7.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
-distlib==0.3.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
-dparse==0.5.1; python_version >= "3.5"
 dynaconf==3.1.7; python_version >= "3.7"
 entrypoints==0.3; python_full_version >= "3.6.1" and python_version >= "3.7"
-et-xmlfile==1.1.0; python_version >= "3.6"
-filelock==3.0.12; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
 fiona==1.8.20; python_version >= "3.6"
-flake8==3.9.2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 geopandas==0.9.0; python_version >= "3.6"
-idna==3.2; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5"
+idna==3.3; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5"
-importlib-metadata==4.8.1; python_version == "3.7" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.5.0" and python_version < "3.8" and python_version >= "3.6") and python_full_version >= "3.6.2"
+importlib-metadata==4.8.1; python_version == "3.7"
-iniconfig==1.1.1; python_version >= "3.6"
+ipdb==0.13.9; python_version >= "2.7"
-ipykernel==6.4.1; python_version >= "3.7"
+ipykernel==6.4.2; python_version >= "3.7"
 ipython-genutils==0.2.0; python_version >= "3.7"
-ipython==7.27.0; python_version >= "3.7"
+ipython==7.28.0; python_version >= "3.7"
 ipywidgets==7.6.5
 isort==5.9.3; python_full_version >= "3.6.1" and python_version < "4.0" and python_version >= "3.6"
 jedi==0.18.0; python_version >= "3.7"
 jellyfish==0.6.1
-jinja2==3.0.1; python_version >= "3.7"
+jinja2==3.0.2; python_version >= "3.7"
-jsonschema==3.2.0; python_version >= "3.5"
+jsonschema==4.1.2; python_version >= "3.7"
-jupyter-client==7.0.3; python_full_version >= "3.6.1" and python_version >= "3.7"
+jupyter-client==7.0.6; python_full_version >= "3.6.1" and python_version >= "3.7"
 jupyter-console==6.4.0; python_version >= "3.6"
 jupyter-contrib-core==0.3.3
 jupyter-contrib-nbextensions==0.5.1
@@ -55,79 +46,64 @@ jupyterlab-pygments==0.1.2; python_version >= "3.7"
 jupyterlab-widgets==1.0.2; python_version >= "3.6"
 kiwisolver==1.3.2; python_version >= "3.7"
 lazy-object-proxy==1.6.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0"
-liccheck==0.6.2; python_version >= "2.7"
+lxml==4.6.5; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
-lxml==4.6.3; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
 markupsafe==2.0.1; python_version >= "3.7"
 matplotlib-inline==0.1.3; python_version >= "3.7"
 matplotlib==3.4.3; python_version >= "3.7"
-mccabe==0.6.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0"
+mccabe==0.6.1; python_version >= "3.6" and python_version < "4.0"
 mistune==0.8.4; python_version >= "3.7"
 munch==2.5.0; python_version >= "3.6"
-mypy-extensions==0.4.3; python_full_version >= "3.6.2" and python_version >= "3.5"
-mypy==0.910; python_version >= "3.5"
 nbclient==0.5.4; python_full_version >= "3.6.1" and python_version >= "3.7"
-nbconvert==6.1.0; python_version >= "3.7"
+nbconvert==6.2.0; python_version >= "3.7"
 nbformat==5.1.3; python_full_version >= "3.6.1" and python_version >= "3.7"
 nest-asyncio==1.5.1; python_full_version >= "3.6.1" and python_version >= "3.7"
-notebook==6.4.4; python_version >= "3.6"
+notebook==6.4.5; python_version >= "3.6"
 numpy==1.21.1; python_version >= "3.7"
-openpyxl==3.0.7; python_version >= "3.6"
+packaging==21.0; python_version >= "3.7"
-packaging==21.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
+pandas==1.3.4; python_full_version >= "3.7.1"
-pandas==1.3.3; python_full_version >= "3.7.1"
 pandocfilters==1.5.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.7"
 parso==0.8.2; python_version >= "3.7"
-pathspec==0.9.0; python_full_version >= "3.6.2"
 pexpect==4.8.0; sys_platform != "win32" and python_version >= "3.7"
 pickleshare==0.7.5; python_version >= "3.7"
-pillow==8.3.2; python_version >= "3.7"
+pillow==8.4.0; python_version >= "3.7"
-platformdirs==2.3.0; python_version >= "3.6" and python_full_version >= "3.6.2" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6")
+platformdirs==2.4.0; python_version >= "3.6" and python_version < "4.0"
-pluggy==1.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 prometheus-client==0.11.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-prompt-toolkit==3.0.20; python_full_version >= "3.6.2" and python_version >= "3.7"
+prompt-toolkit==3.0.21; python_full_version >= "3.6.2" and python_version >= "3.7"
 ptyprocess==0.7.0; sys_platform != "win32" and python_version >= "3.7" and os_name != "nt"
-py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.5.0" and python_version >= "3.6" and implementation_name == "pypy"
+py==1.10.0; implementation_name == "pypy" and python_version >= "3.7" and python_full_version >= "3.6.1"
-pycodestyle==2.7.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
+pycparser==2.20; implementation_name == "pypy" and python_version >= "3.7" and python_full_version >= "3.6.1"
-pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-pyflakes==2.3.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
 pygments==2.10.0; python_version >= "3.7"
 pylint==2.11.1; python_version >= "3.6" and python_version < "4.0"
 pypandoc==1.6.4
 pyparsing==2.4.7; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7"
 pyproj==3.2.1; python_version >= "3.7"
-pyrsistent==0.18.0; python_version >= "3.6"
+pyrsistent==0.18.0; python_version >= "3.7"
-pytest-mock==3.6.1; python_version >= "3.6"
-pytest==6.2.5; python_version >= "3.6"
 python-dateutil==2.8.2; python_full_version >= "3.7.1" and python_version >= "3.7"
-pytz==2021.1; python_full_version >= "3.7.1" and python_version >= "2.7"
+pytz==2021.3; python_full_version >= "3.7.1" and python_version >= "3.6"
-pywin32==301; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_version >= "3.6"
+pywin32==302; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_version >= "3.7"
 pywinpty==1.1.4; os_name == "nt" and python_version >= "3.6"
-pyyaml==5.4.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
+pyyaml==6.0; python_version >= "3.6"
 pyzmq==22.3.0; python_full_version >= "3.6.1" and python_version >= "3.7"
 qtconsole==5.1.1; python_version >= "3.6"
-qtpy==1.11.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+qtpy==1.11.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-regex==2021.8.28; python_full_version >= "3.6.2"
 requests==2.26.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
-safety==1.10.3; python_version >= "3.5"
-semantic-version==2.8.5; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "2.7"
 send2trash==1.8.0; python_version >= "3.6"
 shapely==1.7.1; python_version >= "3.6"
-six==1.16.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
+six==1.16.0; python_full_version >= "3.7.1" and python_version >= "3.7" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7")
 terminado==0.12.1; python_version >= "3.6"
 testpath==0.5.0; python_version >= "3.7"
-toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4.0"
+toml==0.10.2; python_version > "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.3.0" and python_version > "3.6" and python_version < "4.0"
-tomli==1.2.1; python_version >= "3.6" and python_full_version >= "3.6.2"
 tornado==6.1; python_full_version >= "3.6.1" and python_version >= "3.7"
-tox==3.24.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 tqdm==4.62.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
 traitlets==5.1.0; python_full_version >= "3.6.1" and python_version >= "3.7"
-types-requests==2.25.8
+typed-ast==1.4.3; implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6"
-typing-extensions==3.10.0.2; python_version < "3.8" and python_version >= "3.6"
+types-requests==2.25.11
-urllib3==1.26.6; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "2.7"
+typing-extensions==3.10.0.2; python_version >= "3.6" and python_version < "3.8"
+urllib3==1.26.7; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "2.7"
 us==2.0.2
-virtualenv==20.8.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
 wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.7"
 webencodings==0.5.1; python_version >= "3.7"
 widgetsnbextension==3.5.1
-wrapt==1.12.1; python_version >= "3.6" and python_version < "4.0"
+wrapt==1.13.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0"
 xlsxwriter==2.0.0
-zipp==3.5.0; python_version < "3.8" and python_version >= "3.6"
+zipp==3.6.0; python_version < "3.8" and python_version >= "3.6"

data/data-pipeline/tox.ini

@@ -7,15 +7,28 @@ skip_missing_interpreters = true
 
 [testenv:lint]
 # lints python code in src and tests
-basepython = python3.9
 deps = -rrequirements.txt
+# These are "external" because they are dev dependencies that not in
+# "requirements.txt" and therefore not installed in this env, but they will be
+# available where this is being run. See:
+# https://stackoverflow.com/questions/47642747/tox-warningtest-command-found-but-not-installed-in-testenv
+allowlist_externals = black
+                      flake8
+                      pylint
 commands = black data_pipeline
            flake8 data_pipeline
-           pylint data_pipeline
+# Ignore tests this lint check because test dependencies are not installed here.
+           pylint data_pipeline --ignore tests
 
 [testenv:checkdeps]
 # checks the dependencies for security vulnerabilities and open source licenses
 deps = -rrequirements.txt
+# These are "external" because they are dev dependencies that not in
+# "requirements.txt" and therefore not installed in this env, but they will be
+# available where this is being run. See:
+# https://stackoverflow.com/questions/47642747/tox-warningtest-command-found-but-not-installed-in-testenv
+allowlist_externals = safety
+                      liccheck
 commands = safety check
            liccheck
 

mlc_config.json

@@ -14,6 +14,9 @@
     },
     {
       "pattern": "^https://github.com/usds/environmental-justice/"
+    },
+    {
+      "pattern": "^https://.*medium.com"
     }
   ],
   "replacementPatterns": [