publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-08-06 02:24:18 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Starting etl for score (#141)

Browse source 
* starting etl for score

* projection fix

* projection flags
This commit is contained in:
Jorge Escobar 2021-06-16 13:47:27 -04:00 • committed by GitHub
commit 244b3663d1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 385 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

8
score/scripts/download_cbg.py
View file

@ -10,7 +10,7 @@ data_path = Path.cwd() / "data"
with requests.Session() as s:
# the fips_states_2010.csv is generated from data here
# https://www.census.gov/geographies/reference-files/time-series/geo/tallies.html
fips_csv_path = data_path.joinpath("fips_states_2010.csv")
fips_csv_path = data_path / "fips_states_2010.csv"
with open(fips_csv_path) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=",")
line_count = 0
@ -27,10 +27,12 @@ with requests.Session() as s:
if not os.path.isfile(shp_file_path):
print(f"downloading {row[1]}")
# 2020 tiger data is here: https://www2.census.gov/geo/tiger/TIGER2020/BG/
# But using 2010 for now
cbg_state_url = f"https://www2.census.gov/geo/tiger/TIGER2010/BG/2010/tl_2010_{fips}_bg10.zip"
download = s.get(cbg_state_url)
file_contents = download.content
zip_file_path = data_path.joinpath("census", "downloaded.zip")
zip_file_path = data_path / "census" / "downloaded.zip"
zip_file = open(zip_file_path, "wb")
zip_file.write(file_contents)
zip_file.close()
@ -38,7 +40,7 @@ with requests.Session() as s:
print(f"extracting {row[1]}")
with zipfile.ZipFile(zip_file_path, "r") as zip_ref:
shp_dir_path = data_path.joinpath("census", "shp", fips)
shp_dir_path = data_path / "census" / "shp" / fips
zip_ref.extractall(shp_dir_path)
geojson_dir_path = data_path.joinpath(

6
score/scripts/generate_mbtiles.py
View file

@ -22,8 +22,6 @@ for file in os.listdir(geojson_path):
if file.endswith(".json"):
geojson_list += f"/home/data/census/geojson/{file} "
breakpoint()
if geojson_list == "":
print("No GeoJson files found. Please run download_cbg.py first")
@ -35,7 +33,7 @@ else:
cmd = (
'docker run --rm -it -v "'
+ pwd
+ '"/:/home klokantech/tippecanoe tippecanoe -zg -o /home/data/tiles/block2010.mbtiles --drop-densest-as-needed --extend-zooms-if-still-dropping -l cbg2010 '
+ '"/:/home klokantech/tippecanoe tippecanoe -s_srs EPSG:4269 -t_srs EPSG:4326 --drop-densest-as-needed -zg -o /home/data/tiles/block2010.mbtiles --drop-densest-as-needed --extend-zooms-if-still-dropping -l cbg2010 -s_srs EPSG:4269 -t_srs EPSG:4326 '
+ geojson_list
)
print(cmd)
@ -51,7 +49,7 @@ else:
cmd = (
'docker run --rm -it -v "'
+ pwd
+ '"/:/home klokantech/tippecanoe tippecanoe --no-tile-compression -zg -e /home/data/tiles/mvt /home/data/census/geojson/01.json '
+ '"/:/home klokantech/tippecanoe tippecanoe --drop-densest-as-needed --no-tile-compression -zg -e /home/data/tiles/mvt '
+ geojson_list
)
print(cmd)