publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-08-05 20:24:18 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Low-Income Poverty Calculation

Browse source
This commit is contained in:
alene 2024-12-10 14:28:54 -05:00 • committed by Carlos Felix
commit 44f5aae0ca
8 changed files with 5858 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

0
data/data-pipeline/data_pipeline/tests/sources/census_acs/__init__.py Normal file
View file

25
data/data-pipeline/data_pipeline/tests/sources/census_acs/data/transform/README.md Normal file
View file

@ -0,0 +1,25 @@
# How to generate the sample data in this folder
The sample data in this folder can be easily generated by debugging the `data_pipeline/etl/sources/census_acs/etl.py` file
and exporting data using the debugger console. Examples of this exporting are below.
## Why in pickle format?
Exporting as a Pickle file keeps all the metadata about the columns including the data types. If we were to export as CSV then we will need
to code the data types in the test fixtures for all the columns for the comparison to be correct.
### Transform tests input files
- `acs_transform_input.pkl` - this file contains downloaded Census data that is used as input.
- `acs_transform_geojson.geojson` - this file contains the Census GeoJSON data that is used as input.
1. Place a breakpoint in `data_pipeline/etl/sources/census_acs/etl.py` in the `transform` method right at the beginning
and start the debugger running the ETL run command for Census ACS (`etl-run -d census_acs`).
1. Partially export the `self.df` and `self.geo_df` data to files once the debugger pauses at the breakpoint. Use these
sample commands in the debugger console.
```python
t_list = ['01073001100', '01073001400', '01073002000', '01073003802', '01073004000']
self.geo_df[self.geo_df['GEOID10'].isin(t_list)].to_file('data_pipeline/tests/sources/census_acs/data/transform/acs_transform_geojson.geojson')
test_df = self.df[self.df['GEOID10_TRACT'].isin(t_list)].copy()
# Setting this one row to N/A allows the imputations code to succeed
test_df.at[4, self.OFFCAMPUS_UNIVERSITY_BELOW_POVERTY_UNDERGRADUATE] = pd.NA
test_df.to_pickle('data_pipeline/tests/sources/census_acs/data/transform/acs_transform_input.pkl')
```

11
data/data-pipeline/data_pipeline/tests/sources/census_acs/data/transform/acs_transform_geojson.geojson Normal file
View file

@ -0,0 +1,11 @@
{
"type": "FeatureCollection",
"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } },
"features": [
{ "type": "Feature", "properties": { "STATEFP10": "01", "COUNTYFP10": "073", "TRACTCE10": "001400", "GEOID10": "01073001400", "NAME10": "14", "NAMELSAD10": "Census Tract 14", "MTFCC10": "G5020", "FUNCSTAT10": "S", "ALAND10": 2241287, "AWATER10": 0, "INTPTLAT10": "+33.5261497", "INTPTLON10": "-086.8351469" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -86.840884, 33.527586 ], [ -86.837824, 33.528871 ], [ -86.83693, 33.530023 ], [ -86.83679, 33.530205 ], [ -86.83639, 33.530805 ], [ -86.835352, 33.531179 ], [ -86.833843, 33.532003 ], [ -86.832035, 33.532595 ], [ -86.831245, 33.532854 ], [ -86.829135, 33.533924 ], [ -86.827029, 33.534708 ], [ -86.826575, 33.534968 ], [ -86.825634, 33.535508 ], [ -86.824369, 33.536271 ], [ -86.823926, 33.536515 ], [ -86.823758, 33.536606 ], [ -86.823683, 33.536184 ], [ -86.823591, 33.535542 ], [ -86.823596, 33.53486 ], [ -86.823637, 33.534404 ], [ -86.823739, 33.533793 ], [ -86.82384, 33.533389 ], [ -86.824083, 33.532657 ], [ -86.824469, 33.531855 ], [ -86.825589, 33.530001 ], [ -86.825945, 33.529412 ], [ -86.826278, 33.528861 ], [ -86.82662, 33.528272 ], [ -86.826703, 33.528081 ], [ -86.826785, 33.5278 ], [ -86.826911, 33.527375 ], [ -86.827001, 33.526685 ], [ -86.827025, 33.525543 ], [ -86.827091, 33.524614 ], [ -86.827086, 33.523811 ], [ -86.827086, 33.523767 ], [ -86.827091, 33.523677 ], [ -86.827125, 33.523051 ], [ -86.827117, 33.522607 ], [ -86.827088, 33.522283 ], [ -86.827006, 33.521836 ], [ -86.826987, 33.521701 ], [ -86.826807, 33.521107 ], [ -86.829003, 33.520829 ], [ -86.829532, 33.520498 ], [ -86.830151, 33.52015 ], [ -86.830507, 33.520037 ], [ -86.830666, 33.519986 ], [ -86.831415, 33.519802 ], [ -86.831968, 33.519782 ], [ -86.83261, 33.519835 ], [ -86.834123, 33.520139 ], [ -86.834279, 33.52017 ], [ -86.834582, 33.520223 ], [ -86.836148, 33.520497 ], [ -86.837263, 33.520682 ], [ -86.837708, 33.52073 ], [ -86.837983, 33.520761 ], [ -86.838571, 33.520791 ], [ -86.839295, 33.520792 ], [ -86.83942, 33.520792 ], [ -86.839596, 33.520792 ], [ -86.840901, 33.520808 ], [ -86.842445, 33.520826 ], [ -86.843989, 33.520846 ], [ -86.845617, 33.520865 ], [ -86.846194, 33.520873 ], [ -86.84942, 33.520908 ], [ -86.849421, 33.521133 ], [ -86.84944, 33.521596 ], [ -86.849499, 33.521692 ], [ -86.849433, 33.522393 ], [ -86.849582, 33.523085 ], [ -86.849667, 33.523435 ], [ -86.849748, 33.523804 ], [ -86.849783, 33.523978 ], [ -86.849812, 33.52412 ], [ -86.84985, 33.52431 ], [ -86.849984, 33.524884 ], [ -86.850106, 33.525442 ], [ -86.85023, 33.525973 ], [ -86.850303, 33.526624 ], [ -86.850312, 33.526707 ], [ -86.850132, 33.526701 ], [ -86.84979, 33.526605 ], [ -86.849721, 33.526576 ], [ -86.849042, 33.526293 ], [ -86.848434, 33.526035 ], [ -86.847961, 33.525834 ], [ -86.847891, 33.525805 ], [ -86.847787, 33.525787 ], [ -86.845591, 33.525405 ], [ -86.842991, 33.526505 ], [ -86.840884, 33.527586 ] ] ] } },
{ "type": "Feature", "properties": { "STATEFP10": "01", "COUNTYFP10": "073", "TRACTCE10": "001100", "GEOID10": "01073001100", "NAME10": "11", "NAMELSAD10": "Census Tract 11", "MTFCC10": "G5020", "FUNCSTAT10": "S", "ALAND10": 11428313, "AWATER10": 3862, "INTPTLAT10": "+33.5423337", "INTPTLON10": "-086.8765161" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -86.882435, 33.552326 ], [ -86.881871, 33.552684 ], [ -86.881177, 33.552987 ], [ -86.880549, 33.553159 ], [ -86.880024, 33.553247 ], [ -86.879772, 33.553268 ], [ -86.879039, 33.553269 ], [ -86.877437, 33.553193 ], [ -86.873225, 33.553055 ], [ -86.871055, 33.55294 ], [ -86.870488, 33.552953 ], [ -86.870028, 33.552996 ], [ -86.869495, 33.553081 ], [ -86.86902, 33.553198 ], [ -86.868549, 33.553418 ], [ -86.868064, 33.553783 ], [ -86.867614, 33.554299 ], [ -86.867234, 33.554811 ], [ -86.866179, 33.556229 ], [ -86.865886, 33.556531 ], [ -86.865428, 33.556909 ], [ -86.8651, 33.557135 ], [ -86.863343, 33.558141 ], [ -86.861725, 33.55911 ], [ -86.860721, 33.559702 ], [ -86.859731, 33.560316 ], [ -86.858524, 33.561035 ], [ -86.858002, 33.561372 ], [ -86.857324, 33.561719 ], [ -86.856471, 33.562004 ], [ -86.856129, 33.562064 ], [ -86.851595, 33.559404 ], [ -86.848891, 33.559805 ], [ -86.852991, 33.552104 ], [ -86.853223, 33.551239 ], [ -86.855191, 33.543905 ], [ -86.855694, 33.538151 ], [ -86.85606, 33.537889 ], [ -86.856269, 33.537719 ], [ -86.856472, 33.53752 ], [ -86.856779, 33.53719 ], [ -86.857174, 33.536763 ], [ -86.85758, 33.536346 ], [ -86.857795, 33.536127 ], [ -86.858137, 33.535851 ], [ -86.858444, 33.535632 ], [ -86.858851, 33.53539 ], [ -86.85921, 33.535203 ], [ -86.860817, 33.53405 ], [ -86.860936, 33.534028 ], [ -86.864589, 33.533319 ], [ -86.86658, 33.532927 ], [ -86.867357, 33.53278 ], [ -86.867868, 33.532651 ], [ -86.868475, 33.53246 ], [ -86.870603, 33.531691 ], [ -86.870854, 33.531601 ], [ -86.875846, 33.529909 ], [ -86.878638, 33.528917 ], [ -86.878921, 33.528817 ], [ -86.879007, 33.528787 ], [ -86.878999, 33.529822 ], [ -86.8796, 33.529851 ], [ -86.88017, 33.529879 ], [ -86.880793, 33.529909 ], [ -86.880872, 33.529919 ], [ -86.881324, 33.529935 ], [ -86.88209, 33.529947 ], [ -86.882375, 33.529953 ], [ -86.882779, 33.529958 ], [ -86.88349, 33.529994 ], [ -86.884213, 33.530005 ], [ -86.885074, 33.530017 ], [ -86.885649, 33.530026 ], [ -86.885884, 33.530034 ], [ -86.886535, 33.530037 ], [ -86.887886, 33.530039 ], [ -86.887968, 33.530043 ], [ -86.889407, 33.530064 ], [ -86.889415, 33.529614 ], [ -86.889422, 33.529159 ], [ -86.890877, 33.529167 ], [ -86.891569, 33.52917 ], [ -86.891643, 33.52917 ], [ -86.891832, 33.528964 ], [ -86.892208, 33.528556 ], [ -86.893389, 33.527586 ], [ -86.894533, 33.526558 ], [ -86.897192, 33.528505 ], [ -86.896992, 33.530504 ], [ -86.901104, 33.532548 ], [ -86.903492, 33.533105 ], [ -86.908792, 33.535505 ], [ -86.910592, 33.536105 ], [ -86.909292, 33.538305 ], [ -86.909392, 33.539005 ], [ -86.908162, 33.539693 ], [ -86.908095, 33.539893 ], [ -86.907692, 33.540605 ], [ -86.907192, 33.542205 ], [ -86.905392, 33.541305 ], [ -86.903103, 33.542516 ], [ -86.901996, 33.543221 ], [ -86.901608, 33.543456 ], [ -86.901167, 33.54412 ], [ -86.900913, 33.544384 ], [ -86.899614, 33.545658 ], [ -86.899456, 33.545794 ], [ -86.896838, 33.547888 ], [ -86.896206, 33.548394 ], [ -86.895817, 33.548705 ], [ -86.895692, 33.548805 ], [ -86.894818, 33.548368 ], [ -86.894292, 33.548105 ], [ -86.893192, 33.548205 ], [ -86.893149, 33.548341 ], [ -86.892544, 33.550241 ], [ -86.892492, 33.550404 ], [ -86.889392, 33.550505 ], [ -86.888392, 33.549304 ], [ -86.886592, 33.550205 ], [ -86.884997, 33.550205 ], [ -86.8849, 33.551105 ], [ -86.883737, 33.551726 ], [ -86.883615, 33.551775 ], [ -86.883463, 33.551713 ], [ -86.883374, 33.551707 ], [ -86.883263, 33.551733 ], [ -86.883102, 33.551828 ], [ -86.882615, 33.552213 ], [ -86.882435, 33.552326 ] ] ] } },
{ "type": "Feature", "properties": { "STATEFP10": "01", "COUNTYFP10": "073", "TRACTCE10": "002000", "GEOID10": "01073002000", "NAME10": "20", "NAMELSAD10": "Census Tract 20", "MTFCC10": "G5020", "FUNCSTAT10": "S", "ALAND10": 3605025, "AWATER10": 0, "INTPTLAT10": "+33.5591908", "INTPTLON10": "-086.7233518" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -86.713902, 33.539302 ], [ -86.714347, 33.539121 ], [ -86.714768, 33.539008 ], [ -86.715254, 33.538933 ], [ -86.715891, 33.538922 ], [ -86.716497, 33.538825 ], [ -86.716288, 33.539004 ], [ -86.719691, 33.540211 ], [ -86.720436, 33.543861 ], [ -86.72065, 33.544904 ], [ -86.72064, 33.545552 ], [ -86.720624, 33.545854 ], [ -86.720641, 33.546602 ], [ -86.720666, 33.547704 ], [ -86.720671, 33.548361 ], [ -86.720685, 33.549495 ], [ -86.720702, 33.550497 ], [ -86.720715, 33.551287 ], [ -86.720184, 33.551274 ], [ -86.720099, 33.551271 ], [ -86.719672, 33.551264 ], [ -86.719527, 33.551266 ], [ -86.718937, 33.55126 ], [ -86.718379, 33.551245 ], [ -86.717792, 33.551257 ], [ -86.717277, 33.551242 ], [ -86.716713, 33.551233 ], [ -86.716142, 33.551223 ], [ -86.715577, 33.551214 ], [ -86.715006, 33.5512 ], [ -86.714442, 33.551191 ], [ -86.713877, 33.551181 ], [ -86.713881, 33.552963 ], [ -86.713323, 33.552953 ], [ -86.712752, 33.552939 ], [ -86.71275, 33.553417 ], [ -86.712753, 33.554307 ], [ -86.713324, 33.554333 ], [ -86.713888, 33.554326 ], [ -86.714462, 33.55435 ], [ -86.71507, 33.554366 ], [ -86.715588, 33.554385 ], [ -86.71612, 33.554384 ], [ -86.716664, 33.554437 ], [ -86.717222, 33.554653 ], [ -86.717741, 33.554882 ], [ -86.71829, 33.55513 ], [ -86.718989, 33.555448 ], [ -86.719231, 33.555565 ], [ -86.719603, 33.555718 ], [ -86.72003, 33.555922 ], [ -86.720245, 33.556011 ], [ -86.720876, 33.556274 ], [ -86.721511, 33.556456 ], [ -86.7218, 33.556658 ], [ -86.722005, 33.556801 ], [ -86.722554, 33.557118 ], [ -86.723614, 33.556025 ], [ -86.723496, 33.555943 ], [ -86.723105, 33.55567 ], [ -86.72257, 33.555315 ], [ -86.723206, 33.554639 ], [ -86.723346, 33.554398 ], [ -86.72355, 33.554328 ], [ -86.723773, 33.55433 ], [ -86.724149, 33.554569 ], [ -86.724693, 33.554905 ], [ -86.725189, 33.555287 ], [ -86.725711, 33.555637 ], [ -86.726233, 33.555992 ], [ -86.726803, 33.556395 ], [ -86.727325, 33.556739 ], [ -86.727847, 33.557094 ], [ -86.728377, 33.557456 ], [ -86.728891, 33.557804 ], [ -86.729406, 33.558156 ], [ -86.729942, 33.55852 ], [ -86.730464, 33.558875 ], [ -86.731032, 33.559263 ], [ -86.729985, 33.560323 ], [ -86.728915, 33.561436 ], [ -86.728813, 33.56155 ], [ -86.727701, 33.563268 ], [ -86.726796, 33.564729 ], [ -86.72602, 33.565979 ], [ -86.725629, 33.56581 ], [ -86.725297, 33.565678 ], [ -86.725149, 33.565606 ], [ -86.724716, 33.565426 ], [ -86.724148, 33.565125 ], [ -86.72354, 33.564846 ], [ -86.722958, 33.56459 ], [ -86.722363, 33.564322 ], [ -86.721769, 33.564055 ], [ -86.721173, 33.563787 ], [ -86.720598, 33.563519 ], [ -86.719989, 33.563246 ], [ -86.719394, 33.562973 ], [ -86.718812, 33.562717 ], [ -86.718355, 33.56251 ], [ -86.718218, 33.562438 ], [ -86.717693, 33.562221 ], [ -86.717621, 33.562149 ], [ -86.717117, 33.56196 ], [ -86.716665, 33.56175 ], [ -86.716194, 33.561544 ], [ -86.715728, 33.561336 ], [ -86.715252, 33.56112 ], [ -86.714781, 33.560924 ], [ -86.714634, 33.561079 ], [ -86.714179, 33.560881 ], [ -86.712726, 33.560236 ], [ -86.712584, 33.560436 ], [ -86.710637, 33.559646 ], [ -86.709203, 33.559076 ], [ -86.708861, 33.558979 ], [ -86.708777, 33.558949 ], [ -86.707958, 33.558616 ], [ -86.707954, 33.558597 ], [ -86.707908, 33.558385 ], [ -86.707745, 33.558329 ], [ -86.70031, 33.555477 ], [ -86.700624, 33.554907 ], [ -86.701888, 33.552604 ], [ -86.702782, 33.55136 ], [ -86.703551, 33.550291 ], [ -86.703873, 33.549843 ], [ -86.704188, 33.549404 ], [ -86.707152, 33.54685 ], [ -86.707466, 33.546616 ], [ -86.707683, 33.546405 ], [ -86.707985, 33.545992 ], [ -86.708486, 33.54538 ], [ -86.709318, 33.544365 ], [ -86.709708, 33.543917 ], [ -86.712316, 33.541302 ], [ -86.712613, 33.541023 ], [ -86.712812, 33.540799 ], [ -86.713745, 33.539422 ], [ -86.713902, 33.539302 ] ] ] } },
{ "type": "Feature", "properties": { "STATEFP10": "01", "COUNTYFP10": "073", "TRACTCE10": "004000", "GEOID10": "01073004000", "NAME10": "40", "NAMELSAD10": "Census Tract 40", "MTFCC10": "G5020", "FUNCSTAT10": "S", "ALAND10": 2364675, "AWATER10": 0, "INTPTLAT10": "+33.4953245", "INTPTLON10": "-086.8516236" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -86.854631, 33.487542 ], [ -86.855535, 33.486967 ], [ -86.856538, 33.486322 ], [ -86.857046, 33.485979 ], [ -86.859584, 33.484308 ], [ -86.860059, 33.484021 ], [ -86.86053, 33.483714 ], [ -86.860755, 33.484005 ], [ -86.861035, 33.484352 ], [ -86.861374, 33.484803 ], [ -86.861571, 33.485078 ], [ -86.861742, 33.485286 ], [ -86.862107, 33.48576 ], [ -86.862507, 33.486283 ], [ -86.862427, 33.486486 ], [ -86.861595, 33.487797 ], [ -86.861383, 33.488126 ], [ -86.861291, 33.488236 ], [ -86.861169, 33.488332 ], [ -86.860211, 33.488939 ], [ -86.859816, 33.489189 ], [ -86.859266, 33.489539 ], [ -86.85833, 33.490136 ], [ -86.857567, 33.490628 ], [ -86.856969, 33.491013 ], [ -86.857431, 33.491536 ], [ -86.857855, 33.492015 ], [ -86.858305, 33.492492 ], [ -86.858765, 33.492994 ], [ -86.859221, 33.493495 ], [ -86.859706, 33.494023 ], [ -86.860181, 33.494549 ], [ -86.860593, 33.49501 ], [ -86.860637, 33.495047 ], [ -86.861002, 33.495451 ], [ -86.861354, 33.495838 ], [ -86.861804, 33.495554 ], [ -86.861962, 33.495725 ], [ -86.862138, 33.495924 ], [ -86.862286, 33.496091 ], [ -86.862342, 33.496145 ], [ -86.862401, 33.496183 ], [ -86.86248, 33.496203 ], [ -86.862523, 33.496203 ], [ -86.862657, 33.496178 ], [ -86.862747, 33.496129 ], [ -86.865424, 33.494422 ], [ -86.865495, 33.494501 ], [ -86.865766, 33.494783 ], [ -86.86583, 33.49488 ], [ -86.865848, 33.494895 ], [ -86.865876, 33.494905 ], [ -86.8659, 33.494906 ], [ -86.865968, 33.494895 ], [ -86.866332, 33.494779 ], [ -86.866881, 33.494588 ], [ -86.866885, 33.495165 ], [ -86.866882, 33.495367 ], [ -86.862491, 33.497006 ], [ -86.860903, 33.497855 ], [ -86.856845, 33.500199 ], [ -86.854735, 33.500131 ], [ -86.853969, 33.50011 ], [ -86.852812, 33.500079 ], [ -86.851686, 33.500051 ], [ -86.851194, 33.500034 ], [ -86.850371, 33.500006 ], [ -86.849077, 33.499969 ], [ -86.848377, 33.499947 ], [ -86.848378, 33.500321 ], [ -86.848378, 33.500493 ], [ -86.848378, 33.500678 ], [ -86.84838, 33.50083 ], [ -86.848379, 33.501047 ], [ -86.848371, 33.501387 ], [ -86.848002, 33.501458 ], [ -86.847752, 33.50152 ], [ -86.847574, 33.501633 ], [ -86.847191, 33.501852 ], [ -86.847057, 33.501953 ], [ -86.847392, 33.502364 ], [ -86.847897, 33.502965 ], [ -86.848046, 33.503141 ], [ -86.848257, 33.503394 ], [ -86.848336, 33.503486 ], [ -86.848157, 33.503554 ], [ -86.847219, 33.503858 ], [ -86.846292, 33.504089 ], [ -86.843699, 33.504587 ], [ -86.843203, 33.504334 ], [ -86.843156, 33.504274 ], [ -86.842845, 33.503874 ], [ -86.842556, 33.503511 ], [ -86.842525, 33.503473 ], [ -86.842194, 33.503061 ], [ -86.841884, 33.502679 ], [ -86.841545, 33.50226 ], [ -86.84134, 33.502009 ], [ -86.841222, 33.501863 ], [ -86.840803, 33.50135 ], [ -86.840537, 33.501444 ], [ -86.840278, 33.501514 ], [ -86.840164, 33.501534 ], [ -86.839567, 33.50155 ], [ -86.839587, 33.500984 ], [ -86.839566, 33.500861 ], [ -86.8395, 33.500695 ], [ -86.839428, 33.500655 ], [ -86.839565, 33.500597 ], [ -86.839661, 33.500445 ], [ -86.839589, 33.500419 ], [ -86.839586, 33.499992 ], [ -86.839591, 33.499212 ], [ -86.83959, 33.498823 ], [ -86.839596, 33.498071 ], [ -86.839592, 33.497372 ], [ -86.839593, 33.496633 ], [ -86.840423, 33.496212 ], [ -86.841274, 33.495781 ], [ -86.842156, 33.495302 ], [ -86.843044, 33.494824 ], [ -86.844948, 33.493727 ], [ -86.846142, 33.493001 ], [ -86.84733, 33.492275 ], [ -86.849147, 33.49107 ], [ -86.85242, 33.488964 ], [ -86.85329, 33.488405 ], [ -86.853724, 33.488135 ], [ -86.854431, 33.48767 ], [ -86.854631, 33.487542 ] ] ] } },
{ "type": "Feature", "properties": { "STATEFP10": "01", "COUNTYFP10": "073", "TRACTCE10": "003802", "GEOID10": "01073003802", "NAME10": "38.02", "NAMELSAD10": "Census Tract 38.02", "MTFCC10": "G5020", "FUNCSTAT10": "S", "ALAND10": 3245083, "AWATER10": 0, "INTPTLAT10": "+33.4785702", "INTPTLON10": "-086.8900020" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -86.903171, 33.471768 ], [ -86.902839, 33.472127 ], [ -86.902225, 33.47278 ], [ -86.901954, 33.473078 ], [ -86.901641, 33.473416 ], [ -86.90141, 33.473658 ], [ -86.900796, 33.474323 ], [ -86.900673, 33.474449 ], [ -86.898775, 33.47648 ], [ -86.898283, 33.477004 ], [ -86.897626, 33.477708 ], [ -86.896539, 33.478865 ], [ -86.896263, 33.479165 ], [ -86.895093, 33.48043 ], [ -86.894581, 33.480994 ], [ -86.894456, 33.481132 ], [ -86.894021, 33.481637 ], [ -86.893961, 33.48169 ], [ -86.893382, 33.482321 ], [ -86.893258, 33.482453 ], [ -86.892746, 33.482886 ], [ -86.892337, 33.483302 ], [ -86.891263, 33.484422 ], [ -86.889966, 33.485827 ], [ -86.888649, 33.487226 ], [ -86.887767, 33.488186 ], [ -86.887057, 33.488954 ], [ -86.886728, 33.489294 ], [ -86.886305, 33.489638 ], [ -86.885219, 33.490482 ], [ -86.884374, 33.491114 ], [ -86.883539, 33.491751 ], [ -86.882698, 33.492396 ], [ -86.881861, 33.493024 ], [ -86.881309, 33.49345 ], [ -86.881062, 33.49364 ], [ -86.880805, 33.493833 ], [ -86.880411, 33.494131 ], [ -86.88002, 33.494431 ], [ -86.879626, 33.494732 ], [ -86.878975, 33.495227 ], [ -86.878498, 33.494588 ], [ -86.878186, 33.494157 ], [ -86.87756, 33.493268 ], [ -86.877271, 33.49286 ], [ -86.876969, 33.492421 ], [ -86.876655, 33.491999 ], [ -86.87634, 33.491571 ], [ -86.876027, 33.491146 ], [ -86.875717, 33.49072 ], [ -86.875757, 33.490693 ], [ -86.876011, 33.49057 ], [ -86.876337, 33.490403 ], [ -86.876675, 33.490237 ], [ -86.877239, 33.489948 ], [ -86.877115, 33.489769 ], [ -86.87693, 33.489514 ], [ -86.876672, 33.48915 ], [ -86.876629, 33.48908 ], [ -86.876559, 33.488988 ], [ -86.87702, 33.488297 ], [ -86.878015, 33.486803 ], [ -86.878888, 33.485874 ], [ -86.879365, 33.485176 ], [ -86.879779, 33.484583 ], [ -86.880179, 33.484014 ], [ -86.880814, 33.483217 ], [ -86.884791, 33.478206 ], [ -86.889113, 33.472352 ], [ -86.889455, 33.47189 ], [ -86.890077, 33.471066 ], [ -86.89052, 33.47046 ], [ -86.890747, 33.470166 ], [ -86.891276, 33.469468 ], [ -86.891699, 33.468898 ], [ -86.892196, 33.4683 ], [ -86.895724, 33.464729 ], [ -86.897917, 33.46251 ], [ -86.899914, 33.460456 ], [ -86.90018, 33.460195 ], [ -86.900652, 33.459741 ], [ -86.900614, 33.460009 ], [ -86.900652, 33.460414 ], [ -86.900791, 33.461207 ], [ -86.900777, 33.463217 ], [ -86.900792, 33.464061 ], [ -86.900791, 33.464861 ], [ -86.900368, 33.466247 ], [ -86.900487, 33.467212 ], [ -86.900642, 33.468474 ], [ -86.900602, 33.469456 ], [ -86.900561, 33.471007 ], [ -86.900554, 33.471353 ], [ -86.900515, 33.471735 ], [ -86.902983, 33.471766 ], [ -86.903171, 33.471768 ] ] ] } }
]
}

BIN
data/data-pipeline/data_pipeline/tests/sources/census_acs/data/transform/acs_transform_input.pkl Normal file
View file

Binary file not shown.

119
data/data-pipeline/data_pipeline/tests/sources/census_acs/test_etl.py Normal file
View file

@ -0,0 +1,119 @@
import pytest
import pandas as pd
import geopandas as gpd
from pathlib import Path
from data_pipeline.etl.sources.census_acs.etl import CensusACSETL
def _check_fields_exist(df: pd.DataFrame, field_names: list):
for field in field_names:
assert field in df.columns
@pytest.fixture
def transform_census_input_fixture() -> pd.DataFrame:
"""
Load the Census input data for the transform method tests.
Returns:
Dataframe: the input data
"""
file = (
Path(__file__).parents[0]
/ "data"
/ "transform"
/ "acs_transform_input.pkl"
)
return pd.read_pickle(file)
@pytest.fixture
def transform_census_geojson_fixture() -> gpd.GeoDataFrame:
"""
Load the Census GeoJSON input data for the transform method.
Returns:
Dataframe: the Census GeoJSON input data
"""
file = (
Path(__file__).parents[0]
/ "data"
/ "transform"
/ "acs_transform_geojson.geojson"
)
return gpd.read_file(file)
@pytest.fixture
def transformed_data_fixture(
transform_census_input_fixture: pd.DataFrame,
transform_census_geojson_fixture: gpd.GeoDataFrame,
) -> pd.DataFrame:
"""
Tranform the test input data.
Returns:
DataFrame: the transformed data
"""
acs = CensusACSETL()
acs.df = transform_census_input_fixture
acs.geo_df = transform_census_geojson_fixture
acs.transform()
return acs.output_df
#################
# Transform tests
#################
def test_poverty_fields(transformed_data_fixture: pd.DataFrame):
result = transformed_data_fixture
# Test that the poverty and collect fields were added.
acs = CensusACSETL()
fields_to_test = [
acs.POVERTY_LESS_THAN_200_PERCENT_FPL_COUNT_FIELD_NAME,
acs.POVERTY_LESS_THAN_100_PERCENT_FPL_COUNT_FIELD_NAME,
]
_check_fields_exist(result, fields_to_test)
assert (
result.iloc[0][acs.POVERTY_LESS_THAN_200_PERCENT_FPL_COUNT_FIELD_NAME]
== 1743
)
assert (
result.iloc[0][acs.POVERTY_LESS_THAN_100_PERCENT_FPL_COUNT_FIELD_NAME]
== 700
)
assert (
result.iloc[1][acs.POVERTY_LESS_THAN_200_PERCENT_FPL_COUNT_FIELD_NAME]
== 941
)
assert (
result.iloc[1][acs.POVERTY_LESS_THAN_100_PERCENT_FPL_COUNT_FIELD_NAME]
== 548
)
def test_college_undergrad_fields(transformed_data_fixture: pd.DataFrame):
result = transformed_data_fixture
# Test that the poverty and collect fields were added.
acs = CensusACSETL()
fields_to_test = [
acs.OFFCAMPUS_UNDERGRADUATE_POVERTY_FIELD,
acs.OFFCAMPUS_UNDERGRADUATE_FIELD,
acs.OFFCAMPUS_UNIVERSITY_POVERTY_FIELD,
acs.OFFCAMPUS_UNIVERSITY_FIELD,
]
_check_fields_exist(result, fields_to_test)
assert result.iloc[0][acs.OFFCAMPUS_UNDERGRADUATE_POVERTY_FIELD] == 0
assert result.iloc[0][acs.OFFCAMPUS_UNDERGRADUATE_FIELD] == 296
assert result.iloc[0][acs.OFFCAMPUS_UNIVERSITY_POVERTY_FIELD] == 44
assert result.iloc[0][acs.OFFCAMPUS_UNIVERSITY_FIELD] == 340
assert result.iloc[1][acs.OFFCAMPUS_UNDERGRADUATE_POVERTY_FIELD] == 45
assert result.iloc[1][acs.OFFCAMPUS_UNDERGRADUATE_FIELD] == 97
assert result.iloc[1][acs.OFFCAMPUS_UNIVERSITY_POVERTY_FIELD] == 45
assert result.iloc[1][acs.OFFCAMPUS_UNIVERSITY_FIELD] == 128

0
data/data-pipeline/data_pipeline/tests/sources/census_decennial/__init__.py Normal file
View file