mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-02-22 09:41:26 -08:00
Fargate Serverless Workers for Census Data Enrichment and Tile Generation (#230)
* add basic infrastructure
* add cloudfront distribution
* WIP checkpoint
* add ecs cluster
* add conditions and route53 dns entry to cloudfront
* WIP checkin
* Added a raw execution mode for demo/testing
* Add pre-defined Task for ogr2ogr
* Tweak Task Definition name
* Mostly working except for logging error
* Add additional logging permissions
* Succesfully executed ogr2ogr in fargate. S3 permissions needs to be addresses
* Add multipart permissions
* Add a few more actions
* Put IAM Policy on the correct resource
* Deploy lambda and update events
* fix iam permissions 🤦🏻♂️
* Add reference to Tippecanoe container
* Clean up to only use named actions
* Refactor resources to include support for tippecanoe
* Make a more interesting GDAL command
* Pull all ECS variables into environment file; successful test of running tippecanoe container
* Support pre/post commands
* Refactor codebase and enable linting
* Implement many-to-many enrichment between USDS CSV files and Census zipped shapefiles
* Change the GDAL image to one with the built-in drivers
* Add some additional fixes to support the enrichment use case
* Clean up old hello-world example
* Expand the README to include ways to execute the lambdas
* Validate scheduled lambda execution and then comment out
Co-authored-by: Tim Zwolak <timothypage@gmail.com>
This commit is contained in:
Lucas Scharenbroich
GitHub
parent
92efc5c937
commit
38fff9cea8
27 changed files with 7271 additions and 0 deletions
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -139,3 +139,9 @@ score/data/tiles
|
|||
score/data/tmp
|
||||
score/data/dataset
|
||||
score/data/score
|
||||
|
||||
# node
|
||||
node_modules
|
||||
|
||||
# serverless
|
||||
.serverless
|
||||
|
|
|
|||
40
infrastructure/README.md
Normal file
40
infrastructure/README.md
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
## create acm certificate
|
||||
|
||||
This only needs to be run once for the `sit` environment. stg and prd, we're assuming some other certificate arn will be used
|
||||
|
||||
npx serverless create-cert
|
||||
|
||||
you'll have to grab the arn of the certificate from the log output or go into the console to get it, looks like the plugin doesn't work any more. Set CLOUDFRONT_CERTIFICATE_ARN in sit to that value
|
||||
|
||||
## deploy
|
||||
|
||||
sls deploy --aws-profile geoplatform --stage sit --verbose
|
||||
|
||||
If it's the first time deploying, you'll have to create a dns entry that points to the cloudfront distribution.
|
||||
|
||||
## testing
|
||||
|
||||
The examples can be run several different ways
|
||||
|
||||
### local
|
||||
|
||||
The `package.json` file incluses several examples to run against the local source code. The actual
|
||||
tasks will execute within AWS, so an `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` must be set in
|
||||
the `test.env` file.
|
||||
|
||||
```bash
|
||||
$ cd ./functions/detect-changes-for-worker
|
||||
$ npm run test:gdal
|
||||
```
|
||||
|
||||
### lambda invoke
|
||||
|
||||
The deployed lambda functions can be directly invoked with the `serverless invoke` function.
|
||||
|
||||
```bash
|
||||
$ cat ./functions/detect-changes-for-worker/events/gdal.json | sls invoke -s sit -f DetectChangesForWorker
|
||||
```
|
||||
|
||||
New event files can be created to perform one-off data processes.
|
||||
|
||||
|
||||
11
infrastructure/conditions.yml
Normal file
11
infrastructure/conditions.yml
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
Conditions:
|
||||
|
||||
ShouldOnlyCreateResourcesInSIT:
|
||||
Fn::Equals:
|
||||
- ${self:provider.stage}
|
||||
- sit
|
||||
|
||||
ShouldOnlyCreateResourcesInPRD:
|
||||
Fn::Equals:
|
||||
- ${self:provider.stage}
|
||||
- prd
|
||||
37
infrastructure/environment.yml
Normal file
37
infrastructure/environment.yml
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
sit:
|
||||
DEPLOYMENT_BUCKET_PREFIX: ${self:custom.namespaceShort}
|
||||
STACK_NAME_PREFIX: "${self:custom.namespaceShort}-"
|
||||
DATA_BUCKET: ${self:custom.environment.DEPLOYMENT_BUCKET_PREFIX}-${self:provider.stage}-${self:service}-data
|
||||
SHOULD_CREATE_SSL_CERTIFICATE: true
|
||||
HOSTED_ZONE_ID_DOMAIN: Z104704314NAAG3GV4SN1
|
||||
HOSTED_ZONE_SUBDOMAIN: ${self:provider.stage}-${self:service}
|
||||
HOSTED_ZONE_DOMAIN: geoplatform.info
|
||||
CLOUDFRONT_CERTIFICATE_ARN: arn:aws:acm:us-east-1:998343784597:certificate/083641d4-9df6-4f89-b79d-6697f428f5b9
|
||||
GDAL_TASK_DEFINITION_NAME: ${self:provider.stage}-${self:service}-gdal
|
||||
GDAL_CONTAINER_DEFINITION_NAME: ${self:provider.stage}-${self:service}-osgeo-gdal
|
||||
TIPPECANOE_TASK_DEFINITION_NAME: ${self:provider.stage}-${self:service}-tippecanoe
|
||||
TIPPECANOE_CONTAINER_DEFINITION_NAME: ${self:provider.stage}-${self:service}-mgiddens-tippecanoe
|
||||
|
||||
stg:
|
||||
DEPLOYMENT_BUCKET_PREFIX: ${self:custom.namespaceShort}
|
||||
STACK_NAME_PREFIX: "${self:custom.namespaceShort}-"
|
||||
DATA_BUCKET: ${self:custom.environment.DEPLOYMENT_BUCKET_PREFIX}-${self:provider.stage}-${self:service}-data
|
||||
SHOULD_CREATE_SSL_CERTIFICATE: false
|
||||
HOSTED_ZONE_ID_DOMAIN: TBD
|
||||
HOSTED_ZONE_SUBDOMAIN: ${self:provider.stage}-${self:service}
|
||||
HOSTED_ZONE_DOMAIN: TBD
|
||||
CLOUDFRONT_CERTIFICATE_ARN: TBD
|
||||
GDAL_CONTAINER_DEFINITION_NAME: ${self:provider.stage}-${self:service}-osgeo-gdal
|
||||
TIPPECANOE_CONTAINER_DEFINITION_NAME: ${self:provider.stage}-${self:service}-tippecanoe
|
||||
|
||||
prd:
|
||||
DEPLOYMENT_BUCKET_PREFIX: ${self:custom.namespaceShort}
|
||||
STACK_NAME_PREFIX: "${self:custom.namespaceShort}-"
|
||||
DATA_BUCKET: ${self:custom.environment.DEPLOYMENT_BUCKET_PREFIX}-${self:provider.stage}-${self:service}-data
|
||||
SHOULD_CREATE_SSL_CERTIFICATE: false
|
||||
HOSTED_ZONE_ID_DOMAIN: TBD
|
||||
HOSTED_ZONE_SUBDOMAIN: ${self:service}
|
||||
HOSTED_ZONE_DOMAIN: TBD
|
||||
CLOUDFRONT_CERTIFICATE_ARN: TBD
|
||||
GDAL_CONTAINER_DEFINITION_NAME: ${self:provider.stage}-${self:service}-osgeo-gdal
|
||||
TIPPECANOE_CONTAINER_DEFINITION_NAME: ${self:provider.stage}-${self:service}-tippecanoe
|
||||
71
infrastructure/functions.yml
Normal file
71
infrastructure/functions.yml
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
DetectChangesForWorker:
|
||||
handler: functions/detect-changes-for-worker/index.handler
|
||||
name: ${self:provider.stage}-DetectChangesForWorker
|
||||
description: Scans an S3 bucket (with prefix) for items that have changes recently and sends them to ECS Tasks for processing
|
||||
runtime: nodejs12.x
|
||||
memorySize: 512
|
||||
timeout: 900
|
||||
environment:
|
||||
REGION: ${self:provider.region}
|
||||
STAGE: ${self:provider.stage}
|
||||
ECS_CLUSTER: !Ref ECSCluster
|
||||
VPC_SUBNET_ID:
|
||||
Fn::ImportValue: ${self:provider.stage}-PrivateSubnetOne
|
||||
GDAL_TASK_DEFINITION: ${self:custom.environment.GDAL_TASK_DEFINITION_NAME}
|
||||
GDAL_CONTAINER_DEFINITION: ${self:custom.environment.GDAL_CONTAINER_DEFINITION_NAME}
|
||||
TIPPECANOE_TASK_DEFINITION: ${self:custom.environment.TIPPECANOE_TASK_DEFINITION_NAME}
|
||||
TIPPECANOE_CONTAINER_DEFINITION: ${self:custom.environment.TIPPECANOE_CONTAINER_DEFINITION_NAME}
|
||||
|
||||
# The ECS Tasks can be kicked of my invoking the lambda on a schedule. This can provide the
|
||||
# ability to do nightly refreshed of the data.
|
||||
# events:
|
||||
# - schedule:
|
||||
# rate: cron(*/2 * * * ? *) # Fire every 2 minutes
|
||||
# input:
|
||||
# action: "gdal"
|
||||
# command:
|
||||
# - "ogrinfo"
|
||||
# - "-al"
|
||||
# - "-so"
|
||||
# - "-ro"
|
||||
# - "/vsizip//vsicurl/https://j40-sit-justice40-data-harvester-data.s3.amazonaws.com/census/tabblock2010_01_pophu.zip"
|
||||
# - schedule:
|
||||
# rate: cron(0 5 * * ? *) # Scan for updated data at Midnight Eastern Time
|
||||
# input:
|
||||
# action: enrichment
|
||||
# sourceBucketName: !Ref DataBucket
|
||||
# sourceBucketPrefix: usds/custom.csv
|
||||
# age: 86400 # Seconds
|
||||
# censusBucketName: j40-sit-justice40-data-harvester-data
|
||||
# censusBucketPrefix: census/tabblock2010_01_pophu.zip
|
||||
# pre:
|
||||
# - Fn::Join: ['', ["wget https://j40-sit-justice40-data-harvester-data.s3.amazonaws.com/usds/$", "{source.Key} -O /tmp/custom.csv"]]
|
||||
# command:
|
||||
# - "-f"
|
||||
# - "GeoJSON"
|
||||
# - "-sql"
|
||||
# - Fn::Join: ['', ["SELECT * FROM $", "{census.Key:base} LEFT JOIN '/tmp/custom.csv'.custom ON $", "{census.Key:base}.BLOCKID10 = custom.BLOCKID10"]]
|
||||
# - Fn::Join: ['', ["/vsis3/j40-sit-justice40-data-harvester-data/joined/$", "{source.Key:base}-$", "{census.Key:base}.json"]]
|
||||
# - Fn::Join: ['', ["/vsizip//vsicurl/https://j40-sit-justice40-data-harvester-data.s3.amazonaws.com/census/$", "{census.Key}"]]
|
||||
# - schedule:
|
||||
# rate: cron(0 7 * * ? *) # Run two hours after the generating any GeoJSON
|
||||
# input:
|
||||
# action: tippecanoe
|
||||
# pre:
|
||||
# - "curl https://gp-sit-tileservice-tile-cache.s3.amazonaws.com/usds/usa.csv -o /tmp/usa.csv"
|
||||
# - "curl https://gp-sit-tileservice-tile-cache.s3.amazonaws.com/usds/tristate.mbtiles -o /tmp/tristate.mbtiles"
|
||||
# post:
|
||||
# - "aws s3 cp /tmp/tl_2010_bg_with_data.mbtiles s3://j40-sit-justice40-data-harvester-data/output/tl_2010_bg_with_data.mbtiles"
|
||||
# - "tile-join --force -pk -pC -n tl_2010_bg -e /tmp/tiles /tmp/tl_2010_bg_with_data.mbtiles"
|
||||
# - "aws s3 sync /tmp/tiles s3://j40-sit-justice40-data-harvester-data/output/tiles"
|
||||
# command:
|
||||
# - "tile-join"
|
||||
# - "--force"
|
||||
# - "-pk"
|
||||
# - "-n"
|
||||
# - "tl_2010_bg"
|
||||
# - "-o"
|
||||
# - "/tmp/tl_2010_bg_with_data.mbtiles"
|
||||
# - "-c"
|
||||
# - "/tmp/usa.csv"
|
||||
# - "/tmp/tristate.mbtiles"
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
module.exports = {
|
||||
"env": {
|
||||
"node": true,
|
||||
"commonjs": true,
|
||||
"es2020": true
|
||||
},
|
||||
"extends": "eslint:recommended",
|
||||
"parserOptions": {
|
||||
"ecmaVersion": 11
|
||||
},
|
||||
"rules": {
|
||||
}
|
||||
};
|
||||
187
infrastructure/functions/detect-changes-for-worker/ecs.js
Normal file
187
infrastructure/functions/detect-changes-for-worker/ecs.js
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
/**
|
||||
* Load an ECS Task Definition template and apply variable substitution
|
||||
*/
|
||||
async function createECSTaskDefinition(options, templateName, taskVars) {
|
||||
const { fs, path } = options.deps;
|
||||
const { util } = options.deps.local;
|
||||
|
||||
// Load the task template
|
||||
const templatePath = path.join(__dirname, 'taskDefinitions', `${templateName}.json`);
|
||||
const rawTaskTemplate = await fs.promises.readFile(templatePath, 'utf8');
|
||||
|
||||
// Perform variable substitution
|
||||
const taskTemplate = util.applyVariableSubstitution(options, taskVars, rawTaskTemplate);
|
||||
|
||||
// Parse into a JSON object and return
|
||||
return JSON.parse(taskTemplate);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Takes the event parameters and performs some variable substitution for the
|
||||
* SQL query based on the actual S3 items being processed.
|
||||
*/
|
||||
function createECSTaskVariablesFromS3Record(options, record) {
|
||||
const { event } = options;
|
||||
const { REGION } = options.env;
|
||||
const { util } = options.deps;
|
||||
|
||||
// Create substituion variables from the S3 record
|
||||
const vars = util.createSubstitutionVariablesFromS3Record(options, record, 's3');
|
||||
|
||||
// Apply them to the SQL clause
|
||||
const sql = util.applyVariableSubstitution(options, vars, event.sql);
|
||||
|
||||
// Return the modified event record
|
||||
return {
|
||||
...event,
|
||||
REGION,
|
||||
sql
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Small utility function to look at a bash command line element and decide if it needs to
|
||||
* be quoted and/or any characters escaped.
|
||||
*
|
||||
* Currently, it just takes are of double-quotes and does not do full nested escapes.
|
||||
*/
|
||||
function quoteAndEscape(s) {
|
||||
// Escape any single quote chars using ASCII codes
|
||||
// @see https://stackoverflow.com/a/42341860/332406
|
||||
//
|
||||
// Throw an exception if there are double-quotes in the command itself, soo much nested
|
||||
// escaping for now....
|
||||
if (s.includes('"')) {
|
||||
throw new Error(`Double-quotes are not allowed in the container arguments`);
|
||||
}
|
||||
|
||||
// If there are any space in the string, wrap it in escaped double-quotes
|
||||
if (s.includes(' ')) {
|
||||
return `"${s}"`;
|
||||
}
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
/**
|
||||
* Take an array of commands and modify it with a list of pre- and post-
|
||||
* command to run in the image. This is primarily used to move files in
|
||||
* and out of the container ephemeral storage.
|
||||
*/
|
||||
function wrapContainerCommand(options, pre, command, post) {
|
||||
// We will run all of the commands as a chained bash command, so merge everything
|
||||
// together using '&&' chaining.
|
||||
//
|
||||
// We expect the pre/post arrays to be full commands, while the command array is a list
|
||||
// on individual pieces of a single command.
|
||||
if (!pre && !post) {
|
||||
return command;
|
||||
}
|
||||
|
||||
const allCommands = [];
|
||||
|
||||
// Pre-commands come first
|
||||
allCommands.push(...(pre || []));
|
||||
|
||||
// Turn the primary array of command line arguments into a single command line string. Be sure to
|
||||
// quote/escape elements with spaces and double-quotes
|
||||
allCommands.push(command.map(c => quoteAndEscape(c)).join(' '));
|
||||
|
||||
// And add in the post-commands last
|
||||
allCommands.push(...(post || []));
|
||||
|
||||
// Return a new array of commands with everything chained using '&&' so that execution will terminate
|
||||
// as soon as any command in the chain fails.
|
||||
return ['/bin/sh', '-c', allCommands.join(' && ')];
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the appropriate ECS Task Definition name based on the current action
|
||||
*/
|
||||
function getTaskDefinitionName(options, event) {
|
||||
const { GDAL_TASK_DEFINITION, TIPPECANOE_TASK_DEFINITION } = options.env;
|
||||
const { action } = event;
|
||||
|
||||
switch (action) {
|
||||
case 'gdal':
|
||||
case 'ogr2ogr':
|
||||
case 'enrichment':
|
||||
return GDAL_TASK_DEFINITION;
|
||||
case 'tippecanoe':
|
||||
return TIPPECANOE_TASK_DEFINITION;
|
||||
}
|
||||
|
||||
throw new Error(`No Fargate Task Definition defined for ${action}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the appropriate ECS Container Definition name based on the current action
|
||||
*/
|
||||
function getFargateContainerDefinitionName(options, event) {
|
||||
const { GDAL_CONTAINER_DEFINITION, TIPPECANOE_CONTAINER_DEFINITION } = options.env;
|
||||
const { action } = event;
|
||||
|
||||
switch (action) {
|
||||
case 'gdal':
|
||||
case 'ogr2ogr':
|
||||
case 'enrichment':
|
||||
return GDAL_CONTAINER_DEFINITION;
|
||||
case 'tippecanoe':
|
||||
return TIPPECANOE_CONTAINER_DEFINITION
|
||||
}
|
||||
|
||||
throw new Error(`No Fargate Container Definition Name defined for ${action}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes a (known) container in Fargate with the provided command line parameters.
|
||||
*/
|
||||
async function executeRawCommand(options, event) {
|
||||
const { ecs, logger } = options.deps;
|
||||
const { env } = options;
|
||||
const { ECS_CLUSTER, VPC_SUBNET_ID } = env;
|
||||
|
||||
// If there are pre- or post- commands defined, wrap up the primary command
|
||||
const containerCommand = wrapContainerCommand(options, event.pre, event.command, event.post);
|
||||
|
||||
// Get the name of the container that we are using
|
||||
const containerDefinitionName = getFargateContainerDefinitionName(options, event);
|
||||
|
||||
// Create the full Task parameter object and execute
|
||||
const params = {
|
||||
taskDefinition: getTaskDefinitionName(options, event),
|
||||
cluster: ECS_CLUSTER,
|
||||
launchType: 'FARGATE',
|
||||
count: 1,
|
||||
networkConfiguration: { // Must be specified for tasks with `awsvpc` networking and awsvpc networking is required for FARGATE launch types
|
||||
awsvpcConfiguration: {
|
||||
subnets: [
|
||||
VPC_SUBNET_ID
|
||||
],
|
||||
assignPublicIp: 'DISABLED',
|
||||
securityGroups: []
|
||||
}
|
||||
},
|
||||
overrides: {
|
||||
containerOverrides: [
|
||||
{
|
||||
name: containerDefinitionName,
|
||||
command: containerCommand
|
||||
}
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
logger.info(`Executing ECS Task...`, JSON.stringify(params, null, 2));
|
||||
return await ecs.runTask(params).promise();
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createECSTaskDefinition,
|
||||
createECSTaskVariablesFromS3Record,
|
||||
executeRawCommand,
|
||||
getFargateContainerDefinitionName,
|
||||
getTaskDefinitionName,
|
||||
wrapContainerCommand
|
||||
};
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"action": "enrichment",
|
||||
"sourceBucketName": "j40-sit-justice40-data-harvester-data",
|
||||
"sourceBucketPrefix": "usds/custom.csv",
|
||||
"age": 86400,
|
||||
"censusBucketName": "j40-sit-justice40-data-harvester-data",
|
||||
"censusBucketPrefix": "census/tabblock2010_01_pophu.zip",
|
||||
"pre": [
|
||||
"wget https://j40-sit-justice40-data-harvester-data.s3.amazonaws.com/usds/${source.Key} -O /tmp/custom.csv"
|
||||
],
|
||||
"command": [
|
||||
"--debug", "ON",
|
||||
"-f", "GeoJSON",
|
||||
"-sql", "SELECT * FROM ${census.Key:base} LEFT JOIN '/tmp/custom.csv'.custom ON ${census.Key:base}.BLOCKID10 = custom.BLOCKID10",
|
||||
"/vsis3/j40-sit-justice40-data-harvester-data/joined/${source.Key:base}-${census.Key:base}.json",
|
||||
"/vsizip//vsicurl/https://j40-sit-justice40-data-harvester-data.s3.amazonaws.com/census/${census.Key}"
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"action": "gdal",
|
||||
"command": [
|
||||
"ogrinfo",
|
||||
"-al",
|
||||
"-so",
|
||||
"-ro",
|
||||
"/vsizip//vsicurl/https://j40-sit-justice40-data-harvester-data.s3.amazonaws.com/census/tabblock2010_01_pophu.zip"
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"action": "ogr2ogr",
|
||||
"command": [
|
||||
"--debug", "ON",
|
||||
"-f",
|
||||
"GeoJSON",
|
||||
"/vsis3/j40-sit-justice40-data-harvester-data/sources/tabblock2010_01_pophu.json",
|
||||
"/vsizip//vsicurl/https://j40-sit-justice40-data-harvester-data.s3.amazonaws.com/census/tabblock2010_01_pophu.zip"
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
{
|
||||
"action": "tippecanoe",
|
||||
"pre": [
|
||||
"curl https://gp-sit-tileservice-tile-cache.s3.amazonaws.com/usds/usa.csv -o /tmp/usa.csv",
|
||||
"curl https://gp-sit-tileservice-tile-cache.s3.amazonaws.com/usds/tristate.mbtiles -o /tmp/tristate.mbtiles"
|
||||
],
|
||||
"post": [
|
||||
"aws s3 cp /tmp/tl_2010_bg_with_data.mbtiles s3://j40-sit-justice40-data-harvester-data/output/tl_2010_bg_with_data.mbtiles",
|
||||
"tile-join --force -pk -pC -n tl_2010_bg -e /tmp/tiles /tmp/tl_2010_bg_with_data.mbtiles",
|
||||
"aws s3 sync /tmp/tiles s3://j40-sit-justice40-data-harvester-data/output/tiles"
|
||||
],
|
||||
"command": [
|
||||
"tile-join",
|
||||
"--force",
|
||||
"-pk",
|
||||
"-n",
|
||||
"tl_2010_bg",
|
||||
"-o",
|
||||
"/tmp/tl_2010_bg_with_data.mbtiles",
|
||||
"-c",
|
||||
"/tmp/usa.csv",
|
||||
"/tmp/tristate.mbtiles"
|
||||
]
|
||||
}
|
||||
11
infrastructure/functions/detect-changes-for-worker/gdal.js
Normal file
11
infrastructure/functions/detect-changes-for-worker/gdal.js
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
/**
|
||||
* Create an appropriate GDAL path to an S3 object
|
||||
*
|
||||
function buildDestinationVSIS3Path(_options, name) {
|
||||
return `/vsis3/${bucket}/${key}`;
|
||||
}
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
// buildDestinationVSIS3Path
|
||||
}
|
||||
130
infrastructure/functions/detect-changes-for-worker/index.js
Normal file
130
infrastructure/functions/detect-changes-for-worker/index.js
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
// Standard modules
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { DateTime } = require('luxon');
|
||||
const logger = console;
|
||||
|
||||
// AWS APIs
|
||||
const AWS = require('aws-sdk');
|
||||
|
||||
// Local modules
|
||||
const util = require('./util');
|
||||
const gdal = require('./gdal');
|
||||
const s3 = require('./s3');
|
||||
const ecs = require('./ecs');
|
||||
|
||||
async function handler(event) {
|
||||
// Build the options for the lambda
|
||||
const options = initialize(event);
|
||||
|
||||
// Determine what action to take
|
||||
switch (event.action) {
|
||||
// Execute a raw command against the gdal container
|
||||
case 'gdal':
|
||||
return await ecs.executeRawCommand(options, event);
|
||||
|
||||
// Assume that we're running ogr2ogr
|
||||
case 'ogr2ogr':
|
||||
return await ecs.executeRawCommand(options, {
|
||||
...event,
|
||||
command: ['ogr2ogr', ...event.command]
|
||||
});
|
||||
|
||||
case 'tippecanoe':
|
||||
return await ecs.executeRawCommand(options, event);
|
||||
|
||||
// Combine USDS data with external data sources
|
||||
case 'enrichment':
|
||||
return await enrichDataWithUSDSAttributes(options, event);
|
||||
|
||||
default:
|
||||
logger.warn(`Unknown action ${event.action}. Exiting`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
async function enrichDataWithUSDSAttributes(options, event) {
|
||||
const { logger } = options.deps;
|
||||
const { util, ecs, s3 } = options.deps.local;
|
||||
|
||||
// Use the event.age to calculate the custoff for any input files
|
||||
const cutoff = util.getTimestampCutoff(options);
|
||||
logger.info(`Cutoff time of ${cutoff}`);
|
||||
|
||||
// Scan the source S3 bucket for items that need to be processed
|
||||
const { sourceBucketName, sourceBucketPrefix } = event;
|
||||
const sourceS3Records = await s3.fetchUpdatedS3Objects(options, sourceBucketName, sourceBucketPrefix, cutoff);
|
||||
|
||||
// If there are no input record, exit early
|
||||
if (sourceS3Records.length === 0) {
|
||||
logger.info(`There are no objects in s3://${sourceBucketName}/${sourceBucketPrefix} that have been modified after the cutoff date`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Scan for the census records
|
||||
const { censusBucketName, censusBucketPrefix } = event;
|
||||
const censusS3Records = await s3.fetchS3Objects(options, censusBucketName, censusBucketPrefix);
|
||||
|
||||
// If there are no census datasets, exit early
|
||||
if (censusS3Records.length === 0) {
|
||||
logger.info(`There are no objects in s3://${censusBucketName}/${censusBucketPrefix}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Create a set of substitution variables for each S3 record that will be applied to the
|
||||
// action template
|
||||
const censusVariables = censusS3Records.map(r => util.createSubstitutionVariablesFromS3Record(options, r, 'census'));
|
||||
const sourceVariables = sourceS3Records.map(r => util.createSubstitutionVariablesFromS3Record(options, r, 'source'));
|
||||
|
||||
// Kick off an ECS task for each (source, census) pair.
|
||||
for ( const census of censusVariables ) {
|
||||
for ( const source of sourceVariables) {
|
||||
// Merge the variables together
|
||||
const vars = { ...census, ...source };
|
||||
|
||||
// Let the logs know what's happening
|
||||
logger.info(`Enriching ${vars['census.Key']} with ${vars['source.Key']}...`);
|
||||
|
||||
// Apply the substitutions to the pre, post, and command arrays
|
||||
const pre = util.applyVariableSubstitutionToArray(options, vars, event.pre);
|
||||
const post = util.applyVariableSubstitutionToArray(options, vars, event.post);
|
||||
const command = util.applyVariableSubstitutionToArray(options, vars, event.command);
|
||||
|
||||
await ecs.executeRawCommand(options, {
|
||||
...event,
|
||||
pre,
|
||||
command: ['ogr2ogr', ...command],
|
||||
post
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap all dependencies in an object in order to inject as appropriate.
|
||||
*/
|
||||
function initialize(event) {
|
||||
logger.debug('event:', JSON.stringify(event, null, 2));
|
||||
return {
|
||||
deps: {
|
||||
DateTime,
|
||||
fs,
|
||||
logger,
|
||||
path,
|
||||
s3: new AWS.S3(),
|
||||
ecs: new AWS.ECS(),
|
||||
local: {
|
||||
ecs,
|
||||
gdal,
|
||||
s3,
|
||||
util
|
||||
}
|
||||
},
|
||||
env: process.env,
|
||||
event
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
handler
|
||||
};
|
||||
968
infrastructure/functions/detect-changes-for-worker/package-lock.json
generated
Normal file
968
infrastructure/functions/detect-changes-for-worker/package-lock.json
generated
Normal file
|
|
@ -0,0 +1,968 @@
|
|||
{
|
||||
"name": "detect-changes-for-worker",
|
||||
"version": "0.0.1",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
"@babel/code-frame": {
|
||||
"version": "7.12.11",
|
||||
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.11.tgz",
|
||||
"integrity": "sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@babel/highlight": "^7.10.4"
|
||||
}
|
||||
},
|
||||
"@babel/helper-validator-identifier": {
|
||||
"version": "7.14.5",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.5.tgz",
|
||||
"integrity": "sha512-5lsetuxCLilmVGyiLEfoHBRX8UCFD+1m2x3Rj97WrW3V7H3u4RWRXA4evMjImCsin2J2YT0QaVDGf+z8ondbAg==",
|
||||
"dev": true
|
||||
},
|
||||
"@babel/highlight": {
|
||||
"version": "7.14.5",
|
||||
"resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.14.5.tgz",
|
||||
"integrity": "sha512-qf9u2WFWVV0MppaL877j2dBtQIDgmidgjGk5VIMw3OadXvYaXn66U1BFlH2t4+t3i+8PhedppRv+i40ABzd+gg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@babel/helper-validator-identifier": "^7.14.5",
|
||||
"chalk": "^2.0.0",
|
||||
"js-tokens": "^4.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"chalk": {
|
||||
"version": "2.4.2",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
|
||||
"integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"ansi-styles": "^3.2.1",
|
||||
"escape-string-regexp": "^1.0.5",
|
||||
"supports-color": "^5.3.0"
|
||||
}
|
||||
},
|
||||
"escape-string-regexp": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
|
||||
"integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=",
|
||||
"dev": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"@eslint/eslintrc": {
|
||||
"version": "0.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.2.tgz",
|
||||
"integrity": "sha512-8nmGq/4ycLpIwzvhI4tNDmQztZ8sp+hI7cyG8i1nQDhkAbRzHpXPidRAHlNvCZQpJTKw5ItIpMw9RSToGF00mg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"ajv": "^6.12.4",
|
||||
"debug": "^4.1.1",
|
||||
"espree": "^7.3.0",
|
||||
"globals": "^13.9.0",
|
||||
"ignore": "^4.0.6",
|
||||
"import-fresh": "^3.2.1",
|
||||
"js-yaml": "^3.13.1",
|
||||
"minimatch": "^3.0.4",
|
||||
"strip-json-comments": "^3.1.1"
|
||||
}
|
||||
},
|
||||
"acorn": {
|
||||
"version": "7.4.1",
|
||||
"resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz",
|
||||
"integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==",
|
||||
"dev": true
|
||||
},
|
||||
"acorn-jsx": {
|
||||
"version": "5.3.1",
|
||||
"resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.1.tgz",
|
||||
"integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==",
|
||||
"dev": true
|
||||
},
|
||||
"ajv": {
|
||||
"version": "6.12.6",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
|
||||
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"fast-deep-equal": "^3.1.1",
|
||||
"fast-json-stable-stringify": "^2.0.0",
|
||||
"json-schema-traverse": "^0.4.1",
|
||||
"uri-js": "^4.2.2"
|
||||
}
|
||||
},
|
||||
"ansi-colors": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz",
|
||||
"integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==",
|
||||
"dev": true
|
||||
},
|
||||
"ansi-regex": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
|
||||
"integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
|
||||
"dev": true
|
||||
},
|
||||
"ansi-styles": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
|
||||
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"color-convert": "^1.9.0"
|
||||
}
|
||||
},
|
||||
"argparse": {
|
||||
"version": "1.0.10",
|
||||
"resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
|
||||
"integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"sprintf-js": "~1.0.2"
|
||||
}
|
||||
},
|
||||
"astral-regex": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz",
|
||||
"integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==",
|
||||
"dev": true
|
||||
},
|
||||
"balanced-match": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
|
||||
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
|
||||
"dev": true
|
||||
},
|
||||
"brace-expansion": {
|
||||
"version": "1.1.11",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
|
||||
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"balanced-match": "^1.0.0",
|
||||
"concat-map": "0.0.1"
|
||||
}
|
||||
},
|
||||
"callsites": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
|
||||
"integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
|
||||
"dev": true
|
||||
},
|
||||
"chalk": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
|
||||
"integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"ansi-styles": "^4.1.0",
|
||||
"supports-color": "^7.1.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"ansi-styles": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
|
||||
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"color-convert": "^2.0.1"
|
||||
}
|
||||
},
|
||||
"color-convert": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
|
||||
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"color-name": "~1.1.4"
|
||||
}
|
||||
},
|
||||
"color-name": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||
"dev": true
|
||||
},
|
||||
"has-flag": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
|
||||
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
|
||||
"dev": true
|
||||
},
|
||||
"supports-color": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
|
||||
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"has-flag": "^4.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"color-convert": {
|
||||
"version": "1.9.3",
|
||||
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
|
||||
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"color-name": "1.1.3"
|
||||
}
|
||||
},
|
||||
"color-name": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
|
||||
"integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=",
|
||||
"dev": true
|
||||
},
|
||||
"concat-map": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
|
||||
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
|
||||
"dev": true
|
||||
},
|
||||
"cross-spawn": {
|
||||
"version": "7.0.3",
|
||||
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
|
||||
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"path-key": "^3.1.0",
|
||||
"shebang-command": "^2.0.0",
|
||||
"which": "^2.0.1"
|
||||
}
|
||||
},
|
||||
"debug": {
|
||||
"version": "4.3.1",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
|
||||
"integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"ms": "2.1.2"
|
||||
}
|
||||
},
|
||||
"deep-is": {
|
||||
"version": "0.1.3",
|
||||
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
|
||||
"integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=",
|
||||
"dev": true
|
||||
},
|
||||
"doctrine": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
|
||||
"integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"esutils": "^2.0.2"
|
||||
}
|
||||
},
|
||||
"emoji-regex": {
|
||||
"version": "8.0.0",
|
||||
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
|
||||
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
|
||||
"dev": true
|
||||
},
|
||||
"enquirer": {
|
||||
"version": "2.3.6",
|
||||
"resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz",
|
||||
"integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"ansi-colors": "^4.1.1"
|
||||
}
|
||||
},
|
||||
"escape-string-regexp": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
|
||||
"integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==",
|
||||
"dev": true
|
||||
},
|
||||
"eslint": {
|
||||
"version": "7.29.0",
|
||||
"resolved": "https://registry.npmjs.org/eslint/-/eslint-7.29.0.tgz",
|
||||
"integrity": "sha512-82G/JToB9qIy/ArBzIWG9xvvwL3R86AlCjtGw+A29OMZDqhTybz/MByORSukGxeI+YPCR4coYyITKk8BFH9nDA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@babel/code-frame": "7.12.11",
|
||||
"@eslint/eslintrc": "^0.4.2",
|
||||
"ajv": "^6.10.0",
|
||||
"chalk": "^4.0.0",
|
||||
"cross-spawn": "^7.0.2",
|
||||
"debug": "^4.0.1",
|
||||
"doctrine": "^3.0.0",
|
||||
"enquirer": "^2.3.5",
|
||||
"escape-string-regexp": "^4.0.0",
|
||||
"eslint-scope": "^5.1.1",
|
||||
"eslint-utils": "^2.1.0",
|
||||
"eslint-visitor-keys": "^2.0.0",
|
||||
"espree": "^7.3.1",
|
||||
"esquery": "^1.4.0",
|
||||
"esutils": "^2.0.2",
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"file-entry-cache": "^6.0.1",
|
||||
"functional-red-black-tree": "^1.0.1",
|
||||
"glob-parent": "^5.1.2",
|
||||
"globals": "^13.6.0",
|
||||
"ignore": "^4.0.6",
|
||||
"import-fresh": "^3.0.0",
|
||||
"imurmurhash": "^0.1.4",
|
||||
"is-glob": "^4.0.0",
|
||||
"js-yaml": "^3.13.1",
|
||||
"json-stable-stringify-without-jsonify": "^1.0.1",
|
||||
"levn": "^0.4.1",
|
||||
"lodash.merge": "^4.6.2",
|
||||
"minimatch": "^3.0.4",
|
||||
"natural-compare": "^1.4.0",
|
||||
"optionator": "^0.9.1",
|
||||
"progress": "^2.0.0",
|
||||
"regexpp": "^3.1.0",
|
||||
"semver": "^7.2.1",
|
||||
"strip-ansi": "^6.0.0",
|
||||
"strip-json-comments": "^3.1.0",
|
||||
"table": "^6.0.9",
|
||||
"text-table": "^0.2.0",
|
||||
"v8-compile-cache": "^2.0.3"
|
||||
}
|
||||
},
|
||||
"eslint-scope": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz",
|
||||
"integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"esrecurse": "^4.3.0",
|
||||
"estraverse": "^4.1.1"
|
||||
}
|
||||
},
|
||||
"eslint-utils": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz",
|
||||
"integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"eslint-visitor-keys": "^1.1.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"eslint-visitor-keys": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
|
||||
"integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
|
||||
"dev": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"eslint-visitor-keys": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz",
|
||||
"integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==",
|
||||
"dev": true
|
||||
},
|
||||
"espree": {
|
||||
"version": "7.3.1",
|
||||
"resolved": "https://registry.npmjs.org/espree/-/espree-7.3.1.tgz",
|
||||
"integrity": "sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"acorn": "^7.4.0",
|
||||
"acorn-jsx": "^5.3.1",
|
||||
"eslint-visitor-keys": "^1.3.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"eslint-visitor-keys": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
|
||||
"integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
|
||||
"dev": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"esprima": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
|
||||
"integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
|
||||
"dev": true
|
||||
},
|
||||
"esquery": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz",
|
||||
"integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"estraverse": "^5.1.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"estraverse": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
|
||||
"integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==",
|
||||
"dev": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"esrecurse": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
|
||||
"integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"estraverse": "^5.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"estraverse": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
|
||||
"integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==",
|
||||
"dev": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"estraverse": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz",
|
||||
"integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==",
|
||||
"dev": true
|
||||
},
|
||||
"esutils": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
|
||||
"integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
|
||||
"dev": true
|
||||
},
|
||||
"fast-deep-equal": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
|
||||
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
|
||||
"dev": true
|
||||
},
|
||||
"fast-json-stable-stringify": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
|
||||
"integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
|
||||
"dev": true
|
||||
},
|
||||
"fast-levenshtein": {
|
||||
"version": "2.0.6",
|
||||
"resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
|
||||
"integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=",
|
||||
"dev": true
|
||||
},
|
||||
"file-entry-cache": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
|
||||
"integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"flat-cache": "^3.0.4"
|
||||
}
|
||||
},
|
||||
"flat-cache": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz",
|
||||
"integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"flatted": "^3.1.0",
|
||||
"rimraf": "^3.0.2"
|
||||
}
|
||||
},
|
||||
"flatted": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/flatted/-/flatted-3.1.1.tgz",
|
||||
"integrity": "sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA==",
|
||||
"dev": true
|
||||
},
|
||||
"fs.realpath": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
|
||||
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
|
||||
"dev": true
|
||||
},
|
||||
"functional-red-black-tree": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz",
|
||||
"integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=",
|
||||
"dev": true
|
||||
},
|
||||
"glob": {
|
||||
"version": "7.1.7",
|
||||
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz",
|
||||
"integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"fs.realpath": "^1.0.0",
|
||||
"inflight": "^1.0.4",
|
||||
"inherits": "2",
|
||||
"minimatch": "^3.0.4",
|
||||
"once": "^1.3.0",
|
||||
"path-is-absolute": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"glob-parent": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
|
||||
"integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"is-glob": "^4.0.1"
|
||||
}
|
||||
},
|
||||
"globals": {
|
||||
"version": "13.9.0",
|
||||
"resolved": "https://registry.npmjs.org/globals/-/globals-13.9.0.tgz",
|
||||
"integrity": "sha512-74/FduwI/JaIrr1H8e71UbDE+5x7pIPs1C2rrwC52SszOo043CsWOZEMW7o2Y58xwm9b+0RBKDxY5n2sUpEFxA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"type-fest": "^0.20.2"
|
||||
}
|
||||
},
|
||||
"has-flag": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
|
||||
"integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
|
||||
"dev": true
|
||||
},
|
||||
"ignore": {
|
||||
"version": "4.0.6",
|
||||
"resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
|
||||
"integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
|
||||
"dev": true
|
||||
},
|
||||
"import-fresh": {
|
||||
"version": "3.3.0",
|
||||
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
|
||||
"integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"parent-module": "^1.0.0",
|
||||
"resolve-from": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"imurmurhash": {
|
||||
"version": "0.1.4",
|
||||
"resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
|
||||
"integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=",
|
||||
"dev": true
|
||||
},
|
||||
"inflight": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
|
||||
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"once": "^1.3.0",
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"inherits": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
|
||||
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
|
||||
"dev": true
|
||||
},
|
||||
"is-extglob": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
|
||||
"integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=",
|
||||
"dev": true
|
||||
},
|
||||
"is-fullwidth-code-point": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
|
||||
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
|
||||
"dev": true
|
||||
},
|
||||
"is-glob": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz",
|
||||
"integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"is-extglob": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"isexe": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
|
||||
"integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
|
||||
"dev": true
|
||||
},
|
||||
"js-tokens": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
|
||||
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
|
||||
"dev": true
|
||||
},
|
||||
"js-yaml": {
|
||||
"version": "3.14.1",
|
||||
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",
|
||||
"integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"argparse": "^1.0.7",
|
||||
"esprima": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"json-schema-traverse": {
|
||||
"version": "0.4.1",
|
||||
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
|
||||
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
|
||||
"dev": true
|
||||
},
|
||||
"json-stable-stringify-without-jsonify": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
|
||||
"integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=",
|
||||
"dev": true
|
||||
},
|
||||
"levn": {
|
||||
"version": "0.4.1",
|
||||
"resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
|
||||
"integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"prelude-ls": "^1.2.1",
|
||||
"type-check": "~0.4.0"
|
||||
}
|
||||
},
|
||||
"lodash.clonedeep": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
|
||||
"integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=",
|
||||
"dev": true
|
||||
},
|
||||
"lodash.merge": {
|
||||
"version": "4.6.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
|
||||
"integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
|
||||
"dev": true
|
||||
},
|
||||
"lodash.truncate": {
|
||||
"version": "4.4.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz",
|
||||
"integrity": "sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM=",
|
||||
"dev": true
|
||||
},
|
||||
"lru-cache": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
|
||||
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"yallist": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"luxon": {
|
||||
"version": "1.27.0",
|
||||
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.27.0.tgz",
|
||||
"integrity": "sha512-VKsFsPggTA0DvnxtJdiExAucKdAnwbCCNlMM5ENvHlxubqWd0xhZcdb4XgZ7QFNhaRhilXCFxHuoObP5BNA4PA=="
|
||||
},
|
||||
"minimatch": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
|
||||
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
}
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
|
||||
"dev": true
|
||||
},
|
||||
"natural-compare": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
|
||||
"integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
|
||||
"dev": true
|
||||
},
|
||||
"once": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
||||
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"optionator": {
|
||||
"version": "0.9.1",
|
||||
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
|
||||
"integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"deep-is": "^0.1.3",
|
||||
"fast-levenshtein": "^2.0.6",
|
||||
"levn": "^0.4.1",
|
||||
"prelude-ls": "^1.2.1",
|
||||
"type-check": "^0.4.0",
|
||||
"word-wrap": "^1.2.3"
|
||||
}
|
||||
},
|
||||
"parent-module": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
|
||||
"integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"callsites": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"path-is-absolute": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
|
||||
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
|
||||
"dev": true
|
||||
},
|
||||
"path-key": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
|
||||
"integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
|
||||
"dev": true
|
||||
},
|
||||
"prelude-ls": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
|
||||
"integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==",
|
||||
"dev": true
|
||||
},
|
||||
"progress": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
|
||||
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
|
||||
"dev": true
|
||||
},
|
||||
"punycode": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
|
||||
"integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
|
||||
"dev": true
|
||||
},
|
||||
"regexpp": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz",
|
||||
"integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==",
|
||||
"dev": true
|
||||
},
|
||||
"require-from-string": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
|
||||
"integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==",
|
||||
"dev": true
|
||||
},
|
||||
"resolve-from": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
|
||||
"integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
|
||||
"dev": true
|
||||
},
|
||||
"rimraf": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
|
||||
"integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"glob": "^7.1.3"
|
||||
}
|
||||
},
|
||||
"semver": {
|
||||
"version": "7.3.5",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
|
||||
"integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"lru-cache": "^6.0.0"
|
||||
}
|
||||
},
|
||||
"shebang-command": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
|
||||
"integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"shebang-regex": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"shebang-regex": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
|
||||
"integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
|
||||
"dev": true
|
||||
},
|
||||
"slice-ansi": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz",
|
||||
"integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"ansi-styles": "^4.0.0",
|
||||
"astral-regex": "^2.0.0",
|
||||
"is-fullwidth-code-point": "^3.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"ansi-styles": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
|
||||
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"color-convert": "^2.0.1"
|
||||
}
|
||||
},
|
||||
"color-convert": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
|
||||
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"color-name": "~1.1.4"
|
||||
}
|
||||
},
|
||||
"color-name": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||
"dev": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"sprintf-js": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
|
||||
"integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
|
||||
"dev": true
|
||||
},
|
||||
"string-width": {
|
||||
"version": "4.2.2",
|
||||
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
|
||||
"integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"emoji-regex": "^8.0.0",
|
||||
"is-fullwidth-code-point": "^3.0.0",
|
||||
"strip-ansi": "^6.0.0"
|
||||
}
|
||||
},
|
||||
"strip-ansi": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
|
||||
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"ansi-regex": "^5.0.0"
|
||||
}
|
||||
},
|
||||
"strip-json-comments": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
|
||||
"integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
|
||||
"dev": true
|
||||
},
|
||||
"supports-color": {
|
||||
"version": "5.5.0",
|
||||
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
|
||||
"integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"has-flag": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"table": {
|
||||
"version": "6.7.1",
|
||||
"resolved": "https://registry.npmjs.org/table/-/table-6.7.1.tgz",
|
||||
"integrity": "sha512-ZGum47Yi6KOOFDE8m223td53ath2enHcYLgOCjGr5ngu8bdIARQk6mN/wRMv4yMRcHnCSnHbCEha4sobQx5yWg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"ajv": "^8.0.1",
|
||||
"lodash.clonedeep": "^4.5.0",
|
||||
"lodash.truncate": "^4.4.2",
|
||||
"slice-ansi": "^4.0.0",
|
||||
"string-width": "^4.2.0",
|
||||
"strip-ansi": "^6.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"ajv": {
|
||||
"version": "8.6.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.6.0.tgz",
|
||||
"integrity": "sha512-cnUG4NSBiM4YFBxgZIj/In3/6KX+rQ2l2YPRVcvAMQGWEPKuXoPIhxzwqh31jA3IPbI4qEOp/5ILI4ynioXsGQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"fast-deep-equal": "^3.1.1",
|
||||
"json-schema-traverse": "^1.0.0",
|
||||
"require-from-string": "^2.0.2",
|
||||
"uri-js": "^4.2.2"
|
||||
}
|
||||
},
|
||||
"json-schema-traverse": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
|
||||
"integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
|
||||
"dev": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"text-table": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
|
||||
"integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
|
||||
"dev": true
|
||||
},
|
||||
"type-check": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
|
||||
"integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"prelude-ls": "^1.2.1"
|
||||
}
|
||||
},
|
||||
"type-fest": {
|
||||
"version": "0.20.2",
|
||||
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
|
||||
"integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
|
||||
"dev": true
|
||||
},
|
||||
"uri-js": {
|
||||
"version": "4.4.1",
|
||||
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
|
||||
"integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"punycode": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"v8-compile-cache": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz",
|
||||
"integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==",
|
||||
"dev": true
|
||||
},
|
||||
"which": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
|
||||
"integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"isexe": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"word-wrap": {
|
||||
"version": "1.2.3",
|
||||
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
|
||||
"integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
|
||||
"dev": true
|
||||
},
|
||||
"wrappy": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
|
||||
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
|
||||
"dev": true
|
||||
},
|
||||
"yallist": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
|
||||
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
|
||||
"dev": true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"name": "detect-changes-for-worker",
|
||||
"version": "0.0.1",
|
||||
"description": "",
|
||||
"scripts": {
|
||||
"test:gdal": "cat ./events/gdal.json | docker run --rm -v ${PWD}:/var/task --env-file ./test.env -i -e DOCKER_LAMBDA_USE_STDIN=1 lambci/lambda:nodejs12.x index.handler",
|
||||
"test:tippecanoe": "cat ./events/tippecanoe.json | docker run --rm -v ${PWD}:/var/task --env-file ./test.env -i -e DOCKER_LAMBDA_USE_STDIN=1 lambci/lambda:nodejs12.x index.handler",
|
||||
"test:enrichment": "cat ./events/enrichment.json | docker run --rm -v ${PWD}:/var/task --env-file ./test.env -i -e DOCKER_LAMBDA_USE_STDIN=1 lambci/lambda:nodejs12.x index.handler"
|
||||
},
|
||||
"author": "Xentity",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"luxon": "^1.27.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"eslint": "^7.29.0"
|
||||
}
|
||||
}
|
||||
72
infrastructure/functions/detect-changes-for-worker/s3.js
Normal file
72
infrastructure/functions/detect-changes-for-worker/s3.js
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
/**
|
||||
* Helper function to determine if we should interpret an S3 object as
|
||||
* a "simple" file or a folder.
|
||||
*/
|
||||
function isSimpleObject(c, prefix) {
|
||||
// If the object ends with a separator charater, interpret that as a folder
|
||||
if (c.Key.endsWith('/')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// If the object is more deeply nested than the prefix, then ignore, e.g.
|
||||
// prefix = /foo/bar = two separators
|
||||
// c.Key = /foo/bar/baz = three separators [skip]
|
||||
// c.Key = /foo/bar.txt = two separators [pass]
|
||||
// This doesn't give the *exact* count, but all we really care about is that
|
||||
// the value is the same for the prefix and the S3 Key.
|
||||
const separatorCount = c.Key.split('/').length;
|
||||
const prefixSeparatorCount = prefix.split('/').length;
|
||||
|
||||
return separatorCount === prefixSeparatorCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return all of the simple S3 objects from a prefix that have a LastModified
|
||||
* date after a cutoff date.
|
||||
*
|
||||
* This returns objects that have recently changed for re-processing.
|
||||
*/
|
||||
function fetchUpdatedS3Objects (options, bucket, prefix, cutoff) {
|
||||
// Define a filter function that only looks at object on a single level of the
|
||||
// bucket and removes any objects with a LastModified timestamp prior to the cutoff
|
||||
const threshold = cutoff.toMillis();
|
||||
const filterFunc = (c) => isSimpleObject(c, prefix) && (threshold < c.LastModified.getTime());
|
||||
|
||||
return fetchS3Objects(options, bucket, prefix, filterFunc);
|
||||
}
|
||||
|
||||
/**
|
||||
* Basic utility function to return S3 object from a bucket that match a given prefix. An
|
||||
* optional filtering function can be passed in.
|
||||
*/
|
||||
async function fetchS3Objects (options, bucket, prefix, filterFunc = () => true) {
|
||||
const { s3 } = options.deps;
|
||||
const objects = [];
|
||||
|
||||
// Limit the results to items in this bucket with a specific prefix
|
||||
const params = {
|
||||
Bucket: bucket,
|
||||
Prefix: prefix
|
||||
};
|
||||
|
||||
do {
|
||||
// Get all of the initial objects
|
||||
const response = await s3.listObjectsV2(params).promise();
|
||||
|
||||
// Optionally, filter out objects
|
||||
const contents = response.Contents.filter(filterFunc);
|
||||
objects.push(...contents);
|
||||
|
||||
params.ContinuationToken = response.IsTruncated
|
||||
? response.NextContinuationToken
|
||||
: null;
|
||||
} while (params.ContinuationToken);
|
||||
|
||||
return objects;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
isSimpleObject,
|
||||
fetchS3Objects,
|
||||
fetchUpdatedS3Objects
|
||||
};
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"containerDefinitions": [
|
||||
{
|
||||
"name": "ECSUSDSJustice40Worker",
|
||||
"image": "osgeo/gdal:alpine-small-latest",
|
||||
"cpu": 1024,
|
||||
"environment": [
|
||||
{
|
||||
"name": "AWS_REGION",
|
||||
"value": "${REGION}"
|
||||
}
|
||||
],
|
||||
"command": [
|
||||
"ogr2ogr",
|
||||
"-f", "GeoJSON",
|
||||
"-sql", "${sql}",
|
||||
"${output}",
|
||||
"${input}"
|
||||
],
|
||||
"memory": 1024,
|
||||
"essential": true
|
||||
}
|
||||
],
|
||||
"family": ""
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"containerDefinitions": [
|
||||
{
|
||||
"name": "ECSUSDSJustice40Worker",
|
||||
"image": "osgeo/gdal:alpine-small-latest",
|
||||
"cpu": 1024,
|
||||
"environment": [
|
||||
{
|
||||
"name": "AWS_REGION",
|
||||
"value": "${REGION}"
|
||||
}
|
||||
],
|
||||
"command": [
|
||||
"ogr2ogr"
|
||||
],
|
||||
"memory": 1024,
|
||||
"essential": true
|
||||
}
|
||||
],
|
||||
"family": ""
|
||||
}
|
||||
19
infrastructure/functions/detect-changes-for-worker/test.env
Normal file
19
infrastructure/functions/detect-changes-for-worker/test.env
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
REGION=us-east-1
|
||||
STAGE=sit
|
||||
|
||||
# ESC Cluster name that will from the containers
|
||||
ECS_CLUSTER=j40-sit-justice40-data-harvester-ECSCluster-ktXGGU9zjwkb
|
||||
|
||||
# VPC Private Subnet that has a NAT GAteway
|
||||
VPC_SUBNET_ID=subnet-07e68cb57322f7b1f
|
||||
|
||||
# Names of the container and task names.
|
||||
GDAL_TASK_DEFINITION=sit-justice40-data-harvester-gdal
|
||||
GDAL_CONTAINER_DEFINITION=sit-justice40-data-harvester-osgeo-gdal
|
||||
TIPPECANOE_TASK_DEFINITION=sit-justice40-data-harvester-tippecanoe
|
||||
TIPPECANOE_CONTAINER_DEFINITION=sit-justice40-data-harvester-mgiddens-tippecanoe
|
||||
|
||||
# AWS Credentials
|
||||
AWS_ACCESS_KEY_ID=
|
||||
AWS_SECRET_ACCESS_KEY=
|
||||
AWS_DEFAULT_REGION=us-east-1
|
||||
70
infrastructure/functions/detect-changes-for-worker/util.js
Normal file
70
infrastructure/functions/detect-changes-for-worker/util.js
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
/**
|
||||
* Create a luxon object representing a cutoff based on the `age`
|
||||
* passed in from the event.
|
||||
*
|
||||
* A null or zero value for age returns the current time.
|
||||
*/
|
||||
function getTimestampCutoff(options) {
|
||||
const { event } = options;
|
||||
const { DateTime } = options.deps;
|
||||
|
||||
if (!event.age) {
|
||||
return DateTime.fromMillis(0);
|
||||
}
|
||||
|
||||
return DateTime.now().minus({ seconds: event.age });
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a set of substitution variables from an S3 record
|
||||
*/
|
||||
function createSubstitutionVariablesFromS3Record(options, record, prefix) {
|
||||
const { path } = options.deps;
|
||||
|
||||
const fullKey = record.Key;
|
||||
const baseKey = path.basename(fullKey);
|
||||
const baseKeyExt = path.extname(baseKey);
|
||||
const baseKeyNoExt = path.basename(baseKey, baseKeyExt);
|
||||
|
||||
// Define all of the valid substitution variables
|
||||
const vars = {};
|
||||
vars[`${prefix}.Key:full`] = fullKey;
|
||||
vars[`${prefix}.Key`] = baseKey;
|
||||
vars[`${prefix}.Key:base`] = baseKeyNoExt;
|
||||
vars[`${prefix}.Key:ext`] = baseKeyExt;
|
||||
|
||||
return vars;
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a collection of key/value input variables, replace
|
||||
* occurences of ${key} in the input with the corresponding
|
||||
* values
|
||||
*/
|
||||
function applyVariableSubstitution(options, vars, input) {
|
||||
let result = input;
|
||||
for (const [key, value] of Object.entries(vars)) {
|
||||
const token = '${' + key + '}';
|
||||
|
||||
// Use the split-join-method because the tokens have special characters which
|
||||
// confuses the Regular Expression constructor
|
||||
// @see https://stackoverflow.com/a/17606289/332406
|
||||
result = result.split(token).join(value);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generaliztion of the previsou function.
|
||||
*/
|
||||
function applyVariableSubstitutionToArray(options, vars, inputs) {
|
||||
return (inputs || []).map(input => applyVariableSubstitution(options, vars, input));
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
applyVariableSubstitution,
|
||||
applyVariableSubstitutionToArray,
|
||||
createSubstitutionVariablesFromS3Record,
|
||||
getTimestampCutoff
|
||||
};
|
||||
5120
infrastructure/package-lock.json
generated
Normal file
5120
infrastructure/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
20
infrastructure/package.json
Normal file
20
infrastructure/package.json
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
{
|
||||
"name": "infrastructure",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "none"
|
||||
},
|
||||
"author": "",
|
||||
"license": "UNLICENSED",
|
||||
"devDependencies": {
|
||||
"serverless": "^2.48.0",
|
||||
"serverless-certificate-creator": "^1.5.3",
|
||||
"serverless-pseudo-parameters": "^2.5.0"
|
||||
}
|
||||
}
|
||||
84
infrastructure/resources-cloudfront.yml
Normal file
84
infrastructure/resources-cloudfront.yml
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
Resources:
|
||||
|
||||
S3DataBucketPolicyCDN:
|
||||
Type: AWS::S3::BucketPolicy
|
||||
Properties:
|
||||
Bucket:
|
||||
Ref: DataBucket
|
||||
PolicyDocument:
|
||||
Statement:
|
||||
- Effect: "Allow"
|
||||
Action:
|
||||
- "s3:GetObject"
|
||||
Resource:
|
||||
Fn::Join:
|
||||
- ""
|
||||
- - "arn:aws:s3:::"
|
||||
- Ref: DataBucket
|
||||
- "/*"
|
||||
Principal: "*"
|
||||
|
||||
DataBucketCachePolicy:
|
||||
Type: AWS::CloudFront::CachePolicy
|
||||
Properties:
|
||||
CachePolicyConfig:
|
||||
Name: ${self:provider.stage}-${self:service}-cloudfront-cache-policy
|
||||
Comment: CloudFront Cache Policy for justice40 data harvester
|
||||
DefaultTTL: "86400" # one day, only if Origin does _not_ send `Cache-Control` or `Expires` headers
|
||||
MaxTTL: "31536000" # one year, used to validate when origin sends `Cache-Control` or `Expires` headers
|
||||
MinTTL: "1" # one second
|
||||
ParametersInCacheKeyAndForwardedToOrigin:
|
||||
EnableAcceptEncodingGzip: false
|
||||
EnableAcceptEncodingBrotli: false
|
||||
CookiesConfig:
|
||||
CookieBehavior: none
|
||||
HeadersConfig:
|
||||
HeaderBehavior: none
|
||||
QueryStringsConfig:
|
||||
QueryStringBehavior: none
|
||||
|
||||
DataDistribution:
|
||||
Type: AWS::CloudFront::Distribution
|
||||
Properties:
|
||||
DistributionConfig:
|
||||
Origins:
|
||||
- Id: DataBucket
|
||||
DomainName:
|
||||
# e.g. j40-sit-justice40-data-harvester-data.s3-website-us-east-1.amazonaws.com
|
||||
Fn::Join:
|
||||
- ""
|
||||
- - ${self:custom.namespaceShort}-
|
||||
- ${self:provider.stage}-
|
||||
- ${self:service}-
|
||||
- data
|
||||
- ".s3-website-"
|
||||
- Ref: AWS::Region
|
||||
- ".amazonaws.com"
|
||||
CustomOriginConfig:
|
||||
HTTPPort: '80'
|
||||
HTTPSPort: '443'
|
||||
OriginProtocolPolicy: http-only
|
||||
OriginSSLProtocols: [ "TLSv1", "TLSv1.1", "TLSv1.2" ]
|
||||
OriginCustomHeaders:
|
||||
- HeaderName: Origin # if the `Origin` header isn't present, S3 won't send CORS headers, this forces CORS to always be included
|
||||
HeaderValue: geoplatform.gov # this doesn't need to be anything specific, since Allow-Origin: * is our CORS policy, it just has to have a value
|
||||
|
||||
Enabled: true
|
||||
HttpVersion: http2
|
||||
Comment: CDN for justice40 data bucket
|
||||
Aliases:
|
||||
- ${self:custom.environment.HOSTED_ZONE_SUBDOMAIN}.${self:custom.environment.HOSTED_ZONE_DOMAIN}
|
||||
PriceClass: PriceClass_All
|
||||
DefaultCacheBehavior:
|
||||
AllowedMethods: [HEAD, GET, OPTIONS]
|
||||
CachedMethods: [HEAD, GET]
|
||||
CachePolicyId:
|
||||
Ref: DataBucketCachePolicy
|
||||
MinTTL: '0'
|
||||
DefaultTTL: '0'
|
||||
TargetOriginId: DataBucket
|
||||
ViewerProtocolPolicy: redirect-to-https
|
||||
CustomErrorResponses: []
|
||||
ViewerCertificate:
|
||||
AcmCertificateArn: ${self:custom.environment.CLOUDFRONT_CERTIFICATE_ARN}
|
||||
SslSupportMethod: sni-only
|
||||
164
infrastructure/resources-ecs.yml
Normal file
164
infrastructure/resources-ecs.yml
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
Parameters:
|
||||
ServiceNameOgr2Ogr:
|
||||
Type: String
|
||||
Default: ogr2ogr-gdal-3.6
|
||||
Description: The name of the service
|
||||
|
||||
Resources:
|
||||
|
||||
ECSCluster:
|
||||
Type: AWS::ECS::Cluster
|
||||
Properties:
|
||||
Tags:
|
||||
- Key: Stage
|
||||
Value: ${self:provider.stage}
|
||||
- Key: Namespace
|
||||
Value: ${self:custom.namespace}
|
||||
- Key: Name
|
||||
Value: ${self:custom.namespaceShort}-${self:provider.stage}-ecs-cluster
|
||||
|
||||
# Task execution role allowing access to resources.
|
||||
ECSTaskExecutionRoleShared:
|
||||
Type: AWS::IAM::Role
|
||||
Properties:
|
||||
AssumeRolePolicyDocument:
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Principal:
|
||||
Service: [ecs-tasks.amazonaws.com]
|
||||
Action: ['sts:AssumeRole']
|
||||
Path: /
|
||||
Policies:
|
||||
- PolicyName: AmazonECSTaskExecutionRolePolicy
|
||||
PolicyDocument:
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Action:
|
||||
# Allow the ECS tasks to upload logs to CloudWatch
|
||||
- 'logs:CreateLogStream'
|
||||
- 'logs:PutLogEvents'
|
||||
- 'logs:CreateLogStream'
|
||||
- 'logs:DescribeLogStreams'
|
||||
Resource: '*'
|
||||
|
||||
ECSTaskRoleShared:
|
||||
Type: AWS::IAM::Role
|
||||
Properties:
|
||||
AssumeRolePolicyDocument:
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Principal:
|
||||
Service: "ecs-tasks.amazonaws.com"
|
||||
Action: ['sts:AssumeRole']
|
||||
Path: /
|
||||
Policies:
|
||||
- PolicyName: "${self:provider.stage}-${self:service}-task-policy"
|
||||
PolicyDocument:
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- secretsmanager:GetSecretValue
|
||||
Resource: "*"
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- kms:Decrypt
|
||||
Resource: "*"
|
||||
# EventBridge permissions.
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- events:PutEvents
|
||||
Resource:
|
||||
- arn:aws:events:${self:provider.region}:#{AWS::AccountId}:*
|
||||
# Allow the ECS Tasks to access our specific S3 bucket
|
||||
# @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html for Multi-Part Upload requirement
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- s3:GetBucketAcl
|
||||
- s3:ListBucket
|
||||
- s3:ListBucketMultipartUploads
|
||||
Resource:
|
||||
- arn:aws:s3:::${self:custom.environment.DATA_BUCKET}
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- s3:PutObject
|
||||
- s3:PutObjectAcl
|
||||
- s3:GetObject
|
||||
- s3:GetObjectAcl
|
||||
- s3:GetObjectVersion
|
||||
- s3:GetObjectVersionAcl
|
||||
- s3:DeleteObject
|
||||
- s3:DeleteObjectVersion
|
||||
- s3:AbortMultipartUpload
|
||||
- s3:ListMultipartUploadParts
|
||||
Resource:
|
||||
- arn:aws:s3:::${self:custom.environment.DATA_BUCKET}/*
|
||||
|
||||
TaskDefinitionOgr2Ogr:
|
||||
Type: AWS::ECS::TaskDefinition
|
||||
Properties:
|
||||
Family: ${self:custom.environment.GDAL_TASK_DEFINITION_NAME}
|
||||
Cpu: 1024
|
||||
Memory: 2048
|
||||
NetworkMode: awsvpc
|
||||
RequiresCompatibilities:
|
||||
- FARGATE
|
||||
ExecutionRoleArn:
|
||||
Fn::GetAtt: [ ECSTaskExecutionRoleShared, Arn ]
|
||||
TaskRoleArn:
|
||||
Fn::GetAtt: [ ECSTaskRoleShared, Arn ]
|
||||
ContainerDefinitions:
|
||||
- Name: ${self:custom.environment.GDAL_CONTAINER_DEFINITION_NAME}
|
||||
Cpu: 1024
|
||||
Memory: 2048
|
||||
Image: osgeo/gdal:alpine-small-latest
|
||||
Environment:
|
||||
- Name: REGION
|
||||
Value: ${self:provider.region}
|
||||
- Name: STAGE
|
||||
Value: ${self:provider.stage}
|
||||
- Name: NODE_ENV
|
||||
Value: ${self:provider.stage}
|
||||
- Name: ENV_NAME
|
||||
Value: ${self:provider.stage}
|
||||
|
||||
LogConfiguration:
|
||||
LogDriver: 'awslogs'
|
||||
Options:
|
||||
awslogs-group: ${self:provider.stage}-${self:service}
|
||||
awslogs-region: ${self:provider.region}
|
||||
awslogs-stream-prefix: ${self:service}
|
||||
|
||||
TaskDefinitionTippecanoe:
|
||||
Type: AWS::ECS::TaskDefinition
|
||||
Properties:
|
||||
Family: ${self:custom.environment.TIPPECANOE_TASK_DEFINITION_NAME}
|
||||
Cpu: 1024
|
||||
Memory: 2048
|
||||
NetworkMode: awsvpc
|
||||
RequiresCompatibilities:
|
||||
- FARGATE
|
||||
ExecutionRoleArn:
|
||||
Fn::GetAtt: [ ECSTaskExecutionRoleShared, Arn ]
|
||||
TaskRoleArn:
|
||||
Fn::GetAtt: [ ECSTaskRoleShared, Arn ]
|
||||
ContainerDefinitions:
|
||||
- Name: ${self:custom.environment.TIPPECANOE_CONTAINER_DEFINITION_NAME}
|
||||
Cpu: 1024
|
||||
Memory: 2048
|
||||
Image: mikegiddens/tippecanoe:latest
|
||||
Environment:
|
||||
- Name: REGION
|
||||
Value: ${self:provider.region}
|
||||
- Name: STAGE
|
||||
Value: ${self:provider.stage}
|
||||
- Name: NODE_ENV
|
||||
Value: ${self:provider.stage}
|
||||
- Name: ENV_NAME
|
||||
Value: ${self:provider.stage}
|
||||
|
||||
LogConfiguration:
|
||||
LogDriver: 'awslogs'
|
||||
Options:
|
||||
awslogs-group: ${self:provider.stage}-${self:service}
|
||||
awslogs-region: ${self:provider.region}
|
||||
awslogs-stream-prefix: ${self:service}
|
||||
17
infrastructure/resources-route53.yml
Normal file
17
infrastructure/resources-route53.yml
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
Resources:
|
||||
|
||||
ARecordDataHarvester:
|
||||
Type: AWS::Route53::RecordSetGroup
|
||||
Condition: ShouldOnlyCreateResourcesInSIT
|
||||
DependsOn:
|
||||
- DataDistribution
|
||||
Properties:
|
||||
HostedZoneId: ${self:custom.environment.HOSTED_ZONE_ID_DOMAIN}
|
||||
RecordSets:
|
||||
- Name: ${self:custom.environment.HOSTED_ZONE_SUBDOMAIN}.${self:custom.environment.HOSTED_ZONE_DOMAIN}.
|
||||
Type: A
|
||||
AliasTarget:
|
||||
HostedZoneId: Z2FDTNDATAQYW2 # AWS global value https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-route53-aliastarget.html#cfn-route53-aliastarget-hostedzoneid
|
||||
DNSName:
|
||||
Fn::GetAtt: [ DataDistribution, DomainName ]
|
||||
|
||||
18
infrastructure/resources-s3.yml
Normal file
18
infrastructure/resources-s3.yml
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
Resources:
|
||||
|
||||
DataBucket:
|
||||
Type: AWS::S3::Bucket
|
||||
Properties:
|
||||
BucketName: ${self:custom.environment.DATA_BUCKET}
|
||||
AccessControl: PublicRead
|
||||
CorsConfiguration:
|
||||
CorsRules:
|
||||
- AllowedOrigins:
|
||||
- "*"
|
||||
AllowedMethods:
|
||||
- GET
|
||||
AllowedHeaders:
|
||||
- Content-Length
|
||||
WebsiteConfiguration:
|
||||
IndexDocument: index.html
|
||||
ErrorDocument: error.html
|
||||
87
infrastructure/serverless.yml
Normal file
87
infrastructure/serverless.yml
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
service: justice40-data-harvester
|
||||
configValidationMode: error
|
||||
|
||||
frameworkVersion: ">=2.48.0"
|
||||
|
||||
provider:
|
||||
name: aws
|
||||
runtime: nodejs12.x
|
||||
stage: ${opt:stage, 'sit'}
|
||||
region: ${opt:region, 'us-east-1'}
|
||||
profile: ${self:provider.stage}
|
||||
lambdaHashingVersion: "20201221"
|
||||
deploymentBucket:
|
||||
name: ${self:custom.environment.DEPLOYMENT_BUCKET_PREFIX}-${self:provider.stage}-${self:provider.region}-${self:service}
|
||||
blockPublicAccess: true
|
||||
maxPreviousDeploymentArtifacts: 5
|
||||
stackName: ${self:custom.environment.STACK_NAME_PREFIX}${self:provider.stage}-${self:service}
|
||||
|
||||
iam:
|
||||
role:
|
||||
statements:
|
||||
- Effect: "Allow"
|
||||
# Condition:
|
||||
# ArnEquals:
|
||||
# ecs:cluster:
|
||||
# Fn::GetAtt: [ ECSCluster, Arn ]
|
||||
Action: "ecs:RunTask"
|
||||
Resource: "*"
|
||||
- Effect: "Allow"
|
||||
# Condition:
|
||||
# ArnEquals:
|
||||
# ecs:cluster:
|
||||
# Fn::GetAtt: [ ECSCluster, Arn ]
|
||||
Action:
|
||||
- "iam:ListInstanceProfiles"
|
||||
- "iam:ListRoles"
|
||||
- "iam:PassRole"
|
||||
Resource: "*"
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- "s3:ListBucket"
|
||||
Resource:
|
||||
- Fn::Join:
|
||||
- ""
|
||||
- - "arn:aws:s3:::"
|
||||
- Ref: DataBucket
|
||||
- "/*"
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- "s3:DeleteObject"
|
||||
- "s3:GetObject"
|
||||
- "s3:PutObject"
|
||||
- "s3:PutObjectAcl"
|
||||
Resource:
|
||||
- Fn::Join:
|
||||
- ""
|
||||
- - "arn:aws:s3:::"
|
||||
- Ref: DataBucket
|
||||
|
||||
plugins:
|
||||
- serverless-certificate-creator
|
||||
- serverless-pseudo-parameters
|
||||
|
||||
custom:
|
||||
environment: ${file(./environment.yml):${self:provider.stage}}
|
||||
namespace: justice40 # Used to tag resources with a "Namespace".
|
||||
namespaceShort: j40 # Used to prefix stack name, deployment bucket, resource "Name" tags, etc.
|
||||
|
||||
customCertificate:
|
||||
certificateName: ${self:provider.stage}-${self:service}.${self:custom.environment.HOSTED_ZONE_DOMAIN}
|
||||
hostedZoneIds: ${self:custom.environment.HOSTED_ZONE_ID_DOMAIN}
|
||||
region: ${self:provider.region}
|
||||
tags:
|
||||
Name: ${self:provider.stage}-${self:service}.${self:custom.environment.HOSTED_ZONE_DOMAIN}
|
||||
Environment: ${self:provider.stage}
|
||||
rewriteRecords: true
|
||||
enabled: ${self:custom.environment.SHOULD_CREATE_SSL_CERTIFICATE}
|
||||
|
||||
|
||||
functions: ${file(./functions.yml)}
|
||||
|
||||
resources:
|
||||
- ${file(./conditions.yml)}
|
||||
- ${file(./resources-s3.yml)}
|
||||
- ${file(./resources-cloudfront.yml)}
|
||||
- ${file(./resources-ecs.yml)}
|
||||
- ${file(./resources-route53.yml)}
|
||||
Loading…
Add table
Reference in a new issue