j40-cejst-2/infrastructure/functions/detect-changes-for-worker/util.js
Lucas Scharenbroich 38fff9cea8
Fargate Serverless Workers for Census Data Enrichment and Tile Generation (#230)
* add basic infrastructure

* add cloudfront distribution

* WIP checkpoint

* add ecs cluster

* add conditions and route53 dns entry to cloudfront

* WIP checkin

* Added a raw execution mode for demo/testing

* Add pre-defined Task for ogr2ogr

* Tweak Task Definition name

* Mostly working except for logging error

* Add additional logging permissions

* Succesfully executed ogr2ogr in fargate.  S3 permissions needs to be addresses

* Add multipart permissions

* Add a few more actions

* Put IAM Policy on the correct resource

* Deploy lambda and update events

* fix iam permissions 🤦🏻‍♂️

* Add reference to Tippecanoe container

* Clean up to only use named actions

* Refactor resources to include support for tippecanoe

* Make a more interesting GDAL command

* Pull all ECS variables into environment file; successful test of running tippecanoe container

* Support pre/post commands

* Refactor codebase and enable linting

* Implement many-to-many enrichment between USDS CSV files and Census zipped shapefiles

* Change the GDAL image to one with the built-in drivers

* Add some additional fixes to support the enrichment use case

* Clean up old hello-world example

* Expand the README to include ways to execute the lambdas

* Validate scheduled lambda execution and then comment out

Co-authored-by: Tim Zwolak <timothypage@gmail.com>
2021-06-30 09:29:01 -04:00

70 lines
No EOL
2 KiB
JavaScript

/**
* Create a luxon object representing a cutoff based on the `age`
* passed in from the event.
*
* A null or zero value for age returns the current time.
*/
function getTimestampCutoff(options) {
const { event } = options;
const { DateTime } = options.deps;
if (!event.age) {
return DateTime.fromMillis(0);
}
return DateTime.now().minus({ seconds: event.age });
}
/**
* Create a set of substitution variables from an S3 record
*/
function createSubstitutionVariablesFromS3Record(options, record, prefix) {
const { path } = options.deps;
const fullKey = record.Key;
const baseKey = path.basename(fullKey);
const baseKeyExt = path.extname(baseKey);
const baseKeyNoExt = path.basename(baseKey, baseKeyExt);
// Define all of the valid substitution variables
const vars = {};
vars[`${prefix}.Key:full`] = fullKey;
vars[`${prefix}.Key`] = baseKey;
vars[`${prefix}.Key:base`] = baseKeyNoExt;
vars[`${prefix}.Key:ext`] = baseKeyExt;
return vars;
}
/**
* Given a collection of key/value input variables, replace
* occurences of ${key} in the input with the corresponding
* values
*/
function applyVariableSubstitution(options, vars, input) {
let result = input;
for (const [key, value] of Object.entries(vars)) {
const token = '${' + key + '}';
// Use the split-join-method because the tokens have special characters which
// confuses the Regular Expression constructor
// @see https://stackoverflow.com/a/17606289/332406
result = result.split(token).join(value);
}
return result;
}
/**
* Generaliztion of the previsou function.
*/
function applyVariableSubstitutionToArray(options, vars, inputs) {
return (inputs || []).map(input => applyVariableSubstitution(options, vars, input));
}
module.exports = {
applyVariableSubstitution,
applyVariableSubstitutionToArray,
createSubstitutionVariablesFromS3Record,
getTimestampCutoff
};