Fargate Serverless Workers for Census Data Enrichment and Tile Generation (#230)

* add basic infrastructure

* add cloudfront distribution

* WIP checkpoint

* add ecs cluster

* add conditions and route53 dns entry to cloudfront

* WIP checkin

* Added a raw execution mode for demo/testing

* Add pre-defined Task for ogr2ogr

* Tweak Task Definition name

* Mostly working except for logging error

* Add additional logging permissions

* Succesfully executed ogr2ogr in fargate.  S3 permissions needs to be addresses

* Add multipart permissions

* Add a few more actions

* Put IAM Policy on the correct resource

* Deploy lambda and update events

* fix iam permissions 🤦🏻‍♂️

* Add reference to Tippecanoe container

* Clean up to only use named actions

* Refactor resources to include support for tippecanoe

* Make a more interesting GDAL command

* Pull all ECS variables into environment file; successful test of running tippecanoe container

* Support pre/post commands

* Refactor codebase and enable linting

* Implement many-to-many enrichment between USDS CSV files and Census zipped shapefiles

* Change the GDAL image to one with the built-in drivers

* Add some additional fixes to support the enrichment use case

* Clean up old hello-world example

* Expand the README to include ways to execute the lambdas

* Validate scheduled lambda execution and then comment out

Co-authored-by: Tim Zwolak <timothypage@gmail.com>

infrastructure/functions/detect-changes-for-worker/util.js

@@ -0,0 +1,70 @@
+/**
+ * Create a luxon object representing a cutoff based on the `age`
+ * passed in from the event.
+ * 
+ * A null or zero value for age returns the current time.
+ */
+function getTimestampCutoff(options) {
+    const { event } = options;
+    const { DateTime } = options.deps;
+    
+    if (!event.age) {
+        return DateTime.fromMillis(0);
+    }
+
+    return DateTime.now().minus({ seconds: event.age });
+}
+
+/**
+ * Create a set of substitution variables from an S3 record
+ */
+function createSubstitutionVariablesFromS3Record(options, record, prefix) {
+    const { path } = options.deps;
+
+    const fullKey = record.Key;
+    const baseKey = path.basename(fullKey);
+    const baseKeyExt = path.extname(baseKey);
+    const baseKeyNoExt = path.basename(baseKey, baseKeyExt);
+
+    // Define all of the valid substitution variables
+    const vars = {};
+    vars[`${prefix}.Key:full`] = fullKey;
+    vars[`${prefix}.Key`] = baseKey;
+    vars[`${prefix}.Key:base`] = baseKeyNoExt;
+    vars[`${prefix}.Key:ext`] = baseKeyExt;
+
+    return vars;
+}
+
+/**
+ * Given a collection of key/value input variables, replace
+ * occurences of ${key} in the input with the corresponding
+ * values
+ */
+function applyVariableSubstitution(options, vars, input) {
+    let result = input;
+    for (const [key, value] of Object.entries(vars)) {
+        const token = '${' + key + '}';
+
+        // Use the split-join-method because the tokens have special characters which
+        // confuses the Regular Expression constructor
+        //  @see https://stackoverflow.com/a/17606289/332406
+        result = result.split(token).join(value);
+    }
+
+    return result;
+}
+
+/**
+ * Generaliztion of the previsou function.
+ */
+function applyVariableSubstitutionToArray(options, vars, inputs) {
+    return (inputs || []).map(input => applyVariableSubstitution(options, vars, input));
+}
+
+module.exports = {
+    applyVariableSubstitution,
+    applyVariableSubstitutionToArray,
+    createSubstitutionVariablesFromS3Record,
+    getTimestampCutoff
+};