nist-gov/actev.nist.gov/index.html
2025-03-05 18:59:57 +00:00

1443 lines
68 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!DOCTYPE html>
<html lang="en-US">
<noscript>
<META HTTP-EQUIV="Refresh" CONTENT="0;URL=nojs.html">
</noscript>
<head>
<title>content</title>
<link rel="stylesheet" media="screen" href="/packs/css/stylesheets-857d6a9a.css" />
<link rel="stylesheet" media="screen" href="/packs/css/uswds-a5f13679.css" />
<link rel="preload" as="image" href="/img/aria/Hero-Banner.webp" />
<link rel="preload" as="image" href="/img/aria/Overview-Picture.webp" />
<link rel="preload" as="image" href="/img/genai/GenAI-Hero_Back.webp" />
<link rel="preload" as="image" href="/img/Challenge_Problem-Hero_Back.webp" />
<script src="/js/uswds-init.js"></script>
<script src="/packs/js/application-31d17ca45bd614013b7b.js" defer="defer"></script>
<link rel="shortcut icon" type="image/x-icon" href="/packs/media/images/favicon-67b3cdd948447611fc48fa182b794440.ico" />
<meta name="title" content="content"/>
<meta property="og:title" content="content"/>
<meta property="og:site_name" content="NIST AI Challenge Problems"/>
<meta name="description" content="The Activity Extended Video (ActEV) challenge main focus is on human activity detection in multi-camera video streams. Activity detection has been an active research area in computer vision in recent years. The ability to detect human activities is an important task in computer vision due to its potential in a wide range of applications such as public safety and security, crime prevention, traffic monitoring and control, eldercare/childcare, human-computer interaction, human-robot interaction, smart homes, hospital activity monitoring, and many more. Here, by activity detection, we mean the detection of visual events (people/objects engaged in particular activities) in a large collection of video data. The ActEV challenge (https://actev.nist.gov) that we are currently running is based on the VIRAT V1"/>
<meta property="og:description" content="The Activity Extended Video (ActEV) challenge main focus is on human activity detection in multi-camera video streams. Activity detection has been an active research area in computer vision in recent years. The ability to detect human activities is an important task in computer vision due to its potential in a wide range of applications such as public safety and security, crime prevention, traffic monitoring and control, eldercare/childcare, human-computer interaction, human-robot interaction, smart homes, hospital activity monitoring, and many more. Here, by activity detection, we mean the detection of visual events (people/objects engaged in particular activities) in a large collection of video data. The ActEV challenge (https://actev.nist.gov) that we are currently running is based on the VIRAT V1"/>
<meta property="og:image" content="https://actev.nist.gov/img/og_image.webp" />
<meta property="og:image:width" content="1200" />
<meta property="og:image:height" content="631" />
<meta property="og:locale" content="en_US" />
<meta name="keywords" content="Human Activity Detection, Activity Classification and Localization, Multi-Camera Video, Video Analytics, Object Detection and Tracking"/>
<meta property="og:keywords" content="Human Activity Detection, Activity Classification and Localization, Multi-Camera Video, Video Analytics, Object Detection and Tracking"/>
<meta property="og:url" content="actev.nist.gov/" />
<link rel="canonical" href="actev.nist.gov/"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
<meta name="theme-color" content="#29e1cb" />
<meta charset="UTF-8" />
<meta name="robots" content="index, follow"/>
<meta name="twitter:card" content="summary" />
<meta name="csrf-param" content="authenticity_token" />
<meta name="csrf-token" content="x6rvuWbZM5v8vpsYQqv+zD/Ny2jx2hCOP2s/hwTW9bfp+aTeoYUaQRsC6ZUPriJG6mcOBoEO/Gos+EqQ94NBIQ==" />
</head>
<body class="emp-content" id="main">
<a class="usa-skipnav" href="#main-content">Skip to main content</a>
<div class="app-content">
<section class="usa-banner site-banner"
aria-label="Official website of the United States government"
>
<div class="usa-accordion">
<header class="usa-banner__header">
<div class="usa-banner__inner">
<div class="grid-col-auto">
<img
aria-hidden="true"
class="usa-banner__header-flag"
src="/img/us_flag_small.png"
alt="Small US Flag"
/>
</div>
<div class="grid-col-fill tablet:grid-col-auto" aria-hidden="true">
<p class="usa-banner__header-text">
An official website of the United States government
</p>
<p class="usa-banner__header-action">Heres how you know</p>
</div>
<button
type="button"
class="usa-accordion__button usa-banner__button"
aria-expanded="false"
aria-controls="gov-banner-default-default"
>
<span class="usa-banner__button-text">Heres how you know</span>
</button>
</div>
</header>
<div
class="usa-banner__content usa-accordion__content"
id="gov-banner-default-default"
>
<div class="grid-row grid-gap-lg">
<div class="usa-banner__guidance tablet:grid-col-6">
<img
class="usa-banner__icon usa-media-block__img"
src="/img/icon-dot-gov.svg"
role="img"
alt="Small Dot Gov Icon"
aria-hidden="true"
/>
<div class="usa-media-block__body">
<p>
<strong>Official websites use .gov</strong><br />A
<strong>.gov</strong> website belongs to an official government
organization in the United States.
</p>
</div>
</div>
<div class="usa-banner__guidance tablet:grid-col-6">
<img
class="usa-banner__icon usa-media-block__img"
src="/img/icon-https.svg"
role="img"
alt="Small HTTPS Lock Icon"
aria-hidden="true"
/>
<div class="usa-media-block__body">
<p>
<strong>Secure .gov websites use HTTPS</strong><br />A
<strong>lock</strong> (
<span class="icon-lock"
><svg
xmlns="http://www.w3.org/2000/svg"
width="52"
height="64"
viewBox="0 0 52 64"
class="usa-banner__lock-image"
role="img"
aria-labelledby="banner-lock-description-default"
focusable="false"
>
<title id="banner-lock-title-default">Lock</title>
<desc id="banner-lock-description-default">Locked padlock icon</desc>
<path
fill="#000000"
fill-rule="evenodd"
d="M26 0c10.493 0 19 8.507 19 19v9h3a4 4 0 0 1 4 4v28a4 4 0 0 1-4 4H4a4 4 0 0 1-4-4V32a4 4 0 0 1 4-4h3v-9C7 8.507 15.507 0 26 0zm0 8c-5.979 0-10.843 4.77-10.996 10.712L15 19v9h22v-9c0-6.075-4.925-11-11-11z"
/>
</svg> </span
>) or <strong>https://</strong> means youve safely connected to
the .gov website. Share sensitive information only on official,
secure websites.
</p>
</div>
</div>
</div>
</div>
</div>
</section>
<!-- Top Navbar -->
<!-- Taking this out temp. Needs to be tested in different browsers.
<div class="nist-print-header" style="display:none;">
<p class="nist-print-header__url">https://www.nist.gov/national-institute-standards-and-technology</p>
<img class="nist-print-header__logo" width="289" height="38" src="https://actev.nist.gov//img/nist_logo_brand_white.svg" alt="National Institute of Standards and Technology" />
</div>
-->
<header class="usa-header usa-header--extended site-header site-header--dark" role="banner">
<div class="usa-navbar site-header__navbar">
<a href="https://www.nist.gov" title="National Institute of Standards and Technology" aria-label="Home">
<img src="/img/nist_logo_brand_white.svg" role="img" class="nist-logo"
alt= "National Institute of Standards and Technology logo"/>
</a>
<button type="button" class="usa-menu-btn">Menu</button>
</div>
</header>
<div class="usa-overlay"></div>
<nav aria-label="Main Site navigation" class="usa-nav site-nav">
<div class="usa-nav__inner site-nav__inner nav-header">
<button type="button" class="usa-nav__close">
<img src="/img/usa-icons/close.svg" role="img" alt="Close" />
</button>
<ul class="usa-nav__primary usa-accordion emp-nav-list">
<li class="usa-nav__primary-item">
<div class="usa-logo site-logo" id="main-logo">
<a class="navbar-brand emp-logo" href="https://www.iarpa.gov">
<img alt="Platform Brand Logo"
src="/uassets/6"
style="padding-bottom: 0px; height: 30px !important">
</a>
<a class="site-title-a" href="/" title="ActEV">
<span aria-hidden="true" class="site-title--short">ActEV</span>
<span class="site-title--long"><b>ActEV</b></span>
</a>
</div>
</li>
<li class="usa-nav__primary-item flex-fill"></li>
<li class="usa-nav__primary-item padding-top-105">
<a href="uassets/instructions.pdf">Instructions</a>
</li>
<li class="usa-nav__primary-item">
<button type="button" class="usa-accordion__button usa-nav__link site-navbar-button" aria-expanded="false"
aria-controls="tm-profile1">Evaluations
</button>
<ul id="tm-profile1" class="usa-nav__submenu">
<li class="text-muted px-2 py-3"><b>Current Evaluations</b></li>
<li class="px-2 usa-nav__submenu-item"><a href="https://actev.nist.gov/SRL">Self-Reported Leaderboard</a></li>
<li class="text-muted px-2 py-3"><b>Past Evaluations</b></li>
<li class="px-2 usa-nav__submenu-item"><a href="https://actev.nist.gov/sdl">SDL 2021</a></li>
<li class="px-2 usa-nav__submenu-item"><a href="https://actev.nist.gov/trecvid21">TrecVID 2021</a></li>
<li class="px-2 usa-nav__submenu-item"><a href="https://actev.nist.gov/trecvid20">TrecVID 2020</a></li>
<li class="px-2 usa-nav__submenu-item"><a href="https://actev.nist.gov/sdl20">SDL 2019/2020</a></li>
<li class="px-2 usa-nav__submenu-item"><a href="https://actev.nist.gov/trecvid19">TrecVID 2019</a></li>
<li class="px-2 usa-nav__submenu-item"><a href="https://actev.nist.gov/prizechallenge">PrizeChallenge 2019</a></li>
<li class="px-2 usa-nav__submenu-item"><a href="https://actev.nist.gov/1B-Evaluation">1B-Evaluation 2018</a></li>
<li class="px-2 usa-nav__submenu-item"><a href="https://actev.nist.gov/1A-Leaderboard">1A-Leaderboard 2018</a></li>
</ul>
</li>
<li class="usa-nav__primary-item">
<a class="site-nav-elem" href="https://actev.nist.gov/users/sign_in"><span>Sign In</span></a>
</li>
<li class="usa-nav__primary-item">
<a class="site-nav-elem" href="https://actev.nist.gov/users/sign_up"><span>Sign Up</span></a>
</li>
</ul>
</div>
</nav>
<!-- Top Pages Content -->
<!-- Top Alerts Content -->
<!-- Sidebar Menu (admin only) -->
<!-- Embedded Content START -->
<!-- <div class="default-container"> -->
<main id="main-content">
<section class="frontpage-title-blue">
<div class="container">
<div class="row">
<div class="col-xs-12 col-md-12 col-sm-12">
<h1 class="title">ActEV: Activities in Extended Video</h1>
</div>
</div>
</div>
</section>
<section class="content-nav">
<div aria-label="ActEV navigation" class="usa-nav genai-nav genai-shadow">
<div class="grid-container">
<div class="tabbable usa-nav__inner site-nav__inner nav-header ">
<ul class="nav nav-tabs usa-nav__primary" role="tablist">
<li class="usa-nav__primary-item"><a class="usa-nav-link nav-link active" role="tab"
href="#tab_overview" data-toggle="tab">Overview</a></li=>
<li class="usa-nav__primary-item"><a class="usa-nav-link nav-link" role="tab" href="#tab_data"
data-toggle="tab">Video Examples</a></li>
<li class="usa-nav__primary-item"><a class="usa-nav-link nav-link" role="tab" href="#tab_activities"
data-toggle="tab">Activities</a></li>
<li class="usa-nav__primary-item"><a class="usa-nav-link nav-link" role="tab" href="#tab_tasks"
data-toggle="tab">Evaluation Tasks</a></li>
<li class="usa-nav__primary-item"><a class="usa-nav-link nav-link" role="tab" href="#tab_contact"
data-toggle="tab">Contact</a></li>
<li class="usa-nav__primary-item"><a class="usa-nav-link nav-link" role="tab" href="#tab_resources"
data-toggle="tab">Resources</a></li>
</ul>
</div>
</div>
</div>
</section>
<div class="container">
<div class="row">
<div class="col-xs-12 col-md-12 col-sm-12">
<div class="tab-content">
<div class="tab-pane fade" id="tab_resources">
<h3> Datasets</h3>
<ul>
<li> ActEV-supported data sets
<UL>
<li>
Multiview Extended Video with Activities (MEVA) See
the <a href=/uassets/15>License</a> and the <a href=/uassets/16>README</a> for
context.
<ul>
<li>NEW: Annotations from the T&E team are <a
href="https://gitlab.kitware.com/meva/meva-data-repo/tree/master/annotation/DIVA-phase-2/MEVA/kitware-meva-training">available
with additional annotations posted weekly.</a></li>
</ul>
<li> <a href=http://www.viratdata.org>VIRAT</A>
<li>
Data access from <a href=http://mevadata.org>
mevadata.org</a>: <a href=http://mevadata.org/#getting-data>Accessing
and using MEVA </a> and <a
href=http://mevadata.org/resources/README-meva-kf1-data.txt>MEVA
Download Instructions</a></li>
<li>
Data access from git and NIST : See <a
href=https://gitlab.kitware.com/actev/actev-data-repo>actev-data-repo</a>.
Access
credentials provided during signup
</li>
</UL>
</li>
<li> <a
href="https://deepmind.com/research/open-source/open-source-datasets/kinetics/">Kinetics</a>
</li>
<li> <a href="https://research.google.com/ava/">AVA</a>
</li>
<li> <a href="http://moments.csail.mit.edu/">Moments-in-Time</a>
</li>
<li> <a href="http://activity-net.org/download.html">ActivityNet</a>
</li>
<li> <a href="https://www.aicitychallenge.org/">NVIDIA's CityFlow dataset</a>
</li>
<li> <a href="https://visym.github.io/collector/">Live Datasets for Visual AI (Visym
Collector)</a><br />
- People in Public - 175k: 184,379 video clips of the ActEV activity classes for training in
the unknown facility use-case.
</li>
</ul>
<h3> Framework </h3>
The DIVA Framework is a software framework designed to
provide an architecture and a set of software modules which
will facilitate the development of activity recognition
analytics. The Framework is developed as a fully open
source project on GitHub. The following links will help you
get started with the framework:
<ul>
<li><a href="https://github.com/Kitware/DIVA">DIVA Framework Github Repository</a> This is the
main DIVA Framework site, all development of the framework happens here.
</li>
<li><a href="https://github.com/Kitware/DIVA/issues">DIVA Framework Issue Tracker</a> Submit any
bug reports or feature requests for the framework here.
</li>
<li>
<a href="https://kwiver-diva.readthedocs.io/en/latest/">DIVA
Framework Main Documentation Page</a>The source for the
framework documentation is maintained in the Github
repository using <a href="http://www.sphinx-doc.org/en/master/">Sphinx</a>.
A built version is maintained on <a href="https://readthedocs.org/">ReadTheDocs</a> at this
link. A good place to get started in the documentation,
after reading the <a
href="https://kwiver-diva.readthedocs.io/en/latest/introduction.html">Introduction</a>
is the <a href="https://kwiver-diva.readthedocs.io/en/latest/usecases.html">UseCase</a>
section which will walk you though a number of typical
use cases with the framework.
</li>
</ul>
The DIVA Framework is based on <a href="http://www.kwiver.org/">KWIVER</a>, an open source
framework designed for building complex computer vision
systems. The following links will help you learn more about
KWIVER:
<ul>
<li><a href="https://github.com/Kitware/kwiver">KWIVER Github Repository</a> This is the main
KWIVER site, all development of the framework happens here.
</li>
<li><a href="https://github.com/Kitware/kwiver/issues">KWIVER
Issue Tracker</a> Submit any bug reports or feature
requests for the KWIVER here. If there's any question
about whether your issues belongs in the KWIVER or DIVA
framework issues tracker, submit to the DIVA tracker and
we'll sort it out.. </li>
<li>
<a href="https://kwiver.readthedocs.io/en/latest/">KWIVER
Main Documentation Page</a>The source for the KWIVER
documentation is maintained in the Github repository
using <a href="http://www.sphinx-doc.org/en/master/">Sphinx</a>.
A built version is maintained on <a href="https://readthedocs.org/">ReadTheDocs</a> at this
link. A good place to get started in the documentation,
after reading the <a
href="https://kwiver.readthedocs.io/en/latest/introduction.html">Introduction</a>
are the <a
href="https://kwiver.readthedocs.io/en/latest/arrows/architecture.html">Arrows</a>
and <a href="https://kwiver.readthedocs.io/en/latest/sprokit/sprokit.html">Sprokit</a>
sections, both of which are used by the KWIVER
framework.
</li>
</ul>
<h3> Baseline Algorithms </h3>
KITWARE has adapted two "baseline" activity recognition
algorithms to work within the DIVA Framework:
<ul>
<li><a href="https://gitlab.kitware.com/kwiver/R-C3D/tree/kitware/master">R-C3D</a></li>
<li><a href="https://gitlab.kitware.com/kwiver/act_detector/tree/kitware/master">ACT</a></li>
</ul>
<h3>Visualization Tools</h3>
<ul>
<li><a href="https://github.com/visym/vipy">Visym Python Tools for Computer Vision and Machine
Learning</a>
</li>
</ul>
<h3> Annotation Tools </h3>
<ul>
<li><a
href="https://data.kitware.com/#collection/5bdc72e38d777f2179853d1f/folder/5c8a72438d777f072b97f9e1">Kitware
annotation tool (the tool natively supports the DIVA format) </a></li>
<li><a href="http://www.robots.ox.ac.uk/~vgg/software/via/">The VGG Image Annotator</a>
</li>
<li><a href="https://www.scalabel.ai/">Scalabel (used for annotation of Berkeley DeepDrive
project)</a></li>
<li><a href="https://github.com/cvondrick/vatic/tree/contrib">VATIC - Video Annotation Tool </a>
</li>
<li><a href="https://github.com/antingshen/BeaverDam">BeaverDam</a></li>
<li><a href="https://github.com/Microsoft/VoTT">VoTT: Visual Object Tagging Tool </a></li>
<li><a href="https://github.com/opencv/cvat">Computer Vision Annotation Tool (CVAT)</a></li>
<li><a href="http://www.cs.toronto.edu/polyrnn/">Efficient Annotation of Segmentation Datasets
with Polygon-RNN++</a></li>
</ul>
</div>
<!-- ------------ end One Stop tab --------------- -->
<div class="tab-pane fade" id="tab_contact">
<div class="t1">Contact Us</div>
<hr>
<p>For information on data, evaluation code, etc., please email:
<a href="mailto:%20actev-nist@nist.gov">actev-nist@nist.gov</a>
</p>
<p> For ActEV evaluation discussion, please visit
our Google Group: <a
href="https://groups.google.com/a/list.nist.gov/forum/#!forum/trecvid.actev">
https://groups.google.com/a/list.nist.gov/forum/#!forum/trecvid.actev</a>
</p>
</div>
<!-- ------------- end contact tab --------------------- -->
<div class="tab-pane fade" id="tab_activities">
<div class="t1">Activity Examples</div>
<hr>
An ActEV <bold>activity</bold> is defined to be “one or more people performing a specified movement
or
interacting with an object or group of objects”. Activities are annotated by humans using a set of
annotation guidelines that specify how to perform the annotation and the criteria to determine if
the activity occurred. Each activity is formally defined by five elements:
<br />
<ul>
<li> Activity Name - A mnemonic handle for the activity
</li>
<li>Activity Description - Textual description of the activity
</li>
<li>Begin time rule definition - The specification of what determines the beginning time of the
activity
</li>
<li>End time rule definition - The specification of what determines the ending time of the
activity
</li>
<li>Required object type list - The list of objects systems are expected to identify for the
activity. Note: this aspect of an activity not addressed by ActEV-PC.
</li>
</ul>
<br />
<h4> For example: </h4>
<div class="row">
<div class="col-xs-3 col-md-3 col-sm-3">
<b>Activity Name</b>
</div>
<div class="col-xs-9 col-md-9 col-sm-9">
<b> Description and Example Chip Videos</b>
</div>
</div>
<br />
<div class="row">
<div class="col-xs-3 col-md-3 col-sm-3">
<b>person_closes_vehicle_door</b>
</div>
<div class="col-xs-9 col-md-9 col-sm-9">
<ul class="act">
<li>Description: A person closing the door to a vehicle.</li>
<li>Start: The event begins 1 s before the door starts to move.</li>
<li>End: The event ends after the door stops
moving. People in cars who close the car door from
within is a closing event if you can still see the
person within the car. If the person is not visible
once they are in the car, then the closing should not
be annotated as an event.</li>
<li>Objects associated with the activity : Person; and Door or Vehicle</li>
</ul>
</div>
</div>
<br />
<div class="row">
<div class="col-xs-3 col-md-3 col-sm-3">
</div>
<div class="col-xs-9 col-md-9 col-sm-9">
<div class="row">
<div class="col-xs-4 col-md-4 col-sm-4">
<img class="img-responsive"
src="https://mig.nist.gov/public/actev/2017-10-DIVA-v1-preview-annotations/DIVA-v1-preview-annotations/gifs/Closing-00.gif" />
</div>
<div class="col-xs-4 col-md-4 col-sm-4">
<img class="img-responsive"
src="https://mig.nist.gov/public/actev/2017-10-DIVA-v1-preview-annotations/DIVA-v1-preview-annotations/gifs/Closing-01.gif" />
</div>
<div class="col-xs-4 col-md-4 col-sm-4">
<img class="img-responsive"
src="https://mig.nist.gov/public/actev/2017-10-DIVA-v1-preview-annotations/DIVA-v1-preview-annotations/gifs/Closing-02.gif" />
</div>
</div>
</div>
</div>
<br />
<div class="row">
<div class="col-xs-3 col-md-3 col-sm-3">
<b> vehicle_turns_left</b>
</div>
<div class="col-xs-9 col-md-9 col-sm-9">
<ul class="act">
<li>Description: A vehicle turning left or right is determined from the POV of the
driver of the vehicle. The vehicle may not stop for more than 10 s during the turn.
<li>Start: Annotation begins 1 s before vehicle has noticeably changed direction.
<li>End: Annotation ends 1 s after the vehicle is no longer changing direction and
linear motion has resumed. Note: This event is determined after a reasonable
interpretation of the video.
<li>Objects associated with the activity : Vehicle
</ul>
</div>
</div>
<br />
<div class="row">
<div class="col-xs-3 col-md-3 col-sm-3">
</div>
<div class="col-xs-9 col-md-9 col-sm-9">
<div class="row">
<div class="col-xs-12 col-md-12 col-sm-12">
<img class="img-responsive"
src="https://mig.nist.gov/public/actev/2017-10-DIVA-v1-preview-annotations/DIVA-v1-preview-annotations/gifs/vehicle_turning_left-00.gif" />
</div>
</div>
<div class="row">
<div class="col-xs-6 col-md-6 col-sm-6">
<img class="img-responsive"
src="https://mig.nist.gov/public/actev/2017-10-DIVA-v1-preview-annotations/DIVA-v1-preview-annotations/gifs/vehicle_turning_left-01.gif" />
</div>
<div class="col-xs-6 col-md-6 col-sm-6">
<img class="img-responsive"
src="https://mig.nist.gov/public/actev/2017-10-DIVA-v1-preview-annotations/DIVA-v1-preview-annotations/gifs/vehicle_turning_left-04.gif" />
</div>
</div>
</div>
</div>
<br />
<div class="row">
<div class="col-xs-3 col-md-3 col-sm-3">
<b>person_loads_vehicle</b>
</div>
<div class="col-xs-9 col-md-9 col-sm-9">
<ul class="act">
<li>Description: An object moving from person to vehicle.
<li>Start: The event begins 2 s before the cargo to be loaded is extended toward the
vehicle (i.e., before a persons posture changes from one of “carrying” to one of
“loading”).
<li>End: The event ends after the cargo is placed into the vehicle and the person-cargo
contact is lost. In the event of occlusion, it ends when the loss of contact is
visible.
<li>Objects associated with the activity: Person; and Vehicle
</ul>
</div>
</div>
<br />
<div class="row">
<div class="col-xs-3 col-md-3 col-sm-3">
</div>
<div class="col-xs-9 col-md-9 col-sm-9">
<div class="row">
<div class="col-xs-4 col-md-4 col-sm-4">
<img class="img-responsive"
src="https://mig.nist.gov/public/actev/2017-10-DIVA-v1-preview-annotations/DIVA-v1-preview-annotations/gifs/Loading-00.gif" />
</div>
<div class="col-xs-4 col-md-4 col-sm-4">
<img class="img-responsive"
src="https://mig.nist.gov/public/actev/2017-10-DIVA-v1-preview-annotations/DIVA-v1-preview-annotations/gifs/Loading-01.gif" />
</div>
<div class="col-xs-4 col-md-4 col-sm-4">
<img class="img-responsive"
src="https://mig.nist.gov/public/actev/2017-10-DIVA-v1-preview-annotations/DIVA-v1-preview-annotations/gifs/Loading-02.gif" />
</div>
</div>
</div>
</div>
<hr />
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">The names of the 37 Known Activities for ActEV21 SDL </h3>
</div>
<br />
<center>
<table border=0 cellpadding=0 cellspacing=0 width=782 style='border-collapse:
collapse;table-layout:fixed;width:586pt'>
<col width=352 style='mso-width-source:userset;mso-width-alt:11264;width:264pt'>
<col width=247 style='mso-width-source:userset;mso-width-alt:7893;width:185pt'>
<col width=183 style='mso-width-source:userset;mso-width-alt:5845;width:137pt'>
<tr height=21 style='height:16.0pt'>
<td height=21 style='height:16.0pt'>person_abandons_package</td>
<td>person_loads_vehicle</td>
<td>person_stands_up</td>
</tr>
<tr height=21 style='height:16.0pt'>
<td height=21 style='height:16.0pt'>person_closes_facility_door</td>
<td>person_transfers_object</td>
<td>person_talks_on_phone</td>
</tr>
<tr height=21 style='height:16.0pt'>
<td height=21 style='height:16.0pt'>person_closes_trunk</td>
<td>person_opens_facility_door</td>
<td>person_texts_on_phone</td>
</tr>
<tr height=21 style='height:16.0pt'>
<td height=21 style='height:16.0pt'>person_closes_vehicle_door</td>
<td>person_opens_trunk</td>
<td>person_steals_object</td>
</tr>
<tr height=21 style='height:16.0pt'>
<td height=21 style='height:16.0pt'>person_embraces_person</td>
<td>person_opens_vehicle_door</td>
<td>person_unloads_vehicle</td>
</tr>
<tr height=21 style='height:16.0pt'>
<td height=21 style='height:16.0pt'>person_enters_scene_through_structure</td>
<td>person_talks_to_person</td>
<td>vehicle_drops_off_person</td>
</tr>
<tr height=21 style='height:16.0pt'>
<td height=21 style='height:16.0pt'>person_enters_vehicle</td>
<td>person_picks_up_object</td>
<td>vehicle_picks_up_person</td>
</tr>
<tr height=21 style='height:16.0pt'>
<td height=21 style='height:16.0pt'>person_exits_scene_through_structure</td>
<td>person_purchases</td>
<td>vehicle_reverses</td>
</tr>
<tr height=21 style='height:16.0pt'>
<td height=21 style='height:16.0pt'>person_exits_vehicle</td>
<td>person_reads_document</td>
<td>vehicle_starts</td>
</tr>
<tr height=21 style='height:16.0pt'>
<td height=21 style='height:16.0pt'>hand_interacts_with_person</td>
<td>person_rides_bicycle</td>
<td>vehicle_stops</td>
</tr>
<tr height=21 style='height:16.0pt'>
<td height=21 style='height:16.0pt'>person_carries_heavy_object</td>
<td>person_puts_down_object</td>
<td>vehicle_turns_left</td>
</tr>
<tr height=21 style='height:16.0pt'>
<td height=21 style='height:16.0pt'>person_interacts_with_laptop</td>
<td>person_sits_down</td>
<td>vehicle_turns_right</td>
</tr>
<tr height=21 style='height:16.0pt'>
<td height=21 colspan=2 style='height:16.0pt;mso-ignore:colspan'></td>
<td>vehicle_makes_u_turn</td>
</tr>
<![if supportMisalignedColumns]>
<tr height=0 style='display:none'>
<td width=352 style='width:264pt'></td>
<td width=247 style='width:185pt'></td>
<td width=183 style='width:137pt'></td>
</tr>
<![endif]>
</table>
</center>
</div>
</div>
<!-- end activities tab -->
<div class="tab-pane fade in show active" id="tab_overview">
<div class="row">
<div class="col-xs-8 col-md-8 col-sm-8">
<div class="t1">Updates</div>
<div class="c-tp">
We are running the <a href="https://actev.nist.gov/SRL">TRECVID 2023 ActEV Self-Reported
Leaderboard (SRL) Challenge </a>
<ul>
<li> TRECVID'23 ActEV SRL Challenge starts from June 01, 2023.
<li> TRECVID'23 ActEV SRL Challenge results submission deadline : October 02, 2023:
4:00 PM EST
<li> Primary Metric is Activity and Object Detection (AOD) and is based on
Pmiss@0.1RFA.
<li> <strong> The TRECVID'23 ActEV SRL test dataset is the same as for CVPR'22
ActivityNet challenge and the TRECVID'22 ActEV evaluation </strong>
</ul>
</div>
<div class="t1">Summary</div>
<div class="c-tp">
ActEV is a series of evaluations to accelerate the development of robust, multi-camera,
automatic activity detection algorithms for forensic and real-time alerting
applications.
ActEV is an extension of the annual <a href="https://trecvid.nist.gov/"></a>TRECVID</a>
<a
href="https://www.nist.gov/itl/iad/mig/trecvid-surveillance-event-detection-evaluation-track">Surveillance
Event Detection</a> (SED) evaluation
where systems will also detect and track objects involved in the activities.
Each evaluation will challenge systems with new data, system requirements, and/or new
activities. Currently we are running the <a href="https://actev.nist.gov/sdl">ActEV 2021
Sequestered Data Leaderboard (SDL)</a> evaluation that features Unknown Facility and
Surprise Activity Testing and the <a href="https://actev.nist.gov/trecvid20"> ActEV
TRECVID 2020</a> evaluation that features additional known activities for a known
facility.
</div>
<div class="t1">Past Evaluations</div>
<div class="c-tp">
ActEV began with the Summer 2018 Blind and Leaderboard evaluations for 12 activities.
The summer evaluation was followed by the ongoing <a href="1B-Evaluation"> Fall ActEV
Self-Reported Evaluation </a>
which ended in Dec 2018 and included 18 activities.
The <a href=prizechallenge>Activities in Extended Videos Prize Challenge (ActEV-PC) </a>
ran under CVPR'19 ActivityNet workshop</a>. In 2019 we ran two other evaluations, the <a
href="https://actev.nist.gov/sdl20">ActEV 2019 Sequestered Data Leaderboard
(SDL)</a> and the <a href="https://actev.nist.gov/trecvid20"> ActEV TRECVID 2020</a>
evaluations.
</div>
<div class="t1">What is Activity Detection in Extended Videos?</div>
<div class="c-tp">An ActEV <bold><a href="#tab_activities">activity</a></bold> is defined to
be
“one or more people performing a specified movement or interacting with an object or
group of objects”.
Activity detection technologies process <a class="diva-tabs" href="#tab_data"
data-toggle="tab">extended video streams</a>,
such as those from a CCTV camera, and
automatically detects all instances of the activity by:
(1) identifying the type of activity,
(2) producing a confidence score indicating the presence of instance,
(3) temporally localizing the instance by indicating the begin and end times,
and (4) optionally, detecting and tracking the objects (people, vehicles, objects)
involved in the activity.
</div>
<div class="c-tp">Click on the tabs above to see <a href="#tab_data" data-toggle="tab">video
examples</a>,
<a href="#tab_activities" data-toggle="tab">activity</a> examples, and
<a href="#tab_tasks" data-toggle="tab">evaluation tasks</a>
</div>
<div class="t1">What</div>
<div class="c-tp"> The ActEV evaluations are being
conducted to assess the robustness of automatic
activity detection for a multi-camera streaming video
environment.</a>
</div>
<div class="t1">Who</div>
<div class="c-tp"> Everyone. Anyone who <a
href="https://actev.nist.gov/users/sign_up">registers</a> can submit to the
evaluation server. </div>
<div class="t1">How</div>
<div class="c-tp"> <a href="https://actev.nist.gov/users/sign_up">Register
here</a> and then based on the evaluation participants
can either ran their activity detection software on
their compute hardware and submit their system output
to the ActEV Scoring Server or submited their runnable
activity detection software to NIST using the
Evaluation Commandline Interface. See the individual
evaluation pages and evaluation plans for details.
</div>
<div class="t1">Data</div>
<div class="c-tp"> Each ActEV evaluation uses a new
video data set, changes the evaluation tasks, or
adds/changes activities. The data will be provided
in MPEG-4 and AVI formatted files. See the
individual evaluation pages for details.
</div>
<div class="t1">Evaluation Metrics and Tools</div>
<div class="c-tp"> The main scoring metrics will be based on
detection, temporal localization, and spatio-temporal localization using evaluation
measures that include
the probability of
missed detection and rate of false alarm. See details in the evaluation plans of each
evaluation.
<P>
NIST maintains the ActEV Scoring Software on the <a
href="https://github.com/usnistgov/ActEV_Scorer">Scoring software for
the Activities in Extended Video (ActEV) evaluation</a> GitHub repo.
</P>
</div>
</div>
<div class="col-xs-4 col-md-4 col-sm-4">
<div class="t1">News</div>
<div class="row">
<div class="col-md-12">
<div class="date">02<span>Nov</span></div>
<div class="date-list">Deadline for TRECVID ActEV SRL Challenge submission</div>
</div>
</div>
<div class="row">
<div class="col-md-12">
<div class="date">01<span>June</span></div>
<div class="date-list">TRECVID ActEV Challenge Opens</div>
</div>
</div>
<div class="row">
<div class="col-md-12">
<div class="date">6..9<span>Dec</span></div>
<div class="date-list">TRECVID virtual workshop</div>
</div>
</div>
</div>
</div>
</div>
<!-- ------------------- end overview tab --------------- -->
<!-- ---------------- end schedule tab ---------------------- -->
<div class="tab-pane fade" id="tab_data">
<div class="t1"> ActEV: Video Examples</div>
Below you will find four example videos from our data sets. There are two example views each of
indoor and outdoor.
<hr>
<table border=2>
<tr>
<th> Location </th>
<th> View 1 </th>
<th> View 2 </th>
</tr>
<tr>
<td> Indoor</td>
<td>
<!-- was 600x400, 1.5 aspect ratio -->
<video width="500" height="330" controls>
<source
src="https://mig.nist.gov/public/actev/samplevideos/clip-1-G327_2017-08-17_10-36-00_10-46-00.285.mp4"
type="video/mp4">
Your browser does not support the video tag.
</video>
</td>
<td>
<video width="500" height="330" controls>
<source
src="https://mig.nist.gov/public/actev/samplevideos/clip-2-G330_2017-08-16_12-02-38_12-12-38.270.mp4"
type="video/mp4">
Your browser does not support the video tag.
</video>
</td>
</tr>
<tr>
<td> Outdoor</td>
<td>
<video width="500" height="330" controls>
<source
src="https://mig.nist.gov/public/actev/samplevideos/clip-3-G341_2017-08-15_16-46-24_16-56-24.1000.mp4"
type="video/mp4">
Your browser does not support the video tag.
</video>
</td>
<td>
<video width="500" height="330" controls>
<source
src="https://mig.nist.gov/public/actev/samplevideos/clip-4-G301_2017-08-15_15-06-08_15-16-08.785.mp4"
type="video/mp4">
Your browser does not support the video tag.
</video>
</td>
</tr>
</table>
</div>
<!-- -------------- end data tab -------------- -->
<div class="tab-pane fade" id="tab_tasks">
<div class="t1"> ActEV Evaluation Tasks</div>
Activity detection has been researched for many years and remains an unsolved computer vision
challenge
that requires many capabilities beyond the current state of the art.
The ActEV series supports several evaluation tasks each escalating the difficulty by requiring more
specific information from the system.
Presently, there are three evaluation tasks defined: 1) Activity Detection (AD), 2) Activity
and Object Detection (AOD), and (3) Activity and Object Detection and Tracking
(AODT). Each evaluation task is summarized below. For a full description of the evaluation tasks,
read the Evaluation Plan for each specific evaluation.
<div class="t1">Activity Detection (AD)</div>
<br />
For the Activity Detection task, given a target activity, a system automatically detects and
temporally
localizes all instances of the activity. For a
system-identified activity instance to be evaluated as correct, the type of activity must be correct
and
the temporal overlap must fall within a minimal requirement.
<div class="t1">Activity and Object Detection (AOD)</div>
<br />
For the Activity and Object Detection task, given a target activity, a system detects and temporally
localizes all instances of the activity and spatially detects/localizes the people and/or objects
associated
with the target activity. For a system-identified instance to be scored as correct, it must meet the
temporal
overlap criteria for the AD task and in addition meet the spatial overlap of the identified objects
during
the activity instance.
<div class="t1">Activity Object Detection and Tracking (AODT)</div>
<br />
For the Activity Object Detection and Tracking task, given a target activity, a system detects and
temporally
localizes all instances of the activity, spatio-temporally detects/localizes the people and/or
objects
associated with the target activity, and properly assigns IDs the objects play in the activity. For
a
system-identified instance to be scored as correct, it must meet the temporal overlap criteria and
spatio-temporal overlap of the objects for the AOD task and correctly assign the IDs to the objects
as
described in the activity definition.
<br />
</div>
<!-- -------------- end tasks tab -------------- -->
<!-- end tab content -->
</div>
</div>
</div>
</div>
<style>
/* Frontpage CSS */
div.t1{
color: #414751;
font-weight: bold;
font-size: 1.4em;
margin-top: 1em;
margin-bottom: .4em;
}
div.c-tp {
margin-top: 3px;
font-size: 1em;
}
body {
font-family: "Lato", "Helvetica Neue", Helvetica, Arial, sans-serif;
font-size: 16px;
background-color: white;
text-align: left;
overflow-x: hidden;
overflow-y: auto;
}
section{
display: block;
width: 100%;
margin-bottom: 2rem;
}
h1 small {
color: #ddd;
}
.date {
background: #0191c8;
width: 55px;
height: 55px;
border-radius: 35px;
float: left;
margin: 10px 10px 0px 0px;
padding: 15px 0px 0px 0px;
color: #ffffff;
font-size: 24px;
font-weight: 900;
text-align: center;
line-height: 70%;
font-family: 'Open sans', sans-serif;
}
.date span {
font-size: 14px;
text-transform: uppercase;
font-weight: 400;
display: block;
color: #ffffff;
}
/* in order to vertically center text, the div needs some height */
.date-list {
padding-top: 6px;
height: 64px;
display: flex;
align-items: center;
}
/* handle links in page content */
a {
color: #337ab7;
background: none;
text-decoration: none;
border: none;
}
a:visited {
color: #225599;
background: none;
transition: color 5s;
text-decoration: underline;
border: none;
}
a:focus {
color: #114499;
background: none;
transition: color 5s;
text-decoration: underline;
border: none;
}
a:hover {
color: #114499;
background: none;
transition: color 5s;
text-decoration: underline;
border: none;
outline: 0;
}
.nav-tabs {
margin-left: 0px;
}
.nav-pills {
margin-left: 0px;
}
.nav-pills > li {
float: left;
}
.nav-pills > li > a {
border-radius: 0;
color: #0191c8;
text-transform: uppercase;
font-weight: 600;
}
.nav-pills > li + li {
margin-left: 2px;
}
.nav-pills > li.active > a,
.nav-pills > li.active > a:hover,
.nav-pills > li.active > a:focus {
color: #ffffff;
background-color: #0191c8;
text-transform: uppercase;
}
.nav-pills .nav-link {
border-radius: 0px;
}
.nav-stacked > li {
float: none;
}
.nav-stacked > li + li {
margin-top: 2px;
margin-left: 0;
}
.nav-justified {
width: 100%;
}
.nav-justified > li {
float: none;
}
.nav-justified > li > a {
text-align: center;
margin-bottom: 5px;
}
.nav-justified > .dropdown .dropdown-menu {
top: auto;
left: auto;
}
@media (min-width: 768px) {
.nav-justified > li {
display: table-cell;
width: 1%;
}
.nav-justified > li > a {
margin-bottom: 0;
}
}
/* Rules for sizing the icon. */
.material-icons.md-18 { font-size: 18px; }
.material-icons.md-24 { font-size: 24px; }
.material-icons.md-36 { font-size: 36px; }
.material-icons.md-48 { font-size: 48px; }
/* Rules for using icons as black on a light background. */
.material-icons.md-dark { color: rgba(0, 0, 0, 0.54); }
.material-icons.md-dark.md-inactive { color: rgba(0, 0, 0, 0.26); }
/* Rules for using icons as white on a dark background. */
.material-icons.md-light { color: rgba(255, 255, 255, 1); }
.material-icons.md-light.md-inactive { color: rgba(255, 255, 255, 0.3); }
/* ------------------------------------------------------------------------- */
/* Title Section for DIVA specific assets */
/* ------------------------------------------------------------------------- */
section.usa-banner {
margin-bottom: 0px;
}
/* hero images */
section.frontpage-title,
section.frontpage-title-blue,
section.frontpage-title-3,
section.frontpage-title-golden {
margin-bottom: 0px;
}
section.frontpage-title .title,
section.frontpage-title-blue .title,
section.frontpage-title-3 .title,
section.frontpage-title-golden .title {
overflow: hidden;
padding-top: 6rem;
padding-bottom: 6rem;
margin-bottom: 0px;
background-repeat: no-repeat;
background-size: 100%;
text-align: center;
font-size: 3.6rem;
font-weight: bold;
color:#fff;
text-shadow: 4px 4px black;
}
/*
Titles with subtitles use less padding between title and subtitle
and compensate for padding in the subtitle
*/
section.frontpage-title-golden {
padding-bottom: 3rem;
}
section.frontpage-title-golden .title {
padding-top: 4rem;
padding-bottom: 0rem;
}
section.frontpage-title .subtitle,
section.frontpage-title-blue .subtitle,
section.frontpage-title-3 .subtitle,
section.frontpage-title-golden .subtitle {
font-size: 1.7rem;
padding-top: 0.2rem;
text-align: center;
font-weight: bold;
margin:0 auto;
color: #eee;
text-shadow: 4px 4px black;
}
section.frontpage-title {
background-image: url("/uassets/1");
}
section.frontpage-title-3 {
background-image: url("/uassets/23");
}
section.frontpage-title-blue {
background-image: url("/uassets/29");
}
section.frontpage-title-golden {
background-image: url("/uassets/30");
}
section.frontpage-title .banner-image,
section.frontpage-title-blue .banner-image,
section.frontpage-title-3 .banner-image,
section.frontpage-title-golden .banner-image {
position: absolute;
z-index: -5;
width: 100%;
left: 0px;
top: 0px;
margin: 0 auto;
}
ul.act {
margin: 0;
padding-left: 1em;
}
</style>
<script type="text/javascript">window.onload = function (e) {
$("ul.nav-tabs a").click(function (e) {
e.preventDefault();
$(this).tab('show');
});
function go_to_hash_tab() {
var url_parse = $(location).attr('href').split("#");
if (url_parse[1]) {
$('.nav a[href="#' + url_parse[1] + '"]').tab('show');
}
}
// Link xyz/#tab_sth to the right tab
$(window).on('hashchange', go_to_hash_tab);
$(document).ready(go_to_hash_tab);
$(document).on('click', '.nav-tabs li a', function (e) {
href = $(e).attr('target').href
$(location).attr('href', href)
})
}
</script>
</main>
<!-- </div> -->
<!-- Embedded Content END -->
<p style="margin-bottom: 2cm;"></p>
</div> <!-- app-content div -->
<!-- Bottom Footer (auto sticky or non-sticky) -->
<footer class="nist-footer padding-bottom-4">
<div class="grid-container nist-footer__info">
<div class="grid-row">
<div class="tablet:grid-col-6 padding-left-0">
<div class="nist-footer__logo">
<a href="https://www.nist.gov/" title="National Institute of Standards and Technology" class="nist-footer__logo-link" rel="home">
<img src="/img/nist_logo_brand_white.svg" role="img"
alt= "National Institute of Standards and Technology logo" width="300px" height="42px"/>
</a>
</div>
<div class="nist-footer__contact">
<h5 class="nist-footer__contact-heading">HEADQUARTERS</h5>
<address>
100 Bureau Drive<br>
Gaithersburg, MD 20899<br>
<a href="tel:301-975-2000">301-975-2000</a>
</address>
<p>
<a href="mailto:do-webmaster@nist.gov">Webmaster</a> | <a href="https://www.nist.gov/about-nist/contact-us">Contact Us</a> | <a href="https://www.nist.gov/visit">Our Other Offices</a>
</p>
</div>
</div>
<div class="tablet:grid-col-6">
<div class="nist-footer__social-links grid-row">
<a class="nist-social nist-social--twitter-white ext" href=" https://twitter.com/NIST" data-extlink="">
<span>Twitter</span>
<img
class="nist-social-footer-icon filter-white"
src="/img/usa-icons/twitter.svg"
alt="Twitter"
/></a>
<a class="nist-social nist-social--facebook-white ext" href=" https://www.facebook.com/NIST" data-extlink="">
<span>Facebook</span>
<img
class="nist-social-footer-icon filter-white"
src="/img/usa-icons/facebook.svg"
alt="Facebook"
/></a>
<a class="nist-social nist-social--linkedin-white ext" href=" https://www.linkedin.com/company/nist" data-extlink="">
<span>LinkedIn</span>
<img
class="nist-social-footer-icon filter-white"
src="/img/usa-icons/linkedin.svg"
alt="LinkedIn"
/></a>
<a class="nist-social nist-social--instagram-white ext" href=" https://www.instagram.com/nist/" data-extlink="">
<span>Instagram</span>
<img
class="nist-social-footer-icon filter-white"
src="/img/usa-icons/instagram.svg"
alt="Instagram"
/></a>
<a class="nist-social nist-social--youtube-white ext" href=" https://www.youtube.com/NIST" data-extlink="">
<span>YouTube</span>
<img
class="nist-social-footer-icon filter-white"
src="/img/usa-icons/youtube.svg"
alt="YouTube"
/></a>
<a class="nist-social nist-social--rss-white" href=" https://www.nist.gov/news-events/nist-rss-feeds">
<span>RSS Feed</span>
<img
class="nist-social-footer-icon filter-white"
src="/img/usa-icons/rss_feed.svg"
alt="RSS"
/></a>
<a class="nist-social nist-social--envelope-white ext" href=" https://public.govdelivery.com/accounts/USNIST/subscriber/new" data-extlink="">
<span>Mailing List</span>
<img
class="nist-social-footer-icon filter-white"
src="/img/usa-icons/mail.svg"
alt="Subscribe to Mailing List"
/></a>
</div>
<br>
<div class="nist-footer__feedback grid-row">
How are we doing? <a class="margin-left-2 usa-button" rel="nofollow" href="https://www.nist.gov/form/nist-gov-feedback?destination=/national-institute-standards-and-technology" title="Provide feedback">Feedback</a>
</div>
</div>
</div>
</div>
<div class="nist-footer__inner">
<div class="nist-footer__menu" role="navigation">
<ul>
<li class="nist-footer__menu-item">
<a href="https://www.nist.gov/privacy-policy">Site Privacy</a>
</li>
<li class="nist-footer__menu-item">
<a href="https://www.nist.gov/oism/accessibility">Accessibility</a>
</li>
<li class="nist-footer__menu-item">
<a href="https://www.nist.gov/privacy">Privacy Program</a>
</li>
<li class="nist-footer__menu-item">
<a href="https://www.nist.gov/oism/copyrights">Copyrights</a>
</li>
<li class="nist-footer__menu-item">
<a href="https://www.commerce.gov/vulnerability-disclosure-policy">Vulnerability Disclosure</a>
</li>
<li class="nist-footer__menu-item">
<a href="https://www.nist.gov/no-fear-act-policy">No Fear Act Policy</a>
</li>
<li class="nist-footer__menu-item">
<a href="https://www.nist.gov/foia">FOIA</a>
</li>
<li class="nist-footer__menu-item">
<a href="https://www.nist.gov/environmental-policy-statement">Environmental Policy</a>
</li>
<li class="nist-footer__menu-item ">
<a href="https://www.nist.gov/summary-report-scientific-integrity">Scientific Integrity</a>
</li>
<li class="nist-footer__menu-item ">
<a href="https://www.nist.gov/nist-information-quality-standards">Information Quality Standards</a>
</li>
<li class="nist-footer__menu-item">
<a href="https://www.commerce.gov/">Commerce.gov</a>
</li>
<li class="nist-footer__menu-item">
<a href="https://www.science.gov/">Science.gov</a>
</li>
<li class="nist-footer__menu-item">
<a href="https://www.usa.gov/">USA.gov</a>
</li>
<li class="nist-footer__menu-item">
<a href="https://vote.gov/">Vote.gov</a>
</li>
</ul>
</div>
</div>
</footer>
<script defer src="/js/uswds.js"></script>
</body>
</html>