982 lines
80 KiB
Text
982 lines
80 KiB
Text
<!DOCTYPE html>
|
||
<html lang="en" dir="ltr" prefix="content: http://purl.org/rss/1.0/modules/content/ dc: http://purl.org/dc/terms/ foaf: http://xmlns.com/foaf/0.1/ og: http://ogp.me/ns# rdfs: http://www.w3.org/2000/01/rdf-schema# schema: http://schema.org/ sioc: http://rdfs.org/sioc/ns# sioct: http://rdfs.org/sioc/types# skos: http://www.w3.org/2004/02/skos/core# xsd: http://www.w3.org/2001/XMLSchema# ">
|
||
<head>
|
||
<meta charset="utf-8" /><script type="text/javascript">(window.NREUM||(NREUM={})).init={ajax:{deny_list:["bam.nr-data.net"]}};(window.NREUM||(NREUM={})).loader_config={licenseKey:"37b7ccb661",applicationID:"1089704227"};;/*! For license information please see nr-loader-rum-1.283.2.min.js.LICENSE.txt */
|
||
(()=>{var e,t,r={122:(e,t,r)=>{"use strict";r.d(t,{a:()=>i});var n=r(944);function i(e,t){try{if(!e||"object"!=typeof e)return(0,n.R)(3);if(!t||"object"!=typeof t)return(0,n.R)(4);const r=Object.create(Object.getPrototypeOf(t),Object.getOwnPropertyDescriptors(t)),o=0===Object.keys(r).length?e:r;for(let a in o)if(void 0!==e[a])try{if(null===e[a]){r[a]=null;continue}Array.isArray(e[a])&&Array.isArray(t[a])?r[a]=Array.from(new Set([...e[a],...t[a]])):"object"==typeof e[a]&&"object"==typeof t[a]?r[a]=i(e[a],t[a]):r[a]=e[a]}catch(e){(0,n.R)(1,e)}return r}catch(e){(0,n.R)(2,e)}}},555:(e,t,r)=>{"use strict";r.d(t,{Vp:()=>c,fn:()=>s,x1:()=>u});var n=r(384),i=r(122);const o={beacon:n.NT.beacon,errorBeacon:n.NT.errorBeacon,licenseKey:void 0,applicationID:void 0,sa:void 0,queueTime:void 0,applicationTime:void 0,ttGuid:void 0,user:void 0,account:void 0,product:void 0,extra:void 0,jsAttributes:{},userAttributes:void 0,atts:void 0,transactionName:void 0,tNamePlain:void 0},a={};function s(e){try{const t=c(e);return!!t.licenseKey&&!!t.errorBeacon&&!!t.applicationID}catch(e){return!1}}function c(e){if(!e)throw new Error("All info objects require an agent identifier!");if(!a[e])throw new Error("Info for ".concat(e," was never set"));return a[e]}function u(e,t){if(!e)throw new Error("All info objects require an agent identifier!");a[e]=(0,i.a)(t,o);const r=(0,n.nY)(e);r&&(r.info=a[e])}},217:(e,t,r)=>{"use strict";r.d(t,{D0:()=>m,gD:()=>v,xN:()=>h});r(860).K7.genericEvents;const n="experimental.marks",i="experimental.measures",o="experimental.resources",a=e=>{if(!e||"string"!=typeof e)return!1;try{document.createDocumentFragment().querySelector(e)}catch{return!1}return!0};var s=r(614),c=r(944),u=r(384),l=r(122);const d="[data-nr-mask]",f=()=>{const e={feature_flags:[],experimental:{marks:!1,measures:!1,resources:!1},mask_selector:"*",block_selector:"[data-nr-block]",mask_input_options:{color:!1,date:!1,"datetime-local":!1,email:!1,month:!1,number:!1,range:!1,search:!1,tel:!1,text:!1,time:!1,url:!1,week:!1,textarea:!1,select:!1,password:!0}};return{ajax:{deny_list:void 0,block_internal:!0,enabled:!0,autoStart:!0},distributed_tracing:{enabled:void 0,exclude_newrelic_header:void 0,cors_use_newrelic_header:void 0,cors_use_tracecontext_headers:void 0,allowed_origins:void 0},get feature_flags(){return e.feature_flags},set feature_flags(t){e.feature_flags=t},generic_events:{enabled:!0,autoStart:!0},harvest:{interval:30},jserrors:{enabled:!0,autoStart:!0},logging:{enabled:!0,autoStart:!0},metrics:{enabled:!0,autoStart:!0},obfuscate:void 0,page_action:{enabled:!0},page_view_event:{enabled:!0,autoStart:!0},page_view_timing:{enabled:!0,autoStart:!0},performance:{get capture_marks(){return e.feature_flags.includes(n)||e.experimental.marks},set capture_marks(t){e.experimental.marks=t},get capture_measures(){return e.feature_flags.includes(i)||e.experimental.measures},set capture_measures(t){e.experimental.measures=t},capture_detail:!0,resources:{get enabled(){return e.feature_flags.includes(o)||e.experimental.resources},set enabled(t){e.experimental.resources=t},asset_types:[],first_party_domains:[],ignore_newrelic:!0}},privacy:{cookies_enabled:!0},proxy:{assets:void 0,beacon:void 0},session:{expiresMs:s.wk,inactiveMs:s.BB},session_replay:{autoStart:!0,enabled:!1,preload:!1,sampling_rate:10,error_sampling_rate:100,collect_fonts:!1,inline_images:!1,fix_stylesheets:!0,mask_all_inputs:!0,get mask_text_selector(){return e.mask_selector},set mask_text_selector(t){a(t)?e.mask_selector="".concat(t,",").concat(d):""===t||null===t?e.mask_selector=d:(0,c.R)(5,t)},get block_class(){return"nr-block"},get ignore_class(){return"nr-ignore"},get mask_text_class(){return"nr-mask"},get block_selector(){return e.block_selector},set block_selector(t){a(t)?e.block_selector+=",".concat(t):""!==t&&(0,c.R)(6,t)},get mask_input_options(){return e.mask_input_options},set mask_input_options(t){t&&"object"==typeof t?e.mask_input_options={...t,password:!0}:(0,c.R)(7,t)}},session_trace:{enabled:!0,autoStart:!0},soft_navigations:{enabled:!0,autoStart:!0},spa:{enabled:!0,autoStart:!0},ssl:void 0,user_actions:{enabled:!0,elementAttributes:["id","className","tagName","type"]}}},g={},p="All configuration objects require an agent identifier!";function m(e){if(!e)throw new Error(p);if(!g[e])throw new Error("Configuration for ".concat(e," was never set"));return g[e]}function h(e,t){if(!e)throw new Error(p);g[e]=(0,l.a)(t,f());const r=(0,u.nY)(e);r&&(r.init=g[e])}function v(e,t){if(!e)throw new Error(p);var r=m(e);if(r){for(var n=t.split("."),i=0;i<n.length-1;i++)if("object"!=typeof(r=r[n[i]]))return;r=r[n[n.length-1]]}return r}},371:(e,t,r)=>{"use strict";r.d(t,{V:()=>f,f:()=>d});var n=r(122),i=r(384),o=r(154),a=r(324);let s=0;const c={buildEnv:a.F3,distMethod:a.Xs,version:a.xv,originTime:o.WN},u={customTransaction:void 0,disabled:!1,isolatedBacklog:!1,loaderType:void 0,maxBytes:3e4,onerror:void 0,ptid:void 0,releaseIds:{},appMetadata:{},session:void 0,denyList:void 0,timeKeeper:void 0,obfuscator:void 0,harvester:void 0},l={};function d(e){if(!e)throw new Error("All runtime objects require an agent identifier!");if(!l[e])throw new Error("Runtime for ".concat(e," was never set"));return l[e]}function f(e,t){if(!e)throw new Error("All runtime objects require an agent identifier!");l[e]={...(0,n.a)(t,u),...c},Object.hasOwnProperty.call(l[e],"harvestCount")||Object.defineProperty(l[e],"harvestCount",{get:()=>++s});const r=(0,i.nY)(e);r&&(r.runtime=l[e])}},324:(e,t,r)=>{"use strict";r.d(t,{F3:()=>i,Xs:()=>o,xv:()=>n});const n="1.283.2",i="PROD",o="CDN"},154:(e,t,r)=>{"use strict";r.d(t,{OF:()=>c,RI:()=>i,WN:()=>l,bv:()=>o,gm:()=>a,mw:()=>s,sb:()=>u});var n=r(863);const i="undefined"!=typeof window&&!!window.document,o="undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self.navigator instanceof WorkerNavigator||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis.navigator instanceof WorkerNavigator),a=i?window:"undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis),s=Boolean("hidden"===a?.document?.visibilityState),c=/iPad|iPhone|iPod/.test(a.navigator?.userAgent),u=c&&"undefined"==typeof SharedWorker,l=((()=>{const e=a.navigator?.userAgent?.match(/Firefox[/\s](\d+\.\d+)/);Array.isArray(e)&&e.length>=2&&e[1]})(),Date.now()-(0,n.t)())},687:(e,t,r)=>{"use strict";r.d(t,{Ak:()=>c,Ze:()=>d,x3:()=>u});var n=r(836),i=r(606),o=r(860),a=r(646);const s={};function c(e,t){const r={staged:!1,priority:o.P3[t]||0};l(e),s[e].get(t)||s[e].set(t,r)}function u(e,t){e&&s[e]&&(s[e].get(t)&&s[e].delete(t),g(e,t,!1),s[e].size&&f(e))}function l(e){if(!e)throw new Error("agentIdentifier required");s[e]||(s[e]=new Map)}function d(e="",t="feature",r=!1){if(l(e),!e||!s[e].get(t)||r)return g(e,t);s[e].get(t).staged=!0,f(e)}function f(e){const t=Array.from(s[e]);t.every((([e,t])=>t.staged))&&(t.sort(((e,t)=>e[1].priority-t[1].priority)),t.forEach((([t])=>{s[e].delete(t),g(e,t)})))}function g(e,t,r=!0){const o=e?n.ee.get(e):n.ee,s=i.i.handlers;if(!o.aborted&&o.backlog&&s){if(r){const e=o.backlog[t],r=s[t];if(r){for(let t=0;e&&t<e.length;++t)p(e[t],r);Object.entries(r).forEach((([e,t])=>{Object.values(t||{}).forEach((t=>{t[0]?.on&&t[0]?.context()instanceof a.y&&t[0].on(e,t[1])}))}))}}o.isolatedBacklog||delete s[t],o.backlog[t]=null,o.emit("drain-"+t,[])}}function p(e,t){var r=e[1];Object.values(t[r]||{}).forEach((t=>{var r=e[0];if(t[0]===r){var n=t[1],i=e[3],o=e[2];n.apply(i,o)}}))}},836:(e,t,r)=>{"use strict";r.d(t,{P:()=>c,ee:()=>u});var n=r(384),i=r(990),o=r(371),a=r(646),s=r(607);const c="nr@context:".concat(s.W),u=function e(t,r){var n={},s={},l={},d=!1;try{d=16===r.length&&(0,o.f)(r).isolatedBacklog}catch(e){}var f={on:p,addEventListener:p,removeEventListener:function(e,t){var r=n[e];if(!r)return;for(var i=0;i<r.length;i++)r[i]===t&&r.splice(i,1)},emit:function(e,r,n,i,o){!1!==o&&(o=!0);if(u.aborted&&!i)return;t&&o&&t.emit(e,r,n);for(var a=g(n),c=m(e),l=c.length,d=0;d<l;d++)c[d].apply(a,r);var p=v()[s[e]];p&&p.push([f,e,r,a]);return a},get:h,listeners:m,context:g,buffer:function(e,t){const r=v();if(t=t||"feature",f.aborted)return;Object.entries(e||{}).forEach((([e,n])=>{s[n]=t,t in r||(r[t]=[])}))},abort:function(){f._aborted=!0,Object.keys(f.backlog).forEach((e=>{delete f.backlog[e]}))},isBuffering:function(e){return!!v()[s[e]]},debugId:r,backlog:d?{}:t&&"object"==typeof t.backlog?t.backlog:{},isolatedBacklog:d};return Object.defineProperty(f,"aborted",{get:()=>{let e=f._aborted||!1;return e||(t&&(e=t.aborted),e)}}),f;function g(e){return e&&e instanceof a.y?e:e?(0,i.I)(e,c,(()=>new a.y(c))):new a.y(c)}function p(e,t){n[e]=m(e).concat(t)}function m(e){return n[e]||[]}function h(t){return l[t]=l[t]||e(f,t)}function v(){return f.backlog}}(void 0,"globalEE"),l=(0,n.Zm)();l.ee||(l.ee=u)},646:(e,t,r)=>{"use strict";r.d(t,{y:()=>n});class n{constructor(e){this.contextId=e}}},908:(e,t,r)=>{"use strict";r.d(t,{d:()=>n,p:()=>i});var n=r(836).ee.get("handle");function i(e,t,r,i,o){o?(o.buffer([e],i),o.emit(e,t,r)):(n.buffer([e],i),n.emit(e,t,r))}},606:(e,t,r)=>{"use strict";r.d(t,{i:()=>o});var n=r(908);o.on=a;var i=o.handlers={};function o(e,t,r,o){a(o||n.d,i,e,t,r)}function a(e,t,r,i,o){o||(o="feature"),e||(e=n.d);var a=t[o]=t[o]||{};(a[r]=a[r]||[]).push([e,i])}},878:(e,t,r)=>{"use strict";function n(e,t){return{capture:e,passive:!1,signal:t}}function i(e,t,r=!1,i){window.addEventListener(e,t,n(r,i))}function o(e,t,r=!1,i){document.addEventListener(e,t,n(r,i))}r.d(t,{DD:()=>o,jT:()=>n,sp:()=>i})},607:(e,t,r)=>{"use strict";r.d(t,{W:()=>n});const n=(0,r(566).bz)()},566:(e,t,r)=>{"use strict";r.d(t,{LA:()=>s,bz:()=>a});var n=r(154);const i="xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx";function o(e,t){return e?15&e[t]:16*Math.random()|0}function a(){const e=n.gm?.crypto||n.gm?.msCrypto;let t,r=0;return e&&e.getRandomValues&&(t=e.getRandomValues(new Uint8Array(30))),i.split("").map((e=>"x"===e?o(t,r++).toString(16):"y"===e?(3&o()|8).toString(16):e)).join("")}function s(e){const t=n.gm?.crypto||n.gm?.msCrypto;let r,i=0;t&&t.getRandomValues&&(r=t.getRandomValues(new Uint8Array(e)));const a=[];for(var s=0;s<e;s++)a.push(o(r,i++).toString(16));return a.join("")}},614:(e,t,r)=>{"use strict";r.d(t,{BB:()=>a,H3:()=>n,g:()=>u,iL:()=>c,tS:()=>s,uh:()=>i,wk:()=>o});const n="NRBA",i="SESSION",o=144e5,a=18e5,s={STARTED:"session-started",PAUSE:"session-pause",RESET:"session-reset",RESUME:"session-resume",UPDATE:"session-update"},c={SAME_TAB:"same-tab",CROSS_TAB:"cross-tab"},u={OFF:0,FULL:1,ERROR:2}},863:(e,t,r)=>{"use strict";function n(){return Math.floor(performance.now())}r.d(t,{t:()=>n})},944:(e,t,r)=>{"use strict";function n(e,t){"function"==typeof console.debug&&console.debug("New Relic Warning: https://github.com/newrelic/newrelic-browser-agent/blob/main/docs/warning-codes.md#".concat(e),t)}r.d(t,{R:()=>n})},284:(e,t,r)=>{"use strict";r.d(t,{t:()=>c,B:()=>s});var n=r(836),i=r(154);const o="newrelic";const a=new Set,s={};function c(e,t){const r=n.ee.get(t);s[t]??={},e&&"object"==typeof e&&(a.has(t)||(r.emit("rumresp",[e]),s[t]=e,a.add(t),function(e={}){try{i.gm.dispatchEvent(new CustomEvent(o,{detail:e}))}catch(e){}}({loaded:!0})))}},990:(e,t,r)=>{"use strict";r.d(t,{I:()=>i});var n=Object.prototype.hasOwnProperty;function i(e,t,r){if(n.call(e,t))return e[t];var i=r();if(Object.defineProperty&&Object.keys)try{return Object.defineProperty(e,t,{value:i,writable:!0,enumerable:!1}),i}catch(e){}return e[t]=i,i}},389:(e,t,r)=>{"use strict";function n(e,t=500,r={}){const n=r?.leading||!1;let i;return(...r)=>{n&&void 0===i&&(e.apply(this,r),i=setTimeout((()=>{i=clearTimeout(i)}),t)),n||(clearTimeout(i),i=setTimeout((()=>{e.apply(this,r)}),t))}}function i(e){let t=!1;return(...r)=>{t||(t=!0,e.apply(this,r))}}r.d(t,{J:()=>i,s:()=>n})},289:(e,t,r)=>{"use strict";r.d(t,{GG:()=>o,sB:()=>a});var n=r(878);function i(){return"undefined"==typeof document||"complete"===document.readyState}function o(e,t){if(i())return e();(0,n.sp)("load",e,t)}function a(e){if(i())return e();(0,n.DD)("DOMContentLoaded",e)}},384:(e,t,r)=>{"use strict";r.d(t,{NT:()=>o,US:()=>l,Zm:()=>a,bQ:()=>c,dV:()=>s,nY:()=>u,pV:()=>d});var n=r(154),i=r(863);const o={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net"};function a(){return n.gm.NREUM||(n.gm.NREUM={}),void 0===n.gm.newrelic&&(n.gm.newrelic=n.gm.NREUM),n.gm.NREUM}function s(){let e=a();return e.o||(e.o={ST:n.gm.setTimeout,SI:n.gm.setImmediate,CT:n.gm.clearTimeout,XHR:n.gm.XMLHttpRequest,REQ:n.gm.Request,EV:n.gm.Event,PR:n.gm.Promise,MO:n.gm.MutationObserver,FETCH:n.gm.fetch,WS:n.gm.WebSocket}),e}function c(e,t){let r=a();r.initializedAgents??={},t.initializedAt={ms:(0,i.t)(),date:new Date},r.initializedAgents[e]=t}function u(e){let t=a();return t.initializedAgents?.[e]}function l(e,t){a()[e]=t}function d(){return function(){let e=a();const t=e.info||{};e.info={beacon:o.beacon,errorBeacon:o.errorBeacon,...t}}(),function(){let e=a();const t=e.init||{};e.init={...t}}(),s(),function(){let e=a();const t=e.loader_config||{};e.loader_config={...t}}(),a()}},843:(e,t,r)=>{"use strict";r.d(t,{u:()=>i});var n=r(878);function i(e,t=!1,r,i){(0,n.DD)("visibilitychange",(function(){if(t)return void("hidden"===document.visibilityState&&e());e(document.visibilityState)}),r,i)}},434:(e,t,r)=>{"use strict";r.d(t,{Jt:()=>o,YM:()=>c});var n=r(836),i=r(607);const o="nr@original:".concat(i.W);var a=Object.prototype.hasOwnProperty,s=!1;function c(e,t){return e||(e=n.ee),r.inPlace=function(e,t,n,i,o){n||(n="");const a="-"===n.charAt(0);for(let s=0;s<t.length;s++){const c=t[s],u=e[c];l(u)||(e[c]=r(u,a?c+n:n,i,c,o))}},r.flag=o,r;function r(t,r,n,s,c){return l(t)?t:(r||(r=""),nrWrapper[o]=t,function(e,t,r){if(Object.defineProperty&&Object.keys)try{return Object.keys(e).forEach((function(r){Object.defineProperty(t,r,{get:function(){return e[r]},set:function(t){return e[r]=t,t}})})),t}catch(e){u([e],r)}for(var n in e)a.call(e,n)&&(t[n]=e[n])}(t,nrWrapper,e),nrWrapper);function nrWrapper(){var o,a,l,d;try{a=this,o=[...arguments],l="function"==typeof n?n(o,a):n||{}}catch(t){u([t,"",[o,a,s],l],e)}i(r+"start",[o,a,s],l,c);try{return d=t.apply(a,o)}catch(e){throw i(r+"err",[o,a,e],l,c),e}finally{i(r+"end",[o,a,d],l,c)}}}function i(r,n,i,o){if(!s||t){var a=s;s=!0;try{e.emit(r,n,i,t,o)}catch(t){u([t,r,n,i],e)}s=a}}}function u(e,t){t||(t=n.ee);try{t.emit("internal-error",e)}catch(e){}}function l(e){return!(e&&"function"==typeof e&&e.apply&&!e[o])}},993:(e,t,r)=>{"use strict";r.d(t,{A$:()=>o,ET:()=>a,p_:()=>i});var n=r(860);const i={ERROR:"ERROR",WARN:"WARN",INFO:"INFO",DEBUG:"DEBUG",TRACE:"TRACE"},o={OFF:0,ERROR:1,WARN:2,INFO:3,DEBUG:4,TRACE:5},a="log";n.K7.logging},773:(e,t,r)=>{"use strict";r.d(t,{z_:()=>o,XG:()=>s,TZ:()=>n,rs:()=>i,xV:()=>a});r(154),r(566),r(384);const n=r(860).K7.metrics,i="sm",o="cm",a="storeSupportabilityMetrics",s="storeEventMetrics"},630:(e,t,r)=>{"use strict";r.d(t,{T:()=>n});const n=r(860).K7.pageViewEvent},782:(e,t,r)=>{"use strict";r.d(t,{T:()=>n});const n=r(860).K7.pageViewTiming},344:(e,t,r)=>{"use strict";r.d(t,{G4:()=>i});var n=r(614);r(860).K7.sessionReplay;const i={RECORD:"recordReplay",PAUSE:"pauseReplay",REPLAY_RUNNING:"replayRunning",ERROR_DURING_REPLAY:"errorDuringReplay"};n.g.ERROR,n.g.FULL,n.g.OFF},234:(e,t,r)=>{"use strict";r.d(t,{W:()=>o});var n=r(836),i=r(687);class o{constructor(e,t){this.agentIdentifier=e,this.ee=n.ee.get(e),this.featureName=t,this.blocked=!1}deregisterDrain(){(0,i.x3)(this.agentIdentifier,this.featureName)}}},603:(e,t,r)=>{"use strict";r.d(t,{j:()=>K});var n=r(860),i=r(555),o=r(371),a=r(908),s=r(836),c=r(687),u=r(289),l=r(154),d=r(944),f=r(773),g=r(384),p=r(344);const m=["setErrorHandler","finished","addToTrace","addRelease","recordCustomEvent","addPageAction","setCurrentRouteName","setPageViewName","setCustomAttribute","interaction","noticeError","setUserId","setApplicationVersion","start",p.G4.RECORD,p.G4.PAUSE,"log","wrapLogger"],h=["setErrorHandler","finished","addToTrace","addRelease"];var v=r(863),b=r(614),y=r(993);var w=r(646),R=r(434);const A=new Map;function E(e,t,r,n){if("object"!=typeof t||!t||"string"!=typeof r||!r||"function"!=typeof t[r])return(0,d.R)(29);const i=function(e){return(e||s.ee).get("logger")}(e),o=(0,R.YM)(i),a=new w.y(s.P);a.level=n.level,a.customAttributes=n.customAttributes;const c=t[r]?.[R.Jt]||t[r];return A.set(c,a),o.inPlace(t,[r],"wrap-logger-",(()=>A.get(c))),i}function _(){const e=(0,g.pV)();m.forEach((t=>{e[t]=(...r)=>function(t,...r){let n=[];return Object.values(e.initializedAgents).forEach((e=>{e&&e.api?e.exposed&&e.api[t]&&n.push(e.api[t](...r)):(0,d.R)(38,t)})),n.length>1?n:n[0]}(t,...r)}))}const x={};function N(e,t,g=!1){t||(0,c.Ak)(e,"api");const m={};var w=s.ee.get(e),R=w.get("tracer");x[e]=b.g.OFF,w.on(p.G4.REPLAY_RUNNING,(t=>{x[e]=t}));var A="api-",_=A+"ixn-";function N(t,r,n,o){const a=(0,i.Vp)(e);return null===r?delete a.jsAttributes[t]:(0,i.x1)(e,{...a,jsAttributes:{...a.jsAttributes,[t]:r}}),j(A,n,!0,o||null===r?"session":void 0)(t,r)}function k(){}m.log=function(e,{customAttributes:t={},level:r=y.p_.INFO}={}){(0,a.p)(f.xV,["API/log/called"],void 0,n.K7.metrics,w),function(e,t,r={},i=y.p_.INFO){(0,a.p)(f.xV,["API/logging/".concat(i.toLowerCase(),"/called")],void 0,n.K7.metrics,e),(0,a.p)(y.ET,[(0,v.t)(),t,r,i],void 0,n.K7.logging,e)}(w,e,t,r)},m.wrapLogger=(e,t,{customAttributes:r={},level:i=y.p_.INFO}={})=>{(0,a.p)(f.xV,["API/wrapLogger/called"],void 0,n.K7.metrics,w),E(w,e,t,{customAttributes:r,level:i})},h.forEach((e=>{m[e]=j(A,e,!0,"api")})),m.addPageAction=j(A,"addPageAction",!0,n.K7.genericEvents),m.recordCustomEvent=j(A,"recordCustomEvent",!0,n.K7.genericEvents),m.setPageViewName=function(t,r){if("string"==typeof t)return"/"!==t.charAt(0)&&(t="/"+t),(0,o.f)(e).customTransaction=(r||"http://custom.transaction")+t,j(A,"setPageViewName",!0)()},m.setCustomAttribute=function(e,t,r=!1){if("string"==typeof e){if(["string","number","boolean"].includes(typeof t)||null===t)return N(e,t,"setCustomAttribute",r);(0,d.R)(40,typeof t)}else(0,d.R)(39,typeof e)},m.setUserId=function(e){if("string"==typeof e||null===e)return N("enduser.id",e,"setUserId",!0);(0,d.R)(41,typeof e)},m.setApplicationVersion=function(e){if("string"==typeof e||null===e)return N("application.version",e,"setApplicationVersion",!1);(0,d.R)(42,typeof e)},m.start=()=>{try{(0,a.p)(f.xV,["API/start/called"],void 0,n.K7.metrics,w),w.emit("manual-start-all")}catch(e){(0,d.R)(23,e)}},m[p.G4.RECORD]=function(){(0,a.p)(f.xV,["API/recordReplay/called"],void 0,n.K7.metrics,w),(0,a.p)(p.G4.RECORD,[],void 0,n.K7.sessionReplay,w)},m[p.G4.PAUSE]=function(){(0,a.p)(f.xV,["API/pauseReplay/called"],void 0,n.K7.metrics,w),(0,a.p)(p.G4.PAUSE,[],void 0,n.K7.sessionReplay,w)},m.interaction=function(e){return(new k).get("object"==typeof e?e:{})};const T=k.prototype={createTracer:function(e,t){var r={},i=this,o="function"==typeof t;return(0,a.p)(f.xV,["API/createTracer/called"],void 0,n.K7.metrics,w),g||(0,a.p)(_+"tracer",[(0,v.t)(),e,r],i,n.K7.spa,w),function(){if(R.emit((o?"":"no-")+"fn-start",[(0,v.t)(),i,o],r),o)try{return t.apply(this,arguments)}catch(e){const t="string"==typeof e?new Error(e):e;throw R.emit("fn-err",[arguments,this,t],r),t}finally{R.emit("fn-end",[(0,v.t)()],r)}}}};function j(e,t,r,i){return function(){return(0,a.p)(f.xV,["API/"+t+"/called"],void 0,n.K7.metrics,w),i&&(0,a.p)(e+t,[r?(0,v.t)():performance.now(),...arguments],r?null:this,i,w),r?void 0:this}}function I(){r.e(296).then(r.bind(r,778)).then((({setAPI:t})=>{t(e),(0,c.Ze)(e,"api")})).catch((e=>{(0,d.R)(27,e),w.abort()}))}return["actionText","setName","setAttribute","save","ignore","onEnd","getContext","end","get"].forEach((e=>{T[e]=j(_,e,void 0,g?n.K7.softNav:n.K7.spa)})),m.setCurrentRouteName=g?j(_,"routeName",void 0,n.K7.softNav):j(A,"routeName",!0,n.K7.spa),m.noticeError=function(t,r){"string"==typeof t&&(t=new Error(t)),(0,a.p)(f.xV,["API/noticeError/called"],void 0,n.K7.metrics,w),(0,a.p)("err",[t,(0,v.t)(),!1,r,!!x[e]],void 0,n.K7.jserrors,w)},l.RI?(0,u.GG)((()=>I()),!0):I(),m}var k=r(217),T=r(122);const j={accountID:void 0,trustKey:void 0,agentID:void 0,licenseKey:void 0,applicationID:void 0,xpid:void 0},I={};var O=r(284);const S=e=>{const t=e.startsWith("http");e+="/",r.p=t?e:"https://"+e};let P=!1;function K(e,t={},r,n){let{init:a,info:c,loader_config:u,runtime:d={},exposed:f=!0}=t;d.loaderType=r;const p=(0,g.pV)();c||(a=p.init,c=p.info,u=p.loader_config),(0,k.xN)(e.agentIdentifier,a||{}),function(e,t){if(!e)throw new Error("All loader-config objects require an agent identifier!");I[e]=(0,T.a)(t,j);const r=(0,g.nY)(e);r&&(r.loader_config=I[e])}(e.agentIdentifier,u||{}),c.jsAttributes??={},l.bv&&(c.jsAttributes.isWorker=!0),(0,i.x1)(e.agentIdentifier,c);const m=(0,k.D0)(e.agentIdentifier),h=[c.beacon,c.errorBeacon];P||(m.proxy.assets&&(S(m.proxy.assets),h.push(m.proxy.assets)),m.proxy.beacon&&h.push(m.proxy.beacon),_(),(0,g.US)("activatedFeatures",O.B),e.runSoftNavOverSpa&&=!0===m.soft_navigations.enabled&&m.feature_flags.includes("soft_nav")),d.denyList=[...m.ajax.deny_list||[],...m.ajax.block_internal?h:[]],d.ptid=e.agentIdentifier,(0,o.V)(e.agentIdentifier,d),e.ee=s.ee.get(e.agentIdentifier),void 0===e.api&&(e.api=N(e.agentIdentifier,n,e.runSoftNavOverSpa)),void 0===e.exposed&&(e.exposed=f),P=!0}},374:(e,t,r)=>{r.nc=(()=>{try{return document?.currentScript?.nonce}catch(e){}return""})()},860:(e,t,r)=>{"use strict";r.d(t,{$J:()=>u,K7:()=>s,P3:()=>c,XX:()=>i,qY:()=>n,v4:()=>a});const n="events",i="jserrors",o="browser/blobs",a="rum",s={ajax:"ajax",genericEvents:"generic_events",jserrors:i,logging:"logging",metrics:"metrics",pageAction:"page_action",pageViewEvent:"page_view_event",pageViewTiming:"page_view_timing",sessionReplay:"session_replay",sessionTrace:"session_trace",softNav:"soft_navigations",spa:"spa"},c={[s.pageViewEvent]:1,[s.pageViewTiming]:2,[s.metrics]:3,[s.jserrors]:4,[s.spa]:5,[s.ajax]:6,[s.sessionTrace]:7,[s.softNav]:8,[s.sessionReplay]:9,[s.logging]:10,[s.genericEvents]:11},u={[s.pageViewEvent]:a,[s.pageViewTiming]:n,[s.ajax]:n,[s.spa]:n,[s.softNav]:n,[s.metrics]:i,[s.jserrors]:i,[s.sessionTrace]:o,[s.sessionReplay]:o,[s.logging]:"browser/logs",[s.genericEvents]:"ins"}}},n={};function i(e){var t=n[e];if(void 0!==t)return t.exports;var o=n[e]={exports:{}};return r[e](o,o.exports,i),o.exports}i.m=r,i.d=(e,t)=>{for(var r in t)i.o(t,r)&&!i.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:t[r]})},i.f={},i.e=e=>Promise.all(Object.keys(i.f).reduce(((t,r)=>(i.f[r](e,t),t)),[])),i.u=e=>"nr-rum-1.283.2.min.js",i.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),e={},t="NRBA-1.283.2.PROD:",i.l=(r,n,o,a)=>{if(e[r])e[r].push(n);else{var s,c;if(void 0!==o)for(var u=document.getElementsByTagName("script"),l=0;l<u.length;l++){var d=u[l];if(d.getAttribute("src")==r||d.getAttribute("data-webpack")==t+o){s=d;break}}if(!s){c=!0;var f={296:"sha512-2Y8GMAOGF658KnXzOZ/v+DlLch8TBFvV0tTNnOy9wrpvtDa1t5CdZMyX+LubTymBlzPp6NUjllBghMCZqXBPmg=="};(s=document.createElement("script")).charset="utf-8",s.timeout=120,i.nc&&s.setAttribute("nonce",i.nc),s.setAttribute("data-webpack",t+o),s.src=r,0!==s.src.indexOf(window.location.origin+"/")&&(s.crossOrigin="anonymous"),f[a]&&(s.integrity=f[a])}e[r]=[n];var g=(t,n)=>{s.onerror=s.onload=null,clearTimeout(p);var i=e[r];if(delete e[r],s.parentNode&&s.parentNode.removeChild(s),i&&i.forEach((e=>e(n))),t)return t(n)},p=setTimeout(g.bind(null,void 0,{type:"timeout",target:s}),12e4);s.onerror=g.bind(null,s.onerror),s.onload=g.bind(null,s.onload),c&&document.head.appendChild(s)}},i.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.p="https://js-agent.newrelic.com/",(()=>{var e={374:0,840:0};i.f.j=(t,r)=>{var n=i.o(e,t)?e[t]:void 0;if(0!==n)if(n)r.push(n[2]);else{var o=new Promise(((r,i)=>n=e[t]=[r,i]));r.push(n[2]=o);var a=i.p+i.u(t),s=new Error;i.l(a,(r=>{if(i.o(e,t)&&(0!==(n=e[t])&&(e[t]=void 0),n)){var o=r&&("load"===r.type?"missing":r.type),a=r&&r.target&&r.target.src;s.message="Loading chunk "+t+" failed.\n("+o+": "+a+")",s.name="ChunkLoadError",s.type=o,s.request=a,n[1](s)}}),"chunk-"+t,t)}};var t=(t,r)=>{var n,o,[a,s,c]=r,u=0;if(a.some((t=>0!==e[t]))){for(n in s)i.o(s,n)&&(i.m[n]=s[n]);if(c)c(i)}for(t&&t(r);u<a.length;u++)o=a[u],i.o(e,o)&&e[o]&&e[o][0](),e[o]=0},r=self["webpackChunk:NRBA-1.283.2.PROD"]=self["webpackChunk:NRBA-1.283.2.PROD"]||[];r.forEach(t.bind(null,0)),r.push=t.bind(null,r.push.bind(r))})(),(()=>{"use strict";i(374);var e=i(944),t=i(344),r=i(566);class n{agentIdentifier;constructor(){this.agentIdentifier=(0,r.LA)(16)}#e(t,...r){if("function"==typeof this.api?.[t])return this.api[t](...r);(0,e.R)(35,t)}addPageAction(e,t){return this.#e("addPageAction",e,t)}recordCustomEvent(e,t){return this.#e("recordCustomEvent",e,t)}setPageViewName(e,t){return this.#e("setPageViewName",e,t)}setCustomAttribute(e,t,r){return this.#e("setCustomAttribute",e,t,r)}noticeError(e,t){return this.#e("noticeError",e,t)}setUserId(e){return this.#e("setUserId",e)}setApplicationVersion(e){return this.#e("setApplicationVersion",e)}setErrorHandler(e){return this.#e("setErrorHandler",e)}addRelease(e,t){return this.#e("addRelease",e,t)}log(e,t){return this.#e("log",e,t)}}class o extends n{#e(t,...r){if("function"==typeof this.api?.[t])return this.api[t](...r);(0,e.R)(35,t)}start(){return this.#e("start")}finished(e){return this.#e("finished",e)}recordReplay(){return this.#e(t.G4.RECORD)}pauseReplay(){return this.#e(t.G4.PAUSE)}addToTrace(e){return this.#e("addToTrace",e)}setCurrentRouteName(e){return this.#e("setCurrentRouteName",e)}interaction(){return this.#e("interaction")}wrapLogger(e,t,r){return this.#e("wrapLogger",e,t,r)}}var a=i(860),s=i(217);const c=Object.values(a.K7);function u(e){const t={};return c.forEach((r=>{t[r]=function(e,t){return!0===(0,s.gD)(t,"".concat(e,".enabled"))}(r,e)})),t}var l=i(603);var d=i(687),f=i(234),g=i(289),p=i(154),m=i(384);const h=e=>p.RI&&!0===(0,s.gD)(e,"privacy.cookies_enabled");function v(e){return!!(0,m.dV)().o.MO&&h(e)&&!0===(0,s.gD)(e,"session_trace.enabled")}var b=i(389);class y extends f.W{constructor(e,t,r=!0){super(e.agentIdentifier,t),this.auto=r,this.abortHandler=void 0,this.featAggregate=void 0,this.onAggregateImported=void 0,!1===e.init[this.featureName].autoStart&&(this.auto=!1),this.auto?(0,d.Ak)(e.agentIdentifier,t):this.ee.on("manual-start-all",(0,b.J)((()=>{(0,d.Ak)(e.agentIdentifier,this.featureName),this.auto=!0,this.importAggregator(e)})))}importAggregator(t,r={}){if(this.featAggregate||!this.auto)return;let n;this.onAggregateImported=new Promise((e=>{n=e}));const o=async()=>{let o;try{if(h(this.agentIdentifier)){const{setupAgentSession:e}=await i.e(296).then(i.bind(i,861));o=e(t)}}catch(t){(0,e.R)(20,t),this.ee.emit("internal-error",[t]),this.featureName===a.K7.sessionReplay&&this.abortHandler?.()}try{if(!this.#t(this.featureName,o))return(0,d.Ze)(this.agentIdentifier,this.featureName),void n(!1);const{lazyFeatureLoader:e}=await i.e(296).then(i.bind(i,103)),{Aggregate:a}=await e(this.featureName,"aggregate");this.featAggregate=new a(t,r),t.runtime.harvester.initializedAggregates.push(this.featAggregate),n(!0)}catch(t){(0,e.R)(34,t),this.abortHandler?.(),(0,d.Ze)(this.agentIdentifier,this.featureName,!0),n(!1),this.ee&&this.ee.abort()}};p.RI?(0,g.GG)((()=>o()),!0):o()}#t(e,t){switch(e){case a.K7.sessionReplay:return v(this.agentIdentifier)&&!!t;case a.K7.sessionTrace:return!!t;default:return!0}}}var w=i(630);class R extends y{static featureName=w.T;constructor(e,t=!0){super(e,w.T,t),this.importAggregator(e)}}var A=i(908),E=i(843),_=i(878),x=i(782),N=i(863);class k extends y{static featureName=x.T;constructor(e,t=!0){super(e,x.T,t),p.RI&&((0,E.u)((()=>(0,A.p)("docHidden",[(0,N.t)()],void 0,x.T,this.ee)),!0),(0,_.sp)("pagehide",(()=>(0,A.p)("winPagehide",[(0,N.t)()],void 0,x.T,this.ee))),this.importAggregator(e))}}var T=i(773);class j extends y{static featureName=T.TZ;constructor(e,t=!0){super(e,T.TZ,t),this.importAggregator(e)}}new class extends o{constructor(t){super(),p.gm?(this.features={},(0,m.bQ)(this.agentIdentifier,this),this.desiredFeatures=new Set(t.features||[]),this.desiredFeatures.add(R),this.runSoftNavOverSpa=[...this.desiredFeatures].some((e=>e.featureName===a.K7.softNav)),(0,l.j)(this,t,t.loaderType||"agent"),this.run()):(0,e.R)(21)}get config(){return{info:this.info,init:this.init,loader_config:this.loader_config,runtime:this.runtime}}run(){try{const t=u(this.agentIdentifier),r=[...this.desiredFeatures];r.sort(((e,t)=>a.P3[e.featureName]-a.P3[t.featureName])),r.forEach((r=>{if(!t[r.featureName]&&r.featureName!==a.K7.pageViewEvent)return;if(this.runSoftNavOverSpa&&r.featureName===a.K7.spa)return;if(!this.runSoftNavOverSpa&&r.featureName===a.K7.softNav)return;const n=function(e){switch(e){case a.K7.ajax:return[a.K7.jserrors];case a.K7.sessionTrace:return[a.K7.ajax,a.K7.pageViewEvent];case a.K7.sessionReplay:return[a.K7.sessionTrace];case a.K7.pageViewTiming:return[a.K7.pageViewEvent];default:return[]}}(r.featureName).filter((e=>!(e in this.features)));n.length>0&&(0,e.R)(36,{targetFeature:r.featureName,missingDependencies:n}),this.features[r.featureName]=new r(this)}))}catch(t){(0,e.R)(22,t);for(const e in this.features)this.features[e].abortHandler?.();const r=(0,m.Zm)();delete r.initializedAgents[this.agentIdentifier]?.api,delete r.initializedAgents[this.agentIdentifier]?.features,delete this.sharedAggregator;return r.ee.get(this.agentIdentifier).abort(),!1}}}({features:[R,k,j],loaderType:"lite"})})()})();</script>
|
||
<noscript><style>form.antibot * :not(.antibot-message) { display: none !important; }</style>
|
||
</noscript><script async src="https://www.googletagmanager.com/gtag/js?id=G-HEQ0YF2VYL"></script>
|
||
<script>window.dataLayer = window.dataLayer || [];function gtag(){dataLayer.push(arguments)};gtag("js", new Date());gtag("set", "developer_id.dMDhkMT", true);gtag('set', {'cookie_flags': 'SameSite=None;Secure', 'cookie_domain': 'www.nist.gov'});gtag("config", "G-HEQ0YF2VYL", {"groups":"default","page_placeholder":"PLACEHOLDER_page_location","link_attribution":true,"allow_ad_personalization_signals":false});gtag("config", "G-CSLL4ZEK4L", {"groups":"default","page_placeholder":"PLACEHOLDER_page_location","link_attribution":true,"allow_ad_personalization_signals":false});gtag("event", "custom", {"node_title":"Improving the TENOR of Labeling: Re-evaluating Topic Models for Content Analysis","node_id":"1848706","content_type":"Publication"});</script>
|
||
<meta name="description" content="Topic models are a popular tool for understanding text collections, but their evaluation has been a point of contention" />
|
||
<link rel="canonical" href="https://www.nist.gov/publications/improving-tenor-labeling-re-evaluating-topic-models-content-analysis" />
|
||
<link rel="shortlink" href="https://www.nist.gov/node/1848706" />
|
||
<meta name="citation_title" content="Improving the TENOR of Labeling: Re-evaluating Topic Models for Content Analysis" />
|
||
<meta name="citation_author" content="Zongxia Li" />
|
||
<meta name="citation_author" content="Andrew Mao" />
|
||
<meta name="citation_author" content="Jordan Boyd-Graber" />
|
||
<meta name="citation_author" content="Daniel Stephens" />
|
||
<meta name="citation_author" content="Emily Walpole" />
|
||
<meta name="citation_author" content="Alden A. Dima" />
|
||
<meta name="citation_author" content="Juan Fung" />
|
||
<meta name="citation_publication_date" content="2024-03-23T08:00-04:00" />
|
||
<meta name="citation_pdf_url" content="https://tsapps.nist.gov/publication/get_pdf.cfm?pub_id=956761" />
|
||
<meta property="og:site_name" content="NIST" />
|
||
<meta property="og:type" content="Article" />
|
||
<meta property="og:url" content="https://www.nist.gov/publications/improving-tenor-labeling-re-evaluating-topic-models-content-analysis" />
|
||
<meta property="og:title" content="Improving the TENOR of Labeling: Re-evaluating Topic Models for Content Analysis" />
|
||
<meta property="og:description" content="Topic models are a popular tool for understanding text collections, but their evaluation has been a point of contention" />
|
||
<meta property="og:image" content="https://www.nist.gov/themes/custom/nist_www/img/homepage/nist_mark.png" />
|
||
<meta property="article:author" content="Zongxia Li" />
|
||
<meta property="article:author" content="Andrew Mao" />
|
||
<meta property="article:author" content="Jordan Boyd-Graber" />
|
||
<meta property="article:author" content="Daniel Stephens" />
|
||
<meta property="article:author" content="Emily Walpole" />
|
||
<meta property="article:author" content="Alden A. Dima" />
|
||
<meta property="article:author" content="Juan Fung" />
|
||
<meta property="article:published_time" content="2024-03-23T08:00-04:00" />
|
||
<meta property="article:modified_time" content="2024-05-16T16:05-04:00" />
|
||
<meta name="dcterms.title" content="Improving the TENOR of Labeling: Re-evaluating Topic Models for Content Analysis" />
|
||
<meta name="dcterms.creator" content="Zongxia Li, Andrew Mao, Jordan Boyd-Graber, Daniel Stephens, Emily Walpole, Alden A. Dima, Juan Fung" />
|
||
<meta name="dcterms.description" content="Topic models are a popular tool for understanding text collections, but their evaluation has been a point of contention" />
|
||
<meta name="dcterms.publisher" content="Zongxia Li, Andrew Mao, Jordan Boyd-Graber, Daniel Stephens, Emily Walpole, Alden A. Dima, Juan Fung" />
|
||
<meta name="dcterms.date" content="2024-03-23T08:00-04:00" />
|
||
<meta name="dcterms.type" content="text" />
|
||
<meta name="dcterms.format" content="text/html" />
|
||
<meta name="dcterms.identifier" content="https://www.nist.gov/publications/improving-tenor-labeling-re-evaluating-topic-models-content-analysis" />
|
||
<meta name="dcterms.source" content="NIST" />
|
||
<meta name="dcterms.created" content="2024-03-23T08:00-04:00" />
|
||
<meta name="dcterms.modified" content="2024-05-16T16:05-04:00" />
|
||
<meta name="twitter:card" content="summary_large_image" />
|
||
<meta name="twitter:description" content="Topic models are a popular tool for understanding text collections, but their evaluation has been a point of contention" />
|
||
<meta name="twitter:site" content="NIST" />
|
||
<meta name="twitter:title" content="Improving the TENOR of Labeling: Re-evaluating Topic Models for Content Analysis" />
|
||
<meta name="google-site-verification" content="QMu0ODkER3rN5hLcMLqNVf7e3bkjYsNLTuhqfH48jCA" />
|
||
<meta name="Generator" content="Drupal 10 (https://www.drupal.org)" />
|
||
<meta name="MobileOptimized" content="width" />
|
||
<meta name="HandheldFriendly" content="true" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<meta name="nist_search_modified" class="elastic" content="2024-05-16T16:05:00-04:00" />
|
||
<meta name="nist_search_bundle" class="elastic" content="publication" />
|
||
<link rel="icon" href="/themes/custom/nist_www/favicon.ico" type="image/vnd.microsoft.icon" />
|
||
|
||
<title>Improving the TENOR of Labeling: Re-evaluating Topic Models for Content Analysis | NIST</title>
|
||
<link rel="stylesheet" media="all" href="/sites/default/files/css/css_zb71YZl620uz4kH3pYu0tKyafjffOtzVOU80-0NBOTI.css?delta=0&language=en&theme=nist_www&include=eJxFjIEKwzAIRH8o1E8KJpFOamKJttn-flm3UZDDO-5dwrxF13k7pPuPnLWFxubRH1RpZioJe6xaoLBhEorCbQv2Mqc6WaOrD6toQvmyY4y_p6d_-lD6saMsPxtOpmFw6TK3D6GbrJy7Gju9ASWrQTg" />
|
||
<link rel="stylesheet" media="all" href="/sites/default/files/css/css_kGS0ApDpFo-_3JI4Ijxg4RCWlTqsImK3lqH7bCv5v2U.css?delta=1&language=en&theme=nist_www&include=eJxFjIEKwzAIRH8o1E8KJpFOamKJttn-flm3UZDDO-5dwrxF13k7pPuPnLWFxubRH1RpZioJe6xaoLBhEorCbQv2Mqc6WaOrD6toQvmyY4y_p6d_-lD6saMsPxtOpmFw6TK3D6GbrJy7Gju9ASWrQTg" />
|
||
<link rel="stylesheet" media="print" href="/sites/default/files/css/css_-IXwLuTUdm8IGHxib8GX8wWKw3LjKsNWSYYskZ5fEGg.css?delta=2&language=en&theme=nist_www&include=eJxFjIEKwzAIRH8o1E8KJpFOamKJttn-flm3UZDDO-5dwrxF13k7pPuPnLWFxubRH1RpZioJe6xaoLBhEorCbQv2Mqc6WaOrD6toQvmyY4y_p6d_-lD6saMsPxtOpmFw6TK3D6GbrJy7Gju9ASWrQTg" />
|
||
<link rel="stylesheet" media="all" href="/sites/default/files/css/css_jmXeSZNqhh6ayjrVIlS8rDw3mcnQaE5dzE2ZwwPSzNA.css?delta=3&language=en&theme=nist_www&include=eJxFjIEKwzAIRH8o1E8KJpFOamKJttn-flm3UZDDO-5dwrxF13k7pPuPnLWFxubRH1RpZioJe6xaoLBhEorCbQv2Mqc6WaOrD6toQvmyY4y_p6d_-lD6saMsPxtOpmFw6TK3D6GbrJy7Gju9ASWrQTg" />
|
||
|
||
<script src="/sites/default/files/js/js_GM8kl6wk698qKTTwusj83bCoGHHtOZ1UinjY1sE_Qbk.js?scope=header&delta=0&language=en&theme=nist_www&include=eJxdjW0OwjAIhi805UiErtjhWFlatHp7O53RLOEHPLwfgcYZ3fqsEH47XuuQzJIyUiZ9uowVjmDIUh0v0lm0ltUoopceIjlBos-7irMsa7E7wxFsLW_mEy_ce00DFVwsQpRKoQer5PmtgaQWSE-7BSamyOV7YmvtT8EP34wQy20lPe_nC6DrXYU"></script>
|
||
|
||
</head>
|
||
<body class="node-1848706">
|
||
<a href="#main-content" class="visually-hidden focusable" data-elastic-exclude>
|
||
Skip to main content
|
||
</a>
|
||
|
||
<div class="dialog-off-canvas-main-canvas" data-off-canvas-main-canvas>
|
||
|
||
<!-- nist-index-ignore-start -->
|
||
<section data-elastic-exclude class="usa-banner" aria-label="Official government website">
|
||
<div class="usa-accordion">
|
||
<header class="usa-banner__header">
|
||
<div class="usa-banner__inner">
|
||
<div class="grid-col-auto">
|
||
<img class="usa-banner__header-flag" src="/libraries/nist-component-library/dist/img/us_flag_small.png" alt="U.S. flag">
|
||
</div>
|
||
<div class="grid-col-fill tablet:grid-col-auto">
|
||
<p class="usa-banner__header-text">An official website of the United States government</p>
|
||
<p class="usa-banner__header-action" aria-hidden="true">Here’s how you know</p>
|
||
</div>
|
||
<button class="usa-accordion__button usa-banner__button" aria-expanded="false" aria-controls="gov-banner-default">
|
||
<span class="usa-banner__button-text">Here’s how you know</span>
|
||
</button>
|
||
</div>
|
||
</header>
|
||
<div class="usa-banner__content usa-accordion__content" id="gov-banner-default">
|
||
<div class="grid-row grid-gap-lg">
|
||
<div class="usa-banner__guidance tablet:grid-col-6">
|
||
<img class="usa-banner__icon usa-media-block__img" src="/libraries/nist-component-library/dist/img/icon-dot-gov.svg" role="img" alt="" aria-hidden="true">
|
||
<div class="usa-media-block__body">
|
||
<p>
|
||
<strong>Official websites use .gov</strong>
|
||
<br/>
|
||
A <strong>.gov</strong> website belongs to an official government organization in the United States.
|
||
</p>
|
||
</div>
|
||
</div>
|
||
<div class="usa-banner__guidance tablet:grid-col-6">
|
||
<img class="usa-banner__icon usa-media-block__img" src="/libraries/nist-component-library/dist/img/icon-https.svg" role="img" alt="" aria-hidden="true">
|
||
<div class="usa-media-block__body">
|
||
<p>
|
||
<strong>Secure .gov websites use HTTPS</strong>
|
||
<br/>
|
||
A <strong>lock</strong> ( <span class="icon-lock">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="52" height="64" viewbox="0 0 52 64" class="usa-banner__lock-image" role="img" aria-labelledby="banner-lock-title banner-lock-description" focusable="false">
|
||
<title id="banner-lock-title">Lock</title>
|
||
<desc id="banner-lock-description">A locked padlock</desc><path fill="#000000" fill-rule="evenodd" d="M26 0c10.493 0 19 8.507 19 19v9h3a4 4 0 0 1 4 4v28a4 4 0 0 1-4 4H4a4 4 0 0 1-4-4V32a4 4 0 0 1 4-4h3v-9C7 8.507 15.507 0 26 0zm0 8c-5.979 0-10.843 4.77-10.996 10.712L15 19v9h22v-9c0-6.075-4.925-11-11-11z"/></svg>
|
||
</span>
|
||
) or <strong>https://</strong> means you’ve safely connected to the .gov website. Share sensitive information only on official, secure websites.
|
||
</p>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<!-- nist-index-ignore-end -->
|
||
|
||
|
||
<div data-elastic-exclude>
|
||
<!-- nist-index-ignore-start -->
|
||
<div class="nist-print-header" style="display:none;">
|
||
<p class="nist-print-header__url">https://www.nist.gov/publications/improving-tenor-labeling-re-evaluating-topic-models-content-analysis</p>
|
||
<img class="nist-print-header__logo" width="289" height="38" src="/libraries/nist-component-library/dist/img/logo/nist_logo_sidestack.svg" alt="National Institute of Standards and Technology" />
|
||
</div>
|
||
|
||
|
||
<div class="usa-overlay"></div>
|
||
<header class="usa-header nist-header--www nist-header--minimal" role="banner">
|
||
<div class="usa-navbar">
|
||
<div class="usa-logo flex-fill">
|
||
<a href="/" title="National Institute of Standards and Technology" aria-label="Home">
|
||
<img src="/libraries/nist-component-library/dist/img/logo/logo.svg" alt="National Institute of Standards and Technology" width="300px" height="80px" />
|
||
</a>
|
||
</div>
|
||
<div class="usa-header__right grid-row flex-auto">
|
||
<div class="grid-col-fill nist-header__search-group">
|
||
|
||
<form class="usa-search usa-search--small"
|
||
accept-charset="UTF-8"
|
||
action="/search"
|
||
id="search_form"
|
||
method="get">
|
||
<div role="search">
|
||
<label class="usa-sr-only" for="search-form">Search NIST</label>
|
||
<input class="usa-input" id="search-form" type="search" name="s" placeholder="Search NIST" required="" maxlength="128">
|
||
<button class="usa-button" type="submit"><img src="/libraries/nist-component-library/dist/img/usa-icons-bg/search--white.svg" class="usa-search__submit-icon" alt="Search"></button>
|
||
</div>
|
||
</form>
|
||
</div>
|
||
<div class="grid-col-auto padding-left-1">
|
||
<button class="usa-menu-btn">Menu</button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<nav aria-label="Primary navigation" class="usa-nav">
|
||
<div class="usa-nav__inner">
|
||
<button class="usa-nav__close">Close</button>
|
||
<ul class="usa-nav__primary usa-accordion">
|
||
<li class="usa-nav__primary-item">
|
||
<a class="usa-nav__link" href="/publications">Publications</a>
|
||
</li>
|
||
<li class="usa-nav__primary-item">
|
||
<button class="usa-accordion__button usa-nav__link" aria-expanded="false" aria-controls="primary_menu-2"><span>What We Do</span></button>
|
||
<div id="primary_menu-2" class="usa-nav__submenu usa-megamenu">
|
||
<div class="grid-row">
|
||
<div class="tablet:grid-col-fill"><ul class="usa-nav__submenu-list">
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/topics" class="usa-nav__link">All Topics</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/advanced-communications" class="usa-nav__link">Advanced communications</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/artificial-intelligence" class="usa-nav__link">Artificial intelligence</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/bioscience" class="usa-nav__link">Bioscience</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/buildings-construction" class="usa-nav__link">Buildings and construction</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/chemistry" class="usa-nav__link">Chemistry</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/cybersecurity" class="usa-nav__link">Cybersecurity</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/electronics" class="usa-nav__link">Electronics</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/energy" class="usa-nav__link">Energy</a>
|
||
</li>
|
||
</ul></div><div class="column-break tablet:grid-col-fill"><ul class="usa-nav__submenu-list">
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/environment" class="usa-nav__link">Environment</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/fire" class="usa-nav__link">Fire</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/forensic-science" class="usa-nav__link">Forensic science</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/health" class="usa-nav__link">Health</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/information-technology" class="usa-nav__link">Information technology</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/infrastructure" class="usa-nav__link">Infrastructure</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/manufacturing" class="usa-nav__link">Manufacturing</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/materials" class="usa-nav__link">Materials</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/mathematics-statistics" class="usa-nav__link">Mathematics and statistics</a>
|
||
</li>
|
||
</ul></div><div class="column-break tablet:grid-col-fill"><ul class="usa-nav__submenu-list">
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/metrology" class="usa-nav__link">Metrology</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/nanotechnology" class="usa-nav__link">Nanotechnology</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/neutron-research" class="usa-nav__link">Neutron research</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/performance-excellence" class="usa-nav__link">Performance excellence</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/physics" class="usa-nav__link">Physics</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/public-safety" class="usa-nav__link">Public safety</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/quantum-information-science" class="usa-nav__link">Quantum information science</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/resilience" class="usa-nav__link">Resilience</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/standards" class="usa-nav__link">Standards</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/transportation" class="usa-nav__link">Transportation</a>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</li>
|
||
<li class="usa-nav__primary-item">
|
||
<button class="usa-accordion__button usa-nav__link" aria-expanded="false" aria-controls="primary_menu-3"><span>Labs & Major Programs</span></button>
|
||
<div id="primary_menu-3" class="usa-nav__submenu usa-megamenu">
|
||
<div class="grid-row">
|
||
<div class="tablet:grid-col-fill"><ul class="usa-nav__submenu-list">
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/adlp" class="usa-nav__link">Assoc Director of Laboratory Programs</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/laboratories" class="usa-nav__link">Laboratories</a>
|
||
<ul class="usa-nav__submenu-list">
|
||
<li>
|
||
<a href="/ctl" class="usa-nav__link">Communications Technology Laboratory</a>
|
||
</li>
|
||
<li>
|
||
<a href="/el" class="usa-nav__link">Engineering Laboratory</a>
|
||
</li>
|
||
<li>
|
||
<a href="/itl" class="usa-nav__link">Information Technology Laboratory</a>
|
||
</li>
|
||
<li>
|
||
<a href="/mml" class="usa-nav__link">Material Measurement Laboratory</a>
|
||
</li>
|
||
<li>
|
||
<a href="/pml" class="usa-nav__link">Physical Measurement Laboratory</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul></div><div class="column-break tablet:grid-col-fill"><ul class="usa-nav__submenu-list">
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/user-facilities" class="usa-nav__link">User Facilities</a>
|
||
<ul class="usa-nav__submenu-list">
|
||
<li>
|
||
<a href="/ncnr" class="usa-nav__link">NIST Center for Neutron Research</a>
|
||
</li>
|
||
<li>
|
||
<a href="/cnst" class="usa-nav__link">CNST NanoFab</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/labs-major-programs/research-test-beds" class="usa-nav__link">Research Test Beds</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/laboratories/projects-programs" class="usa-nav__link">Research Projects</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/laboratories/tools-instruments" class="usa-nav__link">Tools & Instruments</a>
|
||
</li>
|
||
</ul></div><div class="column-break tablet:grid-col-fill"><ul class="usa-nav__submenu-list">
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/major-programs" class="usa-nav__link">Major Programs</a>
|
||
<ul class="usa-nav__submenu-list">
|
||
<li>
|
||
<a href="/baldrige" class="usa-nav__link">Baldrige Performance Excellence Program</a>
|
||
</li>
|
||
<li>
|
||
<a href="/chips" class="usa-nav__link">CHIPS for America Initiative</a>
|
||
</li>
|
||
<li>
|
||
<a href="/mep" class="usa-nav__link">Manufacturing Extension Partnership (MEP)</a>
|
||
</li>
|
||
<li>
|
||
<a href="/oam" class="usa-nav__link">Office of Advanced Manufacturing</a>
|
||
</li>
|
||
<li>
|
||
<a href="/spo" class="usa-nav__link">Special Programs Office</a>
|
||
</li>
|
||
<li>
|
||
<a href="/tpo" class="usa-nav__link">Technology Partnerships Office</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</li>
|
||
<li class="usa-nav__primary-item">
|
||
<button class="usa-accordion__button usa-nav__link" aria-expanded="false" aria-controls="primary_menu-4"><span>Services & Resources</span></button>
|
||
<div id="primary_menu-4" class="usa-nav__submenu usa-megamenu">
|
||
<div class="grid-row">
|
||
<div class="tablet:grid-col-fill"><ul class="usa-nav__submenu-list">
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/content/standards-measurements" class="usa-nav__link">Standards and Measurements</a>
|
||
<ul class="usa-nav__submenu-list">
|
||
<li>
|
||
<a href="/calibrations" class="usa-nav__link">Calibration Services</a>
|
||
</li>
|
||
<li>
|
||
<a href="/nvlap" class="usa-nav__link">Laboratory Accreditation (NVLAP)</a>
|
||
</li>
|
||
<li>
|
||
<a href="/nist-quality-system" class="usa-nav__link">Quality System</a>
|
||
</li>
|
||
<li>
|
||
<a href="/srm" class="usa-nav__link">Standard Reference Materials (SRMs)</a>
|
||
</li>
|
||
<li>
|
||
<a href="/sri" class="usa-nav__link">Standard Reference Instruments (SRIs)</a>
|
||
</li>
|
||
<li>
|
||
<a href="/standardsgov" class="usa-nav__link">Standards.gov</a>
|
||
</li>
|
||
<li>
|
||
<a href="/pml/time-and-frequency-division/time-services" class="usa-nav__link">Time Services</a>
|
||
</li>
|
||
<li>
|
||
<a href="/pml/owm" class="usa-nav__link">Office of Weights and Measures</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul></div><div class="column-break tablet:grid-col-fill"><ul class="usa-nav__submenu-list">
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/services-resources/software" class="usa-nav__link">Software</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/data" class="usa-nav__link">Data</a>
|
||
<ul class="usa-nav__submenu-list">
|
||
<li>
|
||
<a href="https://webbook.nist.gov/chemistry/" class="usa-nav__link">Chemistry WebBook</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://nvd.nist.gov/" class="usa-nav__link">National Vulnerability Database</a>
|
||
</li>
|
||
<li>
|
||
<a href="/pml/productsservices/physical-reference-data" class="usa-nav__link">Physical Reference Data</a>
|
||
</li>
|
||
<li>
|
||
<a href="/srd" class="usa-nav__link">Standard Reference Data (SRD)</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="https://shop.nist.gov/" class="usa-nav__link">Storefront</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/tpo" class="usa-nav__link">License & Patents</a>
|
||
</li>
|
||
</ul></div><div class="column-break tablet:grid-col-fill"><ul class="usa-nav__submenu-list">
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="https://csrc.nist.gov/" class="usa-nav__link">Computer Security Resource Center (CSRC)</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/nist-research-library" class="usa-nav__link">NIST Research Library</a>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</li>
|
||
<li class="usa-nav__primary-item">
|
||
<button class="usa-accordion__button usa-nav__link" aria-expanded="false" aria-controls="primary_menu-5"><span>News & Events</span></button>
|
||
<div id="primary_menu-5" class="usa-nav__submenu usa-megamenu">
|
||
<div class="grid-row">
|
||
<div class="tablet:grid-col-fill"><ul class="usa-nav__submenu-list">
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/news-events/news" class="usa-nav__link">News</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/news-events/events" class="usa-nav__link">Events</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/blogs" class="usa-nav__link">Blogs</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/feature-stories" class="usa-nav__link">Feature Stories</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/awards" class="usa-nav__link">Awards</a>
|
||
</li>
|
||
</ul></div><div class="column-break tablet:grid-col-fill"><ul class="usa-nav__submenu-list">
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/video-gallery" class="usa-nav__link">Video Gallery</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/image-gallery" class="usa-nav__link">Image Gallery</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/pao/media-contacts" class="usa-nav__link">Media Contacts</a>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</li>
|
||
<li class="usa-nav__primary-item">
|
||
<button class="usa-accordion__button usa-nav__link" aria-expanded="false" aria-controls="primary_menu-6"><span>About NIST</span></button>
|
||
<div id="primary_menu-6" class="usa-nav__submenu usa-megamenu">
|
||
<div class="grid-row">
|
||
<div class="tablet:grid-col-fill"><ul class="usa-nav__submenu-list">
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/about-nist" class="usa-nav__link">About Us</a>
|
||
<ul class="usa-nav__submenu-list">
|
||
<li>
|
||
<a href="/director/leadership" class="usa-nav__link">Leadership</a>
|
||
</li>
|
||
<li>
|
||
<a href="/director/nist-organization-structure" class="usa-nav__link">Organization Structure</a>
|
||
</li>
|
||
<li>
|
||
<a href="/about-nist/budget-planning" class="usa-nav__link">Budget & Planning</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/about-nist/contact-us" class="usa-nav__link">Contact Us</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/about-nist/visit" class="usa-nav__link">Visit</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/careers" class="usa-nav__link">Careers</a>
|
||
<ul class="usa-nav__submenu-list">
|
||
<li>
|
||
<a href="/iaao/academic-affairs-office" class="usa-nav__link">Student programs</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul></div><div class="column-break tablet:grid-col-fill"><ul class="usa-nav__submenu-list">
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/about-nist/work-nist" class="usa-nav__link">Work with NIST</a>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/history" class="usa-nav__link">History</a>
|
||
<ul class="usa-nav__submenu-list">
|
||
<li>
|
||
<a href="http://nistdigitalarchives.contentdm.oclc.org/" class="usa-nav__link">NIST Digital Archives</a>
|
||
</li>
|
||
<li>
|
||
<a href="/nist-museum" class="usa-nav__link">NIST Museum</a>
|
||
</li>
|
||
<li>
|
||
<a href="/nist-and-nobel" class="usa-nav__link">NIST and the Nobel</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="usa-nav__submenu-item">
|
||
<a href="/education" class="usa-nav__link">Educational Resources</a>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</nav>
|
||
|
||
|
||
|
||
|
||
</header>
|
||
<!-- nist-index-ignore-end -->
|
||
</div>
|
||
|
||
<div class="grid-container">
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
<div
|
||
id="block-nist-www-content" class="nist-block"
|
||
>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<section
|
||
class="nist-page__content usa-section clearfix"
|
||
>
|
||
|
||
<a id="main-content" tabindex="-1"></a>
|
||
|
||
|
||
<div class="grid-container margin-top-4">
|
||
<div
|
||
class="nist-page__region nist-page__region--content-top"
|
||
>
|
||
|
||
<div
|
||
class="nist-block"
|
||
>
|
||
|
||
|
||
<a class="usa-button usa-button--accent-cool" href="/publications">PUBLICATIONS</a>
|
||
|
||
</div>
|
||
|
||
<div
|
||
class="nist-block"
|
||
>
|
||
|
||
|
||
<h1 class="nist-page__title">Improving the TENOR of Labeling: Re-evaluating Topic Models for Content Analysis</h1>
|
||
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<div class="grid-container margin-top-4">
|
||
<div class="grid-row grid-gap-6">
|
||
|
||
|
||
|
||
<div
|
||
class="nist-page__region nist-page__region--content tablet-lg:grid-col-12"
|
||
>
|
||
|
||
<div
|
||
class="nist-block"
|
||
>
|
||
|
||
|
||
<div class="nist-field nist-field--label-inline daterange">
|
||
<div class="nist-field__label">Published</div>
|
||
<div class="nist-field__item"><time datetime="2024-03-23T12:00:00Z">March 23, 2024</time>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
<div
|
||
class="nist-block"
|
||
>
|
||
|
||
|
||
<div class="nist-field nist-field--label-above text-long">
|
||
<h3 class="nist-field__label">Author(s)</h3>
|
||
<div class="nist-field__item">Zongxia Li, Andrew Mao, Jordan Boyd-Graber, Daniel Stephens, <span class="nist-author">Emily Walpole</span>, <span class="nist-author">Alden A. Dima</span>, <span class="nist-author">Juan Fung</span></div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
<div
|
||
class="nist-block nist-block--no-margin"
|
||
>
|
||
|
||
|
||
<div class="text-with-summary">
|
||
<h3>Abstract</h3>
|
||
Topic models are a popular tool for understanding text collections, but their evaluation has been a point of contention. Automated evaluation metrics such as coherence are often used, however, their validity has been questioned for neural topic models (NTMs) and can overlook a model's benefits in real-world applications. To this end, we conduct the first evaluation of neural, supervised and classical topic models in an interactive task-based setting. We combine topic models with a classifier and test their ability to help humans conduct content analysis and document annotation. From simulated, real user and expert pilot studies, the Contextual Neural Topic Model does the best on cluster evaluation metrics and human evaluations; however, LDA is competitive with two other NTMs under our simulated experiment and user study results, contrary to what coherence scores suggest. We show that current automated metrics do not provide a complete picture of topic modeling capabilities, but the right choice of NTMs can be better than classical models on practical tasks.
|
||
</div>
|
||
|
||
|
||
</div>
|
||
<div
|
||
class="nist-block nist-block--no-margin"
|
||
>
|
||
|
||
|
||
<div class="nist-field nist-field--label-inline string">
|
||
<div class="nist-field__label">Proceedings Title</div>
|
||
<div class="nist-field__item">Proceedings of the 18th Conference of the European Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
<div
|
||
class="nist-block nist-block--no-margin"
|
||
>
|
||
|
||
|
||
<div class="nist-field nist-field--label-inline string">
|
||
<div class="nist-field__label">Conference Dates</div>
|
||
<div class="nist-field__item">March 17-22, 2024</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
<div
|
||
class="nist-block nist-block--no-margin"
|
||
>
|
||
|
||
|
||
<div class="nist-field nist-field--label-inline string">
|
||
<div class="nist-field__label">Conference Location</div>
|
||
<div class="nist-field__item">St. Julian's, MT</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
<div
|
||
class="nist-block nist-block--no-margin"
|
||
>
|
||
|
||
|
||
<div class="nist-field nist-field--label-inline string">
|
||
<div class="nist-field__label">Conference Title</div>
|
||
<div class="nist-field__item">The 18th Conference of the European Chapter of the Association for Computational Linguistics</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
<div
|
||
class="nist-block nist-block--no-margin"
|
||
>
|
||
|
||
|
||
<div class="nist-field nist-field--label-inline nist-field--link-list entity-reference">
|
||
<div class="nist-field__label">Pub Type</div>
|
||
<div class="nist-field__item">Conferences</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
<div
|
||
class="nist-block block-top-margin"
|
||
>
|
||
|
||
|
||
<div class="nist-field nist-field--label-above nist-field--link-list link">
|
||
<h3 class="nist-field__label">Download Paper</h3>
|
||
<div class="nist-field__items">
|
||
<div class="nist-field__item"><a href="https://tsapps.nist.gov/publication/get_pdf.cfm?pub_id=956761" target="_blank">Local Download</a></div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
<div
|
||
class="nist-block"
|
||
>
|
||
|
||
|
||
<div class="nist-field nist-field--label-above string-long">
|
||
<h3 class="nist-field__label">Keywords</h3>
|
||
<div class="nist-field__item">topic models, active learning, human-in-the-loop, document labeling, annotation, natural language processing, technical language processing</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
<div
|
||
class="nist-block"
|
||
>
|
||
|
||
|
||
<!-- nist-index-ignore-start -->
|
||
<div data-elastic-exclude="" class="nist-tags">
|
||
<a href="/topic-terms/statistical-analysis" hreflang="en">Statistical analysis</a>, <a href="/topic-terms/natural-language-processing" hreflang="en">Natural language processing</a>, <a href="/topic-terms/machine-learning" hreflang="en">Machine learning</a>, <a href="/topic-terms/community-resilience" hreflang="en">Community resilience</a> and <a href="/topic-terms/artificial-intelligence" hreflang="en">Artificial intelligence</a></div>
|
||
<!-- nist-index-ignore-end -->
|
||
|
||
|
||
</div>
|
||
|
||
<div
|
||
class="views-element-container nist-block"
|
||
>
|
||
|
||
<h2
|
||
class="nist-block__title"
|
||
>Citation</h2>
|
||
|
||
<div>
|
||
<div class="js-view-dom-id-15eaae90873586d4f44f7cd70a5d21b493c19bab59cec13455469ae6a84619b3">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<article class="bibcite-reference">
|
||
<div class="bibcite-citation">
|
||
Li, Z.
|
||
, Mao, A.
|
||
, Boyd-Graber, J.
|
||
, Stephens, D.
|
||
, Walpole, E.
|
||
, Dima, A.
|
||
and Fung, J.
|
||
|
||
(2024),
|
||
Improving the TENOR of Labeling: Re-evaluating Topic Models for Content Analysis, Proceedings of the 18th Conference of the European Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), St. Julian's, MT, [online], https://tsapps.nist.gov/publication/get_pdf.cfm?pub_id=956761
|
||
(Accessed March 4, 2025)
|
||
</div>
|
||
|
||
<div class="usa-accordion usa-accordion--bordered bibcite-links margin-top-2" aria-multiselectable="false">
|
||
<h2 class="usa-accordion__heading">
|
||
<button class="usa-accordion__button"
|
||
aria-expanded="false"
|
||
aria-controls="citation_formats">
|
||
Additional citation formats
|
||
</button>
|
||
</h2>
|
||
<div id="citation_formats" class="usa-accordion__content">
|
||
<div class="bibcite-links">
|
||
<ul>
|
||
<li><a href="https://scholar.google.com/scholar?btnG=Search%2BScholar&as_q=%22Improving%2Bthe%2BTENOR%2Bof%2BLabeling%3A%2BRe-evaluating%2BTopic%2BModels%2Bfor%2BContent%2BAnalysis%22&as_occt=any&as_epq=&as_oq=&as_eq=&as_publication=&as_ylo=&as_yhi=&as_sdtAAP=1&as_sdtp=1">Google Scholar</a></li>
|
||
<li><a href="/bibcite/export/bibtex/bibcite_reference/1241171">BibTeX</a></li>
|
||
<li><a href="/bibcite/export/ris/bibcite_reference/1241171">RIS</a></li>
|
||
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
<div
|
||
class="nist-block"
|
||
>
|
||
|
||
|
||
<div class="usa-alert usa-alert--info">
|
||
<div class="usa-alert__body">
|
||
<h4 class="usa-alert__heading">Issues</h4>
|
||
<p class="usa-alert__text">
|
||
If you have any questions about this publication or are having problems accessing it, please contact <a href="mailto:reflib@nist.gov" class="usa-link">reflib@nist.gov</a>.
|
||
</p>
|
||
</div>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<div class="grid-container">
|
||
|
||
<div
|
||
class="nist-page__region nist-page__region--content-bottom"
|
||
>
|
||
|
||
<div
|
||
class="nist-block"
|
||
>
|
||
|
||
|
||
<div class="text-italic font-sans-2xs">
|
||
Created March 23, 2024, Updated May 16, 2024
|
||
</div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
</section>
|
||
|
||
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div data-elastic-exclude>
|
||
<!-- nist-index-ignore-start -->
|
||
|
||
<footer class="nist-footer padding-bottom-4">
|
||
|
||
|
||
<div class="grid-container nist-footer__info">
|
||
<div class="grid-row">
|
||
<div class="tablet:grid-col-6">
|
||
<div class="nist-footer__logo">
|
||
<a href="/" title="National Institute of Standards and Technology" rel="home">
|
||
<img class="nist-footer__logo-img" src="/libraries/nist-component-library/dist/img/logo/NIST-Logo-Brand-White.svg" alt="National Institute of Standards and Technology logo" width="300px" height="42px" />
|
||
</a>
|
||
</div>
|
||
<div class="nist-footer__contact">
|
||
<h3 class="nist-footer__contact-heading">HEADQUARTERS</h3>
|
||
<address>
|
||
100 Bureau Drive<br>
|
||
Gaithersburg, MD 20899<br>
|
||
<a href="tel:301-975-2000">301-975-2000</a>
|
||
</address>
|
||
<p>
|
||
<a href="mailto:do-webmaster@nist.gov">Webmaster</a> | <a href="https://www.nist.gov/about-nist/contact-us">Contact Us</a> | <a href="https://www.nist.gov/visit">Our Other Offices</a>
|
||
</p>
|
||
</div>
|
||
</div>
|
||
<div class="tablet:grid-col-6">
|
||
<div class="nist-footer__social-links">
|
||
|
||
<a class="nist-social nist-social--x-white" href=" https://x.com/NIST">
|
||
<span>X.com</span>
|
||
</a>
|
||
|
||
<a class="nist-social nist-social--facebook-white" href=" https://www.facebook.com/NIST">
|
||
<span>Facebook</span>
|
||
</a>
|
||
|
||
<a class="nist-social nist-social--linkedin-white" href=" https://www.linkedin.com/company/nist">
|
||
<span>LinkedIn</span>
|
||
</a>
|
||
|
||
<a class="nist-social nist-social--instagram-white" href=" https://www.instagram.com/nist/">
|
||
<span>Instagram</span>
|
||
</a>
|
||
|
||
<a class="nist-social nist-social--youtube-white" href=" https://www.youtube.com/NIST">
|
||
<span>YouTube</span>
|
||
</a>
|
||
|
||
<a class="nist-social nist-social--giphy-white" href=" https://giphy.com/nist">
|
||
<span>Giphy</span>
|
||
</a>
|
||
|
||
<a class="nist-social nist-social--rss-white" href=" https://www.nist.gov/news-events/nist-rss-feeds">
|
||
<span>RSS Feed</span>
|
||
</a>
|
||
|
||
<a class="nist-social nist-social--envelope-white" href=" https://public.govdelivery.com/accounts/USNIST/subscriber/new">
|
||
<span>Mailing List</span>
|
||
</a>
|
||
</div>
|
||
<div class="nist-footer__feedback">
|
||
How are we doing? <a class="usa-button" rel="nofollow" href="/form/nist-gov-feedback?destination=/publications/improving-tenor-labeling-re-evaluating-topic-models-content-analysis" title="Provide feedback">Feedback</a>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<div class="grid-container">
|
||
<div class="nist-footer__nav" role="navigation">
|
||
<ul>
|
||
<li class="nist-footer__menu-item">
|
||
<a href="https://www.nist.gov/privacy-policy">Site Privacy</a>
|
||
</li>
|
||
<li class="nist-footer__menu-item">
|
||
<a href="https://www.nist.gov/oism/accessibility">Accessibility</a>
|
||
</li>
|
||
<li class="nist-footer__menu-item">
|
||
<a href="https://www.nist.gov/privacy">Privacy Program</a>
|
||
</li>
|
||
<li class="nist-footer__menu-item">
|
||
<a href="https://www.nist.gov/oism/copyrights">Copyrights</a>
|
||
</li>
|
||
<li class="nist-footer__menu-item">
|
||
<a href="https://www.commerce.gov/vulnerability-disclosure-policy">Vulnerability Disclosure</a>
|
||
</li>
|
||
<li class="nist-footer__menu-item">
|
||
<a href="https://www.nist.gov/no-fear-act-policy">No Fear Act Policy</a>
|
||
</li>
|
||
<li class="nist-footer__menu-item">
|
||
<a href="https://www.nist.gov/office-director/freedom-information-act">FOIA</a>
|
||
</li>
|
||
<li class="nist-footer__menu-item">
|
||
<a href="https://www.nist.gov/environmental-policy-statement">Environmental Policy</a>
|
||
</li>
|
||
<li class="nist-footer__menu-item">
|
||
<a href="https://www.nist.gov/summary-report-scientific-integrity">Scientific Integrity</a>
|
||
</li>
|
||
<li class="nist-footer__menu-item">
|
||
<a href="https://www.nist.gov/nist-information-quality-standards">Information Quality Standards</a>
|
||
</li>
|
||
<li class="nist-footer__menu-item">
|
||
<a href="https://www.commerce.gov/">Commerce.gov</a>
|
||
</li>
|
||
<li class="nist-footer__menu-item">
|
||
<a href="http://www.science.gov/">Science.gov</a>
|
||
</li>
|
||
<li class="nist-footer__menu-item">
|
||
<a href="http://www.usa.gov/">USA.gov</a>
|
||
</li>
|
||
<li class="nist-footer__menu-item">
|
||
<a href="https://vote.gov/">Vote.gov</a>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</footer>
|
||
<!-- nist-index-ignore-end -->
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<script type="application/json" data-drupal-selector="drupal-settings-json">{"path":{"baseUrl":"\/","pathPrefix":"","currentPath":"node\/1848706","currentPathIsAdmin":false,"isFront":false,"currentLanguage":"en"},"pluralDelimiter":"\u0003","suppressDeprecationErrors":true,"back_to_top":{"back_to_top_button_trigger":100,"back_to_top_speed":1200,"back_to_top_prevent_on_mobile":false,"back_to_top_prevent_in_admin":true,"back_to_top_button_type":"image","back_to_top_button_text":"Back to top"},"google_analytics":{"account":"G-HEQ0YF2VYL","trackOutbound":true,"trackMailto":true,"trackTel":true,"trackDownload":true,"trackDownloadExtensions":"7z|aac|arc|arj|asf|asx|avi|bin|bsh|c|csv|doc(x|m)?|dot(x|m)?|dw(fx|g|gd)|dxf|eps|epub|exe|f(90)|flv|gif|gz|gzip|hqx|jar|jpe?g|js|m1v|mp(2|3|4|e?g)|mobi|mov(ie)?|msi|msp|pdf|phps|pl|png|ppt(x|m)?|pot(x|m)?|pps(x|m)?|ppam|sld(x|m)?|thmx|qtm?|ra(m|r)?|rfa|rtf|rvt|sch|sea|sit|swf|tar|tgz|tif|txt|txz|wav|wma|wmv|wpd|wrl|xls(x|m|b)?|xlt(x|m)|xlam|xml|xsd|z|zip"},"nist_search":{"clickTracking":"search-report-click","isDebug":false,"clickTrackEnabled":true,"message":"NIST Search in debug mode. Check the browsers network inspector for Click Track reporting results..."},"data":{"extlink":{"extTarget":false,"extTargetNoOverride":false,"extNofollow":false,"extNoreferrer":false,"extFollowNoOverride":false,"extClass":"ext","extLabel":"(link is external)","extImgClass":false,"extSubdomains":true,"extExclude":"\\.gov\\\/|\\.mil\\\/|\\manufacturingusa\\.com\\\/","extInclude":"","extCssExclude":".ck-editor, .nist-video-thumbnail__lightbox-trigger, a.nist-icon--mail-blue","extCssExplicit":"","extAlert":true,"extAlertText":"Thank you for visiting NIST. We hope your visit was informative. We have provided a link to this site because it has information that may be of interest to our users. NIST does not necessarily endorse the views expressed or the facts presented on this site. Further, NIST does not endorse any commercial products that may be advertised or available on this site. Click OK to be directed to your link.","mailtoClass":"0","mailtoLabel":"(link sends email)","extUseFontAwesome":false,"extIconPlacement":"after","extFaLinkClasses":"fa fa-external-link","extFaMailtoClasses":"fa fa-envelope-o","whitelistedDomains":[]}},"user":{"uid":0,"permissionsHash":"bd6a443844dbe99b4e6942f3b1397526e760137efd2b1ee6b2d588fc414dff20"}}</script>
|
||
<script src="/sites/default/files/js/js_jDaFaXhHm6gPUKstNYX9eIyoQXghVs3g7rbR0QoqpYY.js?scope=footer&delta=0&language=en&theme=nist_www&include=eJxdjW0OwjAIhi805UiErtjhWFlatHp7O53RLOEHPLwfgcYZ3fqsEH47XuuQzJIyUiZ9uowVjmDIUh0v0lm0ltUoopceIjlBos-7irMsa7E7wxFsLW_mEy_ce00DFVwsQpRKoQer5PmtgaQWSE-7BSamyOV7YmvtT8EP34wQy20lPe_nC6DrXYU"></script>
|
||
<script src="https://siteimproveanalytics.com/js/siteanalyze_6017546.js" async></script>
|
||
<script src="/sites/default/files/js/js_709kuJyJYTt2lWrlwglvgNyqkgqlZSSNCLusQNdUubs.js?scope=footer&delta=2&language=en&theme=nist_www&include=eJxdjW0OwjAIhi805UiErtjhWFlatHp7O53RLOEHPLwfgcYZ3fqsEH47XuuQzJIyUiZ9uowVjmDIUh0v0lm0ltUoopceIjlBos-7irMsa7E7wxFsLW_mEy_ce00DFVwsQpRKoQer5PmtgaQWSE-7BSamyOV7YmvtT8EP34wQy20lPe_nC6DrXYU"></script>
|
||
<script type="text/javascript">window.NREUM||(NREUM={});NREUM.info={"beacon":"bam.nr-data.net","licenseKey":"37b7ccb661","applicationID":"1089704227","transactionName":"YFxUN0sADEdYVkBaClkWdwBNCA1aFnFGRhVWVWoNVgUHaHpaWkcXWFVaBks9LFtdUGJaAEB6WQ1NEw1YVVBGHltBUFMU","queueTime":4,"applicationTime":540,"atts":"TBtXQQMaH0k=","errorBeacon":"bam.nr-data.net","agent":""}</script></body>
|
||
</html>
|