diff --git a/www.ssa.gov/robots.txt b/www.ssa.gov/robots.txt new file mode 100644 index 000000000..c8633198f --- /dev/null +++ b/www.ssa.gov/robots.txt @@ -0,0 +1,31 @@ +# www.ssa.gov robots.txt +# 08/07/18 +# 08/06/19 added second sitemap +# 09/25/20 added 2019 contingency plan PDF + html +# Wayne Whitten +# 11/08/22 added search-gov, and /menu/ for Beta launch +# 5/5/23 added 2023-psrw-letter.pdf per Jira ticket + +User-Agent: * + +Disallow: /admin/ +# Disallow: /agency/shutdown/ +Disallow: /cgi-bin/ +Disallow: /dev/ +Disallow: /hlp/ +Disallow: /info/ +Disallow: /javascript/ +Disallow: /myaccount/help/ +Disallow: /myaccount/materials/pdfs/SSA-7004.pdf +Disallow: /onlineservices/eservices/ +Disallow: /ssaexpress/ +Disallow: /w3c/ +Disallow: /agency/shutdown/materials/contingency-plan-08-05-19.pdf +Disallow: /agency/shutdown/materials/contingency-plan-08-05-19.html +Disallow: /news/mip/ +Disallow: /menu/ +Disallow: /news/materials/pdfs/2023-psrw-letter.pdf + +Sitemap: https://www.ssa.gov/sitemap.xml +Sitemap: https://www.ssa.gov/sitemap1.xml +Sitemap: https://www.ssa.gov/search-gov/sitemap.xml \ No newline at end of file