# robots.txt file for http://www.clu-in.org

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf
Disallow : /

User-agent: Teleport+Pro/1.29
Disallow: /

User-Agent: Charlotte
Disallow: /

User-agent: Googlebot
Disallow: /products/chemox/search/
Disallow: /products/altcovers/
Disallow: /products/siteprof/sitedtl.cfm
Disallow: /products/siteprof/remdlist.cfm
Disallow: /products/siteprof/remdsdtl.cfm
Disallow: /products/mtbe/usersearch/
Disallow: /products/thermal/usersearch/
Disallow: /products/nairt/sresults.cfm
Disallow: /products/chemox/submit/
Disallow: /products/altcovers/usersubmit
Disallow: /products/siteprof/frform.cfm
Disallow: /products/mtbe/usersubmit/
Disallow: /products/thermal/usersubmit
# Disallow: /search/default.cfm
# Disallow: /search/
Crawl-delay: 20

User-agent: Xenu Link Sleuth
Disallow: /techdrct/td

User-agent: Slurp
Crawl-delay: 60

User-agent: *
Disallow: /products/tins/display.cfm
Disallow: /plweb-cgi/fastweb.exe
Disallow: /programs/21m2/lit_show.cfm

User-agent: msnbot
Crawl-delay: 60

User-agent: *
Crawl-delay: 20

User-agent: *
Disallow: /search

User-agent: Baiduspider
Disallow: /

Sitemap: https://clu-in.org/sitemap.xml

Sitemap: https://clu-in.org/videositemap.xml