# HTTPD site access
# 040310.clc. let em in, see what happens
# 040805.clc. wildcard many subdirs
# 050620.clc. adjust las entries; inktomisearch was still connecting to las-test
# 050705.clc. Disallow /jg/*  /*?  (dynamic pages)
 
# Everyone go away
# User-agent: *
# Disallow: /
 
User-agent: *
 
Disallow: /cgi-bin
Disallow: /dods-bin
Disallow: /jg/*
Disallow: /Css
Disallow: /data/*
Disallow: /dods/*
Disallow: */ftp-pub/*
Disallow: */images/*
Disallow: */JS_scripts/
Disallow: /las*/*
Disallow: /LAS/*
Disallow: */Oldfiles/*
# Disallow: /PI-NOTES/
Disallow: /smallgifs
Disallow: /statistics
Disallow: /Templates
Disallow: /tmp/*
Disallow: /*?
 
# list of common search engines
# User-agent: Googlebot
# User-agent: InfoNaviRobot
# User-agent: TV33_Mercator
# User-agent: AVSearch
# User-agent: Mercator
# User-agent: Scooter
# User-agent: Slurp
# User-agent: shadow
# User-agent: MultiText
# User-agent: FAST-WebCrawler
# User-agent: Lycos_Spider
# User-agent: Atomz
# User-agent: htdig
# User-agent: spider00.logika.net
# User-agent: NetMechanic
# User-agent: libwww-perl
# User-agent: Teleport Pro
# User-agent: Teoma             # Ask Jeeves