# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:

# User-Agent: *
# Disallow: /

# Go with more specific approach
User-Agent: *
Crawl-Delay: 17
Disallow: /404.html
Disallow: /422.html
Disallow: /500.html
Disallow: /dispatch.cgi
Disallow: /dispatch.fcgi
Disallow: /dispatch.rb
Disallow: /favicon.ico
Disallow: /oldindexpage.html