# http://www.robotstxt.org/norobots-rfc.txt

# Deny all to start, then allow a few popular ones in
# otherwise it is a perpetual maint headache.
User-agent: *
Disallow: /cgi-bin/
Disallow: /vbb/
Disallow: *.php

# Exclude bots from general search of VBB, allow showthread.php only
# as the bots don't need to index anything else.
# Minimize load on webserver and DB.
User-agent: Slurp
User-agent: Googlebot
User-agent: msnbot/1.1
Crawl-delay: 10
Disallow: /cgi-bin/
Disallow: *.php
Disallow: /vbb/
Allow: /vbb/showthread.php