# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism
#
# Examples taken from http://info.webcrawler.com/mak/projects/robots/faq.html#robotstxt
#
# To disallow nothing for webcrawler:
# User-agent: webcrawler
# Disallow:
# 
# To disallow everything for lycra:
# User-agent: lycra
# Disallow: /
# 
# For all other spiders disallow only URLs starting with /tmp or /logs:
# User-agent: *
# Disallow: /tmp
# Disallow: /logs