#
# robots.txt controls Web spiders (i.e.- search engines)
#
User-agent: *
# private stuff
Disallow: /cgi-bin
Disallow: /cgi-bin/
Disallow: /images/
Disallow: /css/
Disallow: /js/

User-Agent: MJ12bot
Crawl-Delay:   60

User-agent: BecomeBot
Disallow: /