# robots.txt for http://www.collectivenet.org User-agent: * Disallow: /stats/ Disallow: /cgi-bin/domains/ User-agent: Slurp Crawl-delay: 5