# don't be a guinea pig User-agent: test Disallow: / User-agent: beta Disallow: / # I don't know who these folks are and they don't obey rules? User-agent: R2 Disallow: / # everybody else is ok, but restrict directories User-agent: * Disallow: /cgi-bin/ Disallow: /Architext/ Disallow: /tmp/ Disallow: /dont.index/ Disallow: /temp/ Disallow: /data/ Disallow: /scripts/ Disallow: /bin/ User-agent: BecomeBot Crawl-Delay: 30 Disallow: /cgi-bin