# robots.txt 
User-agent: *
Disallow: 
# Tell the Internet Archive not to keep a copy
User-agent: ia_archiver
Disallow: /
User-agent: Slurp
Crawl-delay: 20