# # # Robots.txt file from http://www.bitsforpets.com/robots.txt # # All robots will spider the domain User-agent: * Disallow: http://www.bitsforpets.com/product_reviews_write.php Disallow: http://www.bitsforpets.com/product_reviews.php Disallow: /tmp/ Disallow: http://www.bitsforpets.com/blog/2008/03/a Disallow: http://www.bitsforpets.com/blog/category/nature-news/page/2/a Disallow: http://www.bitsforpets.com/blog/page/5/a Disallow: http://www.bitsforpets.com/images/all_mouse.gif%22%3E%3C/p%3E%3Cp Crawl-delay: 60 User-agent: Slurp Crawl-delay: 10 Disallow: cookie_usage.php Disallow: http://www.bitsforpets.com/cookie_usage.php Disallow: /tmp/ User-agent: Yahoo! Disallow: http://www.bitsforpets.com/cookie_usage.php Disallow: cookie_usage.php Crawl-delay: 10 Disallow: /tmp/