# Spiders, crawlers, robots - you're welcome!
User-agent: *
Disallow: /compare/*;*
Disallow: *.js
Crawl-delay: 10
Request-rate: 1/10         # maximum rate is one page every 10 seconds