User-agent: *             # match all bots
# Disallow: /               # keep them out of everything
Crawl-delay: 10           # seconds to wait between successive requests to the same server
Request-rate: 1/2         # maximum rate is one page every 5 seconds
Visit-time: 0200-0845     # only visit between 06:00 and 08:45 UTC (GMT)