User-agent: * # Crawl-delay parameter, set to the number of seconds to wait between successive requests to the same server #Crawl-delay: 10 # disallow all files in these directories Disallow: /~/ Disallow: /sitecore/ Disallow: /img/ User-agent: Googlebot # disallow all files ending with these extensions Disallow: /*.aspx$ Disallow: /*.ashx$ # allow google image bot to search all images User-agent: Googlebot-Image Allow: /* # allow adsense bot on entire site User-agent: Mediapartners-Google* Disallow: /*.aspx$ Disallow: /*.ashx$ # disallow archiving site # disable duggmirror