User-Agent: *
Allow: /
Disallow: /cgi-bin/

User-Agent: Mediapartners-Google
Allow: /
Disallow: /cgi-bin/

User-Agent: Googlebot
Allow: /
Disallow: /cgi-bin/

User-agent: Slurp
Disallow: /cgi-bin/
Disallow: /*agora*
Crawl-delay: 5

User-Agent: Yahoo! Slurp
Disallow: /*cgi-bin
Disallow: /*agora*
Disallow: /*.jpg$
Crawl-delay: 5


Disallow: /webalizer/ 
Disallow: /User-agent: Gigabot
Disallow: /User-agent: Twiceler
Disallow: /User-agent: asterias
Disallow: /User-agent: BackDoorBot/1.0
Disallow: /User-agent: Black Hole
Disallow: /User-agent: BotALot
Disallow: /User-agent: BuiltBotTough
Disallow: /User-agent: WebBandit
Disallow: /User-agent: WebBandit/3.50
Disallow: /User-agent: WebCopier