# This file helps keeps the server fast. DO NOT DELETE!
# If this file is found deleted and a crawler takes down
# the server or overloads it because of your site you
# will be warned only once! Feel free to modify.
User-agent: *
Disallow: /cgi-bin/
Disallow: /images/
Disallow: /webalizer/

User-agent: aipbot
User-agent: becomebot 
User-agent: ia_archiver
User-agent: psbot
User-agent: SurveyBot
Disallow: /

User-agent: gigabot
User-agent: googlebot 
User-agent: msnbot 
User-agent: teoma
User-agent: slurp
Crawl-delay: 20