# /robots.txt for assoziation-a.de
# spammer
User-Agent: sitecheck.internetseek.com
Disallow: /
User-Agent: LinkWalker
Disallow: /
#ConveraCrawler
User-Agent: ConveraCrawler
Disallow: /
# Java - who ever it is ??
User-Agent: Java
Disallow: /
# IDBot => http://www.id-search.org/bot.html
User-Agent: IDBot
# crawl und robot stehen in log-files => test, ob sie so 'draussen bleiben'
Disallow: /
User-Agent: crawl
Disallow: /
User-Agent: robot
Disallow: /
# alle anderen sollen duerfen
User-agent: *
Disallow: /cgi-bin/