# COGENT 1.1

# robot/spider exclusion 

# http://info.webcrawler.com/mak/projects/robots/robots.html
#############################################################
User-agent: AdultsVisit.us/Nutch-1.0
Disallow: /

User-agent: WebCrawler_1.1
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Baiduspider+
Disallow: /

User-agent: Gaisbot/3.0
Disallow: /

User-agent: Pingdom.com/tools
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: Kilroy             # OCLC
Disallow: /

User-agent: Whacker            # WebWhacker
Disallow: /

User-agent: FlashSite
Disallow: /

User-agent: NetJet
Disallow: /

User-agent: Snarf
Disallow: /

User-Agent: TestApp/1.0 libwww/4.1b5
Disallow: /

User-Agent: GETWWW-ROBOT/2.0
Disallow: /

User-Agent: MemoWeb
Disallow: /

User-Agent: net.Sweep/v1.2e-T1-CHI
Disallow: /

User-Agent: FFiNet32.DLL/3.1 libwww/3.1
Disallow: /

User-Agent: Javachapman:05/19/19-15:35
Disallow: /

User-Agent: LaSCoW-trace/1.0
Disallow: /

User-Agent: RufusBot
Disallow: / 

User-Agent: Jyxobot
Disallow: /

User-Agent: NaverBot/1.0
Disallow: /

User-Agent: Steeler/3.3
Disallow: /

User-Agent: QihooBot
Disallow: /

User-Agent: ichiro/2.0
Disallow: /

User-agent: *
Disallow: /home/thunderr/public_html/cgi-bin/
Disallow: /home/thunderr/public_html/temp/
Disallow: /home/thunderr/public_html/scgi-bin/
Disallow: /home/thunderr/public_html/mickalo/
Disallow: /home/thunderr/public_html/mickalo/cgi-bin/
Disallow: /home/thunderr/public_html/projects/
Disallow: /home/thunderr/public_html/projects/cgi-bin/