# robots.txt for http://www.fob.uni-tuebingen.de/
# s. map, search

# all bots:
User-agent: *
Disallow: /cgi-bin
Disallow: /css 
Disallow: /intern 
Disallow: /lib 
Disallow: /links/wiki
Disallow: /media 
Disallow: /news 
Disallow: /scripts
Disallow: /teilnehmer 
Disallow: /tmp
Disallow: /index.tcp.html
Disallow: /index.bau.html

User-agent: googlebot
# only googlebot supports wildcards!
Disallow: *.cgi
Disallow: *.inc

User-Agent: W3C-checklink
Allow: /

# This hits us too fast, and doesn't get the 403 hint!
User-agent: grub-client
Disallow: /

User-agent: grub
Disallow: /

# Misbehaving: requests much too fast:
User-agent: fast
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: Siphon
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: Firefly/1.0
Disallow: /

User-agent: Harvest/1.5
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: HTTrack 3.0
Disallow: /

User-agent: HTTrack 3.32
Disallow: /

User-agent: LinkextractorPro
Disallow: /

User-agent: LinkWalker
Disallow: /

User-agent: lwp-trivial
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Offline Navigator
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: VoidEYE
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: WebBandit/3.50
Disallow: /

User-agent: WebCopy/
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebCopier v.2.2
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebSauger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebWhacker
Disallow: /

User-agent: WebZip
Disallow: /

User-agent: WebZip/4.0
Disallow: /

User-agent: WebZIP/4.21
Disallow: /

User-agent: WebZIP/5.0
Disallow: /

User-agent: WebZIP/6.0
Disallow: /

User-agent: Wget
Disallow: /

User-agent: Wget/1.4.0
Disallow: /

User-agent: Wget/1.5.3
Disallow: /

User-agent: Wget/1.6
Disallow: /

User-agent: Wget/1.7.1
Disallow: /

User-agent: Wget/1.8
Disallow: /

#User-agent: BlackWidow
#User-agent: Bot mailto:craftbot@yahoo.com
#User-agent: ChinaClaw
#User-agent: DISCo
#User-agent: Download Demon
#User-agent: eCatch
#User-agent: EirGrabber
#User-agent: Express WebPictures
#User-agent: EyeNetIE
#User-agent: FlashGet
#User-agent: GetRight
#User-agent: Go!Zilla
#User-agent: Go-Ahead-Got-It
#User-agent: GrabNet
#User-agent: Grafula
#User-agent: HMView
#User-agent: Image Stripper
#User-agent: Image Sucker
#User-agent: InterGET
#User-agent: Internet Ninja
#User-agent: JetCar
#User-agent: JOC Web Spider
#User-agent: larbin
#User-agent: LeechFTP
#User-agent: LWP::Simple/5.63
#User-agent: Mass Downloader
#User-agent: MIDown tool
#User-agent: Mister PiX
#User-agent: Navroad
#User-agent: NearSite
#User-agent: NetAnts
#User-agent: NetSpider
#User-agent: Net Vampire
#User-agent: NetZIP
#User-agent: Octopus
#User-agent: PageGrabber
#User-agent: Papa Foto
#User-agent: pcBrowser
#User-agent: RealDownload
#User-agent: ReGet
#User-agent: SmartDownload
#User-agent: SuperBot
#User-agent: SuperHTTP
#User-agent: Surfbot
#User-agent: tAkeOut
#User-agent: Web Image Collector
#User-agent: Web Sucker
#User-agent: WebAuto
#User-agent: WebCopier
#User-agent: WebFetch
#User-agent: WebReaper
#User-agent: Website eXtractor
#User-agent: Widow
#User-agent: Xaldon WebSpider
#User-agent: Zeus