# Robots.txt spider rights


User-agent: ShopWiki
Disallow: /

User-agent: twiceler
Disallow: /

#User-Agent: MJ12bot
#Disallow: /

#User-agent: Mediapartners-Google
#Disallow: /

#
# Go away, Microsoft URL Control and other nameless MSIE stealth bots
#
User-agent: Microsoft
Disallow: /
User-agent: Control
Disallow: /
User-agent: MSIECrawler
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 98)
Disallow: /
User-agent: MFC Foundation Class Library
Disallow: /

#
# InternetSeer.com: you were not invited, go away
#
User-agent: InternetSeer.com
Disallow: /


#
# Other pests follow
#

User-agent: BaiDuSpider
Disallow: /
User-agent: Cherry Picker
Disallow: /
User-agent: Crescent Internet ToolPak
Disallow: /
User-agent: DISCo Pump
Disallow: /
User-agent: EmailSiphon
Disallow: /
User-agent: grub
Disallow: /
User-agent: htdig
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: LNSpiderGuy
Disallow: /
User-agent: Netprospector
Disallow: /
User-agent: PerMan Surfer
Disallow: /
User-agent: psbot
Disallow: /
# Not sure about these guys -- looks like some kinda polish bot
User-agent: szukacz
Disallow: /
User-agent: RPT-HTTPClient
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: Telesoft
Disallow: /
User-agent: Ultraseek
Disallow: /
User-agent: vspider
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: WebEMailExtractor
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: Webster Pro
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: BecomeBot
Disallow: /
#User-agent: ShopWiki
#Disallow: /

User-agent: *
Disallow: /viewoffer.asp
Disallow: /charts/
Disallow: /postusercomment.asp
Disallow: /reviewretailer.asp
Disallow: /cat/
Disallow: /list.asp?*
Disallow: /*/f-*
Disallow: /*/s-*
Disallow: /s/*
Disallow: /*/*/wpr/*
Disallow: /*/*/spc/*