User-agent: *
Disallow: /Demos/
Disallow: /images/
Disallow: 404b.htm
# This bans all robots from our Demos and images folder


# The below code restricts access to only known, registered, and good robots.
User-agent: Mozilla/3.0 (compatible;miner;mailto:miner@miner.com.br)
Disallow:

User-agent: WebFerretDisallow:
User-agent: noDisallow:
User-agent: 'Ahoy! The Homepage Finder'Disallow:
User-agent: ArachnophiliaDisallow:
User-agent: ArchitextSpiderDisallow:
User-agent: ASpider/0.09Disallow:
User-agent: AURESYS/1.0Disallow:
User-agent: BackRub/*.*Disallow:
User-agent: Big BrotherDisallow:
User-agent: BlackWidowDisallow:
User-agent: BSpider/1.0 libwww-perl/0.40Disallow:
User-agent: CACTVS Chemistry SpiderDisallow:
User-agent: Digimarc CGIReader/1.0Disallow:
User-agent: Checkbot/x.xx LWP/5.xDisallow:
User-agent: root/0.1Disallow:
User-agent: CS-HKUST-IndexServer/1.0Disallow:
User-agent: CyberSpyder/2.1Disallow:
User-agent: Deweb/1.01Disallow:
User-agent: DragonBot/1.0 libwww/5.0Disallow:
User-agent: EIT-Link-Verifier-Robot/0.2Disallow:
User-agent: Emacs-w3/v[0-9\.]+Disallow:
User-agent: EmailSiphon
Disallow:
User-agent: EMC SpiderDisallow:
User-agent: explorersearchDisallow:
User-agent: Explorer
Disallow:
User-agent: ExtractorPro
Disallow:
User-agent: FelixIDE/1.0Disallow:
User-agent: KIT-Fireball/2.0 libwww/5.0aDisallow:
User-agent: Fish-Search-RobotDisallow:
User-agent: Mozilla/2.0 (compatible fouineur v2.0; fouineur.9bit.qc.ca)Disallow:
User-agent: FreecrawlDisallow:
User-agent: FunnelWeb-1.0Disallow:
User-agent: GetURL.rexx v1.05Disallow:
User-agent: Golem/1.1Disallow:
User-agent: Gromit/1.0Disallow:
User-agent: Gulliver/1.1Disallow:
User-agent: AITCSRobot/1.1Disallow:
User-agent: wired-digital-newsbot/1.5Disallow:
User-agent: htdig/3.0b3Disallow:
User-agent: IBM_Planetwide,Disallow:
User-agent: InformantDisallow:
User-agent: InfoSeek Robot 1.0Disallow:
User-agent: Infoseek SidewinderDisallow:
User-agent: InfoSpiders/0.1Disallow:
User-agent: JCrawler/0.2Disallow:
User-agent: Jeeves v0.05alpha (PERL, LWP, lglb@doc.ic.ac.uk)Disallow:
User-agent: Jobot/0.1alpha libwww-perl/4.0Disallow:
User-agent: JoeBot,Disallow:
User-agent: JubiiRobot
Disallow:
User-agent: jumpstationDisallow:
User-agent: LinkWalkerDisallow:
User-agent: logo.gif crawlerDisallow:
User-agent: Lycos/x.xDisallow:
User-agent: Lycos_Spider_(T-Rex)
Disallow:
User-agent: Monster/vX.X.X -$TYPE ($OSTYPE)Disallow:
User-agent: Motor/0.2Disallow:
User-agent: MuscatFerretDisallow:
User-agent: MwdSearch/0.1Disallow:
User-agent: NetCarta CyberPilot ProDisallow:
User-agent: NetMechanicDisallow:
User-agent: NetScoop/1.0 libwww/5.0aDisallow:
User-agent: NHSEWalker/3.0Disallow:
User-agent: Nomad-V2.xDisallow:
User-agent: NorthStarDisallow:
User-agent: Occam/1.0Disallow:
User-agent: HKU WWW Robot,Disallow:
User-agent: Orbsearch/1.0Disallow:
User-agent: PackRat/1.0Disallow:
User-agent: Patric/0.01aDisallow:
User-agent: Peregrinator-Mathematics/0.7Disallow:
User-agent: DuppiesDisallow:
User-agent: PioneerDisallow:
User-agent: PGP-KA/1.2Disallow:
User-agent: Resume RobotDisallow:
User-agent: Road Runner: ImageScape Robot (lim@cs.leidenuniv.nl)Disallow:
User-agent: Robbie/0.1Disallow:
User-agent: ComputingSite Robi/1.0 (robi@computingsite.com)Disallow:
User-agent: RoverbotDisallow:
User-agent: SafetyNet Robot 0.1,Disallow:
User-agent: Scooter/1.0Disallow:
User-agent: SG-ScoutDisallow:
User-agent: SimBot/1.0Disallow:
User-agent: Open Text Site Crawler V1.0Disallow:
User-agent: SiteTech-RoverDisallow:
User-agent: Slurp/2.0Disallow:
User-agent: Snooper/b97_01Disallow:
User-agent: Solbot/1.0 LWP/5.07Disallow:
User-agent: Mozilla/3.0 (Black Widow v1.1.0; Linux 2.0.27; Dec 31 1997 12:25:00Disallow:
User-agent: Tarantula/1.0Disallow:
User-agent: TITAN/0.1Disallow:
User-agent: UCSD-CrawlerDisallow:
User-agent: Valkyrie/1.0 libwww-perl/0.40Disallow:
User-agent: Victoria/1.0Disallow:
User-agent: vision-search/3.0'Disallow:
User-agent: VWbot_K/4.2Disallow:
User-agent: w3indexDisallow:
User-agent: W3M2/x.xxxDisallow:
User-agent: WWWWanderer v3.0Disallow:
User-agent: WebCopy/Disallow:
User-agent: WebCrawler/3.0 Robot libwww/5.0aDisallow:
User-agent: WebFetcher/0.8,Disallow:
User-agent: weblayers/0.0Disallow:
User-agent: WebMoose/0.0.0000Disallow:
User-agent: Digimarc WebReader/1.2Disallow:
User-agent: webs@recruit.co.jpDisallow:
User-agent: webvac/1.0Disallow:
User-agent: webwalkDisallow:
User-agent: WebWalker/1.10Disallow:
User-agent: WebWatchDisallow:
User-agent: Wget/1.4.0Disallow:
User-agent: WWWC/0.25 (Win95)Disallow:
User-agent: XGET/0.7Disallow:
User-agent: Nederland.zoekDisallow:
User-agent: CaliforniaBrownSpiderDisallow:
User-agent: EI*Net/0.1 libwww/0.1Disallow:
User-agent: Ibot/1.0 libwww-perl/0.40Disallow:
User-agent: Merritt/1.0Disallow:
User-agent: StatFetcher/1.0Disallow:
User-agent: TeacherSoft/1.0 libwww/2.17Disallow:
User-agent: wobot/1.0 from 206.214.202.45Disallow:
User-agent: Libertech-Rover www.libertech.com?Disallow:
User-agent: WhoWhere RobotDisallow:
User-agent: spyder3.microsys.comDisallow:
User-agent: www.freeloader.com.Disallow:
User-agent: GooglebotDisallow: