# robots.txt for webapps at www.there.com
#

# disallow all robots not listed in this file

User-agent: *
Disallow: /

# White list of who may crawl
User-agent: Scooter			# alta vista
User-agent: FAST-WebCrawler		# alltheweb
User-agent: askjeeves 			# Self explanitory
User-agent: ArchitextSpider		# Excite
User-agent: Googlebot 			# Google
User-agent: Ultraseek 			# Infoseek free and commercial
User-agent: marvin/infoseek	 	# webseek 
User-agent: Slurp			# Inktomi Bot (hotbot)
User-agent: Lycos_Spider_(T-Rex)	# Lycos
User-agent: LNSpiderguy			# Lexis Nexis
User-agent: MSNBOT			# MSN Search
User-agent: YahooBot			# Yahoo
User-agent: YahooSeeker			# Yahoo
User-agent: Mediapartners-Google	# Google AdSense
# Disallow all folders except approved. Remove others as appropriate.
Disallow: /about/
Disallow: /activities/
Disallow: /activityflag/
Disallow: /administeravatar/
Disallow: /administerbilling/
Disallow: /administeruser/
Disallow: /astro/
Disallow: /avannot/
Disallow: /banker/
Disallow: /billing_common/
Disallow: /captcha/
# allow: /catalog/
Disallow: /cgi-bin/			# should always be disallowed
Disallow: /changepassword/
Disallow: /checkmail/
Disallow: /clubs/
Disallow: /commercehistory/
Disallow: /common/
Disallow: /conversations/
Disallow: /developers/
Disallow: /download/ 			# this should probably always be disallowed due to dynamic content
Disallow: /dynamic/			# always should be disallowed due to dynamic content
Disallow: /email/
Disallow: /eventmgmt/
Disallow: /farp/
Disallow: /frequent_fun/
Disallow: /funzone/
Disallow: /gamekit/
Disallow: /gear/
Disallow: /groups/
Disallow: /help/
Disallow: /helpers/
# allow items
Disallow: /jobs/
Disallow: /listings/
Disallow: /login/
Disallow: /map/
Disallow: /memprog/
Disallow: /mentor/
Disallow: /moderation/
Disallow: /newcommon/
Disallow: /newyou/
Disallow: /onlinebuddies/
Disallow: /partners/
Disallow: /patcher/
Disallow: /personals/
Disallow: /ph/
Disallow: /places/
Disallow: /popular/
Disallow: /problemtext/
Disallow: /profiles/
Disallow: /promo/
Disallow: /qualsurvey/
Disallow: /raf/
Disallow: /register/
Disallow: /registration/
Disallow: /returnit/
Disallow: /scripts/	# python dirs are dynamic.
Disallow: /sg/
Disallow: /sign/
Disallow: /signup/
Disallow: /skillsredirect/
Disallow: /storykit/
Disallow: /sysstat/
# allow: /therecentral/
Disallow: /theredoc/
Disallow: /toprankings/
Disallow: /transfer/
Disallow: /vendmac/
Disallow: /viewlisting/
Disallow: /vignette/
Disallow: /visitors_guide/
Disallow: /VoiceTrainer/
Disallow: /whatsthis/
Disallow: /yahoo/
Disallow: /fgftvx_wa_clientloginpage.gif
Disallow: /fgftvx_wa_enteravinfo.gif
Disallow: /fgftvx_wa_loginpage.gif
Disallow: /fgftvx_wa_regacctcreated.gif
Disallow: /fgftvx_wa_regforward.gif
Disallow: /fgftvx_wa_regsubmit.gif
Disallow: /fgftvx_www_loginpage.gif
Disallow: /global_nav.swf
Disallow: /index.html
Disallow: /links.xml
Disallow: /metarefresh.xsl
Disallow: /status.xml
Disallow: /weblogfact.gif