# small robots.txt
# more information about this file can be found at
# http://www.robotstxt.org/wc/robots.html
# lines beginning with the pund ("#") sign are comments and can be deleted.
# if case your drupal site is in a directory
# lower than your docroot (e.g. /drupal)
# please add this before the /-es below
# to stop a polite robot indexing an exampledir
# add a line like (delte the #'s)
# user-agent: polite-bot
# Disallow: /exampledir/
# a list of know bots can be found at
# http://www.robotstxt.org/wc/active/html/index.html
# see http://www.sxw.org.uk/computing/robots/check.html
# for syntax checking

User-agent: *
Crawl-Delay: 10
Allow: /*.js*
Allow: /*.css*
Disallow: /cgi-bin
Disallow: /aggregator
Disallow: /tracker
#Disallow: /newsletter_confirm
Disallow: /wesell_confirm
Disallow: /adprimes_confirm
Disallow: /sekindo_confirm
#Disallow: /get_ebook
#Disallow: /comment/reply
Disallow: /node/add
Disallow: /user
#Disallow: /files
Disallow: /search
Disallow: /book/print
Disallow: /?q=admin
Disallow: /admin/
Disallow: /cron.php
Disallow: /xmlrpc.php
Disallow: /database/
Disallow: /vtigercrm/
#Disallow: /sendstudio/
Disallow: /feng/
Disallow: /emyth/
Disallow: /projects/
Disallow: /stuffed/
Disallow: /report/
Disallow: /includes/
#Disallow: /modules/
Disallow: /scripts/
#Disallow: /themes/
Disallow: /node/240
Disallow: /bonus_int
#Disallow: /artmako
Disallow: /afftrack
Disallow: */add/
Disallow: /landing/newsletter_confirm
Disallow: /landing/newsletter_confirm_woman
Disallow: /landing/newsletter_confirm_business
Disallow: /about1
Disallow: /order
Disallow: /form_test.php

User-agent: ia_archiver
Disallow: /

User-agent: gigabot
Disallow: /
User-agent: BecomeBot
Disallow: /
User-agent: gigablast
Disallow: /
User-agent: www.agiftfor.co.uk
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: memento
Disallow: /