# DN Robots
# Crawlers Setup 
User-agent: * 
Crawl-delay: 30 

# Base Feed
Allow: /var/export/base_feed.csv

# Directories 
# Disallow: /404/ 
Disallow: /learn/gloosary/* 
Disallow: /app/ 
Disallow: /cgi-bin/ 
Disallow: /downloader/ 
Disallow: /includes/ 
#Disallow: /js/ 
Disallow: /lib/ 
Disallow: /magento/ 
Disallow: /pkginfo/ 
Disallow: /report/ 
#Disallow: /skin/ 
Disallow: /stats/ 
Disallow: /var/ 

# Paths (clean URLs) 
Disallow: /index.php/ 
Disallow: /catalog/product_compare/ 
Disallow: /catalog/category/view/ 
Disallow: /catalog/product/view/ 
Disallow: /catalogsearch/ 
Disallow: /checkout/ 
Disallow: /control/ 
Disallow: /contacts/ 
Disallow: /customer/ 
Disallow: /customize/ 
Disallow: /newsletter/ 
Disallow: /poll/ 
Disallow: /review/ 
Disallow: /sendfriend/ 
Disallow: /tag/ 
Disallow: /wishlist/ 
Disallow: /forms/Ajax/
Disallow: /fcprofile/
Disallow: /fcform/

# Files 
Disallow: /cron.php 
Disallow: /cron.sh 
Disallow: /error_log 
Disallow: /install.php 
Disallow: /LICENSE.html 
Disallow: /LICENSE.txt 
Disallow: /LICENSE_AFL.txt 
Disallow: /STATUS.txt 

# Paths (no clean URLs) 
#Disallow: /*.js$ 
#Disallow: /*.css$ 
Disallow: /*?p=*& 
Disallow: /*?SID= 

# Sitemaps
Sitemap: http://www.diamondnexus.com/sitemap.xml
host: diamondnexus.com
host: www.diamondnexus.com