# Crawlers Setup

    User-agent: *

    Crawl-delay: 10

Sitemap: https://www.audiodubai.ae/sitemaps/ae/sitemap.xml

    # Allowable Index

    Allow: /*?p=



    Allow: /media/



    # Directories

    Disallow: /404/

    Disallow: /app/

    Disallow: /cgi-bin/

    Disallow: /downloader/

    Disallow: /includes/

    Disallow: /js/

    Disallow: /lib/

    Disallow: /magento/

    # Allow: /media/

    Disallow: /pkginfo/

    Disallow: /report/

    Allow: /skin/

    Disallow: /stats/

    Disallow: /var/



    # Paths (clean URLs)

    Disallow: /index.php/

    Disallow: /catalog/product_compare/

    Disallow: /catalog/category/view/

    Disallow: /catalog/product/view/

    Disallow: /catalogsearch/

    Disallow: /checkout/

    Disallow: /control/

    Disallow: /contacts/

    Disallow: /customer/

    Disallow: /customize/

    Disallow: /newsletter/

    Disallow: /poll/

    Disallow: /review/

    Disallow: /sendfriend/

    Disallow: /tag/

    Disallow: /wishlist/



    # Files

    Disallow: /cron.php

    Disallow: /cron.sh

    Disallow: /error_log

    Disallow: /install.php

    Disallow: /LICENSE.html

    Disallow: /LICENSE.txt

    Disallow: /LICENSE_AFL.txt

    Disallow: /STATUS.txt



    # Paths (no clean URLs)

    Allow: /*.js$

    Allow: /*.css$

#Allow: /pro-sitemaps-4013797.php
#Allow: /pro-sitemaps-4013798.php
#Allow: /pro-sitemaps-4013743.php
#Allow: /pro-sitemaps-4013800.php
#Allow: /pro-sitemaps-4013793.php
#Allow: /pro-sitemaps-4013794.php
#Allow: /pro-sitemaps-4013799.php
#Allow: /pro-sitemaps-4013795.php
#Allow: /pro-sitemaps-4013796.php

    Disallow: /*.php$

    Disallow: /*?p=*&

    Disallow: /*?SID=