Sitemap: https://www.fastbuildsupplies.co.uk/sitemap.xml

Crawl-delay: 4

User-agent: googlebot
User-agent: *
 
# # Do not allow indexing files and folders that are required during development: CVS, SVN directory and dump files
Disallow: /CVS
Disallow: /*.Svn$
Disallow: /*.Idea$
Disallow: /*.Sql$
Disallow: /*.Tgz$

# # Do not index the general technical Magento directory
Disallow: /errors/
Disallow: /quickview/
Disallow: /wishlist/
Disallow: /media/
Disallow: /directory/currency/switch/currency/

# # Do not index the page subcategories that are sorted or filtered.
Disallow: *?
Allow: /*?child_id=*
Disallow: /*?dir=*
Disallow: /*?dir=desc
Disallow: /*?dir=asc
Disallow: /*?mode=*
Disallow: /*?limit=*
 
# # Do not index the second copy of the home page (example.com / index.php /). Un-comment only if you have activated Magento SEO URLs.
Disallow: /index.php/
 
# # Do not index the link from the session ID
Disallow: /*?*__SID=
Disallow: /*?*___SID=
Disallow: /*?*SID=
 
# # Do not index the page checkout and user account
Disallow: /checkout/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/account/login/
 
# # Do not index the search page and CEO, non-optimized link categories
Disallow: /catalogsearch/
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
 
# # Do not index the general technical directories and files on a server
Disallow: /cgi-bin/
Disallow: /phpinfo.php

# # Block Bad Bots
User-Agent: rogerbot
Disallow: /
User-Agent: DotBot
Disallow: /
User-Agent: spbot
Disallow: /
User-Agent: careerbot
Disallow: /
User-agent: UbiCrawler
Disallow: /
User-agent: DOC
Disallow: /
User-agent: Zao
Disallow: /
User-agent: Zealbot
Disallow: /
User-agent: MSIECrawler
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: Fetch
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: linko
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: Microsoft.URL.Control
Disallow: /
User-agent: Xenu
Disallow: /
User-agent: larbin
Disallow: /
User-agent: libwww
Disallow: /
User-agent: ZyBORG
Disallow: /
User-agent: wget
Disallow: /
User-agent: grub-client
Disallow: /
User-agent: k2spider
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: SearchmetricsBot
Disallow: /
User-agent: YandexImages
Disallow: /
User-agent: OmniExplorer_Bot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: pixraybot
Disallow: /
User-agent: Yandex
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: sistrix
Disallow: /
User-agent: montastic-monitor
Disallow: /
User-agent: Xenu Link Sleuth/1.3.8
Disallow: /
User-agent: BLP_bbot/0.1
Disallow: /
User-agent: Nutch
Disallow: /
User-agent: Nutch-1.4
Disallow: /
User-agent: BLP_bbot
Disallow: /
User-agent: ezooms
Disallow: /
User-agent: Esribot
Disallow: /
User-agent: DynamicTemplates/2.0
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: Sogou web spider/4.0
Disallow: /
User-agent: sogou
Disallow: /
User-agent: sogou web spider
Disallow: /
User-agent: YoudaoBot
Disallow: /
User-agent: moget
Disallow: /
User-agent: ichiro
Disallow: /
User-agent: 360Spider
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: NaverBot
Disallow: /
User-agent: Yet
Disallow: /
User-agent: 008
Disallow: /
User-agent: EasouSpider
Disallow: /
User-agent: Grapeshot
Disallow: /
User-agent: MojeekBot
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: coccoc
Disallow: /
User-agent: Sogou web spider
Disallow: /
User-agent: JikeSpider
Disallow: /
User-agent: Sosospider
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: 360spider
Disallow: /
User-agent: Baiduspider-image
Disallow: /
User-agent: YoudaoBot
Disallow: /
User-agent: NPBot
Disallow: /
User-agent: WebReaper
Disallow: /

# # handle images
User-Agent: Googlebot-Image
Allow: /media/
User-Agent: Googlebot
Allow: /media/
User-agent: SemrushBot
Allow: /media/

User-Agent: Bingbot
Crawl-delay: 20

User-Agent: Googlebot
Disallow: 

User-Agent: Googlebot-image
Disallow: