Ejemplo de archivo ROBOTS.TXT, Prestashop | Develop Site

# robots.txt automaticaly generated by PrestaShop e-commerce open-source solution # http://www.prestashop.com - http://www.prestashop.com/forums # This file is to prevent the crawling and indexing of certain parts # of your site by web crawlers and spiders run by sites like Yahoo! # and Google. By telling these "robots" where not to go on your site, # you save bandwidth and server resources. # For more information about the robots.txt standard, see: # http://www.robotstxt.org/wc/robots.html

# SOBRECARGA Crawl-Delay: 60

User-agent: Orthogaffe Disallow: /

User-agent: UbiCrawler Disallow: /

User-agent: DOC Disallow: /

User-agent: Zao Disallow: /

User-agent: sitecheck.internetseer.com Disallow: /

User-agent: Zealbot Disallow: /

User-agent: MSIECrawler Disallow: /

User-agent: SiteSnagger Disallow: /

User-agent: WebStripper Disallow: /

User-agent: WebCopier Disallow: /

User-agent: Fetch Disallow: /

User-agent: Offline Explorer Disallow: /

User-agent: Teleport Disallow: /

User-agent: TeleportPro Disallow: /

User-agent: WebZIP Disallow: /

User-agent: linko Disallow: /

User-agent: HTTrack Disallow: /

User-agent: Microsoft.URL.Control Disallow: /

User-agent: Xenu Disallow: /

User-agent: larbin Disallow: /

User-agent: libwww Disallow: /

User-agent: ZyBORG Disallow: /

User-agent: Download Ninja Disallow: /

User-agent: wget Disallow: /

User-agent: grub-client Disallow: /

User-agent: k2spider Disallow: /

User-agent: NPBot Disallow: /

User-agent: WebReaper Disallow: /

User-agent: YandexBot Disallow: /

User-agent: YandexMedia Disallow: /

User-agent: YandexImages Disallow: /

User-agent: YandexCatalog Disallow: /

User-agent: YaDirectFetcher Disallow: /

User-agent: YandexBlogs Disallow: /

User-agent: YandexNews Disallow: /

User-agent: YandexPagechecker Disallow: /

User-agent: YandexMetrika Disallow: /

User-agent: YandexMarket Disallow: /

User-agent: YandexCalendar Disallow: /

user-agent: AhrefsBot disallow: /

user-agent: AhrefsBot/5.0 disallow: /

user-agent: BaiDuSpider disallow: /

user-agent: BaiDuSpider/2.0 disallow: /

User-agent: worldwebheritage.org disallow: /

User-agent: worldwebheritage.org/1.0 disallow: /

User-agent: * # Private pages Disallow: /*orderby= Disallow: /*orderway= Disallow: /*tag= Disallow: /*id_currency= Disallow: /*search_query= Disallow: /*back= Disallow: /*n= Disallow: /*controller=addresses Disallow: /*controller=address Disallow: /*controller=authentication Disallow: /*controller=cart Disallow: /*controller=discount Disallow: /*controller=footer Disallow: /*controller=get-file Disallow: /*controller=header Disallow: /*controller=history Disallow: /*controller=identity Disallow: /*controller=images.inc Disallow: /*controller=init Disallow: /*controller=my-account Disallow: /*controller=order Disallow: /*controller=order-opc Disallow: /*controller=order-slip Disallow: /*controller=order-detail Disallow: /*controller=order-follow Disallow: /*controller=order-return Disallow: /*controller=order-confirmation Disallow: /*controller=pagination Disallow: /*controller=password Disallow: /*controller=pdf-invoice Disallow: /*controller=pdf-order-return Disallow: /*controller=pdf-order-slip Disallow: /*controller=product-sort Disallow: /*controller=search Disallow: /*controller=statistics Disallow: /*controller=attachment Disallow: /*controller=guest-tracking # Directories Disallow: */classes/ Disallow: */config/ Disallow: */download/ Disallow: */mails/ Disallow: */modules/ Disallow: */translations/ Disallow: */tools/ # Files Disallow: /*es/contrasena-olvidado Disallow: /*es/direccion Disallow: /*es/direcciones Disallow: /*es/autenticacion Disallow: /*es/carro-de-la-compra Disallow: /*es/descuento Disallow: /*es/historial-de-pedidos Disallow: /*es/identidad Disallow: /*es/mi-cuenta Disallow: /*es/devolucion-de-productos Disallow: /*es/vales Disallow: /*es/carrito Disallow: /*es/buscar Disallow: /*es/pedido-rapido Disallow: /*es/estado-pedido Disallow: /*es/confirmacion-pedido

Español