Magento/Robots.md

+# Ceicom Commerce Magento Robots.txt 12/08/2019
+#
+# robots.txt
+#
+# Este arquivo é para evitar o rastreamento e indexação de certas partes
+# do seu site, por robôs e são executados por sites como Yahoo!, google etc.
+# Dizer a esses "robôs" onde não ir em seu site,
+# Economizando largura de banda e recursos de servidor.
+
+location /robots.txt { return 200 "### Autogenerated robots.txt\n
+
+# Sitemap
+Sitemap: http://www.meusite.com.br/sitemap.xml
+
+# Crawlers Setup
+User-agent: *
+Crawl-delay: 20
+
+User-agent: Facebot/1.0
+Crawl-delay: 20 # seconds
+
+User-agent: facebookexternalhit/1.1
+Crawl-delay: 20 # seconds
+
+# Allowable Index
+Allow: /*?p=
+Allow: /index.php/blog/
+Allow: /catalog/seo_sitemap/category/
+
+
+# Directories
+Disallow: /404/
+Disallow: /app/
+Disallow: /cgi-bin/
+Disallow: /downloader/
+Disallow: /includes/
+Disallow: /lib/
+Disallow: /magento/
+Disallow: /pkginfo/
+Disallow: /report/
+Disallow: /stats/
+Disallow: /var/
+
+# Paths (clean URLs)
+Disallow: /index.php/
+Disallow: /catalog/product_compare/
+Disallow: /catalog/category/view/
+Disallow: /catalog/product/view/
+Disallow: /catalogsearch/
+Disallow: /catalogsearch/result/
+Disallow: /checkout/
+Disallow: /control/
+Disallow: /contacts/
+Disallow: /customer/
+Disallow: /customize/
+Disallow: /*/filter/
+#Disallow: /*/l/
+Disallow: /filter/
+#Disallow: /l/
+Disallow: /newsletter/
+Disallow: /poll/
+Disallow: /review/
+Disallow: /review/s/
+Disallow: /sendfriend/
+Disallow: /tag/
+Disallow: /wishlist/
+Disallow: /productalert
+Disallow: /captcha/refresh/
+
+# Files
+Disallow: /cron.php
+Disallow: /cron.sh
+Disallow: /mage
+Disallow: /error_log
+Disallow: /install.php
+
+# CE
+Disallow: /LICENSE.html
+Disallow: /LICENSE.txt
+Disallow: /LICENSE_AFL.txt
+
+# Enterprise Edition
+Disallow: /LICENSE_EE.html
+Disallow: /LICENSE_EE.txt
+Disallow: /RELEASE_NOTES.txt
+Disallow: /STATUS.txt
+
+# Paths (no clean URLs)
+Disallow: /*.php
+Disallow: /*?p=*
+Disallow: /*?SID=
+
+# Disallow user interactive pages
+Disallow: /*review
+Disallow: /product_info
+Disallow: /popup_image
+
+## Disallow osCommerce links
+Disallow: *cPath=*
+
+";
+}
\ No newline at end of file