robots.txt 299 B

1234567891011121314
  1. ---
  2. layout: null
  3. permalink: /robots.txt
  4. ---
  5. {% for bot in site.bad_bots %}User-Agent: {{ bot }}
  6. Disallow: /
  7. {% endfor %}User-Agent: *
  8. Disallow: /404.html{% for page in site.pages %}{% if page.sitemap == false %}
  9. Disallow: {{ page.permalink }}{% endif %}{% endfor %}
  10. Allow: /
  11. Sitemap: /sitemap.xml