summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--extensions/OpenGraph/template/en/default/hook/robots-end.txt.tmpl4
-rw-r--r--extensions/SiteMapIndex/template/en/default/hook/robots-end.txt.tmpl3
-rw-r--r--template/en/default/robots.txt.tmpl2
3 files changed, 9 insertions, 0 deletions
diff --git a/extensions/OpenGraph/template/en/default/hook/robots-end.txt.tmpl b/extensions/OpenGraph/template/en/default/hook/robots-end.txt.tmpl
new file mode 100644
index 000000000..217924001
--- /dev/null
+++ b/extensions/OpenGraph/template/en/default/hook/robots-end.txt.tmpl
@@ -0,0 +1,4 @@
+[%# comment lines are required to produce line breaks %]
+#
+Allow: /extensions/OpenGraph/web/
+#
diff --git a/extensions/SiteMapIndex/template/en/default/hook/robots-end.txt.tmpl b/extensions/SiteMapIndex/template/en/default/hook/robots-end.txt.tmpl
index 818afb151..8805de0aa 100644
--- a/extensions/SiteMapIndex/template/en/default/hook/robots-end.txt.tmpl
+++ b/extensions/SiteMapIndex/template/en/default/hook/robots-end.txt.tmpl
@@ -1,2 +1,5 @@
+[%# comment lines are required to produce line breaks %]
+#
Allow: /data/SiteMapIndex/sitemap*.xml.gz
Sitemap: [% SITEMAP_URL %]
+#
diff --git a/template/en/default/robots.txt.tmpl b/template/en/default/robots.txt.tmpl
index 7ef83c0f1..a4f8aed24 100644
--- a/template/en/default/robots.txt.tmpl
+++ b/template/en/default/robots.txt.tmpl
@@ -19,6 +19,8 @@ Disallow: /show_bug.cgi*format=multiple*
Allow: /describecomponents.cgi
Allow: /describekeywords.cgi
+[% FILTER remove('#.*') %]
[% Hook.process("end") %]
+[% END %]
[% END %]