summaryrefslogtreecommitdiffstats
path: root/extensions/SiteMapIndex/robots.txt
diff options
context:
space:
mode:
Diffstat (limited to 'extensions/SiteMapIndex/robots.txt')
-rw-r--r--extensions/SiteMapIndex/robots.txt10
1 files changed, 10 insertions, 0 deletions
diff --git a/extensions/SiteMapIndex/robots.txt b/extensions/SiteMapIndex/robots.txt
new file mode 100644
index 000000000..7adeebd32
--- /dev/null
+++ b/extensions/SiteMapIndex/robots.txt
@@ -0,0 +1,10 @@
+User-agent: *
+Disallow: /*.cgi
+Disallow: /*show_bug.cgi*ctype=*
+Allow: /
+Allow: /*index.cgi
+Allow: /*page.cgi
+Allow: /*show_bug.cgi
+Allow: /*describecomponents.cgi
+Allow: /*data/SiteMapIndex/sitemap*.xml.gz
+Sitemap: SITEMAP_URL