summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--extensions/SiteMapIndex/Extension.pm2
-rw-r--r--extensions/SiteMapIndex/robots.txt14
-rw-r--r--robots.txt27
3 files changed, 26 insertions, 17 deletions
diff --git a/extensions/SiteMapIndex/Extension.pm b/extensions/SiteMapIndex/Extension.pm
index f36fa8c81..23c460fb8 100644
--- a/extensions/SiteMapIndex/Extension.pm
+++ b/extensions/SiteMapIndex/Extension.pm
@@ -134,7 +134,7 @@ sub _fix_robots_txt {
{ local $/; $current_contents = <$current_fh> }
$current_fh->close();
- return if $current_contents =~ m{^Allow: \/\*show_bug\.cgi}ms;
+ return if $current_contents =~ /^Sitemap:/m;
my $backup_name = "$cgi_path/robots.txt.old";
print get_text('sitemap_fixing_robots', { current => $robots_file,
backup => $backup_name }), "\n";
diff --git a/extensions/SiteMapIndex/robots.txt b/extensions/SiteMapIndex/robots.txt
index 7adeebd32..74cc63074 100644
--- a/extensions/SiteMapIndex/robots.txt
+++ b/extensions/SiteMapIndex/robots.txt
@@ -1,10 +1,10 @@
User-agent: *
Disallow: /*.cgi
-Disallow: /*show_bug.cgi*ctype=*
-Allow: /
-Allow: /*index.cgi
-Allow: /*page.cgi
-Allow: /*show_bug.cgi
-Allow: /*describecomponents.cgi
-Allow: /*data/SiteMapIndex/sitemap*.xml.gz
+Disallow: /show_bug.cgi*ctype=*
+Allow: /$
+Allow: /index.cgi
+Allow: /page.cgi
+Allow: /show_bug.cgi
+Allow: /describecomponents.cgi
+Allow: /data/SiteMapIndex/sitemap*.xml.gz
Sitemap: SITEMAP_URL
diff --git a/robots.txt b/robots.txt
index b26f019f7..9a64b62d8 100644
--- a/robots.txt
+++ b/robots.txt
@@ -1,10 +1,19 @@
User-agent: *
-Disallow: /*.cgi
-Disallow: /*show_bug.cgi*ctype=*
-Allow: /
-Allow: /*index.cgi
-Allow: /*page.cgi
-Allow: /*show_bug.cgi
-Allow: /*describecomponents.cgi
-Allow: /*data/SiteMapIndex/sitemap*.xml.gz
-Sitemap: https://bugzilla.mozilla.org/page.cgi?id=sitemap/sitemap.xml
+Disallow: /
+
+Allow: /$
+Allow: /index.cgi
+
+Allow: /page.cgi
+Disallow: /page.cgi*id=voting*
+Disallow: /page.cgi*id=productdashboard*
+
+Allow: /show_bug.cgi
+Disallow: /show_bug.cgi*ctype=*
+Disallow: /show_bug.cgi*format=multiple*
+
+Allow: /describecomponents.cgi
+Allow: /describekeywords.cgi
+
+Allow: /data/SiteMapIndex/sitemap*.xml.gz
+Sitemap: http://bugzilla.mozilla.org/page.cgi?id=sitemap/sitemap.xml