From 6df160df8d911bc11f6dbe8889bdaa0e89460fd3 Mon Sep 17 00:00:00 2001
From: Byron Jones <bjones@mozilla.com>
Date: Tue, 2 Jul 2013 13:06:36 +0800
Subject: Bug 888829: updated robots.txt is being replaced by the SiteMapIndex
 extension

---
 extensions/SiteMapIndex/Extension.pm |  2 +-
 extensions/SiteMapIndex/robots.txt   | 14 +++++++-------
 robots.txt                           | 27 ++++++++++++++++++---------
 3 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/extensions/SiteMapIndex/Extension.pm b/extensions/SiteMapIndex/Extension.pm
index f36fa8c81..23c460fb8 100644
--- a/extensions/SiteMapIndex/Extension.pm
+++ b/extensions/SiteMapIndex/Extension.pm
@@ -134,7 +134,7 @@ sub _fix_robots_txt {
     { local $/; $current_contents = <$current_fh> }
     $current_fh->close();
 
-    return if $current_contents =~ m{^Allow: \/\*show_bug\.cgi}ms;
+    return if $current_contents =~ /^Sitemap:/m;
     my $backup_name = "$cgi_path/robots.txt.old";
     print get_text('sitemap_fixing_robots', { current => $robots_file,
                                               backup  => $backup_name }), "\n";
diff --git a/extensions/SiteMapIndex/robots.txt b/extensions/SiteMapIndex/robots.txt
index 7adeebd32..74cc63074 100644
--- a/extensions/SiteMapIndex/robots.txt
+++ b/extensions/SiteMapIndex/robots.txt
@@ -1,10 +1,10 @@
 User-agent: *
 Disallow: /*.cgi
-Disallow: /*show_bug.cgi*ctype=*
-Allow: /
-Allow: /*index.cgi
-Allow: /*page.cgi
-Allow: /*show_bug.cgi
-Allow: /*describecomponents.cgi
-Allow: /*data/SiteMapIndex/sitemap*.xml.gz
+Disallow: /show_bug.cgi*ctype=*
+Allow: /$
+Allow: /index.cgi
+Allow: /page.cgi
+Allow: /show_bug.cgi
+Allow: /describecomponents.cgi
+Allow: /data/SiteMapIndex/sitemap*.xml.gz
 Sitemap: SITEMAP_URL
diff --git a/robots.txt b/robots.txt
index b26f019f7..9a64b62d8 100644
--- a/robots.txt
+++ b/robots.txt
@@ -1,10 +1,19 @@
 User-agent: *
-Disallow: /*.cgi
-Disallow: /*show_bug.cgi*ctype=*
-Allow: /
-Allow: /*index.cgi
-Allow: /*page.cgi
-Allow: /*show_bug.cgi
-Allow: /*describecomponents.cgi
-Allow: /*data/SiteMapIndex/sitemap*.xml.gz
-Sitemap: https://bugzilla.mozilla.org/page.cgi?id=sitemap/sitemap.xml
+Disallow: /
+
+Allow: /$
+Allow: /index.cgi
+
+Allow: /page.cgi
+Disallow: /page.cgi*id=voting*
+Disallow: /page.cgi*id=productdashboard*
+
+Allow: /show_bug.cgi
+Disallow: /show_bug.cgi*ctype=*
+Disallow: /show_bug.cgi*format=multiple*
+
+Allow: /describecomponents.cgi
+Allow: /describekeywords.cgi
+
+Allow: /data/SiteMapIndex/sitemap*.xml.gz
+Sitemap: http://bugzilla.mozilla.org/page.cgi?id=sitemap/sitemap.xml
-- 
cgit v1.2.3-24-g4f1b