Bug 1386336 - Generate robots.txt at checksetup.pl from a template

3 files changed, 5 insertions, 45 deletions

diff --git a/extensions/SiteMapIndex/Extension.pm b/extensions/SiteMapIndex/Extension.pm
index 1e2010adb..a3f093485 100644
--- a/extensions/SiteMapIndex/Extension.pm
+++ b/extensions/SiteMapIndex/Extension.pm
@@ -90,9 +90,6 @@ sub install_before_final_checks {
     }
 
     return if (correct_urlbase() ne 'https://bugzilla.mozilla.org/');
-
-
-    $self->_fix_robots_txt();
 }
 
 sub install_filesystem {
@@ -127,38 +124,9 @@ EOT
     };
 }
 
-sub _fix_robots_txt {
-    my ($self) = @_;
-    my $cgi_path = bz_locations()->{'cgi_path'};
-    my $robots_file = "$cgi_path/robots.txt";
-    my $current_fh = new IO::File("$cgi_path/robots.txt", 'r');
-    if (!$current_fh) {
-        warn "$robots_file: $!";
-        return;
-    }
-
-    my $current_contents;
-    { local $/; $current_contents = <$current_fh> }
-    $current_fh->close();
-
-    return if $current_contents =~ /^Sitemap:/m;
-    my $backup_name = "$cgi_path/robots.txt.old";
-    print get_text('sitemap_fixing_robots', { current => $robots_file,
-                                              backup  => $backup_name }), "\n";
-    rename $robots_file, $backup_name or die "backup failed: $!";
-
-    my $new_fh = new IO::File($self->package_dir . '/robots.txt', 'r');
-    $new_fh || die "Could not open new robots.txt template file: $!";
-    my $new_contents;
-    { local $/; $new_contents = <$new_fh> }
-    $new_fh->close() || die "Could not close new robots.txt template file: $!";
-
-    my $sitemap_url = correct_urlbase() . SITEMAP_URL;
-    $new_contents =~ s/SITEMAP_URL/$sitemap_url/;
-    $new_fh = new IO::File("$cgi_path/robots.txt", 'w');
-    $new_fh || die "Could not open new robots.txt file: $!";
-    print $new_fh $new_contents;
-    $new_fh->close() || die "Could not close new robots.txt file: $!";
+sub before_robots_txt {
+    my ($self, $args) = @_;
+    $args->{vars}{SITEMAP_URL} = correct_urlbase() . SITEMAP_URL;
 }
 
 __PACKAGE__->NAME;
diff --git a/extensions/SiteMapIndex/robots.txt b/extensions/SiteMapIndex/robots.txt
deleted file mode 100644
index 74cc63074..000000000
--- a/extensions/SiteMapIndex/robots.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-User-agent: *
-Disallow: /*.cgi
-Disallow: /show_bug.cgi*ctype=*
-Allow: /$
-Allow: /index.cgi
-Allow: /page.cgi
-Allow: /show_bug.cgi
-Allow: /describecomponents.cgi
-Allow: /data/SiteMapIndex/sitemap*.xml.gz
-Sitemap: SITEMAP_URL
diff --git a/extensions/SiteMapIndex/template/en/default/hook/robots-end.txt.tmpl b/extensions/SiteMapIndex/template/en/default/hook/robots-end.txt.tmpl
new file mode 100644
index 000000000..818afb151
--- /dev/null
+++ b/extensions/SiteMapIndex/template/en/default/hook/robots-end.txt.tmpl
@@ -0,0 +1,2 @@
+Allow: /data/SiteMapIndex/sitemap*.xml.gz
+Sitemap: [% SITEMAP_URL %]