From 63dd3061a164345095847e502bb5395bb5570c3a Mon Sep 17 00:00:00 2001 From: Mary Umoh Date: Tue, 8 Aug 2017 17:12:08 -0400 Subject: Bug 1386336 - Generate robots.txt at checksetup.pl from a template --- Bugzilla/Install/Filesystem.pm | 18 ++++++++-- extensions/SiteMapIndex/Extension.pm | 38 ++-------------------- extensions/SiteMapIndex/robots.txt | 10 ------ .../template/en/default/hook/robots-end.txt.tmpl | 2 ++ robots.txt | 20 ------------ template/en/default/robots.txt.tmpl | 19 +++++++++++ 6 files changed, 39 insertions(+), 68 deletions(-) delete mode 100644 extensions/SiteMapIndex/robots.txt create mode 100644 extensions/SiteMapIndex/template/en/default/hook/robots-end.txt.tmpl delete mode 100644 robots.txt create mode 100644 template/en/default/robots.txt.tmpl diff --git a/Bugzilla/Install/Filesystem.pm b/Bugzilla/Install/Filesystem.pm index adc1815c1..3114d64be 100644 --- a/Bugzilla/Install/Filesystem.pm +++ b/Bugzilla/Install/Filesystem.pm @@ -214,11 +214,10 @@ sub FILESYSTEM { 'Bugzilla.pm' => { perms => CGI_READ }, "$localconfig*" => { perms => CGI_READ }, - 'META.*' => { perms => CGI_READ }, - 'MYMETA.*' => { perms => CGI_READ }, + 'META.*' => { perms => CGI_READ }, + 'MYMETA.*' => { perms => CGI_READ }, 'bugzilla.dtd' => { perms => WS_SERVE }, 'mod_perl.pl' => { perms => WS_SERVE }, - 'robots.txt' => { perms => WS_SERVE }, '.htaccess' => { perms => WS_SERVE }, 'cvs-update.log' => { perms => WS_SERVE }, 'scripts/sendunsentbugmail.pl' => { perms => WS_EXECUTE }, @@ -405,6 +404,9 @@ sub FILESYSTEM { "skins/yui3.css" => { perms => CGI_READ, overwrite => 1, contents => $yui3_all_css }, + "robots.txt" => { perms => CGI_READ, + overwrite => 1, + contents => \&robots_txt}, ); # Because checksetup controls the creation of index.html separately @@ -952,6 +954,16 @@ sub _check_web_server_group { return $group_id; } +sub robots_txt { + my $output = ''; + my %vars; + Bugzilla::Hook::process("before_robots_txt", { vars => \%vars }); + Bugzilla->template->process("robots.txt.tmpl", \%vars, \$output) + or die Bugzilla->template->error; + return $output; +} + + 1; __END__ diff --git a/extensions/SiteMapIndex/Extension.pm b/extensions/SiteMapIndex/Extension.pm index 1e2010adb..a3f093485 100644 --- a/extensions/SiteMapIndex/Extension.pm +++ b/extensions/SiteMapIndex/Extension.pm @@ -90,9 +90,6 @@ sub install_before_final_checks { } return if (correct_urlbase() ne 'https://bugzilla.mozilla.org/'); - - - $self->_fix_robots_txt(); } sub install_filesystem { @@ -127,38 +124,9 @@ EOT }; } -sub _fix_robots_txt { - my ($self) = @_; - my $cgi_path = bz_locations()->{'cgi_path'}; - my $robots_file = "$cgi_path/robots.txt"; - my $current_fh = new IO::File("$cgi_path/robots.txt", 'r'); - if (!$current_fh) { - warn "$robots_file: $!"; - return; - } - - my $current_contents; - { local $/; $current_contents = <$current_fh> } - $current_fh->close(); - - return if $current_contents =~ /^Sitemap:/m; - my $backup_name = "$cgi_path/robots.txt.old"; - print get_text('sitemap_fixing_robots', { current => $robots_file, - backup => $backup_name }), "\n"; - rename $robots_file, $backup_name or die "backup failed: $!"; - - my $new_fh = new IO::File($self->package_dir . '/robots.txt', 'r'); - $new_fh || die "Could not open new robots.txt template file: $!"; - my $new_contents; - { local $/; $new_contents = <$new_fh> } - $new_fh->close() || die "Could not close new robots.txt template file: $!"; - - my $sitemap_url = correct_urlbase() . SITEMAP_URL; - $new_contents =~ s/SITEMAP_URL/$sitemap_url/; - $new_fh = new IO::File("$cgi_path/robots.txt", 'w'); - $new_fh || die "Could not open new robots.txt file: $!"; - print $new_fh $new_contents; - $new_fh->close() || die "Could not close new robots.txt file: $!"; +sub before_robots_txt { + my ($self, $args) = @_; + $args->{vars}{SITEMAP_URL} = correct_urlbase() . SITEMAP_URL; } __PACKAGE__->NAME; diff --git a/extensions/SiteMapIndex/robots.txt b/extensions/SiteMapIndex/robots.txt deleted file mode 100644 index 74cc63074..000000000 --- a/extensions/SiteMapIndex/robots.txt +++ /dev/null @@ -1,10 +0,0 @@ -User-agent: * -Disallow: /*.cgi -Disallow: /show_bug.cgi*ctype=* -Allow: /$ -Allow: /index.cgi -Allow: /page.cgi -Allow: /show_bug.cgi -Allow: /describecomponents.cgi -Allow: /data/SiteMapIndex/sitemap*.xml.gz -Sitemap: SITEMAP_URL diff --git a/extensions/SiteMapIndex/template/en/default/hook/robots-end.txt.tmpl b/extensions/SiteMapIndex/template/en/default/hook/robots-end.txt.tmpl new file mode 100644 index 000000000..818afb151 --- /dev/null +++ b/extensions/SiteMapIndex/template/en/default/hook/robots-end.txt.tmpl @@ -0,0 +1,2 @@ +Allow: /data/SiteMapIndex/sitemap*.xml.gz +Sitemap: [% SITEMAP_URL %] diff --git a/robots.txt b/robots.txt deleted file mode 100644 index cececd632..000000000 --- a/robots.txt +++ /dev/null @@ -1,20 +0,0 @@ -User-agent: * -Disallow: / - -Allow: /$ -Allow: /index.cgi - -Allow: /page.cgi -Disallow: /page.cgi*id=voting* -Disallow: /page.cgi*id=productdashboard* -Disallow: /page.cgi*id=user_activity* - -Allow: /show_bug.cgi -Disallow: /show_bug.cgi*ctype=* -Disallow: /show_bug.cgi*format=multiple* - -Allow: /describecomponents.cgi -Allow: /describekeywords.cgi - -Allow: /data/SiteMapIndex/sitemap*.xml.gz -Sitemap: http://bugzilla.mozilla.org/page.cgi?id=sitemap/sitemap.xml diff --git a/template/en/default/robots.txt.tmpl b/template/en/default/robots.txt.tmpl new file mode 100644 index 000000000..762b000ec --- /dev/null +++ b/template/en/default/robots.txt.tmpl @@ -0,0 +1,19 @@ +User-agent: * +Disallow: / + +Allow: /$ +Allow: /index.cgi + +Allow: /page.cgi +Disallow: /page.cgi*id=voting* +Disallow: /page.cgi*id=productdashboard* +Disallow: /page.cgi*id=user_activity* + +Allow: /show_bug.cgi +Disallow: /show_bug.cgi*ctype=* +Disallow: /show_bug.cgi*format=multiple* + +Allow: /describecomponents.cgi +Allow: /describekeywords.cgi + +[% Hook.process("end") %] -- cgit v1.2.3-24-g4f1b