diff options
| -rw-r--r-- | extensions/SiteMapIndex/Extension.pm | 2 | ||||
| -rw-r--r-- | extensions/SiteMapIndex/robots.txt | 14 | ||||
| -rw-r--r-- | robots.txt | 27 |
3 files changed, 26 insertions, 17 deletions
diff --git a/extensions/SiteMapIndex/Extension.pm b/extensions/SiteMapIndex/Extension.pm index f36fa8c81..23c460fb8 100644 --- a/extensions/SiteMapIndex/Extension.pm +++ b/extensions/SiteMapIndex/Extension.pm @@ -134,7 +134,7 @@ sub _fix_robots_txt { { local $/; $current_contents = <$current_fh> } $current_fh->close(); - return if $current_contents =~ m{^Allow: \/\*show_bug\.cgi}ms; + return if $current_contents =~ /^Sitemap:/m; my $backup_name = "$cgi_path/robots.txt.old"; print get_text('sitemap_fixing_robots', { current => $robots_file, backup => $backup_name }), "\n"; diff --git a/extensions/SiteMapIndex/robots.txt b/extensions/SiteMapIndex/robots.txt index 7adeebd32..74cc63074 100644 --- a/extensions/SiteMapIndex/robots.txt +++ b/extensions/SiteMapIndex/robots.txt @@ -1,10 +1,10 @@ User-agent: * Disallow: /*.cgi -Disallow: /*show_bug.cgi*ctype=* -Allow: / -Allow: /*index.cgi -Allow: /*page.cgi -Allow: /*show_bug.cgi -Allow: /*describecomponents.cgi -Allow: /*data/SiteMapIndex/sitemap*.xml.gz +Disallow: /show_bug.cgi*ctype=* +Allow: /$ +Allow: /index.cgi +Allow: /page.cgi +Allow: /show_bug.cgi +Allow: /describecomponents.cgi +Allow: /data/SiteMapIndex/sitemap*.xml.gz Sitemap: SITEMAP_URL diff --git a/robots.txt b/robots.txt index b26f019f7..9a64b62d8 100644 --- a/robots.txt +++ b/robots.txt @@ -1,10 +1,19 @@ User-agent: * -Disallow: /*.cgi -Disallow: /*show_bug.cgi*ctype=* -Allow: / -Allow: /*index.cgi -Allow: /*page.cgi -Allow: /*show_bug.cgi -Allow: /*describecomponents.cgi -Allow: /*data/SiteMapIndex/sitemap*.xml.gz -Sitemap: https://bugzilla.mozilla.org/page.cgi?id=sitemap/sitemap.xml +Disallow: / + +Allow: /$ +Allow: /index.cgi + +Allow: /page.cgi +Disallow: /page.cgi*id=voting* +Disallow: /page.cgi*id=productdashboard* + +Allow: /show_bug.cgi +Disallow: /show_bug.cgi*ctype=* +Disallow: /show_bug.cgi*format=multiple* + +Allow: /describecomponents.cgi +Allow: /describekeywords.cgi + +Allow: /data/SiteMapIndex/sitemap*.xml.gz +Sitemap: http://bugzilla.mozilla.org/page.cgi?id=sitemap/sitemap.xml |