path: root/extensions/SiteMapIndex/lib/Util.pm

# -*- Mode: perl; indent-tabs-mode: nil -*-
#
# The contents of this file are subject to the Mozilla Public
# License Version 1.1 (the "License"); you may not use this file
# except in compliance with the License. You may obtain a copy of
# the License at http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS
# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
# implied. See the License for the specific language governing
# rights and limitations under the License.
#
# The Original Code is the Sitemap Bugzilla Extension.
#
# The Initial Developer of the Original Code is Everything Solved, Inc.
# Portions created by the Initial Developer are Copyright (C) 2010 the
# Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Max Kanat-Alexander <mkanat@bugzilla.org>
#   Dave Lawrence <dkl@mozilla.com>

package Bugzilla::Extension::SiteMapIndex::Util;

use 5.10.1;
use strict;
use warnings;

use base qw(Exporter);
our @EXPORT = qw(
    generate_sitemap
    bug_is_ok_to_index
);

use Bugzilla::Extension::SiteMapIndex::Constants;

use Bugzilla::Util qw(datetime_from url_quote);
use Bugzilla::Constants qw(bz_locations);

use Scalar::Util qw(blessed);
use IO::Compress::Gzip qw(gzip $GzipError);

sub too_young_date {
    my $hours_ago = DateTime->now(time_zone => Bugzilla->local_timezone);
    $hours_ago->subtract(hours => SITEMAP_DELAY);
    return $hours_ago;
}

sub bug_is_ok_to_index {
    my ($bug) = @_;
    return 1 unless blessed($bug) && $bug->isa('Bugzilla::Bug') && !$bug->{error};
    my $creation_ts = datetime_from($bug->creation_ts);
    return ($creation_ts && $creation_ts lt too_young_date()) ? 1 : 0;
}

# We put two things in the Sitemap: a list of Browse links for products,
# and links to bugs.
sub generate_sitemap {
    my ($extension_name) = @_;

    # If file is less than SITEMAP_AGE hours old, then read in and send to caller.
    # If greater, then regenerate and send the new version.
    my  $index_file = bz_locations->{'datadir'} . "/$extension_name/sitemap_index.xml";
    if (-e $index_file) {
        my $index_mtime = (stat($index_file))[9];
        my $index_hours = sprintf("%d", (time() - $index_mtime) / 60 / 60); # in hours
        if ($index_hours < SITEMAP_AGE) {
            my $index_fh = new IO::File($index_file, 'r');
            $index_fh || die "Could not open current sitemap index: $!";
            my $index_xml;
            { local $/; $index_xml = <$index_fh> }
            $index_fh->close() || die "Could not close current sitemap index: $!";

            return $index_xml;
        }
    }

    # Set the atime and mtime of the index file to the current time
    # in case another request is made before we finish.
    utime(undef, undef, $index_file);

    # Sitemaps must never contain private data.
    Bugzilla->logout_request();
    my $user = Bugzilla->user;
    my $products = $user->get_accessible_products;

    my $num_bugs = SITEMAP_MAX - scalar(@$products);
    # We do this date math outside of the database because databases
    # usually do better with a straight comparison value.
    my $hours_ago = too_young_date();

    # We don't use Bugzilla::Bug objects, because this could be a tremendous
    # amount of data, and we only want a little. Also, we only display
    # bugs that are not in any group. We show the last $num_bugs
    # most-recently-updated bugs.
    my $dbh = Bugzilla->dbh;
    my $bug_sth = $dbh->prepare(
        'SELECT bugs.bug_id, bugs.delta_ts
           FROM bugs
                LEFT JOIN bug_group_map ON bugs.bug_id = bug_group_map.bug_id
          WHERE bug_group_map.bug_id IS NULL AND creation_ts < ?
        ' . $dbh->sql_limit($num_bugs, '?'));

    my $filecount = 1;
    my $filelist = [];
    my $offset = 0;

    while (1) {
        my $bugs = [];

        $bug_sth->execute($hours_ago, $offset);

        while (my ($bug_id, $delta_ts) = $bug_sth->fetchrow_array()) {
            push(@$bugs, { bug_id => $bug_id, delta_ts => $delta_ts });
        }

        last if !@$bugs;

        # We only need the product links in the first sitemap file
        $products = [] if $filecount > 1;

        push(@$filelist, _generate_sitemap_file($extension_name, $filecount, $products, $bugs));

        $filecount++;
        $offset += $num_bugs;
    }

    # Generate index file
    return _generate_sitemap_index($extension_name, $filelist);
}

sub _generate_sitemap_index {
    my ($extension_name, $filelist) = @_;

    my $dbh = Bugzilla->dbh;
    my $timestamp = $dbh->selectrow_array(
        "SELECT " . $dbh->sql_date_format('NOW()', '%Y-%m-%d'));

    my $index_xml = <<END;
<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
END

    foreach my $filename (@$filelist) {
        $index_xml .= "
  <sitemap>
    <loc>" . Bugzilla->localconfig->{urlbase} . "data/$extension_name/$filename</loc>
    <lastmod>$timestamp</lastmod>
  </sitemap>
";
    }

    $index_xml .= <<END;
</sitemapindex>
END

    my  $index_file = bz_locations->{'datadir'} . "/$extension_name/sitemap_index.xml";
    my $index_fh = new IO::File($index_file, 'w');
    $index_fh || die "Could not open new sitemap index: $!";
    print $index_fh $index_xml;
    $index_fh->close() || die "Could not close new sitemap index: $!";

    return $index_xml;
}

sub _generate_sitemap_file {
    my ($extension_name, $filecount, $products, $bugs) = @_;

    my $bug_url = Bugzilla->localconfig->{urlbase} . 'show_bug.cgi?id=';
    my $product_url = Bugzilla->localconfig->{urlbase} . 'describecomponents.cgi?product=';

    my $sitemap_xml = <<END;
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
END

    foreach my $product (@$products) {
        $sitemap_xml .= "
  <url>
    <loc>" . $product_url . url_quote($product->name) . "</loc>
    <changefreq>daily</changefreq>
    <priority>0.4</priority>
  </url>
";
    }

    foreach my $bug (@$bugs) {
        $sitemap_xml .= "
  <url>
    <loc>" . $bug_url . $bug->{bug_id} . "</loc>
    <lastmod>" . datetime_from($bug->{delta_ts}, 'UTC')->iso8601 . 'Z' . "</lastmod>
  </url>
";
    }

    $sitemap_xml .= <<END;
</urlset>
END

    # Write the compressed sitemap data to a file in the cgi root so that they can
    # be accessed by the search engines.
    my $filename = "sitemap$filecount.xml.gz";
    gzip \$sitemap_xml => bz_locations->{'datadir'} . "/$extension_name/$filename"
        || die "gzip failed: $GzipError\n";

    return $filename;
}

1;