Bug 284184: Allow Bugzilla to use an asynchronous job queue for sending mail.

Patch By Max Kanat-Alexander <mkanat@bugzilla.org> and Mark Smith <mark@plogs.net> r=glob, a=mkanat

12 files changed, 396 insertions, 6 deletions

diff --git a/Bugzilla/Config/Common.pm b/Bugzilla/Config/Common.pm
index d105d9db8..b6aa1a108 100644
--- a/Bugzilla/Config/Common.pm
+++ b/Bugzilla/Config/Common.pm
@@ -49,7 +49,7 @@ use base qw(Exporter);
        check_opsys check_shadowdb check_urlbase check_webdotbase
        check_netmask check_user_verify_class check_image_converter
        check_mail_delivery_method check_notification check_utf8
-       check_bug_status check_smtp_auth
+       check_bug_status check_smtp_auth check_theschwartz_available
 );
 
 # Checking functions for the various values
@@ -335,6 +335,15 @@ sub check_smtp_auth {
     return "";
 }
 
+sub check_theschwartz_available {
+    if (!eval { require TheSchwartz; require Daemon::Generic; }) {
+        return "Using the job queue requires that you have certain Perl"
+               . " modules installed. See the output of checksetup.pl"
+               . " for more information";
+    }
+    return "";
+}
+
 # OK, here are the parameter definitions themselves.
 #
 # Each definition is a hash with keys:
diff --git a/Bugzilla/Config/MTA.pm b/Bugzilla/Config/MTA.pm
index 37d99d967..c7843e286 100644
--- a/Bugzilla/Config/MTA.pm
+++ b/Bugzilla/Config/MTA.pm
@@ -58,6 +58,13 @@ sub get_param_list {
   },
 
   {
+   name => 'use_mailer_queue',
+   type => 'b',
+   default => 0,
+   checker => \&check_theschwartz_available,
+  },
+
+  {
    name => 'sendmailnow',
    type => 'b',
    default => 1
@@ -90,7 +97,6 @@ sub get_param_list {
    default => 7,
    checker => \&check_numeric
   },
-  
   {
    name => 'globalwatchers',
    type => 't',
diff --git a/Bugzilla/DB/Mysql.pm b/Bugzilla/DB/Mysql.pm
index 889b1f0da..f85bd31f1 100644
--- a/Bugzilla/DB/Mysql.pm
+++ b/Bugzilla/DB/Mysql.pm
@@ -63,7 +63,7 @@ sub new {
     my ($class, $user, $pass, $host, $dbname, $port, $sock) = @_;
 
     # construct the DSN from the parameters we got
-    my $dsn = "DBI:mysql:host=$host;database=$dbname";
+    my $dsn = "dbi:mysql:host=$host;database=$dbname";
     $dsn .= ";port=$port" if $port;
     $dsn .= ";mysql_socket=$sock" if $sock;
 
@@ -79,6 +79,9 @@ sub new {
     # a prefix 'private_'. See DBI documentation.
     $self->{private_bz_tables_locked} = "";
 
+    # Needed by TheSchwartz
+    $self->{private_bz_dsn} = $dsn;
+
     bless ($self, $class);
     
     # Bug 321645 - disable MySQL strict mode, if set
diff --git a/Bugzilla/DB/Oracle.pm b/Bugzilla/DB/Oracle.pm
index 833fce635..4153751fd 100644
--- a/Bugzilla/DB/Oracle.pm
+++ b/Bugzilla/DB/Oracle.pm
@@ -65,13 +65,15 @@ sub new {
     $ENV{'NLS_LANG'} = '.AL32UTF8' if Bugzilla->params->{'utf8'};
 
     # construct the DSN from the parameters we got
-    my $dsn = "DBI:Oracle:host=$host;sid=$dbname";
+    my $dsn = "dbi:Oracle:host=$host;sid=$dbname";
     $dsn .= ";port=$port" if $port;
     my $attrs = { FetchHashKeyName => 'NAME_lc',  
                   LongReadLen => ( Bugzilla->params->{'maxattachmentsize'}
                                      || 1000 ) * 1024, 
                 };
     my $self = $class->db_new($dsn, $user, $pass, $attrs);
+    # Needed by TheSchwartz
+    $self->{private_bz_dsn} = $dsn;
 
     bless ($self, $class);
 
diff --git a/Bugzilla/DB/Pg.pm b/Bugzilla/DB/Pg.pm
index 66ad4b1ec..18f9abf88 100644
--- a/Bugzilla/DB/Pg.pm
+++ b/Bugzilla/DB/Pg.pm
@@ -60,7 +60,7 @@ sub new {
     $dbname ||= 'template1';
 
     # construct the DSN from the parameters we got
-    my $dsn = "DBI:Pg:dbname=$dbname";
+    my $dsn = "dbi:Pg:dbname=$dbname";
     $dsn .= ";host=$host" if $host;
     $dsn .= ";port=$port" if $port;
 
@@ -75,6 +75,8 @@ sub new {
     # all class local variables stored in DBI derived class needs to have
     # a prefix 'private_'. See DBI documentation.
     $self->{private_bz_tables_locked} = "";
+    # Needed by TheSchwartz
+    $self->{private_bz_dsn} = $dsn;
 
     bless ($self, $class);
 
diff --git a/Bugzilla/DB/Schema.pm b/Bugzilla/DB/Schema.pm
index ed1245d98..f11c86e75 100644
--- a/Bugzilla/DB/Schema.pm
+++ b/Bugzilla/DB/Schema.pm
@@ -1388,6 +1388,93 @@ use constant ABSTRACT_SCHEMA => {
         ],
      },
 
+    # THESCHWARTZ TABLES
+    # ------------------
+    # Note: In the standard TheSchwartz schema, most integers are unsigned,
+    # but we didn't implement unsigned ints for Bugzilla schemas, so we
+    # just create signed ints, which should be fine.
+
+    ts_funcmap => {
+        FIELDS => [
+            funcid   => {TYPE => 'INTSERIAL', PRIMARYKEY => 1, NOTNULL => 1},
+            funcname => {TYPE => 'varchar(255)', NOTNULL => 1},
+        ],
+        INDEXES => [
+            ts_funcmap_funcname_idx => {FIELDS => ['funcname'], 
+                                          TYPE => 'UNIQUE'},
+        ],
+    },
+
+    ts_job => {
+        FIELDS => [
+            # In a standard TheSchwartz schema, this is a BIGINT, but we
+            # don't have those and I didn't want to add them just for this.
+            jobid         => {TYPE => 'INTSERIAL', PRIMARYKEY => 1, 
+                              NOTNULL => 1},
+            funcid        => {TYPE => 'INT4', NOTNULL => 1},
+            # In standard TheSchwartz, this is a MEDIUMBLOB.
+            arg           => {TYPE => 'LONGBLOB'},
+            uniqkey       => {TYPE => 'varchar(255)'},
+            insert_time   => {TYPE => 'INT4'},
+            run_after     => {TYPE => 'INT4', NOTNULL => 1},
+            grabbed_until => {TYPE => 'INT4', NOTNULL => 1},
+            priority      => {TYPE => 'INT2'},
+            coalesce      => {TYPE => 'varchar(255)'},
+        ],
+        INDEXES => [
+            ts_job_funcid_idx => {FIELDS => [qw(funcid uniqkey)],
+                                  TYPE   => 'UNIQUE'},
+            # In a standard TheSchewartz schema, these both go in the other
+            # direction, but there's no reason to have three indexes that
+            # all start with the same column, and our naming scheme doesn't
+            # allow it anyhow.
+            ts_job_run_after_idx => [qw(run_after funcid)],
+            ts_job_coalesce_idx  => [qw(coalesce funcid)],
+        ],
+    },
+
+    ts_note => {
+        FIELDS => [
+            # This is a BIGINT in standard TheSchwartz schemas.
+            jobid   => {TYPE => 'INT4', NOTNULL => 1},
+            notekey => {TYPE => 'varchar(255)'},
+            value   => {TYPE => 'LONGBLOB'},
+        ],
+        INDEXES => [
+            ts_note_jobid_idx => {FIELDS => [qw(jobid notekey)], 
+                                    TYPE => 'UNIQUE'},
+        ],
+    },
+
+    ts_error => {
+        FIELDS => [
+            error_time => {TYPE => 'INT4', NOTNULL => 1},
+            jobid      => {TYPE => 'INT4', NOTNULL => 1},
+            message    => {TYPE => 'varchar(255)', NOTNULL => 1},
+            funcid     => {TYPE => 'INT4', NOTNULL => 1, DEFAULT => 0},
+        ],
+        INDEXES => [
+            ts_error_funcid_idx     => [qw(funcid error_time)],
+            ts_error_error_time_idx => ['error_time'],
+            ts_error_jobid_idx      => ['jobid'],
+        ],
+    },
+
+    ts_exitstatus => {
+        FIELDS => [
+            jobid           => {TYPE => 'INTSERIAL', PRIMARYKEY => 1,
+                                NOTNULL => 1},
+            funcid          => {TYPE => 'INT4', NOTNULL => 1, DEFAULT => 0},
+            status          => {TYPE => 'INT2'},
+            completion_time => {TYPE => 'INT4'},
+            delete_after    => {TYPE => 'INT4'},
+        ],
+        INDEXES => [
+            ts_exitstatus_funcid_idx       => ['funcid'],
+            ts_exitstatus_delete_after_idx => ['delete_after'],
+        ],
+    },
+
     # SCHEMA STORAGE
     # --------------
 
diff --git a/Bugzilla/Install/Filesystem.pm b/Bugzilla/Install/Filesystem.pm
index 64783e301..8afaa3384 100644
--- a/Bugzilla/Install/Filesystem.pm
+++ b/Bugzilla/Install/Filesystem.pm
@@ -114,6 +114,7 @@ sub FILESYSTEM {
         'customfield.pl'  => { perms => $owner_executable },
         'email_in.pl'     => { perms => $ws_executable },
         'sanitycheck.pl'  => { perms => $ws_executable },
+        'jobqueue.pl'     => { perms => $owner_executable },
         'install-module.pl' => { perms => $owner_executable },
 
         'docs/makedocs.pl'   => { perms => $owner_executable },
diff --git a/Bugzilla/Install/Requirements.pm b/Bugzilla/Install/Requirements.pm
index 19d54af73..5456fc7d4 100644
--- a/Bugzilla/Install/Requirements.pm
+++ b/Bugzilla/Install/Requirements.pm
@@ -231,6 +231,20 @@ sub OPTIONAL_MODULES {
         feature => 'Inbound Email'
     },
 
+    # Mail Queueing
+    {
+        package => 'TheSchwartz',
+        module  => 'TheSchwartz',
+        version => 0,
+        feature => 'Mail Queueing',
+    },
+    {
+        package => 'Daemon-Generic',
+        module  => 'Daemon::Generic',
+        version => 0,
+        feature => 'Mail Queueing',
+    },
+
     # mod_perl
     {
         package => 'mod_perl',
diff --git a/Bugzilla/Job/Mailer.pm b/Bugzilla/Job/Mailer.pm
new file mode 100644
index 000000000..a421c59f2
--- /dev/null
+++ b/Bugzilla/Job/Mailer.pm
@@ -0,0 +1,56 @@
+# -*- Mode: perl; indent-tabs-mode: nil -*-
+#
+# The contents of this file are subject to the Mozilla Public
+# License Version 1.1 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of
+# the License at http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS
+# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# rights and limitations under the License.
+#
+# The Original Code is the Bugzilla Bug Tracking System.
+#
+# The Initial Developer of the Original Code is Mozilla Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2008
+# Mozilla Corporation. All Rights Reserved.
+#
+# Contributor(s): 
+#   Mark Smith <mark@mozilla.com>
+#   Max Kanat-Alexander <mkanat@bugzilla.org>
+
+package Bugzilla::Job::Mailer;
+use Bugzilla::Mailer;
+BEGIN { eval "use base qw(TheSchwartz::Worker)"; }
+
+# The longest we expect a job to possibly take, in seconds.
+use constant grab_for => 300;
+# We don't want email to fail permanently very easily. Retry for 30 days.
+use constant max_retries => 725;
+
+# The first few retries happen quickly, but after that we wait an hour for
+# each retry.
+sub retry_delay {
+    my $num_retries = shift;
+    if ($num_retries < 5) {
+        return (10, 30, 60, 300, 600)[$num_retries];
+    }
+    # One hour
+    return 60*60;
+}
+
+sub work {
+    my ($class, $job) = @_;
+    my $msg = $job->arg->{msg};
+    my $success = eval { MessageToMTA($msg, 1); 1; };
+    if (!$success) {
+        $job->failed($@);
+        undef $@;
+    } 
+    else {
+        $job->completed;
+    }
+}
+
+1;
diff --git a/Bugzilla/JobQueue.pm b/Bugzilla/JobQueue.pm
new file mode 100644
index 000000000..102f58bc6
--- /dev/null
+++ b/Bugzilla/JobQueue.pm
@@ -0,0 +1,108 @@
+# -*- Mode: perl; indent-tabs-mode: nil -*-
+#
+# The contents of this file are subject to the Mozilla Public
+# License Version 1.1 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of
+# the License at http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS
+# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# rights and limitations under the License.
+#
+# The Original Code is the Bugzilla Bug Tracking System.
+#
+# The Initial Developer of the Original Code is Mozilla Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2008
+# Mozilla Corporation. All Rights Reserved.
+#
+# Contributor(s): 
+#   Mark Smith <mark@mozilla.com>
+#   Max Kanat-Alexander <mkanat@bugzilla.org>
+
+package Bugzilla::JobQueue;
+
+use strict;
+
+use Bugzilla::Constants;
+use Bugzilla::Error;
+use Bugzilla::Install::Util qw(install_string);
+BEGIN { eval "use base qw(TheSchwartz)"; }
+
+# This maps job names for Bugzilla::JobQueue to the appropriate modules.
+# If you add new types of jobs, you should add a mapping here.
+use constant JOB_MAP => {
+    send_mail => 'Bugzilla::Job::Mailer',
+};
+
+sub new {
+    my $class = shift;
+
+    if (!eval { require TheSchwartz; }) {
+        ThrowCodeError('jobqueue_not_configured');
+    }
+
+    my $lc = Bugzilla->localconfig;
+    my $self = $class->SUPER::new(
+        databases => [{
+            dsn    => Bugzilla->dbh->{private_bz_dsn},
+            user   => $lc->{db_user},
+            pass   => $lc->{db_pass},
+            prefix => 'ts_',
+        }],
+    );
+
+    return $self;
+}
+
+# A way to get access to the underlying databases directly.
+sub bz_databases {
+    my $self = shift;
+    my @hashes = keys %{ $self->{databases} };
+    return map { $self->driver_for($_) } @hashes;
+}
+
+# inserts a job into the queue to be processed and returns immediately
+sub insert {
+    my $self = shift;
+    my $job = shift;
+
+    my $mapped_job = JOB_MAP->{$job};
+    ThrowCodeError('jobqueue_no_job_mapping', { job => $job })
+        if !$mapped_job;
+    unshift(@_, $mapped_job);
+
+    my $retval = $self->SUPER::insert(@_);
+    # XXX Need to get an error message here if insert fails, but
+    # I don't see any way to do that in TheSchwartz.
+    ThrowCodeError('jobqueue_insert_failed', { job => $job, errmsg => $@ })
+        if !$retval;
+ 
+    return $retval;
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+Bugzilla::JobQueue - Interface between Bugzilla and TheSchwartz.
+
+=head1 SYNOPSIS
+
+ use Bugzilla;
+
+ my $obj = Bugzilla->job_queue();
+ $obj->insert('send_mail', { msg => $message });
+
+=head1 DESCRIPTION
+
+Certain tasks should be done asyncronously.  The job queue system allows
+Bugzilla to use some sort of service to schedule jobs to happen asyncronously.
+
+=head2 Inserting a Job
+
+See the synopsis above for an easy to follow example on how to insert a
+job into the queue.  Give it a name and some arguments and the job will
+be sent away to be done later.
diff --git a/Bugzilla/JobQueue/Runner.pm b/Bugzilla/JobQueue/Runner.pm
new file mode 100644
index 000000000..35cfec5dd
--- /dev/null
+++ b/Bugzilla/JobQueue/Runner.pm
@@ -0,0 +1,97 @@
+# -*- Mode: perl; indent-tabs-mode: nil -*-
+#
+# The contents of this file are subject to the Mozilla Public
+# License Version 1.1 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of
+# the License at http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS
+# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# rights and limitations under the License.
+#
+# The Original Code is the Bugzilla Bug Tracking System.
+#
+# The Initial Developer of the Original Code is Mozilla Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2008
+# Mozilla Corporation. All Rights Reserved.
+#
+# Contributor(s): 
+#   Mark Smith <mark@mozilla.com>
+#   Max Kanat-Alexander <mkanat@bugzilla.org>
+
+# XXX In order to support Windows, we have to make gd_redirect_output
+# use Log4Perl or something instead of calling "logger". We probably
+# also need to use Win32::Daemon or something like that to daemonize.
+
+package Bugzilla::JobQueue::Runner;
+
+use strict;
+use File::Basename;
+use Pod::Usage;
+
+use Bugzilla::Constants;
+use Bugzilla::JobQueue;
+use Bugzilla::Util qw(get_text);
+BEGIN { eval "use base qw(Daemon::Generic)"; }
+
+# Required because of a bug in Daemon::Generic where it won't use the
+# "version" key from DAEMON_CONFIG.
+our $VERSION = BUGZILLA_VERSION;
+
+use constant DAEMON_CONFIG => (
+    progname => basename($0),
+    pidfile  => bz_locations()->{datadir} . '/' . basename($0) . '.pid',
+    version  => BUGZILLA_VERSION,
+);
+
+sub gd_preconfig {
+    return DAEMON_CONFIG;
+}
+
+sub gd_usage {
+    pod2usage({ -verbose => 0, -exitval => 'NOEXIT' });
+    return 0
+}
+
+sub gd_check {
+    my $self = shift;
+
+    # Get a count of all the jobs currently in the queue.
+    my $jq = Bugzilla->job_queue();
+    my @dbs = $jq->bz_databases();
+    my $count = 0;
+    foreach my $driver (@dbs) {
+        $count += $driver->select_one('SELECT COUNT(*) FROM ts_job', []);
+    }
+    print get_text('job_queue_depth', { count => $count }) . "\n";
+}
+
+sub gd_run {
+    my $self = shift;
+
+    my $jq = Bugzilla->job_queue();
+    $jq->set_verbose($self->{debug});
+    foreach my $module (values %{ Bugzilla::JobQueue::JOB_MAP() }) {
+        eval "use $module";
+        $jq->can_do($module);
+    }
+    $jq->work;
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+Bugzilla::JobQueue::Runner - A class representing the daemon that runs the
+job queue.
+
+ use Bugzilla::JobQueue::Runner;
+ Bugzilla::JobQueue::Runner->new();
+
+=head1 DESCRIPTION
+
+This is a subclass of L<Daemon::Generic> that is used by L<jobqueue>
+to run the Bugzilla job queue.
diff --git a/Bugzilla/Mailer.pm b/Bugzilla/Mailer.pm
index 1ffbd44e3..d3810b72b 100644
--- a/Bugzilla/Mailer.pm
+++ b/Bugzilla/Mailer.pm
@@ -53,10 +53,15 @@ use Email::MIME::Modifier;
 use Email::Send;
 
 sub MessageToMTA {
-    my ($msg) = (@_);
+    my ($msg, $send_now) = (@_);
     my $method = Bugzilla->params->{'mail_delivery_method'};
     return if $method eq 'None';
 
+    if (Bugzilla->params->{'use_mailer_queue'} and !$send_now) {
+        Bugzilla->job_queue->insert('send_mail', { msg => $msg });
+        return;
+    }
+
     my $email = ref($msg) ? $msg : Email::MIME->new($msg);
 
     # We add this header to uniquely identify all email that we