Merge branch 'master' into unstable

20 files changed, 438 insertions, 136 deletions

diff --git a/Bugzilla/BugMail.pm b/Bugzilla/BugMail.pm
index defe7c84f..915405a0e 100644
--- a/Bugzilla/BugMail.pm
+++ b/Bugzilla/BugMail.pm
@@ -421,7 +421,7 @@ sub sendMail {
         bugmailtype        => $bugmailtype,
     };
 
-    if (Bugzilla->params->{'use_mailer_queue'}) {
+    if (Bugzilla->get_param_with_override('use_mailer_queue')) {
         enqueue($vars);
     } else {
         MessageToMTA(_generate_bugmail($vars));
diff --git a/Bugzilla/CGI.pm b/Bugzilla/CGI.pm
index ba82c83d0..b932116a2 100644
--- a/Bugzilla/CGI.pm
+++ b/Bugzilla/CGI.pm
@@ -11,6 +11,7 @@ use 5.10.1;
 use strict;
 use warnings;
 
+use Bugzilla::Logging;
 use CGI;
 use base qw(CGI);
 
@@ -597,7 +598,20 @@ sub header {
 sub param {
     my $self = shift;
 
+    # We don't let CGI.pm warn about list context, but we do it ourselves.
     local $CGI::LIST_CONTEXT_WARN = 0;
+    if (0) {
+        state $has_warned = {};
+
+        ## no critic (Freenode::Wantarray)
+        if ( wantarray && @_ ) {
+            my ( $package, $filename, $line ) = caller;
+            if ( $package ne 'CGI' && ! $has_warned->{"$filename:$line"}++) {
+                WARN("Bugzilla::CGI::param called in list context from $package $filename:$line");
+            }
+        }
+        ## use critic
+    }
 
     # When we are just requesting the value of a parameter...
     if (scalar(@_) == 1) {
diff --git a/Bugzilla/Constants.pm b/Bugzilla/Constants.pm
index 2971c7a53..65b37dced 100644
--- a/Bugzilla/Constants.pm
+++ b/Bugzilla/Constants.pm
@@ -19,7 +19,6 @@ use Memoize;
 
 @Bugzilla::Constants::EXPORT = qw(
     BUGZILLA_VERSION
-    REST_DOC
 
     REMOTE_FILE
     LOCAL_FILE
@@ -211,10 +210,6 @@ sub BUGZILLA_VERSION {
     eval { Bugzilla->VERSION } || $bugzilla_version;
 }
 
-# A base link to the current REST Documentation. We place it here
-# as it will need to be updated to whatever the current release is.
-use constant REST_DOC => "https://bugzilla.readthedocs.io/en/latest/api/";
-
 # Location of the remote and local XML files to track new releases.
 use constant REMOTE_FILE => 'http://updates.bugzilla.org/bugzilla-update.xml';
 use constant LOCAL_FILE  => 'bugzilla-update.xml'; # Relative to datadir.
diff --git a/Bugzilla/DB.pm b/Bugzilla/DB.pm
index a2cff0bd4..15acfd0d9 100644
--- a/Bugzilla/DB.pm
+++ b/Bugzilla/DB.pm
@@ -109,8 +109,13 @@ sub connect_shadow {
     my $connect_params = dclone(Bugzilla->localconfig);
     $connect_params->{db_host} = Bugzilla->get_param_with_override('shadowdbhost');
     $connect_params->{db_name} = Bugzilla->get_param_with_override('shadowdb');
-    $connect_params->{db_port} = Bugzilla->get_param_with_override('shadowport');
-    $connect_params->{db_sock} = Bugzilla->get_param_with_override('shadowsock');
+    $connect_params->{db_port} = Bugzilla->get_param_with_override('shadowdbport');
+    $connect_params->{db_sock} = Bugzilla->get_param_with_override('shadowdbsock');
+
+    if ( Bugzilla->localconfig->{'shadowdb_user'} && Bugzilla->localconfig->{'shadowdb_pass'} ) {
+        $connect_params->{db_user} = Bugzilla->localconfig->{'shadowdb_user'};
+        $connect_params->{db_pass} = Bugzilla->localconfig->{'shadowdb_pass'};
+    }
 
     return _connect($connect_params);
 }
diff --git a/Bugzilla/DaemonControl.pm b/Bugzilla/DaemonControl.pm
index b7f7bcbe9..6586cc01b 100644
--- a/Bugzilla/DaemonControl.pm
+++ b/Bugzilla/DaemonControl.pm
@@ -28,7 +28,8 @@ use POSIX qw(setsid WEXITSTATUS);
 use base qw(Exporter);
 
 our @EXPORT_OK = qw(
-    run_httpd run_cereal run_cereal_and_httpd
+    run_httpd run_cereal run_jobqueue
+    run_cereal_and_httpd run_cereal_and_jobqueue
     catch_signal on_finish on_exception
     assert_httpd assert_database assert_selenium
 );
@@ -39,10 +40,12 @@ our %EXPORT_TAGS = (
     utils => [qw(catch_signal on_exception on_finish)],
 );
 
-use constant CEREAL_BIN    => realpath(catfile( bz_locations->{cgi_path}, 'scripts', 'cereal.pl'));
-
-use constant HTTPD_BIN     => '/usr/sbin/httpd';
-use constant HTTPD_CONFIG  => realpath(catfile( bz_locations->{confdir}, 'httpd.conf' ));
+use constant {
+    JOBQUEUE_BIN => realpath( catfile( bz_locations->{cgi_path}, 'jobqueue.pl' ) ),
+    CEREAL_BIN   => realpath( catfile( bz_locations->{cgi_path}, 'scripts', 'cereal.pl' ) ),
+    HTTPD_BIN    => '/usr/sbin/httpd',
+    HTTPD_CONFIG => realpath( catfile( bz_locations->{confdir}, 'httpd.conf' ) ),
+};
 
 sub catch_signal {
     my ($name, @done)   = @_;
@@ -75,7 +78,7 @@ sub run_cereal {
     my $cereal = IO::Async::Process->new(
         command      => [CEREAL_BIN],
         on_finish    => on_finish($exit_f),
-        on_exception => on_exception( "cereal", $exit_f ),
+        on_exception => on_exception( 'cereal', $exit_f ),
     );
     $exit_f->on_cancel( sub { $cereal->kill('TERM') } );
     $loop->add($cereal);
@@ -85,15 +88,18 @@ sub run_cereal {
 
 sub run_httpd {
     my (@args) = @_;
-    my $loop = IO::Async::Loop->new;
 
+    my $loop   = IO::Async::Loop->new;
     my $exit_f = $loop->new_future;
     my $httpd  = IO::Async::Process->new(
         code => sub {
+
             # we have to setsid() to make a new process group
             # or else apache will kill its parent.
             setsid();
-            exec HTTPD_BIN, '-DFOREGROUND', '-f' => HTTPD_CONFIG, @args;
+            my @command = ( HTTPD_BIN, '-DFOREGROUND', '-f' => HTTPD_CONFIG, @args );
+            exec @command
+              or die "failed to exec $command[0] $!";
         },
         on_finish    => on_finish($exit_f),
         on_exception => on_exception( 'httpd', $exit_f ),
@@ -104,21 +110,52 @@ sub run_httpd {
     return $exit_f;
 }
 
+sub run_jobqueue {
+    my (@args) = @_;
+
+    my $loop     = IO::Async::Loop->new;
+    my $exit_f   = $loop->new_future;
+    my $jobqueue = IO::Async::Process->new(
+        command   => [ JOBQUEUE_BIN, 'start', '-f', '-d', @args ],
+        on_finish => on_finish($exit_f),
+        on_exception => on_exception( 'httpd', $exit_f ),
+    );
+    $exit_f->on_cancel( sub { $jobqueue->kill('TERM') } );
+    $loop->add($jobqueue);
+
+    return $exit_f;
+}
+
+sub run_cereal_and_jobqueue {
+    my (@jobqueue_args) = @_;
+
+    my $signal_f      = catch_signal('TERM', 0);
+    my $cereal_exit_f = run_cereal();
+
+    return assert_cereal()->then(
+        sub {
+            my $jobqueue_exit_f = run_jobqueue(@jobqueue_args);
+            return Future->wait_any($cereal_exit_f, $jobqueue_exit_f, $signal_f);
+        }
+    );
+}
+
 sub run_cereal_and_httpd {
     my @httpd_args = @_;
 
-    push @httpd_args, '-DNETCAT_LOGS';
-    my $signal_f      = catch_signal("TERM", 0);
+    my $signal_f      = catch_signal('TERM', 0);
     my $cereal_exit_f = run_cereal();
 
     return assert_cereal()->then(
         sub {
+            push @httpd_args, '-DNETCAT_LOGS';
+
             my $lc = Bugzilla::Install::Localconfig::read_localconfig();
             if ( ($lc->{inbound_proxies} // '') eq '*' && $lc->{urlbase} =~ /^https/) {
                 push @httpd_args, '-DHTTPS';
             }
-            elsif (not $lc->{urlbase} =~ /^https/) {
-                WARN("HTTPS urlbase but inbound_proxies is not '*'");
+            elsif ($lc->{urlbase} =~ /^https/) {
+                WARN('HTTPS urlbase but inbound_proxies is not "*"');
             }
             my $httpd_exit_f  = run_httpd(@httpd_args);
 
@@ -140,24 +177,23 @@ sub assert_httpd {
         my $f = shift;
         ( $f->get =~ /^httpd OK/ );
     };
-    my $timeout = $loop->timeout_future(after => 20)->else_fail("assert_httpd timeout");
+    my $timeout = $loop->timeout_future(after => 20)->else_fail('assert_httpd timeout');
     return Future->wait_any($repeat, $timeout);
 }
 
-
 sub assert_selenium {
     my ($host, $port) = @_;
     $host //= 'localhost';
     $port //= 4444;
 
-    return assert_connect($host, $port, "assert_selenium");
+    return assert_connect($host, $port, 'assert_selenium');
 }
 
 sub assert_cereal {
     return assert_connect(
         'localhost',
         $ENV{LOGGING_PORT} // 5880,
-        "assert_cereal"
+        'assert_cereal'
     );
 }
 
@@ -199,7 +235,7 @@ sub assert_database {
         );
     } until => sub { defined shift->get };
 
-    my $timeout = $loop->timeout_future( after => 20 )->else_fail("assert_database timeout");
+    my $timeout = $loop->timeout_future( after => 20 )->else_fail('assert_database timeout');
     my $any_f = Future->wait_any( $repeat, $timeout );
     return $any_f->transform(
         done => sub { return },
diff --git a/Bugzilla/Error.pm b/Bugzilla/Error.pm
index e7a99dba0..d67571848 100644
--- a/Bugzilla/Error.pm
+++ b/Bugzilla/Error.pm
@@ -38,15 +38,14 @@ sub _in_eval {
 
 sub _throw_error {
     my ($name, $error, $vars) = @_;
-    my $dbh = Bugzilla->dbh;
     $vars ||= {};
-
     $vars->{error} = $error;
 
     # Make sure any transaction is rolled back (if supported).
     # If we are within an eval(), do not roll back transactions as we are
     # eval'uating some test on purpose.
-    $dbh->bz_rollback_transaction() if ($dbh->bz_in_transaction() && !_in_eval());
+    my $dbh = eval { Bugzilla->dbh };
+    $dbh->bz_rollback_transaction() if ($dbh && $dbh->bz_in_transaction() && !_in_eval());
 
     my $datadir = bz_locations()->{'datadir'};
     # If a writable $datadir/errorlog exists, log error details there.
@@ -191,10 +190,9 @@ sub ThrowCodeError {
 
 sub ThrowTemplateError {
     my ($template_err) = @_;
-    my $dbh = Bugzilla->dbh;
-
+    my $dbh = eval { Bugzilla->dbh };
     # Make sure the transaction is rolled back (if supported).
-    $dbh->bz_rollback_transaction() if $dbh->bz_in_transaction();
+    $dbh->bz_rollback_transaction() if $dbh && $dbh->bz_in_transaction();
 
     if (blessed($template_err) && $template_err->isa('Template::Exception')) {
         my $type = $template_err->type;
diff --git a/Bugzilla/Install/Filesystem.pm b/Bugzilla/Install/Filesystem.pm
index 08b824cad..5e51dd9cc 100644
--- a/Bugzilla/Install/Filesystem.pm
+++ b/Bugzilla/Install/Filesystem.pm
@@ -271,6 +271,7 @@ sub FILESYSTEM {
         'metrics.pl'      => { perms => WS_EXECUTE },
         'Makefile.PL'     => { perms => OWNER_EXECUTE },
         'gen-cpanfile.pl' => { perms => OWNER_EXECUTE },
+        'jobqueue-worker.pl' => { perms => OWNER_EXECUTE },
         'clean-bug-user-last-visit.pl' => { perms => WS_EXECUTE },
 
         'Bugzilla.pm'    => { perms => CGI_READ },
diff --git a/Bugzilla/Install/Localconfig.pm b/Bugzilla/Install/Localconfig.pm
index ba8e8dc57..7a913358c 100644
--- a/Bugzilla/Install/Localconfig.pm
+++ b/Bugzilla/Install/Localconfig.pm
@@ -43,7 +43,7 @@ our @EXPORT_OK = qw(
 
 # might want to change this for upstream
 use constant ENV_PREFIX     => 'BMO_';
-use constant PARAM_OVERRIDE => qw( shadowdb shadowdbhost shadowdbport shadowdbsock );
+use constant PARAM_OVERRIDE => qw( use_mailer_queue mail_delivery_method shadowdb shadowdbhost shadowdbport shadowdbsock );
 
 sub _sensible_group {
     return '' if ON_WINDOWS;
@@ -135,12 +135,12 @@ use constant LOCALCONFIG_VARS => (
     {
         name    => 'param_override',
         default => {
-            memcached_servers   => undef,
-            memcached_namespace => undef,
-            shadowdb            => undef,
-            shadowdbhost        => undef,
-            shadowdbport        => undef,
-            shadowdbsock        => undef,
+            use_mailer_queue     => undef,
+            mail_delivery_method => undef,
+            shadowdb             => undef,
+            shadowdbhost         => undef,
+            shadowdbport         => undef,
+            shadowdbsock         => undef,
         },
     },
     {
@@ -175,6 +175,14 @@ use constant LOCALCONFIG_VARS => (
         name => 'inbound_proxies',
         default => _migrate_param( 'inbound_proxies', '' ),
     },
+    {
+        name => 'shadowdb_user',
+        default => '',
+    },
+    {
+        name => 'shadowdb_pass',
+        default => '',
+    }
 );
 
 
diff --git a/Bugzilla/JobQueue.pm b/Bugzilla/JobQueue.pm
index 55d40bfb8..53b088c6e 100644
--- a/Bugzilla/JobQueue.pm
+++ b/Bugzilla/JobQueue.pm
@@ -11,9 +11,14 @@ use 5.10.1;
 use strict;
 use warnings;
 
+use Bugzilla::Logging;
 use Bugzilla::Constants;
 use Bugzilla::Error;
 use Bugzilla::Install::Util qw(install_string);
+use Bugzilla::DaemonControl qw(catch_signal);
+use IO::Async::Timer::Periodic;
+use IO::Async::Loop;
+use Future;
 use base qw(TheSchwartz);
 
 # This maps job names for Bugzilla::JobQueue to the appropriate modules.
@@ -91,6 +96,32 @@ sub insert {
     return $retval;
 }
 
+sub debug {
+    my ($self, @args) = @_;
+    my $caller_pkg = caller;
+    local $Log::Log4perl::caller_depth = $Log::Log4perl::caller_depth + 1;
+    my $logger = Log::Log4perl->get_logger($caller_pkg);
+    $logger->info(@args);
+}
+
+sub work {
+    my ($self, $delay) = @_;
+    $delay ||= 1;
+    my $loop  = IO::Async::Loop->new;
+    my $timer = IO::Async::Timer::Periodic->new(
+        first_interval => 0,
+        interval       => $delay,
+        reschedule     => 'drift',
+        on_tick        => sub { $self->work_once }
+    );
+    DEBUG("working every $delay seconds");
+    $loop->add($timer);
+    $timer->start;
+    Future->wait_any(map { catch_signal($_) } qw( INT TERM HUP ))->get;
+    $timer->stop;
+    $loop->remove($timer);
+}
+
 # Clear the request cache at the start of each run.
 sub work_once {
     my $self = shift;
diff --git a/Bugzilla/JobQueue/Runner.pm b/Bugzilla/JobQueue/Runner.pm
index 5b3164ef9..0177de40a 100644
--- a/Bugzilla/JobQueue/Runner.pm
+++ b/Bugzilla/JobQueue/Runner.pm
@@ -14,23 +14,34 @@ package Bugzilla::JobQueue::Runner;
 use 5.10.1;
 use strict;
 use warnings;
+use autodie qw(open close unlink system);
 
+use Bugzilla::Logging;
+use Bugzilla::Constants;
+use Bugzilla::DaemonControl qw(:utils);
+use Bugzilla::JobQueue::Worker;
+use Bugzilla::JobQueue;
+use Bugzilla::Util qw(get_text);
 use Cwd qw(abs_path);
+use English qw(-no_match_vars $PROGRAM_NAME $EXECUTABLE_NAME);
 use File::Basename;
 use File::Copy;
+use File::Spec::Functions qw(catfile tmpdir);
+use Future;
+use Future::Utils qw(fmap_void);
+use IO::Async::Loop;
+use IO::Async::Process;
+use IO::Async::Signal;
 use Pod::Usage;
 
-use Bugzilla::Constants;
-use Bugzilla::JobQueue;
-use Bugzilla::Util qw(get_text);
-BEGIN { eval "use base qw(Daemon::Generic)"; }
+use parent qw(Daemon::Generic);
 
-our $VERSION = BUGZILLA_VERSION;
+our $VERSION = 2;
 
 # Info we need to install/uninstall the daemon.
-our $chkconfig = "/sbin/chkconfig";
-our $initd = "/etc/init.d";
-our $initscript = "bugzilla-queue";
+our $chkconfig  = '/sbin/chkconfig';
+our $initd      = '/etc/init.d';
+our $initscript = 'bugzilla-queue';
 
 # The Daemon::Generic docs say that it uses all sorts of
 # things from gd_preconfig, but in fact it does not. The
@@ -40,11 +51,10 @@ sub gd_preconfig {
     my $self = shift;
 
     my $pidfile = $self->{gd_args}{pidfile};
-    if (!$pidfile) {
-        $pidfile = bz_locations()->{datadir} . '/' . $self->{gd_progname}
-                   . ".pid";
+    if ( !$pidfile ) {
+        $pidfile = catfile(tmpdir(),  $self->{gd_progname} . '.pid');
     }
-    return (pidfile => $pidfile);
+    return ( pidfile => $pidfile );
 }
 
 # All config other than the pidfile has to be done in gd_getopt
@@ -54,24 +64,30 @@ sub gd_getopt {
 
     $self->SUPER::gd_getopt();
 
-    if ($self->{gd_args}{progname}) {
+    if ( $self->{gd_args}{progname} ) {
         $self->{gd_progname} = $self->{gd_args}{progname};
     }
     else {
-        $self->{gd_progname} = basename($0);
+        $self->{gd_progname} = basename($PROGRAM_NAME);
     }
 
-    # There are places that Daemon Generic's new() uses $0 instead of
+    # There are places that Daemon Generic's new() uses $PROGRAM_NAME instead of
     # gd_progname, which it really shouldn't, but this hack fixes it.
-    $self->{_original_zero} = $0;
-    $0 = $self->{gd_progname};
+    $self->{_original_program_name} = $PROGRAM_NAME;
+
+    ## no critic (Variables::RequireLocalizedPunctuationVars)
+    $PROGRAM_NAME = $self->{gd_progname};
+    ## use critic
 }
 
 sub gd_postconfig {
     my $self = shift;
+
     # See the hack above in gd_getopt. This just reverses it
     # in case anything else needs the accurate $0.
-    $0 = delete $self->{_original_zero};
+    ## no critic (Variables::RequireLocalizedPunctuationVars)
+    $PROGRAM_NAME = delete $self->{_original_program_name};
+    ## use critic
 }
 
 sub gd_more_opt {
@@ -79,12 +95,13 @@ sub gd_more_opt {
     return (
         'pidfile=s' => \$self->{gd_args}{pidfile},
         'n=s'       => \$self->{gd_args}{progname},
+        'jobs|j=i'  => \$self->{gd_args}{jobs},
     );
 }
 
 sub gd_usage {
-    pod2usage({ -verbose => 0, -exitval => 'NOEXIT' });
-    return 0
+    pod2usage( { -verbose => 0, -exitval => 'NOEXIT' } );
+    return 0;
 }
 
 sub gd_can_install {
@@ -95,66 +112,63 @@ sub gd_can_install {
     my $sysconfig   = '/etc/sysconfig';
     my $config_file = "$sysconfig/$initscript";
 
-    if (!-x $chkconfig  or !-d $initd) {
+    if ( !-x $chkconfig || !-d $initd ) {
         return $self->SUPER::gd_can_install(@_);
     }
 
     return sub {
-        if (!-w $initd) {
+        if ( !-w $initd ) {
             print "You must run the 'install' command as root.\n";
             return;
         }
-        if (-e $dest_file) {
+        if ( -e $dest_file ) {
             print "$initscript already in $initd.\n";
         }
         else {
-            copy($source_file, $dest_file)
+            copy( $source_file, $dest_file )
                 or die "Could not copy $source_file to $dest_file: $!";
-            chmod(0755, $dest_file)
+            chmod 0755, $dest_file
                 or die "Could not change permissions on $dest_file: $!";
         }
 
-        system($chkconfig, '--add', $initscript);
-        print "$initscript installed.",
-              " To start the daemon, do \"$dest_file start\" as root.\n";
+        system $chkconfig, '--add', $initscript;
+        print "$initscript installed.", " To start the daemon, do \"$dest_file start\" as root.\n";
 
-        if (-d $sysconfig and -w $sysconfig) {
-            if (-e $config_file) {
+        if ( -d $sysconfig and -w $sysconfig ) {
+            if ( -e $config_file ) {
                 print "$config_file already exists.\n";
                 return;
             }
 
-            open(my $config_fh, ">", $config_file)
-                or die "Could not write to $config_file: $!";
-            my $directory = abs_path(dirname($self->{_original_zero}));
-            my $owner_id = (stat $self->{_original_zero})[4];
-            my $owner = getpwuid($owner_id);
-            print $config_fh <<END;
+            open my $config_fh, '>', $config_file;
+            my $directory = abs_path( dirname( $self->{_original_program_name} ) );
+            my $owner_id  = ( stat $self->{_original_program_name} )[4];
+            my $owner     = getpwuid $owner_id;
+            print $config_fh <<"END";
 #!/bin/sh
 BUGZILLA="$directory"
 USER=$owner
 END
-            close($config_fh);
+            close $config_fh;
         }
         else {
             print "Please edit $dest_file to configure the daemon.\n";
         }
-    }
+        }
 }
 
 sub gd_can_uninstall {
     my $self = shift;
 
-    if (-x $chkconfig and -d $initd) {
+    if ( -x $chkconfig and -d $initd ) {
         return sub {
-            if (!-e "$initd/$initscript") {
+            if ( !-e "$initd/$initscript" ) {
                 print "$initscript not installed.\n";
                 return;
             }
-            system($chkconfig, '--del', $initscript);
-            print "$initscript disabled.",
-                  " To stop it, run: $initd/$initscript stop\n";
-        }
+            system $chkconfig, '--del', $initscript;
+            print "$initscript disabled.", " To stop it, run: $initd/$initscript stop\n";
+            }
     }
 
     return $self->SUPER::gd_can_install(@_);
@@ -164,49 +178,76 @@ sub gd_check {
     my $self = shift;
 
     # Get a count of all the jobs currently in the queue.
-    my $jq = Bugzilla->job_queue();
-    my @dbs = $jq->bz_databases();
+    my $jq    = Bugzilla->job_queue();
+    my @dbs   = $jq->bz_databases();
     my $count = 0;
     foreach my $driver (@dbs) {
-        $count += $driver->select_one('SELECT COUNT(*) FROM ts_job', []);
+        $count += $driver->select_one( 'SELECT COUNT(*) FROM ts_job', [] );
     }
-    print get_text('job_queue_depth', { count => $count }) . "\n";
+    print get_text( 'job_queue_depth', { count => $count } ) . "\n";
 }
 
+# override this to use IO::Async.
 sub gd_setup_signals {
-    my $self = shift;
-    $self->SUPER::gd_setup_signals();
-    $SIG{TERM} = sub { $self->gd_quit_event(); }
+    my $self    = shift;
+    my @signals = qw( INT HUP TERM );
+    $self->{_signal_future} = Future->wait_any( map { catch_signal( $_, $_ ) } @signals );
 }
 
 sub gd_other_cmd {
     my ($self) = shift;
-    if ($ARGV[0] eq "once") {
-        $self->_do_work("work_once");
-
-        exit(0);
+    if ( $ARGV[0] eq 'once' ) {
+        Bugzilla::JobQueue::Worker->run('work_once');
+        exit;
     }
-    
+
     $self->SUPER::gd_other_cmd();
 }
 
-sub gd_run {
-    my $self = shift;
+sub gd_quit_event     { FATAL('gd_quit_event() should never be called') }
+sub gd_reconfig_event { FATAL('gd_reconfig_event() should never be called') }
 
-    $self->_do_work("work");
+sub gd_run {
+    my $self      = shift;
+    my $jobs      = $self->{gd_args}{jobs} // 1;
+    my $signal_f  = $self->{_signal_future};
+    my $workers_f = fmap_void { $self->run_worker() }
+        concurrent => $jobs,
+        generate   => sub { !$signal_f->is_ready };
+
+    # This is so the process shows up in (h)top in a useful way.
+    local $PROGRAM_NAME = "$self->{gd_progname} [supervisor]";
+    Future->wait_any($signal_f, $workers_f)->get;
+    unlink $self->{gd_pidfile};
+    exit 0;
 }
 
-sub _do_work {
-    my ($self, $fn) = @_;
+# This executes the script "jobqueue-worker.pl"
+# $EXECUTABLE_NAME is the name of the perl interpreter.
+sub run_worker {
+    my ( $self ) = @_;
 
-    my $jq = Bugzilla->job_queue();
-    $jq->set_verbose($self->{debug});
-    foreach my $module (values %{ Bugzilla::JobQueue->job_map() }) {
-        eval "use $module";
-        $jq->can_do($module);
+    my $script = catfile( bz_locations->{cgi_path}, 'jobqueue-worker.pl' );
+    my @command = ( $EXECUTABLE_NAME, $script);
+    if ( $self->{gd_args}{progname} ) {
+        push @command, '--name' => "$self->{gd_args}{progname} [worker]";
     }
 
-    $jq->$fn;
+    my $loop   = IO::Async::Loop->new;
+    my $exit_f = $loop->new_future;
+    my $worker = IO::Async::Process->new(
+        command      => \@command,
+        on_finish    => on_finish($exit_f),
+        on_exception => on_exception( 'jobqueue worker', $exit_f )
+    );
+    $exit_f->on_cancel(
+        sub {
+            DEBUG('terminate worker');
+            $worker->kill('TERM');
+        }
+    );
+    $loop->add($worker);
+    return $exit_f;
 }
 
 1;
diff --git a/Bugzilla/JobQueue/Worker.pm b/Bugzilla/JobQueue/Worker.pm
new file mode 100644
index 000000000..db8ebe35e
--- /dev/null
+++ b/Bugzilla/JobQueue/Worker.pm
@@ -0,0 +1,30 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This Source Code Form is "Incompatible With Secondary Licenses", as
+# defined by the Mozilla Public License, v. 2.0.
+
+package Bugzilla::JobQueue::Worker;
+use 5.10.1;
+use strict;
+use warnings;
+
+use Bugzilla::Logging;
+use Module::Runtime qw(require_module);
+
+sub run {
+    my ( $class, $fn ) = @_;
+    DEBUG("Starting up for $fn");
+    my $jq = Bugzilla->job_queue();
+
+    DEBUG('Loading jobqueue modules');
+    foreach my $module ( values %{ Bugzilla::JobQueue->job_map() } ) {
+        DEBUG("JobQueue can do $module");
+        require_module($module);
+        $jq->can_do($module);
+    }
+    $jq->$fn;
+}
+
+1;
diff --git a/Bugzilla/Mailer.pm b/Bugzilla/Mailer.pm
index 6e46d1862..1dec3d4ff 100644
--- a/Bugzilla/Mailer.pm
+++ b/Bugzilla/Mailer.pm
@@ -37,10 +37,10 @@ use Bugzilla::Version qw(vers_cmp);
 
 sub MessageToMTA {
     my ($msg, $send_now) = (@_);
-    my $method = Bugzilla->params->{'mail_delivery_method'};
+    my $method = Bugzilla->get_param_with_override('mail_delivery_method');
     return if $method eq 'None';
 
-    if (Bugzilla->params->{'use_mailer_queue'} and !$send_now) {
+    if (Bugzilla->get_param_with_override('use_mailer_queue') and !$send_now) {
         Bugzilla->job_queue->insert('send_mail', { msg => $msg });
         return;
     }
@@ -66,7 +66,7 @@ sub MessageToMTA {
     }
 
     # Ensure that we are not sending emails too quickly to recipients.
-    if (Bugzilla->params->{use_mailer_queue}
+    if (Bugzilla->get_param_with_override('use_mailer_queue')
         && (EMAIL_LIMIT_PER_MINUTE || EMAIL_LIMIT_PER_HOUR))
     {
         $dbh->do(
@@ -226,7 +226,7 @@ sub MessageToMTA {
     }
 
     # insert into email_rates
-    if (Bugzilla->params->{use_mailer_queue}
+    if (Bugzilla->get_param_with_override('use_mailer_queue')
         && (EMAIL_LIMIT_PER_MINUTE || EMAIL_LIMIT_PER_HOUR))
     {
         $dbh->do(
@@ -252,15 +252,14 @@ sub build_thread_marker {
         $sitespec = "-$2$sitespec"; # Put the port number back in, before the '@'
     }
 
-    my $threadingmarker;
+    my $threadingmarker = "References: <bug-$bug_id-$user_id$sitespec>";
     if ($is_new) {
-        $threadingmarker = "Message-ID: <bug-$bug_id-$user_id$sitespec>";
+        $threadingmarker .= "\nMessage-ID: <bug-$bug_id-$user_id$sitespec>";
     }
     else {
         my $rand_bits = generate_random_password(10);
-        $threadingmarker = "Message-ID: <bug-$bug_id-$user_id-$rand_bits$sitespec>" .
-                           "\nIn-Reply-To: <bug-$bug_id-$user_id$sitespec>" .
-                           "\nReferences: <bug-$bug_id-$user_id$sitespec>";
+        $threadingmarker .= "\nMessage-ID: <bug-$bug_id-$user_id-$rand_bits$sitespec>" .
+                            "\nIn-Reply-To: <bug-$bug_id-$user_id$sitespec>";
     }
 
     return $threadingmarker;
diff --git a/Bugzilla/Memcached.pm b/Bugzilla/Memcached.pm
index 0ceed97c0..d34aaa595 100644
--- a/Bugzilla/Memcached.pm
+++ b/Bugzilla/Memcached.pm
@@ -16,7 +16,7 @@ use Log::Log4perl qw(:easy);
 use Bugzilla::Error;
 use Scalar::Util qw(blessed);
 use List::Util qw(sum);
-use Bugzilla::Util qw(trick_taint);
+use Bugzilla::Util qw(trick_taint trim);
 use URI::Escape;
 use Encode;
 use Sys::Syslog qw(:DEFAULT);
@@ -36,11 +36,25 @@ sub _new {
     if (Bugzilla->feature('memcached') && $servers) {
         $self->{namespace} = Bugzilla->localconfig->{memcached_namespace};
         TRACE("connecting servers: $servers, namespace: $self->{namespace}");
-        $self->{memcached} = Cache::Memcached::Fast->new({
-            servers   => [ split(/[, ]+/, $servers) ],
-            namespace => $self->{namespace},
-            max_size  => 1024 * 1024 * 4,
-        });
+        $self->{memcached} = Cache::Memcached::Fast->new(
+            {
+                servers         => [ _parse_memcached_server_list($servers) ],
+                namespace       => $self->{namespace},
+                max_size        => 1024 * 1024 * 4,
+                max_failures    => 1,
+                failure_timeout => 60,
+                io_timeout      => 0.2,
+                connect_timeout => 0.2,
+            }
+        );
+        my $versions = $self->{memcached}->server_versions;
+        if (keys %$versions) {
+            # this is needed to ensure forked processes don't start out with a connected memcached socket.
+            $self->{memcached}->disconnect_all;
+        }
+        else {
+            WARN("No memcached servers");
+        }
     }
     else {
         TRACE("memcached feature is not enabled");
@@ -48,6 +62,13 @@ sub _new {
     return bless($self, $class);
 }
 
+sub _parse_memcached_server_list {
+    my ($server_list) = @_;
+    my @servers = split(/[, ]+/, trim($server_list));
+
+    return map { /:[0-9]+$/s ? $_ : "$_:11211" } @servers;
+}
+
 sub enabled {
     return $_[0]->{memcached} ? 1 : 0;
 }
@@ -206,6 +227,8 @@ sub should_rate_limit {
     my $prefix    = RATE_LIMIT_PREFIX . $name . ':';
     my $memcached = $self->{memcached};
 
+    return 0 unless $memcached;
+
     $tries //= 3;
 
     for (0 .. $tries) {
@@ -272,7 +295,7 @@ sub _inc_prefix {
     delete Bugzilla->request_cache->{"memcached_prefix_$name"};
 
     # BMO - log that we've wiped the cache
-    INFO("$name cache cleared");
+    TRACE("$name cache cleared");
 }
 
 sub _global_prefix {
@@ -315,7 +338,7 @@ sub _get {
 
     my $enc_key = $self->_encode_key($key)
         or return;
-    my $val = $self->{memcached}->get($key);
+    my $val = $self->{memcached}->get($enc_key);
     TRACE("get $enc_key: " . (defined $val ? "HIT" : "MISS"));
     return $val;
 }
diff --git a/Bugzilla/ModPerl.pm b/Bugzilla/ModPerl.pm
index a5c840897..120dd8210 100644
--- a/Bugzilla/ModPerl.pm
+++ b/Bugzilla/ModPerl.pm
@@ -20,6 +20,7 @@ use Carp ();
 use Template ();
 
 use Bugzilla::ModPerl::BlockIP;
+use Bugzilla::ModPerl::Hostage;
 
 sub apache_config {
     my ($class, $cgi_path) = @_;
@@ -74,6 +75,7 @@ __DATA__
 # the built-in rand(), even though we never use it in Bugzilla itself,
 # so we need to srand() both of them.)
 PerlChildInitHandler "sub { Bugzilla::RNG::srand(); srand(); }"
+PerlInitHandler Bugzilla::ModPerl::Hostage
 PerlAccessHandler Bugzilla::ModPerl::BlockIP
 
 # It is important to specify ErrorDocuments outside of all directories.
@@ -84,6 +86,12 @@ ErrorDocument 403 /errors/403.html
 ErrorDocument 404 /errors/404.html
 ErrorDocument 500 /errors/500.html
 
+<Location /helper>
+    SetHandler perl-script
+    PerlResponseHandler Plack::Handler::Apache2
+    PerlSetVar psgi_app [% cgi_path %]/helper.psgi
+</Location>
+
 <Directory "[% cgi_path %]">
     AddHandler perl-script .cgi
     # No need to PerlModule these because they're already defined in mod_perl.pl
diff --git a/Bugzilla/ModPerl/BasicAuth.pm b/Bugzilla/ModPerl/BasicAuth.pm
index e93680e9d..7248a19f3 100644
--- a/Bugzilla/ModPerl/BasicAuth.pm
+++ b/Bugzilla/ModPerl/BasicAuth.pm
@@ -25,18 +25,22 @@ use warnings;
 # AUTH_VAR_NAME and AUTH_VAR_PASS are the names of variables defined in
 # `localconfig` which hold the authentication credentials.
 
-use Apache2::Const -compile => qw(OK HTTP_UNAUTHORIZED);
+use Apache2::Const -compile => qw(OK HTTP_UNAUTHORIZED); ## no critic (Freenode::ModPerl)
+use Bugzilla::Logging;
 use Bugzilla ();
 
 sub handler {
     my $r = shift;
     my ($status, $password) = $r->get_basic_auth_pw;
-    return $status if $status != Apache2::Const::OK;
+    if ($status != Apache2::Const::OK) {
+        WARN("Got non-OK status: $status when trying to get password");
+        return $status
+    }
 
     my $auth_var_name = $ENV{AUTH_VAR_NAME};
     my $auth_var_pass = $ENV{AUTH_VAR_PASS};
     unless ($auth_var_name && $auth_var_pass) {
-        warn "AUTH_VAR_NAME and AUTH_VAR_PASS environmental vars not set\n";
+        ERROR('AUTH_VAR_NAME and AUTH_VAR_PASS environmental vars not set');
         $r->note_basic_auth_failure;
         return Apache2::Const::HTTP_UNAUTHORIZED;
     }
@@ -44,13 +48,14 @@ sub handler {
     my $auth_user = Bugzilla->localconfig->{$auth_var_name};
     my $auth_pass = Bugzilla->localconfig->{$auth_var_pass};
     unless ($auth_user && $auth_pass) {
-        warn "$auth_var_name and $auth_var_pass not configured\n";
+        ERROR("$auth_var_name and $auth_var_pass not configured");
         $r->note_basic_auth_failure;
         return Apache2::Const::HTTP_UNAUTHORIZED;
     }
 
     unless ($r->user eq $auth_user && $password eq $auth_pass) {
         $r->note_basic_auth_failure;
+        WARN('username and password do not match');
         return Apache2::Const::HTTP_UNAUTHORIZED;
     }
 
diff --git a/Bugzilla/ModPerl/Hostage.pm b/Bugzilla/ModPerl/Hostage.pm
new file mode 100644
index 000000000..a3bdfac58
--- /dev/null
+++ b/Bugzilla/ModPerl/Hostage.pm
@@ -0,0 +1,71 @@
+package Bugzilla::ModPerl::Hostage;
+use 5.10.1;
+use strict;
+use warnings;
+
+use Apache2::Const qw(:common); ## no critic (Freenode::ModPerl)
+
+sub _attachment_root {
+    my ($base) = @_;
+    return undef unless $base;
+    return $base =~ m{^https?://(?:bug)?\%bugid\%\.([a-zA-Z\.-]+)}
+        ? $1
+        : undef;
+}
+
+sub _attachment_host_regex {
+    my ($base) = @_;
+    return undef unless $base;
+    my $val   = $base;
+    $val =~ s{^https?://}{}s;
+    $val =~ s{/$}{}s;
+    my $regex = quotemeta $val;
+    $regex =~ s/\\\%bugid\\\%/\\d+/g;
+    return qr/^$regex$/s;
+}
+
+sub handler {
+    my $r = shift;
+    state $urlbase               = Bugzilla->localconfig->{urlbase};
+    state $urlbase_uri           = URI->new($urlbase);
+    state $urlbase_host          = $urlbase_uri->host;
+    state $urlbase_host_regex    = qr/^bug(\d+)\.\Q$urlbase_host\E$/;
+    state $attachment_base       = Bugzilla->localconfig->{attachment_base};
+    state $attachment_root       = _attachment_root($attachment_base);
+    state $attachment_host_regex = _attachment_host_regex($attachment_base);
+
+    my $hostname  = $r->hostname;
+    return OK if $hostname eq $urlbase_host;
+
+    my $path = $r->uri;
+    return OK if $path eq '/__lbheartbeat__';
+
+    if ($attachment_base && $hostname eq $attachment_root) {
+        $r->headers_out->set(Location => $urlbase);
+        return REDIRECT;
+    }
+    elsif ($attachment_base && $hostname =~ $attachment_host_regex) {
+        if ($path =~ m{^/attachment\.cgi}s) {
+            return OK;
+        } else {
+            my $new_uri = URI->new($r->unparsed_uri);
+            $new_uri->scheme($urlbase_uri->scheme);
+            $new_uri->host($urlbase_host);
+            $r->headers_out->set(Location => $new_uri);
+            return REDIRECT;
+        }
+    }
+    elsif (my ($id) = $hostname =~ $urlbase_host_regex) {
+        my $new_uri = $urlbase_uri->clone;
+        $new_uri->path('/show_bug.cgi');
+        $new_uri->query_form(id => $id);
+        $r->headers_out->set(Location => $new_uri);
+        return REDIRECT;
+    }
+    else {
+        $r->headers_out->set(Location => $urlbase);
+        return REDIRECT;
+    }
+}
+
+1;
\ No newline at end of file
diff --git a/Bugzilla/Send/Sendmail.pm b/Bugzilla/Send/Sendmail.pm
index 71c1f67ce..81c2190e5 100644
--- a/Bugzilla/Send/Sendmail.pm
+++ b/Bugzilla/Send/Sendmail.pm
@@ -37,7 +37,7 @@ sub send {
     unless (close $pipe) {
         return failure "error when closing pipe to $mailer: $!" if $!;
         my ($error_message, $is_transient) = _map_exitcode($? >> 8);
-        if (Bugzilla->params->{'use_mailer_queue'}) {
+        if (Bugzilla->get_param_with_override('use_mailer_queue')) {
             # Return success for errors which are fatal so Bugzilla knows to
             # remove them from the queue
             if ($is_transient) {
diff --git a/Bugzilla/Template.pm b/Bugzilla/Template.pm
index 2ec813303..3ace60cf8 100644
--- a/Bugzilla/Template.pm
+++ b/Bugzilla/Template.pm
@@ -284,7 +284,8 @@ sub get_attachment_link {
 
         $link_text =~ s/ \[details\]$//;
         $link_text =~ s/ \[diff\]$//;
-        my $linkval = "attachment.cgi?id=$attachid";
+        state $urlbase = Bugzilla->localconfig->{urlbase};
+        my $linkval = "${urlbase}attachment.cgi?id=$attachid";
 
         # If the attachment is a patch and patch_viewer feature is
         # enabled, add link to the diff.
@@ -572,7 +573,9 @@ sub create {
         ABSOLUTE => 1,
         RELATIVE => 1,
 
-        COMPILE_DIR => bz_locations()->{'template_cache'},
+        # Only use an on-disk template cache if we're running as the web
+        # server.  This ensures the permissions of the cache remain correct.
+        COMPILE_DIR => is_webserver_group() ? bz_locations()->{'template_cache'} : undef,
 
         # Don't check for a template update until 1 hour has passed since the
         # last check.
@@ -1071,6 +1074,8 @@ our %_templates_to_precompile;
 sub precompile_templates {
     my ($output) = @_;
 
+    return unless is_webserver_group();
+
     # Remove the compiled templates.
     my $cache_dir = bz_locations()->{'template_cache'};
     my $datadir = bz_locations()->{'datadir'};
diff --git a/Bugzilla/Util.pm b/Bugzilla/Util.pm
index 7d85a4dfd..a1316c7ef 100644
--- a/Bugzilla/Util.pm
+++ b/Bugzilla/Util.pm
@@ -17,7 +17,8 @@ use base qw(Exporter);
                              with_writable_database with_readonly_database
                              html_quote url_quote xml_quote
                              css_class_quote html_light_quote
-                             i_am_cgi i_am_webservice correct_urlbase remote_ip
+                             i_am_cgi i_am_webservice is_webserver_group
+                             correct_urlbase remote_ip
                              validate_ip do_ssl_redirect_if_required use_attachbase
                              diff_arrays on_main_db css_url_rewrite
                              trim wrap_hard wrap_comment find_wrap_point
@@ -32,19 +33,20 @@ use base qw(Exporter);
 use Bugzilla::Constants;
 use Bugzilla::RNG qw(irand);
 
-use Date::Parse;
 use Date::Format;
-use DateTime;
+use Date::Parse;
 use DateTime::TimeZone;
+use DateTime;
 use Digest;
 use Email::Address;
-use List::MoreUtils qw(none);
-use Scalar::Util qw(tainted blessed);
-use Text::Wrap;
 use Encode qw(encode decode resolve_alias);
 use Encode::Guess;
+use English qw(-no_match_vars $EGID);
+use List::MoreUtils qw(any none);
 use POSIX qw(floor ceil);
+use Scalar::Util qw(tainted blessed);
 use Taint::Util qw(untaint);
+use Text::Wrap;
 use Try::Tiny;
 
 sub with_writable_database(&) {
@@ -280,6 +282,30 @@ sub i_am_webservice {
            || $usage_mode == USAGE_MODE_REST;
 }
 
+sub is_webserver_group {
+    my @effective_gids = split(/ /, $EGID);
+
+    state $web_server_gid;
+    if (!defined $web_server_gid) {
+        my $web_server_group = Bugzilla->localconfig->{webservergroup};
+
+        if ($web_server_group eq '' || ON_WINDOWS) {
+            $web_server_gid = $effective_gids[0];
+        }
+
+        elsif ($web_server_group =~ /^\d+$/) {
+            $web_server_gid = $web_server_group;
+        }
+
+        else {
+            $web_server_gid = eval { getgrnam($web_server_group) };
+            $web_server_gid //= 0;
+        }
+    }
+
+    return any { $web_server_gid == $_ } @effective_gids;
+}
+
 # This exists as a separate function from Bugzilla::CGI::redirect_to_https
 # because we don't want to create a CGI object during XML-RPC calls
 # (doing so can mess up XML-RPC).
@@ -1071,6 +1097,11 @@ in a command-line script.
 Tells you whether or not the current usage mode is WebServices related
 such as JSONRPC or XMLRPC.
 
+=item C<is_webserver_group()>
+
+Tells you whether or not the current process's group matches that
+configured as webservergroup.
+
 =item C<remote_ip()>
 
 Returns the IP address of the remote client. If Bugzilla is behind
diff --git a/Bugzilla/WebService/Server/REST.pm b/Bugzilla/WebService/Server/REST.pm
index 6fb86fdd4..b8884b753 100644
--- a/Bugzilla/WebService/Server/REST.pm
+++ b/Bugzilla/WebService/Server/REST.pm
@@ -132,7 +132,8 @@ sub response {
     if (exists $json_data->{error}) {
         $result = $json_data->{error};
         $result->{error} = $self->type('boolean', 1);
-        $result->{documentation} = REST_DOC;
+
+        $result->{documentation} = Bugzilla->params->{docs_urlbase} . "api/";
         delete $result->{'name'}; # Remove JSONRPCError
     }
     elsif (exists $json_data->{result}) {