Bug 1480878 - Monitor the health of Push connector job processing

4 files changed, 80 insertions, 23 deletions

diff --git a/Bugzilla.pm b/Bugzilla.pm
index 5e48d21f4..f26819d93 100644
--- a/Bugzilla.pm
+++ b/Bugzilla.pm
@@ -780,6 +780,27 @@ sub memcached {
     return request_cache->{memcached} ||= Bugzilla::Memcached->_new();
 }
 
+# Connector to the Datadog metrics collection daemon.
+sub datadog {
+    my ($class, $namespace) = @_;
+    my $host      = $class->localconfig->{datadog_host};
+    my $port      = $class->localconfig->{datadog_port};
+
+    $namespace //= '';
+
+    if ($class->has_feature('datadog') && $host) {
+        require DataDog::DogStatsd;
+        return request_cache->{datadog}{$namespace} //= DataDog::DogStatsd->new(
+            host      => $host,
+            port      => $port,
+            namespace => $namespace ? "$namespace." : '',
+        );
+    }
+    else {
+        return undef;
+    }
+}
+
 sub elastic {
     my ($class) = @_;
     $class->process_cache->{elastic} //= Bugzilla::Elastic->new();
diff --git a/Bugzilla/Install/Localconfig.pm b/Bugzilla/Install/Localconfig.pm
index e1a8e0909..e524535ac 100644
--- a/Bugzilla/Install/Localconfig.pm
+++ b/Bugzilla/Install/Localconfig.pm
@@ -186,7 +186,15 @@ use constant LOCALCONFIG_VARS => (
     {
         name => 'shadowdb_pass',
         default => '',
-    }
+    },
+    {
+        name => 'datadog_host',
+        default => '',
+    },
+    {
+        name => 'datadog_port',
+        default => 8125,
+    },
 );
 
 
diff --git a/extensions/Push/lib/Push.pm b/extensions/Push/lib/Push.pm
index 670b2aa56..ab640da81 100644
--- a/extensions/Push/lib/Push.pm
+++ b/extensions/Push/lib/Push.pm
@@ -8,8 +8,7 @@
 package Bugzilla::Extension::Push::Push;
 
 use 5.10.1;
-use strict;
-use warnings;
+use Moo;
 
 use Bugzilla::Logging;
 use Bugzilla::Extension::Push::BacklogMessage;
@@ -23,22 +22,12 @@ use Bugzilla::Extension::Push::Option;
 use Bugzilla::Extension::Push::Queue;
 use Bugzilla::Extension::Push::Util;
 use DateTime;
+use Try::Tiny;
 
-sub new {
-    my ($class) = @_;
-    my $self = {};
-    bless($self, $class);
-    $self->{is_daemon} = 0;
-    return $self;
-}
-
-sub is_daemon {
-    my ($self, $value) = @_;
-    if (defined $value) {
-        $self->{is_daemon} = $value ? 1 : 0;
-    }
-    return $self->{is_daemon};
-}
+has 'is_daemon' => (
+    is      => 'rw',
+    default => 0,
+);
 
 sub start {
     my ($self) = @_;
@@ -50,12 +39,49 @@ sub start {
         $connector->backlog->reset_backoff();
     }
 
-    while(1) {
-        if ($self->_dbh_check()) {
-            $self->_reload();
-            $self->push();
+    my $pushd_loop = IO::Async::Loop->new;
+    my $main_timer = IO::Async::Timer::Periodic->new(
+        first_interval => 0,
+        interval       => POLL_INTERVAL_SECONDS,
+        reschedule     => 'drift',
+        on_tick        => sub {
+            if ( $self->_dbh_check() ) {
+                $self->_reload();
+                try {
+                    $self->push();
+                }
+                catch {
+                    FATAL($_);
+                };
+            }
+        },
+    );
+    if ( Bugzilla->datadog ) {
+        my $dog_timer = IO::Async::Timer::Periodic->new(
+            interval   => 120,
+            reschedule => 'drift',
+            on_tick    => sub { $self->heartbeat },
+        );
+        $pushd_loop->add($dog_timer);
+        $dog_timer->start;
+    }
+
+    $pushd_loop->add($main_timer);
+    $main_timer->start;
+    $pushd_loop->run;
+}
+
+sub heartbeat {
+    my ($self) = @_;
+    my $dd = Bugzilla->datadog('bugzilla.pushd');
+
+    $dd->gauge('scheduled_jobs', Bugzilla->dbh->selectrow_array('SELECT COUNT(*) FROM push'));
+
+    foreach my $connector ($self->connectors->list) {
+        if ($connector->enabled) {
+            my $lcname = lc $connector->name;
+            $dd->gauge("${lcname}.backlog", Bugzilla->dbh->selectrow_array('SELECT COUNT(*) FROM push_backlog WHERE connector = ?', undef, $connector->name));
         }
-        sleep(POLL_INTERVAL_SECONDS);
     }
 }
 
diff --git a/template/en/default/setup/strings.txt.pl b/template/en/default/setup/strings.txt.pl
index 363a2d5fd..adb79884a 100644
--- a/template/en/default/setup/strings.txt.pl
+++ b/template/en/default/setup/strings.txt.pl
@@ -138,6 +138,8 @@ END
 If you want to use the CVS integration of the Patch Viewer, please specify
 the full path to the "cvs" executable here.
 END
+    localconfig_datadog_host => 'hostname of datadog stats daemon',
+    localconfig_datadog_port => 'port of datadog stats daemon, defaults to 8125',
     localconfig_db_check => <<'END',
 Should checksetup.pl try to verify that your database setup is correct?
 With some combinations of database servers/Perl modules/moonphase this