Bug 698345: report errors and warnings to arecibo

7 files changed, 327 insertions, 45 deletions

diff --git a/Bugzilla/Arecibo.pm b/Bugzilla/Arecibo.pm
new file mode 100644
index 000000000..c2fd94b44
--- /dev/null
+++ b/Bugzilla/Arecibo.pm
@@ -0,0 +1,265 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This Source Code Form is "Incompatible With Secondary Licenses", as
+# defined by the Mozilla Public License, v. 2.0.
+
+package Bugzilla::Arecibo;
+
+use strict;
+use warnings;
+
+use base qw(Exporter);
+our @EXPORT = qw(
+    arecibo_handle_error
+    arecibo_generate_id
+    arecibo_should_notify
+);
+
+use Apache2::Log;
+use Apache2::SubProcess;
+use Carp;
+use Email::Date::Format 'email_gmdate';
+use LWP::UserAgent;
+use POSIX 'setsid';
+use Sys::Hostname;
+
+use Bugzilla::Util;
+
+use constant CONFIG => {
+    # 'types' maps from the error message to types and priorities
+    types => [
+        {
+            type  => 'the_schwartz',
+            boost => -10,
+            match => [
+                qr/TheSchwartz\.pm/,
+            ],
+        },
+        {
+            type  => 'database_error',
+            boost => -10,
+            match => [
+                qr/DBD::mysql/,
+                qr/Can't connect to the database/,
+            ],
+        },
+        {
+            type  => 'patch_reader',
+            boost => +5,
+            match => [
+                qr#/PatchReader/#,
+            ],
+        },
+        {
+            type  => 'uninitialized_warning',
+            boost => 0,
+            match => [
+                qr/Use of uninitialized value/,
+            ],
+        },
+    ],
+
+    # 'codes' lists the code-errors which are sent to arecibo
+    codes => [qw(
+        bug_error
+        chart_datafile_corrupt
+        chart_dir_nonexistent
+        chart_file_open_fail
+        illegal_content_type_method
+        jobqueue_insert_failed
+        ldap_bind_failed
+        mail_send_error
+        template_error
+        token_generation_error
+    )],
+
+    # any error messages matching these regex's will not be sent to arecibo
+    ignore => [
+        qr/^Software caused connection abort/,
+    ],
+};
+
+sub arecibo_generate_id {
+    return sprintf("%s.%s", (time), $$);
+}
+
+sub arecibo_should_notify {
+    my $code_error = shift;
+    return grep { $_ eq $code_error } @{CONFIG->{codes}};
+}
+
+sub arecibo_handle_error {
+    my $class = shift;
+    my @message = split(/\n/, shift);
+    my $id = shift || arecibo_generate_id();
+
+    my $is_error = $class eq 'error';
+    if ($class ne 'error' && $class ne 'warning') {
+        # it's a code-error
+        return 0 unless arecibo_should_notify($class);
+        $is_error = 1;
+    }
+
+    # build traceback
+    my $traceback;
+    {
+        # for now don't show function arguments, in case they contain
+        # confidential data.  waiting on bug 700683
+        #local $Carp::MaxArgLen  = 256;
+        #local $Carp::MaxArgNums = 0;
+        local $Carp::MaxArgNums = -1;
+        local $Carp::CarpInternal{'CGI::Carp'} = 1;
+        local $Carp::CarpInternal{'Bugzilla::Error'}   = 1;
+        local $Carp::CarpInternal{'Bugzilla::Arecibo'} = 1;
+        $traceback = Carp::longmess();
+    }
+
+    # strip timestamp
+    foreach my $line (@message) {
+        $line =~ s/^\[[^\]]+\] //;
+    }
+    my $message = join(" ", map { trim($_) } grep { $_ ne '' } @message);
+
+    # don't send to arecibo unless configured
+    my $arecibo_server = Bugzilla->params->{arecibo_server};
+    my $send_to_arecibo = $arecibo_server ne '';
+    if ($send_to_arecibo) {
+        # message content filtering
+        foreach my $re (@{CONFIG->{ignore}}) {
+            if ($message =~ $re) {
+                $send_to_arecibo = 0;
+                last;
+            }
+        }
+    }
+
+    # log to apache's error_log
+    if ($send_to_arecibo) {
+        $message .= " [#$id]";
+    } else {
+        $traceback =~ s/\n/ /g;
+        $message .= " $traceback";
+    }
+    _write_to_error_log($message, $is_error);
+
+    return 0 unless $send_to_arecibo;
+
+    # set the error type and priority from the message content
+    $message = join("\n", grep { $_ ne '' } @message);
+    my $type = '';
+    my $priority = $class eq 'error' ? 3 : 10;
+    foreach my $rh_type (@{CONFIG->{types}}) {
+        foreach my $re (@{$rh_type->{match}}) {
+            if ($message =~ $re) {
+                $type = $rh_type->{type};
+                $priority += $rh_type->{boost};
+                last;
+            }
+        }
+        last if $type ne '';
+    }
+    $type ||= $class;
+    $priority = 1 if $priority < 1;
+    $priority = 10 if $priority > 10;
+
+    my $username = '';
+    eval { $username = Bugzilla->user->login };
+
+    my $data = [
+        msg        => $message,
+        priority   => $priority,
+        server     => hostname(),
+        status     => '500',
+        timestamp  => email_gmdate(),
+        traceback  => $traceback,
+        type       => $type,
+        uid        => $id,
+        url        => Bugzilla->cgi->self_url,
+        user_agent => $ENV{HTTP_USER_AGENT},
+        username   => $username,
+    ];
+
+    # fork then post
+    $SIG{CHLD} = 'IGNORE';
+    my $pid = fork();
+    if (defined($pid) && $pid == 0) {
+        # detach
+        chdir('/');
+        open(STDIN, '</dev/null');
+        open(STDOUT, '>/dev/null');
+        open(STDERR, '>/dev/null');
+        setsid();
+
+        # post to arecibo (ignore any errors)
+        my $agent = LWP::UserAgent->new(
+            agent   => 'bugzilla.mozilla.org',
+            timeout => 10, # seconds
+        );
+        $agent->post($arecibo_server, $data);
+
+        CORE::exit(0);
+    }
+    return 1;
+}
+
+sub _write_to_error_log {
+    my ($message, $is_error) = @_;
+    if ($ENV{MOD_PERL}) {
+        if ($is_error) {
+            Apache2::ServerRec::log_error($message);
+        } else {
+            Apache2::ServerRec::warn($message);
+        }
+    } else {
+        print STDERR "$message\n";
+    }
+}
+
+# lifted from Bugzilla::Error
+sub _in_eval {
+    my $in_eval = 0;
+    for (my $stack = 1; my $sub = (caller($stack))[3]; $stack++) {
+        last if $sub =~ /^ModPerl/;
+        $in_eval = 1 if $sub =~ /^\(eval\)/;
+    }
+    return $in_eval;
+}
+
+BEGIN {
+    require CGI::Carp;
+    CGI::Carp::set_die_handler(sub {
+        return if _in_eval();
+        my $message = shift;
+        my $is_compilation_failure = $message =~ /\bcompilation aborted\b/;
+        if (!$is_compilation_failure) {
+            eval { Bugzilla::Error::ThrowTemplateError($message) };
+        }
+        if ($is_compilation_failure || $@) {
+            print "Content-type: text/html\n\n";
+            my $uid = arecibo_generate_id();
+            my $notified = arecibo_handle_error('error', $message, $uid);
+            my $maintainer = html_quote(Bugzilla->params->{'maintainer'});
+            $message = html_quote($message);
+            $uid = html_quote($uid);
+            print qq(
+                <h1>Bugzilla has suffered an internal error</h1>
+                <pre>$message</pre>
+            );
+            if ($notified) {
+                print qq(
+                    The <a href="mailto:$maintainer">Bugzilla maintainers</a> have
+                    been notified of this error [#$uid].
+                );
+            };
+            exit;
+        }
+    });
+    $main::SIG{__WARN__} = sub {
+        return if _in_eval();
+        arecibo_handle_error('warning', shift);
+    };
+}
+
+1;
diff --git a/Bugzilla/Config/Advanced.pm b/Bugzilla/Config/Advanced.pm
index f5653ee86..b70dcb9e3 100644
--- a/Bugzilla/Config/Advanced.pm
+++ b/Bugzilla/Config/Advanced.pm
@@ -68,6 +68,12 @@ use constant get_param_list => (
    type => 'b',
    default => 0
   },
+
+  {
+   name => 'arecibo_server',
+   type => 't',
+   default => '',
+  },
 );
 
 1;
diff --git a/Bugzilla/DB.pm b/Bugzilla/DB.pm
index 0c841632f..2f708d065 100644
--- a/Bugzilla/DB.pm
+++ b/Bugzilla/DB.pm
@@ -159,7 +159,7 @@ sub _handle_error {
     # Cut down the error string to a reasonable size
     $_[0] = substr($_[0], 0, 2000) . ' ... ' . substr($_[0], -2000)
         if length($_[0]) > 4000;
-    $_[0] = Carp::longmess($_[0]);
+    # BMO: stracktrace disabled: $_[0] = Carp::longmess($_[0]);
     return 0; # Now let DBI handle raising the error
 }
 
diff --git a/Bugzilla/Error.pm b/Bugzilla/Error.pm
index 17a7a948a..c2fcfdd55 100644
--- a/Bugzilla/Error.pm
+++ b/Bugzilla/Error.pm
@@ -28,6 +28,7 @@ use base qw(Exporter);
 
 @Bugzilla::Error::EXPORT = qw(ThrowCodeError ThrowTemplateError ThrowUserError ThrowErrorPage);
 
+use Bugzilla::Arecibo;
 use Bugzilla::Constants;
 use Bugzilla::WebService::Constants;
 use Bugzilla::Util;
@@ -93,9 +94,22 @@ sub _throw_error {
 
     my $template = Bugzilla->template;
     if (Bugzilla->error_mode == ERROR_MODE_WEBPAGE) {
+        if (arecibo_should_notify($vars->{error})) {
+            $vars->{maintainers_notified} = 1;
+            $vars->{uid} = arecibo_generate_id();
+            $vars->{processed} = {};
+        } else {
+            $vars->{maintainers_notified} = 0;
+        }
+
         print Bugzilla->cgi->header();
         $template->process($name, $vars)
           || ThrowTemplateError($template->error());
+
+        if ($vars->{maintainers_notified}) {
+            arecibo_handle_error(
+                $vars->{error}, $vars->{processed}->{error_message}, $vars->{uid});
+        }
     }
     # There are some tests that throw and catch a lot of errors,
     # and calling $template->process over and over for those errors
@@ -181,31 +195,33 @@ sub ThrowTemplateError {
     $vars->{'template_error_msg'} = $template_err;
     $vars->{'error'} = "template_error";
 
+    $vars->{'uid'} = arecibo_generate_id();
+    arecibo_handle_error('error', $template_err, $vars->{'uid'});
+
     my $template = Bugzilla->template;
 
     # Try a template first; but if this one fails too, fall back
     # on plain old print statements.
     if (!$template->process("global/code-error.html.tmpl", $vars)) {
-        my $maintainer = Bugzilla->params->{'maintainer'};
+        my $maintainer = html_quote(Bugzilla->params->{'maintainer'});
         my $error = html_quote($vars->{'template_error_msg'});
         my $error2 = html_quote($template->error());
+        my $uid = html_quote($vars->{'uid'});
         print <<END;
         <tt>
           <p>
-            Bugzilla has suffered an internal error. Please save this page and 
-            send it to $maintainer with details of what you were doing at the 
-            time this message appeared.
+            Bugzilla has suffered an internal error:
+          </p>
+          <p>
+            $error
+          </p>
+          <!-- template error, no real need to show this to the user
+          $error2
+          -->
+          <p>
+            The <a href="mailto:$maintainer">Bugzilla maintainers</a> have
+            been notified of this error [#$uid].
           </p>
-          <script type="text/javascript"> <!--
-          document.write("<p>URL: " + 
-                          document.location.href.replace(/&/g,"&amp;")
-                                                .replace(/</g,"&lt;")
-                                                .replace(/>/g,"&gt;") + "</p>");
-          // -->
-          </script>
-          <p>Template->process() failed twice.<br>
-          First error: $error<br>
-          Second error: $error2</p>
         </tt>
 END
     }
diff --git a/skins/standard/global.css b/skins/standard/global.css
index e8336d5c1..537477296 100644
--- a/skins/standard/global.css
+++ b/skins/standard/global.css
@@ -380,7 +380,7 @@ input.requestee {
 }
 
 #error_msg {
-    font-size: x-large;
+    font-size: large;
 }
 
 .warning {
@@ -388,9 +388,9 @@ input.requestee {
 }
 
 .throw_error {
-    background-color: #ff0000;
+    background-color: #ff6666;
     color: black;
-    font-size: 120%;
+    font-size: large;
     margin: 1em;
     padding: 0.5em 1em;
 }
diff --git a/template/en/default/admin/params/advanced.html.tmpl b/template/en/default/admin/params/advanced.html.tmpl
index 0ba40b375..1cf0c344f 100644
--- a/template/en/default/admin/params/advanced.html.tmpl
+++ b/template/en/default/admin/params/advanced.html.tmpl
@@ -81,4 +81,9 @@
 
   disable_bug_updates =>
     "When enabled, all updates to $terms.bugs will be blocked.",
+
+  arecibo_server =>
+    "When set, important errors and warnings will be sent to the"
+    _ " specified Arecibo server. Enter the Arecibo server's full URL;"
+    _ " eg <code>https://arecibo.example.com/v/1/</code>.",
 } %]
diff --git a/template/en/default/global/code-error.html.tmpl b/template/en/default/global/code-error.html.tmpl
index f100df994..8a4252a7e 100644
--- a/template/en/default/global/code-error.html.tmpl
+++ b/template/en/default/global/code-error.html.tmpl
@@ -503,33 +503,23 @@
    admindocslinks = admindocslinks
 %]
 
-<tt>
-  <p>
-    [% terms.Bugzilla %] has suffered an internal error. Please save this page and send
-    it to [% Param("maintainer") %] with details of what you were doing at
-    the time this message appeared.
-  </p>
-  <script type="text/javascript"> <!--
-    document.write("<p>URL: " + 
-                    document.location.href.replace(/&/g,"&amp;")
-                                          .replace(/</g,"&lt;")
-                                          .replace(/>/g,"&gt;") + "</p>");
-  // -->
-  </script>
-</tt>
-
-<table cellpadding="20">
-  <tr>
-    <td bgcolor="#ff0000">
-      <font size="+2">
-        [% error_message FILTER none %]
-      </font>
-    </td>
-  </tr>
-</table>
-
-<p>Traceback:</p>
-<pre>[% traceback FILTER html %]</pre>
+[%# return the generated error_message for arecibo %]
+[% processed.error_message = error_message %]
+
+<p>
+  [% terms.Bugzilla %] has suffered an internal error:
+</p>
+
+<p class="throw_error">
+  [% error_message FILTER none %]
+</p>
+
+[% IF maintainers_notified %]
+<p>
+  The [% terms.Bugzilla %] maintainers have been notified of this error
+  [#[% uid FILTER html %]].
+</p>
+[% END %]
 
 [% IF variables %]
   <pre>