Bug 1438206 - Process SES email bounces properly

8 files changed, 343 insertions, 1 deletions

diff --git a/Bugzilla/Install/Localconfig.pm b/Bugzilla/Install/Localconfig.pm
index f877829c5..646dbc1a7 100644
--- a/Bugzilla/Install/Localconfig.pm
+++ b/Bugzilla/Install/Localconfig.pm
@@ -163,7 +163,15 @@ use constant LOCALCONFIG_VARS => (
     {
         name    => 'attachment_base',
         default => _migrate_param( "attachment_base", '' ),
-    }
+    },
+    {
+        name    => 'ses_username',
+        default => '',
+    },
+    {
+        name    => 'ses_password',
+        default => '',
+    },
 );
 
 
diff --git a/Bugzilla/ModPerl.pm b/Bugzilla/ModPerl.pm
index 142df63d4..a5c840897 100644
--- a/Bugzilla/ModPerl.pm
+++ b/Bugzilla/ModPerl.pm
@@ -97,6 +97,16 @@ ErrorDocument 500 /errors/500.html
     [% root_htaccess.contents FILTER indent %]
 </Directory>
 
+# AWS SES endpoint for handling mail bounces/complaints
+<Location "/ses">
+    PerlSetEnv AUTH_VAR_NAME ses_username
+    PerlSetEnv AUTH_VAR_PASS ses_password
+    PerlAuthenHandler Bugzilla::ModPerl::BasicAuth
+    AuthName SES
+    AuthType Basic
+    require valid-user
+</Location>
+
 # directory rules for all the other places we have .htaccess files
 [% FOREACH htaccess IN htaccess_files %]
 # from [% htaccess.file %]
diff --git a/Bugzilla/ModPerl/BasicAuth.pm b/Bugzilla/ModPerl/BasicAuth.pm
new file mode 100644
index 000000000..e93680e9d
--- /dev/null
+++ b/Bugzilla/ModPerl/BasicAuth.pm
@@ -0,0 +1,60 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This Source Code Form is "Incompatible With Secondary Licenses", as
+# defined by the Mozilla Public License, v. 2.0.
+package Bugzilla::ModPerl::BasicAuth;
+use 5.10.1;
+use strict;
+use warnings;
+
+# Protects a mod_perl <Location> with Basic HTTP authentication.
+#
+# Example use:
+#
+# <Location "/ses">
+#   PerlAuthenHandler Bugzilla::ModPerl::BasicAuth
+#   PerlSetEnv AUTH_VAR_NAME ses_username
+#   PerlSetEnv AUTH_VAR_PASS ses_password
+#   AuthName SES
+#   AuthType Basic
+#   require valid-user
+# </Location>
+#
+# AUTH_VAR_NAME and AUTH_VAR_PASS are the names of variables defined in
+# `localconfig` which hold the authentication credentials.
+
+use Apache2::Const -compile => qw(OK HTTP_UNAUTHORIZED);
+use Bugzilla ();
+
+sub handler {
+    my $r = shift;
+    my ($status, $password) = $r->get_basic_auth_pw;
+    return $status if $status != Apache2::Const::OK;
+
+    my $auth_var_name = $ENV{AUTH_VAR_NAME};
+    my $auth_var_pass = $ENV{AUTH_VAR_PASS};
+    unless ($auth_var_name && $auth_var_pass) {
+        warn "AUTH_VAR_NAME and AUTH_VAR_PASS environmental vars not set\n";
+        $r->note_basic_auth_failure;
+        return Apache2::Const::HTTP_UNAUTHORIZED;
+    }
+
+    my $auth_user = Bugzilla->localconfig->{$auth_var_name};
+    my $auth_pass = Bugzilla->localconfig->{$auth_var_pass};
+    unless ($auth_user && $auth_pass) {
+        warn "$auth_var_name and $auth_var_pass not configured\n";
+        $r->note_basic_auth_failure;
+        return Apache2::Const::HTTP_UNAUTHORIZED;
+    }
+
+    unless ($r->user eq $auth_user && $password eq $auth_pass) {
+        $r->note_basic_auth_failure;
+        return Apache2::Const::HTTP_UNAUTHORIZED;
+    }
+
+    return Apache2::Const::OK;
+}
+
+1;
diff --git a/ses/index.cgi b/ses/index.cgi
new file mode 100755
index 000000000..aa5b34704
--- /dev/null
+++ b/ses/index.cgi
@@ -0,0 +1,206 @@
+#!/usr/bin/perl
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This Source Code Form is "Incompatible With Secondary Licenses", as
+# defined by the Mozilla Public License, v. 2.0.
+
+use 5.10.1;
+use strict;
+use warnings;
+
+use lib qw(.. ../lib ../local/lib/perl5);
+
+use Bugzilla ();
+use Bugzilla::Constants qw( ERROR_MODE_DIE );
+use Bugzilla::Mailer qw( MessageToMTA );
+use Bugzilla::User ();
+use Bugzilla::Util qw( html_quote remote_ip );
+use JSON::XS qw( decode_json encode_json );
+use LWP::UserAgent ();
+use Try::Tiny qw( try catch );
+
+Bugzilla->error_mode(ERROR_MODE_DIE);
+try {
+    main();
+} catch {
+    warn "SES: Fatal error: $_\n";
+    respond(500 => 'Internal Server Error');
+};
+
+sub main {
+    my $message = decode_json_wrapper(Bugzilla->cgi->param('POSTDATA')) // return;
+    my $message_type = $ENV{HTTP_X_AMZ_SNS_MESSAGE_TYPE} // '(missing)';
+
+    if ($message_type eq 'SubscriptionConfirmation') {
+        confirm_subscription($message);
+    }
+
+    elsif ($message_type eq 'Notification') {
+        my $notification = decode_json_wrapper($message->{Message}) // return;
+
+        my $notification_type = $notification->{notificationType} // '';
+        if ($notification_type eq 'Bounce') {
+            process_bounce($notification);
+        }
+        elsif ($notification_type eq 'Complaint') {
+            process_complaint($notification);
+        }
+        else {
+            warn "SES: Unsupported notification-type: $notification_type\n";
+            respond(200 => 'OK');
+        }
+    }
+
+    else {
+        warn "SES: Unsupported message-type: $message_type\n";
+        respond(200 => 'OK');
+    }
+}
+
+sub confirm_subscription {
+    my ($message) = @_;
+
+    my $subscribe_url = $message->{SubscribeURL};
+    if (!$subscribe_url) {
+        warn "SES: Bad SubscriptionConfirmation request: missing SubscribeURL\n";
+        respond(400 => 'Bad Request');
+        return;
+    }
+
+    my $ua = ua();
+    my $res = $ua->get($message->{SubscribeURL});
+    if (!$res->is_success) {
+        warn "SES: Bad response from SubscribeURL: " . $res->status_line . "\n";
+        respond(400 => 'Bad Request');
+        return;
+    }
+
+    respond(200 => 'OK');
+}
+
+sub process_bounce {
+    my ($notification) = @_;
+    my $type = $notification->{bounce}->{bounceType};
+
+    # these should be infrequent and hopefully small
+    warn("SES: notification: " . encode_json($notification));
+
+    if ($type eq 'Transient') {
+        # just log transient bounces
+        foreach my $recipient (@{ $notification->{bounce}->{bouncedRecipients} }) {
+            my $address = $recipient->{emailAddress};
+            Bugzilla->audit("SES: transient bounce for <$address>");
+        }
+    }
+
+    elsif ($type eq 'Permanent') {
+        # disable each account that is permanently bouncing
+        foreach my $recipient (@{ $notification->{bounce}->{bouncedRecipients} }) {
+            my $address = $recipient->{emailAddress};
+            my $reason = sprintf('(%s) %s', $recipient->{action} // 'error',
+                                            $recipient->{diagnosticCode} // 'unknown');
+
+            my $user = Bugzilla::User->new({ name => $address, cache => 1 });
+            if ($user) {
+                # never auto-disable admin accounts
+                if ($user->in_group('admin')) {
+                    Bugzilla->audit("SES: ignoring permanent bounce for admin <$address>: $reason");
+                }
+
+                else {
+                    my $template = Bugzilla->template_inner();
+                    my $vars = {
+                        mta    => $notification->{bounce}->{reportingMTA} // 'unknown',
+                        reason => $reason,
+                    };
+                    my $disable_text;
+                    $template->process('admin/users/bounce-disabled.txt.tmpl', $vars, \$disable_text)
+                        || die $template->error();
+
+					$user->set_disabledtext($disable_text);
+					$user->set_disable_mail(1);
+					$user->update();
+                    Bugzilla->audit("SES: permanent bounce for <$address> disabled userid-" . $user->id . ": $reason");
+                }
+            }
+
+            else {
+                Bugzilla->audit("SES: permanent bounce for <$address> has no user: $reason");
+            }
+        }
+    }
+
+    else {
+        warn "SES: Unsupported bounce type: $type\n";
+    }
+
+    respond(200 => 'OK');
+}
+
+sub process_complaint {
+    # email notification to bugzilla admin
+    my ($notification) = @_;
+    my $template = Bugzilla->template_inner();
+    my $json = JSON::XS->new->pretty->utf8->canonical;
+
+    foreach my $recipient (@{ $notification->{complaint}->{complainedRecipients} }) {
+        my $reason  = $notification->{complaint}->{complaintFeedbackType} // 'unknown';
+        my $address = $recipient->{emailAddress};
+        Bugzilla->audit("SES: complaint for <$address> for '$reason'");
+        my $vars = {
+            email        => $address,
+            user         => Bugzilla::User->new({ name => $address, cache => 1 }),
+            reason       => $reason,
+            notification => $json->encode($notification),
+        };
+        my $message;
+        $template->process('email/ses-complaint.txt.tmpl', $vars, \$message)
+            || die $template->error();
+        MessageToMTA($message);
+    }
+
+    respond(200 => 'OK');
+}
+
+sub respond {
+    my ($code, $message) = @_;
+    print Bugzilla->cgi->header(
+        -status => "$code $message",
+    );
+    # apache will generate non-200 response pages for us
+    say html_quote($message) if $code == 200;
+}
+
+sub decode_json_wrapper {
+    my ($json) = @_;
+    my $result;
+    if (!defined $json) {
+        warn 'SES: Missing JSON from ' . remote_ip() . "\n";
+        respond(400 => 'Bad Request');
+        return undef;
+    }
+    my $ok = try {
+        $result = decode_json($json);
+    }
+    catch {
+        warn 'SES: Malformed JSON from ' . remote_ip() . "\n";
+        respond(400 => 'Bad Request');
+        return undef;
+    };
+    return $ok ? $result : undef;
+}
+
+sub ua {
+    my $ua = LWP::UserAgent->new();
+    $ua->timeout(10);
+    $ua->protocols_allowed(['http', 'https']);
+    if (my $proxy_url = Bugzilla->params->{'proxy_url'}) {
+        $ua->proxy(['http', 'https'], $proxy_url);
+    }
+    else {
+        $ua->env_proxy;
+    }
+    return $ua;
+}
diff --git a/template/en/default/admin/users/bounce-disabled.txt.tmpl b/template/en/default/admin/users/bounce-disabled.txt.tmpl
new file mode 100644
index 000000000..f4ae6a361
--- /dev/null
+++ b/template/en/default/admin/users/bounce-disabled.txt.tmpl
@@ -0,0 +1,19 @@
+[%# This Source Code Form is subject to the terms of the Mozilla Public
+  # License, v. 2.0. If a copy of the MPL was not distributed with this
+  # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+  #
+  # This Source Code Form is "Incompatible With Secondary Licenses", as
+  # defined by the Mozilla Public License, v. 2.0.
+  #%]
+
+[%# INTERFACE:
+  # mta: mail server reporting error
+  # reason: Reason for bounce (diagnostic code)
+  #%]
+
+[% PROCESS global/variables.none.tmpl %]
+
+Your [% terms.Bugzilla %] account has been disabled due to issues delivering
+emails to your address.<br>
+<br>
+Your mail server ([% mta FILTER html %]) said: [% reason FILTER html %]<br>
diff --git a/template/en/default/email/ses-complaint.txt.tmpl b/template/en/default/email/ses-complaint.txt.tmpl
new file mode 100644
index 000000000..93ad5eff4
--- /dev/null
+++ b/template/en/default/email/ses-complaint.txt.tmpl
@@ -0,0 +1,31 @@
+[%# This Source Code Form is subject to the terms of the Mozilla Public
+  # License, v. 2.0. If a copy of the MPL was not distributed with this
+  # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+  #
+  # This Source Code Form is "Incompatible With Secondary Licenses", as
+  # defined by the Mozilla Public License, v. 2.0.
+  #%]
+
+[%# INTERFACE:
+  # email: email address the complaint pertains to
+  # user: Bugzilla::User object associated with email (may be undef)
+  # reason: Reason for complaint
+  # notification: notification from SES (JSON)
+  #%]
+
+[% PROCESS global/variables.none.tmpl %]
+
+From: [% Param('mailfrom') %]
+To: [% Param('maintainer') %]
+Subject: [% terms.Bugzilla %]: SES Complaint: [% email %]: [% reason %]
+X-Bugzilla-Type: admin
+
+SES Complaint received for [% email %]: [% reason %]
+
+[% IF user %]
+[% urlbase %]/editusers.cgi?action=edit&userid=[% user.id %]
+[% ELSE %]
+Failed to find corresponding user in Bugzilla.
+[% END %]
+
+[%+ notification %]
diff --git a/template/en/default/setup/strings.txt.pl b/template/en/default/setup/strings.txt.pl
index 35a771ff3..ce4785b04 100644
--- a/template/en/default/setup/strings.txt.pl
+++ b/template/en/default/setup/strings.txt.pl
@@ -204,6 +204,12 @@ END
     localconfig_memcached_namespace => <<'END',
 Specify a string to prefix each key on Memcached.
 END
+    localconfig_ses_username => <<'END',
+Username for HTTP Basic Authentication in front of the SES bounce handler.
+END
+    localconfig_ses_password => <<'END',
+Password for HTTP Basic Authentication in front of the SES bounce handler.
+END
     localconfig_site_wide_secret => <<'END',
 This secret key is used by your installation for the creation and
 validation of encrypted tokens. These tokens are used to implement
diff --git a/vagrant_support/checksetup_answers.j2 b/vagrant_support/checksetup_answers.j2
index e0ec2ef96..683a28a6f 100644
--- a/vagrant_support/checksetup_answers.j2
+++ b/vagrant_support/checksetup_answers.j2
@@ -36,3 +36,5 @@ $answer{'defaultpriority'}      = '--';
 $answer{'defaultseverity'}      = 'normal';
 $answer{'skin'}                 = 'Mozilla';
 $answer{'docs_urlbase'}         = 'https://bmo.readthedocs.org/en/latest/';
+$answer{'ses_username'}         = 'ses';
+$answer{'ses_password'}         = 'secret';