Bug 1078314: Missing links and broken unicode characters in some bugmail

4 files changed, 96 insertions, 42 deletions

diff --git a/extensions/BMO/Extension.pm b/extensions/BMO/Extension.pm
index 1e0593c4e..d6839b2a7 100644
--- a/extensions/BMO/Extension.pm
+++ b/extensions/BMO/Extension.pm
@@ -40,6 +40,7 @@ use Bugzilla::Util;
 
 use Date::Parse;
 use DateTime;
+use Email::MIME::ContentType qw(parse_content_type);
 use Encode qw(find_encoding encode_utf8);
 use File::MimeInfo::Magic;
 use List::MoreUtils qw(natatime);
@@ -1132,9 +1133,29 @@ sub _inject_headers_into_body {
 sub _replace_placeholder_in_part {
     my ($part, $replacement) = @_;
 
-    # fix encoding
-    my $body = $part->body;
-    if (Bugzilla->params->{'utf8'}) {
+    _fix_encoding($part);
+
+    # replace
+    my $placeholder = quotemeta('@@body-headers@@');
+    my $body = $part->body_str;
+    $body =~ s/$placeholder/$replacement/;
+    $part->body_str_set($body);
+}
+
+sub _fix_encoding {
+    my $part = shift;
+
+    # don't touch the top-level part of multi-part mail
+    return if $part->parts > 1;
+
+    # nothing to do if the part already has a charset
+    my $ct = parse_content_type($part->content_type);
+    my $charset = $ct->{attributes}{charset}
+        ? $ct->{attributes}{charset}
+        : '';
+    return unless !$charset || $charset eq 'us-ascii';
+
+    if (Bugzilla->params->{utf8}) {
         $part->charset_set('UTF-8');
         my $raw = $part->body_raw;
         if (utf8::is_utf8($raw)) {
@@ -1142,13 +1163,7 @@ sub _replace_placeholder_in_part {
             $part->body_set($raw);
         }
     }
-    $part->encoding_set('quoted-printable') if !is_7bit_clean($body);
-
-    # replace
-    my $placeholder = quotemeta('@@body-headers@@');
-    $body = $part->body_str;
-    $body =~ s/$placeholder/$replacement/;
-    $part->body_str_set($body);
+    $part->encoding_set('quoted-printable');
 }
 
 sub _syslog {
diff --git a/extensions/Profanivore/Extension.pm b/extensions/Profanivore/Extension.pm
index cdec6e1c6..49b2f61c3 100644
--- a/extensions/Profanivore/Extension.pm
+++ b/extensions/Profanivore/Extension.pm
@@ -23,6 +23,7 @@ package Bugzilla::Extension::Profanivore;
 use strict;
 use base qw(Bugzilla::Extension);
 
+use Email::MIME::ContentType qw(parse_content_type);
 use Regexp::Common 'RE_ALL';
 
 use Bugzilla::Util qw(is_7bit_clean);
@@ -117,17 +118,26 @@ sub mailer_before_send {
 
 sub _fix_encoding {
     my $part = shift;
-    my $body = $part->body;
-    if (Bugzilla->params->{'utf8'}) {
+
+    # don't touch the top-level part of multi-part mail
+    return if $part->parts > 1;
+
+    # nothing to do if the part already has a charset
+    my $ct = parse_content_type($part->content_type);
+    my $charset = $ct->{attributes}{charset}
+        ? $ct->{attributes}{charset}
+        : '';
+    return unless !$charset || $charset eq 'us-ascii';
+
+    if (Bugzilla->params->{utf8}) {
         $part->charset_set('UTF-8');
-        # encoding_set works only with bytes, not with utf8 strings.
         my $raw = $part->body_raw;
         if (utf8::is_utf8($raw)) {
             utf8::encode($raw);
             $part->body_set($raw);
         }
     }
-    $part->encoding_set('quoted-printable') if !is_7bit_clean($body);
+    $part->encoding_set('quoted-printable');
 }
 
 sub _filter_text {
diff --git a/extensions/SecureMail/Extension.pm b/extensions/SecureMail/Extension.pm
index 687112955..492ce0cb6 100644
--- a/extensions/SecureMail/Extension.pm
+++ b/extensions/SecureMail/Extension.pm
@@ -36,6 +36,7 @@ use Crypt::OpenPGP::Armour;
 use Crypt::OpenPGP::KeyRing;
 use Crypt::OpenPGP;
 use Crypt::SMIME;
+use Email::MIME::ContentType qw(parse_content_type);
 use Encode;
 use HTML::Tree;
 
@@ -443,7 +444,7 @@ sub _make_secure {
             # stringify all parts for encrypting. We have to retain the old 
             # boundaries as well so that the email client can reconstruct the 
             # original message properly.
-            $email->walk_parts(\&_fix_part);
+            $email->walk_parts(\&_fix_encoding);
 
             $email->walk_parts(sub {
                 my ($part) = @_;
@@ -485,7 +486,7 @@ sub _make_secure {
                                "boundary=\"$new_boundary\"");
         }
         else {
-            _fix_part($email);
+            _fix_encoding($email);
             if ($sanitise_subject) {
                 _insert_subject($email, $subject);
             }
@@ -498,7 +499,7 @@ sub _make_secure {
         # S/MIME Encryption #
         #####################
 
-        $email->walk_parts(\&_fix_part);
+        $email->walk_parts(\&_fix_encoding);
 
         if ($sanitise_subject) {
             $email->walk_parts(sub { _insert_subject($_[0], $subject) });
@@ -598,39 +599,38 @@ sub _insert_subject {
     }
 }
 
-# Copied from Bugzilla/Mailer as this extension runs before
-# this code there and Mailer.pm will no longer see the original
-# message.
-sub _fix_part {
-    my ($part) = @_;
-    return if $part->parts > 1; # Top-level
-    my $content_type = $part->content_type || '';
-    $content_type =~ /charset=['"](.+)['"]/;
-    # If no charset is defined or is the default us-ascii,
-    # then we encode the email to UTF-8 if Bugzilla has utf8 enabled.
-    # XXX - This is a hack to workaround bug 723944.
-    if (!$1 || $1 eq 'us-ascii') {
-        my $body = $part->body;
-        if (Bugzilla->params->{'utf8'}) {
-            $part->charset_set('UTF-8');
-            # encoding_set works only with bytes, not with utf8 strings.
-            my $raw = $part->body_raw;
-            if (utf8::is_utf8($raw)) {
-                utf8::encode($raw);
-                $part->body_set($raw);
-            }
+sub _fix_encoding {
+    my $part = shift;
+
+    # don't touch the top-level part of multi-part mail
+    return if $part->parts > 1;
+
+    # nothing to do if the part already has a charset
+    my $ct = parse_content_type($part->content_type);
+    my $charset = $ct->{attributes}{charset}
+        ? $ct->{attributes}{charset}
+        : '';
+    return unless !$charset || $charset eq 'us-ascii';
+
+    if (Bugzilla->params->{utf8}) {
+        $part->charset_set('UTF-8');
+        my $raw = $part->body_raw;
+        if (utf8::is_utf8($raw)) {
+            utf8::encode($raw);
+            $part->body_set($raw);
         }
-        $part->encoding_set('quoted-printable') if !is_7bit_clean($body);
     }
+    $part->encoding_set('quoted-printable');
 }
 
 sub _filter_bug_links {
     my ($email) = @_;
     $email->walk_parts(sub {
         my $part = shift;
+        _fix_encoding($part);
         my $content_type = $part->content_type;
         return if !$content_type || $content_type !~ /text\/html/;
-        my $tree = HTML::Tree->new->parse_content($part->body);
+        my $tree = HTML::Tree->new->parse_content($part->body_str);
         my @links = $tree->look_down( _tag  => q{a}, class => qr/bz_bug_link/ );
         my $updated = 0;
         foreach my $link (@links) {
diff --git a/extensions/Splinter/lib/Util.pm b/extensions/Splinter/lib/Util.pm
index 3c77239a9..e6cbe7493 100644
--- a/extensions/Splinter/lib/Util.pm
+++ b/extensions/Splinter/lib/Util.pm
@@ -25,6 +25,7 @@ use strict;
 
 use Bugzilla;
 use Bugzilla::Util;
+use Email::MIME::ContentType qw(parse_content_type);
 
 use base qw(Exporter);
 
@@ -130,7 +131,11 @@ sub munge_create_attachment {
 # (\015 and \012 are used because Perl \n is platform-dependent)
 sub add_review_links_to_email {
     my $email = shift;
-    my $body = $email->body;
+    return if $email->parts > 1;
+
+    _fix_encoding($email);
+    my $body = $email->body_str;
+
     my $new_body = 0;
     my $bug;
 
@@ -157,7 +162,31 @@ sub add_review_links_to_email {
         $new_body = 1;
     }
 
-    $email->body_set($body) if $new_body;
+    $email->body_str_set($body) if $new_body;
+}
+
+sub _fix_encoding {
+    my $part = shift;
+
+    # don't touch the top-level part of multi-part mail
+    return if $part->parts > 1;
+
+    # nothing to do if the part already has a charset
+    my $ct = parse_content_type($part->content_type);
+    my $charset = $ct->{attributes}{charset}
+        ? $ct->{attributes}{charset}
+        : '';
+    return unless !$charset || $charset eq 'us-ascii';
+
+    if (Bugzilla->params->{utf8}) {
+        $part->charset_set('UTF-8');
+        my $raw = $part->body_raw;
+        if (utf8::is_utf8($raw)) {
+            utf8::encode($raw);
+            $part->body_set($raw);
+        }
+    }
+    $part->encoding_set('quoted-printable');
 }
 
 1;