diff options
Diffstat (limited to 'extensions/Push/lib')
22 files changed, 3336 insertions, 0 deletions
diff --git a/extensions/Push/lib/Admin.pm b/extensions/Push/lib/Admin.pm new file mode 100644 index 000000000..f579409bd --- /dev/null +++ b/extensions/Push/lib/Admin.pm @@ -0,0 +1,122 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Admin; + +use strict; +use warnings; + +use Bugzilla; +use Bugzilla::Error; +use Bugzilla::Extension::Push::Util; +use Bugzilla::Util qw(trim detaint_natural trick_taint); + +use base qw(Exporter); +our @EXPORT = qw( + admin_config + admin_queues + admin_log +); + +sub admin_config { + my ($vars) = @_; + my $push = Bugzilla->push_ext; + my $input = Bugzilla->input_params; + + if ($input->{save}) { + my $dbh = Bugzilla->dbh; + $dbh->bz_start_transaction(); + _update_config_from_form('global', $push->config); + foreach my $connector ($push->connectors->list) { + _update_config_from_form($connector->name, $connector->config); + } + $push->set_config_last_modified(); + $dbh->bz_commit_transaction(); + $vars->{message} = 'push_config_updated'; + } + + $vars->{push} = $push; + $vars->{connectors} = $push->connectors; +} + +sub _update_config_from_form { + my ($name, $config) = @_; + my $input = Bugzilla->input_params; + + # read values from form + my $values = {}; + foreach my $option ($config->options) { + my $option_name = $option->{name}; + $values->{$option_name} = trim($input->{$name . ".$option_name"}); + } + + # validate + if ($values->{enabled} eq 'Enabled') { + eval { + $config->validate($values); + }; + if ($@) { + ThrowUserError('push_error', { error_message => clean_error($@) }); + } + } + + # update + foreach my $option ($config->options) { + my $option_name = $option->{name}; + trick_taint($values->{$option_name}); + $config->{$option_name} = $values->{$option_name}; + } + $config->update(); +} + +sub admin_queues { + my ($vars, $page) = @_; + my $push = Bugzilla->push_ext; + my $input = Bugzilla->input_params; + + if ($page eq 'push_queues.html') { + $vars->{push} = $push; + + } elsif ($page eq 'push_queues_view.html') { + my $queue; + if ($input->{connector}) { + my $connector = $push->connectors->by_name($input->{connector}) + || ThrowUserError('push_error', { error_message => 'Invalid connector' }); + $queue = $connector->backlog; + } else { + $queue = $push->queue; + } + $vars->{queue} = $queue; + + my $id = $input->{message} || 0; + detaint_natural($id) + || ThrowUserError('push_error', { error_message => 'Invalid message ID' }); + my $message = $queue->by_id($id) + || ThrowUserError('push_error', { error_message => 'Invalid message ID' }); + + if ($input->{delete}) { + $message->remove_from_db(); + $vars->{message} = 'push_message_deleted'; + + } else { + $vars->{message_obj} = $message; + eval { + $vars->{json} = to_json($message->payload_decoded, 1); + }; + } + } +} + +sub admin_log { + my ($vars) = @_; + my $push = Bugzilla->push_ext; + my $input = Bugzilla->input_params; + + $vars->{push} = $push; +} + +1; diff --git a/extensions/Push/lib/BacklogMessage.pm b/extensions/Push/lib/BacklogMessage.pm new file mode 100644 index 000000000..8f5263038 --- /dev/null +++ b/extensions/Push/lib/BacklogMessage.pm @@ -0,0 +1,149 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::BacklogMessage; + +use strict; +use warnings; + +use base 'Bugzilla::Object'; + +use constant AUDIT_CREATES => 0; +use constant AUDIT_UPDATES => 0; +use constant AUDIT_REMOVES => 0; + +use Bugzilla; +use Bugzilla::Error; +use Bugzilla::Extension::Push::Util; +use Bugzilla::Util; +use Encode; + +# +# initialisation +# + +use constant DB_TABLE => 'push_backlog'; +use constant DB_COLUMNS => qw( + id + message_id + push_ts + payload + change_set + routing_key + connector + attempt_ts + attempts + last_error +); +use constant UPDATE_COLUMNS => qw( + attempt_ts + attempts + last_error +); +use constant LIST_ORDER => 'push_ts'; +use constant VALIDATORS => { + payload => \&_check_payload, + change_set => \&_check_change_set, + routing_key => \&_check_routing_key, + connector => \&_check_connector, + attempts => \&_check_attempts, +}; + +# +# constructors +# + +sub create_from_message { + my ($class, $message, $connector) = @_; + my $self = $class->create({ + message_id => $message->id, + push_ts => $message->push_ts, + payload => $message->payload, + change_set => $message->change_set, + routing_key => $message->routing_key, + connector => $connector->name, + attempt_ts => undef, + attempts => 0, + last_error => undef, + }); + return $self; +} + +# +# accessors +# + +sub message_id { return $_[0]->{'message_id'} } +sub push_ts { return $_[0]->{'push_ts'}; } +sub payload { return $_[0]->{'payload'}; } +sub change_set { return $_[0]->{'change_set'}; } +sub routing_key { return $_[0]->{'routing_key'}; } +sub connector { return $_[0]->{'connector'}; } +sub attempt_ts { return $_[0]->{'attempt_ts'}; } +sub attempts { return $_[0]->{'attempts'}; } +sub last_error { return $_[0]->{'last_error'}; } + +sub payload_decoded { + my ($self) = @_; + return from_json($self->{'payload'}); +} + +sub attempt_time { + my ($self) = @_; + if (!exists $self->{'attempt_time'}) { + $self->{'attempt_time'} = datetime_from($self->attempt_ts)->epoch; + } + return $self->{'attempt_time'}; +} + +# +# mutators +# + +sub inc_attempts { + my ($self, $error) = @_; + $self->{attempt_ts} = Bugzilla->dbh->selectrow_array('SELECT NOW()'); + $self->{attempts} = $self->{attempts} + 1; + $self->{last_error} = $error; + $self->update; +} + +# +# validators +# + +sub _check_payload { + my ($invocant, $value) = @_; + length($value) || ThrowCodeError('push_invalid_payload'); + return $value; +} + +sub _check_change_set { + my ($invocant, $value) = @_; + (defined($value) && length($value)) || ThrowCodeError('push_invalid_change_set'); + return $value; +} + +sub _check_routing_key { + my ($invocant, $value) = @_; + (defined($value) && length($value)) || ThrowCodeError('push_invalid_routing_key'); + return $value; +} + +sub _check_connector { + my ($invocant, $value) = @_; + Bugzilla->push_ext->connectors->exists($value) || ThrowCodeError('push_invalid_connector'); + return $value; +} + +sub _check_attempts { + my ($invocant, $value) = @_; + return $value || 0; +} + +1; + diff --git a/extensions/Push/lib/BacklogQueue.pm b/extensions/Push/lib/BacklogQueue.pm new file mode 100644 index 000000000..79b9b72ee --- /dev/null +++ b/extensions/Push/lib/BacklogQueue.pm @@ -0,0 +1,127 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::BacklogQueue; + +use strict; +use warnings; + +use Bugzilla; +use Bugzilla::Extension::Push::BacklogMessage; + +sub new { + my ($class, $connector) = @_; + my $self = {}; + bless($self, $class); + $self->{connector} = $connector; + return $self; +} + +sub count { + my ($self) = @_; + my $dbh = Bugzilla->dbh; + return $dbh->selectrow_array(" + SELECT COUNT(*) + FROM push_backlog + WHERE connector = ?", + undef, + $self->{connector}); +} + +sub oldest { + my ($self) = @_; + my @messages = $self->list( + limit => 1, + filter => 'AND ((next_attempt_ts IS NULL) OR (next_attempt_ts <= NOW()))', + ); + return scalar(@messages) ? $messages[0] : undef; +} + +sub by_id { + my ($self, $id) = @_; + my @messages = $self->list( + limit => 1, + filter => "AND (log.id = $id)", + ); + return scalar(@messages) ? $messages[0] : undef; +} + +sub list { + my ($self, %args) = @_; + $args{limit} ||= 10; + $args{filter} ||= ''; + my @result; + my $dbh = Bugzilla->dbh; + + my $filter_sql = $args{filter} || ''; + my $sth = $dbh->prepare(" + SELECT log.id, message_id, push_ts, payload, change_set, routing_key, attempt_ts, log.attempts + FROM push_backlog log + LEFT JOIN push_backoff off ON off.connector = log.connector + WHERE log.connector = ? ". + $args{filter} . " + ORDER BY push_ts " . + $dbh->sql_limit($args{limit}) + ); + $sth->execute($self->{connector}); + while (my $row = $sth->fetchrow_hashref()) { + push @result, Bugzilla::Extension::Push::BacklogMessage->new({ + id => $row->{id}, + message_id => $row->{message_id}, + push_ts => $row->{push_ts}, + payload => $row->{payload}, + change_set => $row->{change_set}, + routing_key => $row->{routing_key}, + connector => $self->{connector}, + attempt_ts => $row->{attempt_ts}, + attempts => $row->{attempts}, + }); + } + return @result; +} + +# +# backoff +# + +sub backoff { + my ($self) = @_; + if (!$self->{backoff}) { + my $ra = Bugzilla::Extension::Push::Backoff->match({ + connector => $self->{connector} + }); + if (@$ra) { + $self->{backoff} = $ra->[0]; + } else { + $self->{backoff} = Bugzilla::Extension::Push::Backoff->create({ + connector => $self->{connector} + }); + } + } + return $self->{backoff}; +} + +sub reset_backoff { + my ($self) = @_; + my $backoff = $self->backoff; + $backoff->reset(); + $backoff->update(); +} + +sub inc_backoff { + my ($self) = @_; + my $backoff = $self->backoff; + $backoff->inc(); + $backoff->update(); +} + +sub connector { + my ($self) = @_; + return $self->{connector}; +} + +1; diff --git a/extensions/Push/lib/Backoff.pm b/extensions/Push/lib/Backoff.pm new file mode 100644 index 000000000..05ee0a775 --- /dev/null +++ b/extensions/Push/lib/Backoff.pm @@ -0,0 +1,109 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Backoff; + +use strict; +use warnings; + +use base 'Bugzilla::Object'; + +use constant AUDIT_CREATES => 0; +use constant AUDIT_UPDATES => 0; +use constant AUDIT_REMOVES => 0; + +use Bugzilla; +use Bugzilla::Util; + +# +# initialisation +# + +use constant DB_TABLE => 'push_backoff'; +use constant DB_COLUMNS => qw( + id + connector + next_attempt_ts + attempts +); +use constant UPDATE_COLUMNS => qw( + next_attempt_ts + attempts +); +use constant VALIDATORS => { + connector => \&_check_connector, + next_attempt_ts => \&_check_next_attempt_ts, + attempts => \&_check_attempts, +}; +use constant LIST_ORDER => 'next_attempt_ts'; + +# +# accessors +# + +sub connector { return $_[0]->{'connector'}; } +sub next_attempt_ts { return $_[0]->{'next_attempt_ts'}; } +sub attempts { return $_[0]->{'attempts'}; } + +sub next_attempt_time { + my ($self) = @_; + if (!exists $self->{'next_attempt_time'}) { + $self->{'next_attempt_time'} = datetime_from($self->next_attempt_ts)->epoch; + } + return $self->{'next_attempt_time'}; +} + +# +# mutators +# + +sub reset { + my ($self) = @_; + $self->{next_attempt_ts} = Bugzilla->dbh->selectrow_array('SELECT NOW()'); + $self->{attempts} = 0; + Bugzilla->push_ext->logger->debug( + sprintf("resetting backoff for %s", $self->connector) + ); +} + +sub inc { + my ($self) = @_; + my $dbh = Bugzilla->dbh; + + my $attempts = $self->attempts + 1; + my $seconds = $attempts <= 4 ? 5 ** $attempts : 15 * 60; + my ($date) = $dbh->selectrow_array("SELECT " . $dbh->sql_date_math('NOW()', '+', $seconds, 'SECOND')); + + $self->{next_attempt_ts} = $date; + $self->{attempts} = $attempts; + Bugzilla->push_ext->logger->debug( + sprintf("setting next attempt for %s to %s (attempt %s)", $self->connector, $date, $attempts) + ); +} + +# +# validators +# + +sub _check_connector { + my ($invocant, $value) = @_; + Bugzilla->push_ext->connectors->exists($value) || ThrowCodeError('push_invalid_connector'); + return $value; +} + +sub _check_next_attempt_ts { + my ($invocant, $value) = @_; + return $value || Bugzilla->dbh->selectrow_array('SELECT NOW()'); +} + +sub _check_attempts { + my ($invocant, $value) = @_; + return $value || 0; +} + +1; + diff --git a/extensions/Push/lib/Config.pm b/extensions/Push/lib/Config.pm new file mode 100644 index 000000000..7033b4195 --- /dev/null +++ b/extensions/Push/lib/Config.pm @@ -0,0 +1,215 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Config; + +use strict; +use warnings; + +use Bugzilla; +use Bugzilla::Constants; +use Bugzilla::Extension::Push::Option; +use Crypt::CBC; + +sub new { + my ($class, $name, @options) = @_; + my $self = { + _name => $name + }; + bless($self, $class); + + $self->{_options} = [@options]; + unshift @{$self->{_options}}, { + name => 'enabled', + label => 'Status', + help => '', + type => 'select', + values => [ 'Enabled', 'Disabled' ], + default => 'Disabled', + }; + + return $self; +} + +sub options { + my ($self) = @_; + return @{$self->{_options}}; +} + +sub option { + my ($self, $name) = @_; + foreach my $option ($self->options) { + return $option if $option->{name} eq $name; + } + return undef; +} + +sub load { + my ($self) = @_; + my $config = {}; + my $logger = Bugzilla->push_ext->logger; + + # prime $config with defaults + foreach my $rh ($self->options) { + $config->{$rh->{name}} = $rh->{default}; + } + + # override defaults with values from database + my $options = Bugzilla::Extension::Push::Option->match({ + connector => $self->{_name}, + }); + foreach my $option (@$options) { + my $option_config = $self->option($option->name) + || next; + if ($option_config->{type} eq 'password') { + $config->{$option->name} = $self->_decrypt($option->value); + } else { + $config->{$option->name} = $option->value; + } + } + + # validate when running from the daemon + if (Bugzilla->push_ext->is_daemon) { + $self->_validate_config($config); + } + + # done, update self + foreach my $name (keys %$config) { + my $value = $self->option($name)->{type} eq 'password' ? '********' : $config->{$name}; + $logger->debug(sprintf("%s: set %s=%s\n", $self->{_name}, $name, $value || '')); + $self->{$name} = $config->{$name}; + } +} + +sub validate { + my ($self, $config) = @_; + $self->_validate_mandatory($config); + $self->_validate_config($config); +} + +sub update { + my ($self) = @_; + + my @valid_options = map { $_->{name} } $self->options; + + my %options; + my $options_list = Bugzilla::Extension::Push::Option->match({ + connector => $self->{_name}, + }); + foreach my $option (@$options_list) { + $options{$option->name} = $option; + } + + # delete options which are no longer valid + foreach my $name (keys %options) { + if (!grep { $_ eq $name } @valid_options) { + $options{$name}->remove_from_db(); + delete $options{$name}; + } + } + + # update options + foreach my $name (keys %options) { + my $option = $options{$name}; + if ($self->option($name)->{type} eq 'password') { + $option->set_value($self->_encrypt($self->{$name})); + } else { + $option->set_value($self->{$name}); + } + $option->update(); + } + + # add missing options + foreach my $name (@valid_options) { + next if exists $options{$name}; + Bugzilla::Extension::Push::Option->create({ + connector => $self->{_name}, + option_name => $name, + option_value => $self->{$name}, + }); + } +} + +sub _remove_invalid_options { + my ($self, $config) = @_; + my @names; + foreach my $rh ($self->options) { + push @names, $rh->{name}; + } + foreach my $name (keys %$config) { + if ($name =~ /^_/ || !grep { $_ eq $name } @names) { + delete $config->{$name}; + } + } +} + +sub _validate_mandatory { + my ($self, $config) = @_; + $self->_remove_invalid_options($config); + + my @missing; + foreach my $option ($self->options) { + next unless $option->{required}; + my $name = $option->{name}; + if (!exists $config->{$name} || !defined($config->{$name}) || $config->{$name} eq '') { + push @missing, $option; + } + } + if (@missing) { + my $connector = $self->{_name}; + @missing = map { $_->{label} } @missing; + if (scalar @missing == 1) { + die "The option '$missing[0]' for the connector '$connector' is mandatory\n"; + } else { + die "The following options for the connector '$connector' are mandatory:\n " + . join("\n ", @missing) . "\n"; + } + } +} + +sub _validate_config { + my ($self, $config) = @_; + $self->_remove_invalid_options($config); + + my @errors; + foreach my $option ($self->options) { + my $name = $option->{name}; + next unless exists $config->{$name} && exists $option->{validate}; + eval { + $option->{validate}->($config->{$name}, $config); + }; + push @errors, $@ if $@; + } + die join("\n", @errors) if @errors; + + if ($self->{_name} ne 'global') { + my $class = 'Bugzilla::Extension::Push::Connector::' . $self->{_name}; + $class->options_validate($config); + } +} + +sub _cipher { + my ($self) = @_; + $self->{_cipher} ||= Crypt::CBC->new( + -key => Bugzilla->localconfig->{'site_wide_secret'}, + -cipher => 'DES_EDE3'); + return $self->{_cipher}; +} + +sub _decrypt { + my ($self, $value) = @_; + my $result; + eval { $result = $self->_cipher->decrypt_hex($value) }; + return $@ ? '' : $result; +} + +sub _encrypt { + my ($self, $value) = @_; + return $self->_cipher->encrypt_hex($value); +} + +1; diff --git a/extensions/Push/lib/Connector.disabled/ServiceNow.pm b/extensions/Push/lib/Connector.disabled/ServiceNow.pm new file mode 100644 index 000000000..832cc9262 --- /dev/null +++ b/extensions/Push/lib/Connector.disabled/ServiceNow.pm @@ -0,0 +1,434 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Connector::ServiceNow; + +use strict; +use warnings; + +use base 'Bugzilla::Extension::Push::Connector::Base'; + +use Bugzilla::Attachment; +use Bugzilla::Bug; +use Bugzilla::Component; +use Bugzilla::Constants; +use Bugzilla::Extension::Push::Constants; +use Bugzilla::Extension::Push::Serialise; +use Bugzilla::Extension::Push::Util; +use Bugzilla::Field; +use Bugzilla::Mailer; +use Bugzilla::Product; +use Bugzilla::User; +use Bugzilla::Util qw(trim trick_taint); +use Email::MIME; +use FileHandle; +use LWP; +use MIME::Base64; +use Net::LDAP; + +use constant SEND_COMPONENTS => ( + { + product => 'mozilla.org', + component => 'Server Operations: Desktop Issues', + }, +); + +sub options { + return ( + { + name => 'bugzilla_user', + label => 'Bugzilla Service-Now User', + type => 'string', + default => 'service.now@bugzilla.tld', + required => 1, + validate => sub { + Bugzilla::User->new({ name => $_[0] }) + || die "Invalid Bugzilla user ($_[0])\n"; + }, + }, + { + name => 'ldap_scheme', + label => 'Mozilla LDAP Scheme', + type => 'select', + values => [ 'LDAP', 'LDAPS' ], + default => 'LDAPS', + required => 1, + }, + { + name => 'ldap_host', + label => 'Mozilla LDAP Host', + type => 'string', + default => '', + required => 1, + }, + { + name => 'ldap_user', + label => 'Mozilla LDAP Bind Username', + type => 'string', + default => '', + required => 1, + }, + { + name => 'ldap_pass', + label => 'Mozilla LDAP Password', + type => 'password', + default => '', + required => 1, + }, + { + name => 'ldap_poll', + label => 'Mozilla LDAP Poll Frequency', + type => 'string', + default => '3', + required => 1, + help => 'minutes', + validate => sub { + $_[0] =~ /\D/ + && die "LDAP Poll Frequency must be an integer\n"; + $_[0] == 0 + && die "LDAP Poll Frequency cannot be less than one minute\n"; + }, + }, + { + name => 'service_now_url', + label => 'Service Now JSON URL', + type => 'string', + default => 'https://mozilladev.service-now.com', + required => 1, + help => "Must start with https:// and end with ?JSON", + validate => sub { + $_[0] =~ m#^https://[^\.\/]+\.service-now\.com\/# + || die "Invalid Service Now JSON URL\n"; + $_[0] =~ m#\?JSON$# + || die "Invalid Service Now JSON URL (must end with ?JSON)\n"; + }, + }, + { + name => 'service_now_user', + label => 'Service Now JSON Username', + type => 'string', + default => '', + required => 1, + }, + { + name => 'service_now_pass', + label => 'Service Now JSON Password', + type => 'password', + default => '', + required => 1, + }, + ); +} + +sub options_validate { + my ($self, $config) = @_; + my $host = $config->{ldap_host}; + trick_taint($host); + my $scheme = lc($config->{ldap_scheme}); + eval { + my $ldap = Net::LDAP->new($host, scheme => $scheme, onerror => 'die', timeout => 5) + or die $!; + $ldap->bind($config->{ldap_user}, password => $config->{ldap_pass}); + }; + if ($@) { + die sprintf("Failed to connect to %s://%s/: %s\n", $scheme, $host, $@); + } +} + +my $_instance; + +sub init { + my ($self) = @_; + $_instance = $self; +} + +sub load_config { + my ($self) = @_; + $self->SUPER::load_config(@_); + $self->{bugzilla_user} ||= Bugzilla::User->new({ name => $self->config->{bugzilla_user} }); +} + +sub should_send { + my ($self, $message) = @_; + + my $data = $message->payload_decoded; + my $bug_data = $self->_get_bug_data($data) + || return 0; + + # we don't want to send the initial comment in a separate message + # because we inject it into the inital message + if (exists $data->{comment} && $data->{comment}->{number} == 0) { + return 0; + } + + my $target = $data->{event}->{target}; + unless ($target eq 'bug' || $target eq 'comment' || $target eq 'attachment') { + return 0; + } + + # ensure the service-now user can see the bug + if (!$self->{bugzilla_user} || !$self->{bugzilla_user}->is_enabled) { + return 0; + } + $self->{bugzilla_user}->can_see_bug($bug_data->{id}) + || return 0; + + # don't push changes made by the service-now account + $data->{event}->{user}->{id} == $self->{bugzilla_user}->id + && return 0; + + # filter based on the component + my $bug = Bugzilla::Bug->new($bug_data->{id}); + my $send = 0; + foreach my $rh (SEND_COMPONENTS) { + if ($bug->product eq $rh->{product} && $bug->component eq $rh->{component}) { + $send = 1; + last; + } + } + return $send; +} + +sub send { + my ($self, $message) = @_; + my $logger = Bugzilla->push_ext->logger; + my $config = $self->config; + + # should_send intiailises bugzilla_user; make sure we return a useful error message + if (!$self->{bugzilla_user}) { + return (PUSH_RESULT_TRANSIENT, "Invalid bugzilla-user (" . $self->config->{bugzilla_user} . ")"); + } + + # load the bug + my $data = $message->payload_decoded; + my $bug_data = $self->_get_bug_data($data); + my $bug = Bugzilla::Bug->new($bug_data->{id}); + + if ($message->routing_key eq 'bug.create') { + # inject the comment into the data for new bugs + my $comment = shift @{ $bug->comments }; + if ($comment->body ne '') { + $bug_data->{comment} = Bugzilla::Extension::Push::Serialise->instance->object_to_hash($comment, 1); + } + + } elsif ($message->routing_key eq 'attachment.create') { + # inject the attachment payload + my $attachment = Bugzilla::Attachment->new($data->{attachment}->{id}); + $data->{attachment}->{data} = encode_base64($attachment->data); + } + + # map bmo login to ldap login and insert into json payload + $self->_add_ldap_logins($data, {}); + + # flatten json data + $self->_flatten($data); + + # add sysparm_action + $data->{sysparm_action} = 'insert'; + + if ($logger->debugging) { + $logger->debug(to_json(ref($data) ? $data : from_json($data), 1)); + } + + # send to service-now + my $request = HTTP::Request->new(POST => $self->config->{service_now_url}); + $request->content_type('application/json'); + $request->content(to_json($data)); + $request->authorization_basic($self->config->{service_now_user}, $self->config->{service_now_pass}); + + $self->{lwp} ||= LWP::UserAgent->new(agent => Bugzilla->params->{urlbase}); + my $result = $self->{lwp}->request($request); + + # http level errors + if (!$result->is_success) { + # treat these as transient + return (PUSH_RESULT_TRANSIENT, $result->status_line); + } + + # empty response + if (length($result->content) == 0) { + # malformed request, treat as transient to allow code to fix + # may also be misconfiguration on servicenow, also transient + return (PUSH_RESULT_TRANSIENT, "Empty response"); + } + + # json errors + my $result_data; + eval { + $result_data = from_json($result->content); + }; + if ($@) { + return (PUSH_RESULT_TRANSIENT, clean_error($@)); + } + if ($logger->debugging) { + $logger->debug(to_json($result_data, 1)); + } + if (exists $result_data->{error}) { + return (PUSH_RESULT_ERROR, $result_data->{error}); + }; + + # malformed/unexpected json response + if (!exists $result_data->{records} + || ref($result_data->{records}) ne 'ARRAY' + || scalar(@{$result_data->{records}}) == 0 + ) { + return (PUSH_RESULT_ERROR, "Malformed JSON response from ServiceNow: missing or empty 'records' array"); + } + + my $record = $result_data->{records}->[0]; + if (ref($record) ne 'HASH') { + return (PUSH_RESULT_ERROR, "Malformed JSON response from ServiceNow: 'records' array does not contain an object"); + } + + # sys_id is the unique identifier for this action + if (!exists $record->{sys_id} || $record->{sys_id} eq '') { + return (PUSH_RESULT_ERROR, "Malformed JSON response from ServiceNow: 'records object' does not contain a valid sys_id"); + } + + # success + return (PUSH_RESULT_OK, "sys_id: " . $record->{sys_id}); +} + +sub _get_bug_data { + my ($self, $data) = @_; + my $target = $data->{event}->{target}; + if ($target eq 'bug') { + return $data->{bug}; + } elsif (exists $data->{$target}->{bug}) { + return $data->{$target}->{bug}; + } else { + return; + } +} + +sub _flatten { + # service-now expects a flat json object + my ($self, $data) = @_; + + my $target = $data->{event}->{target}; + + # delete unnecessary deep objects + if ($target eq 'comment' || $target eq 'attachment') { + $data->{$target}->{bug_id} = $data->{$target}->{bug}->{id}; + delete $data->{$target}->{bug}; + } + delete $data->{event}->{changes}; + + $self->_flatten_hash($data, $data, 'u'); +} + +sub _flatten_hash { + my ($self, $base_hash, $hash, $prefix) = @_; + foreach my $key (keys %$hash) { + if (ref($hash->{$key}) eq 'HASH') { + $self->_flatten_hash($base_hash, $hash->{$key}, $prefix . "_$key"); + } elsif (ref($hash->{$key}) ne 'ARRAY') { + $base_hash->{$prefix . "_$key"} = $hash->{$key}; + } + delete $hash->{$key}; + } +} + +sub _add_ldap_logins { + my ($self, $rh, $cache) = @_; + if (exists $rh->{login}) { + my $login = $rh->{login}; + $cache->{$login} ||= $self->_bmo_to_ldap($login); + Bugzilla->push_ext->logger->debug("BMO($login) --> LDAP(" . $cache->{$login} . ")"); + $rh->{ldap} = $cache->{$login}; + } + foreach my $key (keys %$rh) { + next unless ref($rh->{$key}) eq 'HASH'; + $self->_add_ldap_logins($rh->{$key}, $cache); + } +} + +sub _bmo_to_ldap { + my ($self, $login) = @_; + my $ldap = $self->_ldap_cache(); + + return '' unless $login =~ /\@mozilla\.(?:com|org)$/; + + foreach my $check ($login, canon_email($login)) { + # check for matching bugmail entry + foreach my $mail (keys %$ldap) { + next unless $ldap->{$mail}{bugmail_canon} eq $check; + return $mail; + } + + # check for matching mail + if (exists $ldap->{$check}) { + return $check; + } + + # check for matching email alias + foreach my $mail (sort keys %$ldap) { + next unless grep { $check eq $_ } @{$ldap->{$mail}{aliases}}; + return $mail; + } + } + + return ''; +} + +sub _ldap_cache { + my ($self) = @_; + my $logger = Bugzilla->push_ext->logger; + my $config = $self->config; + + # cache of all ldap entries; updated infrequently + if (!$self->{ldap_cache_time} || (time) - $self->{ldap_cache_time} > $config->{ldap_poll} * 60) { + $logger->debug('refreshing LDAP cache'); + + my $cache = {}; + + my $host = $config->{ldap_host}; + trick_taint($host); + my $scheme = lc($config->{ldap_scheme}); + my $ldap = Net::LDAP->new($host, scheme => $scheme, onerror => 'die') + or die $!; + $ldap->bind($config->{ldap_user}, password => $config->{ldap_pass}); + foreach my $ldap_base ('o=com,dc=mozilla', 'o=org,dc=mozilla') { + my $result = $ldap->search( + base => $ldap_base, + scope => 'sub', + filter => '(mail=*)', + attrs => ['mail', 'bugzillaEmail', 'emailAlias', 'cn', 'employeeType'], + ); + foreach my $entry ($result->entries) { + my ($name, $bugMail, $mail, $type) = + map { $entry->get_value($_) || '' } + qw(cn bugzillaEmail mail employeeType); + next if $type eq 'DISABLED'; + $mail = lc $mail; + $bugMail = '' if $bugMail !~ /\@/; + $bugMail = trim($bugMail); + if ($bugMail =~ / /) { + $bugMail = (grep { /\@/ } split / /, $bugMail)[0]; + } + $name =~ s/\s+/ /g; + $cache->{$mail}{name} = trim($name); + $cache->{$mail}{bugmail} = $bugMail; + $cache->{$mail}{bugmail_canon} = canon_email($bugMail); + $cache->{$mail}{aliases} = []; + foreach my $alias ( + @{$entry->get_value('emailAlias', asref => 1) || []} + ) { + push @{$cache->{$mail}{aliases}}, canon_email($alias); + } + } + } + + $self->{ldap_cache} = $cache; + $self->{ldap_cache_time} = (time); + } + + return $self->{ldap_cache}; +} + +1; + diff --git a/extensions/Push/lib/Connector/AMQP.pm b/extensions/Push/lib/Connector/AMQP.pm new file mode 100644 index 000000000..7b7d4aa72 --- /dev/null +++ b/extensions/Push/lib/Connector/AMQP.pm @@ -0,0 +1,230 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Connector::AMQP; + +use strict; +use warnings; + +use base 'Bugzilla::Extension::Push::Connector::Base'; + +use Bugzilla::Constants; +use Bugzilla::Extension::Push::Constants; +use Bugzilla::Extension::Push::Util; +use Bugzilla::Util qw(generate_random_password); +use DateTime; + +sub init { + my ($self) = @_; + $self->{mq} = 0; + $self->{channel} = 1; + + if ($self->config->{queue}) { + $self->{queue_name} = $self->config->{queue}; + } else { + my $queue_name = Bugzilla->params->{'urlbase'}; + $queue_name =~ s#^https?://##; + $queue_name =~ s#/$#|#; + $queue_name .= generate_random_password(16); + $self->{queue_name} = $queue_name; + } +} + +sub options { + return ( + { + name => 'host', + label => 'AMQP Hostname', + type => 'string', + default => 'localhost', + required => 1, + }, + { + name => 'port', + label => 'AMQP Port', + type => 'string', + default => '5672', + required => 1, + validate => sub { + $_[0] =~ /\D/ && die "Invalid port (must be numeric)\n"; + }, + }, + { + name => 'username', + label => 'Username', + type => 'string', + default => 'guest', + required => 1, + }, + { + name => 'password', + label => 'Password', + type => 'password', + default => 'guest', + required => 1, + }, + { + name => 'vhost', + label => 'Virtual Host', + type => 'string', + default => '/', + required => 1, + }, + { + name => 'exchange', + label => 'Exchange', + type => 'string', + default => '', + required => 1, + }, + { + name => 'queue', + label => 'Queue', + type => 'string', + }, + ); +} + +sub stop { + my ($self) = @_; + if ($self->{mq}) { + Bugzilla->push_ext->logger->debug('AMQP: disconnecting'); + $self->{mq}->disconnect(); + $self->{mq} = 0; + } +} + +sub _connect { + my ($self) = @_; + my $logger = Bugzilla->push_ext->logger; + my $config = $self->config; + + $self->stop(); + + $logger->debug('AMQP: Connecting to RabbitMQ ' . $config->{host} . ':' . $config->{port}); + require Net::RabbitMQ; + my $mq = Net::RabbitMQ->new(); + $mq->connect( + $config->{host}, + { + port => $config->{port}, + user => $config->{username}, + password => $config->{password}, + } + ); + $self->{mq} = $mq; + + $logger->debug('AMQP: Opening channel ' . $self->{channel}); + $self->{mq}->channel_open($self->{channel}); + + $logger->debug('AMQP: Declaring queue ' . $self->{queue_name}); + $self->{mq}->queue_declare( + $self->{channel}, + $self->{queue_name}, + { + passive => 0, + durable => 1, + exclusive => 0, + auto_delete => 0, + }, + ); +} + +sub _bind { + my ($self, $message) = @_; + my $logger = Bugzilla->push_ext->logger; + my $config = $self->config; + + # bind to queue (also acts to verify the connection is still valid) + if ($self->{mq}) { + eval { + $logger->debug('AMQP: binding queue(' . $self->{queue_name} . ') with exchange(' . $config->{exchange} . ')'); + $self->{mq}->queue_bind( + $self->{channel}, + $self->{queue_name}, + $config->{exchange}, + $message->routing_key, + ); + }; + if ($@) { + $logger->debug('AMQP: ' . clean_error($@)); + $self->{mq} = 0; + } + } + +} + +sub should_send { + my ($self, $message) = @_; + my $logger = Bugzilla->push_ext->logger; + + my $payload = $message->payload_decoded(); + my $target = $payload->{event}->{target}; + my $is_private = $payload->{$target}->{is_private} ? 1 : 0; + if (!$is_private && exists $payload->{$target}->{bug}) { + $is_private = $payload->{$target}->{bug}->{is_private} ? 1 : 0; + } + + if ($is_private) { + # we only want to push the is_private message from the change_set, as + # this is guaranteed to contain public information only + if ($message->routing_key !~ /\.modify:is_private$/) { + $logger->debug('AMQP: Ignoring private message'); + return 0; + } + $logger->debug('AMQP: Sending change of message to is_private'); + } + return 1; +} + +sub send { + my ($self, $message) = @_; + my $logger = Bugzilla->push_ext->logger; + my $config = $self->config; + + # don't push comments to pulse + if ($message->routing_key =~ /^comment\./) { + $logger->debug('AMQP: Ignoring comment'); + return PUSH_RESULT_IGNORED; + } + + # don't push private data + $self->should_push($message) + || return PUSH_RESULT_IGNORED; + + $self->_bind($message); + + eval { + # reconnect if required + if (!$self->{mq}) { + $self->_connect(); + } + + # send message + $logger->debug('AMQP: Publishing message'); + $self->{mq}->publish( + $self->{channel}, + $message->routing_key, + $message->payload, + { + exchange => $config->{exchange}, + }, + { + content_type => 'text/plain', + content_encoding => '8bit', + }, + ); + }; + if ($@) { + return (PUSH_RESULT_TRANSIENT, clean_error($@)); + } + + return PUSH_RESULT_OK; +} + +1; + diff --git a/extensions/Push/lib/Connector/Base.pm b/extensions/Push/lib/Connector/Base.pm new file mode 100644 index 000000000..290ea9740 --- /dev/null +++ b/extensions/Push/lib/Connector/Base.pm @@ -0,0 +1,106 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Connector::Base; + +use strict; +use warnings; + +use Bugzilla; +use Bugzilla::Extension::Push::Config; +use Bugzilla::Extension::Push::BacklogMessage; +use Bugzilla::Extension::Push::BacklogQueue; +use Bugzilla::Extension::Push::Backoff; + +sub new { + my ($class) = @_; + my $self = {}; + bless($self, $class); + ($self->{name}) = $class =~ /^.+:(.+)$/; + $self->init(); + return $self; +} + +sub name { + my $self = shift; + return $self->{name}; +} + +sub init { + my ($self) = @_; + # abstract + # perform any initialisation here + # will be run when created by the web pages or by the daemon + # and also when the configuration needs to be reloaded +} + +sub stop { + my ($self) = @_; + # abstract + # run from the daemon only; disconnect from remote hosts, etc +} + +sub should_send { + my ($self, $message) = @_; + # abstract + # return boolean indicating if the connector will be sending the message. + # this will be called each message, and should be a very quick simple test. + # the connector can perform a more exhaustive test in the send() method. + return 0; +} + +sub send { + my ($self, $message) = @_; + # abstract + # deliver the message, daemon only +} + +sub options { + my ($self) = @_; + # abstract + # return an array of configuration variables + return (); +} + +sub options_validate { + my ($class, $config) = @_; + # abstract, static + # die if a combination of options in $config is invalid +} + +# +# +# + +sub config { + my ($self) = @_; + if (!$self->{config}) { + $self->load_config(); + } + return $self->{config}; +} + +sub load_config { + my ($self) = @_; + my $config = Bugzilla::Extension::Push::Config->new($self->name, $self->options); + $config->load(); + $self->{config} = $config; +} + +sub enabled { + my ($self) = @_; + return $self->config->{enabled} eq 'Enabled'; +} + +sub backlog { + my ($self) = @_; + $self->{backlog} ||= Bugzilla::Extension::Push::BacklogQueue->new($self->name); + return $self->{backlog}; +} + +1; + diff --git a/extensions/Push/lib/Connector/File.pm b/extensions/Push/lib/Connector/File.pm new file mode 100644 index 000000000..2a8f4193d --- /dev/null +++ b/extensions/Push/lib/Connector/File.pm @@ -0,0 +1,68 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Connector::File; + +use strict; +use warnings; + +use base 'Bugzilla::Extension::Push::Connector::Base'; + +use Bugzilla::Constants; +use Bugzilla::Extension::Push::Constants; +use Bugzilla::Extension::Push::Util; +use Encode; +use FileHandle; + +sub init { + my ($self) = @_; +} + +sub options { + return ( + { + name => 'filename', + label => 'Filename', + type => 'string', + default => 'push.log', + required => 1, + validate => sub { + my $filename = shift; + $filename =~ m#^/# + && die "Absolute paths are not permitted\n"; + }, + }, + ); +} + +sub should_send { + my ($self, $message) = @_; + return 1; +} + +sub send { + my ($self, $message) = @_; + + # pretty-format json payload + my $payload = $message->payload_decoded; + $payload = to_json($payload, 1); + + my $filename = bz_locations()->{'datadir'} . '/' . $self->config->{filename}; + Bugzilla->push_ext->logger->debug("File: Appending to $filename"); + my $fh = FileHandle->new(">>$filename"); + $fh->binmode(':utf8'); + $fh->print( + "[" . scalar(localtime) . "]\n" . + $payload . "\n\n" + ); + $fh->close; + + return PUSH_RESULT_OK; +} + +1; + diff --git a/extensions/Push/lib/Connector/TCL.pm b/extensions/Push/lib/Connector/TCL.pm new file mode 100644 index 000000000..6207e2eac --- /dev/null +++ b/extensions/Push/lib/Connector/TCL.pm @@ -0,0 +1,349 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Connector::TCL; + +use strict; +use warnings; + +use base 'Bugzilla::Extension::Push::Connector::Base'; + +use Bugzilla::Constants; +use Bugzilla::Extension::Push::Constants; +use Bugzilla::Extension::Push::Serialise; +use Bugzilla::Extension::Push::Util; +use Bugzilla::User; +use Bugzilla::Attachment; + +use Digest::MD5 qw(md5_hex); +use Encode qw(encode_utf8); + +sub options { + return ( + { + name => 'tcl_user', + label => 'Bugzilla TCL User', + type => 'string', + default => 'tcl@bugzilla.tld', + required => 1, + validate => sub { + Bugzilla::User->new({ name => $_[0] }) + || die "Invalid Bugzilla user ($_[0])\n"; + }, + }, + { + name => 'sftp_host', + label => 'SFTP Host', + type => 'string', + default => '', + required => 1, + }, + { + name => 'sftp_port', + label => 'SFTP Port', + type => 'string', + default => '22', + required => 1, + validate => sub { + $_[0] =~ /\D/ && die "SFTP Port must be an integer\n"; + }, + }, + { + name => 'sftp_user', + label => 'SFTP Username', + type => 'string', + default => '', + required => 1, + }, + { + name => 'sftp_pass', + label => 'SFTP Password', + type => 'password', + default => '', + required => 1, + }, + { + name => 'sftp_remote_path', + label => 'SFTP Remote Path', + type => 'string', + default => '', + required => 0, + }, + ); +} + +my $_instance; + +sub init { + my ($self) = @_; + $_instance = $self; +} + +sub load_config { + my ($self) = @_; + $self->SUPER::load_config(@_); +} + +sub should_send { + my ($self, $message) = @_; + + my $data = $message->payload_decoded; + my $bug_data = $self->_get_bug_data($data) + || return 0; + + # sanity check user + $self->{tcl_user} ||= Bugzilla::User->new({ name => $self->config->{tcl_user} }); + if (!$self->{tcl_user} || !$self->{tcl_user}->is_enabled) { + return 0; + } + + # only send bugs created by the tcl user + unless ($bug_data->{reporter}->{id} == $self->{tcl_user}->id) { + return 0; + } + + # don't push changes made by the tcl user + if ($data->{event}->{user}->{id} == $self->{tcl_user}->id) { + return 0; + } + + # send comments + if ($data->{event}->{routing_key} eq 'comment.create') { + return 0 if $data->{comment}->{is_private}; + return 1; + } + + # send status and resolution updates + foreach my $change (@{ $data->{event}->{changes} }) { + return 1 if $change->{field} eq 'bug_status' || $change->{field} eq 'resolution'; + } + + # send attachments + if ($data->{event}->{routing_key} =~ /^attachment\./) { + return 0 if $data->{attachment}->{is_private}; + return 1; + } + + # and nothing else + return 0; +} + +sub send { + my ($self, $message) = @_; + my $logger = Bugzilla->push_ext->logger; + my $config = $self->config; + + require XML::Simple; + require Net::SFTP; + + $self->{tcl_user} ||= Bugzilla::User->new({ name => $self->config->{tcl_user} }); + if (!$self->{tcl_user}) { + return (PUSH_RESULT_TRANSIENT, "Invalid bugzilla-user (" . $self->config->{tcl_user} . ")"); + } + + # load the bug + my $data = $message->payload_decoded; + my $bug_data = $self->_get_bug_data($data); + + # build payload + my $attachment; + my %xml = ( + Mozilla_ID => $bug_data->{id}, + When => $data->{event}->{time}, + Who => $data->{event}->{user}->{login}, + Status => $bug_data->{status}->{name}, + Resolution => $bug_data->{resolution}, + ); + if ($data->{event}->{routing_key} eq 'comment.create') { + $xml{Comment} = $data->{comment}->{body}; + } elsif ($data->{event}->{routing_key} =~ /^attachment\.(\w+)/) { + my $is_update = $1 eq 'modify'; + if (!$is_update) { + $attachment = Bugzilla::Attachment->new($data->{attachment}->{id}); + } + $xml{Attach} = { + Attach_ID => $data->{attachment}->{id}, + Filename => $data->{attachment}->{file_name}, + Description => $data->{attachment}->{description}, + ContentType => $data->{attachment}->{content_type}, + IsPatch => $data->{attachment}->{is_patch} ? 'true' : 'false', + IsObsolete => $data->{attachment}->{is_obsolete} ? 'true' : 'false', + IsUpdate => $is_update ? 'true' : 'false', + }; + } + + # convert to xml + my $xml = XML::Simple::XMLout( + \%xml, + NoAttr => 1, + RootName => 'sync', + XMLDecl => 1, + ); + $xml = encode_utf8($xml); + + # generate md5 + my $md5 = md5_hex($xml); + + # build filename + my ($sec, $min, $hour, $day, $mon, $year) = localtime(time); + my $change_set = $data->{event}->{change_set}; + $change_set =~ s/\.//g; + my $filename = sprintf( + '%04s%02d%02d%02d%02d%02d%s', + $year + 1900, + $mon + 1, + $day, + $hour, + $min, + $sec, + $change_set, + ); + + # create temp files; + my $temp_dir = File::Temp::Directory->new(); + my $local_dir = $temp_dir->dirname; + _write_file("$local_dir/$filename.sync", $xml); + _write_file("$local_dir/$filename.sync.check", $md5); + _write_file("$local_dir/$filename.done", ''); + if ($attachment) { + _write_file("$local_dir/$filename.sync.attach", $attachment->data); + } + + my $remote_dir = $self->config->{sftp_remote_path} eq '' + ? '' + : $self->config->{sftp_remote_path} . '/'; + + # send files via sftp + $logger->debug("Connecting to " . $self->config->{sftp_host} . ":" . $self->config->{sftp_port}); + my $sftp = Net::SFTP->new( + $self->config->{sftp_host}, + ssh_args => { + port => $self->config->{sftp_port}, + }, + user => $self->config->{sftp_user}, + password => $self->config->{sftp_pass}, + ); + + $logger->debug("Uploading $local_dir/$filename.sync"); + $sftp->put("$local_dir/$filename.sync", "$remote_dir$filename.sync") + or return (PUSH_RESULT_ERROR, "Failed to upload $local_dir/$filename.sync"); + + $logger->debug("Uploading $local_dir/$filename.sync.check"); + $sftp->put("$local_dir/$filename.sync.check", "$remote_dir$filename.sync.check") + or return (PUSH_RESULT_ERROR, "Failed to upload $local_dir/$filename.sync.check"); + + if ($attachment) { + $logger->debug("Uploading $local_dir/$filename.sync.attach"); + $sftp->put("$local_dir/$filename.sync.attach", "$remote_dir$filename.sync.attach") + or return (PUSH_RESULT_ERROR, "Failed to upload $local_dir/$filename.sync.attach"); + } + + $logger->debug("Uploading $local_dir/$filename.done"); + $sftp->put("$local_dir/$filename.done", "$remote_dir$filename.done") + or return (PUSH_RESULT_ERROR, "Failed to upload $local_dir/$filename.done"); + + # success + return (PUSH_RESULT_OK, "uploaded $filename.sync"); +} + +sub _get_bug_data { + my ($self, $data) = @_; + my $target = $data->{event}->{target}; + if ($target eq 'bug') { + return $data->{bug}; + } elsif (exists $data->{$target}->{bug}) { + return $data->{$target}->{bug}; + } else { + return; + } +} + +sub _write_file { + my ($filename, $content) = @_; + open(my $fh, ">$filename") or die "Failed to write to $filename: $!\n"; + binmode($fh); + print $fh $content; + close($fh) or die "Failed to write to $filename: $!\n"; +} + +1; + +# File::Temp->newdir() requires a newer version of File::Temp than we have on +# production, so here's a small inline package which performs the same task. + +package File::Temp::Directory; + +use strict; +use warnings; + +use File::Temp; +use File::Path qw(rmtree); +use File::Spec; + +my @chars; + +sub new { + my ($class) = @_; + my $self = {}; + bless($self, $class); + + @chars = qw/ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z + a b c d e f g h i j k l m n o p q r s t u v w x y z + 0 1 2 3 4 5 6 7 8 9 _ + /; + + $self->{TEMPLATE} = File::Spec->catdir(File::Spec->tmpdir, 'X' x 10); + $self->{DIRNAME} = $self->_mktemp(); + return $self; +} + +sub _mktemp { + my ($self) = @_; + my $path = $self->_random_name(); + while(1) { + if (mkdir($path, 0700)) { + # in case of odd umask + chmod(0700, $path); + return $path; + } else { + # abort with error if the reason for failure was anything except eexist + die "Could not create directory $path: $!\n" unless ($!{EEXIST}); + # loop round for another try + } + $path = $self->_random_name(); + } + + return $path; +} + +sub _random_name { + my ($self) = @_; + my $path = $self->{TEMPLATE}; + $path =~ s/X/$chars[int(rand(@chars))]/ge; + return $path; +} + +sub dirname { + my ($self) = @_; + return $self->{DIRNAME}; +} + +sub DESTROY { + my ($self) = @_; + local($., $@, $!, $^E, $?); + if (-d $self->{DIRNAME}) { + # Some versions of rmtree will abort if you attempt to remove the + # directory you are sitting in. We protect that and turn it into a + # warning. We do this because this occurs during object destruction and + # so can not be caught by the user. + eval { rmtree($self->{DIRNAME}, 0, 0); }; + warn $@ if ($@ && $^W); + } +} + +1; + diff --git a/extensions/Push/lib/Connectors.pm b/extensions/Push/lib/Connectors.pm new file mode 100644 index 000000000..026d3f7f1 --- /dev/null +++ b/extensions/Push/lib/Connectors.pm @@ -0,0 +1,116 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Connectors; + +use strict; +use warnings; + +use Bugzilla::Extension::Push::Util; +use Bugzilla::Constants; +use Bugzilla::Util qw(trick_taint); +use File::Basename; + +sub new { + my ($class) = @_; + my $self = {}; + bless($self, $class); + + $self->{names} = []; + $self->{objects} = {}; + $self->{path} = bz_locations->{'extensionsdir'} . '/Push/lib/Connector'; + + my $logger = Bugzilla->push_ext->logger; + foreach my $file (glob($self->{path} . '/*.pm')) { + my $name = basename($file); + $name =~ s/\.pm$//; + next if $name eq 'Base'; + if (length($name) > 32) { + $logger->info("Ignoring connector '$name': Name longer than 32 characters"); + } + push @{$self->{names}}, $name; + $logger->debug("Found connector '$name'"); + } + + return $self; +} + +sub _load { + my ($self) = @_; + return if scalar keys %{$self->{objects}}; + + my $logger = Bugzilla->push_ext->logger; + foreach my $name (@{$self->{names}}) { + next if exists $self->{objects}->{$name}; + my $file = $self->{path} . "/$name.pm"; + trick_taint($file); + require $file; + my $package = "Bugzilla::Extension::Push::Connector::$name"; + + $logger->debug("Loading connector '$name'"); + my $old_error_mode = Bugzilla->error_mode; + Bugzilla->error_mode(ERROR_MODE_DIE); + eval { + my $connector = $package->new(); + $connector->load_config(); + $self->{objects}->{$name} = $connector; + }; + if ($@) { + $logger->error("Connector '$name' failed to load: " . clean_error($@)); + } + Bugzilla->error_mode($old_error_mode); + } +} + +sub stop { + my ($self) = @_; + my $logger = Bugzilla->push_ext->logger; + foreach my $connector ($self->list) { + next unless $connector->enabled; + $logger->debug("Stopping '" . $connector->name . "'"); + eval { + $connector->stop(); + }; + if ($@) { + $logger->error("Connector '" . $connector->name . "' failed to stop: " . clean_error($@)); + $logger->debug("Connector '" . $connector->name . "' failed to stop: $@"); + } + } +} + +sub reload { + my ($self) = @_; + $self->stop(); + $self->{objects} = {}; + $self->_load(); +} + +sub names { + my ($self) = @_; + return @{$self->{names}}; +} + +sub list { + my ($self) = @_; + $self->_load(); + return sort { $a->name cmp $b->name } values %{$self->{objects}}; +} + +sub exists { + my ($self, $name) = @_; + $self->by_name($name) ? 1 : 0; +} + +sub by_name { + my ($self, $name) = @_; + $self->_load(); + return unless exists $self->{objects}->{$name}; + return $self->{objects}->{$name}; +} + +1; + diff --git a/extensions/Push/lib/Constants.pm b/extensions/Push/lib/Constants.pm new file mode 100644 index 000000000..18b12d511 --- /dev/null +++ b/extensions/Push/lib/Constants.pm @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Constants; + +use strict; +use base 'Exporter'; + +our @EXPORT = qw( + PUSH_RESULT_OK + PUSH_RESULT_IGNORED + PUSH_RESULT_TRANSIENT + PUSH_RESULT_ERROR + PUSH_RESULT_UNKNOWN + push_result_to_string + + POLL_INTERVAL_SECONDS +); + +use constant PUSH_RESULT_OK => 1; +use constant PUSH_RESULT_IGNORED => 2; +use constant PUSH_RESULT_TRANSIENT => 3; +use constant PUSH_RESULT_ERROR => 4; +use constant PUSH_RESULT_UNKNOWN => 5; + +sub push_result_to_string { + my ($result) = @_; + return 'OK' if $result == PUSH_RESULT_OK; + return 'OK-IGNORED' if $result == PUSH_RESULT_IGNORED; + return 'TRANSIENT-ERROR' if $result == PUSH_RESULT_TRANSIENT; + return 'FATAL-ERROR' if $result == PUSH_RESULT_ERROR; + return 'UNKNOWN' if $result == PUSH_RESULT_UNKNOWN; +} + +use constant POLL_INTERVAL_SECONDS => 30; + +1; diff --git a/extensions/Push/lib/Daemon.pm b/extensions/Push/lib/Daemon.pm new file mode 100644 index 000000000..66e15783e --- /dev/null +++ b/extensions/Push/lib/Daemon.pm @@ -0,0 +1,96 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Daemon; + +use strict; +use warnings; + +use Bugzilla::Constants; +use Bugzilla::Extension::Push::Push; +use Bugzilla::Extension::Push::Logger; +use Carp qw(confess); +use Daemon::Generic; +use File::Basename; +use Pod::Usage; + +sub start { + newdaemon(); +} + +# +# daemon::generic config +# + +sub gd_preconfig { + my $self = shift; + my $pidfile = $self->{gd_args}{pidfile}; + if (!$pidfile) { + $pidfile = bz_locations()->{datadir} . '/' . $self->{gd_progname} . ".pid"; + } + return (pidfile => $pidfile); +} + +sub gd_getopt { + my $self = shift; + $self->SUPER::gd_getopt(); + if ($self->{gd_args}{progname}) { + $self->{gd_progname} = $self->{gd_args}{progname}; + } else { + $self->{gd_progname} = basename($0); + } + $self->{_original_zero} = $0; + $0 = $self->{gd_progname}; +} + +sub gd_postconfig { + my $self = shift; + $0 = delete $self->{_original_zero}; +} + +sub gd_more_opt { + my $self = shift; + return ( + 'pidfile=s' => \$self->{gd_args}{pidfile}, + 'n=s' => \$self->{gd_args}{progname}, + ); +} + +sub gd_usage { + pod2usage({ -verbose => 0, -exitval => 'NOEXIT' }); + return 0; +}; + +sub gd_redirect_output { + my $self = shift; + + my $filename = bz_locations()->{datadir} . '/' . $self->{gd_progname} . ".log"; + open(STDERR, ">>$filename") or (print "could not open stderr: $!" && exit(1)); + close(STDOUT); + open(STDOUT, ">&STDERR") or die "redirect STDOUT -> STDERR: $!"; + $SIG{HUP} = sub { + close(STDERR); + open(STDERR, ">>$filename") or (print "could not open stderr: $!" && exit(1)); + }; +} + +sub gd_setup_signals { + my $self = shift; + $self->SUPER::gd_setup_signals(); + $SIG{TERM} = sub { $self->gd_quit_event(); } +} + +sub gd_run { + my $self = shift; + $::SIG{__DIE__} = \&Carp::confess if $self->{debug}; + my $push = Bugzilla->push_ext; + $push->logger->{debug} = $self->{debug}; + $push->is_daemon(1); + $push->start(); +} + +1; diff --git a/extensions/Push/lib/Log.pm b/extensions/Push/lib/Log.pm new file mode 100644 index 000000000..6faabea97 --- /dev/null +++ b/extensions/Push/lib/Log.pm @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Log; + +use strict; +use warnings; + +use Bugzilla; +use Bugzilla::Extension::Push::Message; + +sub new { + my ($class) = @_; + my $self = {}; + bless($self, $class); + return $self; +} + +sub count { + my ($self) = @_; + my $dbh = Bugzilla->dbh; + return $dbh->selectrow_array("SELECT COUNT(*) FROM push_log"); +} + +sub list { + my ($self, %args) = @_; + $args{limit} ||= 10; + $args{filter} ||= ''; + my @result; + my $dbh = Bugzilla->dbh; + + my $ids = $dbh->selectcol_arrayref(" + SELECT id + FROM push_log + ORDER BY processed_ts DESC " . + $dbh->sql_limit(100) + ); + return Bugzilla::Extension::Push::LogEntry->new_from_list($ids); +} + +1; diff --git a/extensions/Push/lib/LogEntry.pm b/extensions/Push/lib/LogEntry.pm new file mode 100644 index 000000000..303c19da4 --- /dev/null +++ b/extensions/Push/lib/LogEntry.pm @@ -0,0 +1,70 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::LogEntry; + +use strict; +use warnings; + +use base 'Bugzilla::Object'; + +use constant AUDIT_CREATES => 0; +use constant AUDIT_UPDATES => 0; +use constant AUDIT_REMOVES => 0; + +use Bugzilla; +use Bugzilla::Error; +use Bugzilla::Extension::Push::Constants; + +# +# initialisation +# + +use constant DB_TABLE => 'push_log'; +use constant DB_COLUMNS => qw( + id + message_id + change_set + routing_key + connector + push_ts + processed_ts + result + data +); +use constant VALIDATORS => { + data => \&_check_data, +}; +use constant NAME_FIELD => ''; +use constant LIST_ORDER => 'processed_ts DESC'; + +# +# accessors +# + +sub message_id { return $_[0]->{'message_id'}; } +sub change_set { return $_[0]->{'change_set'}; } +sub routing_key { return $_[0]->{'routing_key'}; } +sub connector { return $_[0]->{'connector'}; } +sub push_ts { return $_[0]->{'push_ts'}; } +sub processed_ts { return $_[0]->{'processed_ts'}; } +sub result { return $_[0]->{'result'}; } +sub data { return $_[0]->{'data'}; } + +sub result_string { return push_result_to_string($_[0]->result) } + +# +# validators +# + +sub _check_data { + my ($invocant, $value) = @_; + return $value eq '' ? undef : $value; +} + +1; + diff --git a/extensions/Push/lib/Logger.pm b/extensions/Push/lib/Logger.pm new file mode 100644 index 000000000..68cec1e69 --- /dev/null +++ b/extensions/Push/lib/Logger.pm @@ -0,0 +1,70 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Logger; + +use strict; +use warnings; + +use Apache2::Log; +use Bugzilla::Extension::Push::Constants; +use Bugzilla::Extension::Push::LogEntry; + +sub new { + my ($class) = @_; + my $self = {}; + bless($self, $class); + return $self; +} + +sub info { shift->_log_it('INFO', @_) } +sub error { shift->_log_it('ERROR', @_) } +sub debug { shift->_log_it('DEBUG', @_) } + +sub debugging { + my ($self) = @_; + return $self->{debug}; +} + +sub _log_it { + my ($self, $method, $message) = @_; + return if $method eq 'DEBUG' && !$self->debugging; + chomp $message; + if ($ENV{MOD_PERL}) { + Apache2::ServerRec::warn("Push $method: $message"); + } elsif ($ENV{SCRIPT_FILENAME}) { + print STDERR "Push $method: $message\n"; + } else { + print STDERR '[' . localtime(time) ."] $method: $message\n"; + } +} + +sub result { + my ($self, $connector, $message, $result, $data) = @_; + $data ||= ''; + + $self->info(sprintf( + "%s: Message #%s: %s %s", + $connector->name, + $message->message_id, + push_result_to_string($result), + $data + )); + + Bugzilla::Extension::Push::LogEntry->create({ + message_id => $message->message_id, + change_set => $message->change_set, + routing_key => $message->routing_key, + connector => $connector->name, + push_ts => $message->push_ts, + processed_ts => Bugzilla->dbh->selectrow_array('SELECT NOW()'), + result => $result, + data => $data, + }); +} + +1; diff --git a/extensions/Push/lib/Message.pm b/extensions/Push/lib/Message.pm new file mode 100644 index 000000000..ebe32d0ea --- /dev/null +++ b/extensions/Push/lib/Message.pm @@ -0,0 +1,103 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Message; + +use strict; +use warnings; + +use base 'Bugzilla::Object'; + +use constant AUDIT_CREATES => 0; +use constant AUDIT_UPDATES => 0; +use constant AUDIT_REMOVES => 0; + +use Bugzilla; +use Bugzilla::Error; +use Bugzilla::Extension::Push::Util; +use Encode; + +# +# initialisation +# + +use constant DB_TABLE => 'push'; +use constant DB_COLUMNS => qw( + id + push_ts + payload + change_set + routing_key +); +use constant LIST_ORDER => 'push_ts'; +use constant VALIDATORS => { + push_ts => \&_check_push_ts, + payload => \&_check_payload, + change_set => \&_check_change_set, + routing_key => \&_check_routing_key, +}; + +# this creates an object which doesn't exist on the database +sub new_transient { + my $invocant = shift; + my $class = ref($invocant) || $invocant; + my $object = shift; + bless($object, $class) if $object; + return $object; +} + +# take a transient object and commit +sub create_from_transient { + my ($self) = @_; + return $self->create($self); +} + +# +# accessors +# + +sub push_ts { return $_[0]->{'push_ts'}; } +sub payload { return $_[0]->{'payload'}; } +sub change_set { return $_[0]->{'change_set'}; } +sub routing_key { return $_[0]->{'routing_key'}; } +sub message_id { return $_[0]->id; } + +sub payload_decoded { + my ($self) = @_; + return from_json($self->{'payload'}); +} + +# +# validators +# + +sub _check_push_ts { + my ($invocant, $value) = @_; + $value ||= Bugzilla->dbh->selectrow_array('SELECT NOW()'); + return $value; +} + +sub _check_payload { + my ($invocant, $value) = @_; + length($value) || ThrowCodeError('push_invalid_payload'); + return $value; +} + +sub _check_change_set { + my ($invocant, $value) = @_; + (defined($value) && length($value)) || ThrowCodeError('push_invalid_change_set'); + return $value; +} + +sub _check_routing_key { + my ($invocant, $value) = @_; + (defined($value) && length($value)) || ThrowCodeError('push_invalid_routing_key'); + return $value; +} + +1; + diff --git a/extensions/Push/lib/Option.pm b/extensions/Push/lib/Option.pm new file mode 100644 index 000000000..25d529f98 --- /dev/null +++ b/extensions/Push/lib/Option.pm @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Option; + +use strict; +use warnings; + +use base 'Bugzilla::Object'; + +use Bugzilla; +use Bugzilla::Error; +use Bugzilla::Util; + +# +# initialisation +# + +use constant DB_TABLE => 'push_options'; +use constant DB_COLUMNS => qw( + id + connector + option_name + option_value +); +use constant UPDATE_COLUMNS => qw( + option_value +); +use constant VALIDATORS => { + connector => \&_check_connector, +}; +use constant LIST_ORDER => 'connector'; + +# +# accessors +# + +sub connector { return $_[0]->{'connector'}; } +sub name { return $_[0]->{'option_name'}; } +sub value { return $_[0]->{'option_value'}; } + +# +# mutators +# + +sub set_value { $_[0]->{'option_value'} = $_[1]; } + +# +# validators +# + +sub _check_connector { + my ($invocant, $value) = @_; + $value eq '*' + || $value eq 'global' + || Bugzilla->push_ext->connectors->exists($value) + || ThrowCodeError('push_invalid_connector'); + return $value; +} + +1; + diff --git a/extensions/Push/lib/Push.pm b/extensions/Push/lib/Push.pm new file mode 100644 index 000000000..aaac0bbd6 --- /dev/null +++ b/extensions/Push/lib/Push.pm @@ -0,0 +1,264 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Push; + +use strict; +use warnings; + +use Bugzilla::Extension::Push::BacklogMessage; +use Bugzilla::Extension::Push::Config; +use Bugzilla::Extension::Push::Connectors; +use Bugzilla::Extension::Push::Constants; +use Bugzilla::Extension::Push::Log; +use Bugzilla::Extension::Push::Logger; +use Bugzilla::Extension::Push::Message; +use Bugzilla::Extension::Push::Option; +use Bugzilla::Extension::Push::Queue; +use Bugzilla::Extension::Push::Util; +use DateTime; + +sub new { + my ($class) = @_; + my $self = {}; + bless($self, $class); + $self->{is_daemon} = 0; + return $self; +} + +sub is_daemon { + my ($self, $value) = @_; + if (defined $value) { + $self->{is_daemon} = $value ? 1 : 0; + } + return $self->{is_daemon}; +} + +sub start { + my ($self) = @_; + my $connectors = $self->connectors; + $self->{config_last_modified} = $self->get_config_last_modified(); + $self->{config_last_checked} = (time); + + foreach my $connector ($connectors->list) { + $connector->backlog->reset_backoff(); + } + + while(1) { + if ($self->_dbh_check()) { + $self->_reload(); + $self->push(); + } + sleep(POLL_INTERVAL_SECONDS); + } +} + +sub push { + my ($self) = @_; + my $logger = $self->logger; + my $connectors = $self->connectors; + + my $enabled = 0; + foreach my $connector ($connectors->list) { + if ($connector->enabled) { + $enabled = 1; + last; + } + } + return unless $enabled; + + $logger->debug("polling"); + + # process each message + while(my $message = $self->queue->oldest) { + foreach my $connector ($connectors->list) { + next unless $connector->enabled; + next unless $connector->should_send($message); + $logger->debug("pushing to " . $connector->name); + + my $is_backlogged = $connector->backlog->count; + + if (!$is_backlogged) { + # connector isn't backlogged, immediate send + $logger->debug("immediate send"); + my ($result, $data); + eval { + ($result, $data) = $connector->send($message); + }; + if ($@) { + $result = PUSH_RESULT_TRANSIENT; + $data = clean_error($@); + } + if (!$result) { + $logger->error($connector->name . " failed to return a result code"); + $result = PUSH_RESULT_UNKNOWN; + } + $logger->result($connector, $message, $result, $data); + + if ($result == PUSH_RESULT_TRANSIENT) { + $is_backlogged = 1; + } + } + + # if the connector is backlogged, push to the backlog queue + if ($is_backlogged) { + $logger->debug("backlogged"); + my $backlog = Bugzilla::Extension::Push::BacklogMessage->create_from_message($message, $connector); + } + } + + # message processed + $message->remove_from_db(); + } + + # process backlog + foreach my $connector ($connectors->list) { + next unless $connector->enabled; + my $message = $connector->backlog->oldest(); + next unless $message; + + $logger->debug("processing backlog for " . $connector->name); + while ($message) { + my ($result, $data); + eval { + ($result, $data) = $connector->send($message); + }; + if ($@) { + $result = PUSH_RESULT_TRANSIENT; + $data = $@; + } + $message->inc_attempts($result == PUSH_RESULT_OK ? '' : $data); + if (!$result) { + $logger->error($connector->name . " failed to return a result code"); + $result = PUSH_RESULT_UNKNOWN; + } + $logger->result($connector, $message, $result, $data); + + if ($result == PUSH_RESULT_TRANSIENT) { + # connector is still down, stop trying + $connector->backlog->inc_backoff(); + last; + } + + # message was processed + $message->remove_from_db(); + + $message = $connector->backlog->oldest(); + } + } +} + +sub _reload { + my ($self) = @_; + + # check for updated config every 60 seconds + my $now = (time); + if ($now - $self->{config_last_checked} < 60) { + return; + } + $self->{config_last_checked} = $now; + + $self->logger->debug('Checking for updated configuration'); + if ($self->get_config_last_modified eq $self->{config_last_modified}) { + return; + } + $self->{config_last_modified} = $self->get_config_last_modified(); + + $self->logger->debug('Configuration has been updated'); + $self->connectors->reload(); +} + +sub get_config_last_modified { + my ($self) = @_; + my $options_list = Bugzilla::Extension::Push::Option->match({ + connector => '*', + option_name => 'last-modified', + }); + if (@$options_list) { + return $options_list->[0]->value; + } else { + return $self->set_config_last_modified(); + } +} + +sub set_config_last_modified { + my ($self) = @_; + my $options_list = Bugzilla::Extension::Push::Option->match({ + connector => '*', + option_name => 'last-modified', + }); + my $now = DateTime->now->datetime(); + if (@$options_list) { + $options_list->[0]->set_value($now); + $options_list->[0]->update(); + } else { + Bugzilla::Extension::Push::Option->create({ + connector => '*', + option_name => 'last-modified', + option_value => $now, + }); + } + return $now; +} + +sub config { + my ($self) = @_; + if (!$self->{config}) { + $self->{config} = Bugzilla::Extension::Push::Config->new( + 'global', + { + name => 'log_purge', + label => 'Purge logs older than (days)', + type => 'string', + default => '7', + required => '1', + validate => sub { $_[0] =~ /\D/ && die "Invalid purge duration (must be numeric)\n"; }, + }, + ); + $self->{config}->load(); + } + return $self->{config}; +} + +sub logger { + my ($self, $value) = @_; + $self->{logger} = $value if $value; + return $self->{logger}; +} + +sub connectors { + my ($self, $value) = @_; + $self->{connectors} = $value if $value; + return $self->{connectors}; +} + +sub queue { + my ($self) = @_; + $self->{queue} ||= Bugzilla::Extension::Push::Queue->new(); + return $self->{queue}; +} + +sub log { + my ($self) = @_; + $self->{log} ||= Bugzilla::Extension::Push::Log->new(); + return $self->{log}; +} + +sub _dbh_check { + my ($self) = @_; + eval { + Bugzilla->dbh->selectrow_array("SELECT 1 FROM push"); + }; + if ($@) { + $self->logger->error(clean_error($@)); + return 0; + } else { + return 1; + } +} + +1; diff --git a/extensions/Push/lib/Queue.pm b/extensions/Push/lib/Queue.pm new file mode 100644 index 000000000..d89cb23c3 --- /dev/null +++ b/extensions/Push/lib/Queue.pm @@ -0,0 +1,72 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Queue; + +use strict; +use warnings; + +use Bugzilla; +use Bugzilla::Extension::Push::Message; + +sub new { + my ($class) = @_; + my $self = {}; + bless($self, $class); + return $self; +} + +sub count { + my ($self) = @_; + my $dbh = Bugzilla->dbh; + return $dbh->selectrow_array("SELECT COUNT(*) FROM push"); +} + +sub oldest { + my ($self) = @_; + my @messages = $self->list(limit => 1); + return scalar(@messages) ? $messages[0] : undef; +} + +sub by_id { + my ($self, $id) = @_; + my @messages = $self->list( + limit => 1, + filter => "AND (push.id = $id)", + ); + return scalar(@messages) ? $messages[0] : undef; +} + +sub list { + my ($self, %args) = @_; + $args{limit} ||= 10; + $args{filter} ||= ''; + my @result; + my $dbh = Bugzilla->dbh; + + my $sth = $dbh->prepare(" + SELECT id, push_ts, payload, change_set, routing_key + FROM push + WHERE (1 = 1) " . + $args{filter} . " + ORDER BY push_ts " . + $dbh->sql_limit($args{limit}) + ); + $sth->execute(); + while (my $row = $sth->fetchrow_hashref()) { + push @result, Bugzilla::Extension::Push::Message->new({ + id => $row->{id}, + push_ts => $row->{push_ts}, + payload => $row->{payload}, + change_set => $row->{change_set}, + routing_key => $row->{routing_key}, + }); + } + return @result; +} + +1; diff --git a/extensions/Push/lib/Serialise.pm b/extensions/Push/lib/Serialise.pm new file mode 100644 index 000000000..82eb654b5 --- /dev/null +++ b/extensions/Push/lib/Serialise.pm @@ -0,0 +1,322 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Serialise; + +use strict; +use warnings; + +use Bugzilla::Constants; +use Bugzilla::Extension::Push::Util; +use Bugzilla::Version; + +use Scalar::Util 'blessed'; +use JSON (); + +my $_instance; +sub instance { + $_instance ||= Bugzilla::Extension::Push::Serialise->_new(); + return $_instance; +} + +sub _new { + my ($class) = @_; + my $self = {}; + bless($self, $class); + return $self; +} + +# given an object, serliase to a hash +sub object_to_hash { + my ($self, $object, $is_shallow) = @_; + + my $method = lc(blessed($object)); + $method =~ s/::/_/g; + $method =~ s/^bugzilla//; + return unless $self->can($method); + (my $name = $method) =~ s/^_//; + + # check for a cached hash + my $cache = Bugzilla->request_cache; + my $cache_id = "push." . ($is_shallow ? 'shallow.' : 'deep.') . $object; + if (exists($cache->{$cache_id})) { + return wantarray ? ($cache->{$cache_id}, $name) : $cache->{$cache_id}; + } + + # call the right method to serialise to a hash + my $rh = $self->$method($object, $is_shallow); + + # store in cache + if ($cache_id) { + $cache->{$cache_id} = $rh; + } + + return wantarray ? ($rh, $name) : $rh; +} + +# given a changes hash, return an event hash +sub changes_to_event { + my ($self, $changes) = @_; + + my $event = {}; + + # create common (created and modified) fields + $event->{'user'} = $self->object_to_hash(Bugzilla->user); + my $timestamp = + $changes->{'timestamp'} + || Bugzilla->dbh->selectrow_array('SELECT LOCALTIMESTAMP(0)'); + $event->{'time'} = datetime_to_timestamp($timestamp); + + foreach my $change (@{$changes->{'changes'}}) { + if (exists $change->{'field'}) { + # map undef to emtpy + hash_undef_to_empty($change); + + # custom_fields change from undef to empty, ignore these changes + return if ($change->{'added'} || "") eq "" && + ($change->{'removed'} || "") eq ""; + + # use saner field serialisation + my $field = $change->{'field'}; + $change->{'field'} = $field; + + if ($field eq 'priority' || $field eq 'target_milestone') { + $change->{'added'} = _select($change->{'added'}); + $change->{'removed'} = _select($change->{'removed'}); + + } elsif ($field =~ /^cf_/) { + $change->{'added'} = _custom_field($field, $change->{'added'}); + $change->{'removed'} = _custom_field($field, $change->{'removed'}); + } + + $event->{'changes'} = [] unless exists $event->{'changes'}; + push @{$event->{'changes'}}, $change; + } + } + + return $event; +} + +# bugzilla returns '---' or '--' for single-select fields that have no value +# selected. it makes more sense to return an empty string. +sub _select { + my ($value) = @_; + return '' if $value eq '---' or $value eq '--'; + return $value; +} + +# return an object which serialises to a json boolean, but still acts as a perl +# boolean +sub _boolean { + my ($value) = @_; + return $value ? JSON::true : JSON::false; +} + +sub _string { + my ($value) = @_; + return defined($value) ? $value : ''; +} + +sub _time { + my ($value) = @_; + return defined($value) ? datetime_to_timestamp($value) : undef; +} + +sub _integer { + my ($value) = @_; + return defined($value) ? $value + 0 : undef; +} + +sub _array { + my ($value) = @_; + return defined($value) ? $value : []; +} + +sub _custom_field { + my ($field, $value) = @_; + $field = Bugzilla::Field->new({ name => $field }) unless blessed $field; + + if ($field->type == FIELD_TYPE_DATETIME) { + return _time($value); + + } elsif ($field->type == FIELD_TYPE_SINGLE_SELECT) { + return _select($value); + + } elsif ($field->type == FIELD_TYPE_MULTI_SELECT) { + return _array($value); + + } else { + return _string($value); + } +} + +# +# class mappings +# automatically derrived from the class name +# Bugzilla::Bug --> _bug, Bugzilla::User --> _user, etc +# + +sub _bug { + my ($self, $bug) = @_; + + my $version = $bug->can('version_obj') + ? $bug->version_obj + : Bugzilla::Version->new({ name => $bug->version, product => $bug->product_obj }); + + my $milestone; + if (_select($bug->target_milestone) ne '') { + $milestone = $bug->can('target_milestone_obj') + ? $bug->target_milestone_obj + : Bugzilla::Milestone->new({ name => $bug->target_milestone, product => $bug->product_obj }); + } + + my $status = $bug->can('status_obj') + ? $bug->status_obj + : Bugzilla::Status->new({ name => $bug->bug_status }); + + my $rh = { + id => _integer($bug->bug_id), + alias => _string($bug->alias), + assigned_to => $self->_user($bug->assigned_to), + classification => _string($bug->classification), + component => $self->_component($bug->component_obj), + creation_time => _time($bug->creation_ts || $bug->delta_ts), + flags => (mapr { $self->_flag($_) } $bug->flags), + is_private => _boolean(!is_public($bug)), + keywords => (mapr { _string($_->name) } $bug->keyword_objects), + last_change_time => _time($bug->delta_ts), + operating_system => _string($bug->op_sys), + platform => _string($bug->rep_platform), + priority => _select($bug->priority), + product => $self->_product($bug->product_obj), + qa_contact => $self->_user($bug->qa_contact), + reporter => $self->_user($bug->reporter), + resolution => _string($bug->resolution), + severity => _string($bug->bug_severity), + status => $self->_status($status), + summary => _string($bug->short_desc), + target_milestone => $self->_milestone($milestone), + url => _string($bug->bug_file_loc), + version => $self->_version($version), + whiteboard => _string($bug->status_whiteboard), + }; + + # add custom fields + my @custom_fields = Bugzilla->active_custom_fields; + foreach my $field (@custom_fields) { + my $name = $field->name; + + # skip custom fields that are hidded from this product/component + next if Bugzilla::Extension::BMO::cf_hidden_in_product( + $name, $bug->product, $bug->component); + + $rh->{$name} = _custom_field($field, $bug->$name); + } + + return $rh; +} + +sub _user { + my ($self, $user) = @_; + return undef unless $user; + return { + id => _integer($user->id), + login => _string($user->login), + real_name => _string($user->name), + }; +} + +sub _component { + my ($self, $component) = @_; + return { + id => _integer($component->id), + name => _string($component->name), + }; +} + +sub _attachment { + my ($self, $attachment, $is_shallow) = @_; + my $rh = { + id => _integer($attachment->id), + content_type => _string($attachment->contenttype), + creation_time => _time($attachment->attached), + description => _string($attachment->description), + file_name => _string($attachment->filename), + flags => (mapr { $self->_flag($_) } $attachment->flags), + is_obsolete => _boolean($attachment->isobsolete), + is_patch => _boolean($attachment->ispatch), + is_private => _boolean(!is_public($attachment)), + last_change_time => _time($attachment->modification_time), + }; + if (!$is_shallow) { + $rh->{bug} = $self->_bug($attachment->bug); + } + return $rh; +} + +sub _comment { + my ($self, $comment, $is_shallow) = @_; + my $rh = { + id => _integer($comment->bug_id), + body => _string($comment->body), + creation_time => _time($comment->creation_ts), + is_private => _boolean($comment->is_private), + number => _integer($comment->count), + }; + if (!$is_shallow) { + $rh->{bug} = $self->_bug($comment->bug); + } + return $rh; +} + +sub _product { + my ($self, $product) = @_; + return { + id => _integer($product->id), + name => _string($product->name), + }; +} + +sub _flag { + my ($self, $flag) = @_; + my $rh = { + id => _integer($flag->id), + name => _string($flag->type->name), + value => _string($flag->status), + }; + if ($flag->requestee) { + $rh->{'requestee'} = $self->_user($flag->requestee); + } + return $rh; +} + +sub _version { + my ($self, $version) = @_; + return { + id => _integer($version->id), + name => _string($version->name), + }; +} + +sub _milestone { + my ($self, $milestone) = @_; + return undef unless $milestone; + return { + id => _integer($milestone->id), + name => _string($milestone->name), + }; +} + +sub _status { + my ($self, $status) = @_; + return { + id => _integer($status->id), + name => _string($status->name), + }; +} + +1; diff --git a/extensions/Push/lib/Util.pm b/extensions/Push/lib/Util.pm new file mode 100644 index 000000000..f52db6936 --- /dev/null +++ b/extensions/Push/lib/Util.pm @@ -0,0 +1,162 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Extension::Push::Util; + +use strict; +use warnings; + +use Bugzilla; +use Bugzilla::Constants; +use Bugzilla::Util qw(datetime_from trim); +use Data::Dumper; +use Encode; +use JSON (); +use Scalar::Util qw(blessed); +use Time::HiRes; + +use base qw(Exporter); +our @EXPORT = qw( + datetime_to_timestamp + debug_dump + get_first_value + hash_undef_to_empty + is_public + mapr + clean_error + change_set_id + canon_email + to_json from_json +); + +# returns true if the specified object is public +sub is_public { + my ($object) = @_; + + my $default_user = Bugzilla::User->new(); + + if ($object->isa('Bugzilla::Bug')) { + return unless $default_user->can_see_bug($object->bug_id); + return 1; + + } elsif ($object->isa('Bugzilla::Comment')) { + return if $object->is_private; + return unless $default_user->can_see_bug($object->bug_id); + return 1; + + } elsif ($object->isa('Bugzilla::Attachment')) { + return if $object->isprivate; + return unless $default_user->can_see_bug($object->bug_id); + return 1; + + } else { + warn "Unsupported class " . blessed($object) . " passed to is_public()\n"; + } + + return 1; +} + +# return the first existing value from the hashref for the given list of keys +sub get_first_value { + my ($rh, @keys) = @_; + foreach my $field (@keys) { + return $rh->{$field} if exists $rh->{$field}; + } + return; +} + +# wrapper for map that works on array references +sub mapr(&$) { + my ($filter, $ra) = @_; + my @result = map(&$filter, @$ra); + return \@result; +} + + +# convert datetime string (from db) to a UTC json friendly datetime +sub datetime_to_timestamp { + my ($datetime_string) = @_; + return '' unless $datetime_string; + return datetime_from($datetime_string, 'UTC')->datetime(); +} + +# replaces all undef values in a hashref with an empty string (deep) +sub hash_undef_to_empty { + my ($rh) = @_; + foreach my $key (keys %$rh) { + my $value = $rh->{$key}; + if (!defined($value)) { + $rh->{$key} = ''; + } elsif (ref($value) eq 'HASH') { + hash_undef_to_empty($value); + } + } +} + +# debugging methods +sub debug_dump { + my ($object) = @_; + local $Data::Dumper::Sortkeys = 1; + my $output = Dumper($object); + $output =~ s/</</g; + print "<pre>$output</pre>"; +} + +# removes stacktrace and "at /some/path ..." from errors +sub clean_error { + my ($error) = @_; + my $path = bz_locations->{'extensionsdir'}; + $error = $1 if $error =~ /^(.+?) at \Q$path/s; + $path = '/loader/0x'; + $error = $1 if $error =~ /^(.+?) at \Q$path/s; + $error =~ s/(^\s+|\s+$)//g; + return $error; +} + +# generate a new change_set id +sub change_set_id { + return "$$." . Time::HiRes::time(); +} + +# remove guff from email addresses +sub clean_email { + my $email = shift; + $email = trim($email); + $email = $1 if $email =~ /^(\S+)/; + $email =~ s/@/@/; + $email = lc $email; + return $email; +} + +# resolve to canonised email form +# eg. glob+bmo@mozilla.com --> glob@mozilla.com +sub canon_email { + my $email = shift; + $email = clean_email($email); + $email =~ s/^([^\+]+)\+[^\@]+(\@.+)$/$1$2/; + return $email; +} + +# json helpers +sub to_json { + my ($object, $pretty) = @_; + if ($pretty) { + return decode('utf8', JSON->new->utf8(1)->pretty(1)->encode($object)); + } else { + return JSON->new->ascii(1)->shrink(1)->encode($object); + } +} + +sub from_json { + my ($json) = @_; + if (utf8::is_utf8($json)) { + $json = encode('utf8', $json); + } + return JSON->new->utf8(1)->decode($json); +} + +1; |