diff options
| -rw-r--r-- | Bugzilla/Migrate/Flyspray.pm | 528 |
1 files changed, 528 insertions, 0 deletions
diff --git a/Bugzilla/Migrate/Flyspray.pm b/Bugzilla/Migrate/Flyspray.pm new file mode 100644 index 000000000..84dc247ca --- /dev/null +++ b/Bugzilla/Migrate/Flyspray.pm @@ -0,0 +1,528 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Migrate::Flyspray; + +use 5.14.0; +use strict; +use warnings; + +use parent qw(Bugzilla::Migrate); + +use Bugzilla::Constants; +use Bugzilla::Util qw(validate_email_syntax); +use Bugzilla::DB; + +use IO::File; +use List::Util qw(first none); +use SQL::Abstract; + +#use constant REQUIRED_MODULES => [ +#{ +#package => 'SQL-Abstract', +#module => 'SQL::Abstract', +#version => 0, +#}, +#]; + +use constant FIELD_MAP => { + 'BugID' => 'bug_id', + 'Category' => 'product', + 'Subject' => 'short_desc', + 'Responsible' => 'assigned_to', + 'State' => 'bug_status', + 'Class' => 'cf_type', + 'Classification' => '', + 'Originator' => 'reporter', + 'CreationTS' => 'creation_ts', + 'Last-Modified' => 'delta_ts', + 'Release' => 'version', + 'Severity' => 'bug_severity', + 'comment' => 'comment', + 'comments' => 'comments', + 'history' => 'history', + 'attachments' => 'attachments', +}; + +use constant VALUE_MAP => { + bug_severity => { + 'critical' => 'critical', + 'high' => 'high', + 'medium' => 'medium', + 'low' => 'low', + 'very low' => 'very low', + }, + bug_status => { + 'unconfirmed' => 'NEW', + 'new' => 'NEW', + 'unassigned' => 'NEW', + 'assigned' => 'CONFIRMED', + 'researching' => 'CONFIRMED', + 'waiting on response' => 'WAITING', + 'requires testing' => 'WAITING', + }, + resolution => { + 'no response' => 'NORESPONSE', + 'upstream' => 'UPSTREAM', + 'not a bug' => 'NOTABUG', + 'won\'t fix' => 'WONTFIX', + 'won\'t implement' => 'WONTFIX', + 'works for me' => 'WORKSFORME', + 'duplicate' => 'DUPLICATE', + 'deferred' => 'DEFERRED', + 'fixed' => 'FIXED', + 'implemented' => 'FIXED', + }, + bug_severity => { + 'Flash' => 'Flash', + 'Immediate' => 'Immediate', + 'Urgent' => 'Urgent', + 'High' => 'High', + 'Normal' => 'Normal', + 'Low' => 'Low', + }, +}; + +use constant FLYSPRAY_CONFIG_VARS => ( + { + name => 'flyspray_db', + default => 'bugzilla_flyspray', + desc => <<END, +# The database name of the flyspray databse. +END + }, + { + name => 'flyspray_prefix', + default => 'flyspray_', + desc => <<'END', +# The prefix of the flyspray database tables. +END + }, + { + name => 'component_owner', + default => 'bluewind', + desc => <<'END', +# The username of the user that is set as the initial owner for components. +END + }, + { + name => 'unknown_user_email', + default => 'nobody@localhost.localdomain', + desc => <<'END', +# The email address for the fallback user if a task is created by/assigned to an unknown user. +END + }, + { + name => 'attachment_path', + default => './flyspray-attachments', + desc => <<'END', +# The path of the flyspray attachments directory (or a copy). +END + }, +); + +sub CONFIG_VARS { + my $self = shift; + my @vars = (FLYSPRAY_CONFIG_VARS, $self->SUPER::CONFIG_VARS); + my $field_map = first { $_->{name} eq 'translate_fields' } @vars; + $field_map->{default} = FIELD_MAP; + my $value_map = first { $_->{name} eq 'translate_values' } @vars; + $value_map->{default} = VALUE_MAP; + return @vars; +} + +use constant NON_COMMENT_FIELDS => qw( +); + +sub new { + my ($class) = @_; + my $self = {}; + bless $self, $class; + $self->{dbh} = Bugzilla->dbh; + $self->{sql} = SQL::Abstract->new; + return $self; +} + +sub _select_fs { + my ($self, $table, $fields, $where) = @_; + + my @fields = qw(category_name shot_in_list); + my $db = $self->config('flyspray_db'); + my ($stmt, @bind) = $self->{sql}->select( + [ + map { + sprintf("%s.%s%s", + $self->config('flyspray_db'), + $self->config('flyspray_prefix'), $_) + } @$table + ], + $fields, + $where + ); + $self->debug($stmt, 2); + $self->debug(\@bind, 2); + my $sth = $self->{dbh}->prepare($stmt); + $sth->execute(@bind); + + return $sth; +} + +sub _read_users { + my $self = shift; + + my $sth = $self->_select_fs(["users"], + [qw(user_id user_name real_name email_address)]); + + my @result; + while (my $row = $sth->fetchrow_hashref()) { + next unless validate_email_syntax($row->{email_address}); + my $item = { + login_name => $row->{user_name} =~ s/\s//gr =~ s/@//gr, + realname => $row->{real_name}, + email => $row->{email_address}, + cryptpassword => "*", + }; + push @result, $item; + $self->{map_user_id_to_email}->{$row->{user_id}} = $item->{email}; + $self->{map_email_to_name}->{$row->{email_address}} = $item->{login_name}; + } + + # Remove duplicate users based on email; keep the latesst account + my %seen; + @result = reverse grep { !$seen{$_->{email}}++ } reverse @result; + + # fallback user for cases where the real user can not be + # created or has been deleted + push @result, + { + login_name => 'UnknownUser', + realname => 'Unknown User', + email => $self->config("unknown_user_email"), + cryptpassword => "*", + is_enabled => 0, + }; + $self->{map_user_id_to_email}->{-1} = $self->config("unknown_user_email"); + $self->{map_email_to_name}->{$self->config("unknown_user_email")} + = 'UnknownUser'; + + $self->debug(\@result, 3); + + #$self->debug([grep { $_->{email} eq 'xx@x.com' } @result], 2); + $self->debug("Migrating " . (@result + 0) . " users"); + return \@result; +} + +sub _read_products { + my $self = shift; + + my $sth = $self->_select_fs(["projects"], + [qw(project_id project_title intro_message)]); + + my @result; + while (my $row = $sth->fetchrow_hashref()) { + push @result, + { + name => $row->{project_title}, + description => $row->{intro_message} || "N/A", + isactive => 1, + components => $self->_get_fs_project_components($row->{project_id}), + version => "unspecified", + }; + $self->{map_project_id_to_name}->{$row->{project_id}} = $row->{project_title}; + } + + $self->debug(\@result, 3); + $self->debug("Migrating " . (@result + 0) . " products"); + return \@result; +} + +sub _get_fs_project_components { + my $self = shift; + my $project_id = shift; + + my $sth = $self->_select_fs( + ["list_category"], + [qw(category_id category_name show_in_list)], + {project_id => $project_id} + ); + + my @result; + while (my $row = $sth->fetchrow_hashref()) { + push @result, + { + name => $row->{category_name}, + description => "N/A", + initialowner => $self->config("component_owner"), + isactive => 1, + }; + $self->{map_component_id_to_name}->{$row->{category_id}} + = $row->{category_name}; + $self->{project_id_components}->{$project_id}->{$row->{category_name}} = 1; + } + push @result, + { + name => "Uncategorized", + description => "N/A", + initialowner => $self->config("component_owner"), + isactive => 1, + }; + $self->{map_component_id_to_name}->{-1} = "Uncategorized"; + + $self->debug(\@result, 3); + $self->debug("Found " . (@result + 0) . " components in project $project_id"); + return \@result; +} + +sub _read_bugs { + my $self = shift; + my $fsdb = $self->config("flyspray_db"); + my $fsprfx = $self->config("flyspray_prefix"); + +#my $sth = $self->_select_fs([qw(tasks users)], [qw(task_id product_category project_id item_summary detailed_desc user_name)], {'tasks.opened_by' => \' = users.user_id' }); + my $sth = $self->{dbh}->prepare(" + SELECT + t.task_id, + t.product_category, + t.project_id, + t.item_summary, + t.detailed_desc, + t.opened_by, + t.task_priority, + t.item_status, + t.date_opened, + t.last_edited_time, + a.user_id assigned_to, + ls.status_name, + t.is_closed, + lr.resolution_name, + lv.version_name + FROM $fsdb.${fsprfx}tasks t + LEFT OUTER JOIN $fsdb.${fsprfx}assigned a ON a.task_id = t.task_id + LEFT OUTER JOIN $fsdb.${fsprfx}list_status ls ON ls.status_id = t.item_status + LEFT OUTER JOIN $fsdb.${fsprfx}list_resolution lr ON lr.resolution_id = t.resolution_reason + LEFT OUTER JOIN $fsdb.${fsprfx}list_version lv ON lv.version_id = t.product_version + WHERE t.task_id IN (23884, 22102) + GROUP BY t.task_id, a.task_id + "); + $sth->execute(); + + my @result; + while (my $row = $sth->fetchrow_hashref()) { + my $item = { + short_desc => $row->{item_summary}, + product => $self->{map_project_id_to_name}->{$row->{project_id}}, + component => $self->{map_component_id_to_name}->{$row->{product_category}} + // $self->{map_component_id_to_name}->{-1}, + bug_id => $row->{task_id}, + comments => [], + history => [], + attachments => [], + assigned_to => $self->_get_username($row->{assigned_to}), + op_sys => "Linux", + rep_platform => "PC", + bug_status => $row->{status_name} // "Unknown", + bug_severity => $row->{task_priority}, + resolution => $row->{is_closed} ? $row->{resolution_name} // 'UNKNOWN' : '', + reporter => $self->_get_username($row->{opened_by}), + creation_ts => $row->{date_opened}, + delta_ts => $row->{last_edited_time}, + version => $row->{version_name} // "unspecified", + comment => $row->{detailed_desc}, + }; + $item->{version} = "unspecified" if $item->{version} eq "None"; + $item->{resolution} = "UNSPECIFIED" if $item->{resolution} eq "None"; + $item->{bug_status} = "RESOLVED" if $item->{resolution} ne ""; + $item->{component} = $self->{map_component_id_to_name}->{-1} + if not defined $self->{project_id_components}->{$row->{project_id}} + ->{$item->{component}}; + + # handle initial comment/description + if (length($item->{comment}) > 65535) { + $item->{comment} + = "--- The description has been moved to an attachment because it is longer than 65535 characters ---"; + unshift @{$item->{attachments}}, + { + data => $row->{detailed_desc}, + description => "Initial bug description", + mimetype => "text/plain", + filename => "description.txt", + creation_ts => $self->parse_date($row->{date_opened}), + }; + } + + my $comments = {}; + + # fetch comments + my $comments_sth = $self->{dbh}->prepare_cached(" + SELECT comment_id, user_id, date_added, comment_text + FROM $fsdb.${fsprfx}comments WHERE task_id = ?"); + $comments_sth->execute($row->{task_id}); + while (my $comment = $comments_sth->fetchrow_hashref()) { + my $comment_item = { + who => $self->_get_username($comment->{user_id}), + bug_when => $self->parse_date($comment->{date_added}), + thetext => $comment->{comment_text}, + }; + push @{$item->{comments}}, $comment_item; + $comments->{$comment->{comment_id}} = $comment_item; + } + + # get history + my $history_sth = $self->{dbh}->prepare_cached(" + SELECT user_id, + event_type, + event_date, + field_changed, + old_value, + new_value, + lr.resolution_name, + lsn.status_name status_name_new, + lso.status_name status_name_old + FROM $fsdb.${fsprfx}history + LEFT OUTER JOIN $fsdb.${fsprfx}list_resolution lr ON lr.resolution_id = new_value + LEFT OUTER JOIN $fsdb.${fsprfx}list_status lsn ON lsn.status_id = new_value + LEFT OUTER JOIN $fsdb.${fsprfx}list_status lso ON lso.status_id = old_value + WHERE task_id = ?"); + $history_sth->execute($row->{task_id}); + while (my $history = $history_sth->fetchrow_hashref()) { + if ($history->{event_type} == 2) { + + # type 2 -> closed bug + my $closure_comment = $history->{old_value}; + if ($closure_comment ne "") { + push @{$item->{comments}}, + { + who => $self->_get_username($history->{user_id}), + bug_when => $self->parse_date($history->{event_date}), + thetext => $closure_comment, + }; + } + + push @{$item->{history}}, + { + who => $self->_get_username($history->{user_id}), + bug_when => $self->parse_date($history->{event_date}), + field => "resolution", + removed => "", + added => $history->{resolution_name}, + }; + } + elsif ($history->{event_type} == 3) { + + # type 3 -> field changed + if ($history->{field_changed} eq "item_status") { + push @{$item->{history}}, + { + who => $self->_get_username($history->{user_id}), + bug_when => $self->parse_date($history->{event_date}), + field => "bug_status", + removed => $history->{status_name_old} // "", + added => $history->{status_name_new}, + }; + } + + # TODO Add more field type mappings + } + + # TODO Add more events? + # optional for now: CC changes, assignee changes, milestone/version changes + } + + # TODO get attachments + my $attachments_sth = $self->{dbh}->prepare_cached(" + SELECT task_id, orig_name, file_name, file_desc, file_type, file_size, added_by, date_added + FROM $fsdb.${fsprfx}attachments + WHERE task_id = ? + "); + $attachments_sth->execute($row->{task_id}); + my $attpath = $self->config("attachment_path"); + while (my $attachment = $attachments_sth->fetchrow_hashref()) { + push @{$item->{attachments}}, + { + submitter => $self->_get_username($attachment->{added_by}), + creation_ts => $attachment->{date_added}, + description => $attachment->{file_desc} || $attachment->{orig_name}, + filename => $attachment->{orig_name}, + mimetype => $attachment->{file_type}, + ispatch => $attachment->{file_type} =~ m/^text\/x-diff/ ? 1 : 0, + data => IO::File->new($attpath . '/' . $attachment->{file_name}, 'r'), + }; + + # TODO: set extra_data and type of comment when attachment is part of a comment. + # problem: how to get attachment id? not yet in db + # change _insert_attachments/_insert_comments? + } + + push @result, $item; + } + + #$self->debug(\@result, 3); + #@result = grep { $_->{bug_id} == 56677 } @result; + #$self->debug([grep { $_->{bug_id} == 56677 } @result], 2); + #$self->debug([keys %{{map { $_->{version} => 1 } @result}}], 2); + #die; + $self->debug("Migrating " . (@result + 0) . " bugs"); + + # Ensure bug IDs are kept by preventing the validator from unsetting them + my $orig_validator = \&Bugzilla::Bug::run_create_validators; + *{Bugzilla::Bug::run_create_validators} = sub { + my ($class, $params) = @_; + my $bug_id = $params->{bug_id}; + my $ret = $orig_validator->(@_); + $ret->{bug_id} = $bug_id; + return $ret; + }; + + # insert_create_data is not implemented in Bugzilla::Bug + #my $orig_create = \&Bugzilla::Bug::insert_create_data; + my $orig_create = \&Bugzilla::Object::insert_create_data; + *{Bugzilla::Bug::insert_create_data} = sub { + my ($class, $params) = @_; + my $bug_id = $params->{bug_id}; + $orig_create->(@_); + +# XXX: The next lines are copied from Bugzilla::Bug::insert_create_data and may need to be adjusted in the future! +# XXX: They also trigger hooks/audit again which is not really nice + my $object = $class->new($bug_id); + Bugzilla::Hook::process('object_end_of_create', + {class => $class, object => $object}); + $object->audit_log(AUDIT_CREATE) if $object->AUDIT_CREATES; + return $object; + }; + + return \@result; +} + +sub _get_username { + my ($self, $userid) = @_; + my $email = $self->{map_user_id_to_email}->{$userid // -1} + // $self->{map_user_id_to_email}->{-1}; + return $self->{map_email_to_name}->{$email}; +} + + +sub translate_bug { + my ($self, $fields) = @_; + + my ($bug, $other_fields) = $self->SUPER::translate_bug($fields); + + #TODO remove this function if not used + + return wantarray ? ($bug, $other_fields) : $bug; +} + + +# Timestamps from flyspray are unix timestamps which Date::Parse +# can't parse so the original parse_date() sub will just die +sub parse_date { + my ($self, $date) = @_; + my $dt = DateTime->from_epoch(epoch => $date); + $dt->set_time_zone(Bugzilla->local_timezone); + return $dt->iso8601; +} + +1; |