diff options
Diffstat (limited to 'Bugzilla')
113 files changed, 12167 insertions, 1389 deletions
diff --git a/Bugzilla/Attachment.pm b/Bugzilla/Attachment.pm index fa8845358..1302fc716 100644 --- a/Bugzilla/Attachment.pm +++ b/Bugzilla/Attachment.pm @@ -22,10 +22,11 @@ # Marc Schumann <wurblzap@gmail.com> # Frédéric Buclin <LpSolit@gmail.com> -use strict; - package Bugzilla::Attachment; +use 5.10.0; +use strict; + =head1 NAME Bugzilla::Attachment - Bugzilla attachment class. @@ -61,6 +62,7 @@ use Bugzilla::Hook; use File::Copy; use List::Util qw(max); +use Storable qw(dclone); use base qw(Bugzilla::Object); @@ -75,22 +77,19 @@ use constant LIST_ORDER => ID_FIELD; use constant AUDIT_CREATES => 0; use constant AUDIT_UPDATES => 0; -sub DB_COLUMNS { - my $dbh = Bugzilla->dbh; - - return qw( - attach_id - bug_id - description - filename - isobsolete - ispatch - isprivate - mimetype - modification_time - submitter_id), - $dbh->sql_date_format('attachments.creation_ts', '%Y.%m.%d %H:%i') . ' AS creation_ts'; -} +use constant DB_COLUMNS => qw( + attach_id + bug_id + creation_ts + description + filename + isobsolete + ispatch + isprivate + mimetype + modification_time + submitter_id +); use constant REQUIRED_FIELD_MAP => { bug_id => 'bug', @@ -116,7 +115,8 @@ use constant VALIDATORS => { }; use constant VALIDATOR_DEPENDENCIES => { - mimetype => ['ispatch'], + content_type => ['ispatch'], + mimetype => ['ispatch'], }; use constant UPDATE_VALIDATORS => { @@ -142,8 +142,7 @@ the ID of the bug to which the attachment is attached =cut sub bug_id { - my $self = shift; - return $self->{bug_id}; + return $_[0]->{bug_id}; } =over @@ -157,11 +156,8 @@ the bug object to which the attachment is attached =cut sub bug { - my $self = shift; - require Bugzilla::Bug; - $self->{bug} ||= Bugzilla::Bug->new($self->bug_id); - return $self->{bug}; + return $_[0]->{bug} //= Bugzilla::Bug->new({ id => $_[0]->bug_id, cache => 1 }); } =over @@ -175,8 +171,7 @@ user-provided text describing the attachment =cut sub description { - my $self = shift; - return $self->{description}; + return $_[0]->{description}; } =over @@ -190,8 +185,7 @@ the attachment's MIME media type =cut sub contenttype { - my $self = shift; - return $self->{mimetype}; + return $_[0]->{mimetype}; } =over @@ -205,10 +199,8 @@ the user who attached the attachment =cut sub attacher { - my $self = shift; - return $self->{attacher} if exists $self->{attacher}; - $self->{attacher} = new Bugzilla::User($self->{submitter_id}); - return $self->{attacher}; + return $_[0]->{attacher} + //= new Bugzilla::User({ id => $_[0]->{submitter_id}, cache => 1 }); } =over @@ -222,8 +214,7 @@ the date and time on which the attacher attached the attachment =cut sub attached { - my $self = shift; - return $self->{creation_ts}; + return $_[0]->{creation_ts}; } =over @@ -237,8 +228,7 @@ the date and time on which the attachment was last modified. =cut sub modification_time { - my $self = shift; - return $self->{modification_time}; + return $_[0]->{modification_time}; } =over @@ -252,8 +242,7 @@ the name of the file the attacher attached =cut sub filename { - my $self = shift; - return $self->{filename}; + return $_[0]->{filename}; } =over @@ -267,8 +256,7 @@ whether or not the attachment is a patch =cut sub ispatch { - my $self = shift; - return $self->{ispatch}; + return $_[0]->{ispatch}; } =over @@ -282,8 +270,7 @@ whether or not the attachment is obsolete =cut sub isobsolete { - my $self = shift; - return $self->{isobsolete}; + return $_[0]->{isobsolete}; } =over @@ -297,8 +284,7 @@ whether or not the attachment is private =cut sub isprivate { - my $self = shift; - return $self->{isprivate}; + return $_[0]->{isprivate}; } =over @@ -315,8 +301,7 @@ matches, because this will return a value even if it's matched by the generic =cut sub is_viewable { - my $self = shift; - my $contenttype = $self->contenttype; + my $contenttype = $_[0]->contenttype; my $cgi = Bugzilla->cgi; # We assume we can view all text and image types. @@ -390,7 +375,7 @@ the length (in characters) of the attachment content sub datasize { my $self = shift; - return $self->{datasize} if exists $self->{datasize}; + return $self->{datasize} if defined $self->{datasize}; # If we have already retrieved the data, return its size. return length($self->{data}) if exists $self->{data}; @@ -415,6 +400,53 @@ sub datasize { return $self->{datasize}; } +=over + +=item C<linecount> + +the number of lines of the attachment content + +=back + +=cut + +# linecount allows for getting the number of lines of an attachment +# from the database directly if the data has not yet been loaded for +# performance reasons. + +sub linecount { + my ($self) = @_; + + return $self->{linecount} if exists $self->{linecount}; + + # Limit this to just text/* attachments as this could + # cause strange results for binary attachments. + return if $self->contenttype !~ /^text\//; + + # If the data has already been loaded, we can just determine + # line count from the data directly. + if ($self->{data}) { + $self->{linecount} = $self->{data} =~ tr/\n/\n/; + } + else { + $self->{linecount} = + int(Bugzilla->dbh->selectrow_array(" + SELECT LENGTH(attach_data.thedata)-LENGTH(REPLACE(attach_data.thedata,'\n',''))/LENGTH('\n') + FROM attach_data WHERE id = ?", undef, $self->id)); + + } + + # If we still do not have a linecount either the attachment + # is stored in a local file or has been deleted. If the former, + # we call self->data to force a load from the filesystem and + # then do a split on newlines and count again. + unless ($self->{linecount}) { + $self->{linecount} = $self->data =~ tr/\n/\n/; + } + + return $self->{linecount}; +} + sub _get_local_filename { my $self = shift; my $hash = ($self->id % 100) + 100; @@ -433,11 +465,8 @@ flags that have been set on the attachment =cut sub flags { - my $self = shift; - # Don't cache it as it must be in sync with ->flag_types. - $self->{flags} = [map { @{$_->{flags}} } @{$self->flag_types}]; - return $self->{flags}; + return $_[0]->{flags} = [map { @{$_->{flags}} } @{$_[0]->flag_types}]; } =over @@ -458,10 +487,10 @@ sub flag_types { my $vars = { target_type => 'attachment', product_id => $self->bug->product_id, component_id => $self->bug->component_id, - attach_id => $self->id }; + attach_id => $self->id, + active_or_has_flags => $self->bug_id }; - $self->{flag_types} = Bugzilla::Flag->_flag_types($vars); - return $self->{flag_types}; + return $self->{flag_types} = Bugzilla::Flag->_flag_types($vars); } ############################### @@ -573,7 +602,7 @@ sub _check_filename { else { ThrowUserError('file_not_specified'); } - } + } # Remove path info (if any) from the file name. The browser should do this # for us, but some are buggy. This may not work on Mac file names and could @@ -608,12 +637,12 @@ sub _check_is_private { =over -=item C<get_attachments_by_bug($bug_id)> +=item C<get_attachments_by_bug($bug)> Description: retrieves and returns the attachments the currently logged in user can view for the given bug. -Params: C<$bug_id> - integer - the ID of the bug for which +Params: C<$bug> - Bugzilla::Bug object - the bug for which to retrieve and return attachments. Returns: a reference to an array of attachment objects. @@ -621,14 +650,14 @@ Returns: a reference to an array of attachment objects. =cut sub get_attachments_by_bug { - my ($class, $bug_id, $vars) = @_; + my ($class, $bug, $vars) = @_; my $user = Bugzilla->user; my $dbh = Bugzilla->dbh; # By default, private attachments are not accessible, unless the user # is in the insider group or submitted the attachment. my $and_restriction = ''; - my @values = ($bug_id); + my @values = ($bug->id); unless ($user->is_insider) { $and_restriction = 'AND (isprivate = 0 OR submitter_id = ?)'; @@ -645,45 +674,67 @@ sub get_attachments_by_bug { # attachment listed here, we collect all the data at once and # populate $attachment->{flags} ourselves. if ($vars->{preload}) { - $_->{flags} = [] foreach @$attachments; - my %att = map { $_->id => $_ } @$attachments; + # Preload flag types and flags + my $vars = { target_type => 'attachment', + product_id => $bug->product_id, + component_id => $bug->component_id, + attach_id => $attach_ids }; + my $flag_types = Bugzilla::Flag->_flag_types($vars); + + foreach my $attachment (@$attachments) { + $attachment->{flag_types} = []; + my $new_types = dclone($flag_types); + foreach my $new_type (@$new_types) { + $new_type->{flags} = [ grep($_->attach_id == $attachment->id, + @{ $new_type->{flags} }) ]; + push(@{ $attachment->{flag_types} }, $new_type); + } + } - my $flags = Bugzilla::Flag->match({ bug_id => $bug_id, - target_type => 'attachment' }); + # Preload attachers. + my %user_ids = map { $_->{submitter_id} => 1 } @$attachments; + my $users = Bugzilla::User->new_from_list([keys %user_ids]); + my %user_map = map { $_->id => $_ } @$users; + foreach my $attachment (@$attachments) { + $attachment->{attacher} = $user_map{$attachment->{submitter_id}}; + } - # Exclude flags for private attachments you cannot see. - @$flags = grep {exists $att{$_->attach_id}} @$flags; + # Preload datasizes. + my $sizes = + $dbh->selectall_hashref('SELECT attach_id, LENGTH(thedata) AS datasize + FROM attachments LEFT JOIN attach_data ON attach_id = id + WHERE bug_id = ?', + 'attach_id', undef, $bug->id); - push(@{$att{$_->attach_id}->{flags}}, $_) foreach @$flags; - $attachments = [sort {$a->id <=> $b->id} values %att]; + # Force the size of attachments not in the DB to be recalculated. + $_->{datasize} = $sizes->{$_->id}->{datasize} || undef foreach @$attachments; } return $attachments; } =pod -=item C<validate_can_edit($attachment, $product_id)> +=item C<validate_can_edit> Description: validates if the user is allowed to view and edit the attachment. Only the submitter or someone with editbugs privs can edit it. Only the submitter and users in the insider group can view private attachments. -Params: $attachment - the attachment object being edited. - $product_id - the product ID the attachment belongs to. +Params: none Returns: 1 on success, 0 otherwise. =cut sub validate_can_edit { - my ($attachment, $product_id) = @_; + my $self = shift; my $user = Bugzilla->user; # The submitter can edit their attachments. - return ($attachment->attacher->id == $user->id - || ((!$attachment->isprivate || $user->is_insider) - && $user->in_group('editbugs', $product_id))) ? 1 : 0; + return ($self->attacher->id == $user->id + || ((!$self->isprivate || $user->is_insider) + && $user->in_group('editbugs', $self->bug->product_id))) ? 1 : 0; } =item C<validate_obsolete($bug, $attach_ids)> @@ -720,7 +771,7 @@ sub validate_obsolete { || ThrowUserError('invalid_attach_id', $vars); # Check that the user can view and edit this attachment. - $attachment->validate_can_edit($bug->product_id) + $attachment->validate_can_edit || ThrowUserError('illegal_attachment_edit', { attach_id => $attachment->id }); if ($attachment->bug_id != $bug->bug_id) { @@ -852,23 +903,23 @@ sub update { } # Record changes in the activity table. - my $sth = $dbh->prepare('INSERT INTO bugs_activity (bug_id, attach_id, who, bug_when, - fieldid, removed, added) - VALUES (?, ?, ?, ?, ?, ?, ?)'); - + require Bugzilla::Bug; foreach my $field (keys %$changes) { my $change = $changes->{$field}; $field = "attachments.$field" unless $field eq "flagtypes.name"; - my $fieldid = get_field_id($field); - $sth->execute($self->bug_id, $self->id, $user->id, $timestamp, - $fieldid, $change->[0], $change->[1]); + Bugzilla::Bug::LogActivityEntry($self->bug_id, $field, $change->[0], + $change->[1], $user->id, $timestamp, undef, $self->id); } if (scalar(keys %$changes)) { - $dbh->do('UPDATE attachments SET modification_time = ? WHERE attach_id = ?', - undef, ($timestamp, $self->id)); - $dbh->do('UPDATE bugs SET delta_ts = ? WHERE bug_id = ?', - undef, ($timestamp, $self->bug_id)); + $dbh->do('UPDATE attachments SET modification_time = ? WHERE attach_id = ?', + undef, ($timestamp, $self->id)); + $dbh->do('UPDATE bugs SET delta_ts = ? WHERE bug_id = ?', + undef, ($timestamp, $self->bug_id)); + $self->{modification_time} = $timestamp; + # because we updated the attachments table after SUPER::update(), we + # need to ensure the cache is flushed. + Bugzilla->memcached->clear({ table => 'attachments', id => $self->id }); } return $changes; @@ -893,11 +944,21 @@ sub remove_from_db { my $dbh = Bugzilla->dbh; $dbh->bz_start_transaction(); - $dbh->do('DELETE FROM flags WHERE attach_id = ?', undef, $self->id); + my $flag_ids = $dbh->selectcol_arrayref( + 'SELECT id FROM flags WHERE attach_id = ?', undef, $self->id); + $dbh->do('DELETE FROM flags WHERE ' . $dbh->sql_in('id', $flag_ids)) + if @$flag_ids; $dbh->do('DELETE FROM attach_data WHERE id = ?', undef, $self->id); $dbh->do('UPDATE attachments SET mimetype = ?, ispatch = ?, isobsolete = ? WHERE attach_id = ?', undef, ('text/plain', 0, 1, $self->id)); $dbh->bz_commit_transaction(); + + # As we don't call SUPER->remove_from_db we need to manually clear + # memcached here. + Bugzilla->memcached->clear({ table => 'attachments', id => $self->id }); + foreach my $flag_id (@$flag_ids) { + Bugzilla->memcached->clear({ table => 'flags', id => $flag_id }); + } } ############################### diff --git a/Bugzilla/Attachment/PatchReader.pm b/Bugzilla/Attachment/PatchReader.pm index cfc7610f4..1ab14f386 100644 --- a/Bugzilla/Attachment/PatchReader.pm +++ b/Bugzilla/Attachment/PatchReader.pm @@ -19,6 +19,9 @@ use strict; package Bugzilla::Attachment::PatchReader; +use IPC::Open3; +use Symbol 'gensym'; + use Bugzilla::Error; use Bugzilla::Attachment; use Bugzilla::Util; @@ -33,8 +36,8 @@ sub process_diff { my ($reader, $last_reader) = setup_patch_readers(undef, $context); if ($format eq 'raw') { - require PatchReader::DiffPrinter::raw; - $last_reader->sends_data_to(new PatchReader::DiffPrinter::raw()); + require Bugzilla::PatchReader::DiffPrinter::raw; + $last_reader->sends_data_to(new Bugzilla::PatchReader::DiffPrinter::raw()); # Actually print out the patch. print $cgi->header(-type => 'text/plain', -expires => '+3M'); @@ -46,7 +49,7 @@ sub process_diff { if ($lc->{interdiffbin} && $lc->{diffpath}) { # Get the list of attachments that the user can view in this bug. my @attachments = - @{Bugzilla::Attachment->get_attachments_by_bug($attachment->bug_id)}; + @{Bugzilla::Attachment->get_attachments_by_bug($attachment->bug)}; # Extract patches only. @attachments = grep {$_->ispatch == 1} @attachments; # We want them sorted from newer to older. @@ -109,13 +112,37 @@ sub process_interdiff { # Send through interdiff, send output directly to template. # Must hack path so that interdiff will work. $ENV{'PATH'} = $lc->{diffpath}; - open my $interdiff_fh, "$lc->{interdiffbin} $old_filename $new_filename|"; - binmode $interdiff_fh; + + my ($pid, $interdiff_stdout, $interdiff_stderr); + if ($ENV{MOD_PERL}) { + require Apache2::RequestUtil; + require Apache2::SubProcess; + my $request = Apache2::RequestUtil->request; + (undef, $interdiff_stdout, $interdiff_stderr) = $request->spawn_proc_prog( + $lc->{interdiffbin}, [$old_filename, $new_filename] + ); + } else { + $interdiff_stderr = gensym; + my $pid = open3(gensym, $interdiff_stdout, $interdiff_stderr, + $lc->{interdiffbin}, $old_filename, $new_filename); + } + binmode $interdiff_stdout; + + # Check for errors + { + local $/ = undef; + my $error = <$interdiff_stderr>; + if ($error) { + warn($error); + $warning = 'interdiff3'; + } + } + my ($reader, $last_reader) = setup_patch_readers("", $context); if ($format eq 'raw') { - require PatchReader::DiffPrinter::raw; - $last_reader->sends_data_to(new PatchReader::DiffPrinter::raw()); + require Bugzilla::PatchReader::DiffPrinter::raw; + $last_reader->sends_data_to(new Bugzilla::PatchReader::DiffPrinter::raw()); # Actually print out the patch. print $cgi->header(-type => 'text/plain', -expires => '+3M'); @@ -123,7 +150,7 @@ sub process_interdiff { } else { # In case the HTML page is displayed with the UTF-8 encoding. - binmode $interdiff_fh, ':utf8' if Bugzilla->params->{'utf8'}; + binmode $interdiff_stdout, ':utf8' if Bugzilla->params->{'utf8'}; $vars->{'warning'} = $warning if $warning; $vars->{'bugid'} = $new_attachment->bug_id; @@ -134,9 +161,9 @@ sub process_interdiff { setup_template_patch_reader($last_reader, $format, $context, $vars); } - $reader->iterate_fh($interdiff_fh, 'interdiff #' . $old_attachment->id . + $reader->iterate_fh($interdiff_stdout, 'interdiff #' . $old_attachment->id . ' #' . $new_attachment->id); - close $interdiff_fh; + waitpid($pid, 0) if $pid; $ENV{'PATH'} = ''; # Delete temporary files. @@ -152,29 +179,29 @@ sub get_unified_diff { my ($attachment, $format) = @_; # Bring in the modules we need. - require PatchReader::Raw; - require PatchReader::FixPatchRoot; - require PatchReader::DiffPrinter::raw; - require PatchReader::PatchInfoGrabber; + require Bugzilla::PatchReader::Raw; + require Bugzilla::PatchReader::FixPatchRoot; + require Bugzilla::PatchReader::DiffPrinter::raw; + require Bugzilla::PatchReader::PatchInfoGrabber; require File::Temp; $attachment->ispatch || ThrowCodeError('must_be_patch', { 'attach_id' => $attachment->id }); # Reads in the patch, converting to unified diff in a temp file. - my $reader = new PatchReader::Raw; + my $reader = new Bugzilla::PatchReader::Raw; my $last_reader = $reader; # Fixes patch root (makes canonical if possible). if (Bugzilla->params->{'cvsroot'}) { my $fix_patch_root = - new PatchReader::FixPatchRoot(Bugzilla->params->{'cvsroot'}); + new Bugzilla::PatchReader::FixPatchRoot(Bugzilla->params->{'cvsroot'}); $last_reader->sends_data_to($fix_patch_root); $last_reader = $fix_patch_root; } # Grabs the patch file info. - my $patch_info_grabber = new PatchReader::PatchInfoGrabber(); + my $patch_info_grabber = new Bugzilla::PatchReader::PatchInfoGrabber(); $last_reader->sends_data_to($patch_info_grabber); $last_reader = $patch_info_grabber; @@ -184,7 +211,7 @@ sub get_unified_diff { # The HTML page will be displayed with the UTF-8 encoding. binmode $fh, ':utf8'; } - my $raw_printer = new PatchReader::DiffPrinter::raw($fh); + my $raw_printer = new Bugzilla::PatchReader::DiffPrinter::raw($fh); $last_reader->sends_data_to($raw_printer); $last_reader = $raw_printer; @@ -228,13 +255,13 @@ sub setup_patch_readers { # Define the patch readers. # The reader that reads the patch in (whatever its format). - require PatchReader::Raw; - my $reader = new PatchReader::Raw; + require Bugzilla::PatchReader::Raw; + my $reader = new Bugzilla::PatchReader::Raw; my $last_reader = $reader; # Fix the patch root if we have a cvs root. if (Bugzilla->params->{'cvsroot'}) { - require PatchReader::FixPatchRoot; - $last_reader->sends_data_to(new PatchReader::FixPatchRoot(Bugzilla->params->{'cvsroot'})); + require Bugzilla::PatchReader::FixPatchRoot; + $last_reader->sends_data_to(new Bugzilla::PatchReader::FixPatchRoot(Bugzilla->params->{'cvsroot'})); $last_reader->sends_data_to->diff_root($diff_root) if defined($diff_root); $last_reader = $last_reader->sends_data_to; } @@ -243,12 +270,12 @@ sub setup_patch_readers { if ($context ne 'patch' && Bugzilla->localconfig->{cvsbin} && Bugzilla->params->{'cvsroot_get'}) { - require PatchReader::AddCVSContext; + require Bugzilla::PatchReader::AddCVSContext; # We need to set $cvsbin as global, because PatchReader::CVSClient # needs it in order to find 'cvs'. $main::cvsbin = Bugzilla->localconfig->{cvsbin}; $last_reader->sends_data_to( - new PatchReader::AddCVSContext($context, Bugzilla->params->{'cvsroot_get'})); + new Bugzilla::PatchReader::AddCVSContext($context, Bugzilla->params->{'cvsroot_get'})); $last_reader = $last_reader->sends_data_to; } @@ -260,7 +287,7 @@ sub setup_template_patch_reader { my $cgi = Bugzilla->cgi; my $template = Bugzilla->template; - require PatchReader::DiffPrinter::template; + require Bugzilla::PatchReader::DiffPrinter::template; # Define the vars for templates. if (defined $cgi->param('headers')) { @@ -276,10 +303,9 @@ sub setup_template_patch_reader { && Bugzilla->params->{'cvsroot_get'} && !$vars->{'newid'}; # Print everything out. - print $cgi->header(-type => 'text/html', - -expires => '+3M'); + print $cgi->header(-type => 'text/html'); - $last_reader->sends_data_to(new PatchReader::DiffPrinter::template($template, + $last_reader->sends_data_to(new Bugzilla::PatchReader::DiffPrinter::template($template, "attachment/diff-header.$format.tmpl", "attachment/diff-file.$format.tmpl", "attachment/diff-footer.$format.tmpl", diff --git a/Bugzilla/Auth.pm b/Bugzilla/Auth.pm index 45034e166..2c58b52a8 100644 --- a/Bugzilla/Auth.pm +++ b/Bugzilla/Auth.pm @@ -38,6 +38,7 @@ use Bugzilla::User::Setting (); use Bugzilla::Auth::Login::Stack; use Bugzilla::Auth::Verify::Stack; use Bugzilla::Auth::Persist::Cookie; +use Socket; sub new { my ($class, $params) = @_; @@ -123,6 +124,15 @@ sub can_logout { return $getter->can_logout; } +sub login_token { + my ($self) = @_; + my $getter = $self->{_info_getter}->{successful}; + if ($getter && $getter->isa('Bugzilla::Auth::Login::Cookie')) { + return $getter->login_token; + } + return undef; +} + sub user_can_create_account { my ($self) = @_; my $verifier = $self->{_verifier}->{successful}; @@ -215,10 +225,19 @@ sub _handle_login_result { my $default_settings = Bugzilla::User::Setting::get_defaults(); my $template = Bugzilla->template_inner( $default_settings->{lang}->{default_value}); + my $address = $attempts->[0]->{ip_addr}; + # Note: inet_aton will only resolve IPv4 addresses. + # For IPv6 we'll need to use inet_pton which requires Perl 5.12. + my $n = inet_aton($address); + if ($n) { + my $host = gethostbyaddr($n, AF_INET); + $address = "$host ($address)" if $host; + } my $vars = { locked_user => $user, attempts => $attempts, unlock_at => $unlock_at, + address => $address, }; my $message; $template->process('email/lockout.txt.tmpl', $vars, \$message) @@ -416,6 +435,14 @@ Params: None Returns: C<true> if users can change their own email address, C<false> otherwise. +=item C<login_token> + +Description: If a login token was used instead of a cookie then this + will return the current login token data such as user id + and the token itself. +Params: None +Returns: A hash containing C<login_token> and C<user_id>. + =back =head1 STRUCTURE diff --git a/Bugzilla/Auth/Login.pm b/Bugzilla/Auth/Login.pm index 42ce51c62..7e03778b3 100644 --- a/Bugzilla/Auth/Login.pm +++ b/Bugzilla/Auth/Login.pm @@ -17,7 +17,7 @@ package Bugzilla::Auth::Login; use strict; -use fields qw(); +use fields qw(_login_token); # Determines whether or not a user can logout. It's really a subroutine, # but we implement it here as a constant. Override it in subclasses if diff --git a/Bugzilla/Auth/Login/Cookie.pm b/Bugzilla/Auth/Login/Cookie.pm index de9188c64..62a6c58a9 100644 --- a/Bugzilla/Auth/Login/Cookie.pm +++ b/Bugzilla/Auth/Login/Cookie.pm @@ -21,13 +21,15 @@ use base qw(Bugzilla::Auth::Login); use Bugzilla::Constants; use Bugzilla::Util; +use Bugzilla::Error; use List::Util qw(first); use constant requires_persistence => 0; use constant requires_verification => 0; use constant can_login => 0; -use constant is_automatic => 1; + +sub is_automatic { return $_[0]->login_token ? 0 : 1; } # Note that Cookie never consults the Verifier, it always assumes # it has a valid DB account or it fails. @@ -35,24 +37,35 @@ sub get_login_info { my ($self) = @_; my $cgi = Bugzilla->cgi; my $dbh = Bugzilla->dbh; + my ($user_id, $login_cookie); - my $ip_addr = remote_ip(); - my $login_cookie = $cgi->cookie("Bugzilla_logincookie"); - my $user_id = $cgi->cookie("Bugzilla_login"); + if (!Bugzilla->request_cache->{auth_no_automatic_login}) { + $login_cookie = $cgi->cookie("Bugzilla_logincookie"); + $user_id = $cgi->cookie("Bugzilla_login"); - # If cookies cannot be found, this could mean that they haven't - # been made available yet. In this case, look at Bugzilla_cookie_list. - unless ($login_cookie) { - my $cookie = first {$_->name eq 'Bugzilla_logincookie'} - @{$cgi->{'Bugzilla_cookie_list'}}; - $login_cookie = $cookie->value if $cookie; + # If cookies cannot be found, this could mean that they haven't + # been made available yet. In this case, look at Bugzilla_cookie_list. + unless ($login_cookie) { + my $cookie = first {$_->name eq 'Bugzilla_logincookie'} + @{$cgi->{'Bugzilla_cookie_list'}}; + $login_cookie = $cookie->value if $cookie; + } + unless ($user_id) { + my $cookie = first {$_->name eq 'Bugzilla_login'} + @{$cgi->{'Bugzilla_cookie_list'}}; + $user_id = $cookie->value if $cookie; + } } - unless ($user_id) { - my $cookie = first {$_->name eq 'Bugzilla_login'} - @{$cgi->{'Bugzilla_cookie_list'}}; - $user_id = $cookie->value if $cookie; + + # If no cookies were provided, we also look for a login token + # passed in the parameters of a webservice + my $token = $self->login_token; + if ($token && (!$login_cookie || !$user_id)) { + ($user_id, $login_cookie) = ($token->{'user_id'}, $token->{'login_token'}); } + my $ip_addr = remote_ip(); + if ($login_cookie && $user_id) { # Anything goes for these params - they're just strings which # we're going to verify against the db @@ -76,13 +89,43 @@ sub get_login_info { WHERE cookie = ?", undef, $login_cookie); return { user_id => $user_id }; } + elsif (i_am_webservice()) { + ThrowUserError('invalid_cookies_or_token'); + } } - # Either the he cookie is invalid, or we got no cookie. We don't want - # to ever return AUTH_LOGINFAILED, because we don't want Bugzilla to - # actually throw an error when it gets a bad cookie. It should just - # look like there was no cookie to begin with. + # Either the cookie or token is invalid and we are not authenticating + # via a webservice, or we did not receive a cookie or token. We don't + # want to ever return AUTH_LOGINFAILED, because we don't want Bugzilla to + # actually throw an error when it gets a bad cookie or token. It should just + # look like there was no cookie or token to begin with. return { failure => AUTH_NODATA }; } +sub login_token { + my ($self) = @_; + my $input = Bugzilla->input_params; + my $usage_mode = Bugzilla->usage_mode; + + return $self->{'_login_token'} if exists $self->{'_login_token'}; + + if (!i_am_webservice()) { + return $self->{'_login_token'} = undef; + } + + # Check if a token was passed in via requests for WebServices + my $token = trim(delete $input->{'Bugzilla_token'}); + return $self->{'_login_token'} = undef if !$token; + + my ($user_id, $login_token) = split('-', $token, 2); + if (!detaint_natural($user_id) || !$login_token) { + return $self->{'_login_token'} = undef; + } + + return $self->{'_login_token'} = { + user_id => $user_id, + login_token => $login_token + }; +} + 1; diff --git a/Bugzilla/Auth/Persist/Cookie.pm b/Bugzilla/Auth/Persist/Cookie.pm index ace474635..c1d133772 100644 --- a/Bugzilla/Auth/Persist/Cookie.pm +++ b/Bugzilla/Auth/Persist/Cookie.pm @@ -36,6 +36,8 @@ use Bugzilla::Constants; use Bugzilla::Util; use Bugzilla::Token; +use Bugzilla::Auth::Login::Cookie qw(login_token); + use List::Util qw(first); sub new { @@ -107,6 +109,7 @@ sub logout { my $dbh = Bugzilla->dbh; my $cgi = Bugzilla->cgi; + my $input = Bugzilla->input_params; $param = {} unless $param; my $user = $param->{user} || Bugzilla->user; my $type = $param->{type} || LOGOUT_ALL; @@ -120,16 +123,24 @@ sub logout { # The LOGOUT_*_CURRENT options require the current login cookie. # If a new cookie has been issued during this run, that's the current one. # If not, it's the one we've received. + my @login_cookies; my $cookie = first {$_->name eq 'Bugzilla_logincookie'} @{$cgi->{'Bugzilla_cookie_list'}}; - my $login_cookie; if ($cookie) { - $login_cookie = $cookie->value; + push(@login_cookies, $cookie->value); } else { - $login_cookie = $cgi->cookie("Bugzilla_logincookie") || ''; + push(@login_cookies, $cgi->cookie("Bugzilla_logincookie")); + } + + # If we are a webservice using a token instead of cookie + # then add that as well to the login cookies to delete + if (my $login_token = $user->authorizer->login_token) { + push(@login_cookies, $login_token->{'login_token'}); } - trick_taint($login_cookie); + + # Make sure that @login_cookies is not empty to not break SQL statements. + push(@login_cookies, '') unless @login_cookies; # These queries use both the cookie ID and the user ID as keys. Even # though we know the userid must match, we still check it in the SQL @@ -138,12 +149,18 @@ sub logout { # logged in and got the same cookie, we could be logging the other # user out here. Yes, this is very very very unlikely, but why take # chances? - bbaetz + map { trick_taint($_) } @login_cookies; + @login_cookies = map { $dbh->quote($_) } @login_cookies; if ($type == LOGOUT_KEEP_CURRENT) { - $dbh->do("DELETE FROM logincookies WHERE cookie != ? AND userid = ?", - undef, $login_cookie, $user->id); + $dbh->do("DELETE FROM logincookies WHERE " . + $dbh->sql_in('cookie', \@login_cookies, 1) . + " AND userid = ?", + undef, $user->id); } elsif ($type == LOGOUT_CURRENT) { - $dbh->do("DELETE FROM logincookies WHERE cookie = ? AND userid = ?", - undef, $login_cookie, $user->id); + $dbh->do("DELETE FROM logincookies WHERE " . + $dbh->sql_in('cookie', \@login_cookies) . + " AND userid = ?", + undef, $user->id); } else { die("Invalid type $type supplied to logout()"); } diff --git a/Bugzilla/Auth/Verify.pm b/Bugzilla/Auth/Verify.pm index a8cd0af2c..3578631f1 100644 --- a/Bugzilla/Auth/Verify.pm +++ b/Bugzilla/Auth/Verify.pm @@ -97,6 +97,7 @@ sub create_or_update_user { if ($extern_id && $username_user_id && !$extern_user_id) { $dbh->do('UPDATE profiles SET extern_id = ? WHERE userid = ?', undef, $extern_id, $username_user_id); + Bugzilla->memcached->clear({ table => 'profiles', id => $username_user_id }); } # Finally, at this point, one of these will give us a valid user id. @@ -109,23 +110,26 @@ sub create_or_update_user { 'Bugzilla::Auth::Verify::create_or_update_user'}) unless $user_id; - my $user = new Bugzilla::User($user_id); + my $user = new Bugzilla::User({ id => $user_id, cache => 1 }); # Now that we have a valid User, we need to see if any data has to be # updated. + my $user_updated = 0; if ($username && lc($user->login) ne lc($username)) { validate_email_syntax($username) || return { failure => AUTH_ERROR, error => 'auth_invalid_email', details => {addr => $username} }; $user->set_login($username); + $user_updated = 1; } if ($real_name && $user->name ne $real_name) { # $real_name is more than likely tainted, but we only use it # in a placeholder and we never use it after this. trick_taint($real_name); $user->set_name($real_name); + $user_updated = 1; } - $user->update(); + $user->update() if $user_updated; return { user => $user }; } diff --git a/Bugzilla/Auth/Verify/DB.pm b/Bugzilla/Auth/Verify/DB.pm index 2fcfd4017..2840b4ab8 100644 --- a/Bugzilla/Auth/Verify/DB.pm +++ b/Bugzilla/Auth/Verify/DB.pm @@ -90,7 +90,9 @@ sub check_credentials { # whatever hashing system we're using now. my $current_algorithm = PASSWORD_DIGEST_ALGORITHM; if ($real_password_crypted !~ /{\Q$current_algorithm\E}$/) { - $user->set_password($password); + # We can't call $user->set_password because we don't want the password + # complexity rules to apply here. + $user->{cryptpassword} = bz_crypt($password); $user->update(); } @@ -103,6 +105,7 @@ sub change_password { my $cryptpassword = bz_crypt($password); $dbh->do("UPDATE profiles SET cryptpassword = ? WHERE userid = ?", undef, $cryptpassword, $user->id); + Bugzilla->memcached->clear({ table => 'profiles', id => $user->id }); } 1; diff --git a/Bugzilla/Bug.pm b/Bugzilla/Bug.pm index 90bd8b66d..39a0f2596 100644 --- a/Bugzilla/Bug.pm +++ b/Bugzilla/Bug.pm @@ -50,6 +50,7 @@ use Bugzilla::Group; use Bugzilla::Status; use Bugzilla::Comment; use Bugzilla::BugUrl; +use Bugzilla::BugUserLastVisit; use List::MoreUtils qw(firstidx uniq part); use List::Util qw(min max first); @@ -76,11 +77,14 @@ use constant LIST_ORDER => ID_FIELD; # Bugs have their own auditing table, bugs_activity. use constant AUDIT_CREATES => 0; use constant AUDIT_UPDATES => 0; +# This will be enabled later +use constant USE_MEMCACHED => 0; # This is a sub because it needs to call other subroutines. sub DB_COLUMNS { my $dbh = Bugzilla->dbh; - my @custom = grep {$_->type != FIELD_TYPE_MULTI_SELECT} + my @custom = grep {$_->type != FIELD_TYPE_MULTI_SELECT + && $_->type != FIELD_TYPE_EXTENSION} Bugzilla->active_custom_fields; my @custom_names = map {$_->name} @custom; @@ -93,6 +97,7 @@ sub DB_COLUMNS { bug_status cclist_accessible component_id + creation_ts delta_ts estimated_time everconfirmed @@ -111,12 +116,11 @@ sub DB_COLUMNS { version ), 'reporter AS reporter_id', - $dbh->sql_date_format('creation_ts', '%Y.%m.%d %H:%i') . ' AS creation_ts', $dbh->sql_date_format('deadline', '%Y-%m-%d') . ' AS deadline', @custom_names); - + Bugzilla::Hook::process("bug_columns", { columns => \@columns }); - + return @columns; } @@ -166,6 +170,9 @@ sub VALIDATORS { elsif ($field->type == FIELD_TYPE_DATETIME) { $validator = \&_check_datetime_field; } + elsif ($field->type == FIELD_TYPE_DATE) { + $validator = \&_check_date_field; + } elsif ($field->type == FIELD_TYPE_FREETEXT) { $validator = \&_check_freetext_field; } @@ -211,7 +218,8 @@ sub VALIDATOR_DEPENDENCIES { }; sub UPDATE_COLUMNS { - my @custom = grep {$_->type != FIELD_TYPE_MULTI_SELECT} + my @custom = grep {$_->type != FIELD_TYPE_MULTI_SELECT + && $_->type != FIELD_TYPE_EXTENSION} Bugzilla->active_custom_fields; my @custom_names = map {$_->name} @custom; my @columns = qw( @@ -248,7 +256,8 @@ use constant NUMERIC_COLUMNS => qw( ); sub DATE_COLUMNS { - my @fields = @{ Bugzilla->fields({ type => FIELD_TYPE_DATETIME }) }; + my @fields = (@{ Bugzilla->fields({ type => FIELD_TYPE_DATETIME }) }, + @{ Bugzilla->fields({ type => FIELD_TYPE_DATE }) }); return map { $_->name } @fields; } @@ -280,10 +289,6 @@ use constant FIELD_MAP => { summary => 'short_desc', url => 'bug_file_loc', whiteboard => 'status_whiteboard', - - # These are special values for the WebService Bug.search method. - limit => 'LIMIT', - offset => 'OFFSET', }; use constant REQUIRED_FIELD_MAP => { @@ -311,21 +316,11 @@ use constant EXTRA_REQUIRED_FIELDS => qw(creation_ts target_milestone cc qa_cont ##################################################################### -# This and "new" catch every single way of creating a bug, so that we -# can call _create_cf_accessors. -sub _do_list_select { - my $invocant = shift; - $invocant->_create_cf_accessors(); - return $invocant->SUPER::_do_list_select(@_); -} - sub new { my $invocant = shift; my $class = ref($invocant) || $invocant; my $param = shift; - $class->_create_cf_accessors(); - # Remove leading "#" mark if we've just been passed an id. if (!ref $param && $param =~ /^#(\d+)$/) { $param = $1; @@ -333,10 +328,14 @@ sub new { # If we get something that looks like a word (not a number), # make it the "name" param. - if (!defined $param || (!ref($param) && $param !~ /^\d+$/)) { + if (!defined $param + || (!ref($param) && (!$param || $param =~ /\D/)) + || (ref($param) && (!$param->{id} || $param->{id} =~ /\D/))) + { # But only if aliases are enabled. if (Bugzilla->params->{'usebugaliases'} && $param) { - $param = { name => $param }; + $param = { name => ref($param) ? $param->{id} : $param, + cache => ref($param) ? $param->{cache} : 0 }; } else { # Aliases are off, and we got something that's not a number. @@ -370,20 +369,35 @@ sub new { return $self; } -sub check { +sub initialize { + $_[0]->_create_cf_accessors(); +} + +sub object_cache_key { my $class = shift; - my ($id, $field) = @_; + my $key = $class->SUPER::object_cache_key(@_) + || return; + return $key . ',' . Bugzilla->user->id; +} - ThrowUserError('improper_bug_id_field_value', { field => $field }) unless defined $id; +sub check { + my $class = shift; + my ($param, $field) = @_; # Bugzilla::Bug throws lots of special errors, so we don't call # SUPER::check, we just call our new and do our own checks. - my $self = $class->new(trim($id)); - # For error messages, use the id that was returned by new(), because - # it's cleaned up. - $id = $self->id; + my $id = ref($param) + ? ($param->{id} = trim($param->{id})) + : ($param = trim($param)); + ThrowUserError('improper_bug_id_field_value', { field => $field }) unless defined $id; + + my $self = $class->new($param); if ($self->{error}) { + # For error messages, use the id that was returned by new(), because + # it's cleaned up. + $id = $self->id; + if ($self->{error} eq 'NotFound') { ThrowUserError("bug_id_does_not_exist", { bug_id => $id }); } @@ -493,6 +507,59 @@ sub preload { # If we don't do this, can_see_bug will do one call per bug in # the dependency lists, during get_bug_link in Bugzilla::Template. $user->visible_bugs(\@all_dep_ids); + + # We preload comments here in order to allow us to compare the time it + # takes to load comments from the database with the template rendering + # time. + foreach my $bug (@$bugs) { + $bug->comments(); + } + + foreach my $bug (@$bugs) { + $bug->_preload_referenced_bugs(); + } +} + +# Helps load up bugs referenced in comments by retrieving them with a single +# query from the database and injecting bug objects into the object-cache. +sub _preload_referenced_bugs { + my $self = shift; + my @referenced_bug_ids; + + # inject current duplicates into the object-cache first + foreach my $bug (@{ $self->duplicates }) { + $bug->object_cache_set() + unless Bugzilla::Bug->object_cache_get($bug->id); + } + + # preload bugs from comments + require Bugzilla::Template; + foreach my $comment (@{ $self->comments }) { + if ($comment->type == CMT_HAS_DUPE || $comment->type == CMT_DUPE_OF) { + # duplicate bugs that aren't currently in $self->duplicates + push @referenced_bug_ids, $comment->extra_data + unless Bugzilla::Bug->object_cache_get($comment->extra_data); + } + else { + # bugs referenced in comments + Bugzilla::Template::quoteUrls($comment->body, undef, undef, undef, + sub { + my $bug_id = $_[0]; + push @referenced_bug_ids, $bug_id + unless Bugzilla::Bug->object_cache_get($bug_id); + }); + } + } + + # inject into object-cache + my $referenced_bugs = Bugzilla::Bug->new_from_list( + [ uniq @referenced_bug_ids ]); + foreach my $bug (@$referenced_bugs) { + $bug->object_cache_set(); + } + + # preload bug visibility + Bugzilla->user->visible_bugs(\@referenced_bug_ids); } sub possible_duplicates { @@ -512,8 +579,10 @@ sub possible_duplicates { my $dbh = Bugzilla->dbh; my $user = Bugzilla->user; my @words = split(/[\b\s]+/, $short_desc || ''); - # Exclude punctuation from the array. - @words = map { /(\w+)/; $1 } @words; + # Remove leading/trailing punctuation from words + foreach my $word (@words) { + $word =~ s/(?:^\W+|\W+$)//g; + } # And make sure that each word is longer than 2 characters. @words = grep { defined $_ and length($_) > 2 } @words; @@ -615,6 +684,14 @@ sub create { my ($class, $params) = @_; my $dbh = Bugzilla->dbh; + # BMO - allow parameter alteration before creation. also add support for + # fields which are not bug columns (eg bug_mentors). extensions should move + # fields from $params to $stash, then use the bug_end_of_create hook to + # update the database + my $stash = {}; + Bugzilla::Hook::process('bug_before_create', { params => $params, + stash => $stash }); + $dbh->bz_start_transaction(); # These fields have default values which we can use if they are undefined. @@ -640,6 +717,7 @@ sub create { my $blocked = delete $params->{blocked}; my $keywords = delete $params->{keywords}; my $creation_comment = delete $params->{comment}; + my $see_also = delete $params->{see_also}; # We don't want the bug to appear in the system until it's correctly # protected by groups. @@ -676,21 +754,20 @@ sub create { # Set up dependencies (blocked/dependson) my $sth_deps = $dbh->prepare( 'INSERT INTO dependencies (blocked, dependson) VALUES (?, ?)'); - my $sth_bug_time = $dbh->prepare('UPDATE bugs SET delta_ts = ? WHERE bug_id = ?'); foreach my $depends_on_id (@$depends_on) { $sth_deps->execute($bug->bug_id, $depends_on_id); # Log the reverse action on the other bug. LogActivityEntry($depends_on_id, 'blocked', '', $bug->bug_id, $bug->{reporter_id}, $timestamp); - $sth_bug_time->execute($timestamp, $depends_on_id); + _update_delta_ts($depends_on_id, $timestamp); } foreach my $blocked_id (@$blocked) { $sth_deps->execute($blocked_id, $bug->bug_id); # Log the reverse action on the other bug. LogActivityEntry($blocked_id, 'dependson', '', $bug->bug_id, $bug->{reporter_id}, $timestamp); - $sth_bug_time->execute($timestamp, $blocked_id); + _update_delta_ts($blocked_id, $timestamp); } # Insert the values into the multiselect value tables @@ -703,6 +780,25 @@ sub create { } } + # Insert any see_also values + if ($see_also) { + my $see_also_array = $see_also; + if (!ref $see_also_array) { + $see_also = trim($see_also); + $see_also_array = [ split(/[\s,]+/, $see_also) ]; + } + foreach my $value (@$see_also_array) { + $bug->add_see_also($value); + } + foreach my $see_also (@{ $bug->see_also }) { + $see_also->insert_create_data($see_also); + } + foreach my $ref_bug (@{ $bug->{_update_ref_bugs} || [] }) { + $ref_bug->update(); + } + delete $bug->{_update_ref_bugs}; + } + # Comment #0 handling... # We now have a bug id so we can fill this out @@ -712,8 +808,10 @@ sub create { # but sometimes it's blank. Bugzilla::Comment->insert_create_data($creation_comment); + # BMO - add the stash param from bug_start_of_create Bugzilla::Hook::process('bug_end_of_create', { bug => $bug, timestamp => $timestamp, + stash => $stash, }); $dbh->bz_commit_transaction(); @@ -721,7 +819,10 @@ sub create { # Because MySQL doesn't support transactions on the fulltext table, # we do this after we've committed the transaction. That way we're # sure we're inserting a good Bug ID. - $bug->_sync_fulltext('new bug'); + $bug->_sync_fulltext( new_bug => 1 ); + + # BMO - some work should happen outside of the transaction block + Bugzilla::Hook::process('bug_after_create', { bug => $bug, timestamp => $timestamp }); return $bug; } @@ -775,8 +876,9 @@ sub run_create_validators { sub update { my $self = shift; + my $dbh = Bugzilla->dbh; + my $user = Bugzilla->user; - my $dbh = Bugzilla->dbh; # XXX This is just a temporary hack until all updating happens # inside this function. my $delta_ts = shift || $dbh->selectrow_array('SELECT LOCALTIMESTAMP(0)'); @@ -785,6 +887,10 @@ sub update { my ($changes, $old_bug) = $self->SUPER::update(@_); + Bugzilla::Hook::process('bug_start_of_update', + { timestamp => $delta_ts, bug => $self, + old_bug => $old_bug, changes => $changes }); + # Certain items in $changes have to be fixed so that they hold # a name instead of an ID. foreach my $field (qw(product_id component_id)) { @@ -863,10 +969,9 @@ sub update { # Add an activity entry for the other bug. LogActivityEntry($removed_id, $other, $self->id, '', - Bugzilla->user->id, $delta_ts); + $user->id, $delta_ts); # Update delta_ts on the other bug so that we trigger mid-airs. - $dbh->do('UPDATE bugs SET delta_ts = ? WHERE bug_id = ?', - undef, $delta_ts, $removed_id); + _update_delta_ts($removed_id, $delta_ts); } foreach my $added_id (@$added) { $dbh->do("INSERT INTO dependencies ($type, $other) VALUES (?,?)", @@ -874,10 +979,9 @@ sub update { # Add an activity entry for the other bug. LogActivityEntry($added_id, $other, '', $self->id, - Bugzilla->user->id, $delta_ts); + $user->id, $delta_ts); # Update delta_ts on the other bug so that we trigger mid-airs. - $dbh->do('UPDATE bugs SET delta_ts = ? WHERE bug_id = ?', - undef, $delta_ts, $added_id); + _update_delta_ts($added_id, $delta_ts); } if (scalar(@$removed) || scalar(@$added)) { @@ -916,7 +1020,7 @@ sub update { $comment = Bugzilla::Comment->insert_create_data($comment); if ($comment->work_time) { LogActivityEntry($self->id, "work_time", "", $comment->work_time, - Bugzilla->user->id, $delta_ts); + $user->id, $delta_ts); } } @@ -927,7 +1031,7 @@ sub update { my ($from, $to) = $comment->is_private ? (0, 1) : (1, 0); LogActivityEntry($self->id, "longdescs.isprivate", $from, $to, - Bugzilla->user->id, $delta_ts, $comment->id); + $user->id, $delta_ts, $comment->id); } # Clear the cache of comments @@ -974,6 +1078,10 @@ sub update { $changes->{'flagtypes.name'} = [$removed, $added]; } + # BMO - allow extensions to alter what is logged into bugs_activity + Bugzilla::Hook::process('bug_update_before_logging', + { bug => $self, timestamp => $delta_ts, changes => $changes, old_bug => $old_bug }); + # Log bugs_activity items # XXX Eventually, when bugs_activity is able to track the dupe_id, # this code should go below the duplicates-table-updating code below. @@ -981,8 +1089,8 @@ sub update { my $change = $changes->{$field}; my $from = defined $change->[0] ? $change->[0] : ''; my $to = defined $change->[1] ? $change->[1] : ''; - LogActivityEntry($self->id, $field, $from, $to, Bugzilla->user->id, - $delta_ts); + LogActivityEntry($self->id, $field, $from, $to, + $user->id, $delta_ts); } # Check if we have to update the duplicates table and the other bug. @@ -996,7 +1104,7 @@ sub update { $update_dup->update(); } } - + $changes->{'dup_id'} = [$old_dup || undef, $cur_dup || undef]; } @@ -1008,20 +1116,50 @@ sub update { if (scalar(keys %$changes) || $self->{added_comments} || $self->{comment_isprivate}) { - $dbh->do('UPDATE bugs SET delta_ts = ? WHERE bug_id = ?', - undef, ($delta_ts, $self->id)); + _update_delta_ts($self->id, $delta_ts); $self->{delta_ts} = $delta_ts; } + # Update last-visited + if ($user->is_involved_in_bug($self)) { + $self->update_user_last_visit($user, $delta_ts); + } + + # If a user is no longer involved, remove their last visit entry + my $last_visits = Bugzilla::BugUserLastVisit->match({bug_id => $self->id}); + foreach my $lv (@$last_visits) { + $lv->remove_from_db() unless $lv->user->is_involved_in_bug($self); + } + + # Update bug ignore data if user wants to ignore mail for this bug + if (exists $self->{'bug_ignored'}) { + my $bug_ignored_changed; + if ($self->{'bug_ignored'} && !$user->is_bug_ignored($self->id)) { + $dbh->do('INSERT INTO email_bug_ignore + (user_id, bug_id) VALUES (?, ?)', + undef, $user->id, $self->id); + $bug_ignored_changed = 1; + + } + elsif (!$self->{'bug_ignored'} && $user->is_bug_ignored($self->id)) { + $dbh->do('DELETE FROM email_bug_ignore + WHERE user_id = ? AND bug_id = ?', + undef, $user->id, $self->id); + $bug_ignored_changed = 1; + } + delete $user->{bugs_ignored} if $bug_ignored_changed; + } + $dbh->bz_commit_transaction(); # The only problem with this here is that update() is often called # in the middle of a transaction, and if that transaction is rolled # back, this change will *not* be rolled back. As we expect rollbacks # to be extremely rare, that is OK for us. - $self->_sync_fulltext() - if $self->{added_comments} || $changes->{short_desc} - || $self->{comment_isprivate}; + $self->_sync_fulltext( + update_short_desc => $changes->{short_desc}, + update_comments => $self->{added_comments} || $self->{comment_isprivate} + ); # Remove obsolete internal variables. delete $self->{'_old_assigned_to'}; @@ -1029,7 +1167,11 @@ sub update { # Also flush the visible_bugs cache for this bug as the user's # relationship with this bug may have changed. - delete Bugzilla->user->{_visible_bugs_cache}->{$self->id}; + delete $user->{_visible_bugs_cache}->{$self->id}; + + # BMO - some work should happen outside of the transaction block + Bugzilla::Hook::process('bug_after_update', + { bug => $self, timestamp => $delta_ts, changes => $changes, old_bug => $old_bug }); return $changes; } @@ -1055,27 +1197,56 @@ sub _extract_multi_selects { # Should be called any time you update short_desc or change a comment. sub _sync_fulltext { - my ($self, $new_bug) = @_; + my ($self, %options) = @_; my $dbh = Bugzilla->dbh; - if ($new_bug) { - $dbh->do('INSERT INTO bugs_fulltext (bug_id, short_desc) - SELECT bug_id, short_desc FROM bugs WHERE bug_id = ?', - undef, $self->id); + + my($all_comments, $public_comments); + if ($options{new_bug} || $options{update_comments}) { + my $comments = $dbh->selectall_arrayref( + 'SELECT thetext, isprivate FROM longdescs WHERE bug_id = ?', + undef, $self->id); + $all_comments = join("\n", map { $_->[0] } @$comments); + my @no_private = grep { !$_->[1] } @$comments; + $public_comments = join("\n", map { $_->[0] } @no_private); } - else { - $dbh->do('UPDATE bugs_fulltext SET short_desc = ? WHERE bug_id = ?', - undef, $self->short_desc, $self->id); + + if ($options{new_bug}) { + $dbh->do('INSERT INTO bugs_fulltext (bug_id, short_desc, comments, + comments_noprivate) + VALUES (?, ?, ?, ?)', + undef, + $self->id, $self->short_desc, $all_comments, $public_comments); + } else { + my(@names, @values); + if ($options{update_short_desc}) { + push @names, 'short_desc'; + push @values, $self->short_desc; + } + if ($options{update_comments}) { + push @names, ('comments', 'comments_noprivate'); + push @values, ($all_comments, $public_comments); + } + if (@names) { + $dbh->do('UPDATE bugs_fulltext SET ' . + join(', ', map { "$_ = ?" } @names) . + ' WHERE bug_id = ?', + undef, + @values, $self->id); + } } - my $comments = $dbh->selectall_arrayref( - 'SELECT thetext, isprivate FROM longdescs WHERE bug_id = ?', - undef, $self->id); - my $all = join("\n", map { $_->[0] } @$comments); - my @no_private = grep { !$_->[1] } @$comments; - my $nopriv_string = join("\n", map { $_->[0] } @no_private); - $dbh->do('UPDATE bugs_fulltext SET comments = ?, comments_noprivate = ? - WHERE bug_id = ?', undef, $all, $nopriv_string, $self->id); } +# Update a bug's delta_ts without requiring the full object to be loaded. +sub _update_delta_ts { + my ($bug_id, $timestamp) = @_; + Bugzilla->dbh->do( + "UPDATE bugs SET delta_ts = ? WHERE bug_id = ?", + undef, + $timestamp, $bug_id + ); + Bugzilla::Hook::process('bug_end_of_update_delta_ts', + { bug_id => $bug_id, timestamp => $timestamp }); +} # This is the correct way to delete bugs from the DB. # No bug should be deleted from anywhere else except from here. @@ -1164,15 +1335,15 @@ sub send_changes { changer => $user, ); - _send_bugmail({ id => $self->id, type => 'bug', forced => \%forced }, - $vars); + my $recipient_count = _send_bugmail( + { id => $self->id, type => 'bug', forced => \%forced }, $vars); # If the bug was marked as a duplicate, we need to notify users on the # other bug of any changes to that bug. my $new_dup_id = $changes->{'dup_id'} ? $changes->{'dup_id'}->[1] : undef; if ($new_dup_id) { - _send_bugmail({ forced => { changer => $user }, type => "dupe", - id => $new_dup_id }, $vars); + $recipient_count += _send_bugmail( + { forced => { changer => $user }, type => "dupe", id => $new_dup_id }, $vars); } # If there were changes in dependencies, we need to notify those @@ -1191,7 +1362,7 @@ sub send_changes { foreach my $id (@{ $self->blocked }) { $params->{id} = $id; - _send_bugmail($params, $vars); + $recipient_count += _send_bugmail($params, $vars); } } } @@ -1209,15 +1380,17 @@ sub send_changes { delete $changed_deps{''}; foreach my $id (sort { $a <=> $b } (keys %changed_deps)) { - _send_bugmail({ forced => { changer => $user }, type => "dep", - id => $id }, $vars); + $recipient_count += _send_bugmail( + { forced => { changer => $user }, type => "dep", id => $id }, $vars); } # Sending emails for the referenced bugs. foreach my $ref_bug_id (uniq @{ $self->{see_also_changes} || [] }) { - _send_bugmail({ forced => { changer => $user }, - id => $ref_bug_id }, $vars); + $recipient_count += _send_bugmail( + { forced => { changer => $user }, id => $ref_bug_id }, $vars); } + + return $recipient_count; } sub _send_bugmail { @@ -1225,7 +1398,7 @@ sub _send_bugmail { require Bugzilla::BugMail; - my $results = + my $results = Bugzilla::BugMail::Send($params->{'id'}, $params->{'forced'}, $params); if (Bugzilla->usage_mode == USAGE_MODE_BROWSER) { @@ -1236,6 +1409,8 @@ sub _send_bugmail { || ThrowTemplateError($template->error()); $vars->{'header_done'} = 1; } + + return scalar @{ $results->{sent} }; } ##################################################################### @@ -1658,7 +1833,9 @@ sub _check_groups { foreach my $name (@$group_names) { my $group = Bugzilla::Group->check_no_disclose({ %args, name => $name }); - if (!$product->group_is_settable($group)) { + # BMO : allow bugs to be always placed into some groups + if (!$product->group_always_settable($group) + && !$product->group_is_settable($group)) { ThrowUserError('group_restriction_not_allowed', { %args, name => $name }); } $add_groups{$group->id} = $group; @@ -1667,7 +1844,7 @@ sub _check_groups { # Now enforce mandatory groups. $add_groups{$_->id} = $_ foreach @{ $product->groups_mandatory }; - + my @add_groups = values %add_groups; return \@add_groups; } @@ -1989,8 +2166,12 @@ sub _check_field_is_mandatory { } } +sub _check_date_field { + my ($invocant, $date) = @_; + return $invocant->_check_datetime_field($date, undef, {date_only => 1}); +} sub _check_datetime_field { - my ($invocant, $date_time) = @_; + my ($invocant, $date_time, $field, $params) = @_; # Empty datetimes are empty strings or strings only containing # 0's, whitespace, and punctuation. @@ -2004,6 +2185,10 @@ sub _check_datetime_field { ThrowUserError('illegal_date', { date => $date, format => 'YYYY-MM-DD' }); } + if ($time && $params->{date_only}) { + ThrowUserError('illegal_date', { date => $date_time, + format => 'YYYY-MM-DD' }); + } if ($time && !validate_time($time)) { ThrowUserError('illegal_time', { 'time' => $time, format => 'HH:MM:SS' }); @@ -2254,7 +2439,8 @@ sub set_all { $self->_add_remove($params, 'see_also'); # And set custom fields. - my @custom_fields = Bugzilla->active_custom_fields; + my @custom_fields = grep { $_->type != FIELD_TYPE_EXTENSION } + Bugzilla->active_custom_fields; foreach my $field (@custom_fields) { my $fname = $field->name; if (exists $params->{$fname}) { @@ -2269,7 +2455,7 @@ sub set_all { # we have to check that the current assignee, qa, and CCs are still # valid if we've switched products, under strict_isolation. We can only # do that here, because if they *did* change the assignee, qa, or CC, - # then we don't want to check the original ones, only the new ones. + # then we don't want to check the original ones, only the new ones. $self->_check_strict_isolation() if $product_changed; } @@ -2299,6 +2485,7 @@ sub reset_assigned_to { my $comp = $self->component_obj; $self->set_assigned_to($comp->default_assignee); } +sub set_bug_ignored { $_[0]->set('bug_ignored', $_[1]); } sub set_cclist_accessible { $_[0]->set('cclist_accessible', $_[1]); } sub set_comment_is_private { my ($self, $comment_id, $isprivate) = @_; @@ -2796,20 +2983,26 @@ sub add_group { # If the bug is already in this group, then there is nothing to do. return if $self->in_group($group); - - # Make sure that bugs in this product can actually be restricted - # to this group by the current user. - $self->product_obj->group_is_settable($group) - || ThrowUserError('group_restriction_not_allowed', $args); - - # OtherControl people can add groups only during a product change, - # and only when the group is not NA for them. - if (!Bugzilla->user->in_group($group->name)) { - my $controls = $self->product_obj->group_controls->{$group->id}; - if (!$self->{_old_product_name} - || $controls->{othercontrol} == CONTROLMAPNA) - { - ThrowUserError('group_restriction_not_allowed', $args); + # BMO : allow bugs to be always placed into some groups by the bug's + # reporter, or by users with editbugs + my $user = Bugzilla->user; + if (!$self->product_obj->group_always_settable($group) + || ($self->{reporter_id} != $user->id && !$user->in_group('editbugs'))) + { + # Make sure that bugs in this product can actually be restricted + # to this group by the current user. + $self->product_obj->group_is_settable($group) + || ThrowUserError('group_restriction_not_allowed', $args); + + # OtherControl people can add groups only during a product change, + # and only when the group is not NA for them. + if (!$user->in_group($group->name)) { + my $controls = $self->product_obj->group_controls->{$group->id}; + if (!$self->{_old_product_name} + || $controls->{othercontrol} == CONTROLMAPNA) + { + ThrowUserError('group_restriction_not_allowed', $args); + } } } @@ -3156,7 +3349,7 @@ sub attachments { return [] if $self->{'error'}; $self->{'attachments'} = - Bugzilla::Attachment->get_attachments_by_bug($self->bug_id, {preload => 1}); + Bugzilla::Attachment->get_attachments_by_bug($self, {preload => 1}); return $self->{'attachments'}; } @@ -3164,8 +3357,8 @@ sub assigned_to { my ($self) = @_; return $self->{'assigned_to_obj'} if exists $self->{'assigned_to_obj'}; $self->{'assigned_to'} = 0 if $self->{'error'}; - $self->{'assigned_to_obj'} ||= new Bugzilla::User($self->{'assigned_to'}); - return $self->{'assigned_to_obj'}; + return $self->{'assigned_to_obj'} + = new Bugzilla::User({ id => $self->{'assigned_to'}, cache => 1 }); } sub blocked { @@ -3229,11 +3422,8 @@ sub cc_users { sub component { my ($self) = @_; - return $self->{component} if exists $self->{component}; return '' if $self->{error}; - ($self->{component}) = Bugzilla->dbh->selectrow_array( - 'SELECT name FROM components WHERE id = ?', - undef, $self->{component_id}); + ($self->{component}) //= $self->component_obj->name; return $self->{component}; } @@ -3242,7 +3432,8 @@ sub component_obj { my ($self) = @_; return $self->{component_obj} if defined $self->{component_obj}; return {} if $self->{error}; - $self->{component_obj} = new Bugzilla::Component($self->{component_id}); + $self->{component_obj} = + new Bugzilla::Component({ id => $self->{component_id}, cache => 1 }); return $self->{component_obj}; } @@ -3281,6 +3472,26 @@ sub depends_on_obj { return $self->{depends_on_obj}; } +sub duplicates { + my $self = shift; + return $self->{duplicates} if exists $self->{duplicates}; + return [] if $self->{error}; + $self->{duplicates} = Bugzilla::Bug->new_from_list($self->duplicate_ids); + return $self->{duplicates}; +} + +sub duplicate_ids { + my $self = shift; + return $self->{duplicate_ids} if exists $self->{duplicate_ids}; + return [] if $self->{error}; + + my $dbh = Bugzilla->dbh; + $self->{duplicate_ids} = + $dbh->selectcol_arrayref('SELECT dupe FROM duplicates WHERE dupe_of = ?', + undef, $self->id); + return $self->{duplicate_ids}; +} + sub flag_types { my ($self) = @_; return $self->{'flag_types'} if exists $self->{'flag_types'}; @@ -3289,7 +3500,8 @@ sub flag_types { my $vars = { target_type => 'bug', product_id => $self->{product_id}, component_id => $self->{component_id}, - bug_id => $self->bug_id }; + bug_id => $self->bug_id, + active_or_has_flags => $self->bug_id }; $self->{'flag_types'} = Bugzilla::Flag->_flag_types($vars); return $self->{'flag_types'}; @@ -3338,8 +3550,14 @@ sub comments { $comment->{count} = $count++; $comment->{bug} = $self; } - Bugzilla::Comment->preload($self->{'comments'}); + # Some bugs may have no comments when upgrading old installations. + Bugzilla::Comment->preload($self->{'comments'}) if @{ $self->{'comments'} }; + # BMO - for comment deletion support + Bugzilla::Hook::process('bug_comments', + { bug => $self, comments => $self->{'comments'} }); } + return unless defined wantarray; + my @comments = @{ $self->{'comments'} }; my $order = $params->{order} @@ -3378,11 +3596,8 @@ sub percentage_complete { sub product { my ($self) = @_; - return $self->{product} if exists $self->{product}; return '' if $self->{error}; - ($self->{product}) = Bugzilla->dbh->selectrow_array( - 'SELECT name FROM products WHERE id = ?', - undef, $self->{product_id}); + ($self->{product}) //= $self->product_obj->name; return $self->{product}; } @@ -3390,7 +3605,8 @@ sub product { sub product_obj { my $self = shift; return {} if $self->{error}; - $self->{product_obj} ||= new Bugzilla::Product($self->{product_id}); + $self->{product_obj} ||= + new Bugzilla::Product({ id => $self->{product_id}, cache => 1 }); return $self->{product_obj}; } @@ -3400,7 +3616,8 @@ sub qa_contact { return undef if $self->{'error'}; if (Bugzilla->params->{'useqacontact'} && $self->{'qa_contact'}) { - $self->{'qa_contact_obj'} = new Bugzilla::User($self->{'qa_contact'}); + $self->{'qa_contact_obj'} + = new Bugzilla::User({ id => $self->{'qa_contact'}, cache => 1 }); } else { # XXX - This is somewhat inconsistent with the assignee/reporter # methods, which will return an empty User if they get a 0. @@ -3414,8 +3631,8 @@ sub reporter { my ($self) = @_; return $self->{'reporter'} if exists $self->{'reporter'}; $self->{'reporter_id'} = 0 if $self->{'error'}; - $self->{'reporter'} = new Bugzilla::User($self->{'reporter_id'}); - return $self->{'reporter'}; + return $self->{'reporter'} + = new Bugzilla::User({ id => $self->{'reporter_id'}, cache => 1 }); } sub see_also { @@ -3556,6 +3773,49 @@ sub groups { } } + # BMO: if required, hack in groups exposed by -visible membership + # (eg mozilla-employee-confidential-visible), so reporters can add the + # bug to a group on show_bug. + # if the bug is already in the group, the user will not be able to remove + # it unless they are a true group member. + my $user = Bugzilla->user; + if ($self->{'reporter_id'} == $user->id) { + foreach my $group (@{ $user->groups }) { + # map from -visible group to the real one + my $group_name = $group->name; + next unless $group_name =~ s/-visible$//; + next if $user->in_group($group_name); + $group = Bugzilla::Group->new({ name => $group_name, cache => 1 }); + + # only show the group if it's visible to normal members + my ($member_control) = $dbh->selectrow_array( + "SELECT membercontrol + FROM groups + LEFT JOIN group_control_map + ON group_control_map.group_id = groups.id + AND group_control_map.product_id = ? + WHERE groups.id = ?", + undef, + $self->{product_id}, $group->id + ); + + if ( + $member_control + && $member_control == CONTROLMAPSHOWN + && !grep { $_->{bit} == $group->id } @groups) + { + push(@groups, { + bit => $group->id, + name => $group->name, + ison => 0, + ingroup => 1, + mandatory => 0, + description => $group->description, + }); + } + } + } + $self->{'groups'} = \@groups; return $self->{'groups'}; @@ -3674,6 +3934,9 @@ sub editable_bug_fields { # Ensure field exists before attempting to remove it. splice(@fields, $location, 1) if ($location > -1); } + + Bugzilla::Hook::process('bug_editable_bug_fields', { fields => \@fields }); + # Sorted because the old @::log_columns variable, which this replaces, # was sorted. return sort(@fields); @@ -3684,32 +3947,52 @@ sub editable_bug_fields { # Join with bug_status and bugs tables to show bugs with open statuses first, # and then the others sub EmitDependList { - my ($myfield, $targetfield, $bug_id) = (@_); + my ($my_field, $target_field, $bug_id, $exclude_resolved) = @_; + my $cache = Bugzilla->request_cache->{bug_dependency_list} ||= {}; + my $dbh = Bugzilla->dbh; - my $list_ref = $dbh->selectcol_arrayref( - "SELECT $targetfield + $exclude_resolved = $exclude_resolved ? 1 : 0; + my $is_open_clause = $exclude_resolved ? 'AND is_open = 1' : ''; + + $cache->{"${target_field}_sth_$exclude_resolved"} ||= $dbh->prepare( + "SELECT $target_field FROM dependencies - INNER JOIN bugs ON dependencies.$targetfield = bugs.bug_id + INNER JOIN bugs ON dependencies.$target_field = bugs.bug_id INNER JOIN bug_status ON bugs.bug_status = bug_status.value - WHERE $myfield = ? - ORDER BY is_open DESC, $targetfield", - undef, $bug_id); - return $list_ref; + WHERE $my_field = ? $is_open_clause + ORDER BY is_open DESC, $target_field"); + + return $dbh->selectcol_arrayref( + $cache->{"${target_field}_sth_$exclude_resolved"}, + undef, $bug_id); } # Creates a lot of bug objects in the same order as the input array. sub _bugs_in_order { my ($self, $bug_ids) = @_; - my $bugs = $self->new_from_list($bug_ids); - my %bug_map = map { $_->id => $_ } @$bugs; - my @result = map { $bug_map{$_} } @$bug_ids; - return \@result; + my %bug_map; + # there's no need to load bugs from the database if they are already in the + # object-cache + my @missing_ids; + foreach my $bug_id (@$bug_ids) { + if (my $bug = Bugzilla::Bug->object_cache_get($bug_id)) { + $bug_map{$bug_id} = $bug; + } + else { + push @missing_ids, $bug_id; + } + } + my $bugs = $self->new_from_list(\@missing_ids); + foreach my $bug (@$bugs) { + $bug_map{$bug->id} = $bug; + } + return [ map { $bug_map{$_} } @$bug_ids ]; } # Get the activity of a bug, starting from $starttime (if given). # This routine assumes Bugzilla::Bug->check has been previously called. sub GetBugActivity { - my ($bug_id, $attach_id, $starttime) = @_; + my ($bug_id, $attach_id, $starttime, $include_comment_tags) = @_; my $dbh = Bugzilla->dbh; # Arguments passed to the SQL query. @@ -3720,7 +4003,7 @@ sub GetBugActivity { if (defined $starttime) { trick_taint($starttime); push (@args, $starttime); - $datepart = "AND bugs_activity.bug_when > ?"; + $datepart = "AND bug_when > ?"; } my $attachpart = ""; @@ -3741,7 +4024,7 @@ sub GetBugActivity { my $query = "SELECT fielddefs.name, bugs_activity.attach_id, " . $dbh->sql_date_format('bugs_activity.bug_when', '%Y.%m.%d %H:%i:%s') . - ", bugs_activity.removed, bugs_activity.added, profiles.login_name, + " AS bug_when, bugs_activity.removed, bugs_activity.added, profiles.login_name, bugs_activity.comment_id FROM bugs_activity $suppjoins @@ -3752,8 +4035,42 @@ sub GetBugActivity { WHERE bugs_activity.bug_id = ? $datepart $attachpart - $suppwhere - ORDER BY bugs_activity.bug_when"; + $suppwhere "; + + if (Bugzilla->params->{'comment_taggers_group'} + && $include_comment_tags + && !$attach_id) + { + # Only includes comment tag activity for comments the user is allowed to see. + $suppjoins = ""; + $suppwhere = ""; + if (!Bugzilla->user->is_insider) { + $suppjoins = "INNER JOIN longdescs + ON longdescs.comment_id = longdescs_tags_activity.comment_id"; + $suppwhere = "AND longdescs.isprivate = 0"; + } + + $query .= " + UNION ALL + SELECT 'comment_tag' AS name, + NULL AS attach_id," . + $dbh->sql_date_format('longdescs_tags_activity.bug_when', '%Y.%m.%d %H:%i:%s') . " AS bug_when, + longdescs_tags_activity.removed, + longdescs_tags_activity.added, + profiles.login_name, + longdescs_tags_activity.comment_id as comment_id + FROM longdescs_tags_activity + INNER JOIN profiles ON profiles.userid = longdescs_tags_activity.who + $suppjoins + WHERE longdescs_tags_activity.bug_id = ? + $datepart + $suppwhere + "; + push @args, $bug_id; + push @args, $starttime if defined $starttime; + } + + $query .= "ORDER BY bug_when, comment_id"; my $list = $dbh->selectall_arrayref($query, undef, @args); @@ -3808,16 +4125,29 @@ sub GetBugActivity { $changes = []; } + # If this is the same field as the previoius item, then concatenate + # the data into the same change. + if ($operation->{'who'} && $who eq $operation->{'who'} + && $when eq $operation->{'when'} + && $fieldname eq $operation->{'fieldname'} + && ($comment_id || 0) == ($operation->{'comment_id'} || 0) + && ($attachid || 0) == ($operation->{'attachid'} || 0)) + { + my $old_change = pop @$changes; + $removed = _join_activity_entries($fieldname, $old_change->{'removed'}, $removed); + $added = _join_activity_entries($fieldname, $old_change->{'added'}, $added); + } + $operation->{'who'} = $who; $operation->{'when'} = $when; + $operation->{'fieldname'} = $change{'fieldname'} = $fieldname; + $operation->{'attachid'} = $change{'attachid'} = $attachid; - $change{'fieldname'} = $fieldname; - $change{'attachid'} = $attachid; $change{'removed'} = $removed; $change{'added'} = $added; - + if ($comment_id) { - $change{'comment'} = Bugzilla::Comment->new($comment_id); + $operation->{comment_id} = $change{'comment'} = Bugzilla::Comment->new($comment_id); } push (@$changes, \%change); @@ -3832,9 +4162,40 @@ sub GetBugActivity { return(\@operations, $incomplete_data); } +sub _join_activity_entries { + my ($field, $current_change, $new_change) = @_; + # We need to insert characters as these were removed by old + # LogActivityEntry code. + + return $new_change if $current_change eq ''; + + # Buglists and see_also need the comma restored + if ($field eq 'dependson' || $field eq 'blocked' || $field eq 'see_also') { + if (substr($new_change, 0, 1) eq ',' || substr($new_change, 0, 1) eq ' ') { + return $current_change . $new_change; + } else { + return $current_change . ', ' . $new_change; + } + } + + # Assume bug_file_loc contain a single url, don't insert a delimiter + if ($field eq 'bug_file_loc') { + return $current_change . $new_change; + } + + # All other fields get a space unless the first character of the second + # string is a comma or space + if (substr($new_change, 0, 1) eq ',' || substr($new_change, 0, 1) eq ' ') { + return $current_change . $new_change; + } else { + return $current_change . ' ' . $new_change; + } +} + # Update the bugs_activity table to reflect changes made in bugs. sub LogActivityEntry { - my ($i, $col, $removed, $added, $whoid, $timestamp, $comment_id) = @_; + my ($i, $col, $removed, $added, $whoid, $timestamp, $comment_id, + $attach_id) = @_; my $dbh = Bugzilla->dbh; # in the case of CCs, deps, and keywords, there's a possibility that someone # might try to add or remove a lot of them at once, which might take more @@ -3846,7 +4207,6 @@ sub LogActivityEntry { my $commaposition = find_wrap_point($removed, MAX_LINE_LENGTH); $removestr = substr($removed, 0, $commaposition); $removed = substr($removed, $commaposition); - $removed =~ s/^[,\s]+//; # remove any comma or space } else { $removed = ""; # no more entries } @@ -3854,17 +4214,36 @@ sub LogActivityEntry { my $commaposition = find_wrap_point($added, MAX_LINE_LENGTH); $addstr = substr($added, 0, $commaposition); $added = substr($added, $commaposition); - $added =~ s/^[,\s]+//; # remove any comma or space } else { $added = ""; # no more entries } trick_taint($addstr); trick_taint($removestr); my $fieldid = get_field_id($col); - $dbh->do("INSERT INTO bugs_activity - (bug_id, who, bug_when, fieldid, removed, added, comment_id) - VALUES (?, ?, ?, ?, ?, ?, ?)", - undef, ($i, $whoid, $timestamp, $fieldid, $removestr, $addstr, $comment_id)); + $dbh->do( + "INSERT INTO bugs_activity + (bug_id, who, bug_when, fieldid, removed, added, comment_id, attach_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + undef, + ($i, $whoid, $timestamp, $fieldid, $removestr, $addstr, $comment_id, + $attach_id)); + } +} + +# Update bug_user_last_visit table +sub update_user_last_visit { + my ($self, $user, $last_visit_ts) = @_; + my $lv = Bugzilla::BugUserLastVisit->match({ bug_id => $self->id, + user_id => $user->id })->[0]; + + if ($lv) { + $lv->set(last_visit_ts => $last_visit_ts); + $lv->update; + } + else { + Bugzilla::BugUserLastVisit->create({ bug_id => $self->id, + user_id => $user->id, + last_visit_ts => $last_visit_ts }); } } @@ -3941,8 +4320,8 @@ sub check_can_change_field { return 1; } - # Allow anyone to change comments. - if ($field =~ /^longdesc/) { + # Allow anyone to change comments, or set flags + if ($field =~ /^longdesc/ || $field eq 'flagtypes.name') { return 1; } @@ -4148,6 +4527,7 @@ sub _create_cf_accessors { my $fields = Bugzilla->fields({ custom => 1 }); foreach my $field (@$fields) { + next if $field->type == FIELD_TYPE_EXTENSION; my $accessor = $class->_accessor_for($field); my $name = "${class}::" . $field->name; { @@ -4157,6 +4537,8 @@ sub _create_cf_accessors { } } + Bugzilla::Hook::process('bug_create_cf_accessors'); + Bugzilla->request_cache->{"${class}_cf_accessors_created"} = 1; } diff --git a/Bugzilla/BugMail.pm b/Bugzilla/BugMail.pm index 55eeeab25..9980535ce 100644 --- a/Bugzilla/BugMail.pm +++ b/Bugzilla/BugMail.pm @@ -47,7 +47,9 @@ use Bugzilla::Hook; use Date::Parse; use Date::Format; use Scalar::Util qw(blessed); -use List::MoreUtils qw(uniq); +use List::MoreUtils qw(uniq firstidx); +use Sys::Hostname; +use Storable qw(dclone); use constant BIT_DIRECT => 1; use constant BIT_WATCHING => 2; @@ -107,30 +109,53 @@ sub Send { my %user_cache = map { $_->id => $_ } (@assignees, @qa_contacts, @ccs); my @diffs; + my @referenced_bugs; if (!$start) { @diffs = _get_new_bugmail_fields($bug); } if ($params->{dep_only}) { + my $fields = Bugzilla->fields({ by_name => 1 }); push(@diffs, { field_name => 'bug_status', + field_desc => $fields->{bug_status}->description, old => $params->{changes}->{bug_status}->[0], new => $params->{changes}->{bug_status}->[1], login_name => $changer->login, blocker => $params->{blocker} }, { field_name => 'resolution', + field_desc => $fields->{resolution}->description, old => $params->{changes}->{resolution}->[0], new => $params->{changes}->{resolution}->[1], login_name => $changer->login, blocker => $params->{blocker} }); + push(@referenced_bugs, $params->{blocker}->id); } else { - push(@diffs, _get_diffs($bug, $end, \%user_cache)); + my ($diffs, $referenced) = _get_diffs($bug, $end, \%user_cache); + push(@diffs, @$diffs); + push(@referenced_bugs, @$referenced); } my $comments = $bug->comments({ after => $start, to => $end }); # Skip empty comments. @$comments = grep { $_->type || $_->body =~ /\S/ } @$comments; + # Add duplicate bug to referenced bug list + foreach my $comment (@$comments) { + if ($comment->type == CMT_DUPE_OF || $comment->type == CMT_HAS_DUPE) { + push(@referenced_bugs, $comment->extra_data); + } + } + + # Add dependencies to referenced bug list on new bugs + if (!$start) { + push @referenced_bugs, @{ $bug->dependson }; + push @referenced_bugs, @{ $bug->blocked }; + } + + # If no changes have been made, there is no need to process further. + return {'sent' => []} unless scalar(@diffs) || scalar(@$comments); + ########################################################################### # Start of email filtering code ########################################################################### @@ -186,28 +211,30 @@ sub Send { # Make sure %user_cache has every user in it so far referenced foreach my $user_id (keys %recipients) { - $user_cache{$user_id} ||= new Bugzilla::User($user_id); + $user_cache{$user_id} ||= new Bugzilla::User({ id => $user_id, cache => 1 }); } Bugzilla::Hook::process('bugmail_recipients', { bug => $bug, recipients => \%recipients, users => \%user_cache, diffs => \@diffs }); - # Find all those user-watching anyone on the current list, who is not - # on it already themselves. - my $involved = join(",", keys %recipients); + if (scalar keys %recipients) { + # Find all those user-watching anyone on the current list, who is not + # on it already themselves. + my $involved = join(",", keys %recipients); - my $userwatchers = - $dbh->selectall_arrayref("SELECT watcher, watched FROM watch - WHERE watched IN ($involved)"); + my $userwatchers = + $dbh->selectall_arrayref("SELECT watcher, watched FROM watch + WHERE watched IN ($involved)"); - # Mark these people as having the role of the person they are watching - foreach my $watch (@$userwatchers) { - while (my ($role, $bits) = each %{$recipients{$watch->[1]}}) { - $recipients{$watch->[0]}->{$role} |= BIT_WATCHING - if $bits & BIT_DIRECT; + # Mark these people as having the role of the person they are watching + foreach my $watch (@$userwatchers) { + while (my ($role, $bits) = each %{$recipients{$watch->[1]}}) { + $recipients{$watch->[0]}->{$role} |= BIT_WATCHING + if $bits & BIT_DIRECT; + } + push(@{$watching{$watch->[0]}}, $watch->[1]); } - push(@{$watching{$watch->[0]}}, $watch->[1]); } # Global watcher @@ -222,21 +249,25 @@ sub Send { # the bug in question. However, we are not necessarily going to mail them # all - there are preferences, permissions checks and all sorts to do yet. my @sent; - my @excluded; # The email client will display the Date: header in the desired timezone, # so we can always use UTC here. my $date = $params->{dep_only} ? $end : $bug->delta_ts; $date = format_time($date, '%a, %d %b %Y %T %z', 'UTC'); + # Remove duplicate references, and convert to bug objects + @referenced_bugs = @{ Bugzilla::Bug->new_from_list([uniq @referenced_bugs]) }; + foreach my $user_id (keys %recipients) { my %rels_which_want; - my $sent_mail = 0; - $user_cache{$user_id} ||= new Bugzilla::User($user_id); - my $user = $user_cache{$user_id}; + my $user = $user_cache{$user_id} ||= new Bugzilla::User({ id => $user_id, cache => 1 }); # Deleted users must be excluded. next unless $user; + # If email notifications are disabled for this account, or the bug + # is ignored, there is no need to do additional checks. + next if ($user->email_disabled || $user->is_bug_ignored($id)); + if ($user->can_see_bug($id)) { # Go through each role the user has and see if they want mail in # that role. @@ -253,7 +284,7 @@ sub Send { } } } - + if (scalar(%rels_which_want)) { # So the user exists, can see the bug, and wants mail in at least # one role. But do we want to send it to them? @@ -267,10 +298,32 @@ sub Send { } # Make sure the user isn't in the nomail list, and the dep check passed. - if ($user->email_enabled && $dep_ok) { - # OK, OK, if we must. Email the user. - $sent_mail = sendMail( - { to => $user, + # BMO: never send emails to bugs or .tld addresses. this check needs to + # happen after the bugmail_recipients hook. + if ($user->email_enabled && $dep_ok && + ($user->login !~ /bugs$/) && ($user->login !~ /\.tld$/)) + { + # Don't show summaries for bugs the user can't access, and + # provide a hook for extensions such as SecureMail to filter + # this list. + # + # We build an array with the short_desc as a separate item to + # allow extensions to modify the summary without touching the + # bug object. + my $referenced_bugs = []; + foreach my $ref (@{ $user->visible_bugs(\@referenced_bugs) }) { + push @$referenced_bugs, { + bug => $ref, + id => $ref->id, + short_desc => $ref->short_desc, + }; + } + Bugzilla::Hook::process('bugmail_referenced_bugs', + { updated_bug => $bug, + referenced_bugs => $referenced_bugs }); + + my $sent_mail = sendMail( + { to => $user, bug => $bug, comments => $comments, date => $date, @@ -279,16 +332,12 @@ sub Send { $watching{$user_id} : undef, diffs => \@diffs, rels_which_want => \%rels_which_want, + referenced_bugs => $referenced_bugs, + dep_only => $params->{dep_only} }); + push(@sent, $user->login) if $sent_mail; } } - - if ($sent_mail) { - push(@sent, $user->login); - } - else { - push(@excluded, $user->login); - } } # When sending bugmail about a blocker being reopened or resolved, @@ -300,12 +349,12 @@ sub Send { $bug->{lastdiffed} = $end; } - return {'sent' => \@sent, 'excluded' => \@excluded}; + return {'sent' => \@sent}; } sub sendMail { my $params = shift; - + my $user = $params->{to}; my $bug = $params->{bug}; my @send_comments = @{ $params->{comments} }; @@ -314,6 +363,8 @@ sub sendMail { my $watchingRef = $params->{watchers}; my @diffs = @{ $params->{diffs} }; my $relRef = $params->{rels_which_want}; + my $referenced_bugs = $params->{referenced_bugs}; + my $dep_only = $params->{dep_only}; # Only display changes the user is allowed see. my @display_diffs; @@ -352,33 +403,113 @@ sub sendMail { push(@watchingrel, 'None') unless @watchingrel; push @watchingrel, map { user_id_to_login($_) } @$watchingRef; + # BMO: Use field descriptions instead of field names in header + my @changedfields = uniq map { $_->{field_desc} } @display_diffs; + my @changedfieldnames = uniq map { $_->{field_name} } @display_diffs; + + # BMO: Add a field to indicate when a comment was added + if (grep($_->type != CMT_ATTACHMENT_CREATED, @send_comments)) { + push(@changedfields, 'Comment Created'); + push(@changedfieldnames, 'comment'); + } + + # Add attachments.created to changedfields if one or more + # comments contain information about a new attachment + if (grep($_->type == CMT_ATTACHMENT_CREATED, @send_comments)) { + push(@changedfields, 'Attachment Created'); + push(@changedfieldnames, 'attachment.created'); + } + + my $bugmailtype = "changed"; + $bugmailtype = "new" if !$bug->lastdiffed; + $bugmailtype = "dep_changed" if $dep_only; + my $vars = { - date => $date, - to_user => $user, - bug => $bug, - reasons => \@reasons, - reasons_watch => \@reasons_watch, - reasonsheader => join(" ", @headerrel), + date => $date, + to_user => $user, + bug => $bug, + reasons => \@reasons, + reasons_watch => \@reasons_watch, + reasonsheader => join(" ", @headerrel), reasonswatchheader => join(" ", @watchingrel), - changer => $changer, - diffs => \@display_diffs, - changedfields => [uniq map { $_->{field_name} } @display_diffs], - new_comments => \@send_comments, - threadingmarker => build_thread_marker($bug->id, $user->id, !$bug->lastdiffed), + changer => $changer, + diffs => \@display_diffs, + changedfields => \@changedfields, + changedfieldnames => \@changedfieldnames, + new_comments => \@send_comments, + threadingmarker => build_thread_marker($bug->id, $user->id, !$bug->lastdiffed), + referenced_bugs => $referenced_bugs, + bugmailtype => $bugmailtype, }; - my $msg = _generate_bugmail($user, $vars); - MessageToMTA($msg); + + if (Bugzilla->params->{'use_mailer_queue'}) { + enqueue($vars); + } else { + MessageToMTA(_generate_bugmail($vars)); + } return 1; } +sub enqueue { + my ($vars) = @_; + # we need to flatten all objects to a hash before pushing to the job queue. + # the hashes need to be inflated in the dequeue method. + $vars->{bug} = _flatten_object($vars->{bug}); + $vars->{to_user} = _flatten_object($vars->{to_user}); + $vars->{changer} = _flatten_object($vars->{changer}); + $vars->{new_comments} = [ map { _flatten_object($_) } @{ $vars->{new_comments} } ]; + foreach my $diff (@{ $vars->{diffs} }) { + $diff->{who} = _flatten_object($diff->{who}); + if (exists $diff->{blocker}) { + $diff->{blocker} = _flatten_object($diff->{blocker}); + } + } + foreach my $reference (@{ $vars->{referenced_bugs} }) { + $reference->{bug} = _flatten_object($reference->{bug}); + } + Bugzilla->job_queue->insert('bug_mail', { vars => $vars }); +} + +sub dequeue { + my ($payload) = @_; + # clone the payload so we can modify it without impacting TheSchwartz's + # ability to process the job when we've finished + my $vars = dclone($payload); + # inflate objects + $vars->{bug} = Bugzilla::Bug->new_from_hash($vars->{bug}); + $vars->{to_user} = Bugzilla::User->new_from_hash($vars->{to_user}); + $vars->{changer} = Bugzilla::User->new_from_hash($vars->{changer}); + $vars->{new_comments} = [ map { Bugzilla::Comment->new_from_hash($_) } @{ $vars->{new_comments} } ]; + foreach my $diff (@{ $vars->{diffs} }) { + $diff->{who} = Bugzilla::User->new_from_hash($diff->{who}); + if (exists $diff->{blocker}) { + $diff->{blocker} = Bugzilla::Bug->new_from_hash($diff->{blocker}); + } + } + # generate bugmail and send + MessageToMTA(_generate_bugmail($vars), 1); +} + +sub _flatten_object { + my ($object) = @_; + # nothing to do if it's already flattened + return $object unless blessed($object); + # the same objects are used for each recipient, so cache the flattened hash + my $cache = Bugzilla->request_cache->{bugmail_flat_objects} ||= {}; + my $key = blessed($object) . '-' . $object->id; + return $cache->{$key} ||= $object->flatten_to_hash; +} + sub _generate_bugmail { - my ($user, $vars) = @_; + my ($vars) = @_; + my $user = $vars->{to_user}; my $template = Bugzilla->template_inner($user->setting('lang')); my ($msg_text, $msg_html, $msg_header); - + $template->process("email/bugmail-header.txt.tmpl", $vars, \$msg_header) || ThrowTemplateError($template->error()); + $template->process("email/bugmail.txt.tmpl", $vars, \$msg_text) || ThrowTemplateError($template->error()); @@ -395,7 +526,7 @@ sub _generate_bugmail { || ThrowTemplateError($template->error()); push @parts, Email::MIME->create( attributes => { - content_type => "text/html", + content_type => "text/html", }, body => $msg_html, ); @@ -403,6 +534,10 @@ sub _generate_bugmail { # TT trims the trailing newline, and threadingmarker may be ignored. my $email = new Email::MIME("$msg_header\n"); + + # For tracking/diagnostic purposes, add our hostname + $email->header_set('X-Generated-By' => hostname()); + if (scalar(@parts) == 1) { $email->content_type_set($parts[0]->content_type); } else { @@ -426,6 +561,7 @@ sub _get_diffs { my $diffs = $dbh->selectall_arrayref( "SELECT fielddefs.name AS field_name, + fielddefs.description AS field_desc, bugs_activity.bug_when, bugs_activity.removed AS old, bugs_activity.added AS new, bugs_activity.attach_id, bugs_activity.comment_id, bugs_activity.who @@ -434,11 +570,12 @@ sub _get_diffs { ON fielddefs.id = bugs_activity.fieldid WHERE bugs_activity.bug_id = ? $when_restriction - ORDER BY bugs_activity.bug_when", {Slice=>{}}, @args); + ORDER BY bugs_activity.bug_when, fielddefs.description", {Slice=>{}}, @args); + my $referenced_bugs = []; foreach my $diff (@$diffs) { - $user_cache->{$diff->{who}} ||= new Bugzilla::User($diff->{who}); - $diff->{who} = $user_cache->{$diff->{who}}; + $user_cache->{$diff->{who}} ||= new Bugzilla::User({ id => $diff->{who}, cache => 1 }); + $diff->{who} = $user_cache->{$diff->{who}}; if ($diff->{attach_id}) { $diff->{isprivate} = $dbh->selectrow_array( 'SELECT isprivate FROM attachments WHERE attach_id = ?', @@ -449,9 +586,13 @@ sub _get_diffs { $diff->{num} = $comment->count; $diff->{isprivate} = $diff->{new}; } + elsif ($diff->{field_name} eq 'dependson' || $diff->{field_name} eq 'blocked') { + push @$referenced_bugs, grep { /^\d+$/ } split(/[\s,]+/, $diff->{old}); + push @$referenced_bugs, grep { /^\d+$/ } split(/[\s,]+/, $diff->{new}); + } } - return @$diffs; + return ($diffs, $referenced_bugs); } sub _get_new_bugmail_fields { @@ -459,6 +600,20 @@ sub _get_new_bugmail_fields { my @fields = @{ Bugzilla->fields({obsolete => 0, in_new_bugmail => 1}) }; my @diffs; + # Show fields in the same order as the DEFAULT_FIELDS list, which mirrors + # 4.0's behavour and provides sane grouping of similar fields. + # Any additional fields are sorted by descrsiption + my @prepend; + foreach my $name (map { $_->{name} } Bugzilla::Field::DEFAULT_FIELDS) { + my $idx = firstidx { $_->name eq $name } @fields; + if ($idx != -1) { + push(@prepend, $fields[$idx]); + splice(@fields, $idx, 1); + } + } + @fields = sort { $a->description cmp $b->description } @fields; + @fields = (@prepend, @fields); + foreach my $field (@fields) { my $name = $field->name; my $value = $bug->$name; @@ -484,7 +639,9 @@ sub _get_new_bugmail_fields { # If there isn't anything to show, don't include this header. next unless $value; - push(@diffs, {field_name => $name, new => $value}); + push(@diffs, {field_name => $name, + field_desc => $field->description, + new => $value}); } return @diffs; diff --git a/Bugzilla/BugUrl.pm b/Bugzilla/BugUrl.pm index 837c0d4fe..f59ec3601 100644 --- a/Bugzilla/BugUrl.pm +++ b/Bugzilla/BugUrl.pm @@ -69,6 +69,8 @@ use constant SUB_CLASSES => qw( Bugzilla::BugUrl::Trac Bugzilla::BugUrl::MantisBT Bugzilla::BugUrl::SourceForge + Bugzilla::BugUrl::GitHub + Bugzilla::BugUrl::MozSupport ); ############################### diff --git a/Bugzilla/BugUrl/GitHub.pm b/Bugzilla/BugUrl/GitHub.pm new file mode 100644 index 000000000..63be65bed --- /dev/null +++ b/Bugzilla/BugUrl/GitHub.pm @@ -0,0 +1,36 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::BugUrl::GitHub; +use strict; +use base qw(Bugzilla::BugUrl); + +############################### +#### Methods #### +############################### + +sub should_handle { + my ($class, $uri) = @_; + + # GitHub issue URLs have only one form: + # https://github.com/USER_OR_TEAM_OR_ORGANIZATION_NAME/REPOSITORY_NAME/issues/111 + return ($uri->authority =~ /^github.com$/i + and $uri->path =~ m|^/[^/]+/[^/]+/issues/\d+$|) ? 1 : 0; +} + +sub _check_value { + my ($class, $uri) = @_; + + $uri = $class->SUPER::_check_value($uri); + + # GitHub HTTP URLs redirect to HTTPS, so just use the HTTPS scheme. + $uri->scheme('https'); + + return $uri; +} + +1; diff --git a/Bugzilla/BugUrl/MozSupport.pm b/Bugzilla/BugUrl/MozSupport.pm new file mode 100644 index 000000000..f78bec799 --- /dev/null +++ b/Bugzilla/BugUrl/MozSupport.pm @@ -0,0 +1,36 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::BugUrl::MozSupport; +use strict; +use base qw(Bugzilla::BugUrl); + +############################### +#### Methods #### +############################### + +sub should_handle { + my ($class, $uri) = @_; + + # Mozilla support questions normally have the form: + # https://support.mozilla.org/<language>/questions/<id> + return ($uri->authority =~ /^support.mozilla.org$/i + and $uri->path =~ m|^(/[^/]+)?/questions/\d+$|) ? 1 : 0; +} + +sub _check_value { + my ($class, $uri) = @_; + + $uri = $class->SUPER::_check_value($uri); + + # Support.mozilla.org redirects to https automatically + $uri->scheme('https'); + + return $uri; +} + +1; diff --git a/Bugzilla/BugUserLastVisit.pm b/Bugzilla/BugUserLastVisit.pm new file mode 100644 index 000000000..6a7b7f3ef --- /dev/null +++ b/Bugzilla/BugUserLastVisit.pm @@ -0,0 +1,92 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::BugUserLastVisit; + +use 5.10.1; +use strict; + +use parent qw(Bugzilla::Object); + +##################################################################### +# Overriden Constants that are used as methods +##################################################################### + +use constant DB_TABLE => 'bug_user_last_visit'; +use constant DB_COLUMNS => qw( id user_id bug_id last_visit_ts ); +use constant UPDATE_COLUMNS => qw( last_visit_ts ); +use constant VALIDATORS => {}; +use constant LIST_ORDER => 'id'; +use constant NAME_FIELD => 'id'; + +# turn off auditing and exclude these objects from memcached +use constant { AUDIT_CREATES => 0, + AUDIT_UPDATES => 0, + AUDIT_REMOVES => 0, + USE_MEMCACHED => 0 }; + +##################################################################### +# Provide accessors for our columns +##################################################################### + +sub id { return $_[0]->{id} } +sub bug_id { return $_[0]->{bug_id} } +sub user_id { return $_[0]->{user_id} } +sub last_visit_ts { return $_[0]->{last_visit_ts} } + +sub user { + my $self = shift; + + $self->{user} //= Bugzilla::User->new({id => $self->user_id, cache => 1}); + return $self->{user}; +} + +1; +__END__ + +=head1 NAME + +Bugzilla::BugUserLastVisit - Model for BugUserLastVisit bug search data + +=head1 SYNOPSIS + + use Bugzilla::BugUserLastVisit; + + my $lv = Bugzilla::BugUserLastVisit->new($id); + + # Class Functions + $user = Bugzilla::BugUserLastVisit->create({ + bug_id => $bug_id, + user_id => $user_id, + last_visit_ts => $last_visit_ts + }); + +=head1 DESCRIPTION + +This package handles Bugzilla BugUserLastVisit. + +C<Bugzilla::BugUserLastVisit> is an implementation of L<Bugzilla::Object>, and +thus provides all the methods of L<Bugzilla::Object> in addition to the methods +listed below. + +=head1 METHODS + +=head2 Accessor Methods + +=over + +=item C<id> + +=item C<bug_id> + +=item C<user_id> + +=item C<last_visit_ts> + +=item C<user> + +=back diff --git a/Bugzilla/CGI.pm b/Bugzilla/CGI.pm index 4dd223a31..a12fb284b 100644 --- a/Bugzilla/CGI.pm +++ b/Bugzilla/CGI.pm @@ -73,11 +73,22 @@ sub new { # Make sure our outgoing cookie list is empty on each invocation $self->{Bugzilla_cookie_list} = []; + # Path-Info is of no use for Bugzilla and interacts badly with IIS. + # Moreover, it causes unexpected behaviors, such as totally breaking + # the rendering of pages. + my $script = basename($0); + if ($self->path_info) { + my @whitelist = ("rest.cgi"); + Bugzilla::Hook::process('path_info_whitelist', { whitelist => \@whitelist }); + if (!grep($_ eq $script, @whitelist)) { + print $self->redirect($self->url(-path => 0, -query => 1)); + } + } + # Send appropriate charset $self->charset(Bugzilla->params->{'utf8'} ? 'UTF-8' : ''); # Redirect to urlbase/sslbase if we are not viewing an attachment. - my $script = basename($0); if ($self->url_is_attachment_base and $script ne 'attachment.cgi') { $self->redirect_to_urlbase(); } @@ -127,7 +138,8 @@ sub canonicalise_query { my $esc_key = url_quote($key); foreach my $value ($self->param($key)) { - if (defined($value)) { + # Omit params with an empty value + if (defined($value) && $value ne '') { my $esc_value = url_quote($value); push(@parameters, "$esc_key=$esc_value"); @@ -224,6 +236,26 @@ sub clean_search_url { } } +sub check_etag { + my ($self, $valid_etag) = @_; + + # ETag support. + my $if_none_match = $self->http('If-None-Match'); + return if !$if_none_match; + + my @if_none = split(/[\s,]+/, $if_none_match); + foreach my $possible_etag (@if_none) { + # remove quotes from begin and end of the string + $possible_etag =~ s/^\"//g; + $possible_etag =~ s/\"$//g; + if ($possible_etag eq $valid_etag or $possible_etag eq '*') { + return 1; + } + } + + return 0; +} + # Overwrite to ensure nph doesn't get set, and unset HEADERS_ONCE sub multipart_init { my $self = shift; @@ -278,9 +310,24 @@ sub multipart_start { $headers .= "Set-Cookie: ${cookie}${CGI::CRLF}"; } $headers .= $CGI::CRLF; + $self->{_multipart_in_progress} = 1; return $headers; } +sub close_standby_message { + my ($self, $contenttype, $disposition) = @_; + + if ($self->{_multipart_in_progress}) { + print $self->multipart_end(); + print $self->multipart_start(-type => $contenttype, + -content_disposition => $disposition); + } + else { + print $self->header(-type => $contenttype, + -content_disposition => $disposition); + } +} + # Override header so we can add the cookies in sub header { my $self = shift; @@ -316,6 +363,10 @@ sub header { unshift(@_, '-x_frame_options' => 'SAMEORIGIN'); } + if ($self->{'_content_disp'}) { + unshift(@_, '-content_disposition' => $self->{'_content_disp'}); + } + # Add X-XSS-Protection header to prevent simple XSS attacks # and enforce the blocking (rather than the rewriting) mode. unshift(@_, '-x_xss_protection' => '1; mode=block'); @@ -432,6 +483,10 @@ sub remove_cookie { # URLs that get POSTed to buglist.cgi. sub redirect_search_url { my $self = shift; + + # If there is no parameter, there is nothing to do. + return unless $self->param; + # If we're retreiving an old list, we never need to redirect or # do anything related to Bugzilla::Search::Recent. return if $self->param('regetlastlist'); @@ -470,9 +525,9 @@ sub redirect_search_url { # GET requests that lacked a list_id are always redirected. POST requests # are only redirected if they're under the CGI_URI_LIMIT though. - my $uri_length = length($self->self_url()); - if ($self->request_method() ne 'POST' or $uri_length < CGI_URI_LIMIT) { - print $self->redirect(-url => $self->self_url()); + my $self_url = $self->self_url(); + if ($self->request_method() ne 'POST' or length($self_url) < CGI_URI_LIMIT) { + print $self->redirect(-url => $self_url); exit; } } @@ -526,7 +581,23 @@ sub url_is_attachment_base { $regex =~ s/\\\%bugid\\\%/\\d+/; } $regex = "^$regex"; - return ($self->self_url =~ $regex) ? 1 : 0; + return ($self->url =~ $regex) ? 1 : 0; +} + +sub set_dated_content_disp { + my ($self, $type, $prefix, $ext) = @_; + + my @time = localtime(time()); + my $date = sprintf "%04d-%02d-%02d", 1900+$time[5], $time[4]+1, $time[3]; + my $filename = "$prefix-$date.$ext"; + + $filename =~ s/\s/_/g; # Remove whitespace to avoid HTTP header tampering + $filename =~ s/\\/_/g; # Remove backslashes as well + $filename =~ s/"/\\"/g; # escape quotes + + my $disposition = "$type; filename=\"$filename\""; + + $self->{'_content_disp'} = $disposition; } ########################## @@ -603,7 +674,9 @@ I<Bugzilla::CGI> also includes additional functions. =item C<canonicalise_query(@exclude)> -This returns a sorted string of the parameters, suitable for use in a url. +This returns a sorted string of the parameters whose values are non-empty, +suitable for use in a url. + Values in C<@exclude> are not included in the result. =item C<send_cookie> @@ -636,6 +709,11 @@ instead of calling this directly. Redirects from the current URL to one prefixed by the urlbase parameter. +=item C<set_dated_content_disp> + +Sets an appropriate date-dependent value for the Content Disposition header +for a downloadable resource. + =back =head1 SEE ALSO diff --git a/Bugzilla/Classification.pm b/Bugzilla/Classification.pm index 88ec4eb3b..a989a40f2 100644 --- a/Bugzilla/Classification.pm +++ b/Bugzilla/Classification.pm @@ -31,6 +31,8 @@ use base qw(Bugzilla::Field::ChoiceInterface Bugzilla::Object); #### Initialization #### ############################### +use constant IS_CONFIG => 1; + use constant DB_TABLE => 'classifications'; use constant LIST_ORDER => 'sortkey, name'; @@ -56,6 +58,7 @@ use constant VALIDATORS => { ############################### #### Constructors ##### ############################### + sub remove_from_db { my $self = shift; my $dbh = Bugzilla->dbh; @@ -63,9 +66,19 @@ sub remove_from_db { ThrowUserError("classification_not_deletable") if ($self->id == 1); $dbh->bz_start_transaction(); + # Reclassify products to the default classification, if needed. - $dbh->do("UPDATE products SET classification_id = 1 - WHERE classification_id = ?", undef, $self->id); + my $product_ids = $dbh->selectcol_arrayref( + 'SELECT id FROM products WHERE classification_id = ?', undef, $self->id); + + if (@$product_ids) { + $dbh->do('UPDATE products SET classification_id = 1 WHERE ' + . $dbh->sql_in('id', $product_ids)); + foreach my $id (@$product_ids) { + Bugzilla->memcached->clear({ table => 'products', id => $id }); + } + Bugzilla->memcached->clear_config(); + } $self->SUPER::remove_from_db(); diff --git a/Bugzilla/Comment.pm b/Bugzilla/Comment.pm index ee342fb2d..979019541 100644 --- a/Bugzilla/Comment.pm +++ b/Bugzilla/Comment.pm @@ -26,11 +26,14 @@ package Bugzilla::Comment; use base qw(Bugzilla::Object); use Bugzilla::Attachment; +use Bugzilla::Comment::TagWeights; use Bugzilla::Constants; use Bugzilla::Error; +use Bugzilla::Hook; use Bugzilla::User; use Bugzilla::Util; +use List::Util qw(first); use Scalar::Util qw(blessed); ############################### @@ -92,21 +95,91 @@ use constant VALIDATOR_DEPENDENCIES => { sub update { my $self = shift; - my $changes = $self->SUPER::update(@_); - $self->bug->_sync_fulltext(); + my ($changes, $old_comment) = $self->SUPER::update(@_); + + if (exists $changes->{'thetext'} || exists $changes->{'isprivate'}) { + $self->bug->_sync_fulltext( update_comments => 1); + } + + my @old_tags = @{ $old_comment->tags }; + my @new_tags = @{ $self->tags }; + my ($removed_tags, $added_tags) = diff_arrays(\@old_tags, \@new_tags); + + if (@$removed_tags || @$added_tags) { + my $dbh = Bugzilla->dbh; + my $when = $dbh->selectrow_array("SELECT LOCALTIMESTAMP(0)"); + my $sth_delete = $dbh->prepare( + "DELETE FROM longdescs_tags WHERE comment_id = ? AND tag = ?" + ); + my $sth_insert = $dbh->prepare( + "INSERT INTO longdescs_tags(comment_id, tag) VALUES (?, ?)" + ); + my $sth_activity = $dbh->prepare( + "INSERT INTO longdescs_tags_activity + (bug_id, comment_id, who, bug_when, added, removed) + VALUES (?, ?, ?, ?, ?, ?)" + ); + + foreach my $tag (@$removed_tags) { + my $weighted = Bugzilla::Comment::TagWeights->new({ name => $tag }); + if ($weighted) { + if ($weighted->weight == 1) { + $weighted->remove_from_db(); + } else { + $weighted->set_weight($weighted->weight - 1); + $weighted->update(); + } + } + trick_taint($tag); + $sth_delete->execute($self->id, $tag); + $sth_activity->execute( + $self->bug_id, $self->id, Bugzilla->user->id, $when, '', $tag); + } + + foreach my $tag (@$added_tags) { + my $weighted = Bugzilla::Comment::TagWeights->new({ name => $tag }); + if ($weighted) { + $weighted->set_weight($weighted->weight + 1); + $weighted->update(); + } else { + Bugzilla::Comment::TagWeights->create({ tag => $tag, weight => 1 }); + } + trick_taint($tag); + $sth_insert->execute($self->id, $tag); + $sth_activity->execute( + $self->bug_id, $self->id, Bugzilla->user->id, $when, $tag, ''); + } + } + return $changes; } -# Speeds up displays of comment lists by loading all ->author objects -# at once for a whole list. +# Speeds up displays of comment lists by loading all author objects and tags at +# once for a whole list. sub preload { my ($class, $comments) = @_; + # Author my %user_ids = map { $_->{who} => 1 } @$comments; my $users = Bugzilla::User->new_from_list([keys %user_ids]); my %user_map = map { $_->id => $_ } @$users; foreach my $comment (@$comments) { $comment->{author} = $user_map{$comment->{who}}; } + # Tags + my $dbh = Bugzilla->dbh; + my @comment_ids = map { $_->id } @$comments; + my %comment_map = map { $_->id => $_ } @$comments; + my $rows = $dbh->selectall_arrayref( + "SELECT comment_id, " . $dbh->sql_group_concat('tag', "','") . " + FROM longdescs_tags + WHERE " . $dbh->sql_in('comment_id', \@comment_ids) . " + GROUP BY comment_id"); + foreach my $row (@$rows) { + $comment_map{$row->[0]}->{tags} = [ split(/,/, $row->[1]) ]; + } + foreach my $comment (@$comments) { + $comment->{tags} //= []; + } } ############################### @@ -126,6 +199,39 @@ sub work_time { sub type { return $_[0]->{'type'}; } sub extra_data { return $_[0]->{'extra_data'} } +sub tags { + my ($self) = @_; + return [] unless Bugzilla->params->{'comment_taggers_group'}; + $self->{'tags'} ||= Bugzilla->dbh->selectcol_arrayref( + "SELECT tag + FROM longdescs_tags + WHERE comment_id = ? + ORDER BY tag", + undef, $self->id); + return $self->{'tags'}; +} + +sub collapsed { + my ($self) = @_; + return 0 unless Bugzilla->params->{'comment_taggers_group'}; + return $self->{collapsed} if exists $self->{collapsed}; + $self->{collapsed} = 0; + Bugzilla->request_cache->{comment_tags_collapsed} + ||= [ split(/\s*,\s*/, Bugzilla->params->{'collapsed_comment_tags'}) ]; + my @collapsed_tags = @{ Bugzilla->request_cache->{comment_tags_collapsed} }; + foreach my $my_tag (@{ $self->tags }) { + $my_tag = lc($my_tag); + foreach my $collapsed_tag (@collapsed_tags) { + if ($my_tag eq lc($collapsed_tag)) { + $self->{collapsed} = 1; + last; + } + } + last if $self->{collapsed}; + } + return $self->{collapsed}; +} + sub bug { my $self = shift; require Bugzilla::Bug; @@ -143,14 +249,15 @@ sub is_about_attachment { sub attachment { my ($self) = @_; return undef if not $self->is_about_attachment; - $self->{attachment} ||= new Bugzilla::Attachment($self->extra_data); + $self->{attachment} ||= + new Bugzilla::Attachment({ id => $self->extra_data, cache => 1 }); return $self->{attachment}; } sub author { my $self = shift; - $self->{'author'} ||= new Bugzilla::User($self->{'who'}); - return $self->{'author'}; + return $self->{'author'} + ||= new Bugzilla::User({ id => $self->{'who'}, cache => 1 }); } sub body_full { @@ -181,6 +288,30 @@ sub set_is_private { $_[0]->set('isprivate', $_[1]); } sub set_type { $_[0]->set('type', $_[1]); } sub set_extra_data { $_[0]->set('extra_data', $_[1]); } +sub add_tag { + my ($self, $tag) = @_; + $tag = $self->_check_tag($tag); + + my $tags = $self->tags; + return if grep { lc($tag) eq lc($_) } @$tags; + push @$tags, $tag; + $self->{'tags'} = [ sort @$tags ]; + Bugzilla::Hook::process("comment_after_add_tag", + { comment => $self, tag => $tag }); +} + +sub remove_tag { + my ($self, $tag) = @_; + $tag = $self->_check_tag($tag); + + my $tags = $self->tags; + my $index = first { lc($tags->[$_]) eq lc($tag) } 0..scalar(@$tags) - 1; + return unless defined $index; + splice(@$tags, $index, 1); + Bugzilla::Hook::process("comment_after_remove_tag", + { comment => $self, tag => $tag }); +} + ############## # Validators # ############## @@ -323,6 +454,17 @@ sub _check_isprivate { return $isprivate ? 1 : 0; } +sub _check_tag { + my ($invocant, $tag) = @_; + length($tag) < MIN_COMMENT_TAG_LENGTH + and ThrowUserError('comment_tag_too_short', { tag => $tag }); + length($tag) > MAX_COMMENT_TAG_LENGTH + and ThrowUserError('comment_tag_too_long', { tag => $tag }); + $tag =~ /^[\w\d\._-]+$/ + or ThrowUserError('comment_tag_invalid', { tag => $tag }); + return $tag; +} + sub count { my ($self) = @_; @@ -337,7 +479,7 @@ sub count { undef, $self->bug_id, $self->creation_ts); return --$self->{'count'}; -} +} 1; @@ -383,7 +525,7 @@ C<string> Time spent as related to this comment. =item C<is_private> -C<boolean> Comment is marked as private +C<boolean> Comment is marked as private. =item C<already_wrapped> @@ -398,6 +540,54 @@ L<Bugzilla::User> who created the comment. C<int> The position this comment is located in the full list of comments for a bug starting from 0. +=item C<collapsed> + +C<boolean> Comment should be displayed as collapsed by default. + +=item C<tags> + +C<array of strings> The tags attached to the comment. + +=item C<add_tag> + +=over + +=item B<Description> + +Attaches the specified tag to the comment. + +=item B<Params> + +=over + +=item C<tag> + +C<string> The tag to attach. + +=back + +=back + +=item C<remove_tag> + +=over + +=item B<Description> + +Detaches the specified tag from the comment. + +=item B<Params> + +=over + +=item C<tag> + +C<string> The tag to detach. + +=back + +=back + =item C<body_full> =over diff --git a/Bugzilla/Comment/TagWeights.pm b/Bugzilla/Comment/TagWeights.pm new file mode 100644 index 000000000..f1a220a47 --- /dev/null +++ b/Bugzilla/Comment/TagWeights.pm @@ -0,0 +1,77 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Comment::TagWeights; + +use 5.10.1; +use strict; + +use parent qw(Bugzilla::Object); + +use Bugzilla::Constants; + +# No auditing required +use constant AUDIT_CREATES => 0; +use constant AUDIT_UPDATES => 0; +use constant AUDIT_REMOVES => 0; + +use constant DB_COLUMNS => qw( + id + tag + weight +); + +use constant UPDATE_COLUMNS => qw( + weight +); + +use constant DB_TABLE => 'longdescs_tags_weights'; +use constant ID_FIELD => 'id'; +use constant NAME_FIELD => 'tag'; +use constant LIST_ORDER => 'weight DESC'; +use constant VALIDATORS => { }; + +# There's no gain to caching these objects +use constant USE_MEMCACHED => 0; + +sub tag { return $_[0]->{'tag'} } +sub weight { return $_[0]->{'weight'} } + +sub set_weight { $_[0]->set('weight', $_[1]); } + +1; + +=head1 NAME + +Comment::TagWeights - Bugzilla comment weighting class. + +=head1 DESCRIPTION + +TagWeights.pm represents a Comment::TagWeight object. It is an implementation +of L<Bugzilla::Object>, and thus provides all methods that L<Bugzilla::Object> +provides. + +TagWeights is used to quickly find tags and order by their usage count. + +=head1 PROPERTIES + +=over + +=item C<tag> + +C<getter string> The tag + +=item C<weight> + +C<getter int> The tag's weight. The value returned corresponds to the number of +comments with this tag attached. + +=item C<set_weight> + +C<setter int> Set the tag's weight. + +=back diff --git a/Bugzilla/Component.pm b/Bugzilla/Component.pm index dc3cc1b9e..0fe6fb25a 100644 --- a/Bugzilla/Component.pm +++ b/Bugzilla/Component.pm @@ -352,30 +352,30 @@ sub bug_ids { sub default_assignee { my $self = shift; - - if (!defined $self->{'default_assignee'}) { - $self->{'default_assignee'} = - new Bugzilla::User($self->{'initialowner'}); - } - return $self->{'default_assignee'}; + return $self->{'default_assignee'} + ||= new Bugzilla::User({ id => $self->{'initialowner'}, cache => 1 }); } sub default_qa_contact { my $self = shift; if (!defined $self->{'default_qa_contact'}) { - $self->{'default_qa_contact'} = - new Bugzilla::User($self->{'initialqacontact'}); + my $params = $self->{'initialqacontact'} + ? { id => $self->{'initialqacontact'}, cache => 1 } + : $self->{'initialqacontact'}; + $self->{'default_qa_contact'} = new Bugzilla::User($params); } return $self->{'default_qa_contact'}; } sub flag_types { - my $self = shift; + my ($self, $params) = @_; + $params ||= {}; if (!defined $self->{'flag_types'}) { my $flagtypes = Bugzilla::FlagType::match({ product_id => $self->product_id, - component_id => $self->id }); + component_id => $self->id, + %$params }); $self->{'flag_types'} = {}; $self->{'flag_types'}->{'bug'} = diff --git a/Bugzilla/Config.pm b/Bugzilla/Config.pm index 990fd8dd2..3e9b793a6 100644 --- a/Bugzilla/Config.pm +++ b/Bugzilla/Config.pm @@ -35,7 +35,6 @@ use strict; use base qw(Exporter); use Bugzilla::Constants; use Bugzilla::Hook; -use Bugzilla::Install::Filesystem qw(fix_file_permissions); use Data::Dumper; use File::Temp; @@ -301,7 +300,10 @@ sub write_params { rename $tmpname, $param_file or die "Can't rename $tmpname to $param_file: $!"; - fix_file_permissions($param_file); + # It's not common to edit parameters and loading + # Bugzilla::Install::Filesystem is slow. + require Bugzilla::Install::Filesystem; + Bugzilla::Install::Filesystem::fix_file_permissions($param_file); # And now we have to reset the params cache so that Bugzilla will re-read # them. diff --git a/Bugzilla/Config/Admin.pm b/Bugzilla/Config/Admin.pm index e6141cf9e..769e3170b 100644 --- a/Bugzilla/Config/Admin.pm +++ b/Bugzilla/Config/Admin.pm @@ -56,6 +56,13 @@ sub get_param_list { name => 'allowuserdeletion', type => 'b', default => 0 + }, + + { + name => 'last_visit_keep_days', + type => 't', + default => 10, + checker => \&check_numeric }); return @param_list; } diff --git a/Bugzilla/Config/Advanced.pm b/Bugzilla/Config/Advanced.pm index 941cefc4f..4b57df24d 100644 --- a/Bugzilla/Config/Advanced.pm +++ b/Bugzilla/Config/Advanced.pm @@ -63,6 +63,49 @@ use constant get_param_list => ( default => 'off', checker => \&check_multi }, + + { + name => 'disable_bug_updates', + type => 'b', + default => 0 + }, + + { + name => 'sentry_uri', + type => 't', + default => '', + }, + + { + name => 'metrics_enabled', + type => 'b', + default => 0 + }, + { + name => 'metrics_user_ids', + type => 't', + default => '3881,5038,5898,13647,20209,251051,373476,409787' + }, + { + name => 'metrics_elasticsearch_server', + type => 't', + default => '127.0.0.1:9200' + }, + { + name => 'metrics_elasticsearch_index', + type => 't', + default => 'bmo-metrics' + }, + { + name => 'metrics_elasticsearch_type', + type => 't', + default => 'timings' + }, + { + name => 'metrics_elasticsearch_ttl', + type => 't', + default => '1210000000' # 14 days + }, ); 1; diff --git a/Bugzilla/Config/Auth.pm b/Bugzilla/Config/Auth.pm index a61cab5a2..d70c1f81e 100644 --- a/Bugzilla/Config/Auth.pm +++ b/Bugzilla/Config/Auth.pm @@ -97,6 +97,12 @@ sub get_param_list { }, { + name => 'webservice_email_filter', + type => 'b', + default => 0 + }, + + { name => 'emailregexp', type => 't', default => q:^[\\w\\.\\+\\-=]+@[\\w\\.\\-]+\\.[\\w\\-]+$:, diff --git a/Bugzilla/Config/BugFields.pm b/Bugzilla/Config/BugFields.pm index d0de9dac6..490d945ea 100644 --- a/Bugzilla/Config/BugFields.pm +++ b/Bugzilla/Config/BugFields.pm @@ -113,7 +113,13 @@ sub get_param_list { choices => ['', @legal_OS], default => '', checker => \&check_opsys - } ); + }, + + { + name => 'collapsed_comment_tags', + type => 't', + default => 'obsolete, spam', + }); return @param_list; } diff --git a/Bugzilla/Config/Common.pm b/Bugzilla/Config/Common.pm index 00c699217..edd5872e1 100644 --- a/Bugzilla/Config/Common.pm +++ b/Bugzilla/Config/Common.pm @@ -52,6 +52,7 @@ use base qw(Exporter); check_mail_delivery_method check_notification check_utf8 check_bug_status check_smtp_auth check_theschwartz_available check_maxattachmentsize check_email + check_comment_taggers_group ); # Checking functions for the various values @@ -369,6 +370,14 @@ sub check_theschwartz_available { return ""; } +sub check_comment_taggers_group { + my $group_name = shift; + if ($group_name && !Bugzilla->feature('jsonrpc')) { + return "Comment tagging requires installation of the JSONRPC feature"; + } + return check_group($group_name); +} + # OK, here are the parameter definitions themselves. # # Each definition is a hash with keys: @@ -467,4 +476,9 @@ Checks that the value is a valid number Checks that the value is a valid regexp +=item C<check_comment_taggers_group> + +Checks that the required modules for comment tagging are installed, and that a +valid group is provided. + =back diff --git a/Bugzilla/Config/GroupSecurity.pm b/Bugzilla/Config/GroupSecurity.pm index 6296583d9..24eddaf6b 100644 --- a/Bugzilla/Config/GroupSecurity.pm +++ b/Bugzilla/Config/GroupSecurity.pm @@ -79,7 +79,15 @@ sub get_param_list { default => 'editbugs', checker => \&check_group }, - + + { + name => 'comment_taggers_group', + type => 's', + choices => \&_get_all_group_names, + default => 'editbugs', + checker => \&check_comment_taggers_group + }, + { name => 'debug_group', type => 's', @@ -107,4 +115,5 @@ sub _get_all_group_names { unshift(@group_names, ''); return \@group_names; } + 1; diff --git a/Bugzilla/Config/Memcached.pm b/Bugzilla/Config/Memcached.pm new file mode 100644 index 000000000..08d8ce0e7 --- /dev/null +++ b/Bugzilla/Config/Memcached.pm @@ -0,0 +1,32 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Config::Memcached; + +use 5.10.1; +use strict; + +use Bugzilla::Config::Common; + +our $sortkey = 1550; + +sub get_param_list { + return ( + { + name => 'memcached_servers', + type => 't', + default => '' + }, + { + name => 'memcached_namespace', + type => 't', + default => 'bugzilla:', + }, + ); +} + +1; diff --git a/Bugzilla/Constants.pm b/Bugzilla/Constants.pm index 862c18917..1383ca7fe 100644 --- a/Bugzilla/Constants.pm +++ b/Bugzilla/Constants.pm @@ -39,12 +39,15 @@ use Memoize; @Bugzilla::Constants::EXPORT = qw( BUGZILLA_VERSION + REST_DOC REMOTE_FILE LOCAL_FILE bz_locations + CONCATENATE_ASSETS + IS_NULL NOT_NULL @@ -90,6 +93,9 @@ use Memoize; COMMENT_COLS MAX_COMMENT_LENGTH + MIN_COMMENT_TAG_LENGTH + MAX_COMMENT_TAG_LENGTH + CMT_NORMAL CMT_DUPE_OF CMT_HAS_DUPE @@ -105,7 +111,7 @@ use Memoize; POS_EVENTS EVT_OTHER EVT_ADDED_REMOVED EVT_COMMENT EVT_ATTACHMENT EVT_ATTACHMENT_DATA EVT_PROJ_MANAGEMENT EVT_OPENED_CLOSED EVT_KEYWORD EVT_CC EVT_DEPEND_BLOCK - EVT_BUG_CREATED + EVT_BUG_CREATED EVT_COMPONENT NEG_EVENTS EVT_UNCONFIRMED EVT_CHANGED_BY_ME @@ -125,10 +131,14 @@ use Memoize; FIELD_TYPE_MULTI_SELECT FIELD_TYPE_TEXTAREA FIELD_TYPE_DATETIME + FIELD_TYPE_DATE FIELD_TYPE_BUG_ID FIELD_TYPE_BUG_URLS FIELD_TYPE_KEYWORDS + FIELD_TYPE_EXTENSION + FIELD_TYPE_HIGHEST_PLUS_ONE + EMPTY_DATETIME_REGEX ABNORMAL_SELECTS @@ -141,12 +151,14 @@ use Memoize; USAGE_MODE_EMAIL USAGE_MODE_JSON USAGE_MODE_TEST + USAGE_MODE_REST ERROR_MODE_WEBPAGE ERROR_MODE_DIE ERROR_MODE_DIE_SOAP_FAULT ERROR_MODE_JSON_RPC ERROR_MODE_TEST + ERROR_MODE_REST COLOR_ERROR COLOR_SUCCESS @@ -182,6 +194,7 @@ use Memoize; MAX_FREETEXT_LENGTH MAX_BUG_URL_LENGTH MAX_POSSIBLE_DUPLICATES + MAX_WEBDOT_BUGS PASSWORD_DIGEST_ALGORITHM PASSWORD_SALT_LENGTH @@ -204,10 +217,19 @@ use Memoize; # Bugzilla version use constant BUGZILLA_VERSION => "4.2.11"; +# A base link to the current REST Documentation. We place it here +# as it will need to be updated to whatever the current release is. +use constant REST_DOC => "http://www.bugzilla.org/docs/tip/en/html/api/"; + # Location of the remote and local XML files to track new releases. use constant REMOTE_FILE => 'http://updates.bugzilla.org/bugzilla-update.xml'; use constant LOCAL_FILE => 'bugzilla-update.xml'; # Relative to datadir. +# When true CSS and JavaScript assets will be concatanted and minified at +# run-time, to reduce the number of requests required to render a page. +# Setting this to a false value can help debugging. +use constant CONCATENATE_ASSETS => 1; + # These are unique values that are unlikely to match a string or a number, # to be used in criteria for match() functions and other things. They start # and end with spaces because most Bugzilla stuff has trim() called on it, @@ -262,7 +284,8 @@ use constant AUTH_NO_SUCH_USER => 5; use constant AUTH_LOCKOUT => 6; # The minimum length a password must have. -use constant USER_PASSWORD_MIN_LENGTH => 6; +# BMO uses 8 characters. +use constant USER_PASSWORD_MIN_LENGTH => 8; use constant LOGIN_OPTIONAL => 0; use constant LOGIN_NORMAL => 1; @@ -303,6 +326,10 @@ use constant COMMENT_COLS => 80; # Used in _check_comment(). Gives the max length allowed for a comment. use constant MAX_COMMENT_LENGTH => 65535; +# The minimum and maximum length of comment tags. +use constant MIN_COMMENT_TAG_LENGTH => 3; +use constant MAX_COMMENT_TAG_LENGTH => 24; + # The type of bug comments. use constant CMT_NORMAL => 0; use constant CMT_DUPE_OF => 1; @@ -355,11 +382,13 @@ use constant EVT_KEYWORD => 7; use constant EVT_CC => 8; use constant EVT_DEPEND_BLOCK => 9; use constant EVT_BUG_CREATED => 10; +use constant EVT_COMPONENT => 11; use constant POS_EVENTS => EVT_OTHER, EVT_ADDED_REMOVED, EVT_COMMENT, EVT_ATTACHMENT, EVT_ATTACHMENT_DATA, EVT_PROJ_MANAGEMENT, EVT_OPENED_CLOSED, EVT_KEYWORD, - EVT_CC, EVT_DEPEND_BLOCK, EVT_BUG_CREATED; + EVT_CC, EVT_DEPEND_BLOCK, EVT_BUG_CREATED, + EVT_COMPONENT; use constant EVT_UNCONFIRMED => 50; use constant EVT_CHANGED_BY_ME => 51; @@ -389,7 +418,8 @@ use constant SENDMAIL_PATH => '/usr/lib:/usr/sbin:/usr/ucblib'; # only storage but also logic. For example, we might add a "user" field type # whose values are stored in an integer column in the database but for which # we do more than we would do for a standard integer type (f.e. we might -# display a user picker). +# display a user picker). Fields of type FIELD_TYPE_EXTENSION should generally +# be ignored by the core code and is used primary by extensions. use constant FIELD_TYPE_UNKNOWN => 0; use constant FIELD_TYPE_FREETEXT => 1; @@ -400,6 +430,12 @@ use constant FIELD_TYPE_DATETIME => 5; use constant FIELD_TYPE_BUG_ID => 6; use constant FIELD_TYPE_BUG_URLS => 7; use constant FIELD_TYPE_KEYWORDS => 8; +use constant FIELD_TYPE_DATE => 9; +use constant FIELD_TYPE_EXTENSION => 99; + +# Add new field types above this line, and change the below value in the +# obvious fashion +use constant FIELD_TYPE_HIGHEST_PLUS_ONE => 100; use constant EMPTY_DATETIME_REGEX => qr/^[0\-:\sA-Za-z]+$/; @@ -431,8 +467,8 @@ use constant MAX_LOGIN_ATTEMPTS => 5; use constant LOGIN_LOCKOUT_INTERVAL => 30; # The maximum number of seconds the Strict-Transport-Security header -# will remain valid. Default is one week. -use constant MAX_STS_AGE => 604800; +# will remain valid. BMO uses one year. +use constant MAX_STS_AGE => 31536000; # Protocols which are considered as safe. use constant SAFE_PROTOCOLS => ('afs', 'cid', 'ftp', 'gopher', 'http', 'https', @@ -445,15 +481,16 @@ use constant LEGAL_CONTENT_TYPES => ('application', 'audio', 'image', 'message', use constant contenttypes => { - "html"=> "text/html" , - "rdf" => "application/rdf+xml" , - "atom"=> "application/atom+xml" , - "xml" => "application/xml" , - "js" => "application/x-javascript" , - "json"=> "application/json" , - "csv" => "text/csv" , - "png" => "image/png" , - "ics" => "text/calendar" , + "html" => "text/html" , + "rdf" => "application/rdf+xml" , + "atom" => "application/atom+xml" , + "xml" => "application/xml" , + "dtd" => "application/xml-dtd" , + "js" => "application/x-javascript" , + "json" => "application/json" , + "csv" => "text/csv" , + "png" => "image/png" , + "ics" => "text/calendar" , }; # Usage modes. Default USAGE_MODE_BROWSER. Use with Bugzilla->usage_mode. @@ -463,6 +500,7 @@ use constant USAGE_MODE_XMLRPC => 2; use constant USAGE_MODE_EMAIL => 3; use constant USAGE_MODE_JSON => 4; use constant USAGE_MODE_TEST => 5; +use constant USAGE_MODE_REST => 6; # Error modes. Default set by Bugzilla->usage_mode (so ERROR_MODE_WEBPAGE # usually). Use with Bugzilla->error_mode. @@ -471,6 +509,7 @@ use constant ERROR_MODE_DIE => 1; use constant ERROR_MODE_DIE_SOAP_FAULT => 2; use constant ERROR_MODE_JSON_RPC => 3; use constant ERROR_MODE_TEST => 4; +use constant ERROR_MODE_REST => 5; # The ANSI colors of messages that command-line scripts use use constant COLOR_ERROR => 'red'; @@ -562,6 +601,9 @@ use constant MAX_BUG_URL_LENGTH => 255; # will return. use constant MAX_POSSIBLE_DUPLICATES => 25; +# Maximum number of bugs to display in a dependency graph +use constant MAX_WEBDOT_BUGS => 2000; + # This is the name of the algorithm used to hash passwords before storing # them in the database. This can be any string that is valid to pass to # Perl's "Digest" module. Note that if you change this, it won't take @@ -635,7 +677,7 @@ sub _bz_locations { # make sure this still points to the CGIs. 'cgi_path' => $libpath, 'templatedir' => "$libpath/template", - 'template_cache' => "$datadir/template", + 'template_cache' => "$libpath/template_cache", 'project' => $project, 'localconfig' => "$libpath/$localconfig", 'datadir' => $datadir, @@ -650,6 +692,7 @@ sub _bz_locations { # The script should really generate these graphs directly... 'webdotdir' => "$datadir/webdot", 'extensionsdir' => "$libpath/extensions", + 'assetsdir' => "$datadir/assets", }; } diff --git a/Bugzilla/DB.pm b/Bugzilla/DB.pm index 1f9c31518..183f619a5 100644 --- a/Bugzilla/DB.pm +++ b/Bugzilla/DB.pm @@ -43,6 +43,8 @@ use Bugzilla::Util; use Bugzilla::Error; use Bugzilla::DB::Schema; +use Bugzilla::Metrics::Mysql; + use List::Util qw(max); use Storable qw(dclone); @@ -148,6 +150,12 @@ sub _connect { . " localconfig: " . $@); # instantiate the correct DB specific module + + # BMO - enable instrumentation of db calls + if (Bugzilla->metrics_enabled) { + $pkg_module = 'Bugzilla::Metrics::Mysql'; + } + my $dbh = $pkg_module->new($params); return $dbh; @@ -159,7 +167,7 @@ sub _handle_error { # Cut down the error string to a reasonable size $_[0] = substr($_[0], 0, 2000) . ' ... ' . substr($_[0], -2000) if length($_[0]) > 4000; - $_[0] = Carp::longmess($_[0]); + # BMO: stracktrace disabled: $_[0] = Carp::longmess($_[0]); return 0; # Now let DBI handle raising the error } @@ -1383,14 +1391,19 @@ sub _bz_real_schema { my ($self) = @_; return $self->{private_real_schema} if exists $self->{private_real_schema}; - my ($data, $version) = $self->selectrow_array( - "SELECT schema_data, version FROM bz_schema"); + my $bz_schema; + unless ($bz_schema = Bugzilla->memcached->get({ key => 'bz_schema' })) { + $bz_schema = $self->selectrow_arrayref( + "SELECT schema_data, version FROM bz_schema" + ); + Bugzilla->memcached->set({ key => 'bz_schema', value => $bz_schema }); + } (die "_bz_real_schema tried to read the bz_schema table but it's empty!") - if !$data; + if !$bz_schema; - $self->{private_real_schema} = - $self->_bz_schema->deserialize_abstract($data, $version); + $self->{private_real_schema} = + $self->_bz_schema->deserialize_abstract($bz_schema->[0], $bz_schema->[1]); return $self->{private_real_schema}; } @@ -1432,6 +1445,8 @@ sub _bz_store_real_schema { $sth->bind_param(1, $store_me, $self->BLOB_TYPE); $sth->bind_param(2, $schema_version); $sth->execute(); + + Bugzilla->memcached->clear({ key => 'bz_schema' }); } # For bz_populate_enum_tables diff --git a/Bugzilla/DB/Mysql.pm b/Bugzilla/DB/Mysql.pm index 9ddb46622..c430725ef 100644 --- a/Bugzilla/DB/Mysql.pm +++ b/Bugzilla/DB/Mysql.pm @@ -181,7 +181,7 @@ sub sql_fulltext_search { $mode = 'IN BOOLEAN MODE'; # quote un-quoted compound words - my @words = quotewords('[\s()]+', 'delimiters', $text); + my @words = grep { defined } quotewords('[\s()]+', 'delimiters', $text); foreach my $word (@words) { # match words that have non-word chars in the middle of them if ($word =~ /\w\W+\w/ && $word !~ m/"/) { diff --git a/Bugzilla/DB/Schema.pm b/Bugzilla/DB/Schema.pm index 1e598c61e..be392c105 100644 --- a/Bugzilla/DB/Schema.pm +++ b/Bugzilla/DB/Schema.pm @@ -342,6 +342,8 @@ use constant ABSTRACT_SCHEMA => { bugs_activity => { FIELDS => [ + id => {TYPE => 'INTSERIAL', NOTNULL => 1, + PRIMARYKEY => 1}, bug_id => {TYPE => 'INT3', NOTNULL => 1, REFERENCES => {TABLE => 'bugs', COLUMN => 'bug_id', @@ -358,8 +360,8 @@ use constant ABSTRACT_SCHEMA => { REFERENCES => {TABLE => 'fielddefs', COLUMN => 'id'}}, added => {TYPE => 'varchar(255)'}, - removed => {TYPE => 'TINYTEXT'}, - comment_id => {TYPE => 'INT3', + removed => {TYPE => 'varchar(255)'}, + comment_id => {TYPE => 'INT4', REFERENCES => { TABLE => 'longdescs', COLUMN => 'comment_id', DELETE => 'CASCADE'}}, @@ -370,6 +372,7 @@ use constant ABSTRACT_SCHEMA => { bugs_activity_bug_when_idx => ['bug_when'], bugs_activity_fieldid_idx => ['fieldid'], bugs_activity_added_idx => ['added'], + bugs_activity_removed_idx => ['removed'], ], }, @@ -393,7 +396,7 @@ use constant ABSTRACT_SCHEMA => { longdescs => { FIELDS => [ - comment_id => {TYPE => 'MEDIUMSERIAL', NOTNULL => 1, + comment_id => {TYPE => 'INTSERIAL', NOTNULL => 1, PRIMARYKEY => 1}, bug_id => {TYPE => 'INT3', NOTNULL => 1, REFERENCES => {TABLE => 'bugs', @@ -421,6 +424,54 @@ use constant ABSTRACT_SCHEMA => { ], }, + longdescs_tags => { + FIELDS => [ + id => { TYPE => 'MEDIUMSERIAL', NOTNULL => 1, PRIMARYKEY => 1 }, + comment_id => { TYPE => 'INT4', + REFERENCES => { TABLE => 'longdescs', + COLUMN => 'comment_id', + DELETE => 'CASCADE' }}, + tag => { TYPE => 'varchar(24)', NOTNULL => 1 }, + ], + INDEXES => [ + longdescs_tags_idx => { FIELDS => ['comment_id', 'tag'], TYPE => 'UNIQUE' }, + ], + }, + + longdescs_tags_weights => { + FIELDS => [ + id => { TYPE => 'MEDIUMSERIAL', NOTNULL => 1, PRIMARYKEY => 1 }, + tag => { TYPE => 'varchar(24)', NOTNULL => 1 }, + weight => { TYPE => 'INT3', NOTNULL => 1 }, + ], + INDEXES => [ + longdescs_tags_weights_tag_idx => { FIELDS => ['tag'], TYPE => 'UNIQUE' }, + ], + }, + + longdescs_tags_activity => { + FIELDS => [ + id => { TYPE => 'MEDIUMSERIAL', NOTNULL => 1, PRIMARYKEY => 1 }, + bug_id => { TYPE => 'INT3', NOTNULL => 1, + REFERENCES => { TABLE => 'bugs', + COLUMN => 'bug_id', + DELETE => 'CASCADE' }}, + comment_id => { TYPE => 'INT4', + REFERENCES => { TABLE => 'longdescs', + COLUMN => 'comment_id', + DELETE => 'CASCADE' }}, + who => { TYPE => 'INT3', NOTNULL => 1, + REFERENCES => { TABLE => 'profiles', + COLUMN => 'userid' }}, + bug_when => { TYPE => 'DATETIME', NOTNULL => 1 }, + added => { TYPE => 'varchar(24)' }, + removed => { TYPE => 'varchar(24)' }, + ], + INDEXES => [ + longdescs_tags_activity_bug_id_idx => ['bug_id'], + ], + }, + dependencies => { FIELDS => [ blocked => {TYPE => 'INT3', NOTNULL => 1, @@ -433,7 +484,8 @@ use constant ABSTRACT_SCHEMA => { DELETE => 'CASCADE'}}, ], INDEXES => [ - dependencies_blocked_idx => ['blocked'], + dependencies_blocked_idx => {FIELDS => [qw(blocked dependson)], + TYPE => 'UNIQUE'}, dependencies_dependson_idx => ['dependson'], ], }, @@ -466,6 +518,7 @@ use constant ABSTRACT_SCHEMA => { attachments_creation_ts_idx => ['creation_ts'], attachments_modification_time_idx => ['modification_time'], attachments_submitter_id_idx => ['submitter_id', 'bug_id'], + attachments_ispatch_idx => ['ispatch'], ], }, attach_data => { @@ -526,6 +579,9 @@ use constant ABSTRACT_SCHEMA => { added => {TYPE => 'MEDIUMTEXT'}, at_time => {TYPE => 'DATETIME', NOTNULL => 1}, ], + INDEXES => [ + audit_log_class_idx => ['class', 'at_time'], + ], }, # Keywords @@ -651,8 +707,8 @@ use constant ABSTRACT_SCHEMA => { DELETE => 'CASCADE'}}, ], INDEXES => [ - flaginclusions_type_id_idx => - [qw(type_id product_id component_id)], + flaginclusions_type_id_idx => { FIELDS => [qw(type_id product_id component_id)], + TYPE => 'UNIQUE' }, ], }, @@ -672,8 +728,8 @@ use constant ABSTRACT_SCHEMA => { DELETE => 'CASCADE'}}, ], INDEXES => [ - flagexclusions_type_id_idx => - [qw(type_id product_id component_id)], + flagexclusions_type_id_idx => { FIELDS => [qw(type_id product_id component_id)], + TYPE => 'UNIQUE' }, ], }, @@ -889,6 +945,7 @@ use constant ABSTRACT_SCHEMA => { extern_id => {TYPE => 'varchar(64)'}, is_enabled => {TYPE => 'BOOLEAN', NOTNULL => 1, DEFAULT => 'TRUE'}, + last_seen_date => {TYPE => 'DATETIME'}, ], INDEXES => [ profiles_login_name_idx => {FIELDS => ['login_name'], @@ -915,6 +972,8 @@ use constant ABSTRACT_SCHEMA => { profiles_activity => { FIELDS => [ + id => {TYPE => 'MEDIUMSERIAL', NOTNULL => 1, + PRIMARYKEY => 1}, userid => {TYPE => 'INT3', NOTNULL => 1, REFERENCES => {TABLE => 'profiles', COLUMN => 'userid', @@ -952,6 +1011,23 @@ use constant ABSTRACT_SCHEMA => { ], }, + email_bug_ignore => { + FIELDS => [ + user_id => {TYPE => 'INT3', NOTNULL => 1, + REFERENCES => {TABLE => 'profiles', + COLUMN => 'userid', + DELETE => 'CASCADE'}}, + bug_id => {TYPE => 'INT3', NOTNULL => 1, + REFERENCES => {TABLE => 'bugs', + COLUMN => 'bug_id', + DELETE => 'CASCADE'}}, + ], + INDEXES => [ + email_bug_ignore_user_id_idx => {FIELDS => [qw(user_id bug_id)], + TYPE => 'UNIQUE'}, + ], + }, + watch => { FIELDS => [ watcher => {TYPE => 'INT3', NOTNULL => 1, @@ -1638,6 +1714,26 @@ use constant ABSTRACT_SCHEMA => { ], }, + bug_user_last_visit => { + FIELDS => [ + id => {TYPE => 'INTSERIAL', NOTNULL => 1, + PRIMARYKEY => 1}, + user_id => {TYPE => 'INT3', NOTNULL => 1, + REFERENCES => {TABLE => 'profiles', + COLUMN => 'userid', + DELETE => 'CASCADE'}}, + bug_id => {TYPE => 'INT3', NOTNULL => 1, + REFERENCES => {TABLE => 'bugs', + COLUMN => 'bug_id', + DELETE => 'CASCADE'}}, + last_visit_ts => {TYPE => 'DATETIME', NOTNULL => 1}, + ], + INDEXES => [ + bug_user_last_visit_idx => {FIELDS => ['user_id', 'bug_id'], + TYPE => 'UNIQUE'}, + bug_user_last_visit_last_visit_ts_idx => ['last_visit_ts'], + ], + }, }; # Foreign Keys are added in Bugzilla::DB::bz_add_field_tables diff --git a/Bugzilla/DB/Schema/Mysql.pm b/Bugzilla/DB/Schema/Mysql.pm index 8c9ea2dda..5fc50a986 100644 --- a/Bugzilla/DB/Schema/Mysql.pm +++ b/Bugzilla/DB/Schema/Mysql.pm @@ -120,7 +120,7 @@ sub _initialize { LONGBLOB => 'longblob', DATETIME => 'datetime', - + DATE => 'date', }; $self->_adjust_schema; diff --git a/Bugzilla/DB/Schema/Oracle.pm b/Bugzilla/DB/Schema/Oracle.pm index a61b1e323..37b406671 100644 --- a/Bugzilla/DB/Schema/Oracle.pm +++ b/Bugzilla/DB/Schema/Oracle.pm @@ -70,7 +70,7 @@ sub _initialize { LONGBLOB => 'blob', DATETIME => 'date', - + DATE => 'date', }; $self->_adjust_schema; diff --git a/Bugzilla/DB/Schema/Pg.pm b/Bugzilla/DB/Schema/Pg.pm index d21f5099c..662120c03 100644 --- a/Bugzilla/DB/Schema/Pg.pm +++ b/Bugzilla/DB/Schema/Pg.pm @@ -80,7 +80,7 @@ sub _initialize { LONGBLOB => 'bytea', DATETIME => 'timestamp(0) without time zone', - + DATE => 'date', }; $self->_adjust_schema; diff --git a/Bugzilla/DB/Schema/Sqlite.pm b/Bugzilla/DB/Schema/Sqlite.pm index aad1f17bc..7ed9def3e 100644 --- a/Bugzilla/DB/Schema/Sqlite.pm +++ b/Bugzilla/DB/Schema/Sqlite.pm @@ -57,6 +57,7 @@ sub _initialize { LONGBLOB => 'blob', DATETIME => 'DATETIME', + DATE => 'DATETIME', }; $self->_adjust_schema; diff --git a/Bugzilla/Error.pm b/Bugzilla/Error.pm index 178f6f90c..6e3309778 100644 --- a/Bugzilla/Error.pm +++ b/Bugzilla/Error.pm @@ -26,8 +26,9 @@ package Bugzilla::Error; use strict; use base qw(Exporter); -@Bugzilla::Error::EXPORT = qw(ThrowCodeError ThrowTemplateError ThrowUserError); +@Bugzilla::Error::EXPORT = qw(ThrowCodeError ThrowTemplateError ThrowUserError ThrowErrorPage); +use Bugzilla::Sentry; use Bugzilla::Constants; use Bugzilla::WebService::Constants; use Bugzilla::Util; @@ -93,6 +94,7 @@ sub _throw_error { my $template = Bugzilla->template; my $message; + # There are some tests that throw and catch a lot of errors, # and calling $template->process over and over for those errors # is too slow. So instead, we just "die" with a dump of the arguments. @@ -108,8 +110,22 @@ sub _throw_error { message => \$message }); if (Bugzilla->error_mode == ERROR_MODE_WEBPAGE) { - print Bugzilla->cgi->header(); - print $message; + if (sentry_should_notify($vars->{error})) { + $vars->{maintainers_notified} = 1; + $vars->{processed} = {}; + } else { + $vars->{maintainers_notified} = 0; + } + + my $cgi = Bugzilla->cgi; + $cgi->close_standby_message('text/html', 'inline'); + $template->process($name, $vars) + || ThrowTemplateError($template->error()); + print $cgi->multipart_final() if $cgi->{_multipart_in_progress}; + + if ($vars->{maintainers_notified}) { + sentry_handle_error($vars->{error}, $vars->{processed}->{error_message}); + } } elsif (Bugzilla->error_mode == ERROR_MODE_TEST) { die Dumper($vars); @@ -118,7 +134,8 @@ sub _throw_error { die("$message\n"); } elsif (Bugzilla->error_mode == ERROR_MODE_DIE_SOAP_FAULT - || Bugzilla->error_mode == ERROR_MODE_JSON_RPC) + || Bugzilla->error_mode == ERROR_MODE_JSON_RPC + || Bugzilla->error_mode == ERROR_MODE_REST) { # Clone the hash so we aren't modifying the constant. my %error_map = %{ WS_ERROR_CODE() }; @@ -135,13 +152,20 @@ sub _throw_error { } else { my $server = Bugzilla->_json_server; + + my $status_code = 0; + if (Bugzilla->error_mode == ERROR_MODE_REST) { + my %status_code_map = %{ REST_STATUS_CODE_MAP() }; + $status_code = $status_code_map{$code} || $status_code_map{'_default'}; + } # Technically JSON-RPC isn't allowed to have error numbers # higher than 999, but we do this to avoid conflicts with # the internal JSON::RPC error codes. - $server->raise_error(code => 100000 + $code, - message => $message, - id => $server->{_bz_request_id}, - version => $server->version); + $server->raise_error(code => 100000 + $code, + status_code => $status_code, + message => $message, + id => $server->{_bz_request_id}, + version => $server->version); # Most JSON-RPC Throw*Error calls happen within an eval inside # of JSON::RPC. So, in that circumstance, instead of exiting, # we die with no message. JSON::RPC checks raise_error before @@ -183,40 +207,83 @@ sub ThrowTemplateError { die("error: template error: $template_err"); } + # mod_perl overrides exit to call die with this string + # we never want to display this to the user + exit if $template_err =~ /\bModPerl::Util::exit\b/; + $vars->{'template_error_msg'} = $template_err; $vars->{'error'} = "template_error"; + sentry_handle_error('error', $template_err); + $vars->{'template_error_msg'} =~ s/ at \S+ line \d+\.\s*$//; + my $template = Bugzilla->template; # Try a template first; but if this one fails too, fall back # on plain old print statements. if (!$template->process("global/code-error.html.tmpl", $vars)) { - my $maintainer = Bugzilla->params->{'maintainer'}; + my $maintainer = html_quote(Bugzilla->params->{'maintainer'}); my $error = html_quote($vars->{'template_error_msg'}); my $error2 = html_quote($template->error()); print <<END; <tt> <p> - Bugzilla has suffered an internal error. Please save this page and - send it to $maintainer with details of what you were doing at the - time this message appeared. + Bugzilla has suffered an internal error: + </p> + <p> + $error + </p> + <!-- template error, no real need to show this to the user + $error2 + --> + <p> + The <a href="mailto:$maintainer">Bugzilla maintainers</a> have + been notified of this error. </p> - <script type="text/javascript"> <!-- - document.write("<p>URL: " + - document.location.href.replace(/&/g,"&") - .replace(/</g,"<") - .replace(/>/g,">") + "</p>"); - // --> - </script> - <p>Template->process() failed twice.<br> - First error: $error<br> - Second error: $error2</p> </tt> END } exit; } +sub ThrowErrorPage { + # BMO customisation for bug 659231 + my ($template_name, $message) = @_; + + my $dbh = Bugzilla->dbh; + $dbh->bz_rollback_transaction() if $dbh->bz_in_transaction(); + + if (Bugzilla->error_mode == ERROR_MODE_DIE) { + die("error: $message"); + } + + if (Bugzilla->error_mode == ERROR_MODE_DIE_SOAP_FAULT + || Bugzilla->error_mode == ERROR_MODE_JSON_RPC) + { + my $code = ERROR_UNKNOWN_TRANSIENT; + if (Bugzilla->error_mode == ERROR_MODE_DIE_SOAP_FAULT) { + die SOAP::Fault->faultcode($code)->faultstring($message); + } else { + my $server = Bugzilla->_json_server; + $server->raise_error(code => 100000 + $code, + message => $message, + id => $server->{_bz_request_id}, + version => $server->version); + die if _in_eval(); + $server->response($server->error_response_header); + } + } else { + my $cgi = Bugzilla->cgi; + my $template = Bugzilla->template; + my $vars = {}; + $vars->{message} = $message; + print $cgi->header(); + $template->process($template_name, $vars) + || ThrowTemplateError($template->error()); + exit; + } +} + 1; __END__ diff --git a/Bugzilla/Field.pm b/Bugzilla/Field.pm index 81677c7ea..d5f8e33f9 100644 --- a/Bugzilla/Field.pm +++ b/Bugzilla/Field.pm @@ -78,6 +78,8 @@ use Bugzilla::Constants; use Bugzilla::Error; use Bugzilla::Util; use List::MoreUtils qw(any); +use Bugzilla::Config qw(SetParam write_params); +use Bugzilla::Hook; use Scalar::Util qw(blessed); @@ -85,6 +87,8 @@ use Scalar::Util qw(blessed); #### Initialization #### ############################### +use constant IS_CONFIG => 1; + use constant DB_TABLE => 'fielddefs'; use constant LIST_ORDER => 'sortkey, name'; @@ -159,6 +163,7 @@ use constant SQL_DEFINITIONS => { FIELD_TYPE_TEXTAREA, { TYPE => 'MEDIUMTEXT', NOTNULL => 1, DEFAULT => "''"}, FIELD_TYPE_DATETIME, { TYPE => 'DATETIME' }, + FIELD_TYPE_DATE, { TYPE => 'DATE' }, FIELD_TYPE_BUG_ID, { TYPE => 'INT3' }, }; @@ -176,6 +181,9 @@ use constant DEFAULT_FIELDS => ( {name => 'product', desc => 'Product', in_new_bugmail => 1, is_mandatory => 1, type => FIELD_TYPE_SINGLE_SELECT, buglist => 1}, + {name => 'component', desc => 'Component', in_new_bugmail => 1, + is_mandatory => 1, + type => FIELD_TYPE_SINGLE_SELECT, buglist => 1}, {name => 'version', desc => 'Version', in_new_bugmail => 1, is_mandatory => 1, buglist => 1}, {name => 'rep_platform', desc => 'Platform', in_new_bugmail => 1, @@ -196,9 +204,6 @@ use constant DEFAULT_FIELDS => ( type => FIELD_TYPE_SINGLE_SELECT, buglist => 1}, {name => 'priority', desc => 'Priority', in_new_bugmail => 1, type => FIELD_TYPE_SINGLE_SELECT, buglist => 1}, - {name => 'component', desc => 'Component', in_new_bugmail => 1, - is_mandatory => 1, - type => FIELD_TYPE_SINGLE_SELECT, buglist => 1}, {name => 'assigned_to', desc => 'AssignedTo', in_new_bugmail => 1, buglist => 1}, {name => 'reporter', desc => 'ReportedBy', in_new_bugmail => 1, @@ -207,9 +212,11 @@ use constant DEFAULT_FIELDS => ( buglist => 1}, {name => 'cc', desc => 'CC', in_new_bugmail => 1}, {name => 'dependson', desc => 'Depends on', in_new_bugmail => 1, - is_numeric => 1}, + is_numeric => 1, buglist => 1}, {name => 'blocked', desc => 'Blocks', in_new_bugmail => 1, - is_numeric => 1}, + is_numeric => 1, buglist => 1}, + + {name => 'assignee_last_login', desc => 'Assignee Last Login Date', buglist => 1}, {name => 'attachments.description', desc => 'Attachment description'}, {name => 'attachments.filename', desc => 'Attachment filename'}, @@ -261,6 +268,9 @@ use constant DEFAULT_FIELDS => ( {name => 'see_also', desc => "See Also", type => FIELD_TYPE_BUG_URLS}, {name => 'tag', desc => 'Tags'}, + {name => 'last_visit_ts', desc => 'Last Visit', buglist => 1, + type => FIELD_TYPE_DATETIME}, + {name => 'comment_tag', desc => 'Comment Tag'}, ); ################ @@ -347,9 +357,7 @@ sub _check_sortkey { sub _check_type { my ($invocant, $type, undef, $params) = @_; my $saved_type = $type; - # The constant here should be updated every time a new, - # higher field type is added. - (detaint_natural($type) && $type <= FIELD_TYPE_KEYWORDS) + (detaint_natural($type) && $type < FIELD_TYPE_HIGHEST_PLUS_ONE) || ThrowCodeError('invalid_customfield_type', { type => $saved_type }); my $custom = blessed($invocant) ? $invocant->custom : $params->{custom}; @@ -918,53 +926,67 @@ sub remove_from_db { ThrowUserError('customfield_not_obsolete', {'name' => $self->name }); } - $dbh->bz_start_transaction(); + # BMO: disable bug updates during field creation + # using an eval as try/finally + eval { + SetParam('disable_bug_updates', 1); + write_params(); - # Check to see if bug activity table has records (should be fast with index) - my $has_activity = $dbh->selectrow_array("SELECT COUNT(*) FROM bugs_activity - WHERE fieldid = ?", undef, $self->id); - if ($has_activity) { - ThrowUserError('customfield_has_activity', {'name' => $name }); - } + $dbh->bz_start_transaction(); - # Check to see if bugs table has records (slow) - my $bugs_query = ""; + # Check to see if bug activity table has records (should be fast with index) + my $has_activity = $dbh->selectrow_array("SELECT COUNT(*) FROM bugs_activity + WHERE fieldid = ?", undef, $self->id); + if ($has_activity) { + ThrowUserError('customfield_has_activity', {'name' => $name }); + } - if ($self->type == FIELD_TYPE_MULTI_SELECT) { - $bugs_query = "SELECT COUNT(*) FROM bug_$name"; - } - else { - $bugs_query = "SELECT COUNT(*) FROM bugs WHERE $name IS NOT NULL"; - if ($self->type != FIELD_TYPE_BUG_ID && $self->type != FIELD_TYPE_DATETIME) { - $bugs_query .= " AND $name != ''"; + # Check to see if bugs table has records (slow) + my $bugs_query = ""; + + if ($self->type == FIELD_TYPE_MULTI_SELECT) { + $bugs_query = "SELECT COUNT(*) FROM bug_$name"; } - # Ignore the default single select value - if ($self->type == FIELD_TYPE_SINGLE_SELECT) { - $bugs_query .= " AND $name != '---'"; + else { + $bugs_query = "SELECT COUNT(*) FROM bugs WHERE $name IS NOT NULL"; + if ($self->type != FIELD_TYPE_BUG_ID + && $self->type != FIELD_TYPE_DATE + && $self->type != FIELD_TYPE_DATETIME) + { + $bugs_query .= " AND $name != ''"; + } + # Ignore the default single select value + if ($self->type == FIELD_TYPE_SINGLE_SELECT) { + $bugs_query .= " AND $name != '---'"; + } } - } - my $has_bugs = $dbh->selectrow_array($bugs_query); - if ($has_bugs) { - ThrowUserError('customfield_has_contents', {'name' => $name }); - } + my $has_bugs = $dbh->selectrow_array($bugs_query); + if ($has_bugs) { + ThrowUserError('customfield_has_contents', {'name' => $name }); + } - # Once we reach here, we should be OK to delete. - $dbh->do('DELETE FROM fielddefs WHERE id = ?', undef, $self->id); + # Once we reach here, we should be OK to delete. + $dbh->do('DELETE FROM fielddefs WHERE id = ?', undef, $self->id); - my $type = $self->type; + my $type = $self->type; - # the values for multi-select are stored in a seperate table - if ($type != FIELD_TYPE_MULTI_SELECT) { - $dbh->bz_drop_column('bugs', $name); - } + # the values for multi-select are stored in a seperate table + if ($type != FIELD_TYPE_MULTI_SELECT) { + $dbh->bz_drop_column('bugs', $name); + } - if ($self->is_select) { - # Delete the table that holds the legal values for this field. - $dbh->bz_drop_field_tables($self); - } + if ($self->is_select) { + # Delete the table that holds the legal values for this field. + $dbh->bz_drop_field_tables($self); + } - $dbh->bz_commit_transaction() + $dbh->bz_commit_transaction(); + }; + my $error = "$@"; + SetParam('disable_bug_updates', 0); + write_params(); + die $error if $error; } =pod @@ -1012,48 +1034,71 @@ sub create { my ($params) = @_; my $dbh = Bugzilla->dbh; - # This makes sure the "sortkey" validator runs, even if - # the parameter isn't sent to create(). - $params->{sortkey} = undef if !exists $params->{sortkey}; - $params->{type} ||= 0; - # We mark the custom field as obsolete till it has been fully created, - # to avoid race conditions when viewing bugs at the same time. - my $is_obsolete = $params->{obsolete}; - $params->{obsolete} = 1 if $params->{custom}; - - $dbh->bz_start_transaction(); - $class->check_required_create_fields(@_); - my $field_values = $class->run_create_validators($params); - my $visibility_values = delete $field_values->{visibility_values}; - my $field = $class->insert_create_data($field_values); - - $field->set_visibility_values($visibility_values); - $field->_update_visibility_values(); + # BMO: disable bug updates during field creation + # using an eval as try/finally + my $field; + eval { + if ($params->{'custom'}) { + SetParam('disable_bug_updates', 1); + write_params(); + } - $dbh->bz_commit_transaction(); + # This makes sure the "sortkey" validator runs, even if + # the parameter isn't sent to create(). + $params->{sortkey} = undef if !exists $params->{sortkey}; + $params->{type} ||= 0; + # We mark the custom field as obsolete till it has been fully created, + # to avoid race conditions when viewing bugs at the same time. + my $is_obsolete = $params->{obsolete}; + $params->{obsolete} = 1 if $params->{custom}; + + $dbh->bz_start_transaction(); + $class->check_required_create_fields(@_); + my $field_values = $class->run_create_validators($params); + my $visibility_values = delete $field_values->{visibility_values}; + $field = $class->insert_create_data($field_values); + + $field->set_visibility_values($visibility_values); + $field->_update_visibility_values(); + + $dbh->bz_commit_transaction(); + Bugzilla->memcached->clear_config(); + + if ($field->custom) { + my $name = $field->name; + my $type = $field->type; + if (SQL_DEFINITIONS->{$type}) { + # Create the database column that stores the data for this field. + $dbh->bz_add_column('bugs', $name, SQL_DEFINITIONS->{$type}); + } - if ($field->custom) { - my $name = $field->name; - my $type = $field->type; - if (SQL_DEFINITIONS->{$type}) { - # Create the database column that stores the data for this field. - $dbh->bz_add_column('bugs', $name, SQL_DEFINITIONS->{$type}); - } + if ($field->is_select) { + # Create the table that holds the legal values for this field. + $dbh->bz_add_field_tables($field); + } - if ($field->is_select) { - # Create the table that holds the legal values for this field. - $dbh->bz_add_field_tables($field); - } + if ($type == FIELD_TYPE_SINGLE_SELECT) { + # Insert a default value of "---" into the legal values table. + $dbh->do("INSERT INTO $name (value) VALUES ('---')"); + } - if ($type == FIELD_TYPE_SINGLE_SELECT) { - # Insert a default value of "---" into the legal values table. - $dbh->do("INSERT INTO $name (value) VALUES ('---')"); + # Restore the original obsolete state of the custom field. + $dbh->do('UPDATE fielddefs SET obsolete = 0 WHERE id = ?', undef, $field->id) + unless $is_obsolete; + + Bugzilla->memcached->clear({ table => 'fielddefs', id => $field->id }); + Bugzilla->memcached->clear_config(); } + }; - # Restore the original obsolete state of the custom field. - $dbh->do('UPDATE fielddefs SET obsolete = 0 WHERE id = ?', undef, $field->id) - unless $is_obsolete; + my $error = "$@"; + if ($params->{'custom'}) { + SetParam('disable_bug_updates', 0); + write_params(); } + die $error if $error; + + Bugzilla::Hook::process("field_end_of_create", { field => $field }); return $field; } @@ -1066,6 +1111,7 @@ sub update { $dbh->do("UPDATE " . $self->name . " SET visibility_value_id = NULL"); } $self->_update_visibility_values(); + Bugzilla->memcached->clear_config(); return $changes; } diff --git a/Bugzilla/Field/Choice.pm b/Bugzilla/Field/Choice.pm index 773dbd4ce..8d7d61def 100644 --- a/Bugzilla/Field/Choice.pm +++ b/Bugzilla/Field/Choice.pm @@ -37,6 +37,8 @@ use Scalar::Util qw(blessed); # Initialization # ################## +use constant IS_CONFIG => 1; + use constant DB_COLUMNS => qw( id value diff --git a/Bugzilla/Flag.pm b/Bugzilla/Flag.pm index b687532c0..e87f1f674 100644 --- a/Bugzilla/Flag.pm +++ b/Bugzilla/Flag.pm @@ -81,15 +81,21 @@ use constant AUDIT_REMOVES => 0; use constant SKIP_REQUESTEE_ON_ERROR => 1; -use constant DB_COLUMNS => qw( - id - type_id - bug_id - attach_id - requestee_id - setter_id - status -); +sub DB_COLUMNS { + my $dbh = Bugzilla->dbh; + return qw( + id + type_id + bug_id + attach_id + requestee_id + setter_id + status), + $dbh->sql_date_format('creation_date', '%Y.%m.%d %H:%i:%s') . + ' AS creation_date', + $dbh->sql_date_format('modification_date', '%Y.%m.%d %H:%i:%s') . + ' AS modification_date'; +} use constant UPDATE_COLUMNS => qw( requestee_id @@ -134,6 +140,14 @@ Returns the ID of the attachment this flag belongs to, if any. Returns the status '+', '-', '?' of the flag. +=item C<creation_date> + +Returns the timestamp when the flag was created. + +=item C<modification_date> + +Returns the timestamp when the flag was last modified. + =back =cut @@ -146,6 +160,8 @@ sub attach_id { return $_[0]->{'attach_id'}; } sub status { return $_[0]->{'status'}; } sub setter_id { return $_[0]->{'setter_id'}; } sub requestee_id { return $_[0]->{'requestee_id'}; } +sub creation_date { return $_[0]->{'creation_date'}; } +sub modification_date { return $_[0]->{'modification_date'}; } ############################### #### Methods #### @@ -180,22 +196,23 @@ is an attachment flag, else undefined. sub type { my $self = shift; - $self->{'type'} ||= new Bugzilla::FlagType($self->{'type_id'}); - return $self->{'type'}; + return $self->{'type'} + ||= new Bugzilla::FlagType($self->{'type_id'}, cache => 1 ); } sub setter { my $self = shift; - $self->{'setter'} ||= new Bugzilla::User($self->{'setter_id'}); - return $self->{'setter'}; + return $self->{'setter' } + ||= new Bugzilla::User({ id => $self->{'setter_id'}, cache => 1 }); } sub requestee { my $self = shift; if (!defined $self->{'requestee'} && $self->{'requestee_id'}) { - $self->{'requestee'} = new Bugzilla::User($self->{'requestee_id'}); + $self->{'requestee'} + = new Bugzilla::User({ id => $self->{'requestee_id'}, cache => 1 }); } return $self->{'requestee'}; } @@ -205,16 +222,16 @@ sub attachment { return undef unless $self->attach_id; require Bugzilla::Attachment; - $self->{'attachment'} ||= new Bugzilla::Attachment($self->attach_id); - return $self->{'attachment'}; + return $self->{'attachment'} + ||= new Bugzilla::Attachment({ id => $self->attach_id, cache => 1 }); } sub bug { my $self = shift; require Bugzilla::Bug; - $self->{'bug'} ||= new Bugzilla::Bug($self->bug_id); - return $self->{'bug'}; + return $self->{'bug'} + ||= new Bugzilla::Bug({ id => $self->bug_id, cache => 1 }); } ################################ @@ -284,7 +301,7 @@ sub count { sub set_flag { my ($class, $obj, $params) = @_; - my ($bug, $attachment); + my ($bug, $attachment, $obj_flag, $requestee_changed); if (blessed($obj) && $obj->isa('Bugzilla::Attachment')) { $attachment = $obj; $bug = $attachment->bug; @@ -296,6 +313,12 @@ sub set_flag { ThrowCodeError('flag_unexpected_object', { 'caller' => ref $obj }); } + # Make sure the user can change flags + my $privs; + $bug->check_can_change_field('flagtypes.name', 0, 1, \$privs) + || ThrowUserError('illegal_change', + { field => 'flagtypes.name', privs => $privs }); + # Update (or delete) an existing flag. if ($params->{id}) { my $flag = $class->check({ id => $params->{id} }); @@ -322,13 +345,14 @@ sub set_flag { ($obj_flagtype) = grep { $_->id == $flag->type_id } @{$obj->flag_types}; push(@{$obj_flagtype->{flags}}, $flag); } - my ($obj_flag) = grep { $_->id == $flag->id } @{$obj_flagtype->{flags}}; + ($obj_flag) = grep { $_->id == $flag->id } @{$obj_flagtype->{flags}}; # If the flag has the correct type but cannot be found above, this means # the flag is going to be removed (e.g. because this is a pending request # and the attachment is being marked as obsolete). return unless $obj_flag; - $class->_validate($obj_flag, $obj_flagtype, $params, $bug, $attachment); + ($obj_flag, $requestee_changed) = + $class->_validate($obj_flag, $obj_flagtype, $params, $bug, $attachment); } # Create a new flag. elsif ($params->{type_id}) { @@ -360,12 +384,21 @@ sub set_flag { } } - $class->_validate(undef, $obj_flagtype, $params, $bug, $attachment); + ($obj_flag, $requestee_changed) = + $class->_validate(undef, $obj_flagtype, $params, $bug, $attachment); } else { ThrowCodeError('param_required', { function => $class . '->set_flag', param => 'id/type_id' }); } + + if ($obj_flag + && $requestee_changed + && $obj_flag->requestee_id + && $obj_flag->requestee->setting('requestee_cc') eq 'on') + { + $bug->add_cc($obj_flag->requestee); + } } sub _validate { @@ -383,25 +416,27 @@ sub _validate { my $old_requestee_id = $obj_flag->requestee_id; $obj_flag->_set_status($params->{status}); - $obj_flag->_set_requestee($params->{requestee}, $attachment, $params->{skip_roe}); + $obj_flag->_set_requestee($params->{requestee}, $bug, $attachment, $params->{skip_roe}); + + # The requestee ID can be undefined. + my $requestee_changed = ($obj_flag->requestee_id || 0) != ($old_requestee_id || 0); # The setter field MUST NOT be updated if neither the status # nor the requestee fields changed. - if (($obj_flag->status ne $old_status) - # The requestee ID can be undefined. - || (($obj_flag->requestee_id || 0) != ($old_requestee_id || 0))) - { + if (($obj_flag->status ne $old_status) || $requestee_changed) { $obj_flag->_set_setter($params->{setter}); } # If the flag is deleted, remove it from the list. if ($obj_flag->status eq 'X') { @{$flag_type->{flags}} = grep { $_->id != $obj_flag->id } @{$flag_type->{flags}}; + return; } # Add the newly created flag to the list. elsif (!$obj_flag->id) { push(@{$flag_type->{flags}}, $obj_flag); } + return wantarray ? ($obj_flag, $requestee_changed) : $obj_flag; } =pod @@ -418,10 +453,14 @@ Creates a flag record in the database. sub create { my ($class, $flag, $timestamp) = @_; - $timestamp ||= Bugzilla->dbh->selectrow_array('SELECT NOW()'); + $timestamp ||= Bugzilla->dbh->selectrow_array('SELECT LOCALTIMESTAMP(0)'); my $params = {}; my @columns = grep { $_ ne 'id' } $class->_get_db_columns; + + # Some columns use date formatting so use alias instead + @columns = map { /\s+AS\s+(.*)$/ ? $1 : $_ } @columns; + $params->{$_} = $flag->{$_} foreach @columns; $params->{creation_date} = $params->{modification_date} = $timestamp; @@ -433,14 +472,21 @@ sub create { sub update { my $self = shift; my $dbh = Bugzilla->dbh; - my $timestamp = shift || $dbh->selectrow_array('SELECT NOW()'); + my $timestamp = shift || $dbh->selectrow_array('SELECT LOCALTIMESTAMP(0)'); my $changes = $self->SUPER::update(@_); if (scalar(keys %$changes)) { $dbh->do('UPDATE flags SET modification_date = ? WHERE id = ?', undef, ($timestamp, $self->id)); + $self->{'modification_date'} = + format_time($timestamp, '%Y.%m.%d %T', Bugzilla->local_timezone); + Bugzilla->memcached->clear({ table => 'flags', id => $self->id }); } + + # BMO - provide a hook which passes the flag object + Bugzilla::Hook::process('flag_updated', {flag => $self, changes => $changes, timestamp => $timestamp}); + return $changes; } @@ -495,6 +541,10 @@ sub update_flags { # These flags have been deleted. foreach my $old_flag (values %old_flags) { $class->notify(undef, $old_flag, $self, $timestamp); + + # BMO - provide a hook which passes the timestamp, + # because that isn't passed to remove_from_db(). + Bugzilla::Hook::process('flag_deleted', {flag => $old_flag, timestamp => $timestamp}); $old_flag->remove_from_db(); } @@ -586,11 +636,17 @@ sub force_retarget { if ($is_retargetted) { $dbh->do('UPDATE flags SET type_id = ? WHERE id = ?', undef, ($flag->type_id, $flag->id)); + Bugzilla->memcached->clear({ table => 'flags', id => $flag->id }); } else { # Track deleted attachment flags. push(@removed, $class->snapshot([$flag])) if $flag->attach_id; $class->notify(undef, $flag, $bug || $flag->bug); + + # BMO - provide a hook which passes the timestamp, + # because that isn't passed to remove_from_db(). + my ($timestamp) = $dbh->selectrow_array('SELECT LOCALTIMESTAMP(0)'); + Bugzilla::Hook::process('flag_deleted', {flag => $flag, timestamp => $timestamp}); $flag->remove_from_db(); } } @@ -602,10 +658,10 @@ sub force_retarget { ############################### sub _set_requestee { - my ($self, $requestee, $attachment, $skip_requestee_on_error) = @_; + my ($self, $requestee, $bug, $attachment, $skip_requestee_on_error) = @_; $self->{requestee} = - $self->_check_requestee($requestee, $attachment, $skip_requestee_on_error); + $self->_check_requestee($requestee, $bug, $attachment, $skip_requestee_on_error); $self->{requestee_id} = $self->{requestee} ? $self->{requestee}->id : undef; @@ -627,7 +683,7 @@ sub _set_status { } sub _check_requestee { - my ($self, $requestee, $attachment, $skip_requestee_on_error) = @_; + my ($self, $requestee, $bug, $attachment, $skip_requestee_on_error) = @_; # If the flag status is not "?", then no requestee can be defined. return undef if ($self->status ne '?'); @@ -647,15 +703,29 @@ sub _check_requestee { # is specifically requestable. For existing flags, if the requestee # was set before the flag became specifically unrequestable, the # user can either remove him or leave him alone. - ThrowCodeError('flag_requestee_disabled', { type => $self->type }) + ThrowCodeError('flag_type_requestee_disabled', { type => $self->type }) if !$self->type->is_requesteeble; + # BMO customisation: + # You can't ask a disabled account, as they don't have the ability to + # set the flag. + ThrowUserError('flag_requestee_disabled', { requestee => $requestee }) + if !$requestee->is_enabled; + # Make sure the requestee can see the bug. # Note that can_see_bug() will query the DB, so if the bug # is being added/removed from some groups and these changes # haven't been committed to the DB yet, they won't be taken - # into account here. In this case, old restrictions matters. - if (!$requestee->can_see_bug($self->bug_id)) { + # into account here. In this case, old group restrictions matter. + # However, if the user has just been changed to the assignee, + # qa_contact, or added to the cc list of the bug and the bug + # is cclist_accessible, the requestee is allowed. + if (!$requestee->can_see_bug($self->bug_id) + && (!$bug->cclist_accessible + || !grep($_->id == $requestee->id, @{ $bug->cc_users }) + && $requestee->id != $bug->assigned_to->id + && (!$bug->qa_contact || $requestee->id != $bug->qa_contact->id))) + { if ($skip_requestee_on_error) { undef $requestee; } @@ -789,7 +859,7 @@ sub extract_flags_from_cgi { # Extract a list of existing flag IDs. my @flag_ids = map(/^flag-(\d+)$/ ? $1 : (), $cgi->param()); - return () if (!scalar(@flagtype_ids) && !scalar(@flag_ids)); + return ([], []) unless (scalar(@flagtype_ids) || scalar(@flag_ids)); my (@new_flags, @flags); foreach my $flag_id (@flag_ids) { @@ -955,7 +1025,7 @@ sub notify { my %recipients; foreach my $cc (split(/[, ]+/, $cc_list)) { - my $ccuser = new Bugzilla::User({ name => $cc }); + my $ccuser = new Bugzilla::User({ name => $cc, cache => 1 }); next if (scalar(@bug_in_groups) && (!$ccuser || !$ccuser->can_see_bug($bug->bug_id))); next if $attachment_is_private && (!$ccuser || !$ccuser->is_insider); # Prevent duplicated entries due to case sensitivity. @@ -1054,29 +1124,30 @@ sub _flag_types { return $flag_types; } -=head1 SEE ALSO +=head1 B<Methods in need of POD> =over -=item B<Bugzilla::FlagType> +=item update_activity -=back +=item setter_id +=item bug -=head1 CONTRIBUTORS +=item requestee_id -=over +=item DB_COLUMNS -=item Myk Melez <myk@mozilla.org> +=item set_flag -=item Jouni Heikniemi <jouni@heikniemi.net> +=item type_id -=item Kevin Benton <kevin.benton@amd.com> +=item snapshot -=item Frédéric Buclin <LpSolit@gmail.com> +=item update_flags -=back +=item update -=cut +=back 1; diff --git a/Bugzilla/FlagType.pm b/Bugzilla/FlagType.pm index 811530c42..8c958f5ce 100644 --- a/Bugzilla/FlagType.pm +++ b/Bugzilla/FlagType.pm @@ -52,6 +52,7 @@ use Bugzilla::Constants; use Bugzilla::Error; use Bugzilla::Util; use Bugzilla::Group; +use Bugzilla::Hook; use base qw(Bugzilla::Object); @@ -133,6 +134,8 @@ sub create { exclusions => $exclusions }); $flagtype->update(); + Bugzilla::Hook::process('flagtype_end_of_create', { type => $flagtype }); + $dbh->bz_commit_transaction(); return $flagtype; } @@ -197,10 +200,21 @@ sub update { # Silently remove requestees from flags which are no longer # specifically requestable. if (!$self->is_requesteeble) { - $dbh->do('UPDATE flags SET requestee_id = NULL WHERE type_id = ?', - undef, $self->id); + my $ids = $dbh->selectcol_arrayref( + 'SELECT id FROM flags WHERE type_id = ? AND requestee_id IS NOT NULL', + undef, $self->id); + + if (@$ids) { + $dbh->do('UPDATE flags SET requestee_id = NULL WHERE ' . $dbh->sql_in('id', $ids)); + foreach my $id (@$ids) { + Bugzilla->memcached->clear({ table => 'flags', id => $id }); + } + } } + Bugzilla::Hook::process('flagtype_end_of_update', + { type => $self, changed => $changes }); + $dbh->bz_commit_transaction(); return $changes; } @@ -462,7 +476,8 @@ sub grant_list { my @custusers; my @allusers = @{Bugzilla->user->get_userlist}; foreach my $user (@allusers) { - my $user_obj = new Bugzilla::User({name => $user->{login}}); + my $user_obj + = new Bugzilla::User({ name => $user->{login}, cache => 1 }); push(@custusers, $user) if $user_obj->can_set_flag($self); } return \@custusers; @@ -601,7 +616,7 @@ sub match { $tables = join(' ', @$tables); $criteria = join(' AND ', @criteria); - my $flagtype_ids = $dbh->selectcol_arrayref("SELECT id FROM $tables WHERE $criteria"); + my $flagtype_ids = $dbh->selectcol_arrayref("SELECT flagtypes.id FROM $tables WHERE $criteria"); return Bugzilla::FlagType->new_from_list($flagtype_ids); } @@ -652,6 +667,8 @@ by the query. =back +=end private + =cut sub sqlify_criteria { @@ -679,6 +696,11 @@ sub sqlify_criteria { my $is_active = $criteria->{is_active} ? "1" : "0"; push(@criteria, "flagtypes.is_active = $is_active"); } + if (exists($criteria->{active_or_has_flags}) && $criteria->{active_or_has_flags} =~ /^\d+$/) { + push(@$tables, "LEFT JOIN flags AS f ON flagtypes.id = f.type_id " . + "AND f.bug_id = " . $criteria->{active_or_has_flags}); + push(@criteria, "(flagtypes.is_active = 1 OR f.id IS NOT NULL)"); + } if ($criteria->{product_id}) { my $product_id = $criteria->{product_id}; detaint_natural($product_id) @@ -725,32 +747,48 @@ sub sqlify_criteria { push(@criteria, "(flagtypes.grant_group_id = $gid " . " OR flagtypes.request_group_id = $gid)"); } - + return @criteria; } 1; -=end private - -=head1 SEE ALSO +=head1 B<Methods in need of POD> =over -=item B<Bugzilla::Flags> +=item exclusions_as_hash -=back +=item request_group_id -=head1 CONTRIBUTORS +=item set_is_active -=over +=item set_is_multiplicable -=item Myk Melez <myk@mozilla.org> +=item inclusions_as_hash -=item Kevin Benton <kevin.benton@amd.com> +=item set_sortkey -=item Frédéric Buclin <LpSolit@gmail.com> +=item grant_group_id -=back +=item set_cc_list -=cut +=item set_request_group + +=item set_name + +=item set_is_specifically_requestable + +=item set_grant_group + +=item create + +=item set_clusions + +=item set_description + +=item set_is_requestable + +=item update + +=back diff --git a/Bugzilla/Group.pm b/Bugzilla/Group.pm index 382407748..732de348f 100644 --- a/Bugzilla/Group.pm +++ b/Bugzilla/Group.pm @@ -37,6 +37,8 @@ use Bugzilla::Config qw(:admin); ##### Module Initialization ### ############################### +use constant IS_CONFIG => 1; + use constant DB_COLUMNS => qw( groups.id groups.name @@ -119,9 +121,10 @@ sub _get_members { } sub flag_types { - my $self = shift; + my ($self, $params) = @_; + $params ||= {}; require Bugzilla::FlagType; - $self->{flag_types} ||= Bugzilla::FlagType::match({ group => $self->id }); + $self->{flag_types} ||= Bugzilla::FlagType::match({ group => $self->id, %$params }); return $self->{flag_types}; } @@ -230,6 +233,7 @@ sub update { Bugzilla::Hook::process('group_end_of_update', { group => $self, changes => $changes }); $dbh->bz_commit_transaction(); + Bugzilla->memcached->clear_config(); return $changes; } @@ -419,6 +423,7 @@ sub create { Bugzilla::Hook::process('group_end_of_create', { group => $group }); $dbh->bz_commit_transaction(); + Bugzilla->memcached->clear_config(); return $group; } diff --git a/Bugzilla/Hook.pm b/Bugzilla/Hook.pm index c658989a0..5e3dca655 100644 --- a/Bugzilla/Hook.pm +++ b/Bugzilla/Hook.pm @@ -434,6 +434,39 @@ to the user. =back +=head2 bug_start_of_update + +This happens near the beginning of L<Bugzilla::Bug/update>, after L<Bugzilla::Object/update> +is called, but before all other special changes are made to the database. Once use case is +this allows for adding your own entries to the C<changes> hash which gets added to the +bugs_activity table later keeping you from having to do it yourself. Also this is also helpful +if your extension needs to add CC members, flags, keywords, groups, etc. This generally +occurs inside a database transaction. + +Params: + +=over + +=item C<bug> + +The changed bug object, with all fields set to their updated values. + +=item C<old_bug> + +A bug object pulled from the database before the fields were set to +their updated values (so it has the old values available for each field). + +=item C<timestamp> + +The timestamp used for all updates in this transaction, as a SQL date +string. + +=item C<changes> + +The hash of changed fields. C<< $changes->{field} = [old, new] >> + +=back + =head2 buglist_columns This happens in L<Bugzilla::Search/COLUMNS>, which determines legal bug @@ -1289,6 +1322,22 @@ your template. =back +=head2 path_info_whitelist + +By default, Bugzilla removes the Path-Info information from URLs before +passing data to CGI scripts. If this information is needed for your +customizations, you can enumerate the pages you want to whitelist here. + +Params: + +=over + +=item C<whitelist> + +An array of script names that will not have their Path-Info automatically +removed. + +=back =head2 post_bug_after_creation @@ -1545,6 +1594,89 @@ See L<Bugzilla::WebService::Constants/WS_ERROR_CODE> for an example. =back +=head2 webservice_fix_credentials + +This hook allows for altering the credential parameters provided by the client +before authentication actually occurs. For example, this can be used to allow mapping +of custom parameters to the standard Bugzilla_login and Bugzilla_password parameters. + +Params: + +=over + +=item C<params> + +A hash ref containing the parameters passed into the webservice after +they have been obtained from the URL or body of the request. + +=back + +=head2 webservice_rest_request + +This hook allows for altering any of the parameters provided by the client +after authentication has occured. You are able to change things like renaming +of keys, removing values, or adding additional information. + +Params: + +=over + +=item C<params> + +A hash ref containing the parameters passed into the webservice after +they have been obtained from the URL or body of the request. + +=item C<rpc> + +The current JSONRPC, XMLRPC, or REST object. + +=back + +=head2 webservice_rest_resources + +This hook allows for altering of the REST resources data allowing you to +add additional paths to perform additional operations or to override the +resources already provided by the webservice modules. + +Params: + +=over + +=item C<resources> + +A hash returned from each module loaded that is used to determine +which code handler to use based on a regex match of the CGI path. + +=item C<rpc> + +The current JSONRPC, XMLRPC, or REST object. + +=back + +=head2 webservice_rest_response + +This hook allows for altering the result data or response object +that is being returned by the current REST webservice call. + +Params: + +=over + +=item C<response> + +The HTTP response object generated by JSON-RPC library. You can use this +to add headers, etc. + +=item C<result> + +A reference to a hash that contains the result data. + +=item C<rpc> + +The current JSONRPC, XMLRPC, or REST object. + +=back + =head1 SEE ALSO L<Bugzilla::Extension> diff --git a/Bugzilla/Install.pm b/Bugzilla/Install.pm index ce8fe6bad..d5f4f04cd 100644 --- a/Bugzilla/Install.pm +++ b/Bugzilla/Install.pm @@ -93,6 +93,12 @@ sub SETTINGS { # 2011-06-21 glob@mozilla.com -- Bug 589128 email_format => { options => ['html', 'text_only'], default => 'html' }, + # 2011-06-16 glob@mozilla.com -- Bug 663747 + bugmail_new_prefix => { options => ['on', 'off'], default => 'on' }, + # 2013-07-26 joshi_sunil@in.com -- Bug 669535 + possible_duplicates => { options => ['on', 'off'], default => 'on' }, + # 2011-10-11 glob@mozilla.com -- Bug 301656 + requestee_cc => { options => ['on', 'off'], default => 'on' }, } }; diff --git a/Bugzilla/Install/DB.pm b/Bugzilla/Install/DB.pm index 3ac83775a..4fb1f1c83 100644 --- a/Bugzilla/Install/DB.pm +++ b/Bugzilla/Install/DB.pm @@ -261,7 +261,7 @@ sub update_table_definitions { $dbh->bz_add_column("bugs", "cclist_accessible", {TYPE => 'BOOLEAN', NOTNULL => 1, DEFAULT => 'TRUE'}); - $dbh->bz_add_column("bugs_activity", "attach_id", {TYPE => 'INT3'}); + $dbh->bz_add_column("bugs_activity", "attach_id", {TYPE => 'INT5'}); _delete_logincookies_cryptpassword_and_handle_invalid_cookies(); @@ -398,7 +398,7 @@ sub update_table_definitions { "WHERE initialqacontact = 0"); _migrate_email_prefs_to_new_table(); - _initialize_dependency_tree_changes_email_pref(); + _initialize_new_email_prefs(); _change_all_mysql_booleans_to_tinyint(); # make classification_id field type be consistent with DB:Schema @@ -455,7 +455,7 @@ sub update_table_definitions { # 2005-12-07 altlst@sonic.net -- Bug 225221 $dbh->bz_add_column('longdescs', 'comment_id', - {TYPE => 'MEDIUMSERIAL', NOTNULL => 1, PRIMARYKEY => 1}); + {TYPE => 'INTSERIAL', NOTNULL => 1, PRIMARYKEY => 1}); _stop_storing_inactive_flags(); _change_short_desc_from_mediumtext_to_varchar(); @@ -607,7 +607,7 @@ sub update_table_definitions { _fix_series_creator_fk(); # 2009-11-14 dkl@redhat.com - Bug 310450 - $dbh->bz_add_column('bugs_activity', 'comment_id', {TYPE => 'INT3'}); + $dbh->bz_add_column('bugs_activity', 'comment_id', {TYPE => 'INT4'}); # 2010-04-07 LpSolit@gmail.com - Bug 69621 $dbh->bz_drop_column('bugs', 'keywords'); @@ -660,6 +660,9 @@ sub update_table_definitions { # 2011-10-11 miketosh - Bug 690173 _on_delete_set_null_for_audit_log_userid(); + # 2011-11-01 glob@mozilla.com - Bug 240437 + $dbh->bz_add_column('profiles', 'last_seen_date', {TYPE => 'DATETIME'}); + # 2011-11-28 dkl@mozilla.com - Bug 685611 _fix_notnull_defaults(); @@ -669,6 +672,45 @@ sub update_table_definitions { $dbh->bz_add_index('profile_search', 'profile_search_user_id_idx', [qw(user_id)]); } + # 2012-06-06 dkl@mozilla.com - Bug 762288 + $dbh->bz_alter_column('bugs_activity', 'removed', + { TYPE => 'varchar(255)' }); + $dbh->bz_add_index('bugs_activity', 'bugs_activity_removed_idx', ['removed']); + + # 2012-06-13 dkl@mozilla.com - Bug 764457 + $dbh->bz_add_column('bugs_activity', 'id', + {TYPE => 'INTSERIAL', NOTNULL => 1, PRIMARYKEY => 1}); + + # 2012-06-13 dkl@mozilla.com - Bug 764466 + $dbh->bz_add_column('profiles_activity', 'id', + {TYPE => 'MEDIUMSERIAL', NOTNULL => 1, PRIMARYKEY => 1}); + + # 2012-07-24 dkl@mozilla.com - Bug 776972 + # BMO - we change this to BIGSERIAL further down + #$dbh->bz_alter_column('bugs_activity', 'id', + # {TYPE => 'INTSERIAL', NOTNULL => 1, PRIMARYKEY => 1}); + + + # 2012-07-24 dkl@mozilla.com - Bug 776982 + _fix_longdescs_primary_key(); + + # 2012-08-02 dkl@mozilla.com - Bug 756953 + _fix_dependencies_dupes(); + + # 2013-02-04 dkl@mozilla.com - Bug 824346 + _fix_flagclusions_indexes(); + + # 2012-04-15 Frank@Frank-Becker.de - Bug 740536 + $dbh->bz_add_index('audit_log', 'audit_log_class_idx', ['class', 'at_time']); + + # 2013-08-16 glob@mozilla.com - Bug 905925 + $dbh->bz_add_index('attachments', 'attachments_ispatch_idx', ['ispatch']); + + # 2014-06-09 dylan@mozilla.com - Bug 1022923 + $dbh->bz_add_index('bug_user_last_visit', + 'bug_user_last_visit_last_visit_ts_idx', + ['last_visit_ts']); + ################################################################ # New --TABLE-- changes should go *** A B O V E *** this point # ################################################################ @@ -2396,13 +2438,16 @@ sub _migrate_email_prefs_to_new_table { } } -sub _initialize_dependency_tree_changes_email_pref { +sub _initialize_new_email_prefs { my $dbh = Bugzilla->dbh; # Check for any "new" email settings that wouldn't have been ported over # during the block above. Since these settings would have otherwise # fallen under EVT_OTHER, we'll just clone those settings. That way if # folks have already disabled all of that mail, there won't be any change. - my %events = ("Dependency Tree Changes" => EVT_DEPEND_BLOCK); + my %events = ( + "Dependency Tree Changes" => EVT_DEPEND_BLOCK, + "Product/Component Changes" => EVT_COMPONENT, + ); foreach my $desc (keys %events) { my $event = $events{$desc}; @@ -3220,6 +3265,11 @@ sub _populate_bugs_fulltext { print "Populating bugs_fulltext with $num_bugs entries..."; print " (this can take a long time.)\n"; } + + # As recommended by Monty Widenius for GNOME's upgrade. + # mkanat and justdave concur it'll be helpful for bmo, too. + $dbh->do('SET SESSION myisam_sort_buffer_size = 3221225472'); + my $newline = $dbh->quote("\n"); $dbh->do( qq{$command INTO bugs_fulltext (bug_id, short_desc, comments, @@ -3682,6 +3732,70 @@ sub _fix_notnull_defaults { } } +sub _fix_longdescs_primary_key { + my $dbh = Bugzilla->dbh; + if ($dbh->bz_column_info('longdescs', 'comment_id')->{TYPE} ne 'INTSERIAL') { + $dbh->bz_drop_related_fks('longdescs', 'comment_id'); + $dbh->bz_alter_column('bugs_activity', 'comment_id', {TYPE => 'INT4'}); + $dbh->bz_alter_column('longdescs', 'comment_id', + {TYPE => 'INTSERIAL', NOTNULL => 1, PRIMARYKEY => 1}); + } +} + +sub _fix_dependencies_dupes { + my $dbh = Bugzilla->dbh; + my $blocked_idx = $dbh->bz_index_info('dependencies', 'dependencies_blocked_idx'); + if ($blocked_idx && scalar @{$blocked_idx->{'FIELDS'}} < 2) { + # Remove duplicated entries + my $dupes = $dbh->selectall_arrayref(" + SELECT blocked, dependson, COUNT(*) AS count + FROM dependencies " . + $dbh->sql_group_by('blocked, dependson') . " + HAVING COUNT(*) > 1", + { Slice => {} }); + print "Removing duplicated entries from the 'dependencies' table...\n" if @$dupes; + foreach my $dupe (@$dupes) { + $dbh->do("DELETE FROM dependencies + WHERE blocked = ? AND dependson = ?", + undef, $dupe->{blocked}, $dupe->{dependson}); + $dbh->do("INSERT INTO dependencies (blocked, dependson) VALUES (?, ?)", + undef, $dupe->{blocked}, $dupe->{dependson}); + } + $dbh->bz_drop_index('dependencies', 'dependencies_blocked_idx'); + $dbh->bz_add_index('dependencies', 'dependencies_blocked_idx', + { FIELDS => [qw(blocked dependson)], TYPE => 'UNIQUE' }); + } +} + +sub _fix_flagclusions_indexes { + my $dbh = Bugzilla->dbh; + foreach my $table ('flaginclusions', 'flagexclusions') { + my $index = $table . '_type_id_idx'; + my $idx_info = $dbh->bz_index_info($table, $index); + if ($idx_info && $idx_info->{'TYPE'} ne 'UNIQUE') { + # Remove duplicated entries + my $dupes = $dbh->selectall_arrayref(" + SELECT type_id, product_id, component_id, COUNT(*) AS count + FROM $table " . + $dbh->sql_group_by('type_id, product_id, component_id') . " + HAVING COUNT(*) > 1", + { Slice => {} }); + print "Removing duplicated entries from the '$table' table...\n" if @$dupes; + foreach my $dupe (@$dupes) { + $dbh->do("DELETE FROM $table + WHERE type_id = ? AND product_id = ? AND component_id = ?", + undef, $dupe->{type_id}, $dupe->{product_id}, $dupe->{component_id}); + $dbh->do("INSERT INTO $table (type_id, product_id, component_id) VALUES (?, ?, ?)", + undef, $dupe->{type_id}, $dupe->{product_id}, $dupe->{component_id}); + } + $dbh->bz_drop_index($table, $index); + $dbh->bz_add_index($table, $index, + { FIELDS => [qw(type_id product_id component_id)], + TYPE => 'UNIQUE' }); + } + } +} + 1; __END__ diff --git a/Bugzilla/Install/Filesystem.pm b/Bugzilla/Install/Filesystem.pm index c5215ecfa..621ece803 100644 --- a/Bugzilla/Install/Filesystem.pm +++ b/Bugzilla/Install/Filesystem.pm @@ -38,6 +38,7 @@ use File::Find; use File::Path; use File::Basename; use File::Copy qw(move); +use File::Slurp; use IO::File; use POSIX (); @@ -125,6 +126,7 @@ sub FILESYSTEM { my $localconfig = bz_locations()->{'localconfig'}; my $template_cache = bz_locations()->{'template_cache'}; my $graphsdir = bz_locations()->{'graphsdir'}; + my $assetsdir = bz_locations()->{'assetsdir'}; # We want to set the permissions the same for all localconfig files # across all PROJECTs, so we do something special with $localconfig, @@ -159,7 +161,10 @@ sub FILESYSTEM { 'runtests.pl' => { perms => OWNER_EXECUTE }, 'jobqueue.pl' => { perms => OWNER_EXECUTE }, 'migrate.pl' => { perms => OWNER_EXECUTE }, + 'sentry.pl' => { perms => WS_EXECUTE }, + 'metrics.pl' => { perms => WS_EXECUTE }, 'install-module.pl' => { perms => OWNER_EXECUTE }, + 'clean-bug-user-last-visit.pl' => { perms => WS_EXECUTE }, 'Bugzilla.pm' => { perms => CGI_READ }, "$localconfig*" => { perms => CGI_READ }, @@ -170,6 +175,7 @@ sub FILESYSTEM { 'contrib/README' => { perms => OWNER_WRITE }, 'contrib/*/README' => { perms => OWNER_WRITE }, + 'contrib/sendunsentbugmail.pl' => { perms => WS_EXECUTE }, 'docs/bugzilla.ent' => { perms => OWNER_WRITE }, 'docs/makedocs.pl' => { perms => OWNER_EXECUTE }, 'docs/style.css' => { perms => WS_SERVE }, @@ -179,13 +185,16 @@ sub FILESYSTEM { "$datadir/old-params.txt" => { perms => OWNER_WRITE }, "$extensionsdir/create.pl" => { perms => OWNER_EXECUTE }, "$extensionsdir/*/*.pl" => { perms => WS_EXECUTE }, + "$extensionsdir/*/bin/*" => { perms => WS_EXECUTE }, ); # Directories that we want to set the perms on, but not # recurse through. These are directories we didn't create # in checkesetup.pl. + # + # Purpose of BMO change: unknown. my %non_recurse_dirs = ( - '.' => DIR_WS_SERVE, + '.' => 0755, docs => DIR_WS_SERVE, ); @@ -205,6 +214,8 @@ sub FILESYSTEM { dirs => DIR_CGI_WRITE | DIR_ALSO_WS_SERVE }, "$datadir/db" => { files => CGI_WRITE, dirs => DIR_CGI_WRITE }, + $assetsdir => { files => WS_SERVE, + dirs => DIR_CGI_OVERWRITE | DIR_ALSO_WS_SERVE }, # Readable directories "$datadir/mining" => { files => CGI_READ, @@ -243,10 +254,13 @@ sub FILESYSTEM { dirs => DIR_WS_SERVE }, "$extensionsdir/*/web" => { files => WS_SERVE, dirs => DIR_WS_SERVE }, - + + # Purpose: allow webserver to read .bzr so we execute bzr commands + # in backticks and look at the result over the web. Used to show + # bzr history. + '.bzr' => { files => WS_SERVE, + dirs => DIR_WS_SERVE }, # Directories only for the owner, not for the webserver. - '.bzr' => { files => OWNER_WRITE, - dirs => DIR_OWNER_WRITE }, t => { files => OWNER_WRITE, dirs => DIR_OWNER_WRITE }, xt => { files => OWNER_WRITE, @@ -264,7 +278,8 @@ sub FILESYSTEM { # The name of each directory that we should actually *create*, # pointing at its default permissions. my %create_dirs = ( - # This is DIR_ALSO_WS_SERVE because it contains $webdotdir. + # This is DIR_ALSO_WS_SERVE because it contains $webdotdir and + # $assetsdir. $datadir => DIR_CGI_OVERWRITE | DIR_ALSO_WS_SERVE, # Directories that are read-only for cgi scripts "$datadir/mining" => DIR_CGI_READ, @@ -275,6 +290,7 @@ sub FILESYSTEM { $attachdir => DIR_CGI_WRITE, $graphsdir => DIR_CGI_WRITE | DIR_ALSO_WS_SERVE, $webdotdir => DIR_CGI_WRITE | DIR_ALSO_WS_SERVE, + $assetsdir => DIR_CGI_WRITE | DIR_ALSO_WS_SERVE, # Directories that contain content served directly by the web server. "$skinsdir/custom" => DIR_WS_SERVE, "$skinsdir/contrib" => DIR_WS_SERVE, @@ -364,6 +380,18 @@ EOT Deny from all EOT }, + + "$assetsdir/.htaccess" => { perms => WS_SERVE, contents => <<EOT +# Allow access to .css files +<FilesMatch \\.(css|js)\$> + Allow from all +</FilesMatch> + +# And no directory listings, either. +Deny from all +EOT + }, + ); Bugzilla::Hook::process('install_filesystem', { @@ -398,6 +426,7 @@ sub update_filesystem { my $datadir = bz_locations->{'datadir'}; my $graphsdir = bz_locations->{'graphsdir'}; + my $assetsdir = bz_locations->{'assetsdir'}; # If the graphs/ directory doesn't exist, we're upgrading from # a version old enough that we need to update the $datadir/mining # format. @@ -431,6 +460,13 @@ sub update_filesystem { _rename_file($oldparamsfile, "$datadir/$oldparamsfile"); } + # Remove old assets htaccess file to force recreation with correct values. + if (-e "$assetsdir/.htaccess") { + if (read_file("$assetsdir/.htaccess") =~ /<FilesMatch \\\.css\$>/) { + unlink("$assetsdir/.htaccess"); + } + } + _create_files(%files); if ($params->{index_html}) { _create_files(%{$fs->{index_html}}); @@ -474,6 +510,7 @@ EOT _remove_empty_css_files(); _convert_single_file_skins(); + _remove_dynamic_assets(); } sub _remove_empty_css_files { @@ -518,6 +555,27 @@ sub _convert_single_file_skins { } } +# delete all automatically generated css/js files to force recreation at the +# next request. +sub _remove_dynamic_assets { + my @files = ( + glob(bz_locations()->{assetsdir} . '/*.css'), + glob(bz_locations()->{assetsdir} . '/*.js'), + ); + foreach my $file (@files) { + unlink($file); + } + + # remove old skins/assets directory + my $old_path = bz_locations()->{skinsdir} . '/assets'; + if (-d $old_path) { + foreach my $file (glob("$old_path/*.css")) { + unlink($file); + } + rmdir($old_path); + } +} + sub create_htaccess { _create_files(%{FILESYSTEM()->{htaccess}}); diff --git a/Bugzilla/Install/Requirements.pm b/Bugzilla/Install/Requirements.pm index 2e67457ab..1c9c91345 100644 --- a/Bugzilla/Install/Requirements.pm +++ b/Bugzilla/Install/Requirements.pm @@ -288,7 +288,7 @@ sub OPTIONAL_MODULES { package => 'JSON-RPC', module => 'JSON::RPC', version => 0, - feature => ['jsonrpc'], + feature => ['jsonrpc', 'rest'], }, { package => 'JSON-XS', @@ -301,7 +301,7 @@ sub OPTIONAL_MODULES { package => 'Test-Taint', module => 'Test::Taint', version => 0, - feature => ['jsonrpc', 'xmlrpc'], + feature => ['jsonrpc', 'xmlrpc', 'rest'], }, { # We need the 'utf8_mode' method of HTML::Parser, for HTML::Scrubber. @@ -372,6 +372,27 @@ sub OPTIONAL_MODULES { version => '0.96', feature => ['mod_perl'], }, + + # memcached + { + package => 'URI-Escape', + module => 'URI::Escape', + version => 0, + feature => ['memcached'], + }, + { + package => 'Cache-Memcached', + module => 'Cache::Memcached', + version => '0', + feature => ['memcached'], + }, + + # BMO - metrics + { + package => 'ElasticSearch', + module => 'ElasticSearch', + version => '0', + }, ); my $extra_modules = _get_extension_requirements('OPTIONAL_MODULES'); @@ -385,6 +406,7 @@ use constant FEATURE_FILES => ( jsonrpc => ['Bugzilla/WebService/Server/JSONRPC.pm', 'jsonrpc.cgi'], xmlrpc => ['Bugzilla/WebService/Server/XMLRPC.pm', 'xmlrpc.cgi', 'Bugzilla/WebService.pm', 'Bugzilla/WebService/*.pm'], + rest => ['Bugzilla/WebService/Server/REST.pm', 'rest.cgi'], moving => ['importxml.pl'], auth_ldap => ['Bugzilla/Auth/Verify/LDAP.pm'], auth_radius => ['Bugzilla/Auth/Verify/RADIUS.pm'], @@ -393,6 +415,7 @@ use constant FEATURE_FILES => ( 'Bugzilla/JobQueue/*', 'jobqueue.pl'], patch_viewer => ['Bugzilla/Attachment/PatchReader.pm'], updates => ['Bugzilla/Update.pm'], + memcached => ['Bugzilla/Memcache.pm'], ); # This implements the REQUIRED_MODULES and OPTIONAL_MODULES stuff diff --git a/Bugzilla/Install/Util.pm b/Bugzilla/Install/Util.pm index bd8942507..5f6c8bceb 100644 --- a/Bugzilla/Install/Util.pm +++ b/Bugzilla/Install/Util.pm @@ -382,7 +382,10 @@ sub include_languages { # Basically, the way this works is that we have a list of languages # that we *want*, and a list of languages that Bugzilla actually - # supports. + # supports. If there is only one language installed, we take it. + my $supported = supported_languages(); + return @$supported if @$supported == 1; + my $wanted; if ($params->{language}) { # We can pass several languages at once as an arrayref @@ -393,7 +396,6 @@ sub include_languages { else { $wanted = _wanted_languages(); } - my $supported = supported_languages(); my $actual = _wanted_to_actual_languages($wanted, $supported); return @$actual; } diff --git a/Bugzilla/Job/BugMail.pm b/Bugzilla/Job/BugMail.pm new file mode 100644 index 000000000..9c176b005 --- /dev/null +++ b/Bugzilla/Job/BugMail.pm @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Job::BugMail; + +use 5.10.1; +use strict; + +use Bugzilla::BugMail; +BEGIN { eval "use parent qw(Bugzilla::Job::Mailer)"; } + +sub work { + my ($class, $job) = @_; + my $success = eval { + Bugzilla::BugMail::dequeue($job->arg->{vars}); + 1; + }; + if (!$success) { + $job->failed($@); + undef $@; + } + else { + $job->completed; + } +} + +1; diff --git a/Bugzilla/JobQueue.pm b/Bugzilla/JobQueue.pm index 7ea678345..669076dd5 100644 --- a/Bugzilla/JobQueue.pm +++ b/Bugzilla/JobQueue.pm @@ -27,12 +27,15 @@ use strict; use Bugzilla::Constants; use Bugzilla::Error; use Bugzilla::Install::Util qw(install_string); +use File::Slurp; use base qw(TheSchwartz); +use fields qw(_worker_pidfile); # This maps job names for Bugzilla::JobQueue to the appropriate modules. # If you add new types of jobs, you should add a mapping here. use constant JOB_MAP => { send_mail => 'Bugzilla::Job::Mailer', + bug_mail => 'Bugzilla::Job::BugMail', }; # Without a driver cache TheSchwartz opens a new database connection @@ -40,6 +43,10 @@ use constant JOB_MAP => { # across requests. use constant DRIVER_CACHE_TIME => 300; # 5 minutes +# To avoid memory leak/fragmentation, a worker process won't process more than +# MAX_MESSAGES messages. +use constant MAX_MESSAGES => 1000; + sub job_map { if (!defined(Bugzilla->request_cache->{job_map})) { my $job_map = JOB_MAP; @@ -99,6 +106,74 @@ sub insert { return $retval; } +# To avoid memory leaks/fragmentation which tends to happen for long running +# perl processes; check for jobs, and spawn a new process to empty the queue. +sub subprocess_worker { + my $self = shift; + + my $command = "$0 -p '" . $self->{_worker_pidfile} . "' onepass"; + + while (1) { + my $time = (time); + my @jobs = $self->list_jobs({ + funcname => $self->{all_abilities}, + run_after => $time, + grabbed_until => $time, + limit => 1, + }); + if (@jobs) { + $self->debug("Spawning queue worker process"); + # Run the worker as a daemon + system $command; + # And poll the PID to detect when the working has finished. + # We do this instead of system() to allow for the INT signal to + # interrup us and trigger kill_worker(). + my $pid = read_file($self->{_worker_pidfile}, err_mode => 'quiet'); + if ($pid) { + sleep(3) while(kill(0, $pid)); + } + $self->debug("Queue worker process completed"); + } else { + $self->debug("No jobs found"); + } + sleep(5); + } +} + +sub kill_worker { + my $self = Bugzilla->job_queue(); + if ($self->{_worker_pidfile} && -e $self->{_worker_pidfile}) { + my $worker_pid = read_file($self->{_worker_pidfile}); + if ($worker_pid && kill(0, $worker_pid)) { + $self->debug("Stopping worker process"); + system "$0 -f -p '" . $self->{_worker_pidfile} . "' stop"; + } + } +} + +sub set_pidfile { + my ($self, $pidfile) = @_; + $pidfile =~ s/^(.+)(\..+)$/$1.worker$2/; + $self->{_worker_pidfile} = $pidfile; +} + +# Clear the request cache at the start of each run. +sub work_once { + my $self = shift; + Bugzilla->clear_request_cache(); + return $self->SUPER::work_once(@_); +} + +# Never process more than MAX_MESSAGES in one batch, to avoid memory +# leak/fragmentation issues. +sub work_until_done { + my $self = shift; + my $count = 0; + while ($count++ < MAX_MESSAGES) { + $self->work_once or last; + } +} + 1; __END__ diff --git a/Bugzilla/JobQueue/Runner.pm b/Bugzilla/JobQueue/Runner.pm index 26755e78f..d45c36647 100644 --- a/Bugzilla/JobQueue/Runner.pm +++ b/Bugzilla/JobQueue/Runner.pm @@ -51,6 +51,7 @@ our $initscript = "bugzilla-queue"; sub gd_preconfig { my $self = shift; + $self->{_run_command} = 'subprocess_worker'; my $pidfile = $self->{gd_args}{pidfile}; if (!$pidfile) { $pidfile = bz_locations()->{datadir} . '/' . $self->{gd_progname} @@ -196,21 +197,26 @@ sub gd_setup_signals { $SIG{TERM} = sub { $self->gd_quit_event(); } } -sub gd_other_cmd { - my ($self) = shift; - if ($ARGV[0] eq "once") { - $self->_do_work("work_once"); +sub gd_quit_event { + Bugzilla->job_queue->kill_worker(); + exit(1); +} - exit(0); +sub gd_other_cmd { + my ($self, $do, $locked) = @_; + if ($do eq "once") { + $self->{_run_command} = 'work_once'; + } elsif ($do eq "onepass") { + $self->{_run_command} = 'work_until_done'; + } else { + $self->SUPER::gd_other_cmd($do, $locked); } - - $self->SUPER::gd_other_cmd(); } sub gd_run { my $self = shift; - - $self->_do_work("work"); + $SIG{__DIE__} = \&Carp::confess if $self->{debug}; + $self->_do_work($self->{_run_command}); } sub _do_work { @@ -218,6 +224,7 @@ sub _do_work { my $jq = Bugzilla->job_queue(); $jq->set_verbose($self->{debug}); + $jq->set_pidfile($self->{gd_pidfile}); foreach my $module (values %{ Bugzilla::JobQueue->job_map() }) { eval "use $module"; $jq->can_do($module); diff --git a/Bugzilla/Keyword.pm b/Bugzilla/Keyword.pm index e2ecc29e5..cda401c59 100644 --- a/Bugzilla/Keyword.pm +++ b/Bugzilla/Keyword.pm @@ -27,6 +27,8 @@ use Bugzilla::Util; #### Initialization #### ############################### +use constant IS_CONFIG => 1; + use constant DB_COLUMNS => qw( keyworddefs.id keyworddefs.name diff --git a/Bugzilla/Mailer.pm b/Bugzilla/Mailer.pm index 1c4fb6188..381422821 100644 --- a/Bugzilla/Mailer.pm +++ b/Bugzilla/Mailer.pm @@ -55,6 +55,7 @@ BEGIN { $Return::Value::NO_CLUCK = 1; } use Email::Send; +use Sys::Hostname; sub MessageToMTA { my ($msg, $send_now) = (@_); @@ -93,29 +94,6 @@ sub MessageToMTA { # thus to hopefully avoid auto replies. $email->header_set('Auto-Submitted', 'auto-generated'); - $email->walk_parts(sub { - my ($part) = @_; - return if $part->parts > 1; # Top-level - my $content_type = $part->content_type || ''; - $content_type =~ /charset=['"](.+)['"]/; - # If no charset is defined or is the default us-ascii, - # then we encode the email to UTF-8 if Bugzilla has utf8 enabled. - # XXX - This is a hack to workaround bug 723944. - if (!$1 || $1 eq 'us-ascii') { - my $body = $part->body; - if (Bugzilla->params->{'utf8'}) { - $part->charset_set('UTF-8'); - # encoding_set works only with bytes, not with utf8 strings. - my $raw = $part->body_raw; - if (utf8::is_utf8($raw)) { - utf8::encode($raw); - $part->body_set($raw); - } - } - $part->encoding_set('quoted-printable') if !is_7bit_clean($body); - } - }); - # MIME-Version must be set otherwise some mailsystems ignore the charset $email->header_set('MIME-Version', '1.0') if !$email->header('MIME-Version'); @@ -140,7 +118,9 @@ sub MessageToMTA { my $from = $email->header('From'); my ($hostname, @args); + my $mailer_class = $method; if ($method eq "Sendmail") { + $mailer_class = 'Bugzilla::Send::Sendmail'; if (ON_WINDOWS) { $Email::Send::Sendmail::SENDMAIL = SENDMAIL_EXE; } @@ -169,6 +149,12 @@ sub MessageToMTA { } } + # For tracking/diagnostic purposes, add our hostname + my $generated_by = $email->header('X-Generated-By') || ''; + if ($generated_by =~ tr/\/// < 3) { + $email->header_set('X-Generated-By' => $generated_by . '/' . hostname() . "($$)"); + } + if ($method eq "SMTP") { push @args, Host => Bugzilla->params->{"smtpserver"}, username => Bugzilla->params->{"smtp_username"}, @@ -180,6 +166,32 @@ sub MessageToMTA { Bugzilla::Hook::process('mailer_before_send', { email => $email, mailer_args => \@args }); + # Allow for extensions to to drop the bugmail by clearing the 'to' header + return if $email->header('to') eq ''; + + $email->walk_parts(sub { + my ($part) = @_; + return if $part->parts > 1; # Top-level + my $content_type = $part->content_type || ''; + $content_type =~ /charset=['"](.+)['"]/; + # If no charset is defined or is the default us-ascii, + # then we encode the email to UTF-8 if Bugzilla has utf8 enabled. + # XXX - This is a hack to workaround bug 723944. + if (!$1 || $1 eq 'us-ascii') { + my $body = $part->body; + if (Bugzilla->params->{'utf8'}) { + $part->charset_set('UTF-8'); + # encoding_set works only with bytes, not with utf8 strings. + my $raw = $part->body_raw; + if (utf8::is_utf8($raw)) { + utf8::encode($raw); + $part->body_set($raw); + } + } + $part->encoding_set('quoted-printable') if !is_7bit_clean($body); + } + }); + if ($method eq "Test") { my $filename = bz_locations()->{'datadir'} . '/mailer.testfile'; open TESTFILE, '>>', $filename; @@ -190,7 +202,7 @@ sub MessageToMTA { else { # This is useful for both Sendmail and Qmail, so we put it out here. local $ENV{PATH} = SENDMAIL_PATH; - my $mailer = Email::Send->new({ mailer => $method, + my $mailer = Email::Send->new({ mailer => $mailer_class, mailer_args => \@args }); my $retval = $mailer->send($email); ThrowCodeError('mail_send_error', { msg => $retval, mail => $email }) diff --git a/Bugzilla/Memcached.pm b/Bugzilla/Memcached.pm new file mode 100644 index 000000000..fdafa0014 --- /dev/null +++ b/Bugzilla/Memcached.pm @@ -0,0 +1,490 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Memcached; + +use 5.10.1; +use strict; +use warnings; + +use Bugzilla::Error; +use Bugzilla::Util qw(trick_taint); +use Scalar::Util qw(blessed); +use URI::Escape; +use Encode; +use Sys::Syslog qw(:DEFAULT); + +# memcached keys have a maximum length of 250 bytes +use constant MAX_KEY_LENGTH => 250; + +sub _new { + my $invocant = shift; + my $class = ref($invocant) || $invocant; + my $self = {}; + + # always return an object to simplify calling code when memcached is + # disabled. + if (Bugzilla->feature('memcached') + && Bugzilla->params->{memcached_servers}) + { + require Cache::Memcached; + $self->{namespace} = Bugzilla->params->{memcached_namespace} || ''; + $self->{memcached} = + Cache::Memcached->new({ + servers => [ split(/[, ]+/, Bugzilla->params->{memcached_servers}) ], + namespace => $self->{namespace}, + }); + } + return bless($self, $class); +} + +sub enabled { + return $_[0]->{memcached} ? 1 : 0; +} + +sub set { + my ($self, $args) = @_; + return unless $self->{memcached}; + + # { key => $key, value => $value } + if (exists $args->{key}) { + $self->_set($args->{key}, $args->{value}); + } + + # { table => $table, id => $id, name => $name, data => $data } + elsif (exists $args->{table} && exists $args->{id} && exists $args->{name}) { + # For caching of Bugzilla::Object, we have to be able to clear the + # cached values when given either the object's id or name. + my ($table, $id, $name, $data) = @$args{qw(table id name data)}; + $self->_set("$table.id.$id", $data); + if (defined $name) { + $self->_set("$table.name_id.$name", $id); + $self->_set("$table.id_name.$id", $name); + } + } + + else { + ThrowCodeError('params_required', { function => "Bugzilla::Memcached::set", + params => [ 'key', 'table' ] }); + } +} + +sub get { + my ($self, $args) = @_; + return unless $self->{memcached}; + + # { key => $key } + if (exists $args->{key}) { + return $self->_get($args->{key}); + } + + # { table => $table, id => $id } + elsif (exists $args->{table} && exists $args->{id}) { + my ($table, $id) = @$args{qw(table id)}; + return $self->_get("$table.id.$id"); + } + + # { table => $table, name => $name } + elsif (exists $args->{table} && exists $args->{name}) { + my ($table, $name) = @$args{qw(table name)}; + return unless my $id = $self->_get("$table.name_id.$name"); + return $self->_get("$table.id.$id"); + } + + else { + ThrowCodeError('params_required', { function => "Bugzilla::Memcached::get", + params => [ 'key', 'table' ] }); + } +} + +sub set_config { + my ($self, $args) = @_; + return unless $self->{memcached}; + + if (exists $args->{key}) { + return $self->_set($self->_config_prefix . '.' . $args->{key}, $args->{data}); + } + else { + ThrowCodeError('params_required', { function => "Bugzilla::Memcached::set_config", + params => [ 'key' ] }); + } +} + +sub get_config { + my ($self, $args) = @_; + return unless $self->{memcached}; + + if (exists $args->{key}) { + return $self->_get($self->_config_prefix . '.' . $args->{key}); + } + else { + ThrowCodeError('params_required', { function => "Bugzilla::Memcached::get_config", + params => [ 'key' ] }); + } +} + +sub clear { + my ($self, $args) = @_; + return unless $self->{memcached}; + + # { key => $key } + if (exists $args->{key}) { + $self->_delete($args->{key}); + } + + # { table => $table, id => $id } + elsif (exists $args->{table} && exists $args->{id}) { + my ($table, $id) = @$args{qw(table id)}; + my $name = $self->_get("$table.id_name.$id"); + $self->_delete("$table.id.$id"); + $self->_delete("$table.name_id.$name") if defined $name; + $self->_delete("$table.id_name.$id"); + } + + # { table => $table, name => $name } + elsif (exists $args->{table} && exists $args->{name}) { + my ($table, $name) = @$args{qw(table name)}; + return unless my $id = $self->_get("$table.name_id.$name"); + $self->_delete("$table.id.$id"); + $self->_delete("$table.name_id.$name"); + $self->_delete("$table.id_name.$id"); + } + + else { + ThrowCodeError('params_required', { function => "Bugzilla::Memcached::clear", + params => [ 'key', 'table' ] }); + } +} + +sub clear_all { + my ($self) = @_; + return unless $self->{memcached}; + $self->_inc_prefix("global"); +} + +sub clear_config { + my ($self, $args) = @_; + return unless $self->{memcached}; + if ($args && exists $args->{key}) { + $self->_delete($self->_config_prefix . '.' . $args->{key}); + } + else { + $self->_inc_prefix("config"); + } +} + +# in order to clear all our keys, we add a prefix to all our keys. when we +# need to "clear" all current keys, we increment the prefix. +sub _prefix { + my ($self, $name) = @_; + # we don't want to change prefixes in the middle of a request + my $request_cache = Bugzilla->request_cache; + my $request_cache_key = "memcached_prefix_$name"; + if (!$request_cache->{$request_cache_key}) { + my $memcached = $self->{memcached}; + my $prefix = $memcached->get($name); + if (!$prefix) { + $prefix = time(); + if (!$memcached->add($name, $prefix)) { + # if this failed, either another process set the prefix, or + # memcached is down. assume we lost the race, and get the new + # value. if that fails, memcached is down so use a dummy + # prefix for this request. + $prefix = $memcached->get($name) || 0; + } + } + $request_cache->{$request_cache_key} = $prefix; + } + return $request_cache->{$request_cache_key}; +} + +sub _inc_prefix { + my ($self, $name) = @_; + my $memcached = $self->{memcached}; + if (!$memcached->incr($name, 1)) { + $memcached->add($name, time()); + } + delete Bugzilla->request_cache->{"memcached_prefix_$name"}; + + # BMO - log that we've wiped the cache + openlog('apache', 'cons,pid', 'local4'); + syslog('notice', encode_utf8("[memcached] $name cache cleared")); + closelog(); +} + +sub _global_prefix { + return $_[0]->_prefix("global"); +} + +sub _config_prefix { + return $_[0]->_prefix("config"); +} + +sub _encode_key { + my ($self, $key) = @_; + $key = $self->_global_prefix . '.' . uri_escape_utf8($key); + return length($self->{namespace} . $key) > MAX_KEY_LENGTH + ? undef + : $key; +} + +sub _set { + my ($self, $key, $value) = @_; + if (blessed($value)) { + # we don't support blessed objects + ThrowCodeError('param_invalid', { function => "Bugzilla::Memcached::set", + param => "value" }); + } + + $key = $self->_encode_key($key) + or return; + return $self->{memcached}->set($key, $value); +} + +sub _get { + my ($self, $key) = @_; + + $key = $self->_encode_key($key) + or return; + my $value = $self->{memcached}->get($key); + return unless defined $value; + + # detaint returned values + # hashes and arrays are detainted just one level deep + if (ref($value) eq 'HASH') { + _detaint_hashref($value); + } + elsif (ref($value) eq 'ARRAY') { + foreach my $value (@$value) { + next unless defined $value; + # arrays of hashes and arrays are common + if (ref($value) eq 'HASH') { + _detaint_hashref($value); + } + elsif (ref($value) eq 'ARRAY') { + _detaint_arrayref($value); + } + elsif (!ref($value)) { + trick_taint($value); + } + } + } + elsif (!ref($value)) { + trick_taint($value); + } + return $value; +} + +sub _detaint_hashref { + my ($hashref) = @_; + foreach my $value (values %$hashref) { + if (defined($value) && !ref($value)) { + trick_taint($value); + } + } +} + +sub _detaint_arrayref { + my ($arrayref) = @_; + foreach my $value (@$arrayref) { + if (defined($value) && !ref($value)) { + trick_taint($value); + } + } +} + +sub _delete { + my ($self, $key) = @_; + $key = $self->_encode_key($key) + or return; + return $self->{memcached}->delete($key); +} + +1; + +__END__ + +=head1 NAME + +Bugzilla::Memcached - Interface between Bugzilla and Memcached. + +=head1 SYNOPSIS + + use Bugzilla; + + my $memcached = Bugzilla->memcached; + + # grab data from the cache. there is no need to check if memcached is + # available or enabled. + my $data = $memcached->get({ key => 'data_key' }); + if (!defined $data) { + # not in cache, generate the data and populate the cache for next time + $data = some_long_process(); + $memcached->set({ key => 'data_key', value => $data }); + } + # do something with $data + + # updating the profiles table directly shouldn't be attempted unless you know + # what you're doing. if you do update a table directly, you need to clear that + # object from memcached. + $dbh->do("UPDATE profiles SET request_count=10 WHERE login_name=?", undef, $login); + $memcached->clear({ table => 'profiles', name => $login }); + +=head1 DESCRIPTION + +If Memcached is installed and configured, Bugzilla can use it to cache data +across requests and between webheads. Unlike the request and process caches, +only scalars, hashrefs, and arrayrefs can be stored in Memcached. + +Memcached integration is only required for large installations of Bugzilla -- +if you have multiple webheads then configuring Memcache is recommended. + +L<Bugzilla::Memcached> provides an interface to a Memcached server/servers, with +the ability to get, set, or clear entries from the cache. + +The stored value must be an unblessed hashref, unblessed array ref, or a +scalar. Currently nested data structures are supported but require manual +de-tainting after reading from Memcached (flat data structures are automatically +de-tainted). + +All values are stored in the Memcached systems using the prefix configured with +the C<memcached_namespace> parameter, as well as an additional prefix managed +by this class to allow all values to be cleared when C<checksetup.pl> is +executed. + +Do not create an instance of this object directly, instead use +L<Bugzilla-E<gt>memcached()|Bugzilla/memcached>. + +=head1 METHODS + +=over + +=item C<enabled> + +Returns true if Memcached support is available and enabled. + +=back + +=head2 Setting + +Adds a value to Memcached. + +=over + +=item C<set({ key =E<gt> $key, value =E<gt> $value })> + +Adds the C<value> using the specific C<key>. + +=item C<set({ table =E<gt> $table, id =E<gt> $id, name =E<gt> $name, data =E<gt> $data })> + +Adds the C<data> using a keys generated from the C<table>, C<id>, and C<name>. +All three parameters must be provided, however C<name> can be provided but set +to C<undef>. + +This is a convenience method which allows cached data to be later retrieved by +specifying the C<table> and either the C<id> or C<name>. + +=item C<set_config({ key =E<gt> $key, data =E<gt> $data })> + +Adds the C<data> using the C<key> while identifying the data as part of +Bugzilla's configuration (such as fields, products, components, groups, etc). +Values set with C<set_config> are automatically cleared when changes are made +to Bugzilla's configuration. + +=back + +=head2 Getting + +Retrieves a value from Memcached. Returns C<undef> if no matching values were +found in the cache. + +=over + +=item C<get({ key =E<gt> $key })> + +Return C<value> with the specified C<key>. + +=item C<get({ table =E<gt> $table, id =E<gt> $id })> + +Return C<value> with the specified C<table> and C<id>. + +=item C<get({ table =E<gt> $table, name =E<gt> $name })> + +Return C<value> with the specified C<table> and C<name>. + +=item C<get_config({ key =E<gt> $key })> + +Return C<value> with the specified C<key> from the configuration cache. See +C<set_config> for more information. + +=back + +=head2 Clearing + +Removes the matching value from Memcached. + +=over + +=item C<clear({ key =E<gt> $key })> + +Removes C<value> with the specified C<key>. + +=item C<clear({ table =E<gt> $table, id =E<gt> $id })> + +Removes C<value> with the specified C<table> and C<id>, as well as the +corresponding C<table> and C<name> entry. + +=item C<clear({ table =E<gt> $table, name =E<gt> $name })> + +Removes C<value> with the specified C<table> and C<name>, as well as the +corresponding C<table> and C<id> entry. + +=item C<clear_config({ key =E<gt> $key })> + +Remove C<value> with the specified C<key> from the configuration cache. See +C<set_config> for more information. + +=item C<clear_config> + +Removes all configuration related values from the cache. See C<set_config> for +more information. + +=item C<clear_all> + +Removes all values from the cache. + +=back + +=head1 Bugzilla::Object CACHE + +The main driver for Memcached integration is to allow L<Bugzilla::Object> based +objects to be automatically cached in Memcache. This is enabled on a +per-package basis by setting the C<USE_MEMCACHED> constant to any true value. + +The current implementation is an opt-in (USE_MEMCACHED is false by default), +however this will change to opt-out once further testing has been completed +(USE_MEMCACHED will be true by default). + +=head1 DIRECT DATABASE UPDATES + +If an object is cached and the database is updated directly (instead of via +C<$object-E<gt>update()>), then it's possible for the data in the cache to be +out of sync with the database. + +As an example let's consider an extension which adds a timestamp field +C<last_activitiy_ts> to the profiles table and user object which contains the +user's last activity. If the extension were to call C<$user-E<gt>update()>, +then an audit entry would be created for each change to the C<last_activity_ts> +field, which is undesirable. + +To remedy this, the extension updates the table directly. It's critical with +Memcached that it then clears the cache: + + $dbh->do("UPDATE profiles SET last_activity_ts=? WHERE userid=?", + undef, $timestamp, $user_id); + Bugzilla->memcached->clear({ table => 'profiles', id => $user_id }); + diff --git a/Bugzilla/Metrics/Collector.pm b/Bugzilla/Metrics/Collector.pm new file mode 100644 index 000000000..53cbac819 --- /dev/null +++ b/Bugzilla/Metrics/Collector.pm @@ -0,0 +1,171 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Metrics::Collector; + +use strict; +use warnings; +use 5.10.1; + +# the reporter needs to be a constant and use'd here to ensure it's loaded at +# compile time. +use constant REPORTER => 'Bugzilla::Metrics::Reporter::ElasticSearch'; +use Bugzilla::Metrics::Reporter::ElasticSearch; + +# Debugging reporter +#use constant REPORTER => 'Bugzilla::Metrics::Reporter::STDERR'; +#use Bugzilla::Metrics::Reporter::STDERR; + +use Bugzilla::Constants; +use Cwd qw(abs_path); +use File::Basename; +use Time::HiRes qw(gettimeofday clock_gettime CLOCK_MONOTONIC); + +sub new { + my ($class, $name) = @_; + my $self = { + root => undef, + head => undef, + time => scalar(gettimeofday()), + }; + bless($self, $class); + $self->_start_timer({ type => 'main', name => $name }); + return $self; +} + +sub end { + my ($self, $timer) = @_; + my $is_head = $timer ? 0 : 1; + $timer ||= $self->{head}; + $timer->{duration} += clock_gettime(CLOCK_MONOTONIC) - $timer->{start_time}; + $self->{head} = $self->{head}->{parent} if $is_head; +} + +sub cancel { + my ($self) = @_; + delete $self->{head}; +} + +sub DESTROY { + my ($self) = @_; + $self->finish() if $self->{head}; +} + +sub finish { + my ($self) = @_; + $self->end($self->{root}); + delete $self->{head}; + + my $user = Bugzilla->user; + if ($ENV{MOD_PERL}) { + require Apache2::RequestUtil; + my $request = eval { Apache2::RequestUtil->request }; + my $headers = $request ? $request->headers_in() : {}; + $self->{env} = { + referer => $headers->{Referer}, + request_method => $request->method, + request_uri => basename($request->unparsed_uri), + script_name => $request->uri, + user_agent => $headers->{'User-Agent'}, + }; + } + else { + $self->{env} = { + referer => $ENV{HTTP_REFERER}, + request_method => $ENV{REQUEST_METHOD}, + request_uri => $ENV{REQUEST_URI}, + script_name => basename($ENV{SCRIPT_NAME}), + user_agent => $ENV{HTTP_USER_AGENT}, + }; + } + $self->{env}->{name} = $self->{root}->{name}; + $self->{env}->{time} = $self->{time}; + $self->{env}->{user_id} = $user->id; + $self->{env}->{login} = $user->login if $user->id; + + # remove passwords from request_uri + $self->{env}->{request_uri} =~ s/\b((?:bugzilla_)?password=)(?:[^&]+|.+$)/$1x/gi; + + $self->report(); +} + +sub name { + my ($self, $value) = @_; + $self->{root}->{name} = $value if defined $value; + return $self->{root}->{name}; +} + +sub db_start { + my ($self) = @_; + my $timer = $self->_start_timer({ type => 'db' }); + + my @stack; + my $i = 0; + state $path = quotemeta(abs_path(bz_locations()->{cgi_path}) . '/'); + while (1) { + my @caller = caller($i); + last unless @caller; + my $file = $caller[1]; + $file =~ s/^$path//o; + push @stack, "$file:$caller[2]" + unless substr($file, 0, 16) eq 'Bugzilla/Metrics'; + last if $file =~ /\.(?:tmpl|cgi)$/; + $i++; + } + $timer->{stack} = \@stack; + + return $timer; +} + +sub template_start { + my ($self, $file) = @_; + $self->_start_timer({ type => 'tmpl', file => $file }); +} + +sub memcached_start { + my ($self, $key) = @_; + $self->_start_timer({ type => 'memcached', key => $key }); +} + +sub memcached_end { + my ($self, $hit) = @_; + $self->{head}->{result} = $hit ? 'hit' : 'miss'; + $self->end(); +} + +sub resume { + my ($self, $timer) = @_; + $timer->{start_time} = clock_gettime(CLOCK_MONOTONIC); + return $timer; +} + +sub _start_timer { + my ($self, $timer) = @_; + $timer->{start_time} = $timer->{first_time} = clock_gettime(CLOCK_MONOTONIC); + $timer->{duration} = 0; + $timer->{children} = []; + + if ($self->{head}) { + $timer->{parent} = $self->{head}; + push @{ $self->{head}->{children} }, $timer; + } + else { + $timer->{parent} = undef; + $self->{root} = $timer; + } + $self->{head} = $timer; + + return $timer; +} + +sub report { + my ($self) = @_; + my $class = REPORTER; + $class->DETACH ? $class->background($self) : $class->foreground($self); +} + +1; diff --git a/Bugzilla/Metrics/Memcached.pm b/Bugzilla/Metrics/Memcached.pm new file mode 100644 index 000000000..8dccdb323 --- /dev/null +++ b/Bugzilla/Metrics/Memcached.pm @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Metrics::Memcached; + +use strict; +use warnings; + +use parent 'Bugzilla::Memcached'; + +sub _get { + my $self = shift; + Bugzilla->metrics->memcached_start($_[0]); + my $result = $self->SUPER::_get(@_); + Bugzilla->metrics->memcached_end($result); + return $result; +} + +1; diff --git a/Bugzilla/Metrics/Mysql.pm b/Bugzilla/Metrics/Mysql.pm new file mode 100644 index 000000000..7719d1cac --- /dev/null +++ b/Bugzilla/Metrics/Mysql.pm @@ -0,0 +1,148 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Metrics::Mysql; + +use 5.10.1; +use strict; +use warnings; + +use parent 'Bugzilla::DB::Mysql'; + +sub do { + my ($self, @args) = @_; + Bugzilla->metrics->db_start($args[0]); + my $result = $self->SUPER::do(@args); + Bugzilla->metrics->end(); + return $result; +} + +sub selectall_arrayref { + my ($self, @args) = @_; + Bugzilla->metrics->db_start($args[0]); + my $result = $self->SUPER::selectall_arrayref(@args); + Bugzilla->metrics->end(); + return $result; +} + +sub selectall_hashref { + my ($self, @args) = @_; + Bugzilla->metrics->db_start($args[0]); + my $result = $self->SUPER::selectall_hashref(@args); + Bugzilla->metrics->end(); + return $result; +} + +sub selectcol_arrayref { + my ($self, @args) = @_; + Bugzilla->metrics->db_start($args[0]); + my $result = $self->SUPER::selectcol_arrayref(@args); + Bugzilla->metrics->end(); + return $result; +} + +sub selectrow_array { + my ($self, @args) = @_; + Bugzilla->metrics->db_start($args[0]); + my @result = $self->SUPER::selectrow_array(@args); + Bugzilla->metrics->end(); + return wantarray ? @result : $result[0]; +} + +sub selectrow_arrayref { + my ($self, @args) = @_; + Bugzilla->metrics->db_start($args[0]); + my $result = $self->SUPER::selectrow_arrayref(@args); + Bugzilla->metrics->end(); + return $result; +} + +sub selectrow_hashref { + my ($self, @args) = @_; + Bugzilla->metrics->db_start($args[0]); + my $result = $self->SUPER::selectrow_hashref(@args); + Bugzilla->metrics->end(); + return $result; +} + +sub commit { + my ($self, @args) = @_; + Bugzilla->metrics->db_start('COMMIT'); + my $result = $self->SUPER::commit(@args); + Bugzilla->metrics->end(); + return $result; +} + +sub prepare { + my ($self, @args) = @_; + my $sth = $self->SUPER::prepare(@args); + bless($sth, 'Bugzilla::Metrics::st'); + return $sth; +} + +package Bugzilla::Metrics::st; + +use 5.10.1; +use strict; +use warnings; + +use base 'DBI::st'; + +sub execute { + my ($self, @args) = @_; + $self->{private_timer} = Bugzilla->metrics->db_start(); + my $result = $self->SUPER::execute(@args); + Bugzilla->metrics->end(); + return $result; +} + +sub fetchrow_array { + my ($self, @args) = @_; + my $timer = $self->{private_timer}; + Bugzilla->metrics->resume($timer); + my @result = $self->SUPER::fetchrow_array(@args); + Bugzilla->metrics->end($timer); + return wantarray ? @result : $result[0]; +} + +sub fetchrow_arrayref { + my ($self, @args) = @_; + my $timer = $self->{private_timer}; + Bugzilla->metrics->resume($timer); + my $result = $self->SUPER::fetchrow_arrayref(@args); + Bugzilla->metrics->end($timer); + return $result; +} + +sub fetchrow_hashref { + my ($self, @args) = @_; + my $timer = $self->{private_timer}; + Bugzilla->metrics->resume($timer); + my $result = $self->SUPER::fetchrow_hashref(@args); + Bugzilla->metrics->end($timer); + return $result; +} + +sub fetchall_arrayref { + my ($self, @args) = @_; + my $timer = $self->{private_timer}; + Bugzilla->metrics->resume($timer); + my $result = $self->SUPER::fetchall_arrayref(@args); + Bugzilla->metrics->end($timer); + return $result; +} + +sub fetchall_hashref { + my ($self, @args) = @_; + my $timer = $self->{private_timer}; + Bugzilla->metrics->resume($timer); + my $result = $self->SUPER::fetchall_hashref(@args); + Bugzilla->metrics->end($timer); + return $result; +} + +1; diff --git a/Bugzilla/Metrics/Reporter.pm b/Bugzilla/Metrics/Reporter.pm new file mode 100644 index 000000000..88a08c9b8 --- /dev/null +++ b/Bugzilla/Metrics/Reporter.pm @@ -0,0 +1,102 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Metrics::Reporter; + +use strict; +use warnings; + +use Bugzilla::Constants; +use File::Slurp; +use File::Temp; +use JSON; + +# most reporters should detach from the httpd process. +# reporters which do not detach will block completion of the http response. +use constant DETACH => 1; + +# class method to start the delivery script in the background +sub background { + my ($class, $collector) = @_; + + # we need to remove parent links to avoid looped structures, which + # encode_json chokes on + _walk_timers($collector->{root}, sub { delete $_[0]->{parent} }); + + # serialisation + my $json = encode_json({ env => $collector->{env}, times => $collector->{root} }); + + # write to temp filename + my $fh = File::Temp->new( UNLINK => 0 ); + if (!$fh) { + warn "Failed to create temp file: $!\n"; + return; + } + binmode($fh, ':utf8'); + print $fh $json; + close($fh) or die "$fh : $!"; + my $filename = $fh->filename; + + # spawn delivery worker + my $command = bz_locations()->{'cgi_path'} . "/metrics.pl '$class' '$filename' &"; + $ENV{PATH} = ''; + system($command); +} + +# run the reporter immediately +sub foreground { + my ($class, $collector) = @_; + my $reporter = $class->new({ hashref => { env => $collector->{env}, times => $collector->{root} } }); + $reporter->report(); +} + +sub new { + my ($invocant, $args) = @_; + my $class = ref($invocant) || $invocant; + + # load from either a json_filename or hashref + my $self; + if ($args->{json_filename}) { + $self = decode_json(read_file($args->{json_filename}, binmode => ':utf8')); + unlink($args->{json_filename}); + } + else { + $self = $args->{hashref}; + } + bless($self, $class); + + # remove redundant data + $self->walk_timers(sub { + my ($timer) = @_; + $timer->{start_time} = delete $timer->{first_time}; + delete $timer->{children} + if exists $timer->{children} && !scalar(@{ $timer->{children} }); + }); + + return $self; +} + +sub walk_timers { + my ($self, $callback) = @_; + _walk_timers($self->{times}, $callback, undef); +} + +sub _walk_timers { + my ($timer, $callback, $parent) = @_; + $callback->($timer, $parent); + if (exists $timer->{children}) { + foreach my $child (@{ $timer->{children} }) { + _walk_timers($child, $callback, $timer); + } + } +} + +sub report { + die "abstract method call"; +} + +1; diff --git a/Bugzilla/Metrics/Reporter/ElasticSearch.pm b/Bugzilla/Metrics/Reporter/ElasticSearch.pm new file mode 100644 index 000000000..1bab2502c --- /dev/null +++ b/Bugzilla/Metrics/Reporter/ElasticSearch.pm @@ -0,0 +1,104 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Metrics::Reporter::ElasticSearch; + +use strict; +use warnings; + +use parent 'Bugzilla::Metrics::Reporter'; + +use constant DETACH => 1; + +sub report { + my ($self) = @_; + + # build path array and flatten + my @timers; + $self->walk_timers(sub { + my ($timer, $parent) = @_; + $timer->{id} = scalar(@timers); + if ($parent) { + if (exists $timer->{children}) { + if ($timer->{type} eq 'tmpl') { + $timer->{node} = 'tmpl: ' . $timer->{file}; + } + elsif ($timer->{type} eq 'db') { + $timer->{node} = 'db'; + } + else { + $timer->{node} = '?'; + } + } + $timer->{path} = [ @{ $parent->{path} }, $parent->{node} ]; + $timer->{parent} = $parent->{id}; + } + else { + $timer->{path} = [ ]; + $timer->{node} = $timer->{name}; + } + push @timers, $timer; + }); + + # calculate timer-only durations + $self->walk_timers(sub { + my ($timer) = @_; + my $child_duration = 0; + if (exists $timer->{children}) { + foreach my $child (@{ $timer->{children} }) { + $child_duration += $child->{duration}; + } + } + $timer->{this_duration} = $timer->{duration} - $child_duration; + }); + + # massage each timer + my $start_time = $self->{times}->{start_time}; + foreach my $timer (@timers) { + # remove node name and children + delete $timer->{node}; + delete $timer->{children}; + + # show relative times + $timer->{start_time} = $timer->{start_time} - $start_time; + delete $timer->{end_time}; + + # show times in ms instead of fractional seconds + foreach my $field (qw( start_time duration this_duration )) { + $timer->{$field} = sprintf('%.4f', $timer->{$field} * 1000) * 1; + } + } + + # remove private data from env + delete $self->{env}->{user_agent}; + delete $self->{env}->{referer}; + + # throw at ES + require ElasticSearch; + my $es = ElasticSearch->new( + servers => Bugzilla->params->{metrics_elasticsearch_server}, + transport => 'http', + ); + # the ElasticSearch module queries the server for a list of nodes and + # connects directly to a random node. that bypasses our load balancer so we + # disable that by setting the server list directly. + $es->transport->servers(Bugzilla->params->{metrics_elasticsearch_server}); + # as the discovered node list is lazy-loaded, increase _refresh_in so it + # won't call ->refresh_servers() + $es->transport->{_refresh_in} = 1; + $es->index( + index => Bugzilla->params->{metrics_elasticsearch_index}, + type => Bugzilla->params->{metrics_elasticsearch_type}, + ttl => Bugzilla->params->{metrics_elasticsearch_ttl}, + data => { + env => $self->{env}, + times => \@timers, + }, + ); +} + +1; diff --git a/Bugzilla/Metrics/Reporter/STDERR.pm b/Bugzilla/Metrics/Reporter/STDERR.pm new file mode 100644 index 000000000..f5bd38acb --- /dev/null +++ b/Bugzilla/Metrics/Reporter/STDERR.pm @@ -0,0 +1,151 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Metrics::Reporter::STDERR; + +use strict; +use warnings; + +use parent 'Bugzilla::Metrics::Reporter'; + +use Data::Dumper; + +use constant DETACH => 0; + +sub report { + my ($self) = @_; + + # count totals + $self->{total} = $self->{times}->{duration}; + $self->{tmpl_count} = $self->{db_count} = $self->{mem_count} = 0; + $self->{total_tmpl} = $self->{total_db} = $self->{mem_hits} = 0; + $self->{mem_keys} = {}; + $self->_tally($self->{times}); + + # calculate percentages + $self->{other} = $self->{total} - $self->{total_tmpl} - $self->{total_db}; + if ($self->{total} * 1) { + $self->{perc_tmpl} = $self->{total_tmpl} / $self->{total} * 100; + $self->{perc_db} = $self->{total_db} / $self->{total} * 100; + $self->{perc_other} = $self->{other} / $self->{total} * 100; + } else { + $self->{perc_tmpl} = 0; + $self->{perc_db} = 0; + $self->{perc_other} = 0; + } + if ($self->{mem_count}) { + $self->{perc_mem} = $self->{mem_hits} / $self->{mem_count} * 100; + } else { + $self->{perm_mem} = 0; + } + + # convert to ms and format + foreach my $key (qw( total total_tmpl total_db other )) { + $self->{$key} = sprintf("%.4f", $self->{$key} * 1000); + } + foreach my $key (qw( perc_tmpl perc_db perc_other perc_mem )) { + $self->{$key} = sprintf("%.1f", $self->{$key}); + } + + # massage each timer + my $start_time = $self->{times}->{start_time}; + $self->walk_timers(sub { + my ($timer) = @_; + delete $timer->{parent}; + + # show relative times + $timer->{start_time} = $timer->{start_time} - $start_time; + delete $timer->{end_time}; + + # show times in ms instead of fractional seconds + foreach my $field (qw( start_time duration duration_this )) { + $timer->{$field} = sprintf('%.4f', $timer->{$field} * 1000) * 1 + if exists $timer->{$field}; + } + }); + + if (0) { + # dump timers to stderr + local $Data::Dumper::Indent = 1; + local $Data::Dumper::Terse = 1; + local $Data::Dumper::Sortkeys = sub { + my ($rh) = @_; + return [ sort { $b cmp $a } keys %$rh ]; + }; + print STDERR Dumper($self->{env}); + print STDERR Dumper($self->{times}); + } + + # summary summary table too + print STDERR <<EOF; +total time: $self->{total} + tmpl time: $self->{total_tmpl} ($self->{perc_tmpl}%) $self->{tmpl_count} hits + db time: $self->{total_db} ($self->{perc_db}%) $self->{db_count} hits +other time: $self->{other} ($self->{perc_other}%) + memcached: $self->{perc_mem}% ($self->{mem_count} requests) +EOF + my $tmpls = $self->{tmpl}; + my $len = 0; + foreach my $file (keys %$tmpls) { + $len = length($file) if length($file) > $len; + } + foreach my $file (sort { $tmpls->{$b}->{count} <=> $tmpls->{$a}->{count} } keys %$tmpls) { + my $tmpl = $tmpls->{$file}; + printf STDERR + "%${len}s: %2s hits %8.4f total %8.4f avg\n", + $file, + $tmpl->{count}, + $tmpl->{duration} * 1000, + $tmpl->{duration} * 1000 / $tmpl->{count} + ; + } + my $keys = $self->{mem_keys}; + $len = 0; + foreach my $key (keys %$keys) { + $len = length($key) if length($key) > $len; + } + foreach my $key (sort { $keys->{$a} <=> $keys->{$b} or $a cmp $b } keys %$keys) { + printf STDERR "%${len}s: %s\n", $key, $keys->{$key}; + } +} + +sub _tally { + my ($self, $timer) = @_; + if (exists $timer->{children}) { + foreach my $child (@{ $timer->{children} }) { + $self->_tally($child); + } + } + + if ($timer->{type} eq 'db') { + $timer->{duration_this} = $timer->{duration}; + $self->{total_db} += $timer->{duration}; + $self->{db_count}++; + + } elsif ($timer->{type} eq 'tmpl') { + my $child_duration = 0; + if (exists $timer->{children}) { + foreach my $child (@{ $timer->{children} }) { + $child_duration += $child->{duration}; + } + } + $timer->{duration_this} = $timer->{duration} - $child_duration; + + $self->{total_tmpl} += $timer->{duration} - $child_duration; + $self->{tmpl_count}++; + $self->{tmpl}->{$timer->{file}}->{count}++; + $self->{tmpl}->{$timer->{file}}->{duration} += $timer->{duration}; + + } elsif ($timer->{type} eq 'memcached') { + $timer->{duration_this} = $timer->{duration}; + $self->{mem_count}++; + $self->{mem_keys}->{$timer->{key}}++; + $self->{mem_hits}++ if $timer->{result} eq 'hit'; + } +} + +1; diff --git a/Bugzilla/Metrics/Template.pm b/Bugzilla/Metrics/Template.pm new file mode 100644 index 000000000..a3b13f85d --- /dev/null +++ b/Bugzilla/Metrics/Template.pm @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Metrics::Template; + +use strict; +use warnings; + +use parent 'Bugzilla::Template'; + +sub process { + my $self = shift; + Bugzilla->metrics->template_start($_[0]); + my $result = $self->SUPER::process(@_); + Bugzilla->metrics->end(); + return $result; +} + +1; diff --git a/Bugzilla/Metrics/Template/Context.pm b/Bugzilla/Metrics/Template/Context.pm new file mode 100644 index 000000000..ae8470b3f --- /dev/null +++ b/Bugzilla/Metrics/Template/Context.pm @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Metrics::Template::Context; + +use strict; +use warnings; + +use parent 'Bugzilla::Template::Context'; + +sub process { + my $self = shift; + + # we only want to measure files not template blocks + if (ref($_[0]) || substr($_[0], -5) ne '.tmpl') { + return $self->SUPER::process(@_); + } + + Bugzilla->metrics->template_start($_[0]); + my $result = $self->SUPER::process(@_); + Bugzilla->metrics->end(); + return $result; +} + +1; diff --git a/Bugzilla/Migrate.pm b/Bugzilla/Migrate.pm index ee0dcab95..923fc36a4 100644 --- a/Bugzilla/Migrate.pm +++ b/Bugzilla/Migrate.pm @@ -459,8 +459,11 @@ sub translate_value { } my $field_obj = $self->bug_fields->{$field}; - if ($field eq 'creation_ts' or $field eq 'delta_ts' - or ($field_obj and $field_obj->type == FIELD_TYPE_DATETIME)) + if ($field eq 'creation_ts' + or $field eq 'delta_ts' + or ($field_obj and + ($field_obj->type == FIELD_TYPE_DATETIME + or $field_obj->type == FIELD_TYPE_DATE))) { $value = trim($value); return undef if !$value; @@ -827,7 +830,7 @@ sub _insert_comments { $self->_do_table_insert('longdescs', \%copy); $self->debug(" Inserted comment from " . $who->login, 2); } - $bug->_sync_fulltext(); + $bug->_sync_fulltext( update_comments => 1 ); } sub _insert_history { diff --git a/Bugzilla/Milestone.pm b/Bugzilla/Milestone.pm index 92bc2192a..d8fcd79a5 100644 --- a/Bugzilla/Milestone.pm +++ b/Bugzilla/Milestone.pm @@ -108,10 +108,12 @@ sub run_create_validators { sub update { my $self = shift; + my $dbh = Bugzilla->dbh; + + $dbh->bz_start_transaction(); my $changes = $self->SUPER::update(@_); if (exists $changes->{value}) { - my $dbh = Bugzilla->dbh; # The milestone value is stored in the bugs table instead of its ID. $dbh->do('UPDATE bugs SET target_milestone = ? WHERE target_milestone = ? AND product_id = ?', @@ -121,7 +123,11 @@ sub update { $dbh->do('UPDATE products SET defaultmilestone = ? WHERE id = ? AND defaultmilestone = ?', undef, ($self->name, $self->product_id, $changes->{value}->[0])); + Bugzilla->memcached->clear({ table => 'products', id => $self->product_id }); } + $dbh->bz_commit_transaction(); + Bugzilla->memcached->clear_config(); + return $changes; } @@ -129,6 +135,8 @@ sub remove_from_db { my $self = shift; my $dbh = Bugzilla->dbh; + $dbh->bz_start_transaction(); + # The default milestone cannot be deleted. if ($self->name eq $self->product->default_milestone) { ThrowUserError('milestone_is_default', { milestone => $self }); @@ -157,8 +165,9 @@ sub remove_from_db { Bugzilla->user->id, $timestamp); } } + $self->SUPER::remove_from_db(); - $dbh->do('DELETE FROM milestones WHERE id = ?', undef, $self->id); + $dbh->bz_commit_transaction(); } ################################ diff --git a/Bugzilla/Object.pm b/Bugzilla/Object.pm index d4574abd2..8a7bba1c5 100644 --- a/Bugzilla/Object.pm +++ b/Bugzilla/Object.pm @@ -47,6 +47,15 @@ use constant AUDIT_CREATES => 1; use constant AUDIT_UPDATES => 1; use constant AUDIT_REMOVES => 1; +# When USE_MEMCACHED is true, the class is suitable for serialisation to +# Memcached. See documentation in Bugzilla::Memcached for more information. +use constant USE_MEMCACHED => 1; + +# When IS_CONFIG is true, the class is used to track seldom changed +# configuration objects. This includes, but is not limited to, fields, field +# values, keywords, products, classifications, priorities, severities, etc. +use constant IS_CONFIG => 0; + # This allows the JSON-RPC interface to return Bugzilla::Object instances # as though they were hashes. In the future, this may be modified to return # less information. @@ -59,8 +68,44 @@ sub TO_JSON { return { %{ $_[0] } }; } sub new { my $invocant = shift; my $class = ref($invocant) || $invocant; - my $object = $class->_init(@_); - bless($object, $class) if $object; + my $param = shift; + + my $object = $class->_object_cache_get($param); + return $object if $object; + + my ($data, $set_memcached); + if (Bugzilla->memcached->enabled + && $class->USE_MEMCACHED + && ref($param) eq 'HASH' && $param->{cache}) + { + if (defined $param->{id}) { + $data = Bugzilla->memcached->get({ + table => $class->DB_TABLE, + id => $param->{id}, + }); + } + elsif (defined $param->{name}) { + $data = Bugzilla->memcached->get({ + table => $class->DB_TABLE, + name => $param->{name}, + }); + } + $set_memcached = $data ? 0 : 1; + } + $data ||= $class->_load_from_db($param); + + if ($data && $set_memcached) { + Bugzilla->memcached->set({ + table => $class->DB_TABLE, + id => $data->{$class->ID_FIELD}, + name => $data->{$class->NAME_FIELD}, + data => $data, + }); + } + + $object = $class->new_from_hash($data); + $class->_object_cache_set($param, $object); + return $object; } @@ -69,7 +114,7 @@ sub new { # Bugzilla::Object, make sure that you modify bz_setup_database # in Bugzilla::DB::Pg appropriately, to add the right LOWER # index. You can see examples already there. -sub _init { +sub _load_from_db { my $class = shift; my ($param) = @_; my $dbh = Bugzilla->dbh; @@ -82,19 +127,19 @@ sub _init { if (ref $param eq 'HASH') { $id = $param->{id}; } - my $object; + my $object_data; if (defined $id) { # We special-case if somebody specifies an ID, so that we can # validate it as numeric. detaint_natural($id) || ThrowCodeError('param_must_be_numeric', - {function => $class . '::_init'}); + {function => $class . '::_load_from_db'}); # Too large integers make PostgreSQL crash. return if $id > MAX_INT_32; - $object = $dbh->selectrow_hashref(qq{ + $object_data = $dbh->selectrow_hashref(qq{ SELECT $columns FROM $table WHERE $id_field = ?}, undef, $id); } else { @@ -121,11 +166,123 @@ sub _init { } map { trick_taint($_) } @values; - $object = $dbh->selectrow_hashref( + $object_data = $dbh->selectrow_hashref( "SELECT $columns FROM $table WHERE $condition", undef, @values); } + return $object_data; +} - return $object; +sub new_from_list { + my $invocant = shift; + my $class = ref($invocant) || $invocant; + my ($id_list) = @_; + my $id_field = $class->ID_FIELD; + + my @detainted_ids; + foreach my $id (@$id_list) { + detaint_natural($id) || + ThrowCodeError('param_must_be_numeric', + {function => $class . '::new_from_list'}); + # Too large integers make PostgreSQL crash. + next if $id > MAX_INT_32; + push(@detainted_ids, $id); + } + + # We don't do $invocant->match because some classes have + # their own implementation of match which is not compatible + # with this one. However, match() still needs to have the right $invocant + # in order to do $class->DB_TABLE and so on. + my $list = match($invocant, { $id_field => \@detainted_ids }); + + # BMO: Populate the object cache with bug objects, which helps + # inline-history when viewing bugs with dependencies. + if ($class eq 'Bugzilla::Bug') { + foreach my $object (@$list) { + $class->_object_cache_set( + { id => $object->id, cache => 1 }, + $object + ); + } + } + + return $list; +} + +sub new_from_hash { + my $invocant = shift; + my $class = ref($invocant) || $invocant; + my $object_data = shift || return; + $class->_serialisation_keys($object_data); + bless($object_data, $class); + $object_data->initialize(); + return $object_data; +} + +sub initialize { + # abstract +} + +# Provides a mechanism for objects to be cached in the request_cahce + +sub object_cache_get { + my ($class, $id) = @_; + return $class->_object_cache_get( + { id => $id, cache => 1}, + $class + ); +} + +sub object_cache_set { + my $self = shift; + return $self->_object_cache_set( + { id => $self->id, cache => 1 }, + $self + ); +} + +sub _object_cache_get { + my $class = shift; + my ($param) = @_; + my $cache_key = $class->object_cache_key($param) + || return; + return Bugzilla->request_cache->{$cache_key}; +} + +sub _object_cache_set { + my $class = shift; + my ($param, $object) = @_; + my $cache_key = $class->object_cache_key($param) + || return; + Bugzilla->request_cache->{$cache_key} = $object; +} + +sub _object_cache_remove { + my $class = shift; + my ($param, $object) = @_; + $param->{cache} = 1; + my $cache_key = $class->object_cache_key($param) + || return; + delete Bugzilla->request_cache->{$cache_key}; +} + +sub object_cache_key { + my $class = shift; + my ($param) = @_; + if (ref($param) && $param->{cache} && ($param->{id} || $param->{name})) { + $class = blessed($class) if blessed($class); + return $class . ',' . ($param->{id} || $param->{name}); + } else { + return; + } +} + +# To support serialisation, we need to capture the keys in an object's default +# hashref. +sub _serialisation_keys { + my ($class, $object) = @_; + my $cache = Bugzilla->request_cache->{serialisation_keys} ||= {}; + $cache->{$class} = [ keys %$object ] if $object && !exists $cache->{$class}; + return @{ $cache->{$class} }; } sub check { @@ -161,28 +318,6 @@ sub check { return $obj; } -sub new_from_list { - my $invocant = shift; - my $class = ref($invocant) || $invocant; - my ($id_list) = @_; - my $id_field = $class->ID_FIELD; - - my @detainted_ids; - foreach my $id (@$id_list) { - detaint_natural($id) || - ThrowCodeError('param_must_be_numeric', - {function => $class . '::new_from_list'}); - # Too large integers make PostgreSQL crash. - next if $id > MAX_INT_32; - push(@detainted_ids, $id); - } - # We don't do $invocant->match because some classes have - # their own implementation of match which is not compatible - # with this one. However, match() still needs to have the right $invocant - # in order to do $class->DB_TABLE and so on. - return match($invocant, { $id_field => \@detainted_ids }); -} - # Note: Future extensions to this could be: # * Add a MATCH_JOIN constant so that we can join against # certain other tables for the WHERE criteria. @@ -228,8 +363,11 @@ sub match { } next; } - - $class->_check_field($field, 'match'); + + # It's always safe to use the field defined by classes as being + # their ID field. In particular, this means that new_from_list() + # is exempted from this check. + $class->_check_field($field, 'match') unless $field eq $class->ID_FIELD; if (ref $value eq 'ARRAY') { # IN () is invalid SQL, and if we have an empty list @@ -263,23 +401,46 @@ sub _do_list_select { my $cols = join(',', $class->_get_db_columns); my $order = $class->LIST_ORDER; - my $sql = "SELECT $cols FROM $table"; - if (defined $where) { - $sql .= " WHERE $where "; + # Unconditional requests for configuration data are cacheable. + my ($objects, $set_memcached, $memcached_key); + if (!defined $where + && Bugzilla->memcached->enabled + && $class->IS_CONFIG) + { + $memcached_key = "$class:get_all"; + $objects = Bugzilla->memcached->get_config({ key => $memcached_key }); + $set_memcached = $objects ? 0 : 1; } - $sql .= " ORDER BY $order"; - - $sql .= " $postamble" if $postamble; - - my $dbh = Bugzilla->dbh; - # Sometimes the values are tainted, but we don't want to untaint them - # for the caller. So we copy the array. It's safe to untaint because - # they're only used in placeholders here. - my @untainted = @{ $values || [] }; - trick_taint($_) foreach @untainted; - my $objects = $dbh->selectall_arrayref($sql, {Slice=>{}}, @untainted); - bless ($_, $class) foreach @$objects; - return $objects + + if (!$objects) { + my $sql = "SELECT $cols FROM $table"; + if (defined $where) { + $sql .= " WHERE $where "; + } + $sql .= " ORDER BY $order"; + $sql .= " $postamble" if $postamble; + + my $dbh = Bugzilla->dbh; + # Sometimes the values are tainted, but we don't want to untaint them + # for the caller. So we copy the array. It's safe to untaint because + # they're only used in placeholders here. + my @untainted = @{ $values || [] }; + trick_taint($_) foreach @untainted; + $objects = $dbh->selectall_arrayref($sql, {Slice=>{}}, @untainted); + $class->_serialisation_keys($objects->[0]) if @$objects; + } + + if ($objects && $set_memcached) { + Bugzilla->memcached->set_config({ + key => $memcached_key, + data => $objects + }); + } + + foreach my $object (@$objects) { + $object = $class->new_from_hash($object); + } + return $objects; } ############################### @@ -332,12 +493,17 @@ sub set_all { my %field_values = %$params; my @sorted_names = $self->_sort_by_dep(keys %field_values); + foreach my $key (@sorted_names) { # It's possible for one set_ method to delete a key from $params # for another set method, so if that's happened, we don't call the # other set method. next if !exists $field_values{$key}; my $method = "set_$key"; + if (!$self->can($method)) { + my $class = ref($self) || $self; + ThrowCodeError("unknown_method", { method => "${class}::${method}" }); + } $self->$method($field_values{$key}, \%field_values); } Bugzilla::Hook::process('object_end_of_set_all', @@ -398,6 +564,13 @@ sub update { $self->audit_log(\%changes) if $self->AUDIT_UPDATES; $dbh->bz_commit_transaction(); + if ($self->USE_MEMCACHED && @values) { + Bugzilla->memcached->clear({ table => $table, id => $self->id }); + Bugzilla->memcached->clear_config() + if $self->IS_CONFIG; + } + $self->_object_cache_remove({ id => $self->id }); + $self->_object_cache_remove({ name => $self->name }) if $self->name; if (wantarray) { return (\%changes, $old_self); @@ -416,6 +589,13 @@ sub remove_from_db { $self->audit_log(AUDIT_REMOVE) if $self->AUDIT_REMOVES; $dbh->do("DELETE FROM $table WHERE $id_field = ?", undef, $self->id); $dbh->bz_commit_transaction(); + if ($self->USE_MEMCACHED) { + Bugzilla->memcached->clear({ table => $table, id => $self->id }); + Bugzilla->memcached->clear_config() + if $self->IS_CONFIG; + } + $self->_object_cache_remove({ id => $self->id }); + $self->_object_cache_remove({ name => $self->name }) if $self->name; undef $self; } @@ -450,6 +630,13 @@ sub audit_log { } } +sub flatten_to_hash { + my $self = shift; + my $class = blessed($self); + my %hash = map { $_ => $self->{$_} } $class->_serialisation_keys; + return \%hash; +} + ############################### #### Subroutines ###### ############################### @@ -473,6 +660,13 @@ sub create { my $object = $class->insert_create_data($field_values); $dbh->bz_commit_transaction(); + if (Bugzilla->memcached->enabled + && $class->USE_MEMCACHED + && $class->IS_CONFIG) + { + Bugzilla->memcached->clear_config(); + } + return $object; } @@ -568,7 +762,7 @@ sub insert_create_data { sub get_all { my $class = shift; - return @{$class->_do_list_select()}; + return @{ $class->_do_list_select() }; } ############################### @@ -977,6 +1171,17 @@ database matching the parameters you passed in. =back +=item C<initialize> + +=over + +=item B<Description> + +Abstract method to allow subclasses to perform initialization tasks after an +object has been created. + +=back + =item C<check> =over @@ -1267,6 +1472,58 @@ Returns C<1> if the passed-in value is true, C<0> otherwise. =back +=head2 CACHE FUNCTIONS + +=over + +=item C<object_cache_get> + +=over + +=item B<Description> + +Class function which returns an object from the object-cache for the provided +C<$id>. + +=item B<Params> + +Takes an integer C<$id> of the object to retrieve. + +=item B<Returns> + +Returns the object from the cache if found, otherwise returns C<undef>. + +=item B<Example> + +my $bug_from_cache = Bugzilla::Bug->object_cache_get(35); + +=back + +=item C<object_cache_set> + +=over + +=item B<Description> + +Object function which injects the object into the object-cache, using the +object's C<id> as the key. + +=item B<Params> + +(none) + +=item B<Returns> + +(nothing) + +=item B<Example> + +$bug->object_cache_set(); + +=back + +=back + =head1 CLASS FUNCTIONS =over diff --git a/Bugzilla/PatchReader.pm b/Bugzilla/PatchReader.pm new file mode 100644 index 000000000..3b1ba3dda --- /dev/null +++ b/Bugzilla/PatchReader.pm @@ -0,0 +1,117 @@ +package Bugzilla::PatchReader; + +use strict; + +=head1 NAME + +PatchReader - Utilities to read and manipulate patches and CVS + +=head1 SYNOPSIS + + # Script that reads in a patch (in any known format), and prints + # out some information about it. Other common operations are + # outputting the patch in a raw unified diff format, outputting + # the patch information to Template::Toolkit templates, adding + # context to a patch from CVS, and narrowing the patch down to + # apply only to a single file or set of files. + + use PatchReader::Raw; + use PatchReader::PatchInfoGrabber; + my $filename = 'filename.patch'; + + # Create the reader that parses the patch and the object that + # extracts info from the reader's datastream + my $reader = new PatchReader::Raw(); + my $patch_info_grabber = new PatchReader::PatchInfoGrabber(); + $reader->sends_data_to($patch_info_grabber); + + # Iterate over the file + $reader->iterate_file($filename); + + # Print the output + my $patch_info = $patch_info_grabber->patch_info(); + print "Summary of Changed Files:\n"; + while (my ($file, $info) = each %{$patch_info->{files}}) { + print "$file: +$info->{plus_lines} -$info->{minus_lines}\n"; + } + +=head1 ABSTRACT + +This perl library allows you to manipulate patches programmatically by +chaining together a variety of objects that read, manipulate, and output +patch information: + +=over + +=item PatchReader::Raw + +Parse a patch in any format known to this author (unified, normal, cvs diff, +among others) + +=item PatchReader::PatchInfoGrabber + +Grab summary info for sections of a patch in a nice hash + +=item PatchReader::AddCVSContext + +Add context to the patch by grabbing the original files from CVS + +=item PatchReader::NarrowPatch + +Narrow a patch down to only apply to a specific set of files + +=item PatchReader::DiffPrinter::raw + +Output the parsed patch in raw unified diff format + +=item PatchReader::DiffPrinter::template + +Output the parsed patch to L<Template::Toolkit> templates (can be used to make +HTML output or anything else you please) + +=back + +Additionally, it is designed so that you can plug in your own objects that +read the parsed data while it is being parsed (no need for the performance or +memory problems that can come from reading in the entire patch all at once). +You can do this by mimicking one of the existing readers (such as +PatchInfoGrabber) and overriding the methods start_patch, start_file, section, +end_file and end_patch. + +=head1 AUTHORS + + John Keiser <jkeiser@cpan.org> + Teemu Mannermaa <tmannerm@cpan.org> + +=head1 COPYRIGHT AND LICENSE + + Copyright (C) 2003-2004, John Keiser and + Copyright (C) 2011-2012, Teemu Mannermaa. + +This module is free software; you can redistribute it and/or modify it under +the terms of the Artistic License 1.0. For details, see the full text of the +license at + <http://www.perlfoundation.org/artistic_license_1_0>. + +This module is distributed in the hope that it will be useful, but it is +provided "as is" and without any warranty; without even the implied warranty +of merchantability or fitness for a particular purpose. + +Files with different licenses or copyright holders: + +=over + +=item F<lib/PatchReader/CVSClient.pm> + +Portions created by Netscape are +Copyright (C) 2003, Netscape Communications Corporation. All rights reserved. + +This file is subject to the terms of the Mozilla Public License, v. 2.0. + +=back + +=cut + +$Bugzilla::PatchReader::VERSION = '0.9.7'; + +1 diff --git a/Bugzilla/PatchReader/AddCVSContext.pm b/Bugzilla/PatchReader/AddCVSContext.pm new file mode 100644 index 000000000..910e45669 --- /dev/null +++ b/Bugzilla/PatchReader/AddCVSContext.pm @@ -0,0 +1,226 @@ +package Bugzilla::PatchReader::AddCVSContext; + +use Bugzilla::PatchReader::FilterPatch; +use Bugzilla::PatchReader::CVSClient; +use Cwd; +use File::Temp; + +use strict; + +@Bugzilla::PatchReader::AddCVSContext::ISA = qw(Bugzilla::PatchReader::FilterPatch); + +# XXX If you need to, get the entire patch worth of files and do a single +# cvs update of all files as soon as you find a file where you need to do a +# cvs update, to avoid the significant connect overhead +sub new { + my $class = shift; + $class = ref($class) || $class; + my $this = $class->SUPER::new(); + bless $this, $class; + + $this->{CONTEXT} = $_[0]; + $this->{CVSROOT} = $_[1]; + + return $this; +} + +sub my_rmtree { + my ($this, $dir) = @_; + foreach my $file (glob("$dir/*")) { + if (-d $file) { + $this->my_rmtree($file); + } else { + trick_taint($file); + unlink $file; + } + } + trick_taint($dir); + rmdir $dir; +} + +sub end_patch { + my $this = shift; + if (exists($this->{TMPDIR})) { + # Set as variable to get rid of taint + # One would like to use rmtree here, but that is not taint-safe. + $this->my_rmtree($this->{TMPDIR}); + } +} + +sub start_file { + my $this = shift; + my ($file) = @_; + $this->{HAS_CVS_CONTEXT} = !$file->{is_add} && !$file->{is_remove} && + $file->{old_revision}; + $this->{REVISION} = $file->{old_revision}; + $this->{FILENAME} = $file->{filename}; + $this->{SECTION_END} = -1; + $this->{TARGET}->start_file(@_) if $this->{TARGET}; +} + +sub end_file { + my $this = shift; + $this->flush_section(); + + if ($this->{FILE}) { + close $this->{FILE}; + unlink $this->{FILE}; # If it fails, it fails ... + delete $this->{FILE}; + } + $this->{TARGET}->end_file(@_) if $this->{TARGET}; +} + +sub next_section { + my $this = shift; + my ($section) = @_; + $this->{NEXT_PATCH_LINE} = $section->{old_start}; + $this->{NEXT_NEW_LINE} = $section->{new_start}; + foreach my $line (@{$section->{lines}}) { + # If this is a line requiring context ... + if ($line =~ /^[-\+]/) { + # Determine how much context is needed for both the previous section line + # and this one: + # - If there is no old line, start new section + # - If this is file context, add (old section end to new line) context to + # the existing section + # - If old end context line + 1 < new start context line, there is an empty + # space and therefore we end the old section and start the new one + # - Else we add (old start context line through new line) context to + # existing section + if (! exists($this->{SECTION})) { + $this->_start_section(); + } elsif ($this->{CONTEXT} eq "file") { + $this->push_context_lines($this->{SECTION_END} + 1, + $this->{NEXT_PATCH_LINE} - 1); + } else { + my $start_context = $this->{NEXT_PATCH_LINE} - $this->{CONTEXT}; + $start_context = $start_context > 0 ? $start_context : 0; + if (($this->{SECTION_END} + $this->{CONTEXT} + 1) < $start_context) { + $this->flush_section(); + $this->_start_section(); + } else { + $this->push_context_lines($this->{SECTION_END} + 1, + $this->{NEXT_PATCH_LINE} - 1); + } + } + push @{$this->{SECTION}{lines}}, $line; + if (substr($line, 0, 1) eq "+") { + $this->{SECTION}{plus_lines}++; + $this->{SECTION}{new_lines}++; + $this->{NEXT_NEW_LINE}++; + } else { + $this->{SECTION_END}++; + $this->{SECTION}{minus_lines}++; + $this->{SECTION}{old_lines}++; + $this->{NEXT_PATCH_LINE}++; + } + } else { + $this->{NEXT_PATCH_LINE}++; + $this->{NEXT_NEW_LINE}++; + } + # If this is context, for now lose it (later we should try and determine if + # we can just use it instead of pulling the file all the time) + } +} + +sub determine_start { + my ($this, $line) = @_; + return 0 if $line < 0; + if ($this->{CONTEXT} eq "file") { + return 1; + } else { + my $start = $line - $this->{CONTEXT}; + $start = $start > 0 ? $start : 1; + return $start; + } +} + +sub _start_section { + my $this = shift; + + # Add the context to the beginning + $this->{SECTION}{old_start} = $this->determine_start($this->{NEXT_PATCH_LINE}); + $this->{SECTION}{new_start} = $this->determine_start($this->{NEXT_NEW_LINE}); + $this->{SECTION}{old_lines} = 0; + $this->{SECTION}{new_lines} = 0; + $this->{SECTION}{minus_lines} = 0; + $this->{SECTION}{plus_lines} = 0; + $this->{SECTION_END} = $this->{SECTION}{old_start} - 1; + $this->push_context_lines($this->{SECTION}{old_start}, + $this->{NEXT_PATCH_LINE} - 1); +} + +sub flush_section { + my $this = shift; + + if ($this->{SECTION}) { + # Add the necessary context to the end + if ($this->{CONTEXT} eq "file") { + $this->push_context_lines($this->{SECTION_END} + 1, "file"); + } else { + $this->push_context_lines($this->{SECTION_END} + 1, + $this->{SECTION_END} + $this->{CONTEXT}); + } + # Send the section and line notifications + $this->{TARGET}->next_section($this->{SECTION}) if $this->{TARGET}; + delete $this->{SECTION}; + $this->{SECTION_END} = 0; + } +} + +sub push_context_lines { + my $this = shift; + # Grab from start to end + my ($start, $end) = @_; + return if $end ne "file" && $start > $end; + + # If it's an added / removed file, don't do anything + return if ! $this->{HAS_CVS_CONTEXT}; + + # Get and open the file if necessary + if (!$this->{FILE}) { + my $olddir = getcwd(); + if (! exists($this->{TMPDIR})) { + $this->{TMPDIR} = File::Temp::tempdir(); + if (! -d $this->{TMPDIR}) { + die "Could not get temporary directory"; + } + } + chdir($this->{TMPDIR}) or die "Could not cd $this->{TMPDIR}"; + if (Bugzilla::PatchReader::CVSClient::cvs_co_rev($this->{CVSROOT}, $this->{REVISION}, $this->{FILENAME})) { + die "Could not check out $this->{FILENAME} r$this->{REVISION} from $this->{CVSROOT}"; + } + open my $fh, $this->{FILENAME} or die "Could not open $this->{FILENAME}"; + $this->{FILE} = $fh; + $this->{NEXT_FILE_LINE} = 1; + trick_taint($olddir); # $olddir comes from getcwd() + chdir($olddir) or die "Could not cd back to $olddir"; + } + + # Read through the file to reach the line we need + die "File read too far!" if $this->{NEXT_FILE_LINE} && $this->{NEXT_FILE_LINE} > $start; + my $fh = $this->{FILE}; + while ($this->{NEXT_FILE_LINE} < $start) { + my $dummy = <$fh>; + $this->{NEXT_FILE_LINE}++; + } + my $i = $start; + for (; $end eq "file" || $i <= $end; $i++) { + my $line = <$fh>; + last if !defined($line); + $line =~ s/\r\n/\n/g; + push @{$this->{SECTION}{lines}}, " $line"; + $this->{NEXT_FILE_LINE}++; + $this->{SECTION}{old_lines}++; + $this->{SECTION}{new_lines}++; + } + $this->{SECTION_END} = $i - 1; +} + +sub trick_taint { + $_[0] =~ /^(.*)$/s; + $_[0] = $1; + return (defined($_[0])); +} + +1; diff --git a/Bugzilla/PatchReader/Base.pm b/Bugzilla/PatchReader/Base.pm new file mode 100644 index 000000000..f2fd69a68 --- /dev/null +++ b/Bugzilla/PatchReader/Base.pm @@ -0,0 +1,23 @@ +package Bugzilla::PatchReader::Base; + +use strict; + +sub new { + my $class = shift; + $class = ref($class) || $class; + my $this = {}; + bless $this, $class; + + return $this; +} + +sub sends_data_to { + my $this = shift; + if (defined($_[0])) { + $this->{TARGET} = $_[0]; + } else { + return $this->{TARGET}; + } +} + +1 diff --git a/Bugzilla/PatchReader/CVSClient.pm b/Bugzilla/PatchReader/CVSClient.pm new file mode 100644 index 000000000..2f76fc18d --- /dev/null +++ b/Bugzilla/PatchReader/CVSClient.pm @@ -0,0 +1,48 @@ +# -*- Mode: perl; indent-tabs-mode: nil -*- +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::PatchReader::CVSClient; + +use strict; + +sub parse_cvsroot { + my $cvsroot = $_[0]; + # Format: :method:[user[:password]@]server[:[port]]/path + if ($cvsroot =~ /^:([^:]*):(.*?)(\/.*)$/) { + my %retval; + $retval{protocol} = $1; + $retval{rootdir} = $3; + my $remote = $2; + if ($remote =~ /^(([^\@:]*)(:([^\@]*))?\@)?([^:]*)(:(.*))?$/) { + $retval{user} = $2; + $retval{password} = $4; + $retval{server} = $5; + $retval{port} = $7; + return %retval; + } + } + + return ( + rootdir => $cvsroot + ); +} + +sub cvs_co { + my ($cvsroot, @files) = @_; + my $cvs = $::cvsbin || "cvs"; + return system($cvs, "-Q", "-d$cvsroot", "co", @files); +} + +sub cvs_co_rev { + my ($cvsroot, $rev, @files) = @_; + my $cvs = $::cvsbin || "cvs"; + return system($cvs, "-Q", "-d$cvsroot", "co", "-r$rev", @files); +} + +1 diff --git a/Bugzilla/PatchReader/DiffPrinter/raw.pm b/Bugzilla/PatchReader/DiffPrinter/raw.pm new file mode 100644 index 000000000..ceb425800 --- /dev/null +++ b/Bugzilla/PatchReader/DiffPrinter/raw.pm @@ -0,0 +1,61 @@ +package Bugzilla::PatchReader::DiffPrinter::raw; + +use strict; + +sub new { + my $class = shift; + $class = ref($class) || $class; + my $this = {}; + bless $this, $class; + + $this->{OUTFILE} = @_ ? $_[0] : *STDOUT; + my $fh = $this->{OUTFILE}; + + return $this; +} + +sub start_patch { +} + +sub end_patch { +} + +sub start_file { + my $this = shift; + my ($file) = @_; + + my $fh = $this->{OUTFILE}; + if ($file->{rcs_filename}) { + print $fh "Index: $file->{filename}\n"; + print $fh "===================================================================\n"; + print $fh "RCS file: $file->{rcs_filename}\n"; + } + my $old_file = $file->{is_add} ? "/dev/null" : $file->{filename}; + my $old_date = $file->{old_date_str} || ""; + print $fh "--- $old_file\t$old_date"; + print $fh "\t$file->{old_revision}" if $file->{old_revision}; + print $fh "\n"; + my $new_file = $file->{is_remove} ? "/dev/null" : $file->{filename}; + my $new_date = $file->{new_date_str} || ""; + print $fh "+++ $new_file\t$new_date"; + print $fh "\t$file->{new_revision}" if $file->{new_revision}; + print $fh "\n"; +} + +sub end_file { +} + +sub next_section { + my $this = shift; + my ($section) = @_; + + return unless $section->{old_start} || $section->{new_start}; + my $fh = $this->{OUTFILE}; + print $fh "@@ -$section->{old_start},$section->{old_lines} +$section->{new_start},$section->{new_lines} @@ $section->{func_info}\n"; + foreach my $line (@{$section->{lines}}) { + $line =~ s/(\r?\n?)$/\n/; + print $fh $line; + } +} + +1 diff --git a/Bugzilla/PatchReader/DiffPrinter/template.pm b/Bugzilla/PatchReader/DiffPrinter/template.pm new file mode 100644 index 000000000..6545e9336 --- /dev/null +++ b/Bugzilla/PatchReader/DiffPrinter/template.pm @@ -0,0 +1,119 @@ +package Bugzilla::PatchReader::DiffPrinter::template; + +use strict; + +sub new { + my $class = shift; + $class = ref($class) || $class; + my $this = {}; + bless $this, $class; + + $this->{TEMPLATE_PROCESSOR} = $_[0]; + $this->{HEADER_TEMPLATE} = $_[1]; + $this->{FILE_TEMPLATE} = $_[2]; + $this->{FOOTER_TEMPLATE} = $_[3]; + $this->{ARGS} = $_[4] || {}; + + $this->{ARGS}{file_count} = 0; + return $this; +} + +sub start_patch { + my $this = shift; + $this->{TEMPLATE_PROCESSOR}->process($this->{HEADER_TEMPLATE}, $this->{ARGS}) + || ::ThrowTemplateError($this->{TEMPLATE_PROCESSOR}->error()); +} + +sub end_patch { + my $this = shift; + $this->{TEMPLATE_PROCESSOR}->process($this->{FOOTER_TEMPLATE}, $this->{ARGS}) + || ::ThrowTemplateError($this->{TEMPLATE_PROCESSOR}->error()); +} + +sub start_file { + my $this = shift; + $this->{ARGS}{file_count}++; + $this->{ARGS}{file} = shift; + $this->{ARGS}{file}{plus_lines} = 0; + $this->{ARGS}{file}{minus_lines} = 0; + @{$this->{ARGS}{sections}} = (); +} + +sub end_file { + my $this = shift; + my $file = $this->{ARGS}{file}; + if ($file->{canonical} && $file->{old_revision} && $this->{ARGS}{bonsai_url}) { + $this->{ARGS}{bonsai_prefix} = "$this->{ARGS}{bonsai_url}/cvsblame.cgi?file=$file->{filename}&rev=$file->{old_revision}"; + } + if ($file->{canonical} && $this->{ARGS}{lxr_url}) { + # Cut off the lxr root, if any + my $filename = $file->{filename}; + $filename = substr($filename, length($this->{ARGS}{lxr_root})); + $this->{ARGS}{lxr_prefix} = "$this->{ARGS}{lxr_url}/source/$filename"; + } + + $this->{TEMPLATE_PROCESSOR}->process($this->{FILE_TEMPLATE}, $this->{ARGS}) + || ::ThrowTemplateError($this->{TEMPLATE_PROCESSOR}->error()); + @{$this->{ARGS}{sections}} = (); + delete $this->{ARGS}{file}; +} + +sub next_section { + my $this = shift; + my ($section) = @_; + + $this->{ARGS}{file}{plus_lines} += $section->{plus_lines}; + $this->{ARGS}{file}{minus_lines} += $section->{minus_lines}; + + # Get groups of lines and print them + my $last_line_char = ''; + my $context_lines = []; + my $plus_lines = []; + my $minus_lines = []; + foreach my $line (@{$section->{lines}}) { + $line =~ s/\r?\n?$//; + if ($line =~ /^ /) { + if ($last_line_char ne ' ') { + push @{$section->{groups}}, {context => $context_lines, + plus => $plus_lines, + minus => $minus_lines}; + $context_lines = []; + $plus_lines = []; + $minus_lines = []; + } + $last_line_char = ' '; + push @{$context_lines}, substr($line, 1); + } elsif ($line =~ /^\+/) { + if ($last_line_char eq ' ' || $last_line_char eq '-' && @{$plus_lines}) { + push @{$section->{groups}}, {context => $context_lines, + plus => $plus_lines, + minus => $minus_lines}; + $context_lines = []; + $plus_lines = []; + $minus_lines = []; + $last_line_char = ''; + } + $last_line_char = '+'; + push @{$plus_lines}, substr($line, 1); + } elsif ($line =~ /^-/) { + if ($last_line_char eq '+' && @{$minus_lines}) { + push @{$section->{groups}}, {context => $context_lines, + plus => $plus_lines, + minus => $minus_lines}; + $context_lines = []; + $plus_lines = []; + $minus_lines = []; + $last_line_char = ''; + } + $last_line_char = '-'; + push @{$minus_lines}, substr($line, 1); + } + } + + push @{$section->{groups}}, {context => $context_lines, + plus => $plus_lines, + minus => $minus_lines}; + push @{$this->{ARGS}{sections}}, $section; +} + +1 diff --git a/Bugzilla/PatchReader/FilterPatch.pm b/Bugzilla/PatchReader/FilterPatch.pm new file mode 100644 index 000000000..dfe42e750 --- /dev/null +++ b/Bugzilla/PatchReader/FilterPatch.pm @@ -0,0 +1,43 @@ +package Bugzilla::PatchReader::FilterPatch; + +use strict; + +use Bugzilla::PatchReader::Base; + +@Bugzilla::PatchReader::FilterPatch::ISA = qw(Bugzilla::PatchReader::Base); + +sub new { + my $class = shift; + $class = ref($class) || $class; + my $this = $class->SUPER::new(); + bless $this, $class; + + return $this; +} + +sub start_patch { + my $this = shift; + $this->{TARGET}->start_patch(@_) if $this->{TARGET}; +} + +sub end_patch { + my $this = shift; + $this->{TARGET}->end_patch(@_) if $this->{TARGET}; +} + +sub start_file { + my $this = shift; + $this->{TARGET}->start_file(@_) if $this->{TARGET}; +} + +sub end_file { + my $this = shift; + $this->{TARGET}->end_file(@_) if $this->{TARGET}; +} + +sub next_section { + my $this = shift; + $this->{TARGET}->next_section(@_) if $this->{TARGET}; +} + +1 diff --git a/Bugzilla/PatchReader/FixPatchRoot.pm b/Bugzilla/PatchReader/FixPatchRoot.pm new file mode 100644 index 000000000..e67fb2796 --- /dev/null +++ b/Bugzilla/PatchReader/FixPatchRoot.pm @@ -0,0 +1,130 @@ +package Bugzilla::PatchReader::FixPatchRoot; + +use Bugzilla::PatchReader::FilterPatch; +use Bugzilla::PatchReader::CVSClient; + +use strict; + +@Bugzilla::PatchReader::FixPatchRoot::ISA = qw(Bugzilla::PatchReader::FilterPatch); + +sub new { + my $class = shift; + $class = ref($class) || $class; + my $this = $class->SUPER::new(); + bless $this, $class; + + my %parsed = Bugzilla::PatchReader::CVSClient::parse_cvsroot($_[0]); + $this->{REPOSITORY_ROOT} = $parsed{rootdir}; + $this->{REPOSITORY_ROOT} .= "/" if substr($this->{REPOSITORY_ROOT}, -1) ne "/"; + + return $this; +} + +sub diff_root { + my $this = shift; + if (@_) { + $this->{DIFF_ROOT} = $_[0]; + } else { + return $this->{DIFF_ROOT}; + } +} + +sub flush_delayed_commands { + my $this = shift; + return if ! $this->{DELAYED_COMMANDS}; + + my $commands = $this->{DELAYED_COMMANDS}; + delete $this->{DELAYED_COMMANDS}; + $this->{FORCE_COMMANDS} = 1; + foreach my $command_arr (@{$commands}) { + my $command = $command_arr->[0]; + my $arg = $command_arr->[1]; + if ($command eq "start_file") { + $this->start_file($arg); + } elsif ($command eq "end_file") { + $this->end_file($arg); + } elsif ($command eq "section") { + $this->next_section($arg); + } + } +} + +sub end_patch { + my $this = shift; + $this->flush_delayed_commands(); + $this->{TARGET}->end_patch(@_) if $this->{TARGET}; +} + +sub start_file { + my $this = shift; + my ($file) = @_; + # If the file is new, it will not have a filename that fits the repository + # root and therefore needs to be fixed up to have the same root as everyone + # else. At the same time we need to fix DIFF_ROOT too. + if (exists($this->{DIFF_ROOT})) { + # XXX Return error if there are multiple roots in the patch by verifying + # that the DIFF_ROOT is not different from the calculated diff root on this + # filename + + $file->{filename} = $this->{DIFF_ROOT} . $file->{filename}; + + $file->{canonical} = 1; + } elsif ($file->{rcs_filename} && + substr($file->{rcs_filename}, 0, length($this->{REPOSITORY_ROOT})) eq + $this->{REPOSITORY_ROOT}) { + # Since we know the repository we can determine where the user was in the + # repository when they did the diff by chopping off the repository root + # from the rcs filename + $this->{DIFF_ROOT} = substr($file->{rcs_filename}, + length($this->{REPOSITORY_ROOT})); + $this->{DIFF_ROOT} =~ s/,v$//; + # If the RCS file exists in the Attic then we need to correct for + # this, stripping off the '/Attic' suffix in order to reduce the name + # to just the CVS root. + if ($this->{DIFF_ROOT} =~ m/Attic/) { + $this->{DIFF_ROOT} = substr($this->{DIFF_ROOT}, 0, -6); + } + # XXX More error checking--that filename exists and that it is in fact + # part of the rcs filename + $this->{DIFF_ROOT} = substr($this->{DIFF_ROOT}, 0, + -length($file->{filename})); + $this->flush_delayed_commands(); + + $file->{filename} = $this->{DIFF_ROOT} . $file->{filename}; + + $file->{canonical} = 1; + } else { + # DANGER Will Robinson. The first file in the patch is new. We will try + # "delayed command mode" + # + # (if force commands is on we are already in delayed command mode, and sadly + # this means the entire patch was unintelligible to us, so we just output + # whatever the hell was in the patch) + + if (!$this->{FORCE_COMMANDS}) { + push @{$this->{DELAYED_COMMANDS}}, [ "start_file", { %{$file} } ]; + return; + } + } + $this->{TARGET}->start_file($file) if $this->{TARGET}; +} + +sub end_file { + my $this = shift; + if (exists($this->{DELAYED_COMMANDS})) { + push @{$this->{DELAYED_COMMANDS}}, [ "end_file", { %{$_[0]} } ]; + } else { + $this->{TARGET}->end_file(@_) if $this->{TARGET}; + } +} + +sub next_section { + my $this = shift; + if (exists($this->{DELAYED_COMMANDS})) { + push @{$this->{DELAYED_COMMANDS}}, [ "section", { %{$_[0]} } ]; + } else { + $this->{TARGET}->next_section(@_) if $this->{TARGET}; + } +} + +1 diff --git a/Bugzilla/PatchReader/NarrowPatch.pm b/Bugzilla/PatchReader/NarrowPatch.pm new file mode 100644 index 000000000..b6502f2f3 --- /dev/null +++ b/Bugzilla/PatchReader/NarrowPatch.pm @@ -0,0 +1,44 @@ +package Bugzilla::PatchReader::NarrowPatch; + +use Bugzilla::PatchReader::FilterPatch; + +use strict; + +@Bugzilla::PatchReader::NarrowPatch::ISA = qw(Bugzilla::PatchReader::FilterPatch); + +sub new { + my $class = shift; + $class = ref($class) || $class; + my $this = $class->SUPER::new(); + bless $this, $class; + + $this->{INCLUDE_FILES} = [@_]; + + return $this; +} + +sub start_file { + my $this = shift; + my ($file) = @_; + if (grep { $_ eq substr($file->{filename}, 0, length($_)) } @{$this->{INCLUDE_FILES}}) { + $this->{IS_INCLUDED} = 1; + $this->{TARGET}->start_file(@_) if $this->{TARGET}; + } +} + +sub end_file { + my $this = shift; + if ($this->{IS_INCLUDED}) { + $this->{TARGET}->end_file(@_) if $this->{TARGET}; + $this->{IS_INCLUDED} = 0; + } +} + +sub next_section { + my $this = shift; + if ($this->{IS_INCLUDED}) { + $this->{TARGET}->next_section(@_) if $this->{TARGET}; + } +} + +1 diff --git a/Bugzilla/PatchReader/PatchInfoGrabber.pm b/Bugzilla/PatchReader/PatchInfoGrabber.pm new file mode 100644 index 000000000..8c52931ba --- /dev/null +++ b/Bugzilla/PatchReader/PatchInfoGrabber.pm @@ -0,0 +1,45 @@ +package Bugzilla::PatchReader::PatchInfoGrabber; + +use Bugzilla::PatchReader::FilterPatch; + +use strict; + +@Bugzilla::PatchReader::PatchInfoGrabber::ISA = qw(Bugzilla::PatchReader::FilterPatch); + +sub new { + my $class = shift; + $class = ref($class) || $class; + my $this = $class->SUPER::new(); + bless $this, $class; + + return $this; +} + +sub patch_info { + my $this = shift; + return $this->{PATCH_INFO}; +} + +sub start_patch { + my $this = shift; + $this->{PATCH_INFO} = {}; + $this->{TARGET}->start_patch(@_) if $this->{TARGET}; +} + +sub start_file { + my $this = shift; + my ($file) = @_; + $this->{PATCH_INFO}{files}{$file->{filename}} = { %{$file} }; + $this->{FILE} = { %{$file} }; + $this->{TARGET}->start_file(@_) if $this->{TARGET}; +} + +sub next_section { + my $this = shift; + my ($section) = @_; + $this->{PATCH_INFO}{files}{$this->{FILE}{filename}}{plus_lines} += $section->{plus_lines}; + $this->{PATCH_INFO}{files}{$this->{FILE}{filename}}{minus_lines} += $section->{minus_lines}; + $this->{TARGET}->next_section(@_) if $this->{TARGET}; +} + +1 diff --git a/Bugzilla/PatchReader/Raw.pm b/Bugzilla/PatchReader/Raw.pm new file mode 100644 index 000000000..b58ed3a2d --- /dev/null +++ b/Bugzilla/PatchReader/Raw.pm @@ -0,0 +1,268 @@ +package Bugzilla::PatchReader::Raw; + +# +# USAGE: +# use PatchReader::Raw; +# my $parser = new PatchReader::Raw(); +# $parser->sends_data_to($my_target); +# $parser->start_lines(); +# open FILE, "mypatch.patch"; +# while (<FILE>) { +# $parser->next_line($_); +# } +# $parser->end_lines(); +# + +use strict; + +use Bugzilla::PatchReader::Base; + +@Bugzilla::PatchReader::Raw::ISA = qw(Bugzilla::PatchReader::Base); + +sub new { + my $class = shift; + $class = ref($class) || $class; + my $this = $class->SUPER::new(); + bless $this, $class; + + return $this; +} + +# We send these notifications: +# start_patch({ patchname }) +# start_file({ filename, rcs_filename, old_revision, old_date_str, new_revision, new_date_str, is_add, is_remove }) +# next_section({ old_start, new_start, old_lines, new_lines, @lines }) +# end_patch +# end_file +sub next_line { + my $this = shift; + my ($line) = @_; + + return if $line =~ /^\?/; + + # patch header parsing + if ($line =~ /^---\s*([\S ]+)\s*\t([^\t\r\n]*)\s*(\S*)/) { + $this->_maybe_end_file(); + + if ($1 eq "/dev/null") { + $this->{FILE_STATE}{is_add} = 1; + } else { + $this->{FILE_STATE}{filename} = $1; + } + $this->{FILE_STATE}{old_date_str} = $2; + $this->{FILE_STATE}{old_revision} = $3 if $3; + + $this->{IN_HEADER} = 1; + + } elsif ($line =~ /^\+\+\+\s*([\S ]+)\s*\t([^\t\r\n]*)(\S*)/) { + if ($1 eq "/dev/null") { + $this->{FILE_STATE}{is_remove} = 1; + } + $this->{FILE_STATE}{new_date_str} = $2; + $this->{FILE_STATE}{new_revision} = $3 if $3; + + $this->{IN_HEADER} = 1; + + } elsif ($line =~ /^RCS file: ([\S ]+)/) { + $this->{FILE_STATE}{rcs_filename} = $1; + + $this->{IN_HEADER} = 1; + + } elsif ($line =~ /^retrieving revision (\S+)/) { + $this->{FILE_STATE}{old_revision} = $1; + + $this->{IN_HEADER} = 1; + + } elsif ($line =~ /^Index:\s*([\S ]+)/) { + $this->_maybe_end_file(); + + $this->{FILE_STATE}{filename} = $1; + + $this->{IN_HEADER} = 1; + + } elsif ($line =~ /^diff\s*(-\S+\s*)*(\S+)\s*(\S*)/ && $3) { + # Simple diff <dir> <dir> + $this->_maybe_end_file(); + $this->{FILE_STATE}{filename} = $2; + + $this->{IN_HEADER} = 1; + + # section parsing + } elsif ($line =~ /^@@\s*-(\d+),?(\d*)\s*\+(\d+),?(\d*)\s*(?:@@\s*(.*))?/) { + $this->{IN_HEADER} = 0; + + $this->_maybe_start_file(); + $this->_maybe_end_section(); + $2 = 0 if !defined($2); + $4 = 0 if !defined($4); + $this->{SECTION_STATE} = { old_start => $1, old_lines => $2, + new_start => $3, new_lines => $4, + func_info => $5, + minus_lines => 0, plus_lines => 0, + }; + + } elsif ($line =~ /^(\d+),?(\d*)([acd])(\d+),?(\d*)/) { + # Non-universal diff. Calculate as though it were universal. + $this->{IN_HEADER} = 0; + + $this->_maybe_start_file(); + $this->_maybe_end_section(); + + my $old_start; + my $old_lines; + my $new_start; + my $new_lines; + if ($3 eq 'a') { + # 'a' has the old number one off from diff -u ("insert after this line" + # vs. "insert at this line") + $old_start = $1 + 1; + $old_lines = 0; + } else { + $old_start = $1; + $old_lines = $2 ? ($2 - $1 + 1) : 1; + } + if ($3 eq 'd') { + # 'd' has the new number one off from diff -u ("delete after this line" + # vs. "delete at this line") + $new_start = $4 + 1; + $new_lines = 0; + } else { + $new_start = $4; + $new_lines = $5 ? ($5 - $4 + 1) : 1; + } + + $this->{SECTION_STATE} = { old_start => $old_start, old_lines => $old_lines, + new_start => $new_start, new_lines => $new_lines, + minus_lines => 0, plus_lines => 0 + }; + } + + # line parsing (only when inside a section) + return if $this->{IN_HEADER}; + if ($line =~ /^ /) { + push @{$this->{SECTION_STATE}{lines}}, $line; + } elsif ($line =~ /^-/) { + $this->{SECTION_STATE}{minus_lines}++; + push @{$this->{SECTION_STATE}{lines}}, $line; + } elsif ($line =~ /^\+/) { + $this->{SECTION_STATE}{plus_lines}++; + push @{$this->{SECTION_STATE}{lines}}, $line; + } elsif ($line =~ /^< /) { + $this->{SECTION_STATE}{minus_lines}++; + push @{$this->{SECTION_STATE}{lines}}, "-" . substr($line, 2); + } elsif ($line =~ /^> /) { + $this->{SECTION_STATE}{plus_lines}++; + push @{$this->{SECTION_STATE}{lines}}, "+" . substr($line, 2); + } +} + +sub start_lines { + my $this = shift; + die "No target specified: call sends_data_to!" if !$this->{TARGET}; + delete $this->{FILE_STARTED}; + delete $this->{FILE_STATE}; + delete $this->{SECTION_STATE}; + $this->{FILE_NEVER_STARTED} = 1; + + $this->{TARGET}->start_patch(@_); +} + +sub end_lines { + my $this = shift; + $this->_maybe_end_file(); + $this->{TARGET}->end_patch(@_); +} + +sub _init_state { + my $this = shift; + $this->{SECTION_STATE}{minus_lines} ||= 0; + $this->{SECTION_STATE}{plus_lines} ||= 0; +} + +sub _maybe_start_file { + my $this = shift; + $this->_init_state(); + if (exists($this->{FILE_STATE}) && !$this->{FILE_STARTED} || + $this->{FILE_NEVER_STARTED}) { + $this->_start_file(); + } +} + +sub _maybe_end_file { + my $this = shift; + $this->_init_state(); + return if $this->{IN_HEADER}; + + $this->_maybe_end_section(); + if (exists($this->{FILE_STATE})) { + # Handle empty patch sections (if the file has not been started and we're + # already trying to end it, start it first!) + if (!$this->{FILE_STARTED}) { + $this->_start_file(); + } + + # Send end notification and set state + $this->{TARGET}->end_file($this->{FILE_STATE}); + delete $this->{FILE_STATE}; + delete $this->{FILE_STARTED}; + } +} + +sub _start_file { + my $this = shift; + + # Send start notification and set state + if (!$this->{FILE_STATE}) { + $this->{FILE_STATE} = { filename => "file_not_specified_in_diff" }; + } + + # Send start notification and set state + $this->{TARGET}->start_file($this->{FILE_STATE}); + $this->{FILE_STARTED} = 1; + delete $this->{FILE_NEVER_STARTED}; +} + +sub _maybe_end_section { + my $this = shift; + if (exists($this->{SECTION_STATE})) { + $this->{TARGET}->next_section($this->{SECTION_STATE}); + delete $this->{SECTION_STATE}; + } +} + +sub iterate_file { + my $this = shift; + my ($filename) = @_; + + open FILE, $filename or die "Could not open $filename: $!"; + $this->start_lines($filename); + while (<FILE>) { + $this->next_line($_); + } + $this->end_lines($filename); + close FILE; +} + +sub iterate_fh { + my $this = shift; + my ($fh, $filename) = @_; + + $this->start_lines($filename); + while (<$fh>) { + $this->next_line($_); + } + $this->end_lines($filename); +} + +sub iterate_string { + my $this = shift; + my ($id, $data) = @_; + + $this->start_lines($id); + while ($data =~ /([^\n]*(\n|$))/g) { + $this->next_line($1); + } + $this->end_lines($id); +} + +1 diff --git a/Bugzilla/Product.pm b/Bugzilla/Product.pm index a0079a033..3e74b9afd 100644 --- a/Bugzilla/Product.pm +++ b/Bugzilla/Product.pm @@ -41,6 +41,8 @@ use constant DEFAULT_CLASSIFICATION_ID => 1; #### Initialization #### ############################### +use constant IS_CONFIG => 1; + use constant DB_TABLE => 'products'; use constant DB_COLUMNS => qw( @@ -107,6 +109,7 @@ sub create { Bugzilla::Hook::process('product_end_of_create', { product => $product }); $dbh->bz_commit_transaction(); + Bugzilla->memcached->clear_config(); return $product; } @@ -114,7 +117,7 @@ sub create { # for each product in the list, particularly with hundreds or thousands # of products. sub preload { - my ($products, $preload_flagtypes) = @_; + my ($products, $preload_flagtypes, $flagtypes_params) = @_; my %prods = map { $_->id => $_ } @$products; my @prod_ids = keys %prods; return unless @prod_ids; @@ -132,7 +135,7 @@ sub preload { } } if ($preload_flagtypes) { - $_->flag_types foreach @$products; + $_->flag_types($flagtypes_params) foreach @$products; } } @@ -262,6 +265,7 @@ sub update { # Changes have been committed. delete $self->{check_group_controls}; Bugzilla->user->clear_product_cache(); + Bugzilla->memcached->clear_config(); return $changes; } @@ -320,6 +324,7 @@ sub remove_from_db { $dbh->do("DELETE FROM products WHERE id = ?", undef, $self->id); $dbh->bz_commit_transaction(); + Bugzilla->memcached->clear_config(); # We have to delete these internal variables, else we get # the old lists of products and classifications again. @@ -779,7 +784,8 @@ sub user_has_access { } sub flag_types { - my $self = shift; + my ($self, $params) = @_; + $params ||= {}; return $self->{'flag_types'} if defined $self->{'flag_types'}; @@ -787,7 +793,7 @@ sub flag_types { my $cache = Bugzilla->request_cache->{flag_types_per_product} ||= {}; $self->{flag_types} = {}; my $prod_id = $self->id; - my $flagtypes = Bugzilla::FlagType::match({ product_id => $prod_id }); + my $flagtypes = Bugzilla::FlagType::match({ product_id => $prod_id, %$params }); foreach my $type ('bug', 'attachment') { my @flags = grep { $_->target_type eq $type } @$flagtypes; @@ -816,8 +822,8 @@ sub flag_types { sub classification { my $self = shift; - $self->{'classification'} ||= - new Bugzilla::Classification($self->classification_id); + $self->{'classification'} ||= + new Bugzilla::Classification({ id => $self->classification_id, cache => 1 }); return $self->{'classification'}; } diff --git a/Bugzilla/Search.pm b/Bugzilla/Search.pm index 8e419c0ee..86e6764ff 100644 --- a/Bugzilla/Search.pm +++ b/Bugzilla/Search.pm @@ -38,7 +38,6 @@ use base qw(Exporter); @Bugzilla::Search::EXPORT = qw( IsValidQueryType split_order_term - translate_old_column ); use Bugzilla::Error; @@ -48,6 +47,7 @@ use Bugzilla::Group; use Bugzilla::User; use Bugzilla::Field; use Bugzilla::Search::Clause; +use Bugzilla::Search::ClauseGroup; use Bugzilla::Search::Condition qw(condition); use Bugzilla::Status; use Bugzilla::Keyword; @@ -56,9 +56,10 @@ use Data::Dumper; use Date::Format; use Date::Parse; use Scalar::Util qw(blessed); -use List::MoreUtils qw(all part uniq); +use List::MoreUtils qw(all firstidx part uniq); use POSIX qw(INT_MAX); use Storable qw(dclone); +use Time::HiRes qw(gettimeofday tv_interval); # Description Of Boolean Charts # ----------------------------- @@ -130,8 +131,14 @@ use Storable qw(dclone); # Constants # ############# +# BMO - product aliases for searching +use constant PRODUCT_ALIASES => { + 'Boot2Gecko' => 'Firefox OS', +}; + # When doing searches, NULL datetimes are treated as this date. use constant EMPTY_DATETIME => '1970-01-01 00:00:00'; +use constant EMPTY_DATE => '1970-01-01'; # This is the regex for real numbers from Regexp::Common, modified to be # more readable. @@ -182,6 +189,8 @@ use constant OPERATORS => { changedfrom => \&_changedfrom_changedto, changedto => \&_changedfrom_changedto, changedby => \&_changedby, + isempty => \&_isempty, + isnotempty => \&_isnotempty, }; # Some operators are really just standard SQL operators, and are @@ -208,6 +217,8 @@ use constant OPERATOR_REVERSE => { lessthaneq => 'greaterthan', greaterthan => 'lessthaneq', greaterthaneq => 'lessthan', + isempty => 'isnotempty', + isnotempty => 'isempty', # The following don't currently have reversals: # casesubstring, anyexact, allwords, allwordssubstr }; @@ -223,6 +234,12 @@ use constant NON_NUMERIC_OPERATORS => qw( notregexp ); +# These operators ignore the entered value +use constant NO_VALUE_OPERATORS => qw( + isempty + isnotempty +); + use constant MULTI_SELECT_OVERRIDE => { notequals => \&_multiselect_negative, notregexp => \&_multiselect_negative, @@ -271,6 +288,9 @@ use constant OPERATOR_FIELD_OVERRIDE => { 'attach_data.thedata' => MULTI_SELECT_OVERRIDE, # We check all attachment fields against this. attachments => MULTI_SELECT_OVERRIDE, + assignee_last_login => { + _default => \&_assignee_last_login, + }, blocked => MULTI_SELECT_OVERRIDE, bug_file_loc => { _non_changed => \&_nullable }, bug_group => MULTI_SELECT_OVERRIDE, @@ -319,7 +339,8 @@ use constant OPERATOR_FIELD_OVERRIDE => { _non_changed => \&_product_nonchanged, }, tag => MULTI_SELECT_OVERRIDE, - + comment_tag => MULTI_SELECT_OVERRIDE, + # Timetracking Fields deadline => { _non_changed => \&_deadline }, percentage_complete => { @@ -331,11 +352,16 @@ use constant OPERATOR_FIELD_OVERRIDE => { changedafter => \&_work_time_changedbefore_after, _default => \&_work_time, }, + last_visit_ts => { + _non_changed => \&_last_visit_ts, + _default => \&_last_visit_ts_invalid_operator, + }, # Custom Fields FIELD_TYPE_FREETEXT, { _non_changed => \&_nullable }, FIELD_TYPE_BUG_ID, { _non_changed => \&_nullable_int }, FIELD_TYPE_DATETIME, { _non_changed => \&_nullable_datetime }, + FIELD_TYPE_DATE, { _non_changed => \&_nullable_date }, FIELD_TYPE_TEXTAREA, { _non_changed => \&_nullable }, FIELD_TYPE_MULTI_SELECT, MULTI_SELECT_OVERRIDE, FIELD_TYPE_BUG_URLS, MULTI_SELECT_OVERRIDE, @@ -343,18 +369,36 @@ use constant OPERATOR_FIELD_OVERRIDE => { # These are fields where special action is taken depending on the # *value* passed in to the chart, sometimes. -use constant SPECIAL_PARSING => { - # Pronoun Fields (Ones that can accept %user%, etc.) - assigned_to => \&_contact_pronoun, - cc => \&_cc_pronoun, - commenter => \&_commenter_pronoun, - qa_contact => \&_contact_pronoun, - reporter => \&_contact_pronoun, - - # Date Fields that accept the 1d, 1w, 1m, 1y, etc. format. - creation_ts => \&_timestamp_translate, - deadline => \&_timestamp_translate, - delta_ts => \&_timestamp_translate, +# This is a sub because custom fields are dynamic +sub SPECIAL_PARSING { + my $map = { + # Pronoun Fields (Ones that can accept %user%, etc.) + assigned_to => \&_contact_pronoun, + cc => \&_cc_pronoun, + commenter => \&_commenter_pronoun, + qa_contact => \&_contact_pronoun, + reporter => \&_contact_pronoun, + + # Date Fields that accept the 1d, 1w, 1m, 1y, etc. format. + creation_ts => \&_datetime_translate, + deadline => \&_date_translate, + delta_ts => \&_datetime_translate, + + # last_visit field that accept both a 1d, 1w, 1m, 1y format and the + # %last_changed% pronoun. + last_visit_ts => \&_last_visit_datetime, + + # BMO - Add ability to use pronoun for bug mentors field + bug_mentor => \&_commenter_pronoun, + }; + foreach my $field (Bugzilla->active_custom_fields) { + if ($field->type == FIELD_TYPE_DATETIME) { + $map->{$field->name} = \&_datetime_translate; + } elsif ($field->type == FIELD_TYPE_DATE) { + $map->{$field->name} = \&_date_translate; + } + } + return $map; }; # Information about fields that represent "users", used by _user_nonchanged. @@ -387,6 +431,11 @@ use constant USER_FIELDS => { field => 'setter_id', join => { table => 'flags' }, }, + # BMO - Ability to search for bugs with specific mentors + 'bug_mentor' => { + field => 'user_id', + join => { table => 'bug_mentors' }, + } }; # Backwards compatibility for times that we changed the names of fields @@ -424,71 +473,98 @@ use constant COLUMN_DEPENDS => { # certain columns in the buglist. For the most part, Search.pm uses # DB::Schema to figure out what needs to be joined, but for some # fields it needs a little help. -use constant COLUMN_JOINS => { - actual_time => { - table => '(SELECT bug_id, SUM(work_time) AS total' - . ' FROM longdescs GROUP BY bug_id)', - join => 'INNER', - }, - assigned_to => { - from => 'assigned_to', - to => 'userid', - table => 'profiles', - join => 'INNER', - }, - reporter => { - from => 'reporter', - to => 'userid', - table => 'profiles', - join => 'INNER', - }, - qa_contact => { - from => 'qa_contact', - to => 'userid', - table => 'profiles', - }, - component => { - from => 'component_id', - to => 'id', - table => 'components', - join => 'INNER', - }, - product => { - from => 'product_id', - to => 'id', - table => 'products', - join => 'INNER', - }, - classification => { - table => 'classifications', - from => 'map_product.classification_id', - to => 'id', - join => 'INNER', - }, - 'flagtypes.name' => { - as => 'map_flags', - table => 'flags', - extra => ['map_flags.attach_id IS NULL'], - then_to => { - as => 'map_flagtypes', - table => 'flagtypes', - from => 'map_flags.type_id', +sub COLUMN_JOINS { + my $user = Bugzilla->user; + + my $joins = { + actual_time => { + table => '(SELECT bug_id, SUM(work_time) AS total' + . ' FROM longdescs GROUP BY bug_id)', + join => 'INNER', + }, + assigned_to => { + from => 'assigned_to', + to => 'userid', + table => 'profiles', + join => 'INNER', + }, + assignee_last_login => { + as => 'assignee', + from => 'assigned_to', + to => 'userid', + table => 'profiles', + join => 'INNER', + }, + reporter => { + from => 'reporter', + to => 'userid', + table => 'profiles', + join => 'INNER', + }, + qa_contact => { + from => 'qa_contact', + to => 'userid', + table => 'profiles', + }, + component => { + from => 'component_id', to => 'id', + table => 'components', + join => 'INNER', }, - }, - keywords => { - table => 'keywords', - then_to => { - as => 'map_keyworddefs', - table => 'keyworddefs', - from => 'map_keywords.keywordid', + product => { + from => 'product_id', to => 'id', + table => 'products', + join => 'INNER', }, - }, - 'longdescs.count' => { - table => 'longdescs', - join => 'INNER', - }, + classification => { + table => 'classifications', + from => 'map_product.classification_id', + to => 'id', + join => 'INNER', + }, + 'flagtypes.name' => { + as => 'map_flags', + table => 'flags', + extra => ['map_flags.attach_id IS NULL'], + then_to => { + as => 'map_flagtypes', + table => 'flagtypes', + from => 'map_flags.type_id', + to => 'id', + }, + }, + keywords => { + table => 'keywords', + then_to => { + as => 'map_keyworddefs', + table => 'keyworddefs', + from => 'map_keywords.keywordid', + to => 'id', + }, + }, + blocked => { + table => 'dependencies', + to => 'dependson', + }, + dependson => { + table => 'dependencies', + to => 'blocked', + }, + 'longdescs.count' => { + table => 'longdescs', + join => 'INNER', + }, + last_visit_ts => { + as => 'bug_user_last_visit', + table => 'bug_user_last_visit', + extra => ['bug_user_last_visit.user_id = ' . $user->id], + from => 'bug_id', + to => 'bug_id', + }, + }; + return $joins; }; # This constant defines the columns that can be selected in a query @@ -552,15 +628,20 @@ sub COLUMNS { . $dbh->sql_string_concat('map_flagtypes.name', 'map_flags.status')), 'keywords' => $dbh->sql_group_concat('DISTINCT map_keyworddefs.name'), + + blocked => $dbh->sql_group_concat('DISTINCT map_blocked.blocked'), + dependson => $dbh->sql_group_concat('DISTINCT map_dependson.dependson'), 'longdescs.count' => 'COUNT(DISTINCT map_longdescs_count.comment_id)', + last_visit_ts => 'bug_user_last_visit.last_visit_ts', + assignee_last_login => 'assignee.last_seen_date', ); # Backward-compatibility for old field names. Goes new_name => old_name. - # These are here and not in translate_old_column because the rest of the + # These are here and not in _translate_old_column because the rest of the # code actually still uses the old names, while the fielddefs table uses # the new names (which is not the case for the fields handled by - # translate_old_column). + # _translate_old_column). my %old_names = ( creation_ts => 'opendate', delta_ts => 'changeddate', @@ -647,7 +728,9 @@ sub REPORT_COLUMNS { # is here because it *always* goes into the GROUP BY as the first item, # so it should be skipped when determining extra GROUP BY columns. use constant GROUP_BY_SKIP => qw( + blocked bug_id + dependson flagtypes.name keywords longdescs.count @@ -688,7 +771,128 @@ sub new { # Public Accessors # #################### -sub sql { +sub data { + my $self = shift; + return $self->{data} if $self->{data}; + my $dbh = Bugzilla->dbh; + + # If all fields belong to the 'bugs' table, there is no need to split + # the original query into two pieces. Else we override the 'fields' + # argument to first get bug IDs based on the search criteria defined + # by the caller, and the desired fields are collected in the 2nd query. + my @orig_fields = $self->_input_columns; + my $all_in_bugs_table = 1; + foreach my $field (@orig_fields) { + next if $self->COLUMNS->{$field}->{name} =~ /^bugs\.\w+$/; + $self->{fields} = ['bug_id']; + $all_in_bugs_table = 0; + last; + } + + # BMO - to avoid massive amounts of joins, if we're selecting a lot of + # tracking flags, replace them with placeholders. the values will be + # retrieved later and injected into the result. + my %tf_map = map { $_ => 1 } Bugzilla::Extension::TrackingFlags::Flag->get_all_names(); + my @tf_selected = grep { exists $tf_map{$_} } @orig_fields; + # mysql has a limit of 61 joins, and we want to avoid massive amounts of joins + # 30 ensures we won't hit the limit, nor generate too many joins + if (scalar @tf_selected > 30) { + foreach my $column (@tf_selected) { + $self->COLUMNS->{$column}->{name} = "'---'"; + } + $self->{tracking_flags} = \@tf_selected; + } + else { + $self->{tracking_flags} = []; + } + + my $start_time = [gettimeofday()]; + my $sql = $self->_sql; + # Do we just want bug IDs to pass to the 2nd query or all the data immediately? + my $func = $all_in_bugs_table ? 'selectall_arrayref' : 'selectcol_arrayref'; + my $bug_ids = $dbh->$func($sql); + my @extra_data = ({sql => $sql, time => tv_interval($start_time)}); + # Restore the original 'fields' argument, just in case. + $self->{fields} = \@orig_fields unless $all_in_bugs_table; + + # BMO if the caller only wants the count, that's all we need to return + return $bug_ids->[0]->[0] if $self->_params->{count_only}; + + # If there are no bugs found, or all fields are in the 'bugs' table, + # there is no need for another query. + if (!scalar @$bug_ids || $all_in_bugs_table) { + $self->{data} = $bug_ids; + return wantarray ? ($self->{data}, \@extra_data) : $self->{data}; + } + + # Make sure the bug_id will be returned. If not, append it to the list. + my $pos = firstidx { $_ eq 'bug_id' } @orig_fields; + if ($pos < 0) { + push(@orig_fields, 'bug_id'); + $pos = $#orig_fields; + } + + # Now create a query with the buglist above as the single criteria + # and the fields that the caller wants. No need to redo security checks; + # the list has already been validated above. + my $search = $self->new('fields' => \@orig_fields, + 'params' => {bug_id => $bug_ids, bug_id_type => 'anyexact'}, + 'sharer' => $self->_sharer_id, + 'user' => $self->_user, + 'allow_unlimited' => 1, + '_no_security_check' => 1); + + $start_time = [gettimeofday()]; + $sql = $search->_sql; + my $unsorted_data = $dbh->selectall_arrayref($sql); + push(@extra_data, {sql => $sql, time => tv_interval($start_time)}); + # Let's sort the data. We didn't do it in the query itself because + # we already know in which order to sort bugs thanks to the first query, + # and this avoids additional table joins in the SQL query. + my %data = map { $_->[$pos] => $_ } @$unsorted_data; + $self->{data} = [map { $data{$_} } @$bug_ids]; + + # BMO - get tracking flags values, and insert into result + if (@{ $self->{tracking_flags} }) { + # read values + my $values; + $sql = " + SELECT bugs.bug_id, tracking_flags.name, tracking_flags_bugs.value + FROM bugs + LEFT JOIN tracking_flags_bugs ON tracking_flags_bugs.bug_id = bugs.bug_id + LEFT JOIN tracking_flags ON tracking_flags.id = tracking_flags_bugs.tracking_flag_id + WHERE " . $dbh->sql_in('bugs.bug_id', $bug_ids); + $start_time = [gettimeofday()]; + my $rows = $dbh->selectall_arrayref($sql); + push(@extra_data, {sql => $sql, time => tv_interval($start_time)}); + foreach my $row (@$rows) { + $values->{$row->[0]}{$row->[1]} = $row->[2] if defined($row->[2]); + } + + # find the columns of the tracking flags + my %tf_pos; + for (my $i = 0; $i <= $#orig_fields; $i++) { + if (grep { $_ eq $orig_fields[$i] } @{ $self->{tracking_flags} }) { + $tf_pos{$orig_fields[$i]} = $i; + } + } + + # replace the placeholder value with the field's value + foreach my $row (@{ $self->{data} }) { + my $bug_id = $row->[$pos]; + next unless exists $values->{$bug_id}; + foreach my $field (keys %{ $values->{$bug_id} }) { + if (exists $tf_pos{$field}) { + $row->[$tf_pos{$field}] = $values->{$bug_id}{$field}; + } + } + } + } + + return wantarray ? ($self->{data}, \@extra_data) : $self->{data}; +} + +sub _sql { my ($self) = @_; return $self->{sql} if $self->{sql}; my $dbh = Bugzilla->dbh; @@ -710,7 +914,15 @@ sub sql { ? "\nORDER BY " . join(', ', $self->_sql_order_by) : ''; my $limit = $self->_sql_limit; $limit = "\n$limit" if $limit; - + + # BMO allow fetching just the number of matching bugs + if ($self->_params->{count_only}) { + $select = 'COUNT(*) AS count'; + $group_by = ''; + $order_by = ''; + $limit = ''; + } + my $query = <<END; SELECT $select FROM $from @@ -725,12 +937,30 @@ sub search_description { my ($self, $params) = @_; my $desc = $self->{'search_description'} ||= []; if ($params) { + + # BMO - product aliases + # display the new product name on the search results name to avoid a + # disparity between the search summary and the results. + if ($params->{field} eq 'product') { + my $aliased; + my @values = split(/,/, $params->{value}); + foreach my $value (@values) { + if (exists PRODUCT_ALIASES->{lc($value)}) { + $value = PRODUCT_ALIASES->{lc($value)}; + $aliased = 1; + } + } + if ($aliased) { + $params->{value} = join(',', @values); + } + } + push(@$desc, $params); } # Make sure that the description has actually been generated if # people are asking for the whole thing. else { - $self->sql; + $self->_sql; } return $self->{'search_description'}; } @@ -754,6 +984,21 @@ sub boolean_charts_to_custom_search { } } +sub invalid_order_columns { + my ($self) = @_; + my @invalid_columns; + foreach my $order ($self->_input_order) { + next if defined $self->_validate_order_column($order); + push(@invalid_columns, $order); + } + return \@invalid_columns; +} + +sub order { + my ($self) = @_; + return $self->_valid_order; +} + ###################### # Internal Accessors # ###################### @@ -819,7 +1064,7 @@ sub _extra_columns { my ($self) = @_; # Everything that's going to be in the ORDER BY must also be # in the SELECT. - push(@{ $self->{extra_columns} }, $self->_input_order_columns); + push(@{ $self->{extra_columns} }, $self->_valid_order_columns); return @{ $self->{extra_columns} }; } @@ -889,10 +1134,32 @@ sub _sql_select { # The "order" that was requested by the consumer, exactly as it was # requested. sub _input_order { @{ $_[0]->{'order'} || [] } } -# The input order with just the column names, and no ASC or DESC. -sub _input_order_columns { +# Requested order with invalid values removed and old names translated +sub _valid_order { + my ($self) = @_; + return map { ($self->_validate_order_column($_)) } $self->_input_order; +} +# The valid order with just the column names, and no ASC or DESC. +sub _valid_order_columns { my ($self) = @_; - return map { (split_order_term($_))[0] } $self->_input_order; + return map { (split_order_term($_))[0] } $self->_valid_order; +} + +sub _validate_order_column { + my ($self, $order_item) = @_; + + # Translate old column names + my ($field, $direction) = split_order_term($order_item); + $field = $self->_translate_old_column($field); + + # Only accept valid columns + return if (!exists $self->COLUMNS->{$field}); + + # Relevance column can be used only with one or more fulltext searches + return if ($field eq 'relevance' && !$self->COLUMNS->{$field}->{name}); + + $direction = " $direction" if $direction; + return "$field$direction"; } # A hashref that describes all the special stuff that has to be done @@ -924,7 +1191,7 @@ sub _sql_order_by { my ($self) = @_; if (!$self->{sql_order_by}) { my @order_by = map { $self->_translate_order_by_column($_) } - $self->_input_order; + $self->_valid_order; $self->{sql_order_by} = \@order_by; } return @{ $self->{sql_order_by} }; @@ -1069,7 +1336,7 @@ sub _select_order_joins { my @column_join = $self->_column_join($field); push(@joins, @column_join); } - foreach my $field ($self->_input_order_columns) { + foreach my $field ($self->_valid_order_columns) { my $join_info = $self->_special_order->{$field}->{join}; if ($join_info) { # Don't let callers modify SPECIAL_ORDER. @@ -1088,6 +1355,7 @@ sub _standard_joins { my ($self) = @_; my $user = $self->_user; my @joins; + return () if $self->{_no_security_check}; my $security_join = { table => 'bug_group_map', @@ -1126,8 +1394,8 @@ sub _translate_join { die "join with no table: " . Dumper($join_info) if !$join_info->{table}; die "join with no 'as': " . Dumper($join_info) if !$join_info->{as}; - - my $from_table = "bugs"; + + my $from_table = $join_info->{bugs_table} || "bugs"; my $from = $join_info->{from} || "bug_id"; if ($from =~ /^(\w+)\.(\w+)$/) { ($from_table, $from) = ($1, $2); @@ -1164,6 +1432,7 @@ sub _translate_join { # group security. sub _standard_where { my ($self) = @_; + return ('1=1') if $self->{_no_security_check}; # If replication lags badly between the shadow db and the main DB, # it's possible for bugs to show up in searches before their group # controls are properly set. To prevent this, when initially creating @@ -1225,7 +1494,7 @@ sub _sql_group_by { # And all items from ORDER BY must be in the GROUP BY. The above loop # doesn't catch items that were put into the ORDER BY from SPECIAL_ORDER. - foreach my $column ($self->_input_order_columns) { + foreach my $column ($self->_valid_order_columns) { my $special_order = $self->_special_order->{$column}->{order}; next if !$special_order; push(@extra_group_by, @$special_order); @@ -1377,6 +1646,8 @@ sub _special_parse_chfield { @fields = map { $_ eq '[Bug creation]' ? 'creation_ts' : $_ } @fields; + return undef unless ($date_from ne '' || $date_to ne '' || $value_to ne ''); + my $clause = new Bugzilla::Search::Clause(); # It is always safe and useful to push delta_ts into the charts @@ -1398,44 +1669,21 @@ sub _special_parse_chfield { $clause->add('delta_ts', 'lessthaneq', $date_to); } - # Basically, we construct the chart like: - # - # (added_for_field1 = value OR added_for_field2 = value) - # AND (date_field1_changed >= date_from OR date_field2_changed >= date_from) - # AND (date_field1_changed <= date_to OR date_field2_changed <= date_to) - # - # Theoretically, all we *really* would need to do is look for the field id - # in the bugs_activity table, because we've already limited the search - # by delta_ts above, but there's no chart to do that, so we check the - # change date of the fields. - - if ($value_to ne '') { - my $value_clause = new Bugzilla::Search::Clause('OR'); - foreach my $field (@fields) { - $value_clause->add($field, 'changedto', $value_to); - } - $clause->add($value_clause); - } + # chfieldto is supposed to be a relative date or a date of the form + # YYYY-MM-DD, i.e. without the time appended to it. We append the + # time ourselves so that the end date is correctly taken into account. + $date_to .= ' 23:59:59' if $date_to =~ /^\d{4}-\d{1,2}-\d{1,2}$/; - if ($date_from ne '') { - my $from_clause = new Bugzilla::Search::Clause('OR'); - foreach my $field (@fields) { - $from_clause->add($field, 'changedafter', $date_from); - } - $clause->add($from_clause); - } - if ($date_to ne '') { - # chfieldto is supposed to be a relative date or a date of the form - # YYYY-MM-DD, i.e. without the time appended to it. We append the - # time ourselves so that the end date is correctly taken into account. - $date_to .= ' 23:59:59' if $date_to =~ /^\d{4}-\d{1,2}-\d{1,2}$/; + my $join_clause = new Bugzilla::Search::Clause('OR'); - my $to_clause = new Bugzilla::Search::Clause('OR'); - foreach my $field (@fields) { - $to_clause->add($field, 'changedbefore', $date_to); - } - $clause->add($to_clause); + foreach my $field (@fields) { + my $sub_clause = new Bugzilla::Search::ClauseGroup(); + $sub_clause->add(condition($field, 'changedto', $value_to)) if $value_to ne ''; + $sub_clause->add(condition($field, 'changedafter', $date_from)) if $date_from ne ''; + $sub_clause->add(condition($field, 'changedbefore', $date_to)) if $date_to ne ''; + $join_clause->add($sub_clause); } + $clause->add($join_clause); return $clause; } @@ -1472,7 +1720,7 @@ sub _special_parse_email { $type = "anyexact" if $type eq "exact"; my $or_clause = new Bugzilla::Search::Clause('OR'); - foreach my $field (qw(assigned_to reporter cc qa_contact)) { + foreach my $field (qw(assigned_to reporter cc qa_contact bug_mentor)) { if ($params->{"email$field$id"}) { $or_clause->add($field, $type, $email); } @@ -1525,7 +1773,7 @@ sub _charts_to_conditions { my $clause = $self->_charts; my @joins; $clause->walk_conditions(sub { - my ($condition) = @_; + my ($clause, $condition) = @_; return if !$condition->translated; push(@joins, @{ $condition->translated->{joins} }); }); @@ -1545,7 +1793,7 @@ sub _params_to_data_structure { my ($self) = @_; # First we get the "special" charts, representing all the normal - # field son the search page. This may modify _params, so it needs to + # fields on the search page. This may modify _params, so it needs to # happen first. my $clause = $self->_special_charts; @@ -1554,7 +1802,7 @@ sub _params_to_data_structure { # And then process the modern "custom search" format. $clause->add( $self->_custom_search ); - + return $clause; } @@ -1585,7 +1833,9 @@ sub _boolean_charts { my $identifier = "$chart_id-$and_id-$or_id"; my $field = $params->{"field$identifier"}; my $operator = $params->{"type$identifier"}; - my $value = $params->{"value$identifier"}; + my $value = $params->{"value$identifier"}; + # no-value operators ignore the value, however a value needs to be set + $value = ' ' if $operator && grep { $_ eq $operator } NO_VALUE_OPERATORS; $or_clause->add($field, $operator, $value); } $and_clause->add($or_clause); @@ -1601,13 +1851,18 @@ sub _custom_search { my ($self) = @_; my $params = $self->_params; - my $current_clause = new Bugzilla::Search::Clause($params->{j_top}); + my $joiner = $params->{j_top} || ''; + my $current_clause = $joiner eq 'AND_G' + ? new Bugzilla::Search::ClauseGroup() + : new Bugzilla::Search::Clause($joiner); my @clause_stack; foreach my $id ($self->_field_ids) { my $field = $params->{"f$id"}; if ($field eq 'OP') { - my $joiner = $params->{"j$id"}; - my $new_clause = new Bugzilla::Search::Clause($joiner); + my $joiner = $params->{"j$id"} || ''; + my $new_clause = $joiner eq 'AND_G' + ? new Bugzilla::Search::ClauseGroup() + : new Bugzilla::Search::Clause($joiner); $new_clause->negate($params->{"n$id"}); $current_clause->add($new_clause); push(@clause_stack, $current_clause); @@ -1623,6 +1878,8 @@ sub _custom_search { my $operator = $params->{"o$id"}; my $value = $params->{"v$id"}; + # no-value operators ignore the value, however a value needs to be set + $value = ' ' if $operator && grep { $_ eq $operator } NO_VALUE_OPERATORS; my $condition = condition($field, $operator, $value); $condition->negate($params->{"n$id"}); $current_clause->add($condition); @@ -1646,14 +1903,12 @@ sub _field_ids { } sub _handle_chart { - my ($self, $chart_id, $condition) = @_; + my ($self, $chart_id, $clause, $condition) = @_; my $dbh = Bugzilla->dbh; my $params = $self->_params; my ($field, $operator, $value) = $condition->fov; - - $field = FIELD_MAP->{$field} || $field; - return if (!defined $field or !defined $operator or !defined $value); + $field = FIELD_MAP->{$field} || $field; my $string_value; if (ref $value eq 'ARRAY') { @@ -1684,16 +1939,20 @@ sub _handle_chart { # on multiple values, like anyexact. my %search_args = ( - chart_id => $chart_id, - sequence => $chart_id, - field => $field, - full_field => $full_field, - operator => $operator, - value => $string_value, - all_values => $value, - joins => [], - condition => $condition, + chart_id => $chart_id, + sequence => $chart_id, + field => $field, + full_field => $full_field, + operator => $operator, + value => $string_value, + all_values => $value, + joins => [], + bugs_table => 'bugs', + table_suffix => '', + condition => $condition, ); + $clause->update_search_args(\%search_args); + $search_args{quoted} = $self->_quote_unless_numeric(\%search_args); # This should add a "term" selement to %search_args. $self->do_search_function(\%search_args); @@ -1709,7 +1968,12 @@ sub _handle_chart { field => $field, type => $operator, value => $string_value, term => $search_args{term}, }); - + + foreach my $join (@{ $search_args{joins} }) { + $join->{bugs_table} = $search_args{bugs_table}; + $join->{table_suffix} = $search_args{table_suffix}; + } + $condition->translated(\%search_args); } @@ -1830,8 +2094,9 @@ sub _get_column_joins { return $cache->{column_joins} if defined $cache->{column_joins}; my %column_joins = %{ COLUMN_JOINS() }; + # BMO - add search object to hook Bugzilla::Hook::process('buglist_column_joins', - { column_joins => \%column_joins }); + { column_joins => \%column_joins, search => $self }); $cache->{column_joins} = \%column_joins; return $cache->{column_joins}; @@ -1866,6 +2131,13 @@ sub _quote_unless_numeric { sub build_subselect { my ($outer, $inner, $table, $cond, $negate) = @_; + if ($table =~ /\battach_data\b/) { + # It takes a long time to scan the whole attach_data table + # unconditionally, so we return the subselect and let the DB optimizer + # restrict the search based on other search criteria. + my $not = $negate ? "NOT" : ""; + return "$outer $not IN (SELECT DISTINCT $inner FROM $table WHERE $cond)"; + } # Execute subselects immediately to avoid dependent subqueries, which are # large performance hits on MySql my $q = "SELECT DISTINCT $inner FROM $table WHERE $cond"; @@ -1951,22 +2223,44 @@ sub _word_terms { ##################################### sub _timestamp_translate { - my ($self, $args) = @_; + my ($self, $ignore_time, $args) = @_; my $value = $args->{value}; my $dbh = Bugzilla->dbh; return if $value !~ /^(?:[\+\-]?\d+[hdwmy]s?|now)$/i; - # By default, the time is appended to the date, which we don't want - # for deadlines. $value = SqlifyDate($value); - if ($args->{field} eq 'deadline') { + # By default, the time is appended to the date, which we don't always want. + if ($ignore_time) { ($value) = split(/\s/, $value); } $args->{value} = $value; $args->{quoted} = $dbh->quote($value); } +sub _datetime_translate { + return shift->_timestamp_translate(0, @_); +} + +sub _last_visit_datetime { + my ($self, $args) = @_; + my $value = $args->{value}; + + $self->_datetime_translate($args); + if ($value eq $args->{value}) { + # Failed to translate a datetime. let's try the pronoun expando. + if ($value eq '%last_changed%') { + $self->_add_extra_column('changeddate'); + $args->{value} = $args->{quoted} = 'bugs.delta_ts'; + } + } +} + + +sub _date_translate { + return shift->_timestamp_translate(1, @_); +} + sub SqlifyDate { my ($str) = @_; my $fmt = "%Y-%m-%d %H:%M:%S"; @@ -2259,7 +2553,7 @@ sub _user_nonchanged { # For negative operators, the system we're using here # only works properly if we reverse the operator and check IS NULL # in the WHERE. - my $is_negative = $operator =~ /^no/ ? 1 : 0; + my $is_negative = $operator =~ /^(?:no|isempty)/ ? 1 : 0; if ($is_negative) { $args->{operator} = $self->_reverse_operator($operator); } @@ -2339,8 +2633,13 @@ sub _long_desc_changedbefore_after { sub _long_desc_nonchanged { my ($self, $args) = @_; - my ($chart_id, $operator, $value, $joins) = - @$args{qw(chart_id operator value joins)}; + my ($chart_id, $operator, $value, $joins, $bugs_table) = + @$args{qw(chart_id operator value joins bugs_table)}; + + if ($operator =~ /^is(not)?empty$/) { + $args->{term} = $self->_multiselect_isempty($args, $operator eq 'isnotempty'); + return; + } my $dbh = Bugzilla->dbh; my $table = "longdescs_$chart_id"; @@ -2354,6 +2653,7 @@ sub _long_desc_nonchanged { all_values => $value, quoted => $dbh->quote($value), joins => [], + bugs_table => $bugs_table, }; $self->_do_operator_function($join_args); @@ -2481,6 +2781,21 @@ sub _percentage_complete { $self->_add_extra_column('actual_time'); } +sub _last_visit_ts { + my ($self, $args) = @_; + + $args->{full_field} = $self->COLUMNS->{last_visit_ts}->{name}; + $self->_add_extra_column('last_visit_ts'); +} + +sub _last_visit_ts_invalid_operator { + my ($self, $args) = @_; + + ThrowUserError('search_field_operator_invalid', + { field => $args->{field}, + operator => $args->{operator} }); +} + sub _days_elapsed { my ($self, $args) = @_; my $dbh = Bugzilla->dbh; @@ -2489,6 +2804,21 @@ sub _days_elapsed { $dbh->sql_to_days('bugs.delta_ts') . ")"; } +sub _assignee_last_login { + my ($self, $args) = @_; + + push @{ $args->{joins} }, { + as => 'assignee', + table => 'profiles', + from => 'assigned_to', + to => 'userid', + join => 'INNER', + }; + # coalesce to 1998 to make it easy to search for users who haven't logged + # in since we added last_seen_date + $args->{full_field} = "COALESCE(assignee.last_seen_date, '1998-01-01')"; +} + sub _component_nonchanged { my ($self, $args) = @_; @@ -2501,7 +2831,28 @@ sub _component_nonchanged { sub _product_nonchanged { my ($self, $args) = @_; - + + # BMO - product aliases + # swap out old product names for new ones + if (ref($args->{all_values})) { + my $aliased; + foreach my $value (@{ $args->{all_values} }) { + if (exists PRODUCT_ALIASES->{lc($value)}) { + $value = PRODUCT_ALIASES->{lc($value)}; + $aliased = 1; + } + } + if ($aliased) { + $args->{value} = join(',', @{ $args->{all_values} }); + $args->{quoted} = Bugzilla->dbh->quote($args->{value}); + } + } + elsif (exists PRODUCT_ALIASES->{lc($args->{value})}) { + $args->{value} = PRODUCT_ALIASES->{lc($args->{value})}; + $args->{all_values} = $args->{value}; + $args->{quoted} = Bugzilla->dbh->quote($args->{value}); + } + # Generate the restriction condition $args->{full_field} = "products.name"; $self->_do_operator_function($args); @@ -2544,6 +2895,13 @@ sub _nullable_datetime { $args->{full_field} = "COALESCE($field, $empty)"; } +sub _nullable_date { + my ($self, $args) = @_; + my $field = $args->{full_field}; + my $empty = Bugzilla->dbh->quote(EMPTY_DATE); + $args->{full_field} = "COALESCE($field, $empty)"; +} + sub _deadline { my ($self, $args) = @_; my $field = $args->{full_field}; @@ -2586,7 +2944,7 @@ sub _owner_idle_time_greater_less { "$ld_table.who IS NULL AND $act_table.who IS NULL"; } else { $args->{term} = - "$ld_table.who IS NOT NULL OR $act_table.who IS NOT NULL"; + "($ld_table.who IS NOT NULL OR $act_table.who IS NOT NULL)"; } } @@ -2630,8 +2988,14 @@ sub _multiselect_multiple { sub _flagtypes_nonchanged { my ($self, $args) = @_; - my ($chart_id, $operator, $value, $joins, $condition) = - @$args{qw(chart_id operator value joins condition)}; + my ($chart_id, $operator, $value, $joins, $bugs_table, $condition) = + @$args{qw(chart_id operator value joins bugs_table condition)}; + + if ($operator =~ /^is(not)?empty$/) { + $args->{term} = $self->_multiselect_isempty($args, $operator eq 'isnotempty'); + return; + } + my $dbh = Bugzilla->dbh; # For 'not' operators, we need to negate the whole term. @@ -2654,6 +3018,7 @@ sub _flagtypes_nonchanged { all_values => $value, quoted => $dbh->quote($value), joins => [], + bugs_table => "bugs_$chart_id", }; $self->_do_operator_function($subselect_args); my $subselect_term = $subselect_args->{term}; @@ -2661,7 +3026,7 @@ sub _flagtypes_nonchanged { # don't call build_subselect as this must run as a true sub-select $args->{term} = "EXISTS ( SELECT 1 - FROM bugs bugs_$chart_id + FROM $bugs_table bugs_$chart_id LEFT JOIN attachments AS attachments_$chart_id ON bugs_$chart_id.bug_id = attachments_$chart_id.bug_id LEFT JOIN flags AS flags_$chart_id @@ -2670,7 +3035,7 @@ sub _flagtypes_nonchanged { OR flags_$chart_id.attach_id IS NULL) LEFT JOIN flagtypes AS flagtypes_$chart_id ON flags_$chart_id.type_id = flagtypes_$chart_id.id - WHERE bugs_$chart_id.bug_id = bugs.bug_id + WHERE bugs_$chart_id.bug_id = $bugs_table.bug_id AND $subselect_term )"; } @@ -2740,6 +3105,13 @@ sub _multiselect_table { "flags.status"); return "flags INNER JOIN flagtypes ON flags.type_id = flagtypes.id"; } + elsif ($field eq 'comment_tag') { + $args->{_extra_where} = " AND longdescs.isprivate = 0" + if !$self->_user->is_insider; + $args->{full_field} = 'longdescs_tags.tag'; + return "longdescs INNER JOIN longdescs_tags". + " ON longdescs.comment_id = longdescs_tags.comment_id"; + } my $table = "bug_$field"; $args->{full_field} = "bug_$field.value"; return $table; @@ -2747,12 +3119,126 @@ sub _multiselect_table { sub _multiselect_term { my ($self, $args, $not) = @_; + my ($operator) = $args->{operator}; + # 'empty' operators require special handling + return $self->_multiselect_isempty($args, $not) + if $operator =~ /^is(not)?empty$/; my $table = $self->_multiselect_table($args); $self->_do_operator_function($args); my $term = $args->{term}; $term .= $args->{_extra_where} || ''; my $select = $args->{_select_field} || 'bug_id'; - return build_subselect("bugs.bug_id", $select, $table, $term, $not); + return build_subselect("$args->{bugs_table}.bug_id", $select, $table, $term, $not); +} + +# We can't use the normal operator_functions to build isempty queries which +# join to different tables. +sub _multiselect_isempty { + my ($self, $args, $not) = @_; + my ($field, $operator, $joins, $chart_id) = @$args{qw(field operator joins chart_id)}; + my $dbh = Bugzilla->dbh; + $operator = $self->_reverseoperator($operator) if $not; + $not = $operator eq 'isnotempty' ? 'NOT' : ''; + + if ($field eq 'keywords') { + push @$joins, { + table => 'keywords', + as => "keywords_$chart_id", + from => 'bug_id', + to => 'bug_id', + }; + return "keywords_$chart_id.bug_id IS $not NULL"; + } + elsif ($field eq 'bug_group') { + push @$joins, { + table => 'bug_group_map', + as => "bug_group_map_$chart_id", + from => 'bug_id', + to => 'bug_id', + }; + return "bug_group_map_$chart_id.bug_id IS $not NULL"; + } + elsif ($field eq 'flagtypes.name') { + push @$joins, { + table => 'flags', + as => "flags_$chart_id", + from => 'bug_id', + to => 'bug_id', + }; + return "flags_$chart_id.bug_id IS $not NULL"; + } + elsif ($field eq 'blocked' or $field eq 'dependson') { + my $to = $field eq 'blocked' ? 'dependson' : 'blocked'; + push @$joins, { + table => 'dependencies', + as => "dependencies_$chart_id", + from => 'bug_id', + to => $to, + }; + return "dependencies_$chart_id.$to IS $not NULL"; + } + elsif ($field eq 'longdesc') { + my @extra = ( "longdescs_$chart_id.type != " . CMT_HAS_DUPE ); + push @extra, "longdescs_$chart_id.isprivate = 0" + unless $self->_user->is_insider; + push @$joins, { + table => 'longdescs', + as => "longdescs_$chart_id", + from => 'bug_id', + to => 'bug_id', + extra => \@extra, + }; + return $not + ? "longdescs_$chart_id.thetext != ''" + : "longdescs_$chart_id.thetext = ''"; + } + elsif ($field eq 'longdescs.isprivate') { + ThrowUserError('search_field_operator_invalid', { field => $field, + operator => $operator }); + } + elsif ($field =~ /^attachments\.(.+)/) { + my $sub_field = $1; + if ($sub_field eq 'description' || $sub_field eq 'filename' || $sub_field eq 'mimetype') { + # can't be null/empty + return $not ? '1=1' : '1=2'; + } else { + # all other fields which get here are boolean + ThrowUserError('search_field_operator_invalid', { field => $field, + operator => $operator }); + } + } + elsif ($field eq 'attach_data.thedata') { + push @$joins, { + table => 'attachments', + as => "attachments_$chart_id", + from => 'bug_id', + to => 'bug_id', + extra => [ $self->_user->is_insider ? '' : "attachments_$chart_id.isprivate = 0" ], + }; + push @$joins, { + table => 'attach_data', + as => "attach_data_$chart_id", + from => "attachments_$chart_id.attach_id", + to => 'id', + }; + return "attach_data_$chart_id.thedata IS $not NULL"; + } + elsif ($field eq 'tag') { + push @$joins, { + table => 'bug_tag', + as => "bug_tag_$chart_id", + from => 'bug_id', + to => 'bug_id', + }; + push @$joins, { + table => 'tag', + as => "tag_$chart_id", + from => "bug_tag_$chart_id.tag_id", + to => 'id', + extra => [ "tag_$chart_id.user_id = " . ($self->_sharer_id || $self->_user->id) ], + }; + return "tag_$chart_id.id IS $not NULL"; + } } ############################### @@ -2829,14 +3315,14 @@ sub _anywordsubstr { my ($self, $args) = @_; my @terms = $self->_substring_terms($args); - $args->{term} = join("\n\tOR ", @terms); + $args->{term} = @terms ? '(' . join("\n\tOR ", @terms) . ')' : ''; } sub _allwordssubstr { my ($self, $args) = @_; my @terms = $self->_substring_terms($args); - $args->{term} = join("\n\tAND ", @terms); + $args->{term} = @terms ? '(' . join("\n\tAND ", @terms) . ')' : ''; } sub _nowordssubstr { @@ -2848,19 +3334,19 @@ sub _nowordssubstr { sub _anywords { my ($self, $args) = @_; - + my @terms = $self->_word_terms($args); # Because _word_terms uses AND, we need to parenthesize its terms # if there are more than one. @terms = map("($_)", @terms) if scalar(@terms) > 1; - $args->{term} = join("\n\tOR ", @terms); + $args->{term} = @terms ? '(' . join("\n\tOR ", @terms) . ')' : ''; } sub _allwords { my ($self, $args) = @_; - + my @terms = $self->_word_terms($args); - $args->{term} = join("\n\tAND ", @terms); + $args->{term} = @terms ? '(' . join("\n\tAND ", @terms) . ')' : ''; } sub _nowords { @@ -2971,6 +3457,27 @@ sub _changed_security_check { } } +sub _isempty { + my ($self, $args) = @_; + my $full_field = $args->{full_field}; + $args->{term} = "$full_field IS NULL OR $full_field = " . $self->_empty_value($args->{field}); +} + +sub _isnotempty { + my ($self, $args) = @_; + my $full_field = $args->{full_field}; + $args->{term} = "$full_field IS NOT NULL AND $full_field != " . $self->_empty_value($args->{field}); +} + +sub _empty_value { + my ($self, $field) = @_; + my $field_obj = $self->_chart_fields->{$field}; + return "0" if $field_obj->type == FIELD_TYPE_BUG_ID; + return Bugzilla->dbh->quote(EMPTY_DATETIME) if $field_obj->type == FIELD_TYPE_DATETIME; + return Bugzilla->dbh->quote(EMPTY_DATE) if $field_obj->type == FIELD_TYPE_DATE; + return "''"; +} + ###################### # Public Subroutines # ###################### @@ -2979,7 +3486,8 @@ sub _changed_security_check { sub IsValidQueryType { my ($queryType) = @_; - if (grep { $_ eq $queryType } qw(specific advanced)) { + # BMO: Added google and instant + if (grep { $_ eq $queryType } qw(specific advanced google instant)) { return 1; } return 0; @@ -2995,8 +3503,8 @@ sub split_order_term { # Used to translate old SQL fragments from buglist.cgi's "order" argument # into our modern field IDs. -sub translate_old_column { - my ($column) = @_; +sub _translate_old_column { + my ($self, $column) = @_; # All old SQL fragments have a period in them somewhere. return $column if $column !~ /\./; @@ -3010,12 +3518,124 @@ sub translate_old_column { # If it doesn't match the regexps above, check to see if the old # SQL fragment matches the SQL of an existing column - foreach my $key (%{ COLUMNS() }) { - next unless exists COLUMNS->{$key}->{name}; - return $key if COLUMNS->{$key}->{name} eq $column; + foreach my $key (%{ $self->COLUMNS }) { + next unless exists $self->COLUMNS->{$key}->{name}; + return $key if $self->COLUMNS->{$key}->{name} eq $column; } return $column; } +# BMO - make product aliases lowercase +foreach my $name (keys %{ PRODUCT_ALIASES() }) { + PRODUCT_ALIASES->{lc($name)} = PRODUCT_ALIASES->{$name}; + delete PRODUCT_ALIASES->{$name}; +} + 1; + +__END__ + +=head1 NAME + +Bugzilla::Search - Provides methods to run queries against bugs. + +=head1 SYNOPSIS + + use Bugzilla::Search; + + my $search = new Bugzilla::Search({'fields' => \@fields, + 'params' => \%search_criteria, + 'sharer' => $sharer_id, + 'user' => $user_obj, + 'allow_unlimited' => 1}); + + my $data = $search->data; + my ($data, $extra_data) = $search->data; + +=head1 DESCRIPTION + +Search.pm represents a search object. It's the single way to collect +data about bugs in a secure way. The list of bugs matching criteria +defined by the caller are filtered based on the user privileges. + +=head1 METHODS + +=head2 new + +=over + +=item B<Description> + +Create a Bugzilla::Search object. + +=item B<Params> + +=over + +=item C<fields> + +An arrayref representing the bug attributes for which data is desired. +Legal attributes are listed in the fielddefs DB table. At least one field +must be defined, typically the 'bug_id' field. + +=item C<params> + +A hashref representing search criteria. Each key => value pair represents +a search criteria, where the key is the search field and the value is the +value for this field. At least one search criteria must be defined if the +'search_allow_no_criteria' parameter is turned off, else an error is thrown. + +=item C<sharer> + +When a saved search is shared by a user, this is his user ID. + +=item C<user> + +A L<Bugzilla::User> object representing the user to whom the data is addressed. +All security checks are done based on this user object, so it's not safe +to share results of the query with other users as not all users have the +same privileges or have the same role for all bugs in the list. If this +parameter is not defined, then the currently logged in user is taken into +account. If no user is logged in, then only public bugs will be returned. + +=item C<allow_unlimited> + +If set to a true value, the number of bugs retrieved by the query is not +limited. + +=back + +=item B<Returns> + +A L<Bugzilla::Search> object. + +=back + +=head2 data + +=over + +=item B<Description> + +Returns bugs matching search criteria passed to C<new()>. + +=item B<Params> + +None + +=item B<Returns> + +In scalar context, this method returns a reference to a list of bugs. +Each item of the list represents a bug, which is itself a reference to +a list where each item represents a bug attribute, in the same order as +specified in the C<fields> parameter of C<new()>. + +In list context, this methods also returns a reference to a list containing +references to hashes. For each hash, two keys are defined: C<sql> contains +the SQL query which has been executed, and C<time> contains the time spent +to execute the SQL query, in seconds. There can be either a single hash, or +two hashes if two SQL queries have been executed sequentially to get all the +required data. + +=back diff --git a/Bugzilla/Search/Clause.pm b/Bugzilla/Search/Clause.pm index 5f5ea5b50..89210babb 100644 --- a/Bugzilla/Search/Clause.pm +++ b/Bugzilla/Search/Clause.pm @@ -42,6 +42,11 @@ sub children { return $self->{children}; } +sub update_search_args { + my ($self, $search_args) = @_; + # abstract +} + sub joiner { return $_[0]->{joiner} } sub has_translated_conditions { @@ -83,7 +88,7 @@ sub walk_conditions { my ($self, $callback) = @_; foreach my $child (@{ $self->children }) { if ($child->isa('Bugzilla::Search::Condition')) { - $callback->($child); + $callback->($self, $child); } else { $child->walk_conditions($callback); diff --git a/Bugzilla/Search/ClauseGroup.pm b/Bugzilla/Search/ClauseGroup.pm new file mode 100644 index 000000000..5b437afec --- /dev/null +++ b/Bugzilla/Search/ClauseGroup.pm @@ -0,0 +1,96 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Search::ClauseGroup; + +use strict; + +use base qw(Bugzilla::Search::Clause); + +use Bugzilla::Error; +use Bugzilla::Search::Condition qw(condition); +use Bugzilla::Util qw(trick_taint); +use List::MoreUtils qw(uniq); + +use constant UNSUPPORTED_FIELDS => qw( + attach_data.thedata + classification + commenter + component + longdescs.count + product + owner_idle_time +); + +sub new { + my ($class) = @_; + my $self = bless({ joiner => 'AND' }, $class); + # Add a join back to the bugs table which will be used to group conditions + # for this clause + my $condition = Bugzilla::Search::Condition->new({}); + $condition->translated({ + joins => [{ + table => 'bugs', + as => 'bugs_g0', + from => 'bug_id', + to => 'bug_id', + extra => [], + }], + term => '1 = 1', + }); + $self->SUPER::add($condition); + $self->{group_condition} = $condition; + return $self; +} + +sub add { + my ($self, @args) = @_; + my $field = scalar(@args) == 3 ? $args[0] : $args[0]->{field}; + + # We don't support nesting of conditions under this clause + if (scalar(@args) == 1 && !$args[0]->isa('Bugzilla::Search::Condition')) { + ThrowUserError('search_grouped_invalid_nesting'); + } + + # Ensure all conditions use the same field + if (!$self->{_field}) { + $self->{_field} = $field; + } elsif ($field ne $self->{_field}) { + ThrowUserError('search_grouped_field_mismatch'); + } + + # Unsupported fields + if (grep { $_ eq $field } UNSUPPORTED_FIELDS ) { + ThrowUserError('search_grouped_field_invalid', { field => $field }); + } + + $self->SUPER::add(@args); +} + +sub update_search_args { + my ($self, $search_args) = @_; + + # No need to change things if there's only one child condition + return unless scalar(@{ $self->children }) > 1; + + # we want all the terms to use the same join table + if (!exists $self->{_first_chart_id}) { + $self->{_first_chart_id} = $search_args->{chart_id}; + } else { + $search_args->{chart_id} = $self->{_first_chart_id}; + } + + my $suffix = '_g' . $self->{_first_chart_id}; + $self->{group_condition}->{translated}->{joins}->[0]->{as} = "bugs$suffix"; + + $search_args->{full_field} =~ s/^bugs\./bugs$suffix\./; + + $search_args->{table_suffix} = $suffix; + $search_args->{bugs_table} = "bugs$suffix"; +} + +1; diff --git a/Bugzilla/Search/Quicksearch.pm b/Bugzilla/Search/Quicksearch.pm index fd9d796d1..00a970292 100644 --- a/Bugzilla/Search/Quicksearch.pm +++ b/Bugzilla/Search/Quicksearch.pm @@ -116,6 +116,17 @@ use constant FIELD_OPERATOR => { owner_idle_time => 'greaterthan', }; +# Mappings for operators symbols to support operators other than "substring" +use constant OPERATOR_SYMBOLS => { + ':' => 'substring', + '=' => 'equals', + '!=' => 'notequals', + '>=' => 'greaterthaneq', + '<=' => 'lessthaneq', + '>' => 'greaterthan', + '<' => 'lessthan', +}; + # We might want to put this into localconfig or somewhere use constant PRODUCT_EXCEPTIONS => ( 'row', # [Browser] @@ -151,7 +162,7 @@ sub quicksearch { # Retain backslashes and quotes, to know which strings are quoted, # and which ones are not. - my @words = parse_line('\s+', 1, $searchstring); + my @words = _parse_line('\s+', 1, $searchstring); # If parse_line() returns no data, this means strings are badly quoted. # Rather than trying to guess what the user wanted to do, we throw an error. scalar(@words) @@ -161,6 +172,8 @@ sub quicksearch { ThrowUserError('quicksearch_invalid_query') if ($words[0] =~ /^(?:AND|OR)$/ || $words[$#words] =~ /^(?:AND|OR|NOT)$/); + $fulltext = Bugzilla->user->setting('quicksearch_fulltext') eq 'on' ? 1 : 0; + my (@qswords, @or_group); while (scalar @words) { my $word = shift @words; @@ -187,6 +200,10 @@ sub quicksearch { } unshift(@words, "-$word"); } + # --comment and ++comment disable or enable fulltext searching + elsif ($word =~ /^(--|\+\+)comments?$/i) { + $fulltext = $1 eq '--' ? 0 : 1; + } else { # OR groups words together, as OR has higher precedence than AND. push(@or_group, $word); @@ -203,12 +220,12 @@ sub quicksearch { shift(@qswords) if $bug_status_set; my (@unknownFields, %ambiguous_fields); - $fulltext = Bugzilla->user->setting('quicksearch_fulltext') eq 'on' ? 1 : 0; # Loop over all main-level QuickSearch words. foreach my $qsword (@qswords) { - my @or_operand = parse_line('\|', 1, $qsword); + my @or_operand = _parse_line('\|', 1, $qsword); foreach my $term (@or_operand) { + next unless defined $term; my $negate = substr($term, 0, 1) eq '-'; if ($negate) { $term = substr($term, 1); @@ -221,7 +238,7 @@ sub quicksearch { # Having ruled out the special cases, we may now split # by comma, which is another legal boolean OR indicator. # Remove quotes from quoted words, if any. - @words = parse_line(',', 0, $term); + @words = _parse_line(',', 0, $term); foreach my $word (@words) { if (!_special_field_syntax($word, $negate)) { _default_quicksearch_word($word, $negate); @@ -273,13 +290,36 @@ sub quicksearch { # Parts of quicksearch() # ########################## +sub _parse_line { + my ($delim, $keep, $line) = @_; + return () unless defined $line; + + # parse_line always treats ' as a quote character, making it impossible + # to sanely search for contractions. As this behavour isn't + # configurable, we replace ' with a placeholder to hide it from the + # parser. + + # only treat ' at the start or end of words as quotes + # it's easier to do this in reverse with regexes + $line =~ s/(^|\s|:)'/$1\001/g; + $line =~ s/'($|\s)/\001$1/g; + $line =~ s/\\?'/\000/g; + $line =~ tr/\001/'/; + + my @words = parse_line($delim, $keep, $line); + foreach my $word (@words) { + $word =~ tr/\000/'/ if defined $word; + } + return @words; +} + sub _bug_numbers_only { my $searchstring = shift; my $cgi = Bugzilla->cgi; # Allow separation by comma or whitespace. $searchstring =~ s/[,\s]+/,/g; - if ($searchstring !~ /,/) { + if ($searchstring !~ /,/ && !i_am_webservice()) { # Single bug number; shortcut to show_bug.cgi. print $cgi->redirect( -uri => correct_urlbase() . "show_bug.cgi?id=$searchstring"); @@ -298,9 +338,11 @@ sub _handle_alias { if ($searchstring =~ /^([^,\s]+)$/) { my $alias = $1; # We use this direct SQL because we want quicksearch to be VERY fast. - my $is_alias = Bugzilla->dbh->selectrow_array( - q{SELECT 1 FROM bugs WHERE alias = ?}, undef, $alias); - if ($is_alias) { + my $bug_id = Bugzilla->dbh->selectrow_array( + q{SELECT bug_id FROM bugs WHERE alias = ?}, undef, $alias); + # If the user cannot see the bug or if we are using a webservice, + # do not resolve its alias. + if ($bug_id && Bugzilla->user->can_see_bug($bug_id) && !i_am_webservice()) { $alias = url_quote($alias); print Bugzilla->cgi->redirect( -uri => correct_urlbase() . "show_bug.cgi?id=$alias"); @@ -339,6 +381,7 @@ sub _handle_status_and_resolution { sub _handle_special_first_chars { my ($qsword, $negate) = @_; + return 0 if !defined $qsword || length($qsword) <= 1; my $firstChar = substr($qsword, 0, 1); my $baseWord = substr($qsword, 1); @@ -377,23 +420,31 @@ sub _handle_field_names { # Flag and requestee shortcut if ($or_operand =~ /^(?:flag:)?([^\?]+\?)([^\?]*)$/) { - my ($flagtype, $requestee) = ($1, $2); - addChart('flagtypes.name', 'substring', $flagtype, $negate); - if ($requestee) { - # AND - $chart++; - $and = $or = 0; - addChart('requestees.login_name', 'substring', $requestee, $negate); + # BMO: Do not treat custom fields as flags if value is ? + if ($1 !~ /^cf_/) { + my ($flagtype, $requestee) = ($1, $2); + addChart('flagtypes.name', 'substring', $flagtype, $negate); + if ($requestee) { + # AND + $chart++; + $and = $or = 0; + addChart('requestees.login_name', 'substring', $requestee, $negate); + } + return 1; } - return 1; } # Generic field1,field2,field3:value1,value2 notation. # We have to correctly ignore commas and colons in quotes. - my @field_values = parse_line(':', 1, $or_operand); - if (scalar @field_values == 2) { - my @fields = parse_line(',', 1, $field_values[0]); - my @values = parse_line(',', 1, $field_values[1]); + # Longer operators must be tested first as we don't want single character + # operators such as <, > and = to be tested before <=, >= and !=. + my @operators = sort { length($b) <=> length($a) } keys %{ OPERATOR_SYMBOLS() }; + + foreach my $symbol (@operators) { + my @field_values = _parse_line($symbol, 1, $or_operand); + next unless scalar @field_values == 2; + my @fields = _parse_line(',', 1, $field_values[0]); + my @values = _parse_line(',', 1, $field_values[1]); foreach my $field (@fields) { my $translated = _translate_field_name($field); # Skip and record any unknown fields @@ -410,7 +461,10 @@ sub _handle_field_names { $bug_status_set = 1; } foreach my $value (@values) { - my $operator = FIELD_OPERATOR->{$translated} || 'substring'; + next unless defined $value; + my $operator = FIELD_OPERATOR->{$translated} + || OPERATOR_SYMBOLS->{$symbol} + || 'substring'; # If the string was quoted to protect some special # characters such as commas and colons, we need # to remove quotes. @@ -482,6 +536,7 @@ sub _translate_field_name { sub _special_field_syntax { my ($word, $negate) = @_; + return unless defined($word); # P1-5 Syntax if ($word =~ m/^P(\d+)(?:-(\d+))?$/i) { @@ -517,6 +572,7 @@ sub _special_field_syntax { sub _default_quicksearch_word { my ($word, $negate) = @_; + return unless defined($word); if (!grep { lc($word) eq $_ } PRODUCT_EXCEPTIONS and length($word) > 2) { addChart('product', 'substring', $word, $negate); @@ -535,10 +591,15 @@ sub _default_quicksearch_word { addChart('short_desc', 'substring', $word, $negate); addChart('status_whiteboard', 'substring', $word, $negate); addChart('content', 'matches', _matches_phrase($word), $negate) if $fulltext; + + # BMO Bug 664124 - Include the crash signature (sig:) field in default quicksearches + addChart('cf_crash_signature', 'substring', $word, $negate); } sub _handle_urls { my ($word, $negate) = @_; + return unless defined($word); + # URL field (for IP addrs, host.names, # scheme://urls) if ($word =~ m/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/ diff --git a/Bugzilla/Search/Recent.pm b/Bugzilla/Search/Recent.pm index 5f04b180b..b9f6428d7 100644 --- a/Bugzilla/Search/Recent.pm +++ b/Bugzilla/Search/Recent.pm @@ -53,6 +53,9 @@ use constant VALIDATORS => { use constant UPDATE_COLUMNS => qw(bug_list list_order); +# There's no gain to caching these objects +use constant USE_MEMCACHED => 0; + ################### # DB Manipulation # ################### @@ -65,12 +68,13 @@ sub create { my $user_id = $search->user_id; # Enforce there only being SAVE_NUM_SEARCHES per user. - my $min_id = $dbh->selectrow_array( - 'SELECT id FROM profile_search WHERE user_id = ? ORDER BY id DESC ' - . $dbh->sql_limit(1, SAVE_NUM_SEARCHES), undef, $user_id); - if ($min_id) { - $dbh->do('DELETE FROM profile_search WHERE user_id = ? AND id <= ?', - undef, ($user_id, $min_id)); + my @ids = @{ $dbh->selectcol_arrayref( + "SELECT id FROM profile_search WHERE user_id = ? ORDER BY id", + undef, $user_id) }; + if (scalar(@ids) > SAVE_NUM_SEARCHES) { + splice(@ids, - SAVE_NUM_SEARCHES); + $dbh->do( + "DELETE FROM profile_search WHERE id IN (" . join(',', @ids) . ")"); } $dbh->bz_commit_transaction(); return $search; diff --git a/Bugzilla/Search/Saved.pm b/Bugzilla/Search/Saved.pm index 99194112a..6e1587d20 100644 --- a/Bugzilla/Search/Saved.pm +++ b/Bugzilla/Search/Saved.pm @@ -199,6 +199,7 @@ sub rename_field_value { } $dbh->do("UPDATE $table SET query = ? WHERE $id_field = ?", undef, $query, $id); + Bugzilla->memcached->clear({ table => $table, id => $id }); } $dbh->bz_commit_transaction(); @@ -301,9 +302,8 @@ sub url { return $_[0]->{'query'}; } sub user { my ($self) = @_; - return $self->{user} if defined $self->{user}; - $self->{user} = new Bugzilla::User($self->{userid}); - return $self->{user}; + return $self->{user} ||= + Bugzilla::User->new({ id => $self->{userid}, cache => 1 }); } ############ diff --git a/Bugzilla/Send/Sendmail.pm b/Bugzilla/Send/Sendmail.pm new file mode 100644 index 000000000..9513134f4 --- /dev/null +++ b/Bugzilla/Send/Sendmail.pm @@ -0,0 +1,95 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Send::Sendmail; + +use strict; + +use base qw(Email::Send::Sendmail); + +use Return::Value; +use Symbol qw(gensym); + +sub send { + my ($class, $message, @args) = @_; + my $mailer = $class->_find_sendmail; + + return failure "Couldn't find 'sendmail' executable in your PATH" + ." and Email::Send::Sendmail::SENDMAIL is not set" + unless $mailer; + + return failure "Found $mailer but cannot execute it" + unless -x $mailer; + + local $SIG{'CHLD'} = 'DEFAULT'; + + my $pipe = gensym; + + open($pipe, "| $mailer -t -oi @args") + || return failure "Error executing $mailer: $!"; + print($pipe $message->as_string) + || return failure "Error printing via pipe to $mailer: $!"; + unless (close $pipe) { + return failure "error when closing pipe to $mailer: $!" if $!; + my ($error_message, $is_transient) = _map_exitcode($? >> 8); + if (Bugzilla->params->{'use_mailer_queue'}) { + # Return success for errors which are fatal so Bugzilla knows to + # remove them from the queue + if ($is_transient) { + return failure "error when closing pipe to $mailer: $error_message"; + } else { + warn "error when closing pipe to $mailer: $error_message\n"; + return success; + } + } else { + return failure "error when closing pipe to $mailer: $error_message"; + } + } + return success; +} + +sub _map_exitcode { + # Returns (error message, is_transient) + # from the sendmail source (sendmail/sysexit.h) + my $code = shift; + if ($code == 64) { + return ("Command line usage error (EX_USAGE)", 1); + } elsif ($code == 65) { + return ("Data format error (EX_DATAERR)", 1); + } elsif ($code == 66) { + return ("Cannot open input (EX_NOINPUT)", 1); + } elsif ($code == 67) { + return ("Addressee unknown (EX_NOUSER)", 0); + } elsif ($code == 68) { + return ("Host name unknown (EX_NOHOST)", 0); + } elsif ($code == 69) { + return ("Service unavailable (EX_UNAVAILABLE)", 1); + } elsif ($code == 70) { + return ("Internal software error (EX_SOFTWARE)", 1); + } elsif ($code == 71) { + return ("System error (EX_OSERR)", 1); + } elsif ($code == 72) { + return ("Critical OS file missing (EX_OSFILE)", 1); + } elsif ($code == 73) { + return ("Can't create output file (EX_CANTCREAT)", 1); + } elsif ($code == 74) { + return ("Input/output error (EX_IOERR)", 1); + } elsif ($code == 75) { + return ("Temp failure (EX_TEMPFAIL)", 1); + } elsif ($code == 76) { + return ("Remote error in protocol (EX_PROTOCOL)", 1); + } elsif ($code == 77) { + return ("Permission denied (EX_NOPERM)", 1); + } elsif ($code == 78) { + return ("Configuration error (EX_CONFIG)", 1); + } else { + return ("Unknown Error ($code)", 1); + } +} + +1; + diff --git a/Bugzilla/Sentry.pm b/Bugzilla/Sentry.pm new file mode 100644 index 000000000..ce45d4823 --- /dev/null +++ b/Bugzilla/Sentry.pm @@ -0,0 +1,322 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Sentry; + +use strict; +use warnings; + +use base qw(Exporter); +our @EXPORT = qw( + sentry_handle_error + sentry_should_notify +); + +use Carp; +use Data::Dumper; +use DateTime; +use File::Temp; +use LWP::UserAgent; +use Sys::Hostname; +use URI; + +use Bugzilla::Constants; +use Bugzilla::RNG qw(irand); +use Bugzilla::Util; +use Bugzilla::WebService::Constants; + +use constant CONFIG => { + # 'codes' lists the code-errors which are sent to sentry + codes => [qw( + bug_error + chart_datafile_corrupt + chart_dir_nonexistent + chart_file_open_fail + illegal_content_type_method + jobqueue_insert_failed + ldap_bind_failed + mail_send_error + template_error + token_generation_error + )], + + # any error/warning messages matching these regex's will not be logged or + # sent to sentry + ignore => [ + qr/^compiled template :\s*$/, + qr/^Use of uninitialized value \$compiled in concatenation \(\.\) or string/, + ], + + # any error/warning messages matching these regex's will be logged but not + # sent to sentry + sentry_ignore => [ + qr/Software caused connection abort/, + qr/Could not check out .*\/cvsroot/, + qr/Unicode character \S+ is illegal/, + qr/Lost connection to MySQL server during query/, + qr/Call me again when you have some data to chart/, + qr/relative paths are not allowed/, + ], + + # (ab)use the logger to classify error/warning types + logger => [ + { + match => [ + qr/DBD::mysql/, + qr/Can't connect to the database/, + ], + logger => 'database_error', + }, + { + match => [ qr/PatchReader/ ], + logger => 'patchreader', + }, + { + match => [ qr/Use of uninitialized value/ ], + logger => 'uninitialized_warning', + }, + ], +}; + +sub sentry_generate_id { + return sprintf('%04x%04x%04x%04x%04x%04x%04x%04x', + irand(0xffff), irand(0xffff), + irand(0xffff), + irand(0x0fff) | 0x4000, + irand(0x3fff) | 0x8000, + irand(0xffff), irand(0xffff), irand(0xffff) + ); +} + +sub sentry_should_notify { + my $code_error = shift; + return grep { $_ eq $code_error } @{ CONFIG->{codes} }; +} + +sub sentry_handle_error { + my $level = shift; + my @message = split(/\n/, shift); + my $id = sentry_generate_id(); + + my $is_error = $level eq 'error'; + if ($level ne 'error' && $level ne 'warning') { + # it's a code-error + return 0 unless sentry_should_notify($level); + $is_error = 1; + $level = 'error'; + } + + # build traceback + my $traceback; + { + # for now don't show function arguments, in case they contain + # confidential data. waiting on bug 700683 + #local $Carp::MaxArgLen = 256; + #local $Carp::MaxArgNums = 0; + local $Carp::MaxArgNums = -1; + local $Carp::CarpInternal{'CGI::Carp'} = 1; + local $Carp::CarpInternal{'Bugzilla::Error'} = 1; + local $Carp::CarpInternal{'Bugzilla::Sentry'} = 1; + $traceback = trim(Carp::longmess()); + } + + # strip timestamp + foreach my $line (@message) { + $line =~ s/^\[[^\]]+\] //; + } + my $message = join(" ", map { trim($_) } grep { $_ ne '' } @message); + + # message content filtering + foreach my $re (@{ CONFIG->{ignore} }) { + return 0 if $message =~ $re; + } + + # determine logger + my $logger; + foreach my $config (@{ CONFIG->{logger} }) { + foreach my $re (@{ $config->{match} }) { + if ($message =~ $re) { + $logger = $config->{logger}; + last; + } + } + last if $logger; + } + $logger ||= $level; + + # don't send to sentry unless configured + my $send_to_sentry = Bugzilla->params->{sentry_uri} ? 1 : 0; + + # web service filtering + if ($send_to_sentry + && (Bugzilla->error_mode == ERROR_MODE_DIE_SOAP_FAULT || Bugzilla->error_mode == ERROR_MODE_JSON_RPC)) + { + my ($code) = $message =~ /^(-?\d+): /; + if ($code + && !($code == ERROR_UNKNOWN_FATAL || $code == ERROR_UNKNOWN_TRANSIENT)) + { + $send_to_sentry = 0; + } + } + + # message content filtering + if ($send_to_sentry) { + foreach my $re (@{ CONFIG->{sentry_ignore} }) { + if ($message =~ $re) { + $send_to_sentry = 0; + last; + } + } + } + + # for now, don't send patchreader errors to sentry + $send_to_sentry = 0 + if $logger eq 'patchreader'; + + # log to apache's error_log + if ($send_to_sentry) { + _write_to_error_log("$message [#$id]", $is_error); + } else { + $traceback =~ s/\n/ /g; + _write_to_error_log("$message $traceback", $is_error); + } + + return 0 unless $send_to_sentry; + + my $user_data = undef; + eval { + my $user = Bugzilla->user; + if ($user->id) { + $user_data = { + id => $user->login, + name => $user->name, + }; + } + }; + + my $uri = URI->new(Bugzilla->cgi->self_url); + $uri->query(undef); + + foreach my $field (qw( QUERY_STRING REQUEST_URI HTTP_REFERER )) { + $ENV{$field} =~ s/\b((?:Bugzilla_password|password)=)[^ &]+/$1*/gi + if exists $ENV{$field}; + } + + my $data = { + event_id => $id, + message => $message, + timestamp => DateTime->now->iso8601(), + level => $level, + platform => 'Other', + logger => $logger, + server_name => hostname(), + 'sentry.interfaces.User' => $user_data, + 'sentry.interfaces.Http' => { + url => $uri->as_string, + method => $ENV{REQUEST_METHOD}, + query_string => $ENV{QUERY_STRING}, + env => \%ENV, + }, + extra => { + stacktrace => $traceback, + }, + }; + + my $fh = File::Temp->new( UNLINK => 0 ); + if (!$fh) { + warn "Failed to create temp file: $!\n"; + return; + } + print $fh Dumper($data); + close($fh) or die $!; + my $filename = $fh->filename; + + my $command = bz_locations()->{'cgi_path'} . "/sentry.pl '$filename' &"; + system($command); + return 1; +} + +sub _write_to_error_log { + my ($message, $is_error) = @_; + if ($ENV{MOD_PERL}) { + require Apache2::Log; + if ($is_error) { + Apache2::ServerRec::log_error($message); + } else { + Apache2::ServerRec::warn($message); + } + } else { + print STDERR "$message\n"; + } +} + +# lifted from Bugzilla::Error +sub _in_eval { + my $in_eval = 0; + for (my $stack = 1; my $sub = (caller($stack))[3]; $stack++) { + last if $sub =~ /^ModPerl/; + last if $sub =~ /^Bugzilla::Template/; + $in_eval = 1 if $sub =~ /^\(eval\)/; + } + return $in_eval; +} + +sub _sentry_die_handler { + my $message = shift; + $message =~ s/^undef error - //; + + # avoid recursion, and check for CGI::Carp::die failures + my $in_cgi_carp_die = 0; + for (my $stack = 1; my $sub = (caller($stack))[3]; $stack++) { + return if $sub =~ /:_sentry_die_handler$/; + $in_cgi_carp_die = 1 if $sub =~ /CGI::Carp::die$/; + } + + return if _in_eval(); + + # mod_perl overrides exit to call die with this string + exit if $message =~ /\bModPerl::Util::exit\b/; + + my $nested_error = ''; + my $is_compilation_failure = $message =~ /\bcompilation (aborted|failed)\b/i; + + # if we are called via CGI::Carp::die chances are something is seriously + # wrong, so skip trying to use ThrowTemplateError + if (!$in_cgi_carp_die && !$is_compilation_failure) { + eval { Bugzilla::Error::ThrowTemplateError($message) }; + $nested_error = $@ if $@; + } + + if ($is_compilation_failure || + $in_cgi_carp_die || + ($nested_error && $nested_error !~ /\bModPerl::Util::exit\b/) + ) { + sentry_handle_error('error', $message); + + # and call the normal error management + # (ISE for web pages, error response for web services, etc) + CORE::die($message); + } + exit; +} + +sub install_sentry_handler { + require CGI::Carp; + CGI::Carp::set_die_handler(\&_sentry_die_handler); + $main::SIG{__WARN__} = sub { + return if _in_eval(); + sentry_handle_error('warning', shift); + }; +} + +BEGIN { + if ($ENV{SCRIPT_NAME} || $ENV{MOD_PERL}) { + install_sentry_handler(); + } +} + +1; diff --git a/Bugzilla/Status.pm b/Bugzilla/Status.pm index ffef600de..4a82f68d0 100644 --- a/Bugzilla/Status.pm +++ b/Bugzilla/Status.pm @@ -126,11 +126,21 @@ sub _check_value { sub BUG_STATE_OPEN { my $dbh = Bugzilla->dbh; - my $cache = Bugzilla->request_cache; - $cache->{status_bug_state_open} ||= - $dbh->selectcol_arrayref('SELECT value FROM bug_status - WHERE is_open = 1'); - return @{ $cache->{status_bug_state_open} }; + my $request_cache = Bugzilla->request_cache; + my $cache_key = 'status_bug_state_open'; + return @{ $request_cache->{$cache_key} } + if exists $request_cache->{$cache_key}; + + my $rows = Bugzilla->memcached->get_config({ key => $cache_key }); + if (!$rows) { + $rows = $dbh->selectcol_arrayref( + 'SELECT value FROM bug_status WHERE is_open = 1' + ); + Bugzilla->memcached->set_config({ key => $cache_key, data => $rows }); + } + + $request_cache->{$cache_key} = $rows; + return @$rows; } # Tells you whether or not the argument is a valid "open" state. diff --git a/Bugzilla/Template.pm b/Bugzilla/Template.pm index 7fd3f0e8d..9bd0c51bd 100644 --- a/Bugzilla/Template.pm +++ b/Bugzilla/Template.pm @@ -51,9 +51,11 @@ use Bugzilla::Token; use Cwd qw(abs_path); use MIME::Base64; use Date::Format (); +use Digest::MD5 qw(md5_hex); use File::Basename qw(basename dirname); use File::Find; use File::Path qw(rmtree mkpath); +use File::Slurp; use File::Spec; use IO::Dir; use List::MoreUtils qw(firstidx); @@ -154,9 +156,10 @@ sub get_format { # If you want to modify this routine, read the comments carefully sub quoteUrls { - my ($text, $bug, $comment, $user) = @_; + my ($text, $bug, $comment, $user, $bug_link_func) = @_; return $text unless $text; $user ||= Bugzilla->user; + $bug_link_func ||= \&get_bug_link; # We use /g for speed, but uris can have other things inside them # (http://foo/bug#3 for example). Filtering that out filters valid @@ -168,6 +171,10 @@ sub quoteUrls { # until we require Perl 5.13.9 or newer. no warnings 'utf8'; + # If the comment is already wrapped, we should ignore newlines when + # looking for matching regexps. Else we should take them into account. + my $s = ($comment && $comment->already_wrapped) ? qr/\s/ : qr/\h/; + # However, note that adding the title (for buglinks) can affect things # In particular, attachment matches go before bug titles, so that titles # with 'attachment 1' don't double match. @@ -206,7 +213,7 @@ sub quoteUrls { map { qr/$_/ } grep($_, Bugzilla->params->{'urlbase'}, Bugzilla->params->{'sslbase'})) . ')'; $text =~ s~\b(${urlbase_re}\Qshow_bug.cgi?id=\E([0-9]+)(\#c([0-9]+))?)\b - ~($things[$count++] = get_bug_link($3, $1, { comment_num => $5, user => $user })) && + ~($things[$count++] = $bug_link_func->($3, $1, { comment_num => $5, user => $user })) && ("\x{FDD2}" . ($count-1) . "\x{FDD3}") ~egox; @@ -234,7 +241,8 @@ sub quoteUrls { ~<a href=\"mailto:$2\">$1$2</a>~igx; # attachment links - $text =~ s~\b(attachment\s*\#?\s*(\d+)(?:\s+\[details\])?) + # BMO: don't make diff view the default for patches (Bug 652332) + $text =~ s~\b(attachment$s*\#?$s*(\d+)(?:$s+\[diff\])?(?:\s+\[details\])?) ~($things[$count++] = get_attachment_link($2, $1, $user)) && ("\x{FDD2}" . ($count-1) . "\x{FDD3}") ~egmxi; @@ -247,21 +255,21 @@ sub quoteUrls { # Also, we can't use $bug_re?$comment_re? because that will match the # empty string my $bug_word = template_var('terms')->{bug}; - my $bug_re = qr/\Q$bug_word\E\s*\#?\s*(\d+)/i; - my $comment_re = qr/comment\s*\#?\s*(\d+)/i; - $text =~ s~\b($bug_re(?:\s*,?\s*$comment_re)?|$comment_re) + my $bug_re = qr/\Q$bug_word\E$s*\#?$s*(\d+)/i; + my $comment_re = qr/comment$s*\#?$s*(\d+)/i; + $text =~ s~\b($bug_re(?:$s*,?$s*$comment_re)?|$comment_re) ~ # We have several choices. $1 here is the link, and $2-4 are set # depending on which part matched - (defined($2) ? get_bug_link($2, $1, { comment_num => $3, user => $user }) : + (defined($2) ? $bug_link_func->($2, $1, { comment_num => $3, user => $user }) : "<a href=\"$current_bugurl#c$4\">$1</a>") - ~egox; + ~egx; # Old duplicate markers. These don't use $bug_word because they are old # and were never customizable. $text =~ s~(?<=^\*\*\*\ This\ bug\ has\ been\ marked\ as\ a\ duplicate\ of\ ) (\d+) (?=\ \*\*\*\Z) - ~get_bug_link($1, $1, { user => $user }) + ~$bug_link_func->($1, $1, { user => $user }) ~egmx; # Now remove the encoding hacks in reverse order @@ -278,7 +286,7 @@ sub get_attachment_link { my $dbh = Bugzilla->dbh; $user ||= Bugzilla->user; - my $attachment = new Bugzilla::Attachment($attachid); + my $attachment = new Bugzilla::Attachment({ id => $attachid, cache => 1 }); if ($attachment) { my $title = ""; @@ -295,19 +303,21 @@ sub get_attachment_link { $title = html_quote(clean_text($title)); $link_text =~ s/ \[details\]$//; + $link_text =~ s/ \[diff\]$//; my $linkval = "attachment.cgi?id=$attachid"; - # If the attachment is a patch, try to link to the diff rather - # than the text, by default. + # If the attachment is a patch and patch_viewer feature is + # enabled, add link to the diff. my $patchlink = ""; if ($attachment->ispatch and Bugzilla->feature('patch_viewer')) { - $patchlink = '&action=diff'; + $patchlink = qq| <a href="${linkval}&action=diff" title="$title">[diff]</a>|; } # Whitespace matters here because these links are in <pre> tags. return qq|<span class="$className">| - . qq|<a href="${linkval}${patchlink}" name="attach_${attachid}" title="$title">$link_text</a>| + . qq|<a href="${linkval}" name="attach_${attachid}" title="$title">$link_text</a>| . qq| <a href="${linkval}&action=edit" title="$title">[details]</a>| + . qq|${patchlink}| . qq|</span>|; } else { @@ -328,8 +338,8 @@ sub get_bug_link { $options->{user} ||= Bugzilla->user; my $dbh = Bugzilla->dbh; - if (defined $bug) { - $bug = blessed($bug) ? $bug : new Bugzilla::Bug($bug); + if (defined $bug && $bug ne '') { + $bug = blessed($bug) ? $bug : new Bugzilla::Bug({ id => $bug, cache => 1 }); return $link_text if $bug->{error}; } @@ -394,21 +404,18 @@ sub mtime_filter { # Set up the skin CSS cascade: # -# 1. YUI CSS -# 2. Standard Bugzilla stylesheet set (persistent) -# 3. Standard Bugzilla stylesheet set (selectable) -# 4. All third-party "skin" stylesheet sets (selectable) -# 5. Page-specific styles -# 6. Custom Bugzilla stylesheet set (persistent) -# -# "Selectable" skin file sets may be either preferred or alternate. -# Exactly one is preferred, determined by the "skin" user preference. +# 1. standard/global.css +# 2. YUI CSS +# 3. Standard Bugzilla stylesheet set +# 4. Third-party "skin" stylesheet set, per user prefs +# 5. Inline css passed to global/header.html.tmpl +# 6. Custom Bugzilla stylesheet set + sub css_files { my ($style_urls, $yui, $yui_css) = @_; - - # global.css goes on every page, and so does IE-fixes.css. - my @requested_css = ('skins/standard/global.css', @$style_urls, - 'skins/standard/IE-fixes.css'); + + # global.css belongs on every page + my @requested_css = ( 'skins/standard/global.css', @$style_urls ); my @yui_required_css; foreach my $yui_name (@$yui) { @@ -419,21 +426,18 @@ sub css_files { my @css_sets = map { _css_link_set($_) } @requested_css; - my %by_type = (standard => [], alternate => {}, skin => [], custom => []); + my %by_type = (standard => [], skin => [], custom => []); foreach my $set (@css_sets) { foreach my $key (keys %$set) { - if ($key eq 'alternate') { - foreach my $alternate_skin (keys %{ $set->{alternate} }) { - my $files = $by_type{alternate}->{$alternate_skin} ||= []; - push(@$files, $set->{alternate}->{$alternate_skin}); - } - } - else { - push(@{ $by_type{$key} }, $set->{$key}); - } + push(@{ $by_type{$key} }, $set->{$key}); } } - + + # build unified + $by_type{unified_standard_skin} = _concatenate_css($by_type{standard}, + $by_type{skin}); + $by_type{unified_custom} = _concatenate_css($by_type{custom}); + return \%by_type; } @@ -441,42 +445,135 @@ sub _css_link_set { my ($file_name) = @_; my %set = (standard => mtime_filter($file_name)); - - # We use (^|/) to allow Extensions to use the skins system if they - # want. - if ($file_name !~ m{(^|/)skins/standard/}) { + + # We use (?:^|/) to allow Extensions to use the skins system if they want. + if ($file_name !~ m{(?:^|/)skins/standard/}) { return \%set; } - - my $skin_user_prefs = Bugzilla->user->settings->{skin}; + + my $skin = Bugzilla->user->settings->{skin}->{value}; my $cgi_path = bz_locations()->{'cgi_path'}; - # If the DB is not accessible, user settings are not available. - my $all_skins = $skin_user_prefs ? $skin_user_prefs->legal_values : []; - my %skin_urls; - foreach my $option (@$all_skins) { - next if $option eq 'standard'; - my $skin_file_name = $file_name; - $skin_file_name =~ s{(^|/)skins/standard/}{skins/contrib/$option/}; - if (my $mtime = _mtime("$cgi_path/$skin_file_name")) { - $skin_urls{$option} = mtime_filter($skin_file_name, $mtime); - } + my $skin_file_name = $file_name; + $skin_file_name =~ s{(?:^|/)skins/standard/}{skins/contrib/$skin/}; + if (my $mtime = _mtime("$cgi_path/$skin_file_name")) { + $set{skin} = mtime_filter($skin_file_name, $mtime); } - $set{alternate} = \%skin_urls; - - my $skin = $skin_user_prefs->{'value'}; - if ($skin ne 'standard' and defined $set{alternate}->{$skin}) { - $set{skin} = delete $set{alternate}->{$skin}; - } - + my $custom_file_name = $file_name; - $custom_file_name =~ s{(^|/)skins/standard/}{skins/custom/}; + $custom_file_name =~ s{(?:^|/)skins/standard/}{skins/custom/}; if (my $custom_mtime = _mtime("$cgi_path/$custom_file_name")) { $set{custom} = mtime_filter($custom_file_name, $custom_mtime); } - + return \%set; } +sub _concatenate_css { + my @sources = map { @$_ } @_; + return unless @sources; + + my %files = + map { + (my $file = $_) =~ s/(^[^\?]+)\?.+/$1/; + $_ => $file; + } @sources; + + my $cgi_path = bz_locations()->{cgi_path}; + my $skins_path = bz_locations()->{assetsdir}; + + # build minified files + my @minified; + foreach my $source (@sources) { + next unless -e "$cgi_path/$files{$source}"; + my $file = $skins_path . '/' . md5_hex($source) . '.css'; + if (!-e $file) { + my $content = read_file("$cgi_path/$files{$source}"); + + # minify + $content =~ s{/\*.*?\*/}{}sg; # comments + $content =~ s{(^\s+|\s+$)}{}mg; # leading/trailing whitespace + $content =~ s{\n}{}g; # single line + + # rewrite urls + $content =~ s{url\(([^\)]+)\)}{_css_url_rewrite($source, $1)}eig; + + write_file($file, "/* $files{$source} */\n" . $content . "\n"); + } + push @minified, $file; + } + + # concat files + my $file = $skins_path . '/' . md5_hex(join(' ', @sources)) . '.css'; + if (!-e $file) { + my $content = ''; + foreach my $source (@minified) { + $content .= read_file($source); + } + write_file($file, $content); + } + + $file =~ s/^\Q$cgi_path\E\///o; + return mtime_filter($file); +} + +sub _css_url_rewrite { + my ($source, $url) = @_; + # rewrite relative urls as the unified stylesheet lives in a different + # directory from the source + $url =~ s/(^['"]|['"]$)//g; + return $url if substr($url, 0, 1) eq '/'; + return 'url(../../' . dirname($source) . '/' . $url . ')'; +} + +sub _concatenate_js { + return @_ unless CONCATENATE_ASSETS; + my ($sources) = @_; + return [] unless $sources; + $sources = ref($sources) ? $sources : [ $sources ]; + + my %files = + map { + (my $file = $_) =~ s/(^[^\?]+)\?.+/$1/; + $_ => $file; + } @$sources; + + my $cgi_path = bz_locations()->{cgi_path}; + my $skins_path = bz_locations()->{assetsdir}; + + # build minified files + my @minified; + foreach my $source (@$sources) { + next unless -e "$cgi_path/$files{$source}"; + my $file = $skins_path . '/' . md5_hex($source) . '.js'; + if (!-e $file) { + my $content = read_file("$cgi_path/$files{$source}"); + + # minimal minification + $content =~ s#/\*.*?\*/##sg; # block comments + $content =~ s#(^ +| +$)##gm; # leading/trailing spaces + $content =~ s#^//.+$##gm; # single line comments + $content =~ s#\n{2,}#\n#g; # blank lines + $content =~ s#(^\s+|\s+$)##g; # whitespace at the start/end of file + + write_file($file, "/* $files{$source} */\n" . $content . "\n"); + } + push @minified, $file; + } + + # concat files + my $file = $skins_path . '/' . md5_hex(join(' ', @$sources)) . '.js'; + if (!-e $file) { + my $content = ''; + foreach my $source (@minified) { + $content .= read_file($source); + } + write_file($file, $content); + } + + $file =~ s/^\Q$cgi_path\E\///o; + return [ $file ]; +} + # YUI dependency resolution sub yui_resolve_deps { my ($yui, $yui_deps) = @_; @@ -553,10 +650,9 @@ $Template::Stash::SCALAR_OPS->{ 0 } = $Template::Stash::SCALAR_OPS->{ truncate } = sub { my ($string, $length, $ellipsis) = @_; - $ellipsis ||= ""; - return $string if !$length || length($string) <= $length; - + + $ellipsis ||= ''; my $strlen = $length - length($ellipsis); my $newstr = substr($string, 0, $strlen) . $ellipsis; return $newstr; @@ -612,6 +708,10 @@ sub create { COMPILE_DIR => bz_locations()->{'template_cache'}, + # Don't check for a template update until 1 hour has passed since the + # last check. + STAT_TTL => 60 * 60, + # Initialize templates (f.e. by loading plugins like Hook). PRE_PROCESS => ["global/initialize.none.tmpl"], @@ -663,6 +763,18 @@ sub create { $var =~ s/>/\\x3e/g; return $var; }, + + # Sadly, different to the above. See http://www.json.org/ + # for details. + json => sub { + my ($var) = @_; + $var =~ s/([\\\"\/])/\\$1/g; + $var =~ s/\n/\\n/g; + $var =~ s/\r/\\r/g; + $var =~ s/\f/\\f/g; + $var =~ s/\t/\\t/g; + return $var; + }, # Converts data to base64 base64 => sub { @@ -839,9 +951,7 @@ sub create { # (Wrapping the message in the WebService is unnecessary # and causes awkward things like \n's appearing in error # messages in JSON-RPC.) - unless (Bugzilla->usage_mode == USAGE_MODE_JSON - or Bugzilla->usage_mode == USAGE_MODE_XMLRPC) - { + unless (i_am_webservice()) { $var = wrap_comment($var, 72); } $var =~ s/\ / /g; @@ -891,14 +1001,9 @@ sub create { # Currently logged in user, if any # If an sudo session is in progress, this is the user we're faking 'user' => sub { return Bugzilla->user; }, - + # Currenly active language - # XXX Eventually this should probably be replaced with something - # like Bugzilla->language. - 'current_language' => sub { - my ($language) = include_languages(); - return $language; - }, + 'current_language' => sub { return Bugzilla->current_language; }, # If an sudo session is in progress, this is the user who # started the session. @@ -909,7 +1014,7 @@ sub create { # Allow templates to access docs url with users' preferred language 'docs_urlbase' => sub { - my ($language) = include_languages(); + my $language = Bugzilla->current_language; my $docs_urlbase = Bugzilla->params->{'docs_urlbase'}; $docs_urlbase =~ s/\%lang\%/$language/; return $docs_urlbase; @@ -938,9 +1043,18 @@ sub create { Bugzilla->fields({ by_name => 1 }); return $cache->{template_bug_fields}; }, - + + # A general purpose cache to store rendered templates for reuse. + # Make sure to not mix language-specific data. + 'template_cache' => sub { + my $cache = Bugzilla->request_cache->{template_cache} ||= {}; + $cache->{users} ||= {}; + return $cache; + }, + 'css_files' => \&css_files, yui_resolve_deps => \&yui_resolve_deps, + concatenate_js => \&_concatenate_js, # Whether or not keywords are enabled, in this Bugzilla. 'use_keywords' => sub { return Bugzilla::Keyword->any_exist; }, @@ -985,8 +1099,17 @@ sub create { 'default_authorizer' => sub { return Bugzilla::Auth->new() }, }, }; - - local $Template::Config::CONTEXT = 'Bugzilla::Template::Context'; + # Use a per-process provider to cache compiled templates in memory across + # requests. + my $provider_key = join(':', @{ $config->{INCLUDE_PATH} }); + my $shared_providers = Bugzilla->process_cache->{shared_providers} ||= {}; + $shared_providers->{$provider_key} ||= Template::Provider->new($config); + $config->{LOAD_TEMPLATES} = [ $shared_providers->{$provider_key} ]; + + # BMO - use metrics subclass + local $Template::Config::CONTEXT = Bugzilla->metrics_enabled() + ? 'Bugzilla::Metrics::Template::Context' + : 'Bugzilla::Template::Context'; Bugzilla::Hook::process('template_before_create', { config => $config }); my $template = $class->new($config) @@ -1066,6 +1189,9 @@ sub precompile_templates { # If anything created a Template object before now, clear it out. delete Bugzilla->request_cache->{template}; + # Clear out the cached Provider object + Bugzilla->process_cache->{shared_providers} = undef; + print install_string('done') . "\n" if $output; } diff --git a/Bugzilla/Template/Context.pm b/Bugzilla/Template/Context.pm index 7923603e5..b3e3446f6 100644 --- a/Bugzilla/Template/Context.pm +++ b/Bugzilla/Template/Context.pm @@ -95,6 +95,14 @@ sub stash { return $stash; } +sub filter { + my ($self, $name, $args) = @_; + # If we pass an alias for the filter name, the filter code is cached + # instead of looking for it at each call. + # If the filter has arguments, then we can't cache it. + $self->SUPER::filter($name, $args, $args ? undef : $name); +} + # We need a DESTROY sub for the same reason that Bugzilla::CGI does. sub DESTROY { my $self = shift; diff --git a/Bugzilla/Token.pm b/Bugzilla/Token.pm index 9c2242f63..24df470ac 100644 --- a/Bugzilla/Token.pm +++ b/Bugzilla/Token.pm @@ -109,6 +109,8 @@ sub IssueEmailChangeToken { $vars->{'newemailaddress'} = $new_email . $email_suffix; $vars->{'expiration_ts'} = ctime($token_ts + MAX_TOKEN_AGE * 86400); $vars->{'token'} = $token; + # For SecureMail extension + $vars->{'to_user'} = $user; $vars->{'emailaddress'} = $old_email . $email_suffix; my $message; diff --git a/Bugzilla/User.pm b/Bugzilla/User.pm index 0bc49d9b1..259a7ea90 100644 --- a/Bugzilla/User.pm +++ b/Bugzilla/User.pm @@ -50,9 +50,12 @@ use Bugzilla::Product; use Bugzilla::Classification; use Bugzilla::Field; use Bugzilla::Group; +use Bugzilla::Hook; +use Bugzilla::BugUserLastVisit; use DateTime::TimeZone; use List::Util qw(max); +use List::MoreUtils qw(any); use Scalar::Util qw(blessed); use Storable qw(dclone); use URI; @@ -91,16 +94,21 @@ use constant DB_TABLE => 'profiles'; # that you passed in for "name" to new(). That's because historically # Bugzilla::User used "name" for the realname field. This should be # fixed one day. -use constant DB_COLUMNS => ( - 'profiles.userid', - 'profiles.login_name', - 'profiles.realname', - 'profiles.mybugslink AS showmybugslink', - 'profiles.disabledtext', - 'profiles.disable_mail', - 'profiles.extern_id', - 'profiles.is_enabled', -); +sub DB_COLUMNS { + my $dbh = Bugzilla->dbh; + return ( + 'profiles.userid', + 'profiles.login_name', + 'profiles.realname', + 'profiles.mybugslink AS showmybugslink', + 'profiles.disabledtext', + 'profiles.disable_mail', + 'profiles.extern_id', + 'profiles.is_enabled', + $dbh->sql_date_format('last_seen_date', '%Y-%m-%d') . ' AS last_seen_date', + ), +} + use constant NAME_FIELD => 'login_name'; use constant ID_FIELD => 'userid'; use constant LIST_ORDER => NAME_FIELD; @@ -144,7 +152,7 @@ sub new { my $class = ref($invocant) || $invocant; my ($param) = @_; - my $user = DEFAULT_USER; + my $user = { %{ DEFAULT_USER() } }; bless ($user, $class); return $user unless $param; @@ -162,7 +170,7 @@ sub super_user { my $class = ref($invocant) || $invocant; my ($param) = @_; - my $user = dclone(DEFAULT_USER); + my $user = { %{ DEFAULT_USER() } }; $user->{groups} = [Bugzilla::Group->get_all]; $user->{bless_groups} = [Bugzilla::Group->get_all]; bless $user, $class; @@ -171,20 +179,25 @@ sub super_user { sub update { my $self = shift; + my $options = shift; + my $changes = $self->SUPER::update(@_); my $dbh = Bugzilla->dbh; if (exists $changes->{login_name}) { - # If we changed the login, silently delete any tokens. - $dbh->do('DELETE FROM tokens WHERE userid = ?', undef, $self->id); + # Delete all the tokens related to the userid + $dbh->do('DELETE FROM tokens WHERE userid = ?', undef, $self->id) + unless $options->{keep_tokens}; # And rederive regex groups $self->derive_regexp_groups(); } # Logout the user if necessary. - Bugzilla->logout_user($self) - if (exists $changes->{login_name} || exists $changes->{disabledtext} - || exists $changes->{cryptpassword}); + Bugzilla->logout_user($self) + if (!$options->{keep_session} + && (exists $changes->{login_name} + || exists $changes->{disabledtext} + || exists $changes->{cryptpassword})); # XXX Can update profiles_activity here as soon as it understands # field names like login_name. @@ -285,6 +298,24 @@ sub set_disabledtext { $_[0]->set('is_enabled', $_[1] ? 0 : 1); } +sub update_last_seen_date { + my $self = shift; + return unless $self->id; + my $dbh = Bugzilla->dbh; + my $date = $dbh->selectrow_array( + 'SELECT ' . $dbh->sql_date_format('NOW()', '%Y-%m-%d')); + + if (!$self->last_seen_date or $date ne $self->last_seen_date) { + $self->{last_seen_date} = $date; + # We don't use the normal update() routine here as we only + # want to update the last_seen_date column, not any other + # pending changes + $dbh->do("UPDATE profiles SET last_seen_date = ? WHERE userid = ?", + undef, $date, $self->id); + Bugzilla->memcached->clear({ table => 'profiles', id => $self->id }); + } +} + ################################################################################ # Methods ################################################################################ @@ -299,6 +330,7 @@ sub is_enabled { $_[0]->{'is_enabled'} ? 1 : 0; } sub showmybugslink { $_[0]->{showmybugslink}; } sub email_disabled { $_[0]->{disable_mail}; } sub email_enabled { !($_[0]->{disable_mail}); } +sub last_seen_date { $_[0]->{last_seen_date}; } sub cryptpassword { my $self = shift; # We don't store it because we never want it in the object (we @@ -431,6 +463,31 @@ sub tags { return $self->{tags}; } +sub bugs_ignored { + my ($self) = @_; + my $dbh = Bugzilla->dbh; + if (!defined $self->{'bugs_ignored'}) { + $self->{'bugs_ignored'} = $dbh->selectall_arrayref( + 'SELECT bugs.bug_id AS id, + bugs.bug_status AS status, + bugs.short_desc AS summary + FROM bugs + INNER JOIN email_bug_ignore + ON bugs.bug_id = email_bug_ignore.bug_id + WHERE user_id = ?', + { Slice => {} }, $self->id); + # Go ahead and load these into the visible bugs cache + # to speed up can_see_bug checks later + $self->visible_bugs([ map { $_->{'id'} } @{ $self->{'bugs_ignored'} } ]); + } + return $self->{'bugs_ignored'}; +} + +sub is_bug_ignored { + my ($self, $bug_id) = @_; + return (grep {$_->{'id'} == $bug_id} @{$self->bugs_ignored}) ? 1 : 0; +} + ########################## # Saved Recent Bug Lists # ########################## @@ -627,56 +684,104 @@ sub groups { return $self->{groups} if defined $self->{groups}; return [] unless $self->id; - my $dbh = Bugzilla->dbh; - my $groups_to_check = $dbh->selectcol_arrayref( - q{SELECT DISTINCT group_id - FROM user_group_map - WHERE user_id = ? AND isbless = 0}, undef, $self->id); + my $user_groups_key = "user_groups." . $self->id; + my $groups = Bugzilla->memcached->get_config({ + key => $user_groups_key + }); - my $rows = $dbh->selectall_arrayref( - "SELECT DISTINCT grantor_id, member_id - FROM group_group_map - WHERE grant_type = " . GROUP_MEMBERSHIP); + if (!$groups) { + my $dbh = Bugzilla->dbh; + my $groups_to_check = $dbh->selectcol_arrayref( + "SELECT DISTINCT group_id + FROM user_group_map + WHERE user_id = ? AND isbless = 0", undef, $self->id); + + my $grant_type_key = 'group_grant_type_' . GROUP_MEMBERSHIP; + my $membership_rows = Bugzilla->memcached->get_config({ + key => $grant_type_key, + }); + if (!$membership_rows) { + $membership_rows = $dbh->selectall_arrayref( + "SELECT DISTINCT grantor_id, member_id + FROM group_group_map + WHERE grant_type = " . GROUP_MEMBERSHIP); + Bugzilla->memcached->set_config({ + key => $grant_type_key, + data => $membership_rows, + }); + } - my %group_membership; - foreach my $row (@$rows) { - my ($grantor_id, $member_id) = @$row; - push (@{ $group_membership{$member_id} }, $grantor_id); - } - - # Let's walk the groups hierarchy tree (using FIFO) - # On the first iteration it's pre-filled with direct groups - # membership. Later on, each group can add its own members into the - # FIFO. Circular dependencies are eliminated by checking - # $checked_groups{$member_id} hash values. - # As a result, %groups will have all the groups we are the member of. - my %checked_groups; - my %groups; - while (scalar(@$groups_to_check) > 0) { - # Pop the head group from FIFO - my $member_id = shift @$groups_to_check; - - # Skip the group if we have already checked it - if (!$checked_groups{$member_id}) { - # Mark group as checked - $checked_groups{$member_id} = 1; - - # Add all its members to the FIFO check list - # %group_membership contains arrays of group members - # for all groups. Accessible by group number. - my $members = $group_membership{$member_id}; - my @new_to_check = grep(!$checked_groups{$_}, @$members); - push(@$groups_to_check, @new_to_check); - - $groups{$member_id} = 1; + my %group_membership; + foreach my $row (@$membership_rows) { + my ($grantor_id, $member_id) = @$row; + push (@{ $group_membership{$member_id} }, $grantor_id); } - } - $self->{groups} = Bugzilla::Group->new_from_list([keys %groups]); + # Let's walk the groups hierarchy tree (using FIFO) + # On the first iteration it's pre-filled with direct groups + # membership. Later on, each group can add its own members into the + # FIFO. Circular dependencies are eliminated by checking + # $checked_groups{$member_id} hash values. + # As a result, %groups will have all the groups we are the member of. + my %checked_groups; + my %groups; + while (scalar(@$groups_to_check) > 0) { + # Pop the head group from FIFO + my $member_id = shift @$groups_to_check; + + # Skip the group if we have already checked it + if (!$checked_groups{$member_id}) { + # Mark group as checked + $checked_groups{$member_id} = 1; + + # Add all its members to the FIFO check list + # %group_membership contains arrays of group members + # for all groups. Accessible by group number. + my $members = $group_membership{$member_id}; + my @new_to_check = grep(!$checked_groups{$_}, @$members); + push(@$groups_to_check, @new_to_check); + + $groups{$member_id} = 1; + } + } + $groups = [ keys %groups ]; + + Bugzilla->memcached->set_config({ + key => $user_groups_key, + data => $groups, + }); + } + $self->{groups} = Bugzilla::Group->new_from_list($groups); return $self->{groups}; } +sub last_visited { + my ($self) = @_; + + return Bugzilla::BugUserLastVisit->match({ user_id => $self->id }); +} + +sub is_involved_in_bug { + my ($self, $bug) = @_; + my $user_id = $self->id; + my $user_login = $self->login; + + return unless $user_id; + return 1 if $user_id == $bug->assigned_to->id; + return 1 if $user_id == $bug->reporter->id; + + if (Bugzilla->params->{'useqacontact'} and $bug->qa_contact) { + return 1 if $user_id == $bug->qa_contact->id; + } + + # BMO - Bug mentors are considered involved with the bug + return 1 if $bug->is_mentor($self); + + return unless $bug->cc; + return any { $user_login eq $_ } @{ $bug->cc }; +} + # It turns out that calling ->id on objects a few hundred thousand # times is pretty slow. (It showed up as a significant time contributor # when profiling xt/search.t.) So we cache the group ids separately from @@ -707,40 +812,48 @@ sub bless_groups { return $self->{'bless_groups'} if defined $self->{'bless_groups'}; return [] unless $self->id; - if ($self->in_group('editusers')) { - # Users having editusers permissions may bless all groups. + if ($self->in_group('admin')) { + # Users having admin permissions may bless all groups. $self->{'bless_groups'} = [Bugzilla::Group->get_all]; return $self->{'bless_groups'}; } + if (Bugzilla->params->{usevisibilitygroups} + && !$self->visible_groups_inherited) { + return []; + } + my $dbh = Bugzilla->dbh; - # Get all groups for the user where: - # + They have direct bless privileges - # + They are a member of a group that inherits bless privs. - my @group_ids = map {$_->id} @{ $self->groups }; - @group_ids = (-1) if !@group_ids; - my $query = - 'SELECT DISTINCT groups.id - FROM groups, user_group_map, group_group_map AS ggm - WHERE user_group_map.user_id = ? - AND ( (user_group_map.isbless = 1 - AND groups.id=user_group_map.group_id) - OR (groups.id = ggm.grantor_id - AND ggm.grant_type = ' . GROUP_BLESS . ' - AND ' . $dbh->sql_in('ggm.member_id', \@group_ids) - . ') )'; - - # If visibilitygroups are used, restrict the set of groups. - if (Bugzilla->params->{'usevisibilitygroups'}) { - return [] if !$self->visible_groups_as_string; - # Users need to see a group in order to bless it. + # Get all groups for the user where they have direct bless privileges. + my $query = " + SELECT DISTINCT group_id + FROM user_group_map + WHERE user_id = ? + AND isbless = 1"; + if (Bugzilla->params->{usevisibilitygroups}) { $query .= " AND " - . $dbh->sql_in('groups.id', $self->visible_groups_inherited); + . $dbh->sql_in('group_id', $self->visible_groups_inherited); + } + + # Get all groups for the user where they are a member of a group that + # inherits bless privs. + my @group_ids = map { $_->id } @{ $self->groups }; + if (@group_ids) { + $query .= " + UNION + SELECT DISTINCT grantor_id + FROM group_group_map + WHERE grant_type = " . GROUP_BLESS . " + AND " . $dbh->sql_in('member_id', \@group_ids); + if (Bugzilla->params->{usevisibilitygroups}) { + $query .= " AND " + . $dbh->sql_in('grantor_id', $self->visible_groups_inherited); + } } my $ids = $dbh->selectcol_arrayref($query, undef, $self->id); - return $self->{'bless_groups'} = Bugzilla::Group->new_from_list($ids); + return $self->{bless_groups} = Bugzilla::Group->new_from_list($ids); } sub in_group { @@ -778,6 +891,14 @@ sub in_group_id { return grep($_->id == $id, @{ $self->groups }) ? 1 : 0; } +# This is a helper to get all groups which have an icon to be displayed +# besides the name of the commenter. +sub groups_with_icon { + my $self = shift; + + return $self->{groups_with_icon} //= [grep { $_->icon_url } @{ $self->groups }]; +} + sub get_products_by_permission { my ($self, $group) = @_; # Make sure $group exists on a per-product basis. @@ -857,6 +978,14 @@ sub visible_bugs { if (@check_ids) { my $dbh = Bugzilla->dbh; my $user_id = $self->id; + + foreach my $id (@check_ids) { + my $orig_id = $id; + detaint_natural($id) + || ThrowCodeError('param_must_be_numeric', { param => $orig_id, + function => 'Bugzilla::User->visible_bugs'}); + } + my $sth; # Speed up the can_see_bug case. if (scalar(@check_ids) == 1) { @@ -1144,24 +1273,6 @@ sub can_set_flag { || $self->in_group_id($flag_type->grant_group_id)) ? 1 : 0; } -sub direct_group_membership { - my $self = shift; - my $dbh = Bugzilla->dbh; - - if (!$self->{'direct_group_membership'}) { - my $gid = $dbh->selectcol_arrayref('SELECT id - FROM groups - INNER JOIN user_group_map - ON groups.id = user_group_map.group_id - WHERE user_id = ? - AND isbless = 0', - undef, $self->id); - $self->{'direct_group_membership'} = Bugzilla::Group->new_from_list($gid); - } - return $self->{'direct_group_membership'}; -} - - # visible_groups_inherited returns a reference to a list of all the groups # whose members are visible to this user. sub visible_groups_inherited { @@ -1277,6 +1388,8 @@ sub derive_regexp_groups { $group_delete->execute($id, $group, GRANT_REGEXP) if $present; } } + + Bugzilla->memcached->clear_config({ key => "user_groups.$id" }); } sub product_responsibilities { @@ -1635,7 +1748,9 @@ our %names_to_events = ( 'attachments.mimetype' => EVT_ATTACHMENT_DATA, 'attachments.ispatch' => EVT_ATTACHMENT_DATA, 'dependson' => EVT_DEPEND_BLOCK, - 'blocked' => EVT_DEPEND_BLOCK); + 'blocked' => EVT_DEPEND_BLOCK, + 'product' => EVT_COMPONENT, + 'component' => EVT_COMPONENT); # Returns true if the user wants mail for a given bug change. # Note: the "+" signs before the constants suppress bareword quoting. @@ -1654,7 +1769,7 @@ sub wants_bug_mail { } else { # Catch-all for any change not caught by a more specific event - $events{+EVT_OTHER} = 1; + $events{+EVT_OTHER} = 1; } # If the user is in a particular role and the value of that role @@ -1719,7 +1834,19 @@ sub wants_bug_mail { if ($wants_mail && $bug->bug_status eq 'UNCONFIRMED') { $wants_mail &= $self->wants_mail([EVT_UNCONFIRMED], $relationship); } - + + # BMO: add a hook to allow custom bugmail filtering + Bugzilla::Hook::process("user_wants_mail", { + user => $self, + wants_mail => \$wants_mail, + bug => $bug, + relationship => $relationship, + fieldDiffs => $fieldDiffs, + comments => $comments, + dep_mail => $dep_mail, + changer => $changer, + }); + return $wants_mail; } @@ -1810,6 +1937,17 @@ sub is_timetracker { return $self->{'is_timetracker'}; } +sub can_tag_comments { + my $self = shift; + + if (!defined $self->{'can_tag_comments'}) { + my $group = Bugzilla->params->{'comment_taggers_group'}; + $self->{'can_tag_comments'} = + ($group && $self->in_group($group)) ? 1 : 0; + } + return $self->{'can_tag_comments'}; +} + sub get_userlist { my $self = shift; @@ -2008,6 +2146,9 @@ sub check_and_send_account_creation_confirmation { ThrowUserError('account_creation_restricted'); } + # BMO - add a hook to allow extra validation prior to account creation. + Bugzilla::Hook::process("user_verify_login", { login => $login }); + # Create and send a token for this new account. require Bugzilla::Token; Bugzilla::Token::issue_new_user_account_token($login); @@ -2199,6 +2340,34 @@ groups. Returns a hashref with tag IDs as key, and a hashref with tag 'id', 'name' and 'bug_count' as value. +=item C<bugs_ignored> + +Returns an array of hashrefs containing information about bugs currently +being ignored by the user. + +Each hashref contains the following information: + +=over + +=item C<id> + +C<int> The id of the bug. + +=item C<status> + +C<string> The current status of the bug. + +=item C<summary> + +C<string> The current summary of the bug. + +=back + +=item C<is_bug_ignored> + +Returns true if the user does not want email notifications for the +specified bug ID, else returns false. + =back =head2 Saved Recent Bug Lists @@ -2363,7 +2532,7 @@ Determines whether or not a user is in the given group by id. Returns an arrayref of L<Bugzilla::Group> objects. The arrayref consists of the groups the user can bless, taking into account -that having editusers permissions means that you can bless all groups, and +that having admin permissions means that you can bless all groups, and that you need to be able to see a group in order to bless it. =item C<get_products_by_permission($group)> @@ -2515,11 +2684,6 @@ Returns a reference to an array of users. The array is populated with hashrefs containing the login, identity and visibility. Users that are not visible to this user will have 'visible' set to zero. -=item C<direct_group_membership> - -Returns a reference to an array of group objects. Groups the user belong to -by group inheritance are excluded from the list. - =item C<visible_groups_inherited> Returns a list of all groups whose members should be visible to this user. @@ -2569,6 +2733,41 @@ i.e. if the 'insidergroup' parameter is set and the user belongs to this group. Returns true if the user is a global watcher, i.e. if the 'globalwatchers' parameter contains the user. +=item C<can_tag_comments> + +Returns true if the user can attach tags to comments. +i.e. if the 'comment_taggers_group' parameter is set and the user belongs to +this group. + +=item C<last_visited> + +Returns an arrayref L<Bugzilla::BugUserLastVisit> objects. + +=item C<is_involved_in_bug($bug)> + +Returns true if any of the following conditions are met, false otherwise. + +=over + +=item * + +User is the assignee of the bug + +=item * + +User is the reporter of the bug + +=item * + +User is the QA contact of the bug (if Bugzilla is configured to use a QA +contact) + +=item * + +User is in the cc list for the bug. + +=back + =back =head1 CLASS FUNCTIONS diff --git a/Bugzilla/User/Setting.pm b/Bugzilla/User/Setting.pm index 958a95580..e083a251e 100644 --- a/Bugzilla/User/Setting.pm +++ b/Bugzilla/User/Setting.pm @@ -25,8 +25,12 @@ use base qw(Exporter); # Module stuff -@Bugzilla::User::Setting::EXPORT = qw(get_all_settings get_defaults - add_setting); +@Bugzilla::User::Setting::EXPORT = qw( + get_all_settings + get_defaults + add_setting + clear_settings_cache +); use Bugzilla::Error; use Bugzilla::Util qw(trick_taint get_text); @@ -166,21 +170,23 @@ sub add_setting { sub get_all_settings { my ($user_id) = @_; - my $settings = get_defaults($user_id); # first get the defaults + my $settings = {}; my $dbh = Bugzilla->dbh; - my $sth = $dbh->prepare( - q{SELECT name, default_value, is_enabled, setting_value, subclass - FROM setting - LEFT JOIN profile_setting - ON setting.name = profile_setting.setting_name - WHERE profile_setting.user_id = ? - ORDER BY name}); + my $cache_key = "user_settings.$user_id"; + my $rows = Bugzilla->memcached->get_config({ key => $cache_key }); + if (!$rows) { + $rows = $dbh->selectall_arrayref( + q{SELECT name, default_value, is_enabled, setting_value, subclass + FROM setting + LEFT JOIN profile_setting + ON setting.name = profile_setting.setting_name + AND profile_setting.user_id = ?}, undef, ($user_id)); + Bugzilla->memcached->set_config({ key => $cache_key, data => $rows }); + } - $sth->execute($user_id); - while (my ($name, $default_value, $is_enabled, $value, $subclass) - = $sth->fetchrow_array()) - { + foreach my $row (@$rows) { + my ($name, $default_value, $is_enabled, $value, $subclass) = @$row; my $is_default; @@ -192,13 +198,18 @@ sub get_all_settings { } $settings->{$name} = new Bugzilla::User::Setting( - $name, $user_id, $is_enabled, + $name, $user_id, $is_enabled, $default_value, $value, $is_default, $subclass); } return $settings; } +sub clear_settings_cache { + my ($user_id) = @_; + Bugzilla->memcached->clear_config({ key => "user_settings.$user_id" }); +} + sub get_defaults { my ($user_id) = @_; my $dbh = Bugzilla->dbh; @@ -206,13 +217,11 @@ sub get_defaults { $user_id ||= 0; - my $sth = $dbh->prepare(q{SELECT name, default_value, is_enabled, subclass - FROM setting - ORDER BY name}); - $sth->execute(); - while (my ($name, $default_value, $is_enabled, $subclass) - = $sth->fetchrow_array()) - { + my $rows = $dbh->selectall_arrayref(q{SELECT name, default_value, is_enabled, subclass + FROM setting}); + + foreach my $row (@$rows) { + my ($name, $default_value, $is_enabled, $subclass) = @$row; $default_settings->{$name} = new Bugzilla::User::Setting( $name, $user_id, $is_enabled, $default_value, $default_value, 1, @@ -383,6 +392,13 @@ Params: C<$setting_name> - string - the name of the setting C<$is_enabled> - boolean - if false, all users must use the global default Returns: nothing +=item C<clear_settings_cache($user_id)> + +Description: Clears cached settings data for the specified user. Must be + called after updating any user's setting. +Params: C<$user_id> - integer - the user id. +Returns: nothing + =begin private =item C<_setting_exists> diff --git a/Bugzilla/UserAgent.pm b/Bugzilla/UserAgent.pm new file mode 100644 index 000000000..db95150e4 --- /dev/null +++ b/Bugzilla/UserAgent.pm @@ -0,0 +1,259 @@ +# -*- Mode: perl; indent-tabs-mode: nil -*- +# +# The contents of this file are subject to the Mozilla Public +# License Version 1.1 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of +# the License at http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS +# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or +# implied. See the License for the specific language governing +# rights and limitations under the License. +# +# The Original Code is the Bugzilla Bug Tracking System. +# +# The Initial Developer of the Original Code is the Mozilla Foundation +# Portions created by the Initial Developer are Copyright (C) 2011 the +# Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Terry Weissman <terry@mozilla.org> +# Dave Miller <justdave@syndicomm.com> +# Joe Robins <jmrobins@tgix.com> +# Gervase Markham <gerv@gerv.net> +# Shane H. W. Travis <travis@sedsystems.ca> +# Nitish Bezzala <nbezzala@yahoo.com> +# Byron Jones <glob@mozilla.com> + +package Bugzilla::UserAgent; + +use strict; +use base qw(Exporter); +our @EXPORT = qw(detect_platform detect_op_sys); + +use Bugzilla::Field; +use List::MoreUtils qw(natatime); + +use constant DEFAULT_VALUE => 'Other'; + +use constant PLATFORMS_MAP => ( + # PowerPC + qr/\(.*PowerPC.*\)/i => ["PowerPC", "Macintosh"], + # AMD64, Intel x86_64 + qr/\(.*[ix0-9]86 (?:on |\()x86_64.*\)/ => ["IA32", "x86", "PC"], + qr/\(.*amd64.*\)/ => ["AMD64", "x86_64", "PC"], + qr/\(.*x86_64.*\)/ => ["AMD64", "x86_64", "PC"], + # Intel IA64 + qr/\(.*IA64.*\)/ => ["IA64", "PC"], + # Intel x86 + qr/\(.*Intel.*\)/ => ["IA32", "x86", "PC"], + qr/\(.*[ix0-9]86.*\)/ => ["IA32", "x86", "PC"], + # Versions of Windows that only run on Intel x86 + qr/\(.*Win(?:dows |)[39M].*\)/ => ["IA32", "x86", "PC"], + qr/\(.*Win(?:dows |)16.*\)/ => ["IA32", "x86", "PC"], + # Sparc + qr/\(.*sparc.*\)/ => ["Sparc", "Sun"], + qr/\(.*sun4.*\)/ => ["Sparc", "Sun"], + # Alpha + qr/\(.*AXP.*\)/i => ["Alpha", "DEC"], + qr/\(.*[ _]Alpha.\D/i => ["Alpha", "DEC"], + qr/\(.*[ _]Alpha\)/i => ["Alpha", "DEC"], + # MIPS + qr/\(.*IRIX.*\)/i => ["MIPS", "SGI"], + qr/\(.*MIPS.*\)/i => ["MIPS", "SGI"], + # 68k + qr/\(.*68K.*\)/ => ["68k", "Macintosh"], + qr/\(.*680[x0]0.*\)/ => ["68k", "Macintosh"], + # HP + qr/\(.*9000.*\)/ => ["PA-RISC", "HP"], + # ARM + qr/\(.*(?:iPad|iPhone).*\)/ => ["ARM"], + qr/\(.*ARM.*\)/ => ["ARM", "PocketPC"], + # PocketPC intentionally before PowerPC + qr/\(.*Windows CE.*PPC.*\)/ => ["ARM", "PocketPC"], + # PowerPC + qr/\(.*PPC.*\)/ => ["PowerPC", "Macintosh"], + qr/\(.*AIX.*\)/ => ["PowerPC", "Macintosh"], + # Stereotypical and broken + qr/\(.*Windows CE.*\)/ => ["ARM", "PocketPC"], + qr/\(.*Macintosh.*\)/ => ["68k", "Macintosh"], + qr/\(.*Mac OS [89].*\)/ => ["68k", "Macintosh"], + qr/\(.*WOW64.*\)/ => ["x86_64"], + qr/\(.*Win64.*\)/ => ["IA64"], + qr/\(Win.*\)/ => ["IA32", "x86", "PC"], + qr/\(.*Win(?:dows[ -])NT.*\)/ => ["IA32", "x86", "PC"], + qr/\(.*OSF.*\)/ => ["Alpha", "DEC"], + qr/\(.*HP-?UX.*\)/i => ["PA-RISC", "HP"], + qr/\(.*IRIX.*\)/i => ["MIPS", "SGI"], + qr/\(.*(SunOS|Solaris).*\)/ => ["Sparc", "Sun"], + # Braindead old browsers who didn't follow convention: + qr/Amiga/ => ["68k", "Macintosh"], + qr/WinMosaic/ => ["IA32", "x86", "PC"], +); + +use constant OS_MAP => ( + # Sun + qr/\(.*Solaris.*\)/ => ["Solaris"], + qr/\(.*SunOS 5.11.*\)/ => [("OpenSolaris", "Opensolaris", "Solaris 11")], + qr/\(.*SunOS 5.10.*\)/ => ["Solaris 10"], + qr/\(.*SunOS 5.9.*\)/ => ["Solaris 9"], + qr/\(.*SunOS 5.8.*\)/ => ["Solaris 8"], + qr/\(.*SunOS 5.7.*\)/ => ["Solaris 7"], + qr/\(.*SunOS 5.6.*\)/ => ["Solaris 6"], + qr/\(.*SunOS 5.5.*\)/ => ["Solaris 5"], + qr/\(.*SunOS 5.*\)/ => ["Solaris"], + qr/\(.*SunOS.*sun4u.*\)/ => ["Solaris"], + qr/\(.*SunOS.*i86pc.*\)/ => ["Solaris"], + qr/\(.*SunOS.*\)/ => ["SunOS"], + # BSD + qr/\(.*BSD\/(?:OS|386).*\)/ => ["BSDI"], + qr/\(.*FreeBSD.*\)/ => ["FreeBSD"], + qr/\(.*OpenBSD.*\)/ => ["OpenBSD"], + qr/\(.*NetBSD.*\)/ => ["NetBSD"], + # Misc POSIX + qr/\(.*IRIX.*\)/ => ["IRIX"], + qr/\(.*OSF.*\)/ => ["OSF/1"], + qr/\(.*Linux.*\)/ => ["Linux"], + qr/\(.*BeOS.*\)/ => ["BeOS"], + qr/\(.*AIX.*\)/ => ["AIX"], + qr/\(.*OS\/2.*\)/ => ["OS/2"], + qr/\(.*QNX.*\)/ => ["Neutrino"], + qr/\(.*VMS.*\)/ => ["OpenVMS"], + qr/\(.*HP-?UX.*\)/ => ["HP-UX"], + qr/\(.*Android.*\)/ => ["Android"], + # Windows + qr/\(.*Windows XP.*\)/ => ["Windows XP"], + qr/\(.*Windows NT 6\.3.*\)/ => ["Windows 8.1"], + qr/\(.*Windows NT 6\.2.*\)/ => ["Windows 8"], + qr/\(.*Windows NT 6\.1.*\)/ => ["Windows 7"], + qr/\(.*Windows NT 6\.0.*\)/ => ["Windows Vista"], + qr/\(.*Windows NT 5\.2.*\)/ => ["Windows Server 2003"], + qr/\(.*Windows NT 5\.1.*\)/ => ["Windows XP"], + qr/\(.*Windows 2000.*\)/ => ["Windows 2000"], + qr/\(.*Windows NT 5.*\)/ => ["Windows 2000"], + qr/\(.*Win.*9[8x].*4\.9.*\)/ => ["Windows ME"], + qr/\(.*Win(?:dows |)M[Ee].*\)/ => ["Windows ME"], + qr/\(.*Win(?:dows |)98.*\)/ => ["Windows 98"], + qr/\(.*Win(?:dows |)95.*\)/ => ["Windows 95"], + qr/\(.*Win(?:dows |)16.*\)/ => ["Windows 3.1"], + qr/\(.*Win(?:dows[ -]|)NT.*\)/ => ["Windows NT"], + qr/\(.*Windows.*NT.*\)/ => ["Windows NT"], + # OS X + qr/\(.*(?:iPad|iPhone).*OS 7.*\)/ => ["iOS 7"], + qr/\(.*(?:iPad|iPhone).*OS 6.*\)/ => ["iOS 6"], + qr/\(.*(?:iPad|iPhone).*OS 5.*\)/ => ["iOS 5"], + qr/\(.*(?:iPad|iPhone).*OS 4.*\)/ => ["iOS 4"], + qr/\(.*(?:iPad|iPhone).*OS 3.*\)/ => ["iOS 3"], + qr/\(.*(?:iPad|iPhone).*\)/ => ["iOS"], + qr/\(.*Mac OS X (?:|Mach-O |\()10.6.*\)/ => ["Mac OS X 10.6"], + qr/\(.*Mac OS X (?:|Mach-O |\()10.5.*\)/ => ["Mac OS X 10.5"], + qr/\(.*Mac OS X (?:|Mach-O |\()10.4.*\)/ => ["Mac OS X 10.4"], + qr/\(.*Mac OS X (?:|Mach-O |\()10.3.*\)/ => ["Mac OS X 10.3"], + qr/\(.*Mac OS X (?:|Mach-O |\()10.2.*\)/ => ["Mac OS X 10.2"], + qr/\(.*Mac OS X (?:|Mach-O |\()10.1.*\)/ => ["Mac OS X 10.1"], + # Unfortunately, OS X 10.4 was the first to support Intel. This is fallback + # support because some browsers refused to include the OS Version. + qr/\(.*Intel.*Mac OS X.*\)/ => ["Mac OS X 10.4"], + # OS X 10.3 is the most likely default version of PowerPC Macs + # OS X 10.0 is more for configurations which didn't setup 10.x versions + qr/\(.*Mac OS X.*\)/ => [("Mac OS X 10.3", "Mac OS X 10.0", "Mac OS X")], + qr/\(.*Mac OS 9.*\)/ => [("Mac System 9.x", "Mac System 9.0")], + qr/\(.*Mac OS 8\.6.*\)/ => [("Mac System 8.6", "Mac System 8.5")], + qr/\(.*Mac OS 8\.5.*\)/ => ["Mac System 8.5"], + qr/\(.*Mac OS 8\.1.*\)/ => [("Mac System 8.1", "Mac System 8.0")], + qr/\(.*Mac OS 8\.0.*\)/ => ["Mac System 8.0"], + qr/\(.*Mac OS 8[^.].*\)/ => ["Mac System 8.0"], + qr/\(.*Mac OS 8.*\)/ => ["Mac System 8.6"], + qr/\(.*Darwin.*\)/ => [("Mac OS X 10.0", "Mac OS X")], + # Firefox OS + qr/\(Mobile;.*Gecko.*Firefox/ => ["Gonk (Firefox OS)"], + # Silly + qr/\(.*Mac.*PowerPC.*\)/ => ["Mac System 9.x"], + qr/\(.*Mac.*PPC.*\)/ => ["Mac System 9.x"], + qr/\(.*Mac.*68k.*\)/ => ["Mac System 8.0"], + # Evil + qr/Amiga/i => ["Other"], + qr/WinMosaic/ => ["Windows 95"], + qr/\(.*32bit.*\)/ => ["Windows 95"], + qr/\(.*16bit.*\)/ => ["Windows 3.1"], + qr/\(.*PowerPC.*\)/ => ["Mac System 9.x"], + qr/\(.*PPC.*\)/ => ["Mac System 9.x"], + qr/\(.*68K.*\)/ => ["Mac System 8.0"], +); + +sub detect_platform { + my $userAgent = $ENV{'HTTP_USER_AGENT'} || ''; + my @detected; + my $iterator = natatime(2, PLATFORMS_MAP); + while (my($re, $ra) = $iterator->()) { + if ($userAgent =~ $re) { + push @detected, @$ra; + } + } + return _pick_valid_field_value('rep_platform', @detected); +} + +sub detect_op_sys { + my $userAgent = $ENV{'HTTP_USER_AGENT'} || ''; + my @detected; + my $iterator = natatime(2, OS_MAP); + while (my($re, $ra) = $iterator->()) { + if ($userAgent =~ $re) { + push @detected, @$ra; + } + } + push(@detected, "Windows") if grep(/^Windows /, @detected); + push(@detected, "Mac OS") if grep(/^Mac /, @detected); + return _pick_valid_field_value('op_sys', @detected); +} + +# Takes the name of a field and a list of possible values for that field. +# Returns the first value in the list that is actually a valid value for that +# field. +# Returns 'Other' if none of the values match. +sub _pick_valid_field_value { + my ($field, @values) = @_; + foreach my $value (@values) { + return $value if check_field($field, $value, undef, 1); + } + return DEFAULT_VALUE; +} + +1; + +__END__ + +=head1 NAME + +Bugzilla::UserAgent - UserAgent utilities for Bugzilla + +=head1 SYNOPSIS + + use Bugzilla::UserAgent; + printf "platform: %s op-sys: %s\n", detect_platform(), detect_op_sys(); + +=head1 DESCRIPTION + +The functions exported by this module all return information derived from the +remote client's user agent. + +=head1 FUNCTIONS + +=over 4 + +=item C<detect_platform> + +This function attempts to detect the remote client's platform from the +presented user-agent. If a suitable value on the I<platform> field is found, +that field value will be returned. If no suitable value is detected, +C<detect_platform> returns I<Other>. + +=item C<detect_op_sys> + +This function attempts to detect the remote client's operating system from the +presented user-agent. If a suitable value on the I<op_sys> field is found, that +field value will be returned. If no suitable value is detected, +C<detect_op_sys> returns I<Other>. + +=back + diff --git a/Bugzilla/Util.pm b/Bugzilla/Util.pm index 48507ff9e..2349dc9e9 100644 --- a/Bugzilla/Util.pm +++ b/Bugzilla/Util.pm @@ -28,6 +28,7 @@ package Bugzilla::Util; +use 5.10.1; use strict; use base qw(Exporter); @@ -35,7 +36,7 @@ use base qw(Exporter); detaint_signed html_quote url_quote xml_quote css_class_quote html_light_quote - i_am_cgi correct_urlbase remote_ip validate_ip + i_am_cgi i_am_webservice correct_urlbase remote_ip validate_ip do_ssl_redirect_if_required use_attachbase diff_arrays on_main_db trim wrap_hard wrap_comment find_wrap_point @@ -44,7 +45,7 @@ use base qw(Exporter); bz_crypt generate_random_password validate_email_syntax clean_text get_text template_var disable_utf8 - detect_encoding); + enable_utf8 detect_encoding email_filter); use Bugzilla::Constants; use Bugzilla::RNG qw(irand); @@ -57,7 +58,6 @@ use Digest; use Email::Address; use List::Util qw(first); use Scalar::Util qw(tainted blessed); -use Template::Filters; use Text::Wrap; use Encode qw(encode decode resolve_alias); use Encode::Guess; @@ -87,10 +87,17 @@ sub detaint_signed { # visible strings. # Bug 319331: Handle BiDi disruptions. sub html_quote { - my ($var) = Template::Filters::html_filter(@_); + my $var = shift; + $var =~ s/&/&/g; + $var =~ s/</</g; + $var =~ s/>/>/g; + $var =~ s/"/"/g; # Obscure '@'. $var =~ s/\@/\@/g; - if (Bugzilla->params->{'utf8'}) { + + state $use_utf8 = Bugzilla->params->{'utf8'}; + + if ($use_utf8) { # Remove control characters if the encoding is utf8. # Other multibyte encodings may be using this range; so ignore if not utf8. $var =~ s/(?![\t\r\n])[[:cntrl:]]//g; @@ -116,13 +123,16 @@ sub html_quote { # |U+200e|Left-To-Right Mark |0xe2 0x80 0x8e | # |U+200f|Right-To-Left Mark |0xe2 0x80 0x8f | # -------------------------------------------------------- - $var =~ s/[\x{202a}-\x{202e}]//g; + $var =~ tr/\x{202a}-\x{202e}//d; } return $var; } sub html_light_quote { my ($text) = @_; + # admin/table.html.tmpl calls |FILTER html_light| many times. + # There is no need to recreate the HTML::Scrubber object again and again. + my $scrubber = Bugzilla->process_cache->{html_scrubber}; # List of allowed HTML elements having no attributes. my @allow = qw(b strong em i u p br abbr acronym ins del cite code var @@ -144,7 +154,7 @@ sub html_light_quote { $text =~ s#$chr($safe)$chr#<$1>#go; return $text; } - else { + elsif (!$scrubber) { # We can be less restrictive. We can accept elements with attributes. push(@allow, qw(a blockquote q span)); @@ -187,14 +197,14 @@ sub html_light_quote { }, ); - my $scrubber = HTML::Scrubber->new(default => \@default, - allow => \@allow, - rules => \@rules, - comment => 0, - process => 0); - - return $scrubber->scrub($text); + Bugzilla->process_cache->{html_scrubber} = $scrubber = + HTML::Scrubber->new(default => \@default, + allow => \@allow, + rules => \@rules, + comment => 0, + process => 0); } + return $scrubber->scrub($text); } sub email_filter { @@ -253,6 +263,13 @@ sub i_am_cgi { return exists $ENV{'SERVER_SOFTWARE'} ? 1 : 0; } +sub i_am_webservice { + my $usage_mode = Bugzilla->usage_mode; + return $usage_mode == USAGE_MODE_XMLRPC + || $usage_mode == USAGE_MODE_JSON + || $usage_mode == USAGE_MODE_REST; +} + # This exists as a separate function from Bugzilla::CGI::redirect_to_https # because we don't want to create a CGI object during XML-RPC calls # (doing so can mess up XML-RPC). @@ -475,11 +492,11 @@ sub find_wrap_point { if (!$string) { return 0 } if (length($string) < $maxpos) { return length($string) } my $wrappoint = rindex($string, ",", $maxpos); # look for comma - if ($wrappoint < 0) { # can't find comma + if ($wrappoint <= 0) { # can't find comma $wrappoint = rindex($string, " ", $maxpos); # look for space - if ($wrappoint < 0) { # can't find space + if ($wrappoint <= 0) { # can't find space $wrappoint = rindex($string, "-", $maxpos); # look for hyphen - if ($wrappoint < 0) { # can't find hyphen + if ($wrappoint <= 0) { # can't find hyphen $wrappoint = $maxpos; # just truncate it } else { $wrappoint++; # leave hyphen on the left side @@ -531,9 +548,14 @@ sub datetime_from { # In the database, this is the "0" date. return undef if $date =~ /^0000/; - # strptime($date) returns an empty array if $date has an invalid - # date format. - my @time = strptime($date); + my @time; + # Most dates will be in this format, avoid strptime's generic parser + if ($date =~ /^(\d{4})[\.-](\d{2})[\.-](\d{2})(?: (\d{2}):(\d{2}):(\d{2}))?$/) { + @time = ($6, $5, $4, $3, $2 - 1, $1 - 1900, undef); + } + else { + @time = strptime($date); + } unless (scalar @time) { # If an unknown timezone is passed (such as MSK, for Moskow), @@ -545,10 +567,14 @@ sub datetime_from { return undef if !@time; - # strptime() counts years from 1900, and months from 0 (January). - # We have to fix both values. + # strptime() counts years from 1900, except if they are older than 1901 + # in which case it returns the full year (so 1890 -> 1890, but 1984 -> 84, + # and 3790 -> 1890). We make a guess and assume that 1100 <= year < 3000. + $time[5] += 1900 if $time[5] < 1100; + my %args = ( - year => $time[5] + 1900, + year => $time[5], + # Months start from 0 (January). month => $time[4] + 1, day => $time[3], hour => $time[2], @@ -609,13 +635,13 @@ sub bz_crypt { $algorithm = $1; } + # Wide characters cause crypt and Digest to die. + if (Bugzilla->params->{'utf8'}) { + utf8::encode($password) if utf8::is_utf8($password); + } + my $crypted_password; if (!$algorithm) { - # Wide characters cause crypt to die - if (Bugzilla->params->{'utf8'}) { - utf8::encode($password) if utf8::is_utf8($password); - } - # Crypt the password. $crypted_password = crypt($password, $salt); @@ -730,10 +756,12 @@ sub get_text { sub template_var { my $name = shift; - my $cache = Bugzilla->request_cache->{util_template_var} ||= {}; - my $template = Bugzilla->template_inner; - my $lang = $template->context->{bz_language}; + my $request_cache = Bugzilla->request_cache; + my $cache = $request_cache->{util_template_var} ||= {}; + my $lang = $request_cache->{template_current_lang}->[0] || ''; return $cache->{$lang}->{$name} if defined $cache->{$lang}; + + my $template = Bugzilla->template_inner($lang); my %vars; # Note: If we suddenly start needing a lot of template_var variables, # they should move into their own template, not field-descs. @@ -750,11 +778,7 @@ sub template_var { sub display_value { my ($field, $value) = @_; - my $value_descs = template_var('value_descs'); - if (defined $value_descs->{$field}->{$value}) { - return $value_descs->{$field}->{$value}; - } - return $value; + return template_var('value_descs')->{$field}->{$value} // $value; } sub disable_utf8 { @@ -763,6 +787,12 @@ sub disable_utf8 { } } +sub enable_utf8 { + if (Bugzilla->params->{'utf8'}) { + binmode STDOUT, ':utf8'; # Turn on UTF8 encoding. + } +} + use constant UTF8_ACCIDENTAL => qw(shiftjis big5-eten euc-kr euc-jp); sub detect_encoding { diff --git a/Bugzilla/WebService.pm b/Bugzilla/WebService.pm index 166707626..86fbe1fe3 100644 --- a/Bugzilla/WebService.pm +++ b/Bugzilla/WebService.pm @@ -52,15 +52,20 @@ This is the standard API for external programs that want to interact with Bugzilla. It provides various methods in various modules. You can interact with this API via -L<XML-RPC|Bugzilla::WebService::Server::XMLRPC> or -L<JSON-RPC|Bugzilla::WebService::Server::JSONRPC>. +L<XML-RPC|Bugzilla::WebService::Server::XMLRPC>, +L<JSON-RPC|Bugzilla::WebService::Server::JSONRPC> or +L<REST|Bugzilla::WebService::Server::REST>. =head1 CALLING METHODS -Methods are grouped into "packages", like C<Bug> for +Methods are grouped into "packages", like C<Bug> for L<Bugzilla::WebService::Bug>. So, for example, L<Bugzilla::WebService::Bug/get>, is called as C<Bug.get>. +For REST, the "package" is more determined by the path +used to access the resource. See each relevant method +for specific details on how to access via REST. + =head1 PARAMETERS The Bugzilla API takes the following various types of parameters: @@ -79,6 +84,11 @@ A floating-point number. May be null. A string. May be null. +=item C<email> + +A string representing an email address. This value, when returned, +may be filtered based on if the user is logged in or not. May be null. + =item C<dateTime> A date/time. Represented differently in different interfaces to this API. @@ -137,7 +147,7 @@ There are various ways to log in: =item C<User.login> -You can use L<Bugzilla::WebService::User/login> to log in as a Bugzilla +You can use L<Bugzilla::WebService::User/login> to log in as a Bugzilla user. This issues standard HTTP cookies that you must then use in future calls, so your client must be capable of receiving and transmitting cookies. @@ -167,13 +177,24 @@ not expire. =back The C<Bugzilla_restrictlogin> and C<Bugzilla_rememberlogin> options -are only used when you have also specified C<Bugzilla_login> and +are only used when you have also specified C<Bugzilla_login> and C<Bugzilla_password>. Note that Bugzilla will return HTTP cookies along with the method response when you use these arguments (just like the C<User.login> method above). +For REST, you may also use the C<username> and C<password> variable +names instead of C<Bugzilla_login> and C<Bugzilla_password> as a +convenience. + +=item B<Added in Bugzilla 5.0> + +An error is now thrown if you pass invalid cookies or an invalid token. +You will need to log in again to get new cookies or a new token. Previous +releases simply ignored invalid cookies and token support was added in +Bugzilla B<5.0>. + =back =head1 STABLE, EXPERIMENTAL, and UNSTABLE @@ -258,6 +279,14 @@ be returned, the rest will not be included. If you specify an empty array, then this function will return empty hashes. +Some RPC calls support specifying sub fields. + +If an RPC call states that it support sub field restrictions, you can +restrict what information is returned within the first field. For example, +if you call Products.get with an include_fields of components.name, then +only the component name would be returned (and nothing else). You can +include the main field, and exclude a sub field. + Invalid field names are ignored. Example: @@ -268,6 +297,9 @@ would return something like: { users => [{ id => 1, name => 'user@domain.com' }] } +Note for REST, C<include_fields> may instead be a comma delimited string +for GET type requests. + =item C<exclude_fields> C<array> An array of strings, representing the (case-sensitive) names of @@ -277,6 +309,13 @@ the returned hashes. If you specify all the fields, then this function will return empty hashes. +Some RPC calls support specifying sub fields. If an RPC call states that +it support sub field restrictions, you can restrict what information is +returned within the first field. For example, if you call Products.get +with an include_fields of components.name, then only the component name +would be returned (and nothing else). You can include the main field, +and exclude a sub field. + Invalid field names are ignored. Specifying fields here overrides C<include_fields>, so if you specify a @@ -290,6 +329,37 @@ would return something like: { users => [{ id => 1, real_name => 'John Smith' }] } +Note for REST, C<exclude_fields> may instead be a comma delimited string +for GET type requests. + +=back + +There are several shortcut identifiers to ask for only certain groups of +fields to be returned or excluded. + +=over + +=item C<_all> + +All possible fields are returned if C<_all> is specified in C<include_fields>. + +=item C<_default> + +These fields are returned if C<include_fields> is empty or C<_default> +is specified. All fields described in the documentation are returned +by default unless specified otherwise. + +=item C<_extra> + +These fields are not returned by default and need to be manually specified +in C<include_fields> either by field name, or using C<_extra>. + +=item C<_custom> + +Only custom fields are returned if C<_custom> is specified in C<include_fields>. +This is normally specific to bug objects and not relevant for other returned +objects. + =back =head1 SEE ALSO diff --git a/Bugzilla/WebService/Bug.pm b/Bugzilla/WebService/Bug.pm index eb76b4131..4f5b4c39c 100644 --- a/Bugzilla/WebService/Bug.pm +++ b/Bugzilla/WebService/Bug.pm @@ -26,18 +26,27 @@ use strict; use base qw(Bugzilla::WebService); use Bugzilla::Comment; +use Bugzilla::Comment::TagWeights; use Bugzilla::Constants; use Bugzilla::Error; use Bugzilla::Field; use Bugzilla::WebService::Constants; -use Bugzilla::WebService::Util qw(filter filter_wants validate); +use Bugzilla::WebService::Util qw(extract_flags filter filter_wants validate translate); use Bugzilla::Bug; use Bugzilla::BugMail; -use Bugzilla::Util qw(trick_taint trim); +use Bugzilla::Util qw(trick_taint trim detaint_natural); use Bugzilla::Version; use Bugzilla::Milestone; use Bugzilla::Status; use Bugzilla::Token qw(issue_hash_token); +use Bugzilla::Search; +use Bugzilla::Product; +use Bugzilla::FlagType; +use Bugzilla::Search::Quicksearch; + +use List::Util qw(max); +use List::MoreUtils qw(uniq); +use Storable qw(dclone); ############# # Constants # @@ -45,10 +54,25 @@ use Bugzilla::Token qw(issue_hash_token); use constant PRODUCT_SPECIFIC_FIELDS => qw(version target_milestone component); -use constant DATE_FIELDS => { - comments => ['new_since'], - search => ['last_change_time', 'creation_time'], -}; +sub DATE_FIELDS { + my $fields = { + comments => ['new_since'], + create => [], + history => ['new_since'], + search => ['last_change_time', 'creation_time'], + update => [] + }; + + # Add date related custom fields + foreach my $field (Bugzilla->active_custom_fields) { + next unless ($field->type == FIELD_TYPE_DATETIME + || $field->type == FIELD_TYPE_DATE); + push(@{ $fields->{create} }, $field->name); + push(@{ $fields->{update} }, $field->name); + } + + return $fields; +} use constant BASE64_FIELDS => { add_attachment => ['data'], @@ -64,6 +88,20 @@ use constant READ_ONLY => qw( search ); +use constant ATTACHMENT_MAPPED_SETTERS => { + file_name => 'filename', + summary => 'description', +}; + +use constant ATTACHMENT_MAPPED_RETURNS => { + description => 'summary', + ispatch => 'is_patch', + isprivate => 'is_private', + isobsolete => 'is_obsolete', + filename => 'file_name', + mimetype => 'content_type', +}; + ###################################################### # Add aliases here for old method name compatibility # ###################################################### @@ -82,6 +120,8 @@ BEGIN { sub fields { my ($self, $params) = validate(@_, 'ids', 'names'); + Bugzilla->switch_to_shadow_db(); + my @fields; if (defined $params->{ids}) { my $ids = $params->{ids}; @@ -117,11 +157,12 @@ sub fields { my (@values, $has_values); if ( ($field->is_select and $field->name ne 'product') - or grep($_ eq $field->name, PRODUCT_SPECIFIC_FIELDS)) + or grep($_ eq $field->name, PRODUCT_SPECIFIC_FIELDS) + or $field->name eq 'keywords') { $has_values = 1; @values = @{ $self->_legal_field_values({ field => $field }) }; - } + } if (grep($_ eq $field->name, PRODUCT_SPECIFIC_FIELDS)) { $value_field = 'product'; @@ -211,6 +252,15 @@ sub _legal_field_values { } } + elsif ($field_name eq 'keywords') { + my @legal_keywords = Bugzilla::Keyword->get_all; + foreach my $value (@legal_keywords) { + push (@result, { + name => $self->type('string', $value->name), + description => $self->type('string', $value->description), + }); + } + } else { my @values = Bugzilla::Field::Choice->type($field)->get_all(); foreach my $value (@values) { @@ -242,7 +292,7 @@ sub comments { my $bug_ids = $params->{ids} || []; my $comment_ids = $params->{comment_ids} || []; - my $dbh = Bugzilla->dbh; + my $dbh = Bugzilla->switch_to_shadow_db(); my $user = Bugzilla->user; my %bugs; @@ -289,31 +339,75 @@ sub comments { return { bugs => \%bugs, comments => \%comments }; } +sub render_comment { + my ($self, $params) = @_; + + unless (defined $params->{text}) { + ThrowCodeError('params_required', + { function => 'Bug.render_comment', + params => ['text'] }); + } + + Bugzilla->switch_to_shadow_db(); + my $bug = $params->{id} ? Bugzilla::Bug->check($params->{id}) : undef; + + my $tmpl = '[% text FILTER quoteUrls(bug) %]'; + my $html; + my $template = Bugzilla->template; + $template->process( + \$tmpl, + { bug => $bug, text => $params->{text}}, + \$html + ); + + return { html => $html }; +} + # Helper for Bug.comments sub _translate_comment { - my ($self, $comment, $filters) = @_; + my ($self, $comment, $filters, $types, $prefix) = @_; my $attach_id = $comment->is_about_attachment ? $comment->extra_data : undef; - return filter $filters, { + + my $comment_hash = { id => $self->type('int', $comment->id), bug_id => $self->type('int', $comment->bug_id), - creator => $self->type('string', $comment->author->login), - author => $self->type('string', $comment->author->login), + creator => $self->type('email', $comment->author->login), + author => $self->type('email', $comment->author->login), time => $self->type('dateTime', $comment->creation_ts), + creation_time => $self->type('dateTime', $comment->creation_ts), is_private => $self->type('boolean', $comment->is_private), text => $self->type('string', $comment->body_full), attachment_id => $self->type('int', $attach_id), }; + + # Don't load comment tags unless enabled + if (Bugzilla->params->{'comment_taggers_group'}) { + $comment_hash->{tags} = [ + map { $self->type('string', $_) } + @{ $comment->tags } + ]; + } + + return filter($filters, $comment_hash, $types, $prefix); } sub get { my ($self, $params) = validate(@_, 'ids'); + Bugzilla->switch_to_shadow_db() unless Bugzilla->user->id; + my $ids = $params->{ids}; - defined $ids || ThrowCodeError('param_required', { param => 'ids' }); + (defined $ids && scalar @$ids) + || ThrowCodeError('param_required', { param => 'ids' }); + + my (@bugs, @faults, @hashes); + + # Cache permissions for bugs. This highly reduces the number of calls to the DB. + # visible_bugs() is only able to handle bug IDs, so we have to skip aliases. + my @int = grep { $_ =~ /^\d+$/ } @$ids; + Bugzilla->user->visible_bugs(\@int); - my @bugs; - my @faults; foreach my $bug_id (@$ids) { my $bug; if ($params->{permissive}) { @@ -331,10 +425,18 @@ sub get { else { $bug = Bugzilla::Bug->check($bug_id); } - push(@bugs, $self->_bug_to_hash($bug, $params)); + push(@bugs, $bug); + push(@hashes, $self->_bug_to_hash($bug, $params)); } - return { bugs => \@bugs, faults => \@faults }; + # Set the ETag before inserting the update tokens + # since the tokens will always be unique even if + # the data has not changed. + $self->bz_etag(\@hashes); + + $self->_add_update_tokens($params, \@bugs, \@hashes); + + return { bugs => \@hashes, faults => \@faults }; } # this is a function that gets bug activity for list of bug ids @@ -343,34 +445,39 @@ sub get { sub history { my ($self, $params) = validate(@_, 'ids'); + Bugzilla->switch_to_shadow_db(); + my $ids = $params->{ids}; defined $ids || ThrowCodeError('param_required', { param => 'ids' }); - my @return; + my %api_name = reverse %{ Bugzilla::Bug::FIELD_MAP() }; + $api_name{'bug_group'} = 'groups'; + my @return; foreach my $bug_id (@$ids) { my %item; my $bug = Bugzilla::Bug->check($bug_id); $bug_id = $bug->id; $item{id} = $self->type('int', $bug_id); - my ($activity) = Bugzilla::Bug::GetBugActivity($bug_id); + my ($activity) = Bugzilla::Bug::GetBugActivity($bug_id, undef, $params->{new_since}); my @history; foreach my $changeset (@$activity) { my %bug_history; $bug_history{when} = $self->type('dateTime', $changeset->{when}); - $bug_history{who} = $self->type('string', $changeset->{who}); + $bug_history{who} = $self->type('email', $changeset->{who}); $bug_history{changes} = []; foreach my $change (@{ $changeset->{changes} }) { + my $api_field = $api_name{$change->{fieldname}} || $change->{fieldname}; my $attach_id = delete $change->{attachid}; if ($attach_id) { $change->{attachment_id} = $self->type('int', $attach_id); } $change->{removed} = $self->type('string', $change->{removed}); $change->{added} = $self->type('string', $change->{added}); - $change->{field_name} = $self->type('string', - delete $change->{fieldname}); + $change->{field_name} = $self->type('string', $api_field); + delete $change->{fieldname}; push (@{$bug_history{changes}}, $change); } @@ -399,77 +506,127 @@ sub history { sub search { my ($self, $params) = @_; + my $user = Bugzilla->user; + my $dbh = Bugzilla->dbh; - if ( defined($params->{offset}) and !defined($params->{limit}) ) { - ThrowCodeError('param_required', + Bugzilla->switch_to_shadow_db(); + + my $match_params = dclone($params); + delete $match_params->{include_fields}; + delete $match_params->{exclude_fields}; + + # Determine whether this is a quicksearch query + if (exists $match_params->{quicksearch}) { + my $quicksearch = quicksearch($match_params->{'quicksearch'}); + my $cgi = Bugzilla::CGI->new($quicksearch); + $match_params = $cgi->Vars; + } + + if ( defined($match_params->{offset}) and !defined($match_params->{limit}) ) { + ThrowCodeError('param_required', { param => 'limit', function => 'Bug.search()' }); } my $max_results = Bugzilla->params->{max_search_results}; - unless (defined $params->{limit} && $params->{limit} == 0) { - if (!defined $params->{limit} || $params->{limit} > $max_results) { - $params->{limit} = $max_results; + unless (defined $match_params->{limit} && $match_params->{limit} == 0) { + if (!defined $match_params->{limit} || $match_params->{limit} > $max_results) { + $match_params->{limit} = $max_results; } } else { - delete $params->{limit}; - delete $params->{offset}; + delete $match_params->{limit}; + delete $match_params->{offset}; } - $params = Bugzilla::Bug::map_fields($params); - delete $params->{WHERE}; + $match_params = Bugzilla::Bug::map_fields($match_params); - unless (Bugzilla->user->is_timetracker) { - delete $params->{$_} foreach qw(estimated_time remaining_time deadline); - } + my %options = ( fields => ['bug_id'] ); + + # Find the highest custom field id + my @field_ids = grep(/^f(\d+)$/, keys %$match_params); + my $last_field_id = @field_ids ? max @field_ids + 1 : 1; # Do special search types for certain fields. - if ( my $bug_when = delete $params->{delta_ts} ) { - $params->{WHERE}->{'delta_ts >= ?'} = $bug_when; + if (my $change_when = delete $match_params->{'delta_ts'}) { + $match_params->{"f${last_field_id}"} = 'delta_ts'; + $match_params->{"o${last_field_id}"} = 'greaterthaneq'; + $match_params->{"v${last_field_id}"} = $change_when; + $last_field_id++; + } + if (my $creation_when = delete $match_params->{'creation_ts'}) { + $match_params->{"f${last_field_id}"} = 'creation_ts'; + $match_params->{"o${last_field_id}"} = 'greaterthaneq'; + $match_params->{"v${last_field_id}"} = $creation_when; + $last_field_id++; } - if (my $when = delete $params->{creation_ts}) { - $params->{WHERE}->{'creation_ts >= ?'} = $when; + + # Some fields require a search type such as short desc, keywords, etc. + foreach my $param (qw(short_desc longdesc status_whiteboard bug_file_loc)) { + if (defined $match_params->{$param} && !defined $match_params->{$param . '_type'}) { + $match_params->{$param . '_type'} = 'allwordssubstr'; + } } - if (my $summary = delete $params->{short_desc}) { - my @strings = ref $summary ? @$summary : ($summary); - my @likes = ("short_desc LIKE ?") x @strings; - my $clause = join(' OR ', @likes); - $params->{WHERE}->{"($clause)"} = [map { "\%$_\%" } @strings]; + if (defined $match_params->{'keywords'} && !defined $match_params->{'keywords_type'}) { + $match_params->{'keywords_type'} = 'allwords'; } - if (my $whiteboard = delete $params->{status_whiteboard}) { - my @strings = ref $whiteboard ? @$whiteboard : ($whiteboard); - my @likes = ("status_whiteboard LIKE ?") x @strings; - my $clause = join(' OR ', @likes); - $params->{WHERE}->{"($clause)"} = [map { "\%$_\%" } @strings]; + + # Backwards compatibility with old method regarding role search + $match_params->{'reporter'} = delete $match_params->{'creator'} if $match_params->{'creator'}; + foreach my $role (qw(assigned_to reporter qa_contact commenter cc)) { + next if !exists $match_params->{$role}; + my $value = delete $match_params->{$role}; + $match_params->{"f${last_field_id}"} = $role; + $match_params->{"o${last_field_id}"} = "anywordssubstr"; + $match_params->{"v${last_field_id}"} = ref $value ? join(" ", @{$value}) : $value; + $last_field_id++; } # If no other parameters have been passed other than limit and offset - # and a WHERE parameter was not created earlier, then we throw error - # if system is configured to do so. - if (!$params->{WHERE} - && !grep(!/(limit|offset)/i, keys %$params) + # then we throw error if system is configured to do so. + if (!grep(!/^(limit|offset)$/, keys %$match_params) && !Bugzilla->params->{search_allow_no_criteria}) { ThrowUserError('buglist_parameters_required'); } - # We want include_fields and exclude_fields to be passed to - # _bug_to_hash but not to Bugzilla::Bug->match so we copy the - # params and delete those before passing to Bugzilla::Bug->match. - my %match_params = %{ $params }; - delete $match_params{'include_fields'}; - delete $match_params{'exclude_fields'}; + $options{order} = [ split(/\s*,\s*/, delete $match_params->{order}) ] if $match_params->{order}; + $options{params} = $match_params; - my $bugs = Bugzilla::Bug->match(\%match_params); - my $visible = Bugzilla->user->visible_bugs($bugs); - my @hashes = map { $self->_bug_to_hash($_, $params) } @$visible; - return { bugs => \@hashes }; + my $search = new Bugzilla::Search(%options); + my ($data) = $search->data; + + # BMO if the caller only wants the count, that's all we need to return + if ($params->{count_only}) { + if (Bugzilla->usage_mode == USAGE_MODE_XMLRPC) { + return $data; + } + else { + return { bug_count => $data }; + } + } + + if (!scalar @$data) { + return { bugs => [] }; + } + + # Search.pm won't return bugs that the user shouldn't see so no filtering is needed. + my @bug_ids = map { $_->[0] } @$data; + my %bug_objects = map { $_->id => $_ } @{ Bugzilla::Bug->new_from_list(\@bug_ids) }; + my @bugs = map { $bug_objects{$_} } @bug_ids; + @bugs = map { $self->_bug_to_hash($_, $params) } @bugs; + + # BzAPI + Bugzilla->request_cache->{bzapi_search_bugs} = [ map { $bug_objects{$_} } @bug_ids ]; + + return { bugs => \@bugs }; } sub possible_duplicates { my ($self, $params) = validate(@_, 'product'); my $user = Bugzilla->user; + Bugzilla->switch_to_shadow_db(); + # Undo the array-ification that validate() does, for "summary". $params->{summary} || ThrowCodeError('param_required', { function => 'Bug.possible_duplicates', param => 'summary' }); @@ -484,12 +641,19 @@ sub possible_duplicates { { summary => $params->{summary}, products => \@products, limit => $params->{limit} }); my @hashes = map { $self->_bug_to_hash($_, $params) } @$possible_dupes; + $self->_add_update_tokens($params, $possible_dupes, \@hashes); return { bugs => \@hashes }; } sub update { my ($self, $params) = validate(@_, 'ids'); + # BMO: Don't allow updating of bugs if disabled + if (Bugzilla->params->{disable_bug_updates}) { + ThrowErrorPage('bug/process/updates-disabled.html.tmpl', + 'Bug updates are currently disabled.'); + } + my $user = Bugzilla->login(LOGIN_REQUIRED); my $dbh = Bugzilla->dbh; @@ -514,7 +678,8 @@ sub update { # have valid "set_" functions in Bugzilla::Bug, but shouldn't be # called using those field names. delete $values{dependencies}; - delete $values{flags}; + + my $flags = delete $values{flags}; foreach my $bug (@bugs) { if (!$user->can_edit_product($bug->product_obj->id) ) { @@ -523,6 +688,10 @@ sub update { } $bug->set_all(\%values); + if ($flags) { + my ($old_flags, $new_flags) = extract_flags($flags, $bug); + $bug->set_flags($old_flags, $new_flags); + } } my %all_changes; @@ -531,7 +700,7 @@ sub update { $all_changes{$bug->id} = $bug->update(); } $dbh->bz_commit_transaction(); - + foreach my $bug (@bugs) { $bug->send_changes($all_changes{$bug->id}); } @@ -584,16 +753,50 @@ sub update { sub create { my ($self, $params) = @_; + my $dbh = Bugzilla->dbh; + + # BMO: Don't allow updating of bugs if disabled + if (Bugzilla->params->{disable_bug_updates}) { + ThrowErrorPage('bug/process/updates-disabled.html.tmpl', + 'Bug updates are currently disabled.'); + } + Bugzilla->login(LOGIN_REQUIRED); + + # Some fields cannot be sent to Bugzilla::Bug->create + foreach my $key (qw(login password token)) { + delete $params->{$key}; + } + $params = Bugzilla::Bug::map_fields($params); + + my $flags = delete $params->{flags}; + + # We start a nested transaction in case flag setting fails + # we want the bug creation to roll back as well. + $dbh->bz_start_transaction(); + my $bug = Bugzilla::Bug->create($params); - Bugzilla::BugMail::Send($bug->bug_id, { changer => $bug->reporter }); + + # Set bug flags + if ($flags) { + my ($flags, $new_flags) = extract_flags($flags, $bug); + $bug->set_flags($flags, $new_flags); + $bug->update($bug->creation_ts); + } + + $dbh->bz_commit_transaction(); + + $bug->send_changes(); + return { id => $self->type('int', $bug->bug_id) }; } sub legal_values { my ($self, $params) = @_; + Bugzilla->switch_to_shadow_db(); + defined $params->{field} or ThrowCodeError('param_required', { param => 'field' }); @@ -646,6 +849,12 @@ sub add_attachment { my ($self, $params) = validate(@_, 'ids'); my $dbh = Bugzilla->dbh; + # BMO: Don't allow updating of bugs if disabled + if (Bugzilla->params->{disable_bug_updates}) { + ThrowErrorPage('bug/process/updates-disabled.html.tmpl', + 'Bug updates are currently disabled.'); + } + Bugzilla->login(LOGIN_REQUIRED); defined $params->{ids} || ThrowCodeError('param_required', { param => 'ids' }); @@ -662,6 +871,8 @@ sub add_attachment { $dbh->bz_start_transaction(); my $timestamp = $dbh->selectrow_array('SELECT LOCALTIMESTAMP(0)'); + my $flags = delete $params->{flags}; + foreach my $bug (@bugs) { my $attachment = Bugzilla::Attachment->create({ bug => $bug, @@ -673,6 +884,13 @@ sub add_attachment { ispatch => $params->{is_patch}, isprivate => $params->{is_private}, }); + + if ($flags) { + my ($old_flags, $new_flags) = extract_flags($flags, $bug, $attachment); + $attachment->set_flags($old_flags, $new_flags); + } + + $attachment->update($timestamp); my $comment = $params->{comment} || ''; $attachment->bug->add_comment($comment, { isprivate => $attachment->isprivate, @@ -691,9 +909,107 @@ sub add_attachment { return { attachments => \%attachments }; } +sub update_attachment { + my ($self, $params) = validate(@_, 'ids'); + + my $user = Bugzilla->login(LOGIN_REQUIRED); + my $dbh = Bugzilla->dbh; + + my $ids = delete $params->{ids}; + defined $ids || ThrowCodeError('param_required', { param => 'ids' }); + + # Some fields cannot be sent to set_all + foreach my $key (qw(login password token)) { + delete $params->{$key}; + } + + $params = translate($params, ATTACHMENT_MAPPED_SETTERS); + + # Get all the attachments, after verifying that they exist and are editable + my @attachments = (); + my %bugs = (); + foreach my $id (@$ids) { + my $attachment = Bugzilla::Attachment->new($id) + || ThrowUserError("invalid_attach_id", { attach_id => $id }); + my $bug = $attachment->bug; + $attachment->_check_bug; + $attachment->validate_can_edit + || ThrowUserError("illegal_attachment_edit", { attach_id => $id }); + + push @attachments, $attachment; + $bugs{$bug->id} = $bug; + } + + my $flags = delete $params->{flags}; + my $comment = delete $params->{comment}; + + # Update the values + foreach my $attachment (@attachments) { + $attachment->set_all($params); + if ($flags) { + my ($old_flags, $new_flags) = extract_flags($flags, $attachment->bug, $attachment); + $attachment->set_flags($old_flags, $new_flags); + } + } + + $dbh->bz_start_transaction(); + + # Do the actual update and get information to return to user + my @result; + foreach my $attachment (@attachments) { + my $changes = $attachment->update(); + + if ($comment = trim($comment)) { + $attachment->bug->add_comment($comment, + { isprivate => $attachment->isprivate, + type => CMT_ATTACHMENT_UPDATED, + extra_data => $attachment->id }); + } + + $changes = translate($changes, ATTACHMENT_MAPPED_RETURNS); + + my %hash = ( + id => $self->type('int', $attachment->id), + last_change_time => $self->type('dateTime', $attachment->modification_time), + changes => {}, + ); + + foreach my $field (keys %$changes) { + my $change = $changes->{$field}; + + # We normalize undef to an empty string, so that the API + # stays consistent for things like Deadline that can become + # empty. + $hash{changes}->{$field} = { + removed => $self->type('string', $change->[0] // ''), + added => $self->type('string', $change->[1] // '') + }; + } + + push(@result, \%hash); + } + + $dbh->bz_commit_transaction(); + + # Email users about the change + foreach my $bug (values %bugs) { + $bug->update(); + $bug->send_changes(); + } + + # Return the information to the user + return { attachments => \@result }; +} + sub add_comment { my ($self, $params) = @_; + # BMO: Don't allow updating of bugs if disabled + if (Bugzilla->params->{disable_bug_updates}) { + ThrowErrorPage('bug/process/updates-disabled.html.tmpl', + 'Bug updates are currently disabled.'); + } + #The user must login in order add a comment Bugzilla->login(LOGIN_REQUIRED); @@ -738,6 +1054,12 @@ sub add_comment { sub update_see_also { my ($self, $params) = @_; + # BMO: Don't allow updating of bugs if disabled + if (Bugzilla->params->{disable_bug_updates}) { + ThrowErrorPage('bug/process/updates-disabled.html.tmpl', + 'Bug updates are currently disabled.'); + } + my $user = Bugzilla->login(LOGIN_REQUIRED); # Check parameters @@ -785,6 +1107,8 @@ sub update_see_also { sub attachments { my ($self, $params) = validate(@_, 'ids', 'attachment_ids'); + Bugzilla->switch_to_shadow_db() unless Bugzilla->user->id; + if (!(defined $params->{ids} or defined $params->{attachment_ids})) { @@ -821,6 +1145,105 @@ sub attachments { return { bugs => \%bugs, attachments => \%attachments }; } +sub flag_types { + my ($self, $params) = @_; + my $dbh = Bugzilla->switch_to_shadow_db(); + my $user = Bugzilla->user; + + defined $params->{product} + || ThrowCodeError('param_required', + { function => 'Bug.flag_types', + param => 'product' }); + + my $product = delete $params->{product}; + my $component = delete $params->{component}; + + $product = Bugzilla::Product->check({ name => $product, cache => 1 }); + $component = Bugzilla::Component->check( + { name => $component, product => $product, cache => 1 }) if $component; + + my $flag_params = { product_id => $product->id }; + $flag_params->{component_id} = $component->id if $component; + my $matched_flag_types = Bugzilla::FlagType::match($flag_params); + + my $flag_types = { bug => [], attachment => [] }; + foreach my $flag_type (@$matched_flag_types) { + push(@{ $flag_types->{bug} }, $self->_flagtype_to_hash($flag_type, $product)) + if $flag_type->target_type eq 'bug'; + push(@{ $flag_types->{attachment} }, $self->_flagtype_to_hash($flag_type, $product)) + if $flag_type->target_type eq 'attachment'; + } + + return $flag_types; +} + +sub update_comment_tags { + my ($self, $params) = @_; + + my $user = Bugzilla->login(LOGIN_REQUIRED); + Bugzilla->params->{'comment_taggers_group'} + || ThrowUserError("comment_tag_disabled"); + $user->can_tag_comments + || ThrowUserError("auth_failure", + { group => Bugzilla->params->{'comment_taggers_group'}, + action => "update", + object => "comment_tags" }); + + my $comment_id = $params->{comment_id} + // ThrowCodeError('param_required', + { function => 'Bug.update_comment_tags', + param => 'comment_id' }); + + my $comment = Bugzilla::Comment->new($comment_id) + || return []; + $comment->bug->check_is_visible(); + if ($comment->is_private && !$user->is_insider) { + ThrowUserError('comment_is_private', { id => $comment_id }); + } + + my $dbh = Bugzilla->dbh; + $dbh->bz_start_transaction(); + foreach my $tag (@{ $params->{add} || [] }) { + $comment->add_tag($tag) if defined $tag; + } + foreach my $tag (@{ $params->{remove} || [] }) { + $comment->remove_tag($tag) if defined $tag; + } + $comment->update(); + $dbh->bz_commit_transaction(); + + return $comment->tags; +} + +sub search_comment_tags { + my ($self, $params) = @_; + + Bugzilla->login(LOGIN_REQUIRED); + Bugzilla->params->{'comment_taggers_group'} + || ThrowUserError("comment_tag_disabled"); + Bugzilla->user->can_tag_comments + || ThrowUserError("auth_failure", { group => Bugzilla->params->{'comment_taggers_group'}, + action => "search", + object => "comment_tags"}); + + my $query = $params->{query}; + $query + // ThrowCodeError('param_required', { param => 'query' }); + my $limit = $params->{limit} || 7; + detaint_natural($limit) + || ThrowCodeError('param_must_be_numeric', { param => 'limit', + function => 'Bug.search_comment_tags' }); + + + my $tags = Bugzilla::Comment::TagWeights->match({ + WHERE => { + 'tag LIKE ?' => "\%$query\%", + }, + LIMIT => $limit, + }); + return [ map { $_->tag } @$tags ]; +} + ############################## # Private Helper Subroutines # ############################## @@ -836,18 +1259,13 @@ sub _bug_to_hash { # All the basic bug attributes are here, in alphabetical order. # A bug attribute is "basic" if it doesn't require an additional # database call to get the info. - my %item = ( + my %item = %{ filter $params, { alias => $self->type('string', $bug->alias), - classification => $self->type('string', $bug->classification), - component => $self->type('string', $bug->component), - creation_time => $self->type('dateTime', $bug->creation_ts), id => $self->type('int', $bug->bug_id), is_confirmed => $self->type('boolean', $bug->everconfirmed), - last_change_time => $self->type('dateTime', $bug->delta_ts), op_sys => $self->type('string', $bug->op_sys), platform => $self->type('string', $bug->rep_platform), priority => $self->type('string', $bug->priority), - product => $self->type('string', $bug->product), resolution => $self->type('string', $bug->resolution), severity => $self->type('string', $bug->bug_severity), status => $self->type('string', $bug->bug_status), @@ -856,25 +1274,35 @@ sub _bug_to_hash { url => $self->type('string', $bug->bug_file_loc), version => $self->type('string', $bug->version), whiteboard => $self->type('string', $bug->status_whiteboard), - ); + } }; - - # First we handle any fields that require extra SQL calls. - # We don't do the SQL calls at all if the filter would just - # eliminate them anyway. + # First we handle any fields that require extra work (such as date parsing + # or SQL calls). if (filter_wants $params, 'assigned_to') { - $item{'assigned_to'} = $self->type('string', $bug->assigned_to->login); + $item{'assigned_to'} = $self->type('email', $bug->assigned_to->login); + $item{'assigned_to_detail'} = $self->_user_to_hash($bug->assigned_to, $params, undef, 'assigned_to'); } if (filter_wants $params, 'blocks') { my @blocks = map { $self->type('int', $_) } @{ $bug->blocked }; $item{'blocks'} = \@blocks; } + if (filter_wants $params, 'classification') { + $item{classification} = $self->type('string', $bug->classification); + } + if (filter_wants $params, 'component') { + $item{component} = $self->type('string', $bug->component); + } if (filter_wants $params, 'cc') { - my @cc = map { $self->type('string', $_) } @{ $bug->cc || [] }; + my @cc = map { $self->type('email', $_) } @{ $bug->cc || [] }; $item{'cc'} = \@cc; + $item{'cc_detail'} = [ map { $self->_user_to_hash($_, $params, undef, 'cc') } @{ $bug->cc_users } ]; + } + if (filter_wants $params, 'creation_time') { + $item{'creation_time'} = $self->type('dateTime', $bug->creation_ts); } if (filter_wants $params, 'creator') { - $item{'creator'} = $self->type('string', $bug->reporter->login); + $item{'creator'} = $self->type('email', $bug->reporter->login); + $item{'creator_detail'} = $self->_user_to_hash($bug->reporter, $params, undef, 'creator'); } if (filter_wants $params, 'depends_on') { my @depends_on = map { $self->type('int', $_) } @{ $bug->dependson }; @@ -896,25 +1324,40 @@ sub _bug_to_hash { @{ $bug->keyword_objects }; $item{'keywords'} = \@keywords; } + if (filter_wants $params, 'last_change_time') { + $item{'last_change_time'} = $self->type('dateTime', $bug->delta_ts); + } + if (filter_wants $params, 'product') { + $item{product} = $self->type('string', $bug->product); + } if (filter_wants $params, 'qa_contact') { my $qa_login = $bug->qa_contact ? $bug->qa_contact->login : ''; - $item{'qa_contact'} = $self->type('string', $qa_login); + $item{'qa_contact'} = $self->type('email', $qa_login); + if ($bug->qa_contact) { + $item{'qa_contact_detail'} = $self->_user_to_hash($bug->qa_contact, $params, undef, 'qa_contact'); + } } if (filter_wants $params, 'see_also') { my @see_also = map { $self->type('string', $_->name) } @{ $bug->see_also }; $item{'see_also'} = \@see_also; } + if (filter_wants $params, 'flags') { + $item{'flags'} = [ map { $self->_flag_to_hash($_) } @{$bug->flags} ]; + } # And now custom fields - my @custom_fields = Bugzilla->active_custom_fields; + my @custom_fields = Bugzilla->active_custom_fields({ + product => $bug->product_obj, component => $bug->component_obj, bug_id => $bug->id }); foreach my $field (@custom_fields) { my $name = $field->name; - next if !filter_wants $params, $name; + next if !filter_wants($params, $name, ['default', 'custom']); if ($field->type == FIELD_TYPE_BUG_ID) { $item{$name} = $self->type('int', $bug->$name); } - elsif ($field->type == FIELD_TYPE_DATETIME) { + elsif ($field->type == FIELD_TYPE_DATETIME + || $field->type == FIELD_TYPE_DATE) + { $item{$name} = $self->type('dateTime', $bug->$name); } elsif ($field->type == FIELD_TYPE_MULTI_SELECT) { @@ -928,33 +1371,55 @@ sub _bug_to_hash { # Timetracking fields are only sent if the user can see them. if (Bugzilla->user->is_timetracker) { - $item{'estimated_time'} = $self->type('double', $bug->estimated_time); - $item{'remaining_time'} = $self->type('double', $bug->remaining_time); - # No need to format $bug->deadline specially, because Bugzilla::Bug - # already does it for us. - $item{'deadline'} = $self->type('string', $bug->deadline); - } - - if (Bugzilla->user->id) { - my $token = issue_hash_token([$bug->id, $bug->delta_ts]); - $item{'update_token'} = $self->type('string', $token); + if (filter_wants $params, 'estimated_time') { + $item{'estimated_time'} = $self->type('double', $bug->estimated_time); + } + if (filter_wants $params, 'remaining_time') { + $item{'remaining_time'} = $self->type('double', $bug->remaining_time); + } + if (filter_wants $params, 'deadline') { + # No need to format $bug->deadline specially, because Bugzilla::Bug + # already does it for us. + $item{'deadline'} = $self->type('string', $bug->deadline); + } + if (filter_wants $params, 'actual_time') { + $item{'actual_time'} = $self->type('double', $bug->actual_time); + } } # The "accessible" bits go here because they have long names and it # makes the code look nicer to separate them out. - $item{'is_cc_accessible'} = $self->type('boolean', - $bug->cclist_accessible); - $item{'is_creator_accessible'} = $self->type('boolean', - $bug->reporter_accessible); + if (filter_wants $params, 'is_cc_accessible') { + $item{'is_cc_accessible'} = $self->type('boolean', $bug->cclist_accessible); + } + if (filter_wants $params, 'is_creator_accessible') { + $item{'is_creator_accessible'} = $self->type('boolean', $bug->reporter_accessible); + } - return filter $params, \%item; + # BMO - support for special mentors field + if (filter_wants $params, 'mentors') { + $item{'mentors'} + = [ map { $self->type('email', $_->login) } @{ $bug->mentors || [] } ]; + $item{'mentors_detail'} + = [ map { $self->_user_to_hash($_, $params, undef, 'mentors') } @{ $bug->mentors } ]; + } + + return \%item; } -sub _attachment_to_hash { - my ($self, $attach, $filters) = @_; +sub _user_to_hash { + my ($self, $user, $filters, $types, $prefix) = @_; + my $item = filter $filters, { + id => $self->type('int', $user->id), + real_name => $self->type('string', $user->name), + name => $self->type('email', $user->login), + email => $self->type('email', $user->email), + }, $types, $prefix; + return $item; +} - # Skipping attachment flags for now. - delete $attach->{flags}; +sub _attachment_to_hash { + my ($self, $attach, $filters, $types, $prefix) = @_; my $item = filter $filters, { creation_time => $self->type('dateTime', $attach->attached), @@ -968,23 +1433,114 @@ sub _attachment_to_hash { is_private => $self->type('int', $attach->isprivate), is_obsolete => $self->type('int', $attach->isobsolete), is_patch => $self->type('int', $attach->ispatch), - }; + }, $types, $prefix; # creator/attacher require an extra lookup, so we only send them if # the filter wants them. foreach my $field (qw(creator attacher)) { - if (filter_wants $filters, $field) { - $item->{$field} = $self->type('string', $attach->attacher->login); + if (filter_wants $filters, $field, $types, $prefix) { + $item->{$field} = $self->type('email', $attach->attacher->login); } } - if (filter_wants $filters, 'data') { + if (filter_wants $filters, 'data', $types, $prefix) { $item->{'data'} = $self->type('base64', $attach->data); } + if (filter_wants $filters, 'size', $types, $prefix) { + $item->{'size'} = $self->type('int', $attach->datasize); + } + + if (filter_wants $filters, 'flags', $types, $prefix) { + $item->{'flags'} = [ map { $self->_flag_to_hash($_) } @{$attach->flags} ]; + } + + return $item; +} + +sub _flag_to_hash { + my ($self, $flag) = @_; + + my $item = { + id => $self->type('int', $flag->id), + name => $self->type('string', $flag->name), + type_id => $self->type('int', $flag->type_id), + creation_date => $self->type('dateTime', $flag->creation_date), + modification_date => $self->type('dateTime', $flag->modification_date), + status => $self->type('string', $flag->status) + }; + + foreach my $field (qw(setter requestee)) { + my $field_id = $field . "_id"; + $item->{$field} = $self->type('email', $flag->$field->login) + if $flag->$field_id; + } + + return $item; +} + +sub _flagtype_to_hash { + my ($self, $flagtype, $product) = @_; + my $user = Bugzilla->user; + + my @values = ('X'); + push(@values, '?') if ($flagtype->is_requestable && $user->can_request_flag($flagtype)); + push(@values, '+', '-') if $user->can_set_flag($flagtype); + + my $item = { + id => $self->type('int' , $flagtype->id), + name => $self->type('string' , $flagtype->name), + description => $self->type('string' , $flagtype->description), + type => $self->type('string' , $flagtype->target_type), + values => \@values, + is_active => $self->type('boolean', $flagtype->is_active), + is_requesteeble => $self->type('boolean', $flagtype->is_requesteeble), + is_multiplicable => $self->type('boolean', $flagtype->is_multiplicable) + }; + + if ($product) { + my $inclusions = $self->_flagtype_clusions_to_hash($flagtype->inclusions, $product->id); + my $exclusions = $self->_flagtype_clusions_to_hash($flagtype->exclusions, $product->id); + # if we have both inclusions and exclusions, the exclusions are redundant + $exclusions = [] if @$inclusions && @$exclusions; + # no need to return anything if there's just "any component" + $item->{inclusions} = $inclusions if @$inclusions && $inclusions->[0] ne ''; + $item->{exclusions} = $exclusions if @$exclusions && $exclusions->[0] ne ''; + } + return $item; } +sub _flagtype_clusions_to_hash { + my ($self, $clusions, $product_id) = @_; + my $result = []; + foreach my $key (keys %$clusions) { + my ($prod_id, $comp_id) = split(/:/, $clusions->{$key}, 2); + if ($prod_id == 0 || $prod_id == $product_id) { + if ($comp_id) { + my $component = Bugzilla::Component->new({ id => $comp_id, cache => 1 }); + push @$result, $component->name; + } + else { + return [ '' ]; + } + } + } + return $result; +} + +sub _add_update_tokens { + my ($self, $params, $bugs, $hashes) = @_; + + return if !Bugzilla->user->id; + return if !filter_wants($params, 'update_token'); + + for(my $i = 0; $i < @$bugs; $i++) { + my $token = issue_hash_token([$bugs->[$i]->id, $bugs->[$i]->delta_ts]); + $hashes->[$i]->{'update_token'} = $self->type('string', $token); + } +} + 1; __END__ @@ -1004,6 +1560,10 @@ or get information about bugs that have already been filed. See L<Bugzilla::WebService> for a description of how parameters are passed, and what B<STABLE>, B<UNSTABLE>, and B<EXPERIMENTAL> mean. +Although the data input and output is the same for JSONRPC, XMLRPC and REST, +the directions for how to access the data via REST is noted in each method +where applicable. + =head1 Utility Functions =head2 fields @@ -1017,11 +1577,26 @@ B<UNSTABLE> Get information about valid bug fields, including the lists of legal values for each field. +=item B<REST> + +You have several options for retreiving information about fields. The first +part is the request method and the rest is the related path needed. + +To get information about all fields: + +GET /field/bug + +To get information related to a single field: + +GET /field/bug/<id_or_name> + +The returned data format is the same as below. + =item B<Params> You can pass either field ids or field names. -B<Note>: If neither C<ids> nor C<names> is specified, then all +B<Note>: If neither C<ids> nor C<names> is specified, then all non-obsolete fields will be returned. In addition to the parameters below, this method also accepts the @@ -1120,7 +1695,7 @@ values of the field are shown in the user interface. Can be null. This is an array of hashes, representing the legal values for select-type (drop-down and multiple-selection) fields. This is also -populated for the C<component>, C<version>, and C<target_milestone> +populated for the C<component>, C<version>, C<target_milestone>, and C<keywords> fields, but not for the C<product> field (you must use L<Product.get_accessible_products|Bugzilla::WebService::Product/get_accessible_products> for that. @@ -1153,6 +1728,11 @@ if the C<value_field> is set to one of the values listed in this array. Note that for per-product fields, C<value_field> is set to C<'product'> and C<visibility_values> will reflect which product(s) this value appears in. +=item C<description> + +C<string> The description of the value. This item is only included for the +C<keywords> field. + =item C<is_open> C<boolean> For C<bug_status> values, determines whether this status @@ -1206,8 +1786,107 @@ You specified an invalid field name or id. =back +=item REST API call added in Bugzilla B<5.0> + =back +=head2 flag_types + +B<UNSTABLE> + +=over + +=item B<Description> + +Get information about valid flag types that can be set for bugs and attachments. + +=item B<REST> + +You have several options for retreiving information about flag types. The first +part is the request method and the rest is the related path needed. + +To get information about all flag types for a product: + +GET /flag_types/<product> + +To get information about flag_types for a product and component: + +GET /flag_types/<product>/<component> + +The returned data format is the same as below. + +=item B<Params> + +You must pass a product name and an optional component name. + +=over + +=item C<product> (string) - The name of a valid product. + +=item C<component> (string) - An optional valid component name associated with the product. + +=back + +=item B<Returns> + +A hash containing two keys, C<bug> and C<attachment>. Each key value is an array of hashes, +containing the following keys: + +=over + +=item C<id> + +C<int> An integer id uniquely identifying this flag type. + +=item C<name> + +C<string> The name for the flag type. + +=item C<type> + +C<string> The target of the flag type which is either C<bug> or C<attachment>. + +=item C<description> + +C<string> The description of the flag type. + +=item C<values> + +C<array> An array of string values that the user can set on the flag type. + +=item C<is_requesteeble> + +C<boolean> Users can ask specific other users to set flags of this type. + +=item C<is_multiplicable> + +C<boolean> Multiple flags of this type can be set for the same bug or attachment. + +=back + +=item B<Errors> + +=over + +=item 106 (Product Access Denied) + +Either the product does not exist or you don't have access to it. + +=item 51 (Invalid Component) + +The component provided does not exist in the product. + +=back + +=item B<History> + +=over + +=item Added in Bugzilla B<5.0>. + +=back + +=back =head2 legal_values @@ -1219,6 +1898,18 @@ B<DEPRECATED> - Use L</fields> instead. Tells you what values are allowed for a particular field. +=item B<REST> + +To get information on the values for a field based on field name: + +GET /field/bug/<field_name>/values + +To get information based on field name and a specific product: + +GET /field/bug/<field_name>/<product_id>/values + +The returned data format is the same as below. + =item B<Params> =over @@ -1251,6 +1942,14 @@ You specified a field that doesn't exist or isn't a drop-down field. =back +=item B<History> + +=over + +=item REST API call added in Bugzilla B<5.0>. + +=back + =back =head1 Bug Information @@ -1269,6 +1968,18 @@ and/or attachment ids. B<Note>: Private attachments will only be returned if you are in the insidergroup or if you are the submitter of the attachment. +=item B<REST> + +To get all current attachments for a bug: + +GET /bug/<bug_id>/attachment + +To get a specific attachment based on attachment ID: + +GET /bug/attachment/<attachment_id> + +The returned data format is the same as below. + =item B<Params> B<Note>: At least one of C<ids> or C<attachment_ids> is required. @@ -1329,6 +2040,10 @@ diagram above) are: C<base64> The raw data of the attachment, encoded as Base64. +=item C<size> + +C<int> The length (in bytes) of the attachment. + =item C<creation_time> C<dateTime> The time the attachment was created. @@ -1382,6 +2097,48 @@ Also returned as C<attacher>, for backwards-compatibility with older Bugzillas. (However, this backwards-compatibility will go away in Bugzilla 5.0.) +=item C<flags> + +An array of hashes containing the information about flags currently set +for each attachment. Each flag hash contains the following items: + +=over + +=item C<id> + +C<int> The id of the flag. + +=item C<name> + +C<string> The name of the flag. + +=item C<type_id> + +C<int> The type id of the flag. + +=item C<creation_date> + +C<dateTime> The timestamp when this flag was originally created. + +=item C<modification_date> + +C<dateTime> The timestamp when the flag was last modified. + +=item C<status> + +C<string> The current status of the flag. + +=item C<setter> + +C<string> The login name of the user who created or last modified the flag. + +=item C<requestee> + +C<string> The login name of the user this flag has been requested to be granted or denied. +Note, this field is only returned if a requestee is set. + +=back + =back =item B<Errors> @@ -1418,6 +2175,10 @@ C<summary>. =back +=item The C<flags> array was added in Bugzilla B<4.4>. + +=item REST API call added in Bugzilla B<5.0>. + =back @@ -1432,6 +2193,18 @@ B<STABLE> This allows you to get data about comments, given a list of bugs and/or comment ids. +=item B<REST> + +To get all comments for a particular bug using the bug ID or alias: + +GET /bug/<id_or_alias>/comment + +To get a specific comment based on the comment ID: + +GET /bug/comment/<comment_id> + +The returned data format is the same as below. + =item B<Params> B<Note>: At least one of C<ids> or C<comment_ids> is required. @@ -1522,6 +2295,13 @@ Bugzillas. (However, this backwards-compatibility will go away in Bugzilla C<dateTime> The time (in Bugzilla's timezone) that the comment was added. +=item creation_time + +C<dateTime> This is exactly same as the C<time> key. Use this field instead of +C<time> for consistency with other methods including L</get> and L</attachments>. +For compatibility, C<time> is still usable. However, please note that C<time> +may be deprecated and removed in a future release. + =item is_private C<boolean> True if this comment is private (only visible to a certain @@ -1563,6 +2343,10 @@ C<creator>. =back +=item C<creation_time> was added in Bugzilla B<4.4>. + +=item REST API call added in Bugzilla B<5.0>. + =back @@ -1578,6 +2362,14 @@ Gets information about particular bugs in the database. Note: Can also be called as "get_bugs" for compatibilty with Bugzilla 3.0 API. +=item B<REST> + +To get information about a particular bug using its ID or alias: + +GET /bug/<id_or_alias> + +The returned data format is the same as below. + =item B<Params> In addition to the parameters below, this method also accepts the @@ -1620,8 +2412,18 @@ Two items are returned: An array of hashes that contains information about the bugs with the valid ids. Each hash contains the following items: +These fields are returned by default or by specifying C<_default> +in C<include_fields>. + =over +=item C<actual_time> + +C<double> The total number of hours that this bug has taken (so far). + +If you are not in the time-tracking group, this field will not be included +in the return value. + =item C<alias> C<string> The unique alias of this bug. @@ -1630,6 +2432,11 @@ C<string> The unique alias of this bug. C<string> The login name of the user to whom the bug is assigned. +=item C<assigned_to_detail> + +C<hash> A hash containing detailed user information for the assigned_to. To see the +keys included in the user detail hash, see below. + =item C<blocks> C<array> of C<int>s. The ids of bugs that are "blocked" by this bug. @@ -1639,6 +2446,11 @@ C<array> of C<int>s. The ids of bugs that are "blocked" by this bug. C<array> of C<string>s. The login names of users on the CC list of this bug. +=item C<cc_detail> + +C<array> of hashes containing detailed user information for each of the cc list +members. To see the keys included in the user detail hash, see below. + =item C<classification> C<string> The name of the current classification the bug is in. @@ -1655,6 +2467,11 @@ C<dateTime> When the bug was created. C<string> The login name of the person who filed this bug (the reporter). +=item C<creator_detail> + +C<hash> A hash containing detailed user information for the creator. To see the +keys included in the user detail hash, see below. + =item C<deadline> C<string> The day that this bug is due to be completed, in the format @@ -1680,6 +2497,48 @@ take. If you are not in the time-tracking group, this field will not be included in the return value. +=item C<flags> + +An array of hashes containing the information about flags currently set +for the bug. Each flag hash contains the following items: + +=over + +=item C<id> + +C<int> The id of the flag. + +=item C<name> + +C<string> The name of the flag. + +=item C<type_id> + +C<int> The type id of the flag. + +=item C<creation_date> + +C<dateTime> The timestamp when this flag was originally created. + +=item C<modification_date> + +C<dateTime> The timestamp when the flag was last modified. + +=item C<status> + +C<string> The current status of the flag. + +=item C<setter> + +C<string> The login name of the user who created or last modified the flag. + +=item C<requestee> + +C<string> The login name of the user this flag has been requested to be granted or denied. +Note, this field is only returned if a requestee is set. + +=back + =item C<groups> C<array> of C<string>s. The names of all the groups that this bug is in. @@ -1737,6 +2596,11 @@ C<string> The name of the product this bug is in. C<string> The login name of the current QA Contact on the bug. +=item C<qa_contact_detail> + +C<hash> A hash containing detailed user information for the qa_contact. To see the +keys included in the user detail hash, see below. + =item C<remaining_time> C<double> The number of hours of work remaining until work on this bug @@ -1800,7 +2664,11 @@ C<string> The value of the "status whiteboard" field on the bug. Every custom field in this installation will also be included in the return value. Most fields are returned as C<string>s. However, some -field types have different return values: +field types have different return values. + +Normally custom fields are returned by default similar to normal bug +fields or you can specify only custom fields by using C<_custom> in +C<include_fields>. =over @@ -1812,6 +2680,30 @@ field types have different return values: =back +=item I<user detail hashes> + +Each user detail hash contains the following items: + +=over + +=item C<id> + +C<int> The user id for this user. + +=item C<real_name> + +C<string> The 'real' name for this user, if any. + +=item C<name> + +C<string> The user's Bugzilla login. + +=item C<email> + +C<string> The user's email address. Currently this is the same value as the name. + +=back + =back =item C<faults> B<EXPERIMENTAL> @@ -1869,6 +2761,8 @@ You do not have access to the bug_id you specified. =item The following properties were added to this method's return values in Bugzilla B<3.4>: +=item REST API call added in Bugzilla B<5.0> + =over =item For C<bugs> @@ -1907,8 +2801,12 @@ C<op_sys>, C<platform>, C<qa_contact>, C<remaining_time>, C<see_also>, C<target_milestone>, C<update_token>, C<url>, C<version>, C<whiteboard>, and all custom fields. -=back +=item The C<flags> array was added in Bugzilla B<4.4>. +=item The C<actual_time> item was added to the C<bugs> return value +in Bugzilla B<4.4>. + +=back =back @@ -1922,6 +2820,14 @@ B<EXPERIMENTAL> Gets the history of changes for particular bugs in the database. +=item B<REST> + +To get the history for a specific bug ID: + +GET /bug/<bug_id>/history + +The returned data format will be the same as below. + =item B<Params> =over @@ -1933,7 +2839,12 @@ An array of numbers and strings. If an element in the array is entirely numeric, it represents a bug_id from the Bugzilla database to fetch. If it contains any non-numeric characters, it is considered to be a bug alias instead, and the data bug -with that alias will be loaded. +with that alias will be loaded. + +item C<new_since> + +C<dateTime> If specified, the method will only return changes I<newer> +than this time. Note that it's possible for aliases to be disabled in Bugzilla, in which case you will be told that you have specified an invalid bug_id if you @@ -2002,6 +2913,8 @@ present in this hash. =back +=item REST API call added Bugzilla B<5.0>. + =back =item B<Errors> @@ -2014,6 +2927,10 @@ The same as L</get>. =item Added in Bugzilla B<3.4>. +=item Field names changed to be more consistent with other methods in Bugzilla B<4.4>. + +=item As of Bugzilla B<4.4>, field names now match names used by L<Bug.update|/"update"> for consistency. + =back =back @@ -2082,6 +2999,14 @@ B<UNSTABLE> Allows you to search for bugs based on particular criteria. +=item <REST> + +To search for bugs: + +GET /bug + +The URL parameters and the returned data format are the same as below. + =item B<Params> Unless otherwise specified in the description of a parameter, bugs are @@ -2102,10 +3027,19 @@ the "Foo" or "Bar" products, you'd pass: product => ['Foo', 'Bar'] Some Bugzillas may treat your arguments case-sensitively, depending -on what database system they are using. Most commonly, though, Bugzilla is -not case-sensitive with the arguments passed (because MySQL is the +on what database system they are using. Most commonly, though, Bugzilla is +not case-sensitive with the arguments passed (because MySQL is the most-common database to use with Bugzilla, and MySQL is not case sensitive). +In addition to the fields listed below, you may also use criteria that +is similar to what is used in the Advanced Search screen of the Bugzilla +UI. This includes fields specified by C<Search by Change History> and +C<Custom Search>. The easiest way to determine what the field names are and what +format Bugzilla expects, is to first construct your query using the +Advanced Search UI, execute it and use the query parameters in they URL +as your key/value pairs for the WebService call. With REST, you can +just reuse the query parameter portion in the REST call itself. + =over =item C<alias> @@ -2230,6 +3164,15 @@ C<string> Search the "Status Whiteboard" field on bugs for a substring. Works the same as the C<summary> field described above, but searches the Status Whiteboard field. +=item C<count_only> + +C<boolean> If count_only set to true, only a single hash key called C<bug_count> +will be returned which is the number of bugs that matched the search. + +=item C<quicksearch> + +C<string> Search for bugs using quicksearch syntax. + =back =item B<Returns> @@ -2269,6 +3212,13 @@ in Bugzilla B<4.0>. C<limit> is set equal to zero. Otherwise maximum results returned are limited by system configuration. +=item REST API call added in Bugzilla B<5.0>. + +=item Updated to allow for full search capability similar to the Bugzilla UI +in Bugzilla B<5.0>. + +=item Updated to allow quicksearch capability in Bugzilla B<5.0>. + =back =back @@ -2295,10 +3245,19 @@ The WebService interface may allow you to set things other than those listed here, but realize that anything undocumented is B<UNSTABLE> and will very likely change in the future. +=item B<REST> + +To create a new bug in Bugzilla: + +POST /bug + +The params to include in the POST body as well as the returned data format, +are the same as below. + =item B<Params> Some params must be set, or an error will be thrown. These params are -marked B<Required>. +marked B<Required>. Some parameters can have defaults set in Bugzilla, by the administrator. If these parameters have defaults set, you can omit them. These parameters @@ -2372,6 +3331,32 @@ Note that only certain statuses can be set on bug creation. =item C<target_milestone> (string) - A valid target milestone for this product. +=item C<flags> + +C<array> An array of hashes with flags to add to the bug. To create a flag, +at least the status and the type_id or name must be provided. An optional +requestee can be passed if the flag type is requestable to a specific user. + +=over + +=item C<name> + +C<string> The name of the flag type. + +=item C<type_id> + +C<int> The internal flag type id. + +=item C<status> + +C<string> The flags new status (i.e. "?", "+", "-" or "X" to clear a flag). + +=item C<requestee> + +C<string> The login of the requestee if the flag type is requestable to a specific user. + +=back + =back In addition to the above parameters, if your installation has any custom @@ -2424,6 +3409,28 @@ that would cause a circular dependency between bugs. You tried to restrict the bug to a group which does not exist, or which you cannot use with this product. +=item 129 (Flag Status Invalid) + +The flag status is invalid. + +=item 130 (Flag Modification Denied) + +You tried to request, grant, or deny a flag but only a user with the required +permissions may make the change. + +=item 131 (Flag not Requestable from Specific Person) + +You can't ask a specific person for the flag. + +=item 133 (Flag Type not Unique) + +The flag type specified matches several flag types. You must specify +the type id value to update or add a flag. + +=item 134 (Inactive Flag Type) + +The flag type is inactive and cannot be used to create new flags. + =item 504 (Invalid User) Either the QA Contact, Assignee, or CC lists have some invalid user @@ -2453,6 +3460,8 @@ loop errors had a generic code of C<32000>. =back +=item REST API call added in Bugzilla B<5.0>. + =back @@ -2466,6 +3475,16 @@ B<UNSTABLE> This allows you to add an attachment to a bug in Bugzilla. +=item B<REST> + +To create attachment on a current bug: + +POST /bug/<bug_id>/attachment + +The params to include in the POST body, as well as the returned +data format are the same as below. The C<ids> param will be +overridden as it it pulled from the URL path. + =item B<Params> =over @@ -2517,6 +3536,32 @@ to the "insidergroup"), False if the attachment should be public. Defaults to False if not specified. +=item C<flags> + +C<array> An array of hashes with flags to add to the attachment. to create a flag, +at least the status and the type_id or name must be provided. An optional requestee +can be passed if the flag type is requestable to a specific user. + +=over + +=item C<name> + +C<string> The name of the flag type. + +=item C<type_id> + +C<int> The internal flag type id. + +=item C<status> + +C<string> The flags new status (i.e. "?", "+", "-" or "X" to clear a flag). + +=item C<requestee> + +C<string> The login of the requestee if the flag type is requestable to a specific user. + +=back + =back =item B<Returns> @@ -2531,6 +3576,28 @@ This method can throw all the same errors as L</get>, plus: =over +=item 129 (Flag Status Invalid) + +The flag status is invalid. + +=item 130 (Flag Modification Denied) + +You tried to request, grant, or deny a flag but only a user with the required +permissions may make the change. + +=item 131 (Flag not Requestable from Specific Person) + +You can't ask a specific person for the flag. + +=item 133 (Flag Type not Unique) + +The flag type specified matches several flag types. You must specify +the type id value to update or add a flag. + +=item 134 (Inactive Flag Type) + +The flag type is inactive and cannot be used to create new flags. + =item 600 (Attachment Too Large) You tried to attach a file that was larger than Bugzilla will accept. @@ -2564,8 +3631,227 @@ You set the "data" field to an empty string. =back +=item REST API call added in Bugzilla B<5.0>. + +=back + + +=head2 update_attachment + +B<UNSTABLE> + +=over + +=item B<Description> + +This allows you to update attachment metadata in Bugzilla. + +=item B<REST> + +To update attachment metadata on a current attachment: + +PUT /bug/attachment/<attach_id> + +The params to include in the POST body, as well as the returned +data format are the same as below. The C<ids> param will be +overridden as it it pulled from the URL path. + +=item B<Params> + +=over + +=item C<ids> + +B<Required> C<array> An array of integers -- the ids of the attachments you +want to update. + +=item C<file_name> + +C<string> The "file name" that will be displayed +in the UI for this attachment. + +=item C<summary> + +C<string> A short string describing the +attachment. + +=item C<comment> + +C<string> An optional comment to add to the attachment's bug. + +=item C<content_type> + +C<string> The MIME type of the attachment, like +C<text/plain> or C<image/png>. + +=item C<is_patch> + +C<boolean> True if Bugzilla should treat this attachment as a patch. +If you specify this, you do not need to specify a C<content_type>. +The C<content_type> of the attachment will be forced to C<text/plain>. + +=item C<is_private> + +C<boolean> True if the attachment should be private (restricted +to the "insidergroup"), False if the attachment should be public. + +=item C<is_obsolete> + +C<boolean> True if the attachment is obsolete, False otherwise. + +=item C<flags> + +C<array> An array of hashes with changes to the flags. The following values +can be specified. At least the status and one of type_id, id, or name must +be specified. If a type_id or name matches a single currently set flag, +the flag will be updated unless new is specified. + +=over + +=item C<name> + +C<string> The name of the flag that will be created or updated. + +=item C<type_id> + +C<int> The internal flag type id that will be created or updated. You will +need to specify the C<type_id> if more than one flag type of the same name exists. + +=item C<status> + +C<string> The flags new status (i.e. "?", "+", "-" or "X" to clear a flag). + +=item C<requestee> + +C<string> The login of the requestee if the flag type is requestable to a specific user. + +=item C<id> + +C<int> Use id to specify the flag to be updated. You will need to specify the C<id> +if more than one flag is set of the same name. + +=item C<new> + +C<boolean> Set to true if you specifically want a new flag to be created. + +=back + +=item B<Returns> + +A C<hash> with a single field, "attachment". This points to an array of hashes +with the following fields: + +=over + +=item C<id> + +C<int> The id of the attachment that was updated. + +=item C<last_change_time> + +C<dateTime> The exact time that this update was done at, for this attachment. +If no update was done (that is, no fields had their values changed and +no comment was added) then this will instead be the last time the attachment +was updated. + +=item C<changes> + +C<hash> The changes that were actually done on this bug. The keys are +the names of the fields that were changed, and the values are a hash +with two keys: + +=over + +=item C<added> (C<string>) The values that were added to this field. +possibly a comma-and-space-separated list if multiple values were added. + +=item C<removed> (C<string>) The values that were removed from this +field. + +=back + =back +Here's an example of what a return value might look like: + + { + attachments => [ + { + id => 123, + last_change_time => '2010-01-01T12:34:56', + changes => { + summary => { + removed => 'Sample ptach', + added => 'Sample patch' + }, + is_obsolete => { + removed => '0', + added => '1', + } + }, + } + ] + } + +=item B<Errors> + +This method can throw all the same errors as L</get>, plus: + +=over + +=item 129 (Flag Status Invalid) + +The flag status is invalid. + +=item 130 (Flag Modification Denied) + +You tried to request, grant, or deny a flag but only a user with the required +permissions may make the change. + +=item 131 (Flag not Requestable from Specific Person) + +You can't ask a specific person for the flag. + +=item 132 (Flag not Unique) + +The flag specified has been set multiple times. You must specify the id +value to update the flag. + +=item 133 (Flag Type not Unique) + +The flag type specified matches several flag types. You must specify +the type id value to update or add a flag. + +=item 134 (Inactive Flag Type) + +The flag type is inactive and cannot be used to create new flags. + +=item 601 (Invalid MIME Type) + +You specified a C<content_type> argument that was blank, not a valid +MIME type, or not a MIME type that Bugzilla accepts for attachments. + +=item 603 (File Name Not Specified) + +You did not specify a valid for the C<file_name> argument. + +=item 604 (Summary Required) + +You did not specify a value for the C<summary> argument. + +=back + +=item B<History> + +=over + +=item Added in Bugzilla B<5.0>. + +=back + +=back + +=back =head2 add_comment @@ -2577,6 +3863,15 @@ B<STABLE> This allows you to add a comment to a bug in Bugzilla. +=item B<REST> + +To create a comment on a current bug: + +POST /bug/<bug_id>/comment + +The params to include in the POST body as well as the returned data format, +are the same as below. + =item B<Params> =over @@ -2653,6 +3948,10 @@ purposes if you wish. =item Before Bugzilla B<3.6>, error 54 and error 114 had a generic error code of 32000. +=item REST API call added in Bugzilla B<5.0>. + +=item In Bugzilla B<5.0>, the following items were added to the bugs return value: C<assigned_to_detail>, C<creator_detail>, C<qa_contact_detail>. + =back =back @@ -2669,6 +3968,16 @@ B<UNSTABLE> Allows you to update the fields of a bug. Automatically sends emails out about the changes. +=item B<REST> + +To update the fields of a current bug: + +PUT /bug/<bug_id> + +The params to include in the PUT body as well as the returned data format, +are the same as below. The C<ids> param will be overridden as it is +pulled from the URL path. + =item B<Params> =over @@ -2788,6 +4097,43 @@ duplicate bugs. C<double> The total estimate of time required to fix the bug, in hours. This is the I<total> estimate, not the amount of time remaining to fix it. +=item C<flags> + +C<array> An array of hashes with changes to the flags. The following values +can be specified. At least the status and one of type_id, id, or name must +be specified. If a type_id or name matches a single currently set flag, +the flag will be updated unless new is specified. + +=over + +=item C<name> + +C<string> The name of the flag that will be created or updated. + +=item C<type_id> + +C<int> The internal flag type id that will be created or updated. You will +need to specify the C<type_id> if more than one flag type of the same name exists. + +=item C<status> + +C<string> The flags new status (i.e. "?", "+", "-" or "X" to clear a flag). + +=item C<requestee> + +C<string> The login of the requestee if the flag type is requestable to a specific user. + +=item C<id> + +C<int> Use id to specify the flag to be updated. You will need to specify the C<id> +if more than one flag is set of the same name. + +=item C<new> + +C<boolean> Set to true if you specifically want a new flag to be created. + +=back + =item C<groups> C<hash> The groups a bug is in. To modify this field, pass a hash, which @@ -3106,6 +4452,33 @@ field. You tried to change from one status to another, but the status workflow rules don't allow that change. +=item 129 (Flag Status Invalid) + +The flag status is invalid. + +=item 130 (Flag Modification Denied) + +You tried to request, grant, or deny a flag but only a user with the required +permissions may make the change. + +=item 131 (Flag not Requestable from Specific Person) + +You can't ask a specific person for the flag. + +=item 132 (Flag not Unique) + +The flag specified has been set multiple times. You must specify the id +value to update the flag. + +=item 133 (Flag Type not Unique) + +The flag type specified matches several flag types. You must specify +the type id value to update or add a flag. + +=item 134 (Inactive Flag Type) + +The flag type is inactive and cannot be used to create new flags. + =back =item B<History> @@ -3114,6 +4487,10 @@ rules don't allow that change. =item Added in Bugzilla B<4.0>. +=item REST API call added Bugzilla B<5.0>. + +=item Added C<new_since> parameter if Bugzilla B<5.0>. + =back =back @@ -3230,3 +4607,46 @@ this bug. =back =back + +=head2 render_comment + +B<UNSTABLE> + +=over + +=item B<Description> + +Returns the HTML rendering of the provided comment text. + +=item B<Params> + +=over + +=item C<text> + +B<Required> C<strings> Text comment text to render. + +=item C<id> + +C<int> The ID of the bug to render the comment against. + +=back + +=item B<Returns> + +C<html> containing the HTML rendering. + +=item B<Errors> + +This method can throw all of the errors that L</get> throws. + +=item B<History> + +=over + +=item Added in Bugzilla B<5.0>. + +=back + +=back + diff --git a/Bugzilla/WebService/BugUserLastVisit.pm b/Bugzilla/WebService/BugUserLastVisit.pm new file mode 100644 index 000000000..71b637fef --- /dev/null +++ b/Bugzilla/WebService/BugUserLastVisit.pm @@ -0,0 +1,208 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::WebService::BugUserLastVisit; + +use 5.10.1; +use strict; + +use parent qw(Bugzilla::WebService); + +use Bugzilla::Bug; +use Bugzilla::Error; +use Bugzilla::WebService::Util qw( validate filter ); +use Bugzilla::Constants; + +sub update { + my ($self, $params) = validate(@_, 'ids'); + my $user = Bugzilla->user; + my $dbh = Bugzilla->dbh; + + $user->login(LOGIN_REQUIRED); + + my $ids = $params->{ids} // []; + ThrowCodeError('param_required', { param => 'ids' }) unless @$ids; + + # Cache permissions for bugs. This highly reduces the number of calls to the + # DB. visible_bugs() is only able to handle bug IDs, so we have to skip + # aliases. + $user->visible_bugs([grep /^[0-9]$/, @$ids]); + + $dbh->bz_start_transaction(); + my @results; + my $last_visit_ts = $dbh->selectrow_array('SELECT NOW()'); + foreach my $bug_id (@$ids) { + my $bug = Bugzilla::Bug->check({ id => $bug_id, cache => 1 }); + + ThrowUserError('user_not_involved', { bug_id => $bug->id }) + unless $user->is_involved_in_bug($bug); + + $bug->update_user_last_visit($user, $last_visit_ts); + + push( + @results, + $self->_bug_user_last_visit_to_hash( + $bug, $last_visit_ts, $params + )); + } + $dbh->bz_commit_transaction(); + + return \@results; +} + +sub get { + my ($self, $params) = validate(@_, 'ids'); + my $user = Bugzilla->user; + my $ids = $params->{ids}; + + $user->login(LOGIN_REQUIRED); + + if ($ids) { + # Cache permissions for bugs. This highly reduces the number of calls to + # the DB. visible_bugs() is only able to handle bug IDs, so we have to + # skip aliases. + $user->visible_bugs([grep /^[0-9]$/, @$ids]); + } + + my @last_visits = @{ $user->last_visits }; + + if ($ids) { + # remove bugs that we arn't interested in if ids is passed in. + my %id_set = map { ($_ => 1) } @$ids; + @last_visits = grep { $id_set{ $_->bug_id } } @last_visits; + } + + return [ + map { + $self->_bug_user_last_visit_to_hash($_->bug_id, $_->last_visit_ts, + $params) + } @last_visits + ]; +} + +sub _bug_user_last_visit_to_hash { + my ($self, $bug_id, $last_visit_ts, $params) = @_; + + my %result = (id => $self->type('int', $bug_id), + last_visit_ts => $self->type('dateTime', $last_visit_ts)); + + return filter($params, \%result); +} + +1; + +__END__ +=head1 NAME + +Bugzilla::WebService::BugUserLastVisit - Find and Store the last time a user +visited a bug. + +=head1 METHODS + +See L<Bugzilla::WebService> for a description of how parameters are passed, +and what B<STABLE>, B<UNSTABLE>, and B<EXPERIMENTAL> mean. + +Although the data input and output is the same for JSONRPC, XMLRPC and REST, +the directions for how to access the data via REST is noted in each method +where applicable. + +=head2 update + +B<EXPERIMENTAL> + +=over + +=item B<Description> + +Update the last visit time for the specified bug and current user. + +=item B<REST> + +To add a single bug id: + + POST /rest/bug_user_last_visit/<bug-id> + +Tp add one or more bug ids at once: + + POST /rest/bug_user_last_visit + +The returned data format is the same as below. + +=item B<Params> + +=over + +=item C<ids> (array) - One or more bug ids to add. + +=back + +=item B<Returns> + +=over + +=item C<array> - An array of hashes containing the following: + +=over + +=item C<id> - (int) The bug id. + +=item C<last_visit_ts> - (string) The timestamp the user last visited the bug. + +=back + +=back + +=back + +=head2 get + +B<EXPERIMENTAL> + +=over + +=item B<Description> + +Get the last visited timestamp for one or more specified bug ids or get a +list of the last 20 visited bugs and their timestamps. + +=item B<REST> + +To return the last visited timestamp for a single bug id: + +GET /rest/bug_visit/<bug-id> + +To return more than one bug timestamp or the last 20: + +GET /rest/bug_visit + +The returned data format is the same as below. + +=item B<Params> + +=over + +=item C<ids> (integer) - One or more optional bug ids to get. + +=back + +=item B<Returns> + +=over + +=item C<array> - An array of hashes containing the following: + +=over + +=item C<id> - (int) The bug id. + +=item C<last_visit_ts> - (string) The timestamp the user last visited the bug. + +=back + +=back + +=back diff --git a/Bugzilla/WebService/Bugzilla.pm b/Bugzilla/WebService/Bugzilla.pm index efc822311..1fc15c3c3 100644 --- a/Bugzilla/WebService/Bugzilla.pm +++ b/Bugzilla/WebService/Bugzilla.pm @@ -98,6 +98,10 @@ This provides functions that tell you about Bugzilla in general. See L<Bugzilla::WebService> for a description of how parameters are passed, and what B<STABLE>, B<UNSTABLE>, and B<EXPERIMENTAL> mean. +Although the data input and output is the same for JSONRPC, XMLRPC and REST, +the directions for how to access the data via REST is noted in each method +where applicable. + =head2 version B<STABLE> @@ -108,6 +112,12 @@ B<STABLE> Returns the current version of Bugzilla. +=item B<REST> + +GET /version + +The returned data format is the same as below. + =item B<Params> (none) =item B<Returns> @@ -117,6 +127,14 @@ string. =item B<Errors> (none) +=item B<History> + +=over + +=item REST API call added in Bugzilla B<5.0>. + +=back + =back =head2 extensions @@ -130,6 +148,12 @@ B<EXPERIMENTAL> Gets information about the extensions that are currently installed and enabled in this Bugzilla. +=item B<REST> + +GET /extensions + +The returned data format is the same as below. + =item B<Params> (none) =item B<Returns> @@ -160,6 +184,8 @@ The return value looks something like this: that the extensions define themselves. Before 3.6, the names of the extensions depended on the directory they were in on the Bugzilla server. +=item REST API call added in Bugzilla B<5.0>. + =back =back @@ -175,6 +201,12 @@ Use L</time> instead. Returns the timezone that Bugzilla expects dates and times in. +=item B<REST> + +GET /timezone + +The returned data format is the same as below. + =item B<Params> (none) =item B<Returns> @@ -189,6 +221,8 @@ string in (+/-)XXXX (RFC 2822) format. =item As of Bugzilla B<3.6>, the timezone returned is always C<+0000> (the UTC timezone). +=item REST API call added in Bugzilla B<5.0>. + =back =back @@ -205,6 +239,12 @@ B<STABLE> Gets information about what time the Bugzilla server thinks it is, and what timezone it's running in. +=item B<REST> + +GET /time + +The returned data format is the same as below. + =item B<Params> (none) =item B<Returns> @@ -215,7 +255,7 @@ A struct with the following items: =item C<db_time> -C<dateTime> The current time in UTC, according to the Bugzilla +C<dateTime> The current time in UTC, according to the Bugzilla I<database server>. Note that Bugzilla assumes that the database and the webserver are running @@ -225,7 +265,7 @@ rely on for doing searches and other input to the WebService. =item C<web_time> -C<dateTime> This is the current time in UTC, according to Bugzilla's +C<dateTime> This is the current time in UTC, according to Bugzilla's I<web server>. This might be different by a second from C<db_time> since this comes from @@ -241,7 +281,7 @@ versions of Bugzilla before 3.6.) =item C<tz_name> C<string> The literal string C<UTC>. (Exists only for backwards-compatibility -with versions of Bugzilla before 3.6.) +with versions of Bugzilla before 3.6.) =item C<tz_short_name> @@ -265,6 +305,8 @@ with versions of Bugzilla before 3.6.) were in the UTC timezone, instead of returning information in the server's local timezone. +=item REST API call added in Bugzilla B<5.0>. + =back =back diff --git a/Bugzilla/WebService/Classification.pm b/Bugzilla/WebService/Classification.pm new file mode 100644 index 000000000..1c4226fd6 --- /dev/null +++ b/Bugzilla/WebService/Classification.pm @@ -0,0 +1,208 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::WebService::Classification; + +use 5.10.1; +use strict; + +use parent qw (Bugzilla::WebService); + +use Bugzilla::Classification; +use Bugzilla::Error; +use Bugzilla::WebService::Util qw(filter validate params_to_objects); + +use constant READ_ONLY => qw( + get +); + +sub get { + my ($self, $params) = validate(@_, 'names', 'ids'); + + defined $params->{names} || defined $params->{ids} + || ThrowCodeError('params_required', { function => 'Classification.get', + params => ['names', 'ids'] }); + + my $user = Bugzilla->user; + + Bugzilla->params->{'useclassification'} + || $user->in_group('editclassifications') + || ThrowUserError('auth_classification_not_enabled'); + + Bugzilla->switch_to_shadow_db; + + my @classification_objs = @{ params_to_objects($params, 'Bugzilla::Classification') }; + unless ($user->in_group('editclassifications')) { + my %selectable_class = map { $_->id => 1 } @{$user->get_selectable_classifications}; + @classification_objs = grep { $selectable_class{$_->id} } @classification_objs; + } + + my @classifications = map { $self->_classification_to_hash($_, $params) } @classification_objs; + + return { classifications => \@classifications }; +} + +sub _classification_to_hash { + my ($self, $classification, $params) = @_; + + my $user = Bugzilla->user; + return unless (Bugzilla->params->{'useclassification'} || $user->in_group('editclassifications')); + + my $products = $user->in_group('editclassifications') ? + $classification->products : $user->get_selectable_products($classification->id); + + return filter $params, { + id => $self->type('int', $classification->id), + name => $self->type('string', $classification->name), + description => $self->type('string', $classification->description), + sort_key => $self->type('int', $classification->sortkey), + products => [ map { $self->_product_to_hash($_, $params) } @$products ], + }; +} + +sub _product_to_hash { + my ($self, $product, $params) = @_; + + return filter $params, { + id => $self->type('int', $product->id), + name => $self->type('string', $product->name), + description => $self->type('string', $product->description), + }, undef, 'products'; +} + +1; + +__END__ + +=head1 NAME + +Bugzilla::Webservice::Classification - The Classification API + +=head1 DESCRIPTION + +This part of the Bugzilla API allows you to deal with the available Classifications. +You will be able to get information about them as well as manipulate them. + +=head1 METHODS + +See L<Bugzilla::WebService> for a description of how parameters are passed, +and what B<STABLE>, B<UNSTABLE>, and B<EXPERIMENTAL> mean. + +Although the data input and output is the same for JSONRPC, XMLRPC and REST, +the directions for how to access the data via REST is noted in each method +where applicable. + +=head1 Classification Retrieval + +=head2 get + +B<EXPERIMENTAL> + +=over + +=item B<Description> + +Returns a hash containing information about a set of classifications. + +=item B<REST> + +To return information on a single classification: + +GET /classification/<classification_id_or_name> + +The returned data format will be the same as below. + +=item B<Params> + +In addition to the parameters below, this method also accepts the +standard L<include_fields|Bugzilla::WebService/include_fields> and +L<exclude_fields|Bugzilla::WebService/exclude_fields> arguments. + +You could get classifications info by supplying their names and/or ids. +So, this method accepts the following parameters: + +=over + +=item C<ids> + +An array of classification ids. + +=item C<names> + +An array of classification names. + +=back + +=item B<Returns> + +A hash with the key C<classifications> and an array of hashes as the corresponding value. +Each element of the array represents a classification that the user is authorized to see +and has the following keys: + +=over + +=item C<id> + +C<int> The id of the classification. + +=item C<name> + +C<string> The name of the classification. + +=item C<description> + +C<string> The description of the classificaion. + +=item C<sort_key> + +C<int> The value which determines the order the classification is sorted. + +=item C<products> + +An array of hashes. The array contains the products the user is authorized to +access within the classification. Each hash has the following keys: + +=over + +=item C<name> + +C<string> The name of the product. + +=item C<id> + +C<int> The id of the product. + +=item C<description> + +C<string> The description of the product. + +=back + +=back + +=item B<Errors> + +=over + +=item 900 (Classification not enabled) + +Classification is not enabled on this installation. + +=back + +=item B<History> + +=over + +=item Added in Bugzilla B<4.4>. + +=item REST API call added in Bugzilla B<5.0>. + +=back + +=back + diff --git a/Bugzilla/WebService/Constants.pm b/Bugzilla/WebService/Constants.pm index 3207356fa..4d36b877f 100644 --- a/Bugzilla/WebService/Constants.pm +++ b/Bugzilla/WebService/Constants.pm @@ -22,9 +22,22 @@ use base qw(Exporter); our @EXPORT = qw( WS_ERROR_CODE + + STATUS_OK + STATUS_CREATED + STATUS_ACCEPTED + STATUS_NO_CONTENT + STATUS_MULTIPLE_CHOICES + STATUS_BAD_REQUEST + STATUS_NOT_FOUND + STATUS_GONE + REST_STATUS_CODE_MAP + ERROR_UNKNOWN_FATAL ERROR_UNKNOWN_TRANSIENT + XMLRPC_CONTENT_TYPE_WHITELIST + REST_CONTENT_TYPE_WHITELIST WS_DISPATCH ); @@ -93,7 +106,12 @@ use constant WS_ERROR_CODE => { comment_is_private => 110, comment_id_invalid => 111, comment_too_long => 114, - comment_invalid_isprivate => 117, + comment_invalid_isprivate => 117, + # Comment tagging + comment_tag_disabled => 125, + comment_tag_invalid => 126, + comment_tag_too_long => 127, + comment_tag_too_short => 128, # See Also errors bug_url_invalid => 112, bug_url_too_long => 112, @@ -116,6 +134,13 @@ use constant WS_ERROR_CODE => { missing_resolution => 121, resolution_not_allowed => 122, illegal_bug_status_transition => 123, + # Flag errors + flag_status_invalid => 129, + flag_update_denied => 130, + flag_type_requestee_disabled => 131, + flag_not_unique => 132, + flag_type_not_unique => 133, + flag_type_inactive => 134, # Authentication errors are usually 300-400. invalid_username_or_password => 300, @@ -177,8 +202,48 @@ use constant WS_ERROR_CODE => { unknown_method => -32601, json_rpc_post_only => 32610, json_rpc_invalid_callback => 32611, - xmlrpc_illegal_content_type => 32612, - json_rpc_illegal_content_type => 32613, + xmlrpc_illegal_content_type => 32612, + json_rpc_illegal_content_type => 32613, + rest_invalid_resource => 32614, +}; + +# RESTful webservices use the http status code +# to describe whether a call was successful or +# to describe the type of error that occurred. +use constant STATUS_OK => 200; +use constant STATUS_CREATED => 201; +use constant STATUS_ACCEPTED => 202; +use constant STATUS_NO_CONTENT => 204; +use constant STATUS_MULTIPLE_CHOICES => 300; +use constant STATUS_BAD_REQUEST => 400; +use constant STATUS_NOT_AUTHORIZED => 401; +use constant STATUS_NOT_FOUND => 404; +use constant STATUS_GONE => 410; + +# The integer value is the error code above returned by +# the related webvservice call. We choose the appropriate +# http status code based on the error code or use the +# default STATUS_BAD_REQUEST. +use constant REST_STATUS_CODE_MAP => { + 51 => STATUS_NOT_FOUND, + 101 => STATUS_NOT_FOUND, + 102 => STATUS_NOT_AUTHORIZED, + 106 => STATUS_NOT_AUTHORIZED, + 109 => STATUS_NOT_AUTHORIZED, + 110 => STATUS_NOT_AUTHORIZED, + 113 => STATUS_NOT_AUTHORIZED, + 115 => STATUS_NOT_AUTHORIZED, + 120 => STATUS_NOT_AUTHORIZED, + 300 => STATUS_NOT_AUTHORIZED, + 301 => STATUS_NOT_AUTHORIZED, + 302 => STATUS_NOT_AUTHORIZED, + 303 => STATUS_NOT_AUTHORIZED, + 304 => STATUS_NOT_AUTHORIZED, + 410 => STATUS_NOT_AUTHORIZED, + 504 => STATUS_NOT_AUTHORIZED, + 505 => STATUS_NOT_AUTHORIZED, + 32614 => STATUS_NOT_FOUND, + _default => STATUS_BAD_REQUEST }; # These are the fallback defaults for errors not in ERROR_CODE. @@ -192,6 +257,14 @@ use constant XMLRPC_CONTENT_TYPE_WHITELIST => qw( application/xml ); +# The first content type specified is used as the default. +use constant REST_CONTENT_TYPE_WHITELIST => qw( + application/json + application/javascript + text/javascript + text/html +); + sub WS_DISPATCH { # We "require" here instead of "use" above to avoid a dependency loop. require Bugzilla::Hook; @@ -199,11 +272,13 @@ sub WS_DISPATCH { Bugzilla::Hook::process('webservice', { dispatch => \%hook_dispatch }); my $dispatch = { - 'Bugzilla' => 'Bugzilla::WebService::Bugzilla', - 'Bug' => 'Bugzilla::WebService::Bug', - 'User' => 'Bugzilla::WebService::User', - 'Product' => 'Bugzilla::WebService::Product', - 'Group' => 'Bugzilla::WebService::Group', + 'Bugzilla' => 'Bugzilla::WebService::Bugzilla', + 'Bug' => 'Bugzilla::WebService::Bug', + 'Classification' => 'Bugzilla::WebService::Classification', + 'User' => 'Bugzilla::WebService::User', + 'Product' => 'Bugzilla::WebService::Product', + 'Group' => 'Bugzilla::WebService::Group', + 'BugUserLastVisit' => 'Bugzilla::WebService::BugUserLastVisit', %hook_dispatch }; return $dispatch; diff --git a/Bugzilla/WebService/Group.pm b/Bugzilla/WebService/Group.pm index 65feb7a1a..b571a1062 100644 --- a/Bugzilla/WebService/Group.pm +++ b/Bugzilla/WebService/Group.pm @@ -61,6 +61,10 @@ get information about them. See L<Bugzilla::WebService> for a description of how parameters are passed, and what B<STABLE>, B<UNSTABLE>, and B<EXPERIMENTAL> mean. +Although the data input and output is the same for JSONRPC, XMLRPC and REST, +the directions for how to access the data via REST is noted in each method +where applicable. + =head1 Group Creation =head2 create @@ -73,9 +77,16 @@ B<UNSTABLE> This allows you to create a new group in Bugzilla. -=item B<Params> +=item B<REST> + +POST /group + +The params to include in the POST body as well as the returned data format, +are the same as below. -Some params must be set, or an error will be thrown. These params are +=item B<Params> + +Some params must be set, or an error will be thrown. These params are marked B<Required>. =over @@ -96,7 +107,7 @@ name of the group. C<string> A regular expression. Any user whose Bugzilla username matches this regular expression will automatically be granted membership in this group. -=item C<is_active> +=item C<is_active> C<boolean> C<True> if new group can be used for bugs, C<False> if this is a group that will only contain users and no bugs will be restricted @@ -110,7 +121,7 @@ if they are in this group. =back -=item B<Returns> +=item B<Returns> A hash with one element, C<id>. This is the id of the newly-created group. @@ -136,6 +147,14 @@ You specified an invalid regular expression in the C<user_regexp> field. =back -=back +=item B<History> + +=over + +=item REST API call added in Bugzilla B<5.0>. + +=back + +=back =cut diff --git a/Bugzilla/WebService/Product.pm b/Bugzilla/WebService/Product.pm index 3cd0d0a6c..3be46bd6d 100644 --- a/Bugzilla/WebService/Product.pm +++ b/Bugzilla/WebService/Product.pm @@ -47,57 +47,95 @@ BEGIN { *get_products = \&get } # Get the ids of the products the user can search sub get_selectable_products { - return {ids => [map {$_->id} @{Bugzilla->user->get_selectable_products}]}; + Bugzilla->switch_to_shadow_db(); + return {ids => [map {$_->id} @{Bugzilla->user->get_selectable_products}]}; } # Get the ids of the products the user can enter bugs against sub get_enterable_products { - return {ids => [map {$_->id} @{Bugzilla->user->get_enterable_products}]}; + Bugzilla->switch_to_shadow_db(); + return {ids => [map {$_->id} @{Bugzilla->user->get_enterable_products}]}; } # Get the union of the products the user can search and enter bugs against. sub get_accessible_products { - return {ids => [map {$_->id} @{Bugzilla->user->get_accessible_products}]}; + Bugzilla->switch_to_shadow_db(); + return {ids => [map {$_->id} @{Bugzilla->user->get_accessible_products}]}; } # Get a list of actual products, based on list of ids or names sub get { - my ($self, $params) = validate(@_, 'ids', 'names'); - - # Only products that are in the users accessible products, - # can be allowed to be returned - my $accessible_products = Bugzilla->user->get_accessible_products; + my ($self, $params) = validate(@_, 'ids', 'names', 'type'); + my $user = Bugzilla->user; + + defined $params->{ids} || defined $params->{names} || defined $params->{type} + || ThrowCodeError("params_required", { function => "Product.get", + params => ['ids', 'names', 'type'] }); + + Bugzilla->switch_to_shadow_db(); + + my $products = []; + if (defined $params->{type}) { + my %product_hash; + foreach my $type (@{ $params->{type} }) { + my $result = []; + if ($type eq 'accessible') { + $result = $user->get_accessible_products(); + } + elsif ($type eq 'enterable') { + $result = $user->get_enterable_products(); + } + elsif ($type eq 'selectable') { + $result = $user->get_selectable_products(); + } + else { + ThrowUserError('get_products_invalid_type', + { type => $type }); + } + map { $product_hash{$_->id} = $_ } @$result; + } + $products = [ values %product_hash ]; + } + else { + $products = $user->get_accessible_products; + } - my @requested_accessible; + my @requested_products; if (defined $params->{ids}) { # Create a hash with the ids the user wants my %ids = map { $_ => 1 } @{$params->{ids}}; - - # Return the intersection of this, by grepping the ids from + + # Return the intersection of this, by grepping the ids from # accessible products. - push(@requested_accessible, - grep { $ids{$_->id} } @$accessible_products); + push(@requested_products, + grep { $ids{$_->id} } @$products); } if (defined $params->{names}) { # Create a hash with the names the user wants my %names = map { lc($_) => 1 } @{$params->{names}}; - - # Return the intersection of this, by grepping the names from + + # Return the intersection of this, by grepping the names from # accessible products, union'ed with products found by ID to # avoid duplicates foreach my $product (grep { $names{lc $_->name} } - @$accessible_products) { + @$products) { next if grep { $_->id == $product->id } - @requested_accessible; - push @requested_accessible, $product; + @requested_products; + push @requested_products, $product; } } + # If we just requested a specific type of products without + # specifying ids or names, then return the entire list. + if (!defined $params->{ids} && !defined $params->{names}) { + @requested_products = @$products; + } + # Now create a result entry for each. my @products = map { $self->_product_to_hash($params, $_) } - @requested_accessible; + @requested_products; return { products => \@products }; } @@ -105,7 +143,7 @@ sub create { my ($self, $params) = @_; Bugzilla->login(LOGIN_REQUIRED); - Bugzilla->user->in_group('editcomponents') + Bugzilla->user->in_group('editcomponents') || ThrowUserError("auth_failure", { group => "editcomponents", action => "add", object => "products"}); @@ -143,45 +181,88 @@ sub _product_to_hash { }; if (filter_wants($params, 'components')) { $field_data->{components} = [map { - $self->_component_to_hash($_) + $self->_component_to_hash($_, $params) } @{$product->components}]; } if (filter_wants($params, 'versions')) { $field_data->{versions} = [map { - $self->_version_to_hash($_) + $self->_version_to_hash($_, $params) } @{$product->versions}]; } if (filter_wants($params, 'milestones')) { $field_data->{milestones} = [map { - $self->_milestone_to_hash($_) + $self->_milestone_to_hash($_, $params) } @{$product->milestones}]; } return filter($params, $field_data); } sub _component_to_hash { - my ($self, $component) = @_; - return { + my ($self, $component, $params) = @_; + my $field_data = filter $params, { id => $self->type('int', $component->id), name => $self->type('string', $component->name), description => - $self->type('string' , $component->description), + $self->type('string', $component->description), default_assigned_to => - $self->type('string' , $component->default_assignee->login), + $self->type('email', $component->default_assignee->login), default_qa_contact => - $self->type('string' , $component->default_qa_contact->login), + $self->type('email', $component->default_qa_contact->login), sort_key => # sort_key is returned to match Bug.fields 0, is_active => $self->type('boolean', $component->is_active), - }; + }, undef, 'components'; + + if (filter_wants($params, 'flag_types', undef, 'components')) { + $field_data->{flag_types} = { + bug => + [map { + $self->_flag_type_to_hash($_) + } @{$component->flag_types->{'bug'}}], + attachment => + [map { + $self->_flag_type_to_hash($_) + } @{$component->flag_types->{'attachment'}}], + }; + } + + return $field_data; +} + +sub _flag_type_to_hash { + my ($self, $flag_type, $params) = @_; + return filter $params, { + id => + $self->type('int', $flag_type->id), + name => + $self->type('string', $flag_type->name), + description => + $self->type('string', $flag_type->description), + cc_list => + $self->type('string', $flag_type->cc_list), + sort_key => + $self->type('int', $flag_type->sortkey), + is_active => + $self->type('boolean', $flag_type->is_active), + is_requestable => + $self->type('boolean', $flag_type->is_requestable), + is_requesteeble => + $self->type('boolean', $flag_type->is_requesteeble), + is_multiplicable => + $self->type('boolean', $flag_type->is_multiplicable), + grant_group => + $self->type('int', $flag_type->grant_group_id), + request_group => + $self->type('int', $flag_type->request_group_id), + }, undef, 'flag_types'; } sub _version_to_hash { - my ($self, $version) = @_; - return { + my ($self, $version, $params) = @_; + return filter $params, { id => $self->type('int', $version->id), name => @@ -190,12 +271,12 @@ sub _version_to_hash { 0, is_active => $self->type('boolean', $version->is_active), - }; + }, undef, 'versions'; } sub _milestone_to_hash { - my ($self, $milestone) = @_; - return { + my ($self, $milestone, $params) = @_; + return filter $params, { id => $self->type('int', $milestone->id), name => @@ -204,7 +285,7 @@ sub _milestone_to_hash { $self->type('int', $milestone->sortkey), is_active => $self->type('boolean', $milestone->is_active), - }; + }, undef, 'milestones'; } 1; @@ -225,6 +306,10 @@ get information about them. See L<Bugzilla::WebService> for a description of how parameters are passed, and what B<STABLE>, B<UNSTABLE>, and B<EXPERIMENTAL> mean. +Although the data input and output is the same for JSONRPC, XMLRPC and REST, +the directions for how to access the data via REST is noted in each method +where applicable. + =head1 List Products =head2 get_selectable_products @@ -237,15 +322,29 @@ B<EXPERIMENTAL> Returns a list of the ids of the products the user can search on. +=item B<REST> + +GET /product_selectable + +the returned data format is same as below. + =item B<Params> (none) -=item B<Returns> +=item B<Returns> A hash containing one item, C<ids>, that contains an array of product ids. =item B<Errors> (none) +=item B<History> + +=over + +=item REST API call added in Bugzilla B<5.0>. + +=back + =back =head2 get_enterable_products @@ -259,6 +358,12 @@ B<EXPERIMENTAL> Returns a list of the ids of the products the user can enter bugs against. +=item B<REST> + +GET /product_enterable + +the returned data format is same as below. + =item B<Params> (none) =item B<Returns> @@ -268,6 +373,14 @@ ids. =item B<Errors> (none) +=item B<History> + +=over + +=item REST API call added in Bugzilla B<5.0>. + +=back + =back =head2 get_accessible_products @@ -281,6 +394,12 @@ B<UNSTABLE> Returns a list of the ids of the products the user can search or enter bugs against. +=item B<REST> + +GET /product_accessible + +the returned data format is same as below. + =item B<Params> (none) =item B<Returns> @@ -290,6 +409,14 @@ ids. =item B<Errors> (none) +=item B<History> + +=over + +=item REST API call added in Bugzilla B<5.0>. + +=back + =back =head2 get @@ -304,12 +431,32 @@ Returns a list of information about the products passed to it. Note: Can also be called as "get_products" for compatibilty with Bugzilla 3.0 API. +=item B<REST> + +To return information about a specific groups of products such as +C<accessible>, C<selectable>, or C<enterable>: + +GET /product?type=accessible + +To return information about a specific product by C<id> or C<name>: + +GET /product/<product_id_or_name> + +You can also return information about more than one specific product +by using the following in your query string: + +GET /product?ids=1&ids=2&ids=3 or GET /product?names=ProductOne&names=Product2 + +the returned data format is same as below. + =item B<Params> In addition to the parameters below, this method also accepts the standard L<include_fields|Bugzilla::WebService/include_fields> and L<exclude_fields|Bugzilla::WebService/exclude_fields> arguments. +This RPC call supports sub field restrictions. + =over =item C<ids> @@ -320,9 +467,15 @@ An array of product ids An array of product names +=item C<type> + +The group of products to return. Valid values are: C<accessible> (default), +C<selectable>, and C<enterable>. C<type> can be a single value or an array +of values if more than one group is needed with duplicates removed. + =back -=item B<Returns> +=item B<Returns> A hash containing one item, C<products>, that is an array of hashes. Each hash describes a product, and has the following items: @@ -400,6 +553,68 @@ and then secondly by their name. C<boolean> A boolean indicating if the component is active. Inactive components are not enabled for new bugs. +=item C<flag_types> + +A hash containing the two items C<bug> and C<attachment> that each contains an +array of hashes, where each hash describes a flagtype, and has the +following items: + +=over + +=item C<id> + +C<int> Returns the ID of the flagtype. + +=item C<name> + +C<string> Returns the name of the flagtype. + +=item C<description> + +C<string> Returns the description of the flagtype. + +=item C<cc_list> + +C<string> Returns the concatenated CC list for the flagtype, as a single string. + +=item C<sort_key> + +C<int> Returns the sortkey of the flagtype. + +=item C<is_active> + +C<boolean> Returns whether the flagtype is active or disabled. Flags being +in a disabled flagtype are not deleted. It only prevents you from +adding new flags to it. + +=item C<is_requestable> + +C<boolean> Returns whether you can request for the given flagtype +(i.e. whether the '?' flag is available or not). + +=item C<is_requesteeble> + +C<boolean> Returns whether you can ask someone specifically or not. + +=item C<is_multiplicable> + +C<boolean> Returns whether you can have more than one flag for the given +flagtype in a given bug/attachment. + +=item C<grant_group> + +C<int> the group id that is allowed to grant/deny flags of this type. +If the item is not included all users are allowed to grant/deny this +flagtype. + +=item C<request_group> + +C<int> the group id that is allowed to request the flag if the flag +is of the type requestable. If the item is not included all users +are allowed request this flagtype. + +=back + =back =item C<versions> @@ -432,6 +647,11 @@ C<milestones>, C<default_milestone> and C<has_unconfirmed> were added to the fields returned by C<get> as a replacement for C<internals>, which has been removed. +=item REST API call added in Bugzilla B<5.0>. + +=item In Bugzilla B<4.4>, C<flag_types> was added to the fields returned +by C<get>. + =back =back @@ -448,9 +668,16 @@ B<EXPERIMENTAL> This allows you to create a new product in Bugzilla. -=item B<Params> +=item B<REST> -Some params must be set, or an error will be thrown. These params are +POST /product + +The params to include in the POST body as well as the returned data format, +are the same as below. + +=item B<Params> + +Some params must be set, or an error will be thrown. These params are marked B<Required>. =over @@ -464,11 +691,11 @@ within Bugzilla. B<Required> C<string> A description for this product. Allows some simple HTML. -=item C<version> +=item C<version> B<Required> C<string> The default version for this product. -=item C<has_unconfirmed> +=item C<has_unconfirmed> C<boolean> Allow the UNCONFIRMED status to be set on bugs in this product. Default: true. @@ -477,11 +704,11 @@ Default: true. C<string> The name of the Classification which contains this product. -=item C<default_milestone> +=item C<default_milestone> C<string> The default milestone for this product. Default: '---'. -=item C<is_open> +=item C<is_open> C<boolean> True if the product is currently allowing bugs to be entered into it. Default: true. @@ -493,7 +720,7 @@ new product. Default: true. =back -=item B<Returns> +=item B<Returns> A hash with one element, id. This is the id of the newly-filed product. @@ -529,4 +756,12 @@ You must specify a version for this product. =back +=item B<History> + +=over + +=item REST API call added in Bugzilla B<5.0>. + +=back + =back diff --git a/Bugzilla/WebService/Server.pm b/Bugzilla/WebService/Server.pm index 206f0c657..e61a1f600 100644 --- a/Bugzilla/WebService/Server.pm +++ b/Bugzilla/WebService/Server.pm @@ -22,22 +22,34 @@ use Bugzilla::Error; use Bugzilla::Util qw(datetime_from); use Scalar::Util qw(blessed); +use Digest::MD5 qw(md5_base64); + +use Storable qw(freeze); sub handle_login { my ($self, $class, $method, $full_method) = @_; # Throw error if the supplied class does not exist or the method is private ThrowCodeError('unknown_method', {method => $full_method}) if (!$class or $method =~ /^_/); + # BMO - use the class and method as the name, instead of the cgi filename + if (Bugzilla->metrics_enabled) { + Bugzilla->metrics->name("$class $method"); + } + eval "require $class"; ThrowCodeError('unknown_method', {method => $full_method}) if $@; return if ($class->login_exempt($method) and !defined Bugzilla->input_params->{Bugzilla_login}); Bugzilla->login(); + + Bugzilla::Hook::process( + 'webservice_before_call', + { 'method' => $method, full_method => $full_method }); } sub datetime_format_inbound { my ($self, $time) = @_; - + my $converted = datetime_from($time, Bugzilla->local_timezone); if (!defined $converted) { ThrowUserError('illegal_date', { date => $time }); @@ -63,4 +75,71 @@ sub datetime_format_outbound { return $time->iso8601(); } +# ETag support +sub bz_etag { + my ($self, $data) = @_; + my $cache = Bugzilla->request_cache; + if (defined $data) { + # Serialize the data if passed a reference + local $Storable::canonical = 1; + $data = freeze($data) if ref $data; + + # Wide characters cause md5_base64() to die. + utf8::encode($data) if utf8::is_utf8($data); + + # Append content_type to the end of the data + # string as we want the etag to be unique to + # the content_type. We do not need this for + # XMLRPC as text/xml is always returned. + if (blessed($self) && $self->can('content_type')) { + $data .= $self->content_type if $self->content_type; + } + + $cache->{'bz_etag'} = md5_base64($data); + } + return $cache->{'bz_etag'}; +} + 1; + +=head1 NAME + +Bugzilla::WebService::Server - Base server class for the WebService API + +=head1 DESCRIPTION + +Bugzilla::WebService::Server is the base class for the individual WebService API +servers such as XMLRPC, JSONRPC, and REST. You never actually create a +Bugzilla::WebService::Server directly, you only make subclasses of it. + +=head1 FUNCTIONS + +=over + +=item C<bz_etag> + +This function is used to store an ETag value that will be used when returning +the data by the different API server modules such as XMLRPC, or REST. The individual +webservice methods can also set the value earlier in the process if needed such as +before a unique update token is added. If a value is not set earlier, an etag will +automatically be created using the returned data except in some cases when an error +has occurred. + +=back + +=head1 SEE ALSO + +L<Bugzilla::WebService::Server::XMLRPC|XMLRPC>, L<Bugzilla::WebService::Server::JSONRPC|JSONRPC>, +and L<Bugzilla::WebService::Server::REST|REST>. + +=head1 B<Methods in need of POD> + +=over + +=item handle_login + +=item datetime_format_outbound + +=item datetime_format_inbound + +=back diff --git a/Bugzilla/WebService/Server/JSONRPC.pm b/Bugzilla/WebService/Server/JSONRPC.pm index 373aa4fe0..0df4240e0 100644 --- a/Bugzilla/WebService/Server/JSONRPC.pm +++ b/Bugzilla/WebService/Server/JSONRPC.pm @@ -37,8 +37,8 @@ BEGIN { use Bugzilla::Error; use Bugzilla::WebService::Constants; -use Bugzilla::WebService::Util qw(taint_data); -use Bugzilla::Util qw(correct_urlbase trim disable_utf8); +use Bugzilla::WebService::Util qw(taint_data fix_credentials); +use Bugzilla::Util; use HTTP::Message; use MIME::Base64 qw(decode_base64 encode_base64); @@ -87,6 +87,7 @@ sub response_header { sub response { my ($self, $response) = @_; + my $cgi = $self->cgi; # Implement JSONP. if (my $callback = $self->_bz_callback) { @@ -108,9 +109,18 @@ sub response { push(@header_args, "-$name", $value); } } - my $cgi = $self->cgi; - print $cgi->header(-status => $response->code, @header_args); - print $response->content; + + # ETag support + my $etag = $self->bz_etag; + if ($etag && $cgi->check_etag($etag)) { + push(@header_args, "-ETag", $etag); + print $cgi->header(-status => '304 Not Modified', @header_args); + } + else { + push(@header_args, "-ETag", $etag) if $etag; + print $cgi->header(-status => $response->code, @header_args); + print $response->content; + } } # The JSON-RPC 1.1 GET specification is not so great--you can't specify @@ -222,6 +232,9 @@ sub type { utf8::encode($value) if utf8::is_utf8($value); $retval = encode_base64($value, ''); } + elsif ($type eq 'email' && Bugzilla->params->{'webservice_email_filter'}) { + $retval = email_filter($value); + } return $retval; } @@ -267,7 +280,17 @@ sub _handle { my $self = shift; my ($obj) = @_; $self->{_bz_request_id} = $obj->{id}; - return $self->SUPER::_handle(@_); + + my $result = $self->SUPER::_handle(@_); + + # Set the ETag if not already set in the webservice methods. + my $etag = $self->bz_etag; + if (!$etag && ref $result) { + my $data = $self->json->decode($result)->{'result'}; + $self->bz_etag($data); + } + + return $result; } # Make all error messages returned by JSON::RPC go into the 100000 @@ -364,6 +387,10 @@ sub _argument_type_check { } } + # Update the params to allow for several convenience key/values + # use for authentication + fix_credentials($params); + Bugzilla->input_params($params); if ($self->request->method eq 'POST') { diff --git a/Bugzilla/WebService/Server/REST.pm b/Bugzilla/WebService/Server/REST.pm new file mode 100644 index 000000000..69f97ef08 --- /dev/null +++ b/Bugzilla/WebService/Server/REST.pm @@ -0,0 +1,659 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::WebService::Server::REST; + +use 5.10.1; +use strict; + +use parent qw(Bugzilla::WebService::Server::JSONRPC); + +use Bugzilla; +use Bugzilla::Constants; +use Bugzilla::Error; +use Bugzilla::Hook; +use Bugzilla::Util qw(correct_urlbase html_quote disable_utf8 enable_utf8); +use Bugzilla::WebService::Constants; +use Bugzilla::WebService::Util qw(taint_data fix_credentials); + +# Load resource modules +use Bugzilla::WebService::Server::REST::Resources::Bug; +use Bugzilla::WebService::Server::REST::Resources::Bugzilla; +use Bugzilla::WebService::Server::REST::Resources::Classification; +use Bugzilla::WebService::Server::REST::Resources::Group; +use Bugzilla::WebService::Server::REST::Resources::Product; +use Bugzilla::WebService::Server::REST::Resources::User; +use Bugzilla::WebService::Server::REST::Resources::BugUserLastVisit; + +use Scalar::Util qw(blessed reftype); +use MIME::Base64 qw(decode_base64); + +########################### +# Public Method Overrides # +########################### + +sub handle { + my ($self) = @_; + + # Determine how the data should be represented. We do this early so + # errors will also be returned with the proper content type. + # If no accept header was sent or the content types specified were not + # matched, we default to the first type in the whitelist. + $self->content_type($self->_best_content_type(REST_CONTENT_TYPE_WHITELIST())); + + # Using current path information, decide which class/method to + # use to serve the request. Throw error if no resource was found + # unless we were looking for OPTIONS + if (!$self->_find_resource($self->cgi->path_info)) { + if ($self->request->method eq 'OPTIONS' + && $self->bz_rest_options) + { + my $response = $self->response_header(STATUS_OK, ""); + my $options_string = join(', ', @{ $self->bz_rest_options }); + $response->header('Allow' => $options_string, + 'Access-Control-Allow-Methods' => $options_string); + return $self->response($response); + } + + ThrowUserError("rest_invalid_resource", + { path => $self->cgi->path_info, + method => $self->request->method }); + } + + # Dispatch to the proper module + my $class = $self->bz_class_name; + my ($path) = $class =~ /::([^:]+)$/; + $self->path_info($path); + delete $self->{dispatch_path}; + $self->dispatch({ $path => $class }); + + my $params = $self->_retrieve_json_params; + + fix_credentials($params); + + # Fix includes/excludes for each call + rest_include_exclude($params); + + # Set callback name if content-type is 'application/javascript' + if ($params->{'callback'} + || $self->content_type eq 'application/javascript') + { + $self->_bz_callback($params->{'callback'} || 'callback'); + } + + Bugzilla->input_params($params); + + # Set the JSON version to 1.1 and the id to the current urlbase + # also set up the correct handler method + my $obj = { + version => '1.1', + id => correct_urlbase(), + method => $self->bz_method_name, + params => $params + }; + + # Execute the handler + my $result = $self->_handle($obj); + + if (!$self->error_response_header) { + return $self->response( + $self->response_header($self->bz_success_code || STATUS_OK, $result)); + } + + $self->response($self->error_response_header); +} + +sub response { + my ($self, $response) = @_; + + # If we have thrown an error, the 'error' key will exist + # otherwise we use 'result'. JSONRPC returns other data + # along with the result/error such as version and id which + # we will strip off for REST calls. + my $content = $response->content; + + my $json_data = {}; + if ($content) { + # Content is in bytes at this point and needs to be converted + # back to utf8 string. + enable_utf8(); + utf8::decode($content) if !utf8::is_utf8($content); + $json_data = $self->json->decode($content); + } + + my $result = {}; + if (exists $json_data->{error}) { + $result = $json_data->{error}; + $result->{error} = $self->type('boolean', 1); + $result->{documentation} = REST_DOC; + delete $result->{'name'}; # Remove JSONRPCError + } + elsif (exists $json_data->{result}) { + $result = $json_data->{result}; + } + + Bugzilla::Hook::process('webservice_rest_response', + { rpc => $self, result => \$result, response => $response }); + + # Access Control + $response->header("Access-Control-Allow-Origin", "*"); + $response->header("Access-Control-Allow-Headers", "origin, content-type, accept"); + + # ETag support + my $etag = $self->bz_etag; + $self->bz_etag($result) if !$etag; + + # If accessing through web browser, then display in readable format + if ($self->content_type eq 'text/html') { + $result = $self->json->pretty->canonical->allow_nonref->encode($result); + + my $template = Bugzilla->template; + $content = ""; + $template->process("rest.html.tmpl", { result => $result }, \$content) + || ThrowTemplateError($template->error()); + + $response->content_type('text/html'); + } + else { + $content = $self->json->encode($result); + } + + utf8::encode($content) if utf8::is_utf8($content); + disable_utf8(); + + $response->content($content); + + $self->SUPER::response($response); +} + +####################################### +# Bugzilla::WebService Implementation # +####################################### + +sub handle_login { + my $self = shift; + + # If we're being called using GET, we don't allow cookie-based or Env + # login, because GET requests can be done cross-domain, and we don't + # want private data showing up on another site unless the user + # explicitly gives that site their username and password. (This is + # particularly important for JSONP, which would allow a remote site + # to use private data without the user's knowledge, unless we had this + # protection in place.) We do allow this for GET /login as we need to + # for Bugzilla::Auth::Persist::Cookie to create a login cookie that we + # can also use for Bugzilla_token support. This is OK as it requires + # a login and password to be supplied and will fail if they are not + # valid for the user. + if (!grep($_ eq $self->request->method, ('POST', 'PUT')) + && !($self->bz_class_name eq 'Bugzilla::WebService::User' + && $self->bz_method_name eq 'login')) + { + # XXX There's no particularly good way for us to get a parameter + # to Bugzilla->login at this point, so we pass this information + # around using request_cache, which is a bit of a hack. The + # implementation of it is in Bugzilla::Auth::Login::Stack. + Bugzilla->request_cache->{'auth_no_automatic_login'} = 1; + } + + my $class = $self->bz_class_name; + my $method = $self->bz_method_name; + my $full_method = $class . "." . $method; + + # Bypass JSONRPC::handle_login + Bugzilla::WebService::Server->handle_login($class, $method, $full_method); +} + +############################ +# Private Method Overrides # +############################ + +# We do not want to run Bugzilla::WebService::Server::JSONRPC->_find_prodedure +# as it determines the method name differently. +sub _find_procedure { + my $self = shift; + if ($self->isa('JSON::RPC::Server::CGI')) { + return JSON::RPC::Server::_find_procedure($self, @_); + } + else { + return JSON::RPC::Legacy::Server::_find_procedure($self, @_); + } +} + +sub _argument_type_check { + my $self = shift; + my $params; + + if ($self->isa('JSON::RPC::Server::CGI')) { + $params = JSON::RPC::Server::_argument_type_check($self, @_); + } + else { + $params = JSON::RPC::Legacy::Server::_argument_type_check($self, @_); + } + + # JSON-RPC 1.0 requires all parameters to be passed as an array, so + # we just pull out the first item and assume it's an object. + my $params_is_array; + if (ref $params eq 'ARRAY') { + $params = $params->[0]; + $params_is_array = 1; + } + + taint_data($params); + + # Now, convert dateTime fields on input. + my $method = $self->bz_method_name; + my $pkg = $self->{dispatch_path}->{$self->path_info}; + my @date_fields = @{ $pkg->DATE_FIELDS->{$method} || [] }; + foreach my $field (@date_fields) { + if (defined $params->{$field}) { + my $value = $params->{$field}; + if (ref $value eq 'ARRAY') { + $params->{$field} = + [ map { $self->datetime_format_inbound($_) } @$value ]; + } + else { + $params->{$field} = $self->datetime_format_inbound($value); + } + } + } + my @base64_fields = @{ $pkg->BASE64_FIELDS->{$method} || [] }; + foreach my $field (@base64_fields) { + if (defined $params->{$field}) { + $params->{$field} = decode_base64($params->{$field}); + } + } + + # This is the best time to do login checks. + $self->handle_login(); + + # Bugzilla::WebService packages call internal methods like + # $self->_some_private_method. So we have to inherit from + # that class as well as this Server class. + my $new_class = ref($self) . '::' . $pkg; + my $isa_string = 'our @ISA = qw(' . ref($self) . " $pkg)"; + eval "package $new_class;$isa_string;"; + bless $self, $new_class; + + # Allow extensions to modify the params post login + Bugzilla::Hook::process('webservice_rest_request', + { rpc => $self, params => $params }); + + if ($params_is_array) { + $params = [$params]; + } + + return $params; +} + +################### +# Utility Methods # +################### + +sub bz_method_name { + my ($self, $method) = @_; + $self->{_bz_method_name} = $method if $method; + return $self->{_bz_method_name}; +} + +sub bz_class_name { + my ($self, $class) = @_; + $self->{_bz_class_name} = $class if $class; + return $self->{_bz_class_name}; +} + +sub bz_success_code { + my ($self, $value) = @_; + $self->{_bz_success_code} = $value if $value; + return $self->{_bz_success_code}; +} + +sub bz_rest_params { + my ($self, $params) = @_; + $self->{_bz_rest_params} = $params if $params; + return $self->{_bz_rest_params}; +} + +sub bz_rest_options { + my ($self, $options) = @_; + $self->{_bz_rest_options} = $options if $options; + return [ sort { $a cmp $b } @{ $self->{_bz_rest_options} } ]; +} + +sub rest_include_exclude { + my ($params) = @_; + + if (exists $params->{'include_fields'} && !ref $params->{'include_fields'}) { + $params->{'include_fields'} = [ split(/[\s+,]/, $params->{'include_fields'}) ]; + } + if (exists $params->{'exclude_fields'} && !ref $params->{'exclude_fields'}) { + $params->{'exclude_fields'} = [ split(/[\s+,]/, $params->{'exclude_fields'}) ]; + } + + return $params; +} + +########################## +# Private Custom Methods # +########################## + +sub _retrieve_json_params { + my $self = shift; + + # Make a copy of the current input_params rather than edit directly + my $params = {}; + %{$params} = %{ Bugzilla->input_params }; + + # First add any params we were able to pull out of the path + # based on the resource regexp + %{$params} = (%{$params}, %{$self->bz_rest_params}) if $self->bz_rest_params; + + # Merge any additional query key/values with $obj->{params} if not a GET request + # We do this manually cause CGI.pm doesn't understand JSON strings. + if ($self->request->method ne 'GET') { + my $extra_params = {}; + my $json = delete $params->{'POSTDATA'} || delete $params->{'PUTDATA'}; + if ($json) { + eval { $extra_params = $self->json->utf8(0)->decode($json); }; + if ($@) { + ThrowUserError('json_rpc_invalid_params', { err_msg => $@ }); + } + } + + # Allow parameters in the query string if request was not GET. + # Note: query string parameters will override any matching params + # also specified in the request body. + foreach my $param ($self->cgi->url_param()) { + $extra_params->{$param} = $self->cgi->url_param($param); + } + + %{$params} = (%{$params}, %{$extra_params}) if %{$extra_params}; + } + + return $params; +} + +sub _find_resource { + my ($self, $path) = @_; + + # Load in the WebService module from the dispatch map and then call + # $module->rest_resources to get the resources array ref. + my $resources = {}; + foreach my $module (values %{ $self->{dispatch_path} }) { + eval("require $module") || die $@; + next if !$module->can('rest_resources'); + $resources->{$module} = $module->rest_resources; + } + + Bugzilla::Hook::process('webservice_rest_resources', + { rpc => $self, resources => $resources }); + + # Use the resources hash from each module loaded earlier to determine + # which handler to use based on a regex match of the CGI path. + # Also any matches found in the regex will be passed in later to the + # handler for possible use. + my $request_method = $self->request->method; + + my (@matches, $handler_found, $handler_method, $handler_class); + foreach my $class (keys %{ $resources }) { + # The resource data for each module needs to be + # an array ref with an even number of elements + # to work correctly. + next if (ref $resources->{$class} ne 'ARRAY' + || scalar @{ $resources->{$class} } % 2 != 0); + + while (my $regex = shift @{ $resources->{$class} }) { + my $options_data = shift @{ $resources->{$class} }; + next if ref $options_data ne 'HASH'; + + if (@matches = ($path =~ $regex)) { + # If a specific path is accompanied by a OPTIONS request + # method, the user is asking for a list of possible request + # methods for a specific path. + $self->bz_rest_options([ keys %{ $options_data } ]); + + if ($options_data->{$request_method}) { + my $resource_data = $options_data->{$request_method}; + $self->bz_class_name($class); + + # The method key/value can be a simple scalar method name + # or a anonymous subroutine so we execute it here. + my $method = ref $resource_data->{method} eq 'CODE' + ? $resource_data->{method}->($self) + : $resource_data->{method}; + $self->bz_method_name($method); + + # Pull out any parameters parsed from the URL path + # and store them for use by the method. + if ($resource_data->{params}) { + $self->bz_rest_params($resource_data->{params}->(@matches)); + } + + # If a special success code is needed for this particular + # method, then store it for later when generating response. + if ($resource_data->{success_code}) { + $self->bz_success_code($resource_data->{success_code}); + } + $handler_found = 1; + } + } + last if $handler_found; + } + last if $handler_found; + } + + return $handler_found; +} + +sub _best_content_type { + my ($self, @types) = @_; + return ($self->_simple_content_negotiation(@types))[0] || '*/*'; +} + +sub _simple_content_negotiation { + my ($self, @types) = @_; + my @accept_types = $self->_get_content_prefs(); + # Return the types as-is if no accept header sent, since sorting will be a no-op. + if (!@accept_types) { + return @types; + } + my $score = sub { $self->_score_type(shift, @accept_types) }; + return sort {$score->($b) <=> $score->($a)} @types; +} + +sub _score_type { + my ($self, $type, @accept_types) = @_; + my $score = scalar(@accept_types); + for my $accept_type (@accept_types) { + return $score if $type eq $accept_type; + $score--; + } + return 0; +} + +sub _get_content_prefs { + my $self = shift; + my $default_weight = 1; + my @prefs; + + # Parse the Accept header, and save type name, score, and position. + my @accept_types = split /,/, $self->cgi->http('accept') || ''; + my $order = 0; + for my $accept_type (@accept_types) { + my ($weight) = ($accept_type =~ /q=(\d\.\d+|\d+)/); + my ($name) = ($accept_type =~ m#(\S+/[^;]+)#); + next unless $name; + push @prefs, { name => $name, order => $order++}; + if (defined $weight) { + $prefs[-1]->{score} = $weight; + } else { + $prefs[-1]->{score} = $default_weight; + $default_weight -= 0.001; + } + } + + # Sort the types by score, subscore by order, and pull out just the name + @prefs = map {$_->{name}} sort {$b->{score} <=> $a->{score} || + $a->{order} <=> $b->{order}} @prefs; + return @prefs; +} + +1; + +__END__ + +=head1 NAME + +Bugzilla::WebService::Server::REST - The REST Interface to Bugzilla + +=head1 DESCRIPTION + +This documentation describes things about the Bugzilla WebService that +are specific to REST. For a general overview of the Bugzilla WebServices, +see L<Bugzilla::WebService>. The L<Bugzilla::WebService::Server::REST> +module is a sub-class of L<Bugzilla::WebService::Server::JSONRPC> so any +method documentation not found here can be viewed in it's POD. + +Please note that I<everything> about this REST interface is +B<EXPERIMENTAL>. If you want a fully stable API, please use the +C<Bugzilla::WebService::Server::XMLRPC|XML-RPC> interface. + +=head1 CONNECTING + +The endpoint for the REST interface is the C<rest.cgi> script in +your Bugzilla installation. If using Apache and mod_rewrite is installed +and enabled, you can also use /rest/ as your endpoint. For example, if your +Bugzilla is at C<bugzilla.yourdomain.com>, then your REST client would +access the API via: C<http://bugzilla.yourdomain.com/rest/bug/35> which +looks cleaner. + +=head1 BROWSING + +If the Accept: header of a request is set to text/html (as it is by an +ordinary web browser) then the API will return the JSON data as a HTML +page which the browser can display. In other words, you can play with the +API using just your browser and see results in a human-readable form. +This is a good way to try out the various GET calls, even if you can't use +it for POST or PUT. + +=head1 DATA FORMAT + +The REST API only supports JSON input, and either JSON and JSONP output. +So objects sent and received must be in JSON format. Basically since +the REST API is a sub class of the JSONRPC API, you can refer to +L<JSONRPC|Bugzilla::WebService::Server::JSONRPC> for more information +on data types that are valid for REST. + +On every request, you must set both the "Accept" and "Content-Type" HTTP +headers to the MIME type of the data format you are using to communicate with +the API. Content-Type tells the API how to interpret your request, and Accept +tells it how you want your data back. "Content-Type" must be "application/json". +"Accept" can be either that, or "application/javascript" for JSONP - add a "callback" +parameter to name your callback. + +Parameters may also be passed in as part of the query string for non-GET requests +and will override any matching parameters in the request body. + +=head1 AUTHENTICATION + +Along with viewing data as an anonymous user, you may also see private information +if you have a Bugzilla account by providing your login credentials. + +=over + +=item Login name and password + +Pass in as query parameters of any request: + +login=fred@example.com&password=ilovecheese + +Remember to URL encode any special characters, which are often seen in passwords and to +also enable SSL support. + +=item Login token + +By calling GET /login?login=fred@example.com&password=ilovecheese, you get back +a C<token> value which can then be passed to each subsequent call as +authentication. This is useful for third party clients that cannot use cookies +and do not want to store a user's login and password in the client. You can also +pass in "token" as a convenience. + +=back + +=head1 ERRORS + +When an error occurs over REST, a hash structure is returned with the key C<error> +set to C<true>. + +The error contents look similar to: + + { "error": true, "message": "Some message here", "code": 123 } + +Every error has a "code", as described in L<Bugzilla::WebService/ERRORS>. +Errors with a numeric C<code> higher than 100000 are errors thrown by +the JSON-RPC library that Bugzilla uses, not by Bugzilla. + +=head1 UTILITY FUNCTIONS + +=over + +=item B<handle> + +This method overrides the handle method provided by JSONRPC so that certain +actions related to REST such as determining the proper resource to use, +loading query parameters, etc. can be done before the proper WebService +method is executed. + +=item B<response> + +This method overrides the response method provided by JSONRPC so that +the response content can be altered for REST before being returned to +the client. + +=item B<handle_login> + +This method determines the proper WebService all to make based on class +and method name determined earlier. Then calls L<Bugzilla::WebService::Server::handle_login> +which will attempt to authenticate the client. + +=item B<bz_method_name> + +The WebService method name that matches the path used by the client. + +=item B<bz_class_name> + +The WebService class containing the method that matches the path used by the client. + +=item B<bz_rest_params> + +Each REST resource contains a hash key called C<params> that is a subroutine reference. +This subroutine will return a hash structure based on matched values from the path +information that is formatted properly for the WebService method that will be called. + +=item B<bz_rest_options> + +When a client uses the OPTIONS request method along with a specific path, they are +requesting the list of request methods that are valid for the path. Such as for the +path /bug, the valid request methods are GET (search) and POST (create). So the +client would receive in the response header, C<Access-Control-Allow-Methods: GET, POST>. + +=item B<bz_success_code> + +Each resource can specify a specific SUCCESS CODE if the operation completes successfully. +OTherwise STATUS OK (200) is the default returned. + +=item B<rest_include_exclude> + +Normally the WebService methods required C<include_fields> and C<exclude_fields> to be an +array of field names. REST allows for the values for these to be instead comma delimited +string of field names. This method converts the latter into the former so the WebService +methods will not complain. + +=back + +=head1 SEE ALSO + +L<Bugzilla::WebService> diff --git a/Bugzilla/WebService/Server/REST/Resources/Bug.pm b/Bugzilla/WebService/Server/REST/Resources/Bug.pm new file mode 100644 index 000000000..d0f470fcd --- /dev/null +++ b/Bugzilla/WebService/Server/REST/Resources/Bug.pm @@ -0,0 +1,190 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::WebService::Server::REST::Resources::Bug; + +use 5.10.1; +use strict; + +use Bugzilla::WebService::Constants; +use Bugzilla::WebService::Bug; + +BEGIN { + *Bugzilla::WebService::Bug::rest_resources = \&_rest_resources; +}; + +sub _rest_resources { + my $rest_resources = [ + qr{^/bug$}, { + GET => { + method => 'search', + }, + POST => { + method => 'create', + status_code => STATUS_CREATED + } + }, + qr{^/bug/([^/]+)$}, { + GET => { + method => 'get', + params => sub { + return { ids => [ $_[0] ] }; + } + }, + PUT => { + method => 'update', + params => sub { + return { ids => [ $_[0] ] }; + } + } + }, + qr{^/bug/([^/]+)/comment$}, { + GET => { + method => 'comments', + params => sub { + return { ids => [ $_[0] ] }; + } + }, + POST => { + method => 'add_comment', + params => sub { + return { id => $_[0] }; + }, + success_code => STATUS_CREATED + } + }, + qr{^/bug/comment/([^/]+)$}, { + GET => { + method => 'comments', + params => sub { + return { comment_ids => [ $_[0] ] }; + } + } + }, + qr{^/bug/comment/tags/([^/]+)$}, { + GET => { + method => 'search_comment_tags', + params => sub { + return { query => $_[0] }; + }, + }, + }, + qr{^/bug/comment/([^/]+)/tags$}, { + PUT => { + method => 'update_comment_tags', + params => sub { + return { comment_id => $_[0] }; + }, + }, + }, + qr{^/bug/([^/]+)/history$}, { + GET => { + method => 'history', + params => sub { + return { ids => [ $_[0] ] }; + }, + } + }, + qr{^/bug/([^/]+)/attachment$}, { + GET => { + method => 'attachments', + params => sub { + return { ids => [ $_[0] ] }; + } + }, + POST => { + method => 'add_attachment', + params => sub { + return { ids => [ $_[0] ] }; + }, + success_code => STATUS_CREATED + } + }, + qr{^/bug/attachment/([^/]+)$}, { + GET => { + method => 'attachments', + params => sub { + return { attachment_ids => [ $_[0] ] }; + } + }, + PUT => { + method => 'update_attachment', + params => sub { + return { ids => [ $_[0] ] }; + } + } + }, + qr{^/field/bug$}, { + GET => { + method => 'fields', + } + }, + qr{^/field/bug/([^/]+)$}, { + GET => { + method => 'fields', + params => sub { + my $value = $_[0]; + my $param = 'names'; + $param = 'ids' if $value =~ /^\d+$/; + return { $param => [ $_[0] ] }; + } + } + }, + qr{^/field/bug/([^/]+)/values$}, { + GET => { + method => 'legal_values', + params => sub { + return { field => $_[0] }; + } + } + }, + qr{^/field/bug/([^/]+)/([^/]+)/values$}, { + GET => { + method => 'legal_values', + params => sub { + return { field => $_[0], + product_id => $_[1] }; + } + } + }, + qr{^/flag_types/([^/]+)/([^/]+)$}, { + GET => { + method => 'flag_types', + params => sub { + return { product => $_[0], + component => $_[1] }; + } + } + }, + qr{^/flag_types/([^/]+)$}, { + GET => { + method => 'flag_types', + params => sub { + return { product => $_[0] }; + } + } + } + ]; + return $rest_resources; +} + +1; + +__END__ + +=head1 NAME + +Bugzilla::Webservice::Server::REST::Resources::Bug - The REST API for creating, +changing, and getting the details of bugs. + +=head1 DESCRIPTION + +This part of the Bugzilla REST API allows you to file a new bug in Bugzilla, +or get information about bugs that have already been filed. + +See L<Bugzilla::WebService::Bug> for more details on how to use this part of +the REST API. diff --git a/Bugzilla/WebService/Server/REST/Resources/BugUserLastVisit.pm b/Bugzilla/WebService/Server/REST/Resources/BugUserLastVisit.pm new file mode 100644 index 000000000..a434d4bef --- /dev/null +++ b/Bugzilla/WebService/Server/REST/Resources/BugUserLastVisit.pm @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::WebService::Server::REST::Resources::BugUserLastVisit; + +use 5.10.1; +use strict; +use warnings; + +BEGIN { + *Bugzilla::WebService::BugUserLastVisit::rest_resources = \&_rest_resources; +} + +sub _rest_resources { + return [ + # bug-id + qr{^/bug_user_last_visit/(\d+)$}, { + GET => { + method => 'get', + params => sub { + return { ids => $_[0] }; + }, + }, + POST => { + method => 'update', + params => sub { + return { ids => $_[0] }; + }, + }, + }, + ]; +} + +1; +__END__ + +=head1 NAME + +Bugzilla::Webservice::Server::REST::Resources::BugUserLastVisit - The +BugUserLastVisit REST API + +=head1 DESCRIPTION + +This part of the Bugzilla REST API allows you to lookup and update the last time +a user visited a bug. + +See L<Bugzilla::WebService::BugUserLastVisit> for more details on how to use +this part of the REST API. diff --git a/Bugzilla/WebService/Server/REST/Resources/Bugzilla.pm b/Bugzilla/WebService/Server/REST/Resources/Bugzilla.pm new file mode 100644 index 000000000..1c86f77bc --- /dev/null +++ b/Bugzilla/WebService/Server/REST/Resources/Bugzilla.pm @@ -0,0 +1,69 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::WebService::Server::REST::Resources::Bugzilla; + +use 5.10.1; +use strict; + +use Bugzilla::WebService::Constants; +use Bugzilla::WebService::Bugzilla; + +BEGIN { + *Bugzilla::WebService::Bugzilla::rest_resources = \&_rest_resources; +}; + +sub _rest_resources { + my $rest_resources = [ + qr{^/version$}, { + GET => { + method => 'version' + } + }, + qr{^/extensions$}, { + GET => { + method => 'extensions' + } + }, + qr{^/timezone$}, { + GET => { + method => 'timezone' + } + }, + qr{^/time$}, { + GET => { + method => 'time' + } + }, + qr{^/last_audit_time$}, { + GET => { + method => 'last_audit_time' + } + }, + qr{^/parameters$}, { + GET => { + method => 'parameters' + } + } + ]; + return $rest_resources; +} + +1; + +__END__ + +=head1 NAME + +Bugzilla::WebService::Bugzilla - Global functions for the webservice interface. + +=head1 DESCRIPTION + +This provides functions that tell you about Bugzilla in general. + +See L<Bugzilla::WebService::Bugzilla> for more details on how to use this part +of the REST API. diff --git a/Bugzilla/WebService/Server/REST/Resources/Classification.pm b/Bugzilla/WebService/Server/REST/Resources/Classification.pm new file mode 100644 index 000000000..5bb697ac1 --- /dev/null +++ b/Bugzilla/WebService/Server/REST/Resources/Classification.pm @@ -0,0 +1,49 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::WebService::Server::REST::Resources::Classification; + +use 5.10.1; +use strict; + +use Bugzilla::WebService::Constants; +use Bugzilla::WebService::Classification; + +BEGIN { + *Bugzilla::WebService::Classification::rest_resources = \&_rest_resources; +}; + +sub _rest_resources { + my $rest_resources = [ + qr{^/classification/([^/]+)$}, { + GET => { + method => 'get', + params => sub { + my $param = $_[0] =~ /^\d+$/ ? 'ids' : 'names'; + return { $param => [ $_[0] ] }; + } + } + } + ]; + return $rest_resources; +} + +1; + +__END__ + +=head1 NAME + +Bugzilla::Webservice::Server::REST::Resources::Classification - The Classification REST API + +=head1 DESCRIPTION + +This part of the Bugzilla REST API allows you to deal with the available Classifications. +You will be able to get information about them as well as manipulate them. + +See L<Bugzilla::WebService::Bug> for more details on how to use this part +of the REST API. diff --git a/Bugzilla/WebService/Server/REST/Resources/Group.pm b/Bugzilla/WebService/Server/REST/Resources/Group.pm new file mode 100644 index 000000000..9200d609d --- /dev/null +++ b/Bugzilla/WebService/Server/REST/Resources/Group.pm @@ -0,0 +1,56 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::WebService::Server::REST::Resources::Group; + +use 5.10.1; +use strict; + +use Bugzilla::WebService::Constants; +use Bugzilla::WebService::Group; + +BEGIN { + *Bugzilla::WebService::Group::rest_resources = \&_rest_resources; +}; + +sub _rest_resources { + my $rest_resources = [ + qr{^/group$}, { + POST => { + method => 'create', + success_code => STATUS_CREATED + } + }, + qr{^/group/([^/]+)$}, { + PUT => { + method => 'update', + params => sub { + my $param = $_[0] =~ /^\d+$/ ? 'ids' : 'names'; + return { $param => [ $_[0] ] }; + } + } + } + ]; + return $rest_resources; +} + +1; + +__END__ + +=head1 NAME + +Bugzilla::Webservice::Server::REST::Resources::Group - The REST API for +creating, changing, and getting information about Groups. + +=head1 DESCRIPTION + +This part of the Bugzilla REST API allows you to create Groups and +get information about them. + +See L<Bugzilla::WebService::Group> for more details on how to use this part +of the REST API. diff --git a/Bugzilla/WebService/Server/REST/Resources/Product.pm b/Bugzilla/WebService/Server/REST/Resources/Product.pm new file mode 100644 index 000000000..acee3887b --- /dev/null +++ b/Bugzilla/WebService/Server/REST/Resources/Product.pm @@ -0,0 +1,82 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::WebService::Server::REST::Resources::Product; + +use 5.10.1; +use strict; + +use Bugzilla::WebService::Constants; +use Bugzilla::WebService::Product; + +use Bugzilla::Error; + +BEGIN { + *Bugzilla::WebService::Product::rest_resources = \&_rest_resources; +}; + +sub _rest_resources { + my $rest_resources = [ + qr{^/product_accessible$}, { + GET => { + method => 'get_accessible_products' + } + }, + qr{^/product_enterable$}, { + GET => { + method => 'get_enterable_products' + } + }, + qr{^/product_selectable$}, { + GET => { + method => 'get_selectable_products' + } + }, + qr{^/product$}, { + GET => { + method => 'get' + }, + POST => { + method => 'create', + success_code => STATUS_CREATED + } + }, + qr{^/product/([^/]+)$}, { + GET => { + method => 'get', + params => sub { + my $param = $_[0] =~ /^\d+$/ ? 'ids' : 'names'; + return { $param => [ $_[0] ] }; + } + }, + PUT => { + method => 'update', + params => sub { + my $param = $_[0] =~ /^\d+$/ ? 'ids' : 'names'; + return { $param => [ $_[0] ] }; + } + } + }, + ]; + return $rest_resources; +} + +1; + +__END__ + +=head1 NAME + +Bugzilla::Webservice::Server::REST::Resources::Product - The Product REST API + +=head1 DESCRIPTION + +This part of the Bugzilla REST API allows you to list the available Products and +get information about them. + +See L<Bugzilla::WebService::Bug> for more details on how to use this part of +the REST API. diff --git a/Bugzilla/WebService/Server/REST/Resources/User.pm b/Bugzilla/WebService/Server/REST/Resources/User.pm new file mode 100644 index 000000000..badbc94b2 --- /dev/null +++ b/Bugzilla/WebService/Server/REST/Resources/User.pm @@ -0,0 +1,80 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::WebService::Server::REST::Resources::User; + +use 5.10.1; +use strict; + +use Bugzilla::WebService::Constants; +use Bugzilla::WebService::User; + +BEGIN { + *Bugzilla::WebService::User::rest_resources = \&_rest_resources; +}; + +sub _rest_resources { + my $rest_resources = [ + qr{^/valid_login$}, { + GET => { + method => 'valid_login' + } + }, + qr{^/login$}, { + GET => { + method => 'login' + } + }, + qr{^/logout$}, { + GET => { + method => 'logout' + } + }, + qr{^/user$}, { + GET => { + method => 'get' + }, + POST => { + method => 'create', + success_code => STATUS_CREATED + } + }, + qr{^/user/([^/]+)$}, { + GET => { + method => 'get', + params => sub { + my $param = $_[0] =~ /^\d+$/ ? 'ids' : 'names'; + return { $param => [ $_[0] ] }; + } + }, + PUT => { + method => 'update', + params => sub { + my $param = $_[0] =~ /^\d+$/ ? 'ids' : 'names'; + return { $param => [ $_[0] ] }; + } + } + } + ]; + return $rest_resources; +} + +1; + +__END__ + +=head1 NAME + +Bugzilla::Webservice::Server::REST::Resources::User - The User Account REST API + +=head1 DESCRIPTION + +This part of the Bugzilla REST API allows you to get User information as well +as create User Accounts. + +See L<Bugzilla::WebService::Bug> for more details on how to use this part of +the REST API. diff --git a/Bugzilla/WebService/Server/XMLRPC.pm b/Bugzilla/WebService/Server/XMLRPC.pm index fc297421a..8d9108122 100644 --- a/Bugzilla/WebService/Server/XMLRPC.pm +++ b/Bugzilla/WebService/Server/XMLRPC.pm @@ -30,9 +30,10 @@ if ($ENV{MOD_PERL}) { } use Bugzilla::WebService::Constants; +use Bugzilla::Util; -# Allow WebService methods to call XMLRPC::Lite's type method directly BEGIN { + # Allow WebService methods to call XMLRPC::Lite's type method directly *Bugzilla::WebService::type = sub { my ($self, $type, $value) = @_; if ($type eq 'dateTime') { @@ -41,8 +42,19 @@ BEGIN { $value = Bugzilla::WebService::Server->datetime_format_outbound($value); $value =~ s/-//g; } + elsif ($type eq 'email') { + $type = 'string'; + if (Bugzilla->params->{'webservice_email_filter'}) { + $value = email_filter($value); + } + } return XMLRPC::Data->type($type)->value($value); }; + + # Add support for ETags into XMLRPC WebServices + *Bugzilla::WebService::bz_etag = sub { + return Bugzilla::WebService::Server->bz_etag($_[1]); + }; } sub initialize { @@ -56,22 +68,38 @@ sub initialize { sub make_response { my $self = shift; + my $cgi = Bugzilla->cgi; $self->SUPER::make_response(@_); # XMLRPC::Transport::HTTP::CGI doesn't know about Bugzilla carrying around # its cookies in Bugzilla::CGI, so we need to copy them over. - foreach my $cookie (@{Bugzilla->cgi->{'Bugzilla_cookie_list'}}) { + foreach my $cookie (@{$cgi->{'Bugzilla_cookie_list'}}) { $self->response->headers->push_header('Set-Cookie', $cookie); } # Copy across security related headers from Bugzilla::CGI - foreach my $header (split(/[\r\n]+/, Bugzilla->cgi->header)) { + foreach my $header (split(/[\r\n]+/, $cgi->header)) { my ($name, $value) = $header =~ /^([^:]+): (.*)/; if (!$self->response->headers->header($name)) { $self->response->headers->header($name => $value); } } + + # ETag support + my $etag = $self->bz_etag; + if (!$etag) { + my $data = $self->response->as_string; + $etag = $self->bz_etag($data); + } + + if ($etag && $cgi->check_etag($etag)) { + $self->response->headers->push_header('ETag', $etag); + $self->response->headers->push_header('status', '304 Not Modified'); + } + elsif ($etag) { + $self->response->headers->push_header('ETag', $etag); + } } sub handle_login { diff --git a/Bugzilla/WebService/User.pm b/Bugzilla/WebService/User.pm index deb7518ec..988ae3cd5 100644 --- a/Bugzilla/WebService/User.pm +++ b/Bugzilla/WebService/User.pm @@ -28,7 +28,10 @@ use Bugzilla::Error; use Bugzilla::Group; use Bugzilla::User; use Bugzilla::Util qw(trim); -use Bugzilla::WebService::Util qw(filter validate); +use Bugzilla::WebService::Util qw(filter filter_wants validate); +use Bugzilla::Hook; + +use List::Util qw(first); # Don't need auth to login use constant LOGIN_EXEMPT => { @@ -70,14 +73,36 @@ sub login { $input_params->{'Bugzilla_password'} = $params->{password}; $input_params->{'Bugzilla_remember'} = $remember; - Bugzilla->login(); - return { id => $self->type('int', Bugzilla->user->id) }; + my $user = Bugzilla->login(); + + my $result = { id => $self->type('int', $user->id) }; + + # We will use the stored cookie value combined with the user id + # to create a token that can be used with future requests in the + # query parameters + my $login_cookie = first { $_->name eq 'Bugzilla_logincookie' } + @{ Bugzilla->cgi->{'Bugzilla_cookie_list'} }; + if ($login_cookie) { + $result->{'token'} = $user->id . "-" . $login_cookie->value; + } + + return $result; } sub logout { my $self = shift; Bugzilla->logout; - return undef; +} + +sub valid_login { + my ($self, $params) = @_; + defined $params->{login} + || ThrowCodeError('param_required', { param => 'login' }); + Bugzilla->login(); + if (Bugzilla->user->id && Bugzilla->user->login eq $params->{login}) { + return $self->type('boolean', 1); + } + return $self->type('boolean', 0); } ################# @@ -124,7 +149,9 @@ sub create { # $call = $rpc->call( 'User.get', { ids => [1,2,3], # names => ['testusera@redhat.com', 'testuserb@redhat.com'] }); sub get { - my ($self, $params) = validate(@_, 'names', 'ids'); + my ($self, $params) = validate(@_, 'names', 'ids', 'match', 'group_ids', 'groups'); + + Bugzilla->switch_to_shadow_db(); defined($params->{names}) || defined($params->{ids}) || defined($params->{match}) @@ -152,11 +179,11 @@ sub get { } my $in_group = $self->_filter_users_by_group( \@user_objects, $params); - @users = map {filter $params, { + @users = map { filter $params, { id => $self->type('int', $_->id), - real_name => $self->type('string', $_->name), - name => $self->type('string', $_->login), - }} @$in_group; + real_name => $self->type('string', $_->name), + name => $self->type('email', $_->login), + } } @$in_group; return { users => \@users }; } @@ -196,33 +223,47 @@ sub get { } } } - - my $in_group = $self->_filter_users_by_group( - \@user_objects, $params); - if (Bugzilla->user->in_group('editusers')) { - @users = - map {filter $params, { - id => $self->type('int', $_->id), - real_name => $self->type('string', $_->name), - name => $self->type('string', $_->login), - email => $self->type('string', $_->email), - can_login => $self->type('boolean', $_->is_enabled ? 1 : 0), - email_enabled => $self->type('boolean', $_->email_enabled), - login_denied_text => $self->type('string', $_->disabledtext), - }} @$in_group; - - } - else { - @users = - map {filter $params, { - id => $self->type('int', $_->id), - real_name => $self->type('string', $_->name), - name => $self->type('string', $_->login), - email => $self->type('string', $_->email), - can_login => $self->type('boolean', $_->is_enabled ? 1 : 0), - }} @$in_group; + + my $in_group = $self->_filter_users_by_group(\@user_objects, $params); + foreach my $user (@$in_group) { + my $user_info = filter $params, { + id => $self->type('int', $user->id), + real_name => $self->type('string', $user->name), + name => $self->type('email', $user->login), + email => $self->type('email', $user->email), + can_login => $self->type('boolean', $user->is_enabled ? 1 : 0), + }; + + if (Bugzilla->user->in_group('editusers')) { + $user_info->{email_enabled} = $self->type('boolean', $user->email_enabled); + $user_info->{login_denied_text} = $self->type('string', $user->disabledtext); + } + + if (Bugzilla->user->id == $user->id) { + if (filter_wants($params, 'saved_searches')) { + $user_info->{saved_searches} = [ + map { $self->_query_to_hash($_) } @{ $user->queries } + ]; + } + } + + if (filter_wants($params, 'groups')) { + if (Bugzilla->user->id == $user->id || Bugzilla->user->in_group('editusers')) { + $user_info->{groups} = [ + map { $self->_group_to_hash($_) } @{ $user->groups } + ]; + } + else { + $user_info->{groups} = $self->_filter_bless_groups($user->groups); + } + } + + push(@users, $user_info); } + Bugzilla::Hook::process('webservice_user_get', + { webservice => $self, params => $params, users => \@users }); + return { users => \@users }; } @@ -259,6 +300,40 @@ sub _user_in_any_group { return 0; } +sub _filter_bless_groups { + my ($self, $groups) = @_; + my $user = Bugzilla->user; + + my @filtered_groups; + foreach my $group (@$groups) { + next unless ($user->in_group('editusers') || $user->can_bless($group->id)); + push(@filtered_groups, $self->_group_to_hash($group)); + } + + return \@filtered_groups; +} + +sub _group_to_hash { + my ($self, $group) = @_; + my $item = { + id => $self->type('int', $group->id), + name => $self->type('string', $group->name), + description => $self->type('string', $group->description), + }; + return $item; +} + +sub _query_to_hash { + my ($self, $query) = @_; + my $item = { + id => $self->type('int', $query->id), + name => $self->type('string', $query->name), + url => $self->type('string', $query->url), + }; + + return $item; +} + 1; __END__ @@ -277,6 +352,10 @@ log in/out using an existing account. See L<Bugzilla::WebService> for a description of how parameters are passed, and what B<STABLE>, B<UNSTABLE>, and B<EXPERIMENTAL> mean. +Although the data input and output is the same for JSONRPC, XMLRPC and REST, +the directions for how to access the data via REST is noted in each method +where applicable. + =head1 Logging In and Out =head2 login @@ -295,7 +374,7 @@ etc. This method logs in an user. =over -=item C<login> (string) - The user's login name. +=item C<login> (string) - The user's login name. =item C<password> (string) - The user's password. @@ -311,10 +390,14 @@ management of cookies across sessions. =item B<Returns> -On success, a hash containing one item, C<id>, the numeric id of the -user that was logged in. A set of http cookies is also sent with the -response. These cookies must be sent along with any future requests -to the webservice, for the duration of the session. +On success, a hash containing two items, C<id>, the numeric id of the +user that was logged in, and a C<token> which can be passed in +the parameters as authentication in other calls. A set of http cookies +is also sent with the response. These cookies *or* the token can be sent +along with any future requests to the webservice, for the duration of the +session. Note that cookies are not accepted for GET requests for JSONRPC +and REST for security reasons. You may, however, use the token or valid +login parameters for those requests. =item B<Errors> @@ -360,6 +443,50 @@ Log out the user. Does nothing if there is no user logged in. =back +=head2 valid_login + +B<UNSTABLE> + +=over + +=item B<Description> + +This method will verify whether a client's cookies or current login +token is still valid or have expired. A valid username must be provided +as well that matches. + +=item B<Params> + +=over + +=item C<login> + +The login name that matches the provided cookies or token. + +=item C<token> + +(string) Persistent login token current being used for authentication (optional). +Cookies passed by client will be used before the token if both provided. + +=back + +=item B<Returns> + +Returns true/false depending on if the current cookies or token are valid +for the provided username. + +=item B<Errors> (none) + +=item B<History> + +=over + +=item Added in Bugzilla B<5.0>. + +=back + +=back + =head1 Account Creation =head2 offer_account_by_email @@ -419,6 +546,13 @@ actually receive an email. This function does not check that. You must be logged in and have the C<editusers> privilege in order to call this function. +=item B<REST> + +POST /user + +The params to include in the POST body as well as the returned data format, +are the same as below. + =item B<Params> =over @@ -462,6 +596,8 @@ password is under three characters.) =item Error 503 (Password Too Long) removed in Bugzilla B<3.6>. +=item REST API call added in Bugzilla B<5.0>. + =back =back @@ -478,6 +614,18 @@ B<STABLE> Gets information about user accounts in Bugzilla. +=item B<REST> + +To get information about a single user: + +GET /user/<user_id_or_name> + +To search for users by name, group using URL params same as below: + +GET /user + +The returned data format is the same as below. + =item B<Params> B<Note>: At least one of C<ids>, C<names>, or C<match> must be specified. @@ -539,6 +687,14 @@ match string. Setting C<include_disabled> to C<true> will include disabled users in the returned results even if their username doesn't fully match the input string. +=item B<History> + +=over + +=item REST API call added in Bugzilla B<5.0>. + +=back + =back =item B<Returns> @@ -581,10 +737,60 @@ C<string> A text field that holds the reason for disabling a user from logging into bugzilla, if empty then the user account is enabled. Otherwise it is disabled/closed. +=item groups + +C<array> An array of group hashes the user is a member of. Each hash describes +the group and contains the following items: + +=over + +=item id + +C<int> The group id + +=item name + +C<string> The name of the group + +=item description + +C<string> The description for the group + +=back + +=over + +=item saved_searches + +C<array> An array of hashes, each of which represents a user's saved search and has +the following keys: + +=over + +=item id + +C<int> An integer id uniquely identifying the saved search. + +=item name + +C<string> The name of the saved search. + +=item url + +C<string> The CGI parameters for the saved search. + +=back + +B<Note>: The elements of the returned array (i.e. hashes) are ordered by the +name of each saved search. + +=back + B<Note>: If you are not logged in to Bugzilla when you call this function, you will only be returned the C<id>, C<name>, and C<real_name> items. If you are logged in and not in editusers group, you will only be returned the C<id>, C<name>, -C<real_name>, C<email>, and C<can_login> items. +C<real_name>, C<email>, and C<can_login> items. The groups returned are filtered +based on your permission to bless each group. =back @@ -625,9 +831,15 @@ exist or you do not belong to it. =item C<include_disabled> added in Bugzilla B<4.0>. Default behavior for C<match> has changed to only returning enabled accounts. +=item C<groups> Added in Bugzilla B<4.4>. + +=item C<saved_searches> Added in Bugzilla B<4.4>. + =item Error 804 has been added in Bugzilla 4.0.9 and 4.2.4. It's now illegal to pass a group name you don't belong to. =back +=item REST API call added in Bugzilla B<5.0>. + =back diff --git a/Bugzilla/WebService/Util.pm b/Bugzilla/WebService/Util.pm index fe4105ca2..856fd3481 100644 --- a/Bugzilla/WebService/Util.pm +++ b/Bugzilla/WebService/Util.pm @@ -21,6 +21,13 @@ package Bugzilla::WebService::Util; use strict; + +use Bugzilla::Flag; +use Bugzilla::FlagType; +use Bugzilla::Error; + +use Storable qw(dclone); + use base qw(Exporter); # We have to "require", not "use" this, because otherwise it tries to @@ -28,36 +35,165 @@ use base qw(Exporter); require Test::Taint; our @EXPORT_OK = qw( + extract_flags filter filter_wants taint_data validate + translate + params_to_objects + fix_credentials ); -sub filter ($$) { - my ($params, $hash) = @_; +sub extract_flags { + my ($flags, $bug, $attachment) = @_; + my (@new_flags, @old_flags); + + my $flag_types = $attachment ? $attachment->flag_types : $bug->flag_types; + my $current_flags = $attachment ? $attachment->flags : $bug->flags; + + # Copy the user provided $flags as we may call extract_flags more than + # once when editing multiple bugs or attachments. + my $flags_copy = dclone($flags); + + foreach my $flag (@$flags_copy) { + my $id = $flag->{id}; + my $type_id = $flag->{type_id}; + + my $new = delete $flag->{new}; + my $name = delete $flag->{name}; + + if ($id) { + my $flag_obj = grep($id == $_->id, @$current_flags); + $flag_obj || ThrowUserError('object_does_not_exist', + { class => 'Bugzilla::Flag', id => $id }); + } + elsif ($type_id) { + my $type_obj = grep($type_id == $_->id, @$flag_types); + $type_obj || ThrowUserError('object_does_not_exist', + { class => 'Bugzilla::FlagType', id => $type_id }); + if (!$new) { + my @flag_matches = grep($type_id == $_->type->id, @$current_flags); + @flag_matches > 1 && ThrowUserError('flag_not_unique', + { value => $type_id }); + if (!@flag_matches) { + delete $flag->{id}; + } + else { + delete $flag->{type_id}; + $flag->{id} = $flag_matches[0]->id; + } + } + } + elsif ($name) { + my @type_matches = grep($name eq $_->name, @$flag_types); + @type_matches > 1 && ThrowUserError('flag_type_not_unique', + { value => $name }); + @type_matches || ThrowUserError('object_does_not_exist', + { class => 'Bugzilla::FlagType', name => $name }); + if ($new) { + delete $flag->{id}; + $flag->{type_id} = $type_matches[0]->id; + } + else { + my @flag_matches = grep($name eq $_->type->name, @$current_flags); + @flag_matches > 1 && ThrowUserError('flag_not_unique', { value => $name }); + if (@flag_matches) { + $flag->{id} = $flag_matches[0]->id; + } + else { + delete $flag->{id}; + $flag->{type_id} = $type_matches[0]->id; + } + } + } + + if ($flag->{id}) { + push(@old_flags, $flag); + } + else { + push(@new_flags, $flag); + } + } + + return (\@old_flags, \@new_flags); +} + +sub filter($$;$$) { + my ($params, $hash, $types, $prefix) = @_; my %newhash = %$hash; foreach my $key (keys %$hash) { - delete $newhash{$key} if !filter_wants($params, $key); + delete $newhash{$key} if !filter_wants($params, $key, $types, $prefix); } return \%newhash; } -sub filter_wants ($$) { - my ($params, $field) = @_; +sub filter_wants($$;$$) { + my ($params, $field, $types, $prefix) = @_; + + # Since this is operation is resource intensive, we will cache the results + # This assumes that $params->{*_fields} doesn't change between calls + my $cache = Bugzilla->request_cache->{filter_wants} ||= {}; + $field = "${prefix}.${field}" if $prefix; + + if (exists $cache->{$field}) { + return $cache->{$field}; + } + + # Mimic old behavior if no types provided + my %field_types = map { $_ => 1 } (ref $types ? @$types : ($types || 'default')); + my %include = map { $_ => 1 } @{ $params->{'include_fields'} || [] }; my %exclude = map { $_ => 1 } @{ $params->{'exclude_fields'} || [] }; - if (defined $params->{include_fields}) { - return 0 if !$include{$field}; + my %include_types; + my %exclude_types; + + # Only return default fields if nothing is specified + $include_types{default} = 1 if !%include; + + # Look for any field types requested + foreach my $key (keys %include) { + next if $key !~ /^_(.*)$/; + $include_types{$1} = 1; + delete $include{$key}; + } + foreach my $key (keys %exclude) { + next if $key !~ /^_(.*)$/; + $exclude_types{$1} = 1; + delete $exclude{$key}; + } + + # Explicit inclusion/exclusion + return $cache->{$field} = 0 if $exclude{$field}; + return $cache->{$field} = 1 if $include{$field}; + + # If the user has asked to include all or exclude all + return $cache->{$field} = 0 if $exclude_types{'all'}; + return $cache->{$field} = 1 if $include_types{'all'}; + + # If the user has not asked for any fields specifically or if the user has asked + # for one or more of the field's types (and not excluded them) + foreach my $type (keys %field_types) { + return $cache->{$field} = 0 if $exclude_types{$type}; + return $cache->{$field} = 1 if $include_types{$type}; + } + + my $wants = 0; + if ($prefix) { + # Include the field if the parent is include (and this one is not excluded) + $wants = 1 if $include{$prefix}; } - if (defined $params->{exclude_fields}) { - return 0 if $exclude{$field}; + else { + # We want to include this if one of the sub keys is included + my $key = $field . '.'; + my $len = length($key); + $wants = 1 if grep { substr($_, 0, $len) eq $key } keys %include; } - return 1; + return $cache->{$field} = $wants; } sub taint_data { @@ -78,7 +214,7 @@ sub _delete_bad_keys { # However, we need to validate our argument names in some way. # We know that all hash keys passed in to the WebService will # match \w+, so we delete any key that doesn't match that. - if ($key !~ /^\w+$/) { + if ($key !~ /^[\w\.\-]+$/) { delete $item->{$key}; } } @@ -108,6 +244,51 @@ sub validate { return ($self, $params); } +sub translate { + my ($params, $mapped) = @_; + my %changes; + while (my ($key,$value) = each (%$params)) { + my $new_field = $mapped->{$key} || $key; + $changes{$new_field} = $value; + } + return \%changes; +} + +sub params_to_objects { + my ($params, $class) = @_; + my (@objects, @objects_by_ids); + + @objects = map { $class->check($_) } + @{ $params->{names} } if $params->{names}; + + @objects_by_ids = map { $class->check({ id => $_ }) } + @{ $params->{ids} } if $params->{ids}; + + push(@objects, @objects_by_ids); + my %seen; + @objects = grep { !$seen{$_->id}++ } @objects; + return \@objects; +} + +sub fix_credentials { + my ($params) = @_; + # Allow user to pass in login=foo&password=bar as a convenience + # even if not calling GET /login. We also do not delete them as + # GET /login requires "login" and "password". + if (exists $params->{'login'} && exists $params->{'password'}) { + $params->{'Bugzilla_login'} = $params->{'login'}; + $params->{'Bugzilla_password'} = $params->{'password'}; + } + # Allow user to pass token=12345678 as a convenience which becomes + # "Bugzilla_token" which is what the auth code looks for. + if (exists $params->{'token'}) { + $params->{'Bugzilla_token'} = $params->{'token'}; + } + + # Allow extensions to modify the credential data before login + Bugzilla::Hook::process('webservice_fix_credentials', { params => $params }); +} + __END__ =head1 NAME @@ -136,6 +317,13 @@ of WebService methods. Given a hash (the second argument to this subroutine), this will remove any keys that are I<not> in C<include_fields> and then remove any keys that I<are> in C<exclude_fields>. +An optional third option can be passed that prefixes the field name to allow +filtering of data two or more levels deep. + +For example, if you want to filter out the C<id> key/value in components returned +by Product.get, you would use the value C<component.id> in your C<exclude_fields> +list. + =head2 filter_wants Returns C<1> if a filter would preserve the specified field when passing @@ -147,3 +335,31 @@ This helps in the validation of parameters passed into the WebService methods. Currently it converts listed parameters into an array reference if the client only passed a single scalar value. It modifies the parameters hash in place so other parameters should be unaltered. + +=head2 params_to_objects + +Creates objects of the type passed in as the second parameter, using the +parameters passed to a WebService method (the first parameter to this function). +Helps make life simpler for WebService methods that internally create objects +via both "ids" and "names" fields. Also de-duplicates objects that were loaded +by both "ids" and "names". Returns an arrayref of objects. + +=head2 fix_credentials + +Allows for certain parameters related to authentication such as Bugzilla_login, +Bugzilla_password, and Bugzilla_token to have shorter named equivalents passed in. +This function converts the shorter versions to their respective internal names. + +=head2 extract_flags + +Subroutine that takes a list of hashes that are potential flag changes for +both bugs and attachments. Then breaks the list down into two separate lists +based on if the change is to add a new flag or to update an existing flag. + +=head1 B<Methods in need of POD> + +=over + +=item taint_data + +=back |