From 9c26c01867ca3e2af1e70c051140eea59c68c500 Mon Sep 17 00:00:00 2001 From: Dylan William Hardison Date: Tue, 7 Mar 2017 17:47:36 -0500 Subject: Bug 1307485 - Add code to run a subset of buglist.cgi search queries against the ES backend --- Bugzilla.pm | 6 + Bugzilla/Bug.pm | 47 +-- Bugzilla/Elastic.pm | 47 +++ Bugzilla/Elastic/Indexer.pm | 29 +- Bugzilla/Elastic/Role/Search.pm | 16 + Bugzilla/Elastic/Search.pm | 425 +++++++++++++++++++++ Bugzilla/Elastic/Search/FakeCGI.pm | 43 +++ Bugzilla/Search/Quicksearch.pm | 6 +- Bugzilla/User.pm | 7 +- Bugzilla/WebService/Constants.pm | 1 + Bugzilla/WebService/Elastic.pm | 59 +++ Bugzilla/WebService/Server/REST.pm | 1 + .../WebService/Server/REST/Resources/Elastic.pm | 30 ++ buglist.cgi | 95 ++++- js/field.js | 6 +- scripts/search.pl | 13 + scripts/suggest-user.pl | 20 + template/en/default/list/list.html.tmpl | 8 + 18 files changed, 782 insertions(+), 77 deletions(-) create mode 100644 Bugzilla/Elastic.pm create mode 100644 Bugzilla/Elastic/Role/Search.pm create mode 100644 Bugzilla/Elastic/Search.pm create mode 100644 Bugzilla/Elastic/Search/FakeCGI.pm create mode 100644 Bugzilla/WebService/Elastic.pm create mode 100644 Bugzilla/WebService/Server/REST/Resources/Elastic.pm create mode 100644 scripts/search.pl create mode 100644 scripts/suggest-user.pl diff --git a/Bugzilla.pm b/Bugzilla.pm index 55e416933..8e70cb653 100644 --- a/Bugzilla.pm +++ b/Bugzilla.pm @@ -23,6 +23,7 @@ BEGIN { use Bugzilla::Auth; use Bugzilla::Auth::Persist::Cookie; use Bugzilla::CGI; +use Bugzilla::Elastic; use Bugzilla::Config; use Bugzilla::Constants; use Bugzilla::DB; @@ -782,6 +783,11 @@ sub memcached { } } +sub elastic { + my ($class) = @_; + $class->process_cache->{elastic} //= Bugzilla::Elastic->new(); +} + # Private methods # Per-process cleanup. Note that this is a plain subroutine, not a method, diff --git a/Bugzilla/Bug.pm b/Bugzilla/Bug.pm index cba973863..bc099f76e 100644 --- a/Bugzilla/Bug.pm +++ b/Bugzilla/Bug.pm @@ -303,19 +303,15 @@ with 'Bugzilla::Elastic::Role::Object'; sub ES_TYPE {'bug'} sub _bz_field { - my ($field, $type, $analyzer, @fields) = @_; + my ($field, @fields) = @_; return ( $field => { - type => $type, - analyzer => $analyzer, + type => 'string', + analyzer => 'bz_text_analyzer', fields => { - raw => { - type => 'string', - index => 'not_analyzed', - }, eq => { - type => 'string', + type => 'string', analyzer => 'bz_equals_analyzer', }, @fields, @@ -324,32 +320,20 @@ sub _bz_field { ); } -sub _bz_text_field { - my ($field) = @_; - - return _bz_field($field, 'string', 'bz_text_analyzer'); -} - -sub _bz_substring_field { - my ($field, @rest) = @_; - - return _bz_field($field, 'string', 'bz_substring_analyzer', @rest); -} - sub ES_PROPERTIES { return { - priority => { type => 'string', analyzer => 'keyword' }, - bug_severity => { type => 'string', analyzer => 'keyword' }, - bug_status => { type => 'string', analyzer => 'keyword' }, - resolution => { type => 'string', analyzer => 'keyword' }, - keywords => { type => 'string' }, + _bz_field('priority'), + _bz_field('bug_severity'), + _bz_field('bug_status'), + _bz_field('resolution'), status_whiteboard => { type => 'string', analyzer => 'whiteboard_shingle_tokens' }, delta_ts => { type => 'string', index => 'not_analyzed' }, - _bz_substring_field('product'), - _bz_substring_field('component'), - _bz_substring_field('classification'), - _bz_text_field('short_desc'), - _bz_substring_field('assigned_to'), + _bz_field('product'), + _bz_field('component'), + _bz_field('classification'), + _bz_field('short_desc'), + _bz_field('assigned_to'), + _bz_field('reporter'), }; } @@ -426,7 +410,7 @@ sub es_document { bug_id => $self->id, product => $self->product_obj->name, alias => $self->alias, - keywords => $self->keywords, + keywords => [ map { $_->name } @{$self->keyword_objects} ], priority => $self->priority, bug_status => $self->bug_status, resolution => $self->resolution, @@ -435,6 +419,7 @@ sub es_document { status_whiteboard => $self->status_whiteboard, short_desc => $self->short_desc, assigned_to => $self->assigned_to->login, + reporter => $self->reporter->login, delta_ts => $self->delta_ts, bug_severity => $self->bug_severity, }; diff --git a/Bugzilla/Elastic.pm b/Bugzilla/Elastic.pm new file mode 100644 index 000000000..6384269fd --- /dev/null +++ b/Bugzilla/Elastic.pm @@ -0,0 +1,47 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. +package Bugzilla::Elastic; +use 5.10.1; +use Moo; + +use Bugzilla::Elastic::Search; +use Bugzilla::Util qw(trick_taint); + +with 'Bugzilla::Elastic::Role::HasClient'; +with 'Bugzilla::Elastic::Role::HasIndexName'; + +sub suggest_users { + my ($self, $text) = @_; + my $field = 'suggest_user'; + if ($text =~ /^:(.+)$/) { + $text = $1; + $field = 'suggest_nick'; + } + + my $result = eval { + $self->client->suggest( + index => $self->index_name, + body => { + $field => { + text => $text, + completion => { field => $field, size => 25 }, + } + } + ); + }; + if (defined $result) { + return [ map { $_->{payload} } @{$result->{$field}[0]{options}} ]; + } + else { + warn "suggest_users error: $@"; + my $users = Bugzilla::User::match($text, 25, 0); + return [ map { { real_name => $_->name, name => $_->login } } @$users]; + } +} + + +1; diff --git a/Bugzilla/Elastic/Indexer.pm b/Bugzilla/Elastic/Indexer.pm index 82f946af9..dd71a7198 100644 --- a/Bugzilla/Elastic/Indexer.pm +++ b/Bugzilla/Elastic/Indexer.pm @@ -23,7 +23,7 @@ has 'mtime' => ( has 'shadow_dbh' => ( is => 'lazy' ); has 'debug_sql' => ( - is => 'ro', + is => 'ro', default => 0, ); @@ -40,24 +40,24 @@ sub create_index { index => $self->index_name, body => { settings => { - number_of_shards => 1, + number_of_shards => 2, analysis => { + filter => { + asciifolding_original => { + type => "asciifolding", + preserve_original => \1, + }, + }, analyzer => { folding => { - type => 'standard', tokenizer => 'standard', - filter => [ 'lowercase', 'asciifolding' ] + filter => ['standard', 'lowercase', 'asciifolding_original'], }, bz_text_analyzer => { type => 'standard', filter => ['lowercase', 'stop'], max_token_length => '20' }, - bz_substring_analyzer => { - type => 'custom', - filter => ['lowercase'], - tokenizer => 'bz_ngram_tokenizer', - }, bz_equals_analyzer => { type => 'custom', filter => ['lowercase'], @@ -71,25 +71,20 @@ sub create_index { whiteboard_shingle_words => { type => 'custom', tokenizer => 'whiteboard_words_pattern', - filter => ['stop', 'shingle'] + filter => ['stop', 'shingle', 'lowercase'] }, whiteboard_tokens => { type => 'custom', tokenizer => 'whiteboard_tokens_pattern', - filter => ['stop'] + filter => ['stop', 'lowercase'] }, whiteboard_shingle_tokens => { type => 'custom', tokenizer => 'whiteboard_tokens_pattern', - filter => ['stop', 'shingle'] + filter => ['stop', 'shingle', 'lowercase'] } }, tokenizer => { - bz_ngram_tokenizer => { - type => 'nGram', - min_ngram => 2, - max_ngram => 25, - }, whiteboard_tokens_pattern => { type => 'pattern', pattern => '\\s*([,;]*\\[|\\][\\s\\[]*|[;,])\\s*' diff --git a/Bugzilla/Elastic/Role/Search.pm b/Bugzilla/Elastic/Role/Search.pm new file mode 100644 index 000000000..9446e0da8 --- /dev/null +++ b/Bugzilla/Elastic/Role/Search.pm @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. +package Bugzilla::Elastic::Role::Search; + +use 5.10.1; +use strict; +use warnings; +use Role::Tiny; + +requires qw(data search_description invalid_order_columns order); + +1; diff --git a/Bugzilla/Elastic/Search.pm b/Bugzilla/Elastic/Search.pm new file mode 100644 index 000000000..5c60f2353 --- /dev/null +++ b/Bugzilla/Elastic/Search.pm @@ -0,0 +1,425 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. +package Bugzilla::Elastic::Search; + +use 5.10.1; +use Moo; +use Bugzilla::Search; +use Bugzilla::Search::Quicksearch; +use Bugzilla::Util qw(trick_taint); +use namespace::clean; + +use Bugzilla::Elastic::Search::FakeCGI; + + +has 'quicksearch' => ( is => 'ro' ); +has 'limit' => ( is => 'ro', predicate => 'has_limit' ); +has 'offset' => ( is => 'ro', predicate => 'has_offset' ); +has 'fields' => ( is => 'ro', isa => \&_arrayref_of_fields, default => sub { [] } ); +has 'params' => ( is => 'lazy' ); +has 'clause' => ( is => 'lazy' ); +has 'es_query' => ( is => 'lazy' ); +has 'search_description' => (is => 'lazy'); +has 'query_time' => ( is => 'rwp' ); + +has '_input_order' => ( is => 'ro', init_arg => 'order', required => 1); +has '_order' => ( is => 'lazy', init_arg => undef ); +has 'invalid_order_columns' => ( is => 'lazy' ); + +with 'Bugzilla::Elastic::Role::HasClient'; +with 'Bugzilla::Elastic::Role::HasIndexName'; +with 'Bugzilla::Elastic::Role::Search'; + +my @SUPPORTED_FIELDS = qw( + bug_id product component short_desc + priority status_whiteboard bug_status resolution + keywords alias assigned_to reporter delta_ts + longdesc cf_crash_signature classification bug_severity + commenter +); +my %IS_SUPPORTED_FIELD = map { $_ => 1 } @SUPPORTED_FIELDS; + +$IS_SUPPORTED_FIELD{relevance} = 1; + +my @NORMAL_FIELDS = qw( + priority + bug_severity + bug_status + resolution + product + component + classification + short_desc + assigned_to + reporter +); + +my %SORT_MAP = ( + bug_id => '_id', + relevance => '_score', + map { $_ => "$_.eq" } @NORMAL_FIELDS, +); + +my %EQUALS_MAP = ( + map { $_ => "$_.eq" } @NORMAL_FIELDS, +); + +sub _arrayref_of_fields { + my $f = $_; + foreach my $field (@$f) { + Bugzilla::Elastic::Search::UnsupportedField->throw(field => $field) + unless $IS_SUPPORTED_FIELD{$field}; + } +} + +sub data { + my ($self) = @_; + my $body = $self->es_query; + my $result = eval { + $self->client->search( + index => $self->index_name, + type => 'bug', + body => $body, + ); + }; + if (!$result) { + die $@; + } + $self->_set_query_time($result->{took} / 1000); + my (@ids, %hits); + my $fields = $self->fields; + foreach my $hit (@{ $result->{hits}{hits} }) { + push @ids, $hit->{_id}; + my $source = $hit->{_source}; + $source->{relevance} = $hit->{_score}; + foreach my $val (values %$source) { + next unless defined $val; + trick_taint($val); + } + trick_taint($hit->{_id}); + if ($source) { + $hits{$hit->{_id}} = [ @$source{@$fields} ]; + } + else { + $hits{$hit->{_id}} = $hit->{_id}; + } + } + my $visible_ids = Bugzilla->user->visible_bugs(\@ids); + + return [ map { $hits{$_} } @$visible_ids ]; +} + +sub _valid_order { + my ($self) = @_; + + return grep { $IS_SUPPORTED_FIELD{$_->[0]} } @{$self->_order}; +} + +sub order { + my ($self) = @_; + + return map { $_->[0] } $self->_valid_order; +} + +sub _quicksearch_to_params { + my ($quicksearch) = @_; + no warnings 'redefine'; + my $cgi = Bugzilla::Elastic::Search::FakeCGI->new; + local *Bugzilla::cgi = sub { $cgi }; + local $Bugzilla::Search::Quicksearch::ELASTIC = 1; + quicksearch($quicksearch); + + return $cgi->params; +} + +sub _build_fields { return \@SUPPORTED_FIELDS } + +sub _build__order { + my ($self) = @_; + + my @order; + foreach my $order (@{$self->_input_order}) { + if ($order =~ /^(.+)\s+(asc|desc)$/i) { + push @order, [ $1, lc $2 ]; + } + else { + push @order, [ $order ]; + } + } + return \@order; +} + +sub _build_invalid_order_columns { + my ($self) = @_; + + return [ map { $_->[0] } grep { !$IS_SUPPORTED_FIELD{$_->[0]} } @{ $self->_order } ]; +} + +sub _build_params { + my ($self) = @_; + + return _quicksearch_to_params($self->quicksearch); +} + +sub _build_clause { + my ($self) = @_; + my $search = Bugzilla::Search->new(params => $self->params); + + return $search->_params_to_data_structure; +} + +sub _build_search_description { + my ($self) = @_; + + return [_describe($self->clause)]; +} + +sub _describe { + my ($thing) = @_; + + state $class_to_func = { + 'Bugzilla::Search::Condition' => \&_describe_condition, + 'Bugzilla::Search::Clause' => \&_describe_clause + }; + + my $func = $class_to_func->{ref $thing} or die "nothing for $thing\n"; + + return $func->($thing); +} + +sub _describe_clause { + my ($clause) = @_; + + return map { _describe($_) } @{$clause->children}; +} + +sub _describe_condition { + my ($cond) = @_; + + return { field => $cond->field, type => $cond->operator, value => _describe_value($cond->value) }; +} + +sub _describe_value { + my ($val) = @_; + + return ref($val) ? join(", ", @$val) : $val; +} + +sub _build_es_query { + my ($self) = @_; + my @extra; + + if ($self->_valid_order) { + my @sort = map { + my $f = $SORT_MAP{$_->[0]} // $_->[0]; + @$_ > 1 ? { $f => lc $_[1] } : $f; + } $self->_valid_order; + push @extra, sort => \@sort; + } + if ($self->has_offset) { + push @extra, from => $self->offset; + } + my $max_limit = Bugzilla->params->{max_search_results}; + my $limit = Bugzilla->params->{default_search_limit}; + if ($self->has_limit) { + if ($self->limit) { + my $l = $self->limit; + $limit = $l < $max_limit ? $l : $max_limit; + } + else { + $limit = $max_limit; + } + } + push @extra, size => $limit; + return { + _source => @{$self->fields} ? \1 : \0, + query => _query($self->clause), + @extra, + }; +} + +sub _query { + my ($thing) = @_; + state $class_to_func = { + 'Bugzilla::Search::Condition' => \&_query_condition, + 'Bugzilla::Search::Clause' => \&_query_clause, + }; + + my $func = $class_to_func->{ref $thing} or die "nothing for $thing\n"; + + return $func->($thing); +} + +sub _query_condition { + my ($cond) = @_; + state $operator_to_es = { + equals => \&_operator_equals, + substring => \&_operator_substring, + anyexact => \&_operator_anyexact, + anywords => \&_operator_anywords, + allwords => \&_operator_allwords, + }; + + my $field = $cond->field; + my $operator = $cond->operator; + my $value = $cond->value; + + if ($field eq 'resolution') { + $value = [ map { $_ eq '---' ? '' : $_ } ref $value ? @$value : $value ]; + } + + unless ($IS_SUPPORTED_FIELD{$field}) { + Bugzilla::Elastic::Search::UnsupportedField->throw(field => $field); + } + + my $op = $operator_to_es->{$operator} + or Bugzilla::Elastic::Search::UnsupportedOperator->throw(operator => $operator); + + my $result; + if (ref $op) { + $result = $op->($field, $value); + } else { + $result = { $op => { $field => $value } }; + } + + return $result; +} + +# is equal to any of the strings +sub _operator_anyexact { + my ($field, $value) = @_; + my @values = ref $value ? @$value : split(/\s*,\s*/, $value); + if (@values == 1) { + return _operator_equals($field, $values[0]); + } + else { + return { + terms => { + $EQUALS_MAP{$field} // $field => [map { lc } @values], + minimum_should_match => 1, + }, + }; + } +} + +# contains any of the words +sub _operator_anywords { + my ($field, $value) = @_; + return { + match => { + $field => { query => $value, operator => "or" } + }, + }; +} + +# contains all of the words +sub _operator_allwords { + my ($field, $value) = @_; + return { + match => { + $field => { query => $value, operator => "and" } + }, + }; +} + +sub _operator_equals { + my ($field, $value) = @_; + return { + match => { + $EQUALS_MAP{$field} // $field => $value, + }, + }; +} + +sub _operator_substring { + my ($field, $value) = @_; + my $is_insider = Bugzilla->user->is_insider; + + if ($field eq 'longdesc') { + return { + has_child => { + type => 'comment', + query => { + bool => { + must => [ + { match => { body => { query => $value, operator => "and" } } }, + $is_insider ? () : { term => { is_private => \0 } }, + ], + }, + }, + }, + } + } + elsif ($field eq 'reporter' or $field eq 'assigned_to') { + return { + prefix => { + $EQUALS_MAP{$field} // $field => lc $value, + } + } + } + else { + return { + wildcard => { + $EQUALS_MAP{$field} // $field => lc "*$value*", + } + }; + } +} + +sub _query_clause { + my ($clause) = @_; + + state $joiner_to_func = { + AND => \&_join_and, + OR => \&_join_or, + }; + + my @children = grep { !$_->isa('Bugzilla::Search::Clause') || @{$_->children} } @{$clause->children}; + if (@children == 1) { + return _query($children[0]); + } + + return $joiner_to_func->{$clause->joiner}->([ map { _query($_) } @children ]); +} + +sub _join_and { + my ($children) = @_; + return { bool => { must => $children } }, +} + +sub _join_or { + my ($children) = @_; + return { bool => { should => $children } }; +} + +# Exceptions +BEGIN { + package Bugzilla::Elastic::Search::Redirect; + use Moo; + + with 'Throwable'; + + has 'redirect_args' => (is => 'ro', required => 1); + + package Bugzilla::Elastic::Search::UnsupportedField; + use Moo; + use overload q{""} => sub { "Unsupported field: ", $_[0]->field }, fallback => 1; + + with 'Throwable'; + + has 'field' => (is => 'ro', required => 1); + + + package Bugzilla::Elastic::Search::UnsupportedOperator; + use Moo; + + with 'Throwable'; + + has 'operator' => (is => 'ro', required => 1); +} + +1; diff --git a/Bugzilla/Elastic/Search/FakeCGI.pm b/Bugzilla/Elastic/Search/FakeCGI.pm new file mode 100644 index 000000000..827c96c52 --- /dev/null +++ b/Bugzilla/Elastic/Search/FakeCGI.pm @@ -0,0 +1,43 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. +package Bugzilla::Elastic::Search::FakeCGI; +use 5.10.1; +use Moo; +use namespace::clean; + +has 'params' => (is => 'ro', default => sub { {} }); + +# we pretend to be Bugzilla::CGI at times. +sub canonicalise_query { + return Bugzilla::CGI::canonicalise_query(@_); +} + +sub delete { + my ($self, $key) = @_; + delete $self->params->{$key}; +} + +sub redirect { + my ($self, @args) = @_; + + Bugzilla::Elastic::Search::Redirect->throw(redirect_args => \@args); +} + +sub param { + my ($self, $key, $val, @rest) = @_; + if (@_ > 3) { + $self->params->{$key} = [$val, @rest]; + } elsif (@_ == 3) { + $self->params->{$key} = $val; + } elsif (@_ == 2) { + return $self->params->{$key}; + } else { + return $self->params + } +} + +1; diff --git a/Bugzilla/Search/Quicksearch.pm b/Bugzilla/Search/Quicksearch.pm index 4f11a3f54..462a9ba85 100644 --- a/Bugzilla/Search/Quicksearch.pm +++ b/Bugzilla/Search/Quicksearch.pm @@ -127,7 +127,7 @@ use constant COMPONENT_EXCEPTIONS => ( ); # Quicksearch-wide globals for boolean charts. -our ($chart, $and, $or, $fulltext, $bug_status_set); +our ($chart, $and, $or, $fulltext, $bug_status_set, $ELASTIC); sub quicksearch { my ($searchstring) = (@_); @@ -587,7 +587,8 @@ sub _default_quicksearch_word { addChart('alias', 'substring', $word, $negate); addChart('short_desc', 'substring', $word, $negate); addChart('status_whiteboard', 'substring', $word, $negate); - addChart('content', 'matches', _matches_phrase($word), $negate) if $fulltext; + addChart('longdesc', 'substring', $word, $negate) if $ELASTIC; + addChart('content', 'matches', _matches_phrase($word), $negate) if $fulltext && !$ELASTIC; # BMO Bug 664124 - Include the crash signature (sig:) field in default quicksearches addChart('cf_crash_signature', 'substring', $word, $negate); @@ -617,6 +618,7 @@ sub _handle_urls { # Quote and escape a phrase appropriately for a "content matches" search. sub _matches_phrase { my ($phrase) = @_; + return $phrase if $ELASTIC; $phrase =~ s/"/\\"/g; return "\"$phrase\""; } diff --git a/Bugzilla/User.pm b/Bugzilla/User.pm index 69885f57c..e8ddc0be7 100644 --- a/Bugzilla/User.pm +++ b/Bugzilla/User.pm @@ -128,7 +128,7 @@ with 'Bugzilla::Elastic::Role::Object'; sub ES_TYPE { 'user' } -sub ES_OBJECTS_AT_ONCE { 2000 } +sub ES_OBJECTS_AT_ONCE { 5000 } sub ES_SELECT_UPDATED_SQL { my ($class, $mtime) = @_; @@ -150,7 +150,7 @@ sub ES_SELECT_ALL_SQL { my $id = $class->ID_FIELD; my $table = $class->DB_TABLE; - return ("SELECT $id FROM $table WHERE $id > ? AND is_enabled ORDER BY $id", [$last_id // 0]); + return ("SELECT $id FROM $table WHERE $id > ? AND is_enabled AND NOT disabledtext ORDER BY $id", [$last_id // 0]); } sub ES_PROPERTIES { @@ -175,7 +175,6 @@ sub ES_PROPERTIES { sub es_document { my ( $self, $timestamp ) = @_; - my $weight = eval { $self->last_activity_ts ? datetime_from($self->last_activity_ts)->epoch : 0 } // 0; my $doc = { login => $self->login, name => $self->name, @@ -184,7 +183,6 @@ sub es_document { input => [ $self->login, $self->name ], output => $self->identity, payload => { name => $self->login, real_name => $self->name }, - weight => $weight, }, }; if ($self->name && $self->name =~ /:(\w+)/) { @@ -193,7 +191,6 @@ sub es_document { input => [ $ircnick ], output => $self->login, payload => { name => $self->login, real_name => $self->name, ircnick => $ircnick }, - weight => $weight, }; } diff --git a/Bugzilla/WebService/Constants.pm b/Bugzilla/WebService/Constants.pm index bf3a93fd5..1399513c5 100644 --- a/Bugzilla/WebService/Constants.pm +++ b/Bugzilla/WebService/Constants.pm @@ -290,6 +290,7 @@ sub WS_DISPATCH { 'Product' => 'Bugzilla::WebService::Product', 'Group' => 'Bugzilla::WebService::Group', 'BugUserLastVisit' => 'Bugzilla::WebService::BugUserLastVisit', + 'Elastic' => 'Bugzilla::WebService::Elastic', %hook_dispatch }; return $dispatch; diff --git a/Bugzilla/WebService/Elastic.pm b/Bugzilla/WebService/Elastic.pm new file mode 100644 index 000000000..3a33a1dba --- /dev/null +++ b/Bugzilla/WebService/Elastic.pm @@ -0,0 +1,59 @@ +# -*- Mode: perl; indent-tabs-mode: nil -*- +# +# The contents of this file are subject to the Mozilla Public +# License Version 1.1 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of +# the License at http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS +# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or +# implied. See the License for the specific language governing +# rights and limitations under the License. +# +# The Original Code is the Bugzilla Bug Tracking System. +# +# Contributor(s): Marc Schumann +# Max Kanat-Alexander +# Mads Bondo Dydensborg +# Noura Elhawary + +package Bugzilla::WebService::Elastic; + +use 5.10.1; +use strict; +use warnings; +use base qw(Bugzilla::WebService); + +use Bugzilla; +use Bugzilla::Constants; +use Bugzilla::Error; +use Bugzilla::WebService::Util qw(validate); +use Bugzilla::Util qw(trim detaint_natural trick_taint); + +use constant READ_ONLY => qw( suggest_users ); +use constant PUBLIC_METHODS => qw( suggest_users ); + +sub suggest_users { + my ($self, $params) = @_; + + Bugzilla->switch_to_shadow_db(); + + ThrowCodeError('params_required', { function => 'Elastic.suggest_users', params => ['match'] }) + unless defined $params->{match}; + + ThrowUserError('user_access_by_match_denied') + unless Bugzilla->user->id; + + trick_taint($params->{match}); + my $results = Bugzilla->elastic->suggest_users($params->{match} . ""); + my @users = map { + { + real_name => $self->type(string => $_->{real_name}), + name => $self->type(email => $_->{name}), + } + } @$results; + + return { users => \@users }; +} + +1; \ No newline at end of file diff --git a/Bugzilla/WebService/Server/REST.pm b/Bugzilla/WebService/Server/REST.pm index d9381b2c8..6e1944061 100644 --- a/Bugzilla/WebService/Server/REST.pm +++ b/Bugzilla/WebService/Server/REST.pm @@ -29,6 +29,7 @@ use Bugzilla::WebService::Server::REST::Resources::Group; use Bugzilla::WebService::Server::REST::Resources::Product; use Bugzilla::WebService::Server::REST::Resources::User; use Bugzilla::WebService::Server::REST::Resources::BugUserLastVisit; +use Bugzilla::WebService::Server::REST::Resources::Elastic; use List::MoreUtils qw(uniq); use Scalar::Util qw(blessed reftype); diff --git a/Bugzilla/WebService/Server/REST/Resources/Elastic.pm b/Bugzilla/WebService/Server/REST/Resources/Elastic.pm new file mode 100644 index 000000000..2f7c1eaa4 --- /dev/null +++ b/Bugzilla/WebService/Server/REST/Resources/Elastic.pm @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::WebService::Server::REST::Resources::Elastic; + +use 5.10.1; +use strict; +use warnings; + +use Bugzilla::WebService::Constants; +use Bugzilla::WebService::Elastic; + +BEGIN { + *Bugzilla::WebService::Elastic::rest_resources = \&_rest_resources; +}; + +sub _rest_resources { + my $rest_resources = [ + qr{^/elastic/suggest_users$}, { + GET => { method => 'suggest_users' }, + }, + ]; + return $rest_resources; +} + +1; diff --git a/buglist.cgi b/buglist.cgi index 850d79d01..fa6bb060f 100755 --- a/buglist.cgi +++ b/buglist.cgi @@ -687,11 +687,42 @@ if ($format->{'extension'} eq 'html' && !defined $params->param('limit')) { $vars->{'default_limited'} = 1; } -# Generate the basic SQL query that will be used to generate the bug list. -my $search = new Bugzilla::Search('fields' => \@selectcolumns, - 'params' => scalar $params->Vars, - 'order' => \@order_columns, - 'sharer' => $sharer_id); +my $fallback_search = Bugzilla::Search->new(fields => [@selectcolumns], + params => scalar $params->Vars, + order => [@order_columns], + sharer => $sharer_id); + +my $search; +my $elastic = $cgi->param('elastic') // 1; +if (defined $cgi->param('elastic')) { + $vars->{was_elastic} = 1; +} +if ($elastic) { + local $SIG{__DIE__} = undef; + local $SIG{__WARN__} = undef; + my $ok = eval { + my @args = ( params => scalar $params->Vars ); + if ($searchstring) { + @args = (quicksearch => $searchstring); + } + if (defined $params->param('limit')) { + push @args, limit => scalar $params->param('limit'); + } + $search = Bugzilla::Elastic::Search->new( + fields => [@selectcolumns], + order => [@order_columns], + @args, + ); + $search->es_query; + 1; + }; + if (!$ok) { + warn "fallback from elasticsearch: $@\n"; + $search = $fallback_search; + } +} else { + $search = $fallback_search; +} $order = join(',', $search->order); @@ -735,25 +766,44 @@ $::SIG{TERM} = 'DEFAULT'; $::SIG{PIPE} = 'DEFAULT'; # Execute the query. -my ($data, $extra_data) = $search->data; -$vars->{'search_description'} = $search->search_description; +my ($data, $extra_data); +do { + local $SIG{__DIE__} = undef; + local $SIG{__WARN__} = undef; + ($data, $extra_data) = eval { $search->data }; +}; + +if ($elastic && not defined $data) { + warn "fallback from elasticsearch: $@\n"; + $search = $fallback_search; + ($data, $extra_data) = $search->data; + $elastic = 0; +} + +$fulltext = 1 if $elastic; +$vars->{'search_description'} = $search->search_description; if ($cgi->param('debug') && Bugzilla->params->{debug_group} && $user->in_group(Bugzilla->params->{debug_group}) ) { $vars->{'debug'} = 1; - $vars->{'queries'} = $extra_data; - my $query_time = 0; - $query_time += $_->{'time'} foreach @$extra_data; - $vars->{'query_time'} = $query_time; - # Explains are limited to admins because you could use them to figure - # out how many hidden bugs are in a particular product (by doing - # searches and looking at the number of rows the explain says it's - # examining). - if ($user->in_group('admin')) { - foreach my $query (@$extra_data) { - $query->{explain} = $dbh->bz_explain($query->{sql}); + if ($search->isa('Bugzilla::Elastic::Search')) { + $vars->{query_time} = $search->query_time; + } + else { + $vars->{'queries'} = $extra_data; + my $query_time = 0; + $query_time += $_->{'time'} foreach @$extra_data; + $vars->{'query_time'} = $query_time; + # Explains are limited to admins because you could use them to figure + # out how many hidden bugs are in a particular product (by doing + # searches and looking at the number of rows the explain says it's + # examining). + if ($user->in_group('admin')) { + foreach my $query (@$extra_data) { + $query->{explain} = $dbh->bz_explain($query->{sql}); + } } } } @@ -885,6 +935,15 @@ else { # remaining_time <= 0 # Define the variables and functions that will be passed to the UI template. +if ($vars->{elastic} = $search->isa('Bugzilla::Elastic::Search')) { + $vars->{elastic_query_time} = $search->query_time; +} +else { + my $query_time = 0; + $query_time += $_->{'time'} foreach @$extra_data; + $vars->{'query_time'} = $query_time; +} + $vars->{'bugs'} = \@bugs; $vars->{'buglist'} = \@bugidlist; $vars->{'buglist_joined'} = join(',', @bugidlist); diff --git a/js/field.js b/js/field.js index 349e2fae3..b514fa53c 100644 --- a/js/field.js +++ b/js/field.js @@ -713,15 +713,13 @@ $(function() { } var options_user = { - serviceUrl: 'rest/user', + serviceUrl: 'rest/elastic/suggest_users', params: { Bugzilla_api_token: BUGZILLA.api_token, - include_fields: 'name,real_name', - limit: 100 }, paramName: 'match', deferRequestBy: 250, - minChars: 3, + minChars: 2, tabDisabled: true, autoSelectFirst: true, triggerSelectOnValidInput: false, diff --git a/scripts/search.pl b/scripts/search.pl new file mode 100644 index 000000000..6e0f7245d --- /dev/null +++ b/scripts/search.pl @@ -0,0 +1,13 @@ +#!/usr/bin/perl +use strict; +use warnings; +use Bugzilla; +use JSON '-convert_blessed_universally'; + +print JSON->new->pretty->encode( + Bugzilla::Elastic::Search->new( + quicksearch => "@ARGV", + fields => ['bug_id', 'short_desc'], + order => ['bug_id'], + )->es_query +); diff --git a/scripts/suggest-user.pl b/scripts/suggest-user.pl new file mode 100644 index 000000000..dcf24da87 --- /dev/null +++ b/scripts/suggest-user.pl @@ -0,0 +1,20 @@ +#!/usr/bin/perl +use strict; +use warnings; +use FindBin qw($RealBin); +use lib ($RealBin); +use Bugzilla; +use Search::Elasticsearch; +use Bugzilla::Elastic; + +my $elastic = Bugzilla::Elastic->new( + es_client => Search::Elasticsearch->new() +); +my $user = Bugzilla::User->check({name => 'dylan@mozilla.com'}); +Bugzilla->set_user($user); +my $users; + +for (1..4) { + $users = $elastic->suggest_users($ARGV[0]); +} +print "$_->{name}\n" for @$users; diff --git a/template/en/default/list/list.html.tmpl b/template/en/default/list/list.html.tmpl index 5e154f5df..051382a21 100644 --- a/template/en/default/list/list.html.tmpl +++ b/template/en/default/list/list.html.tmpl @@ -317,5 +317,13 @@ [% ELSE %] [% bugs.size %] [%+ terms.bugs %] found. [% END %] + [% IF elastic %] +
+ ElasticSearch took [% elastic_query_time FILTER html %] seconds. + Try without ElasticSearch + [% ELSIF was_elastic %] +
+ Search took [% query_time FILTER html %] seconds. + [% END %] [% END %] -- cgit v1.2.3-24-g4f1b