diff options
author | Dylan William Hardison <dylan@hardison.net> | 2018-07-08 06:59:55 +0200 |
---|---|---|
committer | Dylan William Hardison <dylan@hardison.net> | 2018-07-08 06:59:55 +0200 |
commit | be1f92450788dc89280c9e04a4bf983b5d7fac54 (patch) | |
tree | b5513fd846597fe5f152177dbbda88dca08fdf5f /Bugzilla | |
parent | 9bbd8d368598046c47964ee043620621b6c3634b (diff) | |
parent | 446a08b30b0dbaac9f2b88e0a5cad410f0446140 (diff) | |
download | bugzilla-be1f92450788dc89280c9e04a4bf983b5d7fac54.tar.gz bugzilla-be1f92450788dc89280c9e04a4bf983b5d7fac54.tar.xz |
Merge remote-tracking branch 'bmo/master'
Diffstat (limited to 'Bugzilla')
46 files changed, 621 insertions, 1414 deletions
diff --git a/Bugzilla/Attachment.pm b/Bugzilla/Attachment.pm index 0bdb50c9a..9eac3a147 100644 --- a/Bugzilla/Attachment.pm +++ b/Bugzilla/Attachment.pm @@ -297,10 +297,8 @@ sub is_viewable { # We assume we can view all text and image types. return 1 if ($contenttype =~ /^(text|image)\//); - # Mozilla can view XUL. Note the trailing slash on the Gecko detection to - # avoid sending XUL to Safari. - return 1 if (($contenttype =~ /^application\/vnd\.mozilla\./) - && ($cgi->user_agent() =~ /Gecko\//)); + # Modern browsers support PDF as well. + return 1 if ($contenttype eq 'application/pdf'); # If it's not one of the above types, we check the Accept: header for any # types mentioned explicitly. diff --git a/Bugzilla/Bloomfilter.pm b/Bugzilla/Bloomfilter.pm index 0d329b2ea..ba1d6d6c3 100644 --- a/Bugzilla/Bloomfilter.pm +++ b/Bugzilla/Bloomfilter.pm @@ -13,7 +13,8 @@ use warnings; use Bugzilla::Constants; use Algorithm::BloomFilter; -use File::Temp qw(tempfile); +use File::Slurper qw(write_binary read_binary read_lines); +use File::Spec::Functions qw(catfile); sub _new_bloom_filter { my ($n) = @_; @@ -24,44 +25,37 @@ sub _new_bloom_filter { } sub _filename { - my ($name) = @_; + my ($name, $type) = @_; my $datadir = bz_locations->{datadir}; - return sprintf("%s/%s.bloom", $datadir, $name); + + return catfile($datadir, "$name.$type"); } sub populate { - my ($class, $name, $items) = @_; + my ($class, $name) = @_; my $memcached = Bugzilla->memcached; + my @items = read_lines(_filename($name, 'list')); + my $filter = _new_bloom_filter(@items + 0); - my $filter = _new_bloom_filter(@$items + 0); - foreach my $item (@$items) { - $filter->add($item); - } - - my ($fh, $filename) = tempfile( "${name}XXXXXX", DIR => bz_locations->{datadir}, UNLINK => 0); - binmode $fh, ':bytes'; - print $fh $filter->serialize; - close $fh; - rename($filename, _filename($name)) or die "failed to rename $filename: $!"; + $filter->add($_) foreach @items; + write_binary(_filename($name, 'bloom'), $filter->serialize); $memcached->clear_bloomfilter({name => $name}); } sub lookup { my ($class, $name) = @_; my $memcached = Bugzilla->memcached; - my $filename = _filename($name); + my $filename = _filename($name, 'bloom'); my $filter_data = $memcached->get_bloomfilter( { name => $name } ); if (!$filter_data && -f $filename) { - open my $fh, '<:bytes', $filename; - local $/ = undef; - $filter_data = <$fh>; - close $fh; + $filter_data = read_binary($filename); $memcached->set_bloomfilter({ name => $name, filter => $filter_data }); } - return Algorithm::BloomFilter->deserialize($filter_data); + return Algorithm::BloomFilter->deserialize($filter_data) if $filter_data; + return undef; } 1; diff --git a/Bugzilla/BugMail.pm b/Bugzilla/BugMail.pm index 915405a0e..ebfc95d51 100644 --- a/Bugzilla/BugMail.pm +++ b/Bugzilla/BugMail.pm @@ -277,7 +277,7 @@ sub Send { # BMO: never send emails to bugs or .tld addresses. this check needs to # happen after the bugmail_recipients hook. if ($user->email_enabled && $dep_ok && - ($user->login !~ /bugs$/) && ($user->login !~ /\.tld$/)) + ($user->login !~ /\.(?:bugs|tld)$/)) { # Don't show summaries for bugs the user can't access, and # provide a hook for extensions such as SecureMail to filter diff --git a/Bugzilla/CGI.pm b/Bugzilla/CGI.pm index b932116a2..a69bdd278 100644 --- a/Bugzilla/CGI.pm +++ b/Bugzilla/CGI.pm @@ -42,6 +42,11 @@ sub DEFAULT_CSP { img_src => [ 'self', 'https://secure.gravatar.com', 'https://www.google-analytics.com' ], style_src => [ 'self', 'unsafe-inline' ], object_src => [ 'none' ], + connect_src => [ + 'self', + # This is from extensions/OrangeFactor/web/js/orange_factor.js + 'https://treeherder.mozilla.org/api/failurecount/', + ], form_action => [ 'self', # used in template/en/default/search/search-google.html.tmpl @@ -69,7 +74,7 @@ sub SHOW_BUG_MODAL_CSP { connect_src => [ 'self', # This is from extensions/OrangeFactor/web/js/orange_factor.js - 'https://brasstacks.mozilla.com/orangefactor/api/count', + 'https://treeherder.mozilla.org/api/failurecount/', ], frame_src => [ 'self', ], worker_src => [ 'none', ], @@ -590,6 +595,9 @@ sub header { "skins/standard/fonts/MaterialIcons-Regular.woff2", ); $headers{'-link'} = join(", ", map { sprintf('</static/v%s/%s>; rel="preload"; as="font"', Bugzilla->VERSION, $_) } @fonts); + if (Bugzilla->params->{google_analytics_tracking_id}) { + $headers{'-link'} .= ', <https://www.google-analytics.com>; rel="preconnect"; crossorigin'; + } } return $self->SUPER::header(%headers) || ""; @@ -684,6 +692,8 @@ sub send_cookie { $paramhash{'-secure'} = 1 if lc( $uri->scheme ) eq 'https'; + $paramhash{'-samesite'} = 'Lax'; + push(@{$self->{'Bugzilla_cookie_list'}}, $self->cookie(%paramhash)); } diff --git a/Bugzilla/Config/Advanced.pm b/Bugzilla/Config/Advanced.pm index 2eec11dbe..398f02701 100644 --- a/Bugzilla/Config/Advanced.pm +++ b/Bugzilla/Config/Advanced.pm @@ -36,43 +36,6 @@ use constant get_param_list => ( type => 'b', default => 0 }, - - { - name => 'sentry_uri', - type => 't', - default => '', - }, - - { - name => 'metrics_enabled', - type => 'b', - default => 0 - }, - { - name => 'metrics_user_ids', - type => 't', - default => '3881,5038,5898,13647,20209,251051,373476,409787' - }, - { - name => 'metrics_elasticsearch_server', - type => 't', - default => '127.0.0.1:9200' - }, - { - name => 'metrics_elasticsearch_index', - type => 't', - default => 'bmo-metrics' - }, - { - name => 'metrics_elasticsearch_type', - type => 't', - default => 'timings' - }, - { - name => 'metrics_elasticsearch_ttl', - type => 't', - default => '1210000000' # 14 days - }, ); 1; diff --git a/Bugzilla/Constants.pm b/Bugzilla/Constants.pm index 65b37dced..0b0c857a0 100644 --- a/Bugzilla/Constants.pm +++ b/Bugzilla/Constants.pm @@ -695,9 +695,8 @@ sub _bz_locations { # The script should really generate these graphs directly... 'webdotdir' => "$datadir/webdot", 'extensionsdir' => "$libpath/extensions", + 'logsdir' => "$libpath/logs", 'assetsdir' => "$datadir/assets", - # error_reports store error/warnings destined for sentry - 'error_reports' => "$libpath/error_reports", 'confdir' => $confdir, }; } diff --git a/Bugzilla/DB.pm b/Bugzilla/DB.pm index 15acfd0d9..ec0f058b9 100644 --- a/Bugzilla/DB.pm +++ b/Bugzilla/DB.pm @@ -8,13 +8,22 @@ package Bugzilla::DB; use 5.10.1; -use strict; -use warnings; +use Moo; use DBI; - -# Inherit the DB class from DBI::db. -use base qw(DBI::db); +use DBIx::Connector; +our %Connector; + +has 'dbh' => ( + is => 'lazy', + handles => [ + qw[ + begin_work column_info commit disconnect do errstr get_info last_insert_id ping prepare + primary_key quote_identifier rollback selectall_arrayref selectall_hashref + selectcol_arrayref selectrow_array selectrow_arrayref selectrow_hashref table_info + ] + ], +); use Bugzilla::Constants; use Bugzilla::Install::Requirements; @@ -27,11 +36,14 @@ use Bugzilla::Error; use Bugzilla::DB::Schema; use Bugzilla::Version; -use Bugzilla::Metrics::Mysql; - use List::Util qw(max); use Storable qw(dclone); +has [qw(dsn user pass attrs)] => ( + is => 'ro', + required => 1, +); + ##################################################################### # Constants ##################################################################### @@ -89,7 +101,7 @@ use constant INDEX_DROPS_REQUIRE_FK_DROPS => 1; sub quote { my $self = shift; - my $retval = $self->SUPER::quote(@_); + my $retval = $self->dbh->quote(@_); trick_taint($retval) if defined $retval; return $retval; } @@ -138,11 +150,6 @@ sub _connect { # instantiate the correct DB specific module - # BMO - enable instrumentation of db calls - if (Bugzilla->metrics_enabled) { - $pkg_module = 'Bugzilla::Metrics::Mysql'; - } - my $dbh = $pkg_module->new($params); return $dbh; @@ -206,7 +213,6 @@ sub bz_check_server_version { my ($self, $db, $output) = @_; my $sql_vers = $self->bz_server_version; - $self->disconnect; my $sql_want = $db->{db_version}; my $version_ok = vers_cmp($sql_vers, $sql_want) > -1 ? 1 : 0; @@ -262,7 +268,6 @@ sub bz_create_database { } } - $dbh->disconnect; } # A helper for bz_create_database and bz_check_requirements. @@ -271,6 +276,7 @@ sub _get_no_db_connection { my $dbh; my %connect_params = %{ Bugzilla->localconfig }; $connect_params{db_name} = ''; + local %Connector = (); my $conn_success = eval { $dbh = _connect(\%connect_params); }; @@ -1247,14 +1253,14 @@ sub bz_rollback_transaction { # Subclass Helpers ##################################################################### -sub db_new { - my ($class, $params) = @_; +sub _build_dbh { + my ($self) = @_; my ($dsn, $user, $pass, $override_attrs) = - @$params{qw(dsn user pass attrs)}; + map { $self->$_ } qw(dsn user pass attrs); # set up default attributes used to connect to the database # (may be overridden by DB driver implementations) - my $attributes = { RaiseError => 0, + my $attributes = { RaiseError => 1, AutoCommit => 1, PrintError => 0, ShowErrorStatement => 1, @@ -1272,20 +1278,16 @@ sub db_new { $attributes->{$key} = $override_attrs->{$key}; } } + my $class = ref $self; + if ($class->can('on_dbi_connected')) { + $attributes->{Callbacks} = { + connected => sub { $class->on_dbi_connected(@_); return }, + } + } - # connect using our known info to the specified db - my $self = DBI->connect($dsn, $user, $pass, $attributes) - or die "\nCan't connect to the database.\nError: $DBI::errstr\n" - . " Is your database installed and up and running?\n Do you have" - . " the correct username and password selected in localconfig?\n\n"; - - # RaiseError was only set to 0 so that we could catch the - # above "die" condition. - $self->{RaiseError} = 1; - - bless ($self, $class); + my $connector = $Connector{"$user.$dsn"} //= DBIx::Connector->new($dsn, $user, $pass, $attributes); - return $self; + return $connector->dbh; } ##################################################################### diff --git a/Bugzilla/DB/Mysql.pm b/Bugzilla/DB/Mysql.pm index 727fe2316..d0b9724eb 100644 --- a/Bugzilla/DB/Mysql.pm +++ b/Bugzilla/DB/Mysql.pm @@ -22,10 +22,9 @@ For interface details see L<Bugzilla::DB> and L<DBI>. package Bugzilla::DB::Mysql; use 5.10.1; -use strict; -use warnings; +use Moo; -use base qw(Bugzilla::DB); +extends qw(Bugzilla::DB); use Bugzilla::Constants; use Bugzilla::Install::Util qw(install_string); @@ -43,7 +42,7 @@ use constant MAX_COMMENTS => 50; use constant FULLTEXT_OR => '|'; -sub new { +sub BUILDARGS { my ($class, $params) = @_; my ($user, $pass, $host, $dbname, $port, $sock) = @$params{qw(db_user db_pass db_host db_name db_port db_sock)}; @@ -55,28 +54,20 @@ sub new { my %attrs = ( mysql_enable_utf8 => Bugzilla->params->{'utf8'}, - # Needs to be explicitly specified for command-line processes. - mysql_auto_reconnect => 1, ); - my $self = $class->db_new({ dsn => $dsn, user => $user, - pass => $pass, attrs => \%attrs }); + return { dsn => $dsn, user => $user, pass => $pass, attrs => \%attrs }; +} + +sub on_dbi_connected { + my ($class, $dbh) = @_; # This makes sure that if the tables are encoded as UTF-8, we # return their data correctly. - $self->do("SET NAMES utf8") if Bugzilla->params->{'utf8'}; - - # all class local variables stored in DBI derived class needs to have - # a prefix 'private_'. See DBI documentation. - $self->{private_bz_tables_locked} = ""; - - # Needed by TheSchwartz - $self->{private_bz_dsn} = $dsn; - - bless ($self, $class); + $dbh->do("SET NAMES utf8") if Bugzilla->params->{'utf8'}; # Bug 321645 - disable MySQL strict mode, if set - my ($var, $sql_mode) = $self->selectrow_array( + my ($var, $sql_mode) = $dbh->selectrow_array( "SHOW VARIABLES LIKE 'sql\\_mode'"); if ($sql_mode) { @@ -88,15 +79,13 @@ sub new { split(/,/, $sql_mode)); if ($sql_mode ne $new_sql_mode) { - $self->do("SET SESSION sql_mode = ?", undef, $new_sql_mode); + $dbh->do("SET SESSION sql_mode = ?", undef, $new_sql_mode); } } # Allow large GROUP_CONCATs (largely for inserting comments # into bugs_fulltext). - $self->do('SET SESSION group_concat_max_len = 128000000'); - - return $self; + $dbh->do('SET SESSION group_concat_max_len = 128000000'); } # when last_insert_id() is supported on MySQL by lowest DBI/DBD version diff --git a/Bugzilla/DB/Oracle.pm b/Bugzilla/DB/Oracle.pm index 5c9ea68a3..a519bb796 100644 --- a/Bugzilla/DB/Oracle.pm +++ b/Bugzilla/DB/Oracle.pm @@ -22,10 +22,9 @@ For interface details see L<Bugzilla::DB> and L<DBI>. package Bugzilla::DB::Oracle; use 5.10.1; -use strict; -use warnings; +use Moo; -use base qw(Bugzilla::DB); +extends qw(Bugzilla::DB); use DBD::Oracle; use DBD::Oracle qw(:ora_types); @@ -47,7 +46,7 @@ use constant FULLTEXT_OR => ' OR '; our $fulltext_label = 0; -sub new { +sub BUILDARGS { my ($class, $params) = @_; my ($user, $pass, $host, $dbname, $port) = @$params{qw(db_user db_pass db_host db_name db_port)}; @@ -66,22 +65,20 @@ sub new { LongReadLen => max(Bugzilla->params->{'maxattachmentsize'} || 0, MIN_LONG_READ_LEN) * 1024, }; - my $self = $class->db_new({ dsn => $dsn, user => $user, - pass => $pass, attrs => $attrs }); - # Needed by TheSchwartz - $self->{private_bz_dsn} = $dsn; + return { dsn => $dsn, user => $user, pass => $pass, attrs => $attrs }; +} - bless ($self, $class); +sub on_dbi_connected { + my ($class, $dbh) = @_; # Set the session's default date format to match MySQL - $self->do("ALTER SESSION SET NLS_DATE_FORMAT='YYYY-MM-DD HH24:MI:SS'"); - $self->do("ALTER SESSION SET NLS_TIMESTAMP_FORMAT='YYYY-MM-DD HH24:MI:SS'"); - $self->do("ALTER SESSION SET NLS_LENGTH_SEMANTICS='CHAR'") + $dbh->do("ALTER SESSION SET NLS_DATE_FORMAT='YYYY-MM-DD HH24:MI:SS'"); + $dbh->do("ALTER SESSION SET NLS_TIMESTAMP_FORMAT='YYYY-MM-DD HH24:MI:SS'"); + $dbh->do("ALTER SESSION SET NLS_LENGTH_SEMANTICS='CHAR'") if Bugzilla->params->{'utf8'}; # To allow case insensitive query. - $self->do("ALTER SESSION SET NLS_COMP='ANSI'"); - $self->do("ALTER SESSION SET NLS_SORT='BINARY_AI'"); - return $self; + $dbh->do("ALTER SESSION SET NLS_COMP='ANSI'"); + $dbh->do("ALTER SESSION SET NLS_SORT='BINARY_AI'"); } sub bz_last_key { diff --git a/Bugzilla/DB/Pg.pm b/Bugzilla/DB/Pg.pm index 8bcbc3262..0db349412 100644 --- a/Bugzilla/DB/Pg.pm +++ b/Bugzilla/DB/Pg.pm @@ -22,19 +22,18 @@ For interface details see L<Bugzilla::DB> and L<DBI>. package Bugzilla::DB::Pg; use 5.10.1; -use strict; -use warnings; +use Moo; use Bugzilla::Error; use Bugzilla::Version; use DBD::Pg; # This module extends the DB interface via inheritance -use base qw(Bugzilla::DB); +extends qw(Bugzilla::DB); use constant BLOB_TYPE => { pg_type => DBD::Pg::PG_BYTEA }; -sub new { +sub BUILDARGS { my ($class, $params) = @_; my ($user, $pass, $host, $dbname, $port) = @$params{qw(db_user db_pass db_host db_name db_port)}; @@ -55,18 +54,7 @@ sub new { my $attrs = { pg_enable_utf8 => Bugzilla->params->{'utf8'} }; - my $self = $class->db_new({ dsn => $dsn, user => $user, - pass => $pass, attrs => $attrs }); - - # all class local variables stored in DBI derived class needs to have - # a prefix 'private_'. See DBI documentation. - $self->{private_bz_tables_locked} = ""; - # Needed by TheSchwartz - $self->{private_bz_dsn} = $dsn; - - bless ($self, $class); - - return $self; + return { dsn => $dsn, user => $user, pass => $pass, attrs => $attrs } } # if last_insert_id is supported on PostgreSQL by lowest DBI/DBD version diff --git a/Bugzilla/DB/Schema.pm b/Bugzilla/DB/Schema.pm index 3307464db..67ee9071c 100644 --- a/Bugzilla/DB/Schema.pm +++ b/Bugzilla/DB/Schema.pm @@ -398,7 +398,8 @@ use constant ABSTRACT_SCHEMA => { DEFAULT => 'FALSE'}, type => {TYPE => 'INT2', NOTNULL => 1, DEFAULT => '0'}, - extra_data => {TYPE => 'varchar(255)'} + extra_data => {TYPE => 'varchar(255)'}, + is_markdown => {TYPE => 'BOOLEAN', NOTNULL => 1, DEFAULT => 'FALSE'} ], INDEXES => [ longdescs_bug_id_idx => ['bug_id'], @@ -923,6 +924,8 @@ use constant ABSTRACT_SCHEMA => { cryptpassword => {TYPE => 'varchar(128)'}, realname => {TYPE => 'varchar(255)', NOTNULL => 1, DEFAULT => "''"}, + nickname => {TYPE => 'varchar(255)', NOTNULL => 1, + DEFAULT => "''"}, disabledtext => {TYPE => 'MEDIUMTEXT', NOTNULL => 1, DEFAULT => "''"}, disable_mail => {TYPE => 'BOOLEAN', NOTNULL => 1, @@ -942,7 +945,10 @@ use constant ABSTRACT_SCHEMA => { profiles_login_name_idx => {FIELDS => ['login_name'], TYPE => 'UNIQUE'}, profiles_extern_id_idx => {FIELDS => ['extern_id'], - TYPE => 'UNIQUE'} + TYPE => 'UNIQUE'}, + profiles_nickname_idx => ['nickname'], + profiles_realname_ft_idx => {FIELDS => ['realname'], + TYPE => 'FULLTEXT'}, ], }, diff --git a/Bugzilla/DB/Sqlite.pm b/Bugzilla/DB/Sqlite.pm index 87f45415b..3890d0795 100644 --- a/Bugzilla/DB/Sqlite.pm +++ b/Bugzilla/DB/Sqlite.pm @@ -8,10 +8,9 @@ package Bugzilla::DB::Sqlite; use 5.10.1; -use strict; -use warnings; +use Moo; -use base qw(Bugzilla::DB); +extends qw(Bugzilla::DB); use Bugzilla::Constants; use Bugzilla::Error; @@ -69,7 +68,7 @@ sub _sqlite_position_ci { # Constructor # ############### -sub new { +sub BUILDARGS { my ($class, $params) = @_; my $db_name = $params->{db_name}; @@ -97,10 +96,11 @@ sub new { sqlite_unicode => Bugzilla->params->{'utf8'}, }; - my $self = $class->db_new({ dsn => $dsn, user => '', - pass => '', attrs => $attrs }); - # Needed by TheSchwartz - $self->{private_bz_dsn} = $dsn; + return { dsn => $dsn, user => '', pass => '', attrs => $attrs }; +} + +sub on_dbi_connected { + my ($class, $dbh) = @_; my %pragmas = ( # Make sure that the sqlite file doesn't grow without bound. @@ -122,23 +122,20 @@ sub new { ); while (my ($name, $value) = each %pragmas) { - $self->do("PRAGMA $name = $value"); + $dbh->do("PRAGMA $name = $value"); } - $self->sqlite_create_collation('bugzilla', \&_sqlite_collate_ci); - $self->sqlite_create_function('position', 2, \&_sqlite_position); - $self->sqlite_create_function('iposition', 2, \&_sqlite_position_ci); + $dbh->sqlite_create_collation('bugzilla', \&_sqlite_collate_ci); + $dbh->sqlite_create_function('position', 2, \&_sqlite_position); + $dbh->sqlite_create_function('iposition', 2, \&_sqlite_position_ci); # SQLite has a "substr" function, but other DBs call it "SUBSTRING" # so that's what we use, and I don't know of any way in SQLite to # alias the SQL "substr" function to be called "SUBSTRING". - $self->sqlite_create_function('substring', 3, \&CORE::substr); - $self->sqlite_create_function('mod', 2, \&_sqlite_mod); - $self->sqlite_create_function('now', 0, \&_sqlite_now); - $self->sqlite_create_function('localtimestamp', 1, \&_sqlite_now); - $self->sqlite_create_function('floor', 1, \&POSIX::floor); - - bless ($self, $class); - return $self; + $dbh->sqlite_create_function('substring', 3, \&CORE::substr); + $dbh->sqlite_create_function('mod', 2, \&_sqlite_mod); + $dbh->sqlite_create_function('now', 0, \&_sqlite_now); + $dbh->sqlite_create_function('localtimestamp', 1, \&_sqlite_now); + $dbh->sqlite_create_function('floor', 1, \&POSIX::floor); } ############### diff --git a/Bugzilla/DaemonControl.pm b/Bugzilla/DaemonControl.pm index 6586cc01b..2c6df1b87 100644 --- a/Bugzilla/DaemonControl.pm +++ b/Bugzilla/DaemonControl.pm @@ -81,7 +81,13 @@ sub run_cereal { on_exception => on_exception( 'cereal', $exit_f ), ); $exit_f->on_cancel( sub { $cereal->kill('TERM') } ); + $exit_f->on_ready( + sub { + delete $ENV{LOG4PERL_STDERR_DISABLE}; + } + ); $loop->add($cereal); + $ENV{LOG4PERL_STDERR_DISABLE} = 1; return $exit_f; } diff --git a/Bugzilla/Error.pm b/Bugzilla/Error.pm index d67571848..9fcd16386 100644 --- a/Bugzilla/Error.pm +++ b/Bugzilla/Error.pm @@ -13,9 +13,10 @@ use warnings; use base qw(Exporter); -@Bugzilla::Error::EXPORT = qw(ThrowCodeError ThrowTemplateError ThrowUserError ThrowErrorPage); +## no critic (Modules::ProhibitAutomaticExportation) +our @EXPORT = qw( ThrowCodeError ThrowTemplateError ThrowUserError ThrowErrorPage); +## use critic -use Bugzilla::Sentry; use Bugzilla::Constants; use Bugzilla::WebService::Constants; use Bugzilla::Util; @@ -37,7 +38,7 @@ sub _in_eval { } sub _throw_error { - my ($name, $error, $vars) = @_; + my ($name, $error, $vars, $logfunc) = @_; $vars ||= {}; $vars->{error} = $error; @@ -47,38 +48,6 @@ sub _throw_error { my $dbh = eval { Bugzilla->dbh }; $dbh->bz_rollback_transaction() if ($dbh && $dbh->bz_in_transaction() && !_in_eval()); - my $datadir = bz_locations()->{'datadir'}; - # If a writable $datadir/errorlog exists, log error details there. - if (-w "$datadir/errorlog") { - require Data::Dumper; - my $mesg = ""; - for (1..75) { $mesg .= "-"; }; - $mesg .= "\n[$$] " . time2str("%D %H:%M:%S ", time()); - $mesg .= "$name $error "; - $mesg .= remote_ip(); - $mesg .= Bugzilla->user->login; - $mesg .= (' actually ' . Bugzilla->sudoer->login) if Bugzilla->sudoer; - $mesg .= "\n"; - my %params = Bugzilla->cgi->Vars; - $Data::Dumper::Useqq = 1; - for my $param (sort keys %params) { - my $val = $params{$param}; - # obscure passwords - $val = "*****" if $param =~ /password/i; - # limit line length - $val =~ s/^(.{512}).*$/$1\[CHOP\]/; - $mesg .= "[$$] " . Data::Dumper->Dump([$val],["param($param)"]); - } - for my $var (sort keys %ENV) { - my $val = $ENV{$var}; - $val = "*****" if $val =~ /password|http_pass/i; - $mesg .= "[$$] " . Data::Dumper->Dump([$val],["env($var)"]); - } - open(ERRORLOGFID, ">>", "$datadir/errorlog"); - print ERRORLOGFID "$mesg\n"; - close ERRORLOGFID; - } - my $template = Bugzilla->template; my $message; @@ -97,34 +66,24 @@ sub _throw_error { message => \$message }); if ($Bugzilla::Template::is_processing) { - $name =~ /^global\/(user|code)-error/; - my $type = $1 // 'unknown'; + my ($type) = $name =~ /^global\/(user|code)-error/; + $type //= 'unknown'; die Template::Exception->new("bugzilla.$type.$error", $vars); } if (Bugzilla->error_mode == ERROR_MODE_WEBPAGE) { - if (sentry_should_notify($vars->{error})) { - $vars->{maintainers_notified} = 1; - $vars->{processed} = {}; - } else { - $vars->{maintainers_notified} = 0; - } - my $cgi = Bugzilla->cgi; $cgi->close_standby_message('text/html', 'inline', 'error', 'html'); $template->process($name, $vars) || ThrowTemplateError($template->error()); print $cgi->multipart_final() if $cgi->{_multipart_in_progress}; - - if ($vars->{maintainers_notified}) { - sentry_handle_error($vars->{error}, $vars->{processed}->{error_message}); - } + $logfunc->("webpage error: $error"); } elsif (Bugzilla->error_mode == ERROR_MODE_TEST) { die Dumper($vars); } elsif (Bugzilla->error_mode == ERROR_MODE_DIE) { - die("$message\n"); + die "$message\n"; } elsif (Bugzilla->error_mode == ERROR_MODE_DIE_SOAP_FAULT || Bugzilla->error_mode == ERROR_MODE_JSON_RPC @@ -141,6 +100,7 @@ sub _throw_error { } if (Bugzilla->error_mode == ERROR_MODE_DIE_SOAP_FAULT) { + $logfunc->("XMLRPC error: $error ($code)"); die SOAP::Fault->faultcode($code)->faultstring($message); } else { @@ -150,6 +110,11 @@ sub _throw_error { if (Bugzilla->error_mode == ERROR_MODE_REST) { my %status_code_map = %{ REST_STATUS_CODE_MAP() }; $status_code = $status_code_map{$code} || $status_code_map{'_default'}; + $logfunc->("REST error: $error (HTTP $status_code, internal code $code)"); + } + else { + my $fake_code = 100000 + $code; + $logfunc->("JSONRPC error: $error ($fake_code)"); } # Technically JSON-RPC isn't allowed to have error numbers # higher than 999, but we do this to avoid conflicts with @@ -170,22 +135,44 @@ sub _throw_error { exit; } + +sub _add_vars_to_logging_fields { + my ($vars) = @_; + + foreach my $key (keys %$vars) { + Bugzilla::Logging->fields->{"var_$key"} = $vars->{$key}; + } +} + +sub _make_logfunc { + my ($type) = @_; + my $logger = Log::Log4perl->get_logger("Bugzilla.Error.$type"); + return sub { + local $Log::Log4perl::caller_depth = $Log::Log4perl::caller_depth + 3; + if ($type eq 'User') { + $logger->warn(@_); + } + else { + $logger->error(@_); + } + }; +} + + sub ThrowUserError { - _throw_error("global/user-error.html.tmpl", @_); + my ($error, $vars) = @_; + my $logfunc = _make_logfunc('User'); + _add_vars_to_logging_fields($vars); + + _throw_error( 'global/user-error.html.tmpl', $error, $vars, $logfunc); } sub ThrowCodeError { - my (undef, $vars) = @_; - - # Don't show function arguments, in case they contain - # confidential data. - local $Carp::MaxArgNums = -1; - # Don't show the error as coming from Bugzilla::Error, show it - # as coming from the caller. - local $Carp::CarpInternal{'Bugzilla::Error'} = 1; - $vars->{traceback} //= Carp::longmess(); + my ($error, $vars) = @_; + my $logfunc = _make_logfunc('Code'); + _add_vars_to_logging_fields($vars); - _throw_error("global/code-error.html.tmpl", @_); + _throw_error( 'global/code-error.html.tmpl', $error, $vars, $logfunc ); } sub ThrowTemplateError { @@ -211,10 +198,12 @@ sub ThrowTemplateError { # we never want to display this to the user exit if $template_err =~ /\bModPerl::Util::exit\b/; + state $logger = Log::Log4perl->get_logger('Bugzilla.Error.Template'); + $logger->error($template_err); + $vars->{'template_error_msg'} = $template_err; $vars->{'error'} = "template_error"; - sentry_handle_error('error', $template_err); $vars->{'template_error_msg'} =~ s/ at \S+ line \d+\.\s*$//; my $template = Bugzilla->template; diff --git a/Bugzilla/Extension.pm b/Bugzilla/Extension.pm index a41ac9326..8e173c711 100644 --- a/Bugzilla/Extension.pm +++ b/Bugzilla/Extension.pm @@ -35,7 +35,11 @@ sub INC_HOOK { my $first = 1; untaint($real_file); $INC{$fake_file} = $real_file; - open my $fh, '<', $real_file or die "invalid file: $real_file"; + my $found = open my $fh, '<', $real_file; + unless ($found) { + require Carp; + Carp::croak "Can't locate $fake_file while looking for $real_file in \@INC (\@INC contains: @INC)"; + } return sub { no warnings; if ( !$first ) { diff --git a/Bugzilla/Group.pm b/Bugzilla/Group.pm index 06491f6a0..7f684ea15 100644 --- a/Bugzilla/Group.pm +++ b/Bugzilla/Group.pm @@ -74,6 +74,11 @@ use constant UPDATE_COLUMNS => qw( use constant GROUP_PARAMS => qw(chartgroup insidergroup timetrackinggroup querysharegroup); + +sub DYNAMIC_COLUMNS { + return Bugzilla->usage_mode == USAGE_MODE_CMDLINE; +} + ############################### #### Accessors ###### ############################### diff --git a/Bugzilla/Install.pm b/Bugzilla/Install.pm index 6d47a143f..14fe904eb 100644 --- a/Bugzilla/Install.pm +++ b/Bugzilla/Install.pm @@ -215,6 +215,10 @@ use constant SYSTEM_GROUPS => ( description => 'Can edit or disable users' }, { + name => 'disableusers', + description => 'Can disable users' + }, + { name => 'creategroups', description => 'Can create and destroy groups' }, diff --git a/Bugzilla/Install/DB.pm b/Bugzilla/Install/DB.pm index 0d63f68b9..86edf8a30 100644 --- a/Bugzilla/Install/DB.pm +++ b/Bugzilla/Install/DB.pm @@ -710,6 +710,11 @@ sub update_table_definitions { # 2014-07-27 LpSolit@gmail.com - Bug 1044561 _fix_user_api_keys_indexes(); + + # 2018-06-14 dylan@mozilla.com - Bug 1468818 + $dbh->bz_add_column('longdescs', 'is_markdown', + {TYPE => 'BOOLEAN', NOTNULL => 1, DEFAULT => 'FALSE'}); + # 2014-10-?? dkl@mozilla.com - Bug 1062940 $dbh->bz_alter_column('bugs', 'alias', { TYPE => 'varchar(40)' }); @@ -762,6 +767,13 @@ sub update_table_definitions { $dbh->bz_add_column('components', 'triage_owner_id', {TYPE => 'INT3'}); + $dbh->bz_add_column('profiles', 'nickname', + {TYPE => 'varchar(255)', NOTNULL => 1, DEFAULT => "''"}); + $dbh->bz_add_index('profiles', 'profiles_nickname_idx', [qw(nickname)]); + + $dbh->bz_add_index('profiles', 'profiles_realname_ft_idx', + {TYPE => 'FULLTEXT', FIELDS => ['realname']}); + ################################################################ # New --TABLE-- changes should go *** A B O V E *** this point # ################################################################ diff --git a/Bugzilla/Install/Filesystem.pm b/Bugzilla/Install/Filesystem.pm index 493479d5e..d56c7f4c4 100644 --- a/Bugzilla/Install/Filesystem.pm +++ b/Bugzilla/Install/Filesystem.pm @@ -36,6 +36,7 @@ use Cwd (); use File::Slurp; use IO::File; use POSIX (); +use English qw(-no_match_vars $OSNAME); use base qw(Exporter); our @EXPORT = qw( @@ -106,6 +107,7 @@ use constant HTTPD_ENV => qw( LOCALCONFIG_ENV BUGZILLA_UNSAFE_AUTH_DELEGATION LOG4PERL_CONFIG_FILE + LOG4PERL_STDERR_DISABLE USE_NYTPROF NYTPROF_DIR ); @@ -206,6 +208,8 @@ sub DIR_CGI_OVERWRITE { _group() ? 0770 : 0777 }; # (or their subdirectories) to the user, via the webserver. sub DIR_ALSO_WS_SERVE { _suexec() ? 0001 : 0 }; +sub DIR_ALSO_WS_STICKY { $OSNAME eq 'linux' ? 02000 : 0 } + # This looks like a constant because it effectively is, but # it has to call other subroutines and read the current filesystem, # so it's defined as a sub. This is not exported, so it doesn't have @@ -230,7 +234,7 @@ sub FILESYSTEM { my $template_cache = bz_locations()->{'template_cache'}; my $graphsdir = bz_locations()->{'graphsdir'}; my $assetsdir = bz_locations()->{'assetsdir'}; - my $error_reports = bz_locations()->{'error_reports'}; + my $logsdir = bz_locations()->{'logsdir'}; # We want to set the permissions the same for all localconfig files # across all PROJECTs, so we do something special with $localconfig, @@ -265,8 +269,6 @@ sub FILESYSTEM { 'runtests.pl' => { perms => OWNER_EXECUTE }, 'jobqueue.pl' => { perms => OWNER_EXECUTE }, 'migrate.pl' => { perms => OWNER_EXECUTE }, - 'sentry.pl' => { perms => WS_EXECUTE }, - 'metrics.pl' => { perms => WS_EXECUTE }, 'Makefile.PL' => { perms => OWNER_EXECUTE }, 'gen-cpanfile.pl' => { perms => OWNER_EXECUTE }, 'jobqueue-worker.pl' => { perms => OWNER_EXECUTE }, @@ -315,8 +317,6 @@ sub FILESYSTEM { # Writeable directories $template_cache => { files => CGI_READ, dirs => DIR_CGI_OVERWRITE }, - $error_reports => { files => CGI_READ, - dirs => DIR_CGI_WRITE }, $attachdir => { files => CGI_WRITE, dirs => DIR_CGI_WRITE }, $webdotdir => { files => WS_SERVE, @@ -325,6 +325,8 @@ sub FILESYSTEM { dirs => DIR_CGI_WRITE | DIR_ALSO_WS_SERVE }, "$datadir/db" => { files => CGI_WRITE, dirs => DIR_CGI_WRITE }, + $logsdir => { files => CGI_WRITE, + dirs => DIR_CGI_WRITE | DIR_ALSO_WS_STICKY }, $assetsdir => { files => WS_SERVE, dirs => DIR_CGI_OVERWRITE | DIR_ALSO_WS_SERVE }, @@ -409,7 +411,7 @@ sub FILESYSTEM { $webdotdir => DIR_CGI_WRITE | DIR_ALSO_WS_SERVE, $assetsdir => DIR_CGI_WRITE | DIR_ALSO_WS_SERVE, $template_cache => DIR_CGI_WRITE, - $error_reports => DIR_CGI_WRITE, + $logsdir => DIR_CGI_WRITE | DIR_ALSO_WS_STICKY, # Directories that contain content served directly by the web server. "$skinsdir/custom" => DIR_WS_SERVE, "$skinsdir/contrib" => DIR_WS_SERVE, @@ -544,8 +546,6 @@ sub FILESYSTEM { contents => HT_DEFAULT_DENY }, "$datadir/.htaccess" => { perms => WS_SERVE, contents => HT_DEFAULT_DENY }, - "$error_reports/.htaccess" => { perms => WS_SERVE, - contents => HT_DEFAULT_DENY }, "$graphsdir/.htaccess" => { perms => WS_SERVE, contents => HT_GRAPHS_DIR }, "$webdotdir/.htaccess" => { perms => WS_SERVE, diff --git a/Bugzilla/Install/Localconfig.pm b/Bugzilla/Install/Localconfig.pm index 85092f23a..39063ee63 100644 --- a/Bugzilla/Install/Localconfig.pm +++ b/Bugzilla/Install/Localconfig.pm @@ -126,6 +126,10 @@ use constant LOCALCONFIG_VARS => ( default => sub { dirname( bin_loc('diff') ) }, }, { + name => 'tct_bin', + default => sub { bin_loc('tct') }, + }, + { name => 'site_wide_secret', # 64 characters is roughly the equivalent of a 384-bit key, which @@ -211,6 +215,7 @@ sub _read_localconfig_from_env { else { my $default = $var->{default}; $localconfig{$name} = ref($default) eq 'CODE' ? $default->() : $default; + untaint($localconfig{$name}); } } diff --git a/Bugzilla/JobQueue.pm b/Bugzilla/JobQueue.pm index afb36673f..a78a4d0ae 100644 --- a/Bugzilla/JobQueue.pm +++ b/Bugzilla/JobQueue.pm @@ -59,7 +59,7 @@ sub new { # to write to it. my $self = $class->SUPER::new( databases => [{ - dsn => Bugzilla->dbh_main->{private_bz_dsn}, + dsn => Bugzilla->dbh_main->dsn, user => $lc->{db_user}, pass => $lc->{db_pass}, prefix => 'ts_', diff --git a/Bugzilla/Logging.pm b/Bugzilla/Logging.pm index 769485c86..f334435fc 100644 --- a/Bugzilla/Logging.pm +++ b/Bugzilla/Logging.pm @@ -10,21 +10,30 @@ use 5.10.1; use strict; use warnings; -use Log::Log4perl; +use Log::Log4perl qw(:easy); use Log::Log4perl::MDC; -use File::Spec::Functions qw(rel2abs); +use File::Spec::Functions qw(rel2abs catfile); use Bugzilla::Constants qw(bz_locations); use English qw(-no_match_vars $PROGRAM_NAME); +use Taint::Util qw(untaint); -sub is_interactive { - return not exists $ENV{SERVER_SOFTWARE} +sub logfile { + my ($class, $name) = @_; + + my $file = rel2abs(catfile(bz_locations->{logsdir}, $name)); + untaint($file); + return $file; +} + +sub fields { + return Log::Log4perl::MDC->get_context->{fields} //= {}; } BEGIN { my $file = $ENV{LOG4PERL_CONFIG_FILE} // 'log4perl-syslog.conf'; Log::Log4perl::Logger::create_custom_level('NOTICE', 'WARN', 5, 2); Log::Log4perl->init(rel2abs($file, bz_locations->{confdir})); - Log::Log4perl->get_logger(__PACKAGE__)->trace("logging enabled in $PROGRAM_NAME"); + TRACE("logging enabled in $PROGRAM_NAME"); } # this is copied from Log::Log4perl's :easy handling, diff --git a/Bugzilla/Mailer.pm b/Bugzilla/Mailer.pm index 1dec3d4ff..c9a458b47 100644 --- a/Bugzilla/Mailer.pm +++ b/Bugzilla/Mailer.pm @@ -12,8 +12,9 @@ use strict; use warnings; use base qw(Exporter); -@Bugzilla::Mailer::EXPORT = qw(MessageToMTA build_thread_marker); +our @EXPORT = qw(MessageToMTA build_thread_marker); ## no critic (Modules::ProhibitAutomaticExportation) +use Bugzilla::Logging; use Bugzilla::Constants; use Bugzilla::Error; use Bugzilla::Hook; @@ -25,6 +26,8 @@ use Encode qw(encode); use Encode::MIME::Header; use Email::Address; use Email::MIME; +use Try::Tiny; + # Return::Value 1.666002 pollutes the error log with warnings about this # deprecated module. We have to set NO_CLUCK = 1 before loading Email::Send # to disable these warnings. @@ -62,7 +65,7 @@ sub MessageToMTA { $msg =~ s/(?:\015+)?\012/\015\012/msg; } - $email = new Email::MIME($msg); + $email = Email::MIME->new($msg); } # Ensure that we are not sending emails too quickly to recipients. @@ -182,6 +185,25 @@ sub MessageToMTA { Bugzilla::Hook::process('mailer_before_send', { email => $email, mailer_args => \@args }); + try { + my $to = $email->header('to') or die qq{Unable to find "To:" address\n}; + my @recipients = Email::Address->parse($to); + die qq{Unable to parse "To:" address - $to\n} unless @recipients; + die qq{Did not expect more than one "To:" address in $to\n} if @recipients > 1; + my $recipient = $recipients[0]; + my $badhosts = Bugzilla::Bloomfilter->lookup("badhosts"); + if ($badhosts && $badhosts->test($recipient->host)) { + WARN("Attempted to send email to address in badhosts: $to"); + $email->header_set(to => ''); + } + elsif ($recipient->host =~ /\.(?:bugs|tld)$/) { + WARN("Attempted to send email to fake address: $to"); + $email->header_set(to => ''); + } + } catch { + ERROR($_); + }; + # Allow for extensions to to drop the bugmail by clearing the 'to' header return if $email->header('to') eq ''; diff --git a/Bugzilla/Markdown/GFM.pm b/Bugzilla/Markdown/GFM.pm new file mode 100644 index 000000000..f3f24fc6a --- /dev/null +++ b/Bugzilla/Markdown/GFM.pm @@ -0,0 +1,92 @@ +package Bugzilla::Markdown::GFM; + +use 5.10.1; +use strict; +use warnings; + +use Alien::libcmark_gfm; +use FFI::Platypus; +use FFI::Platypus::Buffer qw( scalar_to_buffer buffer_to_scalar ); +use Exporter qw(import); + +use Bugzilla::Markdown::GFM::SyntaxExtension; +use Bugzilla::Markdown::GFM::SyntaxExtensionList; +use Bugzilla::Markdown::GFM::Parser; +use Bugzilla::Markdown::GFM::Node; + +our @EXPORT_OK = qw(cmark_markdown_to_html); + +my %OPTIONS = ( + default => 0, + sourcepos => ( 1 << 1 ), + hardbreaks => ( 1 << 2 ), + safe => ( 1 << 3 ), + nobreaks => ( 1 << 4 ), + normalize => ( 1 << 8 ), + validate_utf8 => ( 1 << 9 ), + smart => ( 1 << 10 ), + github_pre_lang => ( 1 << 11 ), + liberal_html_tag => ( 1 << 12 ), + footnotes => ( 1 << 13 ), + strikethrough_double_tilde => ( 1 << 14 ), + table_prefer_style_attributes => ( 1 << 15 ), +); + +my $FFI = FFI::Platypus->new( + lib => [grep { not -l $_ } Alien::libcmark_gfm->dynamic_libs], +); + +$FFI->custom_type( + markdown_options_t => { + native_type => 'int', + native_to_perl => sub { + my ($options) = @_; + my $result = {}; + foreach my $key (keys %OPTIONS) { + $result->{$key} = ($options & $OPTIONS{$key}) != 0; + } + return $result; + }, + perl_to_native => sub { + my ($options) = @_; + my $result = 0; + foreach my $key (keys %OPTIONS) { + if ($options->{$key}) { + $result |= $OPTIONS{$key}; + } + } + return $result; + } + } +); + +$FFI->attach(cmark_markdown_to_html => ['opaque', 'int', 'markdown_options_t'] => 'string', + sub { + my $c_func = shift; + my($markdown, $markdown_length) = scalar_to_buffer $_[0]; + return $c_func->($markdown, $markdown_length, $_[1]); + } +); + +# This has to happen after something from the main lib is loaded +$FFI->attach('core_extensions_ensure_registered' => [] => 'void'); + +core_extensions_ensure_registered(); + +Bugzilla::Markdown::GFM::SyntaxExtension->SETUP($FFI); +Bugzilla::Markdown::GFM::SyntaxExtensionList->SETUP($FFI); +Bugzilla::Markdown::GFM::Node->SETUP($FFI); +Bugzilla::Markdown::GFM::Parser->SETUP($FFI); + +1; + +__END__ + +=head1 NAME + +Bugzilla::Markdown::GFM - Sets up the FFI to libcmark_gfm. + +=head1 DESCRIPTION + +This modules mainly just does setup work. See L<Bugzilla::Markdown::GFM::Parser> +to actually render markdown to html. diff --git a/Bugzilla/Markdown/GFM/Node.pm b/Bugzilla/Markdown/GFM/Node.pm new file mode 100644 index 000000000..da5af1a68 --- /dev/null +++ b/Bugzilla/Markdown/GFM/Node.pm @@ -0,0 +1,33 @@ +package Bugzilla::Markdown::GFM::Node; + +use 5.10.1; +use strict; +use warnings; + +sub SETUP { + my ($class, $FFI) = @_; + + $FFI->custom_type( + markdown_node_t => { + native_type => 'opaque', + native_to_perl => sub { + bless \$_[0], $class if $_[0]; + }, + perl_to_native => sub { ${ $_[0] } }, + } + ); + + $FFI->attach( + [ cmark_node_free => 'DESTROY' ], + [ 'markdown_node_t' ] => 'void' + ); + + $FFI->attach( + [ cmark_render_html => 'render_html' ], + [ 'markdown_node_t', 'markdown_options_t', 'markdown_syntax_extension_list_t'] => 'string', + ); +} + +1; + +__END__ diff --git a/Bugzilla/Markdown/GFM/Parser.pm b/Bugzilla/Markdown/GFM/Parser.pm new file mode 100644 index 000000000..5307b49c1 --- /dev/null +++ b/Bugzilla/Markdown/GFM/Parser.pm @@ -0,0 +1,109 @@ +package Bugzilla::Markdown::GFM::Parser; + +use 5.10.1; +use strict; +use warnings; + +use FFI::Platypus::Buffer qw( scalar_to_buffer buffer_to_scalar ); + +sub new { + my ($class, $options) = @_; + my $extensions = delete $options->{extensions} // []; + my $parser = $class->_new($options); + $parser->{_options} = $options; + + eval { + foreach my $name (@$extensions) { + my $extension = Bugzilla::Markdown::GFM::SyntaxExtension->find($name) + or die "unknown extension: $name"; + $parser->attach_syntax_extension($extension); + } + }; + + return $parser; +} + +sub render_html { + my ($self, $markdown) = @_; + $self->feed($markdown); + my $node = $self->finish; + return $node->render_html($self->{_options}, $self->get_syntax_extensions); +} + +sub SETUP { + my ($class, $FFI) = @_; + + $FFI->custom_type( + markdown_parser_t => { + native_type => 'opaque', + native_to_perl => sub { + bless { _pointer => $_[0] }, $class; + }, + perl_to_native => sub { $_[0]->{_pointer} }, + } + ); + + $FFI->attach( + [ cmark_parser_new => '_new' ], + [ 'markdown_options_t' ] => 'markdown_parser_t', + sub { + my $c_func = shift; + return $c_func->($_[1]); + } + ); + + $FFI->attach( + [ cmark_parser_free => 'DESTROY' ], + [ 'markdown_parser_t' ] => 'void' + ); + + $FFI->attach( + [ cmark_parser_feed => 'feed'], + ['markdown_parser_t', 'opaque', 'int'] => 'void', + sub { + my $c_func = shift; + $c_func->($_[0], scalar_to_buffer $_[1]); + } + ); + + $FFI->attach( + [ cmark_parser_finish => 'finish' ], + [ 'markdown_parser_t' ] => 'markdown_node_t', + ); + + $FFI->attach( + [ cmark_parser_attach_syntax_extension => 'attach_syntax_extension' ], + [ 'markdown_parser_t', 'markdown_syntax_extension_t' ] => 'void', + ); + + $FFI->attach( + [ cmark_parser_get_syntax_extensions => 'get_syntax_extensions' ], + [ 'markdown_parser_t' ] => 'markdown_syntax_extension_list_t', + ); +} + +1; + +__END__ + +=head1 NAME + +Bugzilla::Markdown::GFM::Parser - Transforms markdown into HTML via libcmark_gfm. + +=head1 SYNOPSIS + + use Bugzilla::Markdown::GFM; + use Bugzilla::Markdown::GFM::Parser; + + my $parser = Bugzilla::Markdown::GFM::Parser->new({ + extensions => [qw( autolink tagfilter table strikethrough )] + }); + + say $parser->render_html(<<'MARKDOWN'); + # My header + + This is **markdown**! + + - list item 1 + - list item 2 + MARKDOWN diff --git a/Bugzilla/Markdown/GFM/SyntaxExtension.pm b/Bugzilla/Markdown/GFM/SyntaxExtension.pm new file mode 100644 index 000000000..56efa177a --- /dev/null +++ b/Bugzilla/Markdown/GFM/SyntaxExtension.pm @@ -0,0 +1,31 @@ +package Bugzilla::Markdown::GFM::SyntaxExtension; + +use 5.10.1; +use strict; +use warnings; + +sub SETUP { + my ($class, $FFI) = @_; + + $FFI->custom_type( + markdown_syntax_extension_t => { + native_type => 'opaque', + native_to_perl => sub { + bless \$_[0], $class if $_[0]; + }, + perl_to_native => sub { $_[0] ? ${ $_[0] } : 0 }, + } + ); + $FFI->attach( + [ cmark_find_syntax_extension => 'find' ], + [ 'string' ] => 'markdown_syntax_extension_t', + sub { + my $c_func = shift; + return $c_func->($_[1]); + } + ); +} + +1; + +__END__ diff --git a/Bugzilla/Markdown/GFM/SyntaxExtensionList.pm b/Bugzilla/Markdown/GFM/SyntaxExtensionList.pm new file mode 100644 index 000000000..06a9798c2 --- /dev/null +++ b/Bugzilla/Markdown/GFM/SyntaxExtensionList.pm @@ -0,0 +1,23 @@ +package Bugzilla::Markdown::GFM::SyntaxExtensionList; + +use 5.10.1; +use strict; +use warnings; + +sub SETUP { + my ($class, $FFI) = @_; + + $FFI->custom_type( + markdown_syntax_extension_list_t => { + native_type => 'opaque', + native_to_perl => sub { + bless \$_[0], $class if $_[0]; + }, + perl_to_native => sub { $_[0] ? ${ $_[0] } : 0 }, + } + ); +} + +1; + +__END__ diff --git a/Bugzilla/Memcached.pm b/Bugzilla/Memcached.pm index d34aaa595..40755aa29 100644 --- a/Bugzilla/Memcached.pm +++ b/Bugzilla/Memcached.pm @@ -227,11 +227,12 @@ sub should_rate_limit { my $prefix = RATE_LIMIT_PREFIX . $name . ':'; my $memcached = $self->{memcached}; + return 0 unless $name; return 0 unless $memcached; - $tries //= 3; + $tries //= 4; - for (0 .. $tries) { + for my $try (1 .. $tries) { my $now = time; my ($key, @keys) = map { $prefix . ( $now - $_ ) } 0 .. $rate_seconds; $memcached->add($key, 0, $rate_seconds+1); @@ -240,8 +241,9 @@ sub should_rate_limit { $tokens->{$key} = $cas->[1]++; return 1 if sum(values %$tokens) >= $rate_max; return 0 if $memcached->cas($key, @$cas, $rate_seconds+1); + WARN("retry for $prefix (try $try of $tries)"); } - return 1; + return 0; } sub clear_all { diff --git a/Bugzilla/Metrics/Collector.pm b/Bugzilla/Metrics/Collector.pm deleted file mode 100644 index 53cbac819..000000000 --- a/Bugzilla/Metrics/Collector.pm +++ /dev/null @@ -1,171 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# This Source Code Form is "Incompatible With Secondary Licenses", as -# defined by the Mozilla Public License, v. 2.0. - -package Bugzilla::Metrics::Collector; - -use strict; -use warnings; -use 5.10.1; - -# the reporter needs to be a constant and use'd here to ensure it's loaded at -# compile time. -use constant REPORTER => 'Bugzilla::Metrics::Reporter::ElasticSearch'; -use Bugzilla::Metrics::Reporter::ElasticSearch; - -# Debugging reporter -#use constant REPORTER => 'Bugzilla::Metrics::Reporter::STDERR'; -#use Bugzilla::Metrics::Reporter::STDERR; - -use Bugzilla::Constants; -use Cwd qw(abs_path); -use File::Basename; -use Time::HiRes qw(gettimeofday clock_gettime CLOCK_MONOTONIC); - -sub new { - my ($class, $name) = @_; - my $self = { - root => undef, - head => undef, - time => scalar(gettimeofday()), - }; - bless($self, $class); - $self->_start_timer({ type => 'main', name => $name }); - return $self; -} - -sub end { - my ($self, $timer) = @_; - my $is_head = $timer ? 0 : 1; - $timer ||= $self->{head}; - $timer->{duration} += clock_gettime(CLOCK_MONOTONIC) - $timer->{start_time}; - $self->{head} = $self->{head}->{parent} if $is_head; -} - -sub cancel { - my ($self) = @_; - delete $self->{head}; -} - -sub DESTROY { - my ($self) = @_; - $self->finish() if $self->{head}; -} - -sub finish { - my ($self) = @_; - $self->end($self->{root}); - delete $self->{head}; - - my $user = Bugzilla->user; - if ($ENV{MOD_PERL}) { - require Apache2::RequestUtil; - my $request = eval { Apache2::RequestUtil->request }; - my $headers = $request ? $request->headers_in() : {}; - $self->{env} = { - referer => $headers->{Referer}, - request_method => $request->method, - request_uri => basename($request->unparsed_uri), - script_name => $request->uri, - user_agent => $headers->{'User-Agent'}, - }; - } - else { - $self->{env} = { - referer => $ENV{HTTP_REFERER}, - request_method => $ENV{REQUEST_METHOD}, - request_uri => $ENV{REQUEST_URI}, - script_name => basename($ENV{SCRIPT_NAME}), - user_agent => $ENV{HTTP_USER_AGENT}, - }; - } - $self->{env}->{name} = $self->{root}->{name}; - $self->{env}->{time} = $self->{time}; - $self->{env}->{user_id} = $user->id; - $self->{env}->{login} = $user->login if $user->id; - - # remove passwords from request_uri - $self->{env}->{request_uri} =~ s/\b((?:bugzilla_)?password=)(?:[^&]+|.+$)/$1x/gi; - - $self->report(); -} - -sub name { - my ($self, $value) = @_; - $self->{root}->{name} = $value if defined $value; - return $self->{root}->{name}; -} - -sub db_start { - my ($self) = @_; - my $timer = $self->_start_timer({ type => 'db' }); - - my @stack; - my $i = 0; - state $path = quotemeta(abs_path(bz_locations()->{cgi_path}) . '/'); - while (1) { - my @caller = caller($i); - last unless @caller; - my $file = $caller[1]; - $file =~ s/^$path//o; - push @stack, "$file:$caller[2]" - unless substr($file, 0, 16) eq 'Bugzilla/Metrics'; - last if $file =~ /\.(?:tmpl|cgi)$/; - $i++; - } - $timer->{stack} = \@stack; - - return $timer; -} - -sub template_start { - my ($self, $file) = @_; - $self->_start_timer({ type => 'tmpl', file => $file }); -} - -sub memcached_start { - my ($self, $key) = @_; - $self->_start_timer({ type => 'memcached', key => $key }); -} - -sub memcached_end { - my ($self, $hit) = @_; - $self->{head}->{result} = $hit ? 'hit' : 'miss'; - $self->end(); -} - -sub resume { - my ($self, $timer) = @_; - $timer->{start_time} = clock_gettime(CLOCK_MONOTONIC); - return $timer; -} - -sub _start_timer { - my ($self, $timer) = @_; - $timer->{start_time} = $timer->{first_time} = clock_gettime(CLOCK_MONOTONIC); - $timer->{duration} = 0; - $timer->{children} = []; - - if ($self->{head}) { - $timer->{parent} = $self->{head}; - push @{ $self->{head}->{children} }, $timer; - } - else { - $timer->{parent} = undef; - $self->{root} = $timer; - } - $self->{head} = $timer; - - return $timer; -} - -sub report { - my ($self) = @_; - my $class = REPORTER; - $class->DETACH ? $class->background($self) : $class->foreground($self); -} - -1; diff --git a/Bugzilla/Metrics/Memcached.pm b/Bugzilla/Metrics/Memcached.pm deleted file mode 100644 index b640f9280..000000000 --- a/Bugzilla/Metrics/Memcached.pm +++ /dev/null @@ -1,24 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# This Source Code Form is "Incompatible With Secondary Licenses", as -# defined by the Mozilla Public License, v. 2.0. - -package Bugzilla::Metrics::Memcached; - -use 5.10.1; -use strict; -use warnings; - -use base 'Bugzilla::Memcached'; - -sub _get { - my $self = shift; - Bugzilla->metrics->memcached_start($_[0]); - my $result = $self->SUPER::_get(@_); - Bugzilla->metrics->memcached_end($result); - return $result; -} - -1; diff --git a/Bugzilla/Metrics/Mysql.pm b/Bugzilla/Metrics/Mysql.pm deleted file mode 100644 index 60dff78c8..000000000 --- a/Bugzilla/Metrics/Mysql.pm +++ /dev/null @@ -1,148 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# This Source Code Form is "Incompatible With Secondary Licenses", as -# defined by the Mozilla Public License, v. 2.0. - -package Bugzilla::Metrics::Mysql; - -use 5.10.1; -use strict; -use warnings; - -use base 'Bugzilla::DB::Mysql'; - -sub do { - my ($self, @args) = @_; - Bugzilla->metrics->db_start($args[0]); - my $result = $self->SUPER::do(@args); - Bugzilla->metrics->end(); - return $result; -} - -sub selectall_arrayref { - my ($self, @args) = @_; - Bugzilla->metrics->db_start($args[0]); - my $result = $self->SUPER::selectall_arrayref(@args); - Bugzilla->metrics->end(); - return $result; -} - -sub selectall_hashref { - my ($self, @args) = @_; - Bugzilla->metrics->db_start($args[0]); - my $result = $self->SUPER::selectall_hashref(@args); - Bugzilla->metrics->end(); - return $result; -} - -sub selectcol_arrayref { - my ($self, @args) = @_; - Bugzilla->metrics->db_start($args[0]); - my $result = $self->SUPER::selectcol_arrayref(@args); - Bugzilla->metrics->end(); - return $result; -} - -sub selectrow_array { - my ($self, @args) = @_; - Bugzilla->metrics->db_start($args[0]); - my @result = $self->SUPER::selectrow_array(@args); - Bugzilla->metrics->end(); - return wantarray ? @result : $result[0]; -} - -sub selectrow_arrayref { - my ($self, @args) = @_; - Bugzilla->metrics->db_start($args[0]); - my $result = $self->SUPER::selectrow_arrayref(@args); - Bugzilla->metrics->end(); - return $result; -} - -sub selectrow_hashref { - my ($self, @args) = @_; - Bugzilla->metrics->db_start($args[0]); - my $result = $self->SUPER::selectrow_hashref(@args); - Bugzilla->metrics->end(); - return $result; -} - -sub commit { - my ($self, @args) = @_; - Bugzilla->metrics->db_start('COMMIT'); - my $result = $self->SUPER::commit(@args); - Bugzilla->metrics->end(); - return $result; -} - -sub prepare { - my ($self, @args) = @_; - my $sth = $self->SUPER::prepare(@args); - bless($sth, 'Bugzilla::Metrics::st'); - return $sth; -} - -package Bugzilla::Metrics::st; - -use 5.10.1; -use strict; -use warnings; - -use base 'DBI::st'; - -sub execute { - my ($self, @args) = @_; - $self->{private_timer} = Bugzilla->metrics->db_start(); - my $result = $self->SUPER::execute(@args); - Bugzilla->metrics->end(); - return $result; -} - -sub fetchrow_array { - my ($self, @args) = @_; - my $timer = $self->{private_timer}; - Bugzilla->metrics->resume($timer); - my @result = $self->SUPER::fetchrow_array(@args); - Bugzilla->metrics->end($timer); - return wantarray ? @result : $result[0]; -} - -sub fetchrow_arrayref { - my ($self, @args) = @_; - my $timer = $self->{private_timer}; - Bugzilla->metrics->resume($timer); - my $result = $self->SUPER::fetchrow_arrayref(@args); - Bugzilla->metrics->end($timer); - return $result; -} - -sub fetchrow_hashref { - my ($self, @args) = @_; - my $timer = $self->{private_timer}; - Bugzilla->metrics->resume($timer); - my $result = $self->SUPER::fetchrow_hashref(@args); - Bugzilla->metrics->end($timer); - return $result; -} - -sub fetchall_arrayref { - my ($self, @args) = @_; - my $timer = $self->{private_timer}; - Bugzilla->metrics->resume($timer); - my $result = $self->SUPER::fetchall_arrayref(@args); - Bugzilla->metrics->end($timer); - return $result; -} - -sub fetchall_hashref { - my ($self, @args) = @_; - my $timer = $self->{private_timer}; - Bugzilla->metrics->resume($timer); - my $result = $self->SUPER::fetchall_hashref(@args); - Bugzilla->metrics->end($timer); - return $result; -} - -1; diff --git a/Bugzilla/Metrics/Reporter.pm b/Bugzilla/Metrics/Reporter.pm deleted file mode 100644 index 8216ea923..000000000 --- a/Bugzilla/Metrics/Reporter.pm +++ /dev/null @@ -1,103 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# This Source Code Form is "Incompatible With Secondary Licenses", as -# defined by the Mozilla Public License, v. 2.0. - -package Bugzilla::Metrics::Reporter; - -use 5.10.1; -use strict; -use warnings; - -use Bugzilla::Constants; -use File::Slurp; -use File::Temp; -use JSON; - -# most reporters should detach from the httpd process. -# reporters which do not detach will block completion of the http response. -use constant DETACH => 1; - -# class method to start the delivery script in the background -sub background { - my ($class, $collector) = @_; - - # we need to remove parent links to avoid looped structures, which - # encode_json chokes on - _walk_timers($collector->{root}, sub { delete $_[0]->{parent} }); - - # serialisation - my $json = encode_json({ env => $collector->{env}, times => $collector->{root} }); - - # write to temp filename - my $fh = File::Temp->new( UNLINK => 0 ); - if (!$fh) { - warn "Failed to create temp file: $!\n"; - return; - } - binmode($fh, ':utf8'); - print $fh $json; - close($fh) or die "$fh : $!"; - my $filename = $fh->filename; - - # spawn delivery worker - my $command = bz_locations()->{'cgi_path'} . "/metrics.pl '$class' '$filename' &"; - $ENV{PATH} = ''; - system($command); -} - -# run the reporter immediately -sub foreground { - my ($class, $collector) = @_; - my $reporter = $class->new({ hashref => { env => $collector->{env}, times => $collector->{root} } }); - $reporter->report(); -} - -sub new { - my ($invocant, $args) = @_; - my $class = ref($invocant) || $invocant; - - # load from either a json_filename or hashref - my $self; - if ($args->{json_filename}) { - $self = decode_json(read_file($args->{json_filename}, binmode => ':utf8')); - unlink($args->{json_filename}); - } - else { - $self = $args->{hashref}; - } - bless($self, $class); - - # remove redundant data - $self->walk_timers(sub { - my ($timer) = @_; - $timer->{start_time} = delete $timer->{first_time}; - delete $timer->{children} - if exists $timer->{children} && !scalar(@{ $timer->{children} }); - }); - - return $self; -} - -sub walk_timers { - my ($self, $callback) = @_; - _walk_timers($self->{times}, $callback, undef); -} - -sub _walk_timers { - my ($timer, $callback, $parent) = @_; - $callback->($timer, $parent); - if (exists $timer->{children}) { - foreach my $child (@{ $timer->{children} }) { - _walk_timers($child, $callback, $timer); - } - } -} - -sub report { - die "abstract method call"; -} - -1; diff --git a/Bugzilla/Metrics/Reporter/ElasticSearch.pm b/Bugzilla/Metrics/Reporter/ElasticSearch.pm deleted file mode 100644 index c61a1d750..000000000 --- a/Bugzilla/Metrics/Reporter/ElasticSearch.pm +++ /dev/null @@ -1,105 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# This Source Code Form is "Incompatible With Secondary Licenses", as -# defined by the Mozilla Public License, v. 2.0. - -package Bugzilla::Metrics::Reporter::ElasticSearch; - -use 5.10.1; -use strict; -use warnings; - -use base 'Bugzilla::Metrics::Reporter'; - -use constant DETACH => 1; - -sub report { - my ($self) = @_; - - # build path array and flatten - my @timers; - $self->walk_timers(sub { - my ($timer, $parent) = @_; - $timer->{id} = scalar(@timers); - if ($parent) { - if (exists $timer->{children}) { - if ($timer->{type} eq 'tmpl') { - $timer->{node} = 'tmpl: ' . $timer->{file}; - } - elsif ($timer->{type} eq 'db') { - $timer->{node} = 'db'; - } - else { - $timer->{node} = '?'; - } - } - $timer->{path} = [ @{ $parent->{path} }, $parent->{node} ]; - $timer->{parent} = $parent->{id}; - } - else { - $timer->{path} = [ ]; - $timer->{node} = $timer->{name}; - } - push @timers, $timer; - }); - - # calculate timer-only durations - $self->walk_timers(sub { - my ($timer) = @_; - my $child_duration = 0; - if (exists $timer->{children}) { - foreach my $child (@{ $timer->{children} }) { - $child_duration += $child->{duration}; - } - } - $timer->{this_duration} = $timer->{duration} - $child_duration; - }); - - # massage each timer - my $start_time = $self->{times}->{start_time}; - foreach my $timer (@timers) { - # remove node name and children - delete $timer->{node}; - delete $timer->{children}; - - # show relative times - $timer->{start_time} = $timer->{start_time} - $start_time; - delete $timer->{end_time}; - - # show times in ms instead of fractional seconds - foreach my $field (qw( start_time duration this_duration )) { - $timer->{$field} = sprintf('%.4f', $timer->{$field} * 1000) * 1; - } - } - - # remove private data from env - delete $self->{env}->{user_agent}; - delete $self->{env}->{referer}; - - # throw at ES - require ElasticSearch; - my $es = ElasticSearch->new( - servers => Bugzilla->params->{metrics_elasticsearch_server}, - transport => 'http', - ); - # the ElasticSearch module queries the server for a list of nodes and - # connects directly to a random node. that bypasses our load balancer so we - # disable that by setting the server list directly. - $es->transport->servers(Bugzilla->params->{metrics_elasticsearch_server}); - # as the discovered node list is lazy-loaded, increase _refresh_in so it - # won't call ->refresh_servers() - $es->transport->{_refresh_in} = 1; - $es->index( - index => Bugzilla->params->{metrics_elasticsearch_index}, - type => Bugzilla->params->{metrics_elasticsearch_type}, - ttl => Bugzilla->params->{metrics_elasticsearch_ttl}, - data => { - env => $self->{env}, - times => \@timers, - }, - ); -} - -1; diff --git a/Bugzilla/Metrics/Reporter/STDERR.pm b/Bugzilla/Metrics/Reporter/STDERR.pm deleted file mode 100644 index a61cdb25f..000000000 --- a/Bugzilla/Metrics/Reporter/STDERR.pm +++ /dev/null @@ -1,152 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# This Source Code Form is "Incompatible With Secondary Licenses", as -# defined by the Mozilla Public License, v. 2.0. - -package Bugzilla::Metrics::Reporter::STDERR; - -use 5.10.1; -use strict; -use warnings; - -use base 'Bugzilla::Metrics::Reporter'; - -use Data::Dumper; - -use constant DETACH => 0; - -sub report { - my ($self) = @_; - - # count totals - $self->{total} = $self->{times}->{duration}; - $self->{tmpl_count} = $self->{db_count} = $self->{mem_count} = 0; - $self->{total_tmpl} = $self->{total_db} = $self->{mem_hits} = 0; - $self->{mem_keys} = {}; - $self->_tally($self->{times}); - - # calculate percentages - $self->{other} = $self->{total} - $self->{total_tmpl} - $self->{total_db}; - if ($self->{total} * 1) { - $self->{perc_tmpl} = $self->{total_tmpl} / $self->{total} * 100; - $self->{perc_db} = $self->{total_db} / $self->{total} * 100; - $self->{perc_other} = $self->{other} / $self->{total} * 100; - } else { - $self->{perc_tmpl} = 0; - $self->{perc_db} = 0; - $self->{perc_other} = 0; - } - if ($self->{mem_count}) { - $self->{perc_mem} = $self->{mem_hits} / $self->{mem_count} * 100; - } else { - $self->{perm_mem} = 0; - } - - # convert to ms and format - foreach my $key (qw( total total_tmpl total_db other )) { - $self->{$key} = sprintf("%.4f", $self->{$key} * 1000); - } - foreach my $key (qw( perc_tmpl perc_db perc_other perc_mem )) { - $self->{$key} = sprintf("%.1f", $self->{$key}); - } - - # massage each timer - my $start_time = $self->{times}->{start_time}; - $self->walk_timers(sub { - my ($timer) = @_; - delete $timer->{parent}; - - # show relative times - $timer->{start_time} = $timer->{start_time} - $start_time; - delete $timer->{end_time}; - - # show times in ms instead of fractional seconds - foreach my $field (qw( start_time duration duration_this )) { - $timer->{$field} = sprintf('%.4f', $timer->{$field} * 1000) * 1 - if exists $timer->{$field}; - } - }); - - if (0) { - # dump timers to stderr - local $Data::Dumper::Indent = 1; - local $Data::Dumper::Terse = 1; - local $Data::Dumper::Sortkeys = sub { - my ($rh) = @_; - return [ sort { $b cmp $a } keys %$rh ]; - }; - print STDERR Dumper($self->{env}); - print STDERR Dumper($self->{times}); - } - - # summary summary table too - print STDERR <<EOF; -total time: $self->{total} - tmpl time: $self->{total_tmpl} ($self->{perc_tmpl}%) $self->{tmpl_count} hits - db time: $self->{total_db} ($self->{perc_db}%) $self->{db_count} hits -other time: $self->{other} ($self->{perc_other}%) - memcached: $self->{perc_mem}% ($self->{mem_count} requests) -EOF - my $tmpls = $self->{tmpl}; - my $len = 0; - foreach my $file (keys %$tmpls) { - $len = length($file) if length($file) > $len; - } - foreach my $file (sort { $tmpls->{$b}->{count} <=> $tmpls->{$a}->{count} } keys %$tmpls) { - my $tmpl = $tmpls->{$file}; - printf STDERR - "%${len}s: %2s hits %8.4f total %8.4f avg\n", - $file, - $tmpl->{count}, - $tmpl->{duration} * 1000, - $tmpl->{duration} * 1000 / $tmpl->{count} - ; - } - my $keys = $self->{mem_keys}; - $len = 0; - foreach my $key (keys %$keys) { - $len = length($key) if length($key) > $len; - } - foreach my $key (sort { $keys->{$a} <=> $keys->{$b} or $a cmp $b } keys %$keys) { - printf STDERR "%${len}s: %s\n", $key, $keys->{$key}; - } -} - -sub _tally { - my ($self, $timer) = @_; - if (exists $timer->{children}) { - foreach my $child (@{ $timer->{children} }) { - $self->_tally($child); - } - } - - if ($timer->{type} eq 'db') { - $timer->{duration_this} = $timer->{duration}; - $self->{total_db} += $timer->{duration}; - $self->{db_count}++; - - } elsif ($timer->{type} eq 'tmpl') { - my $child_duration = 0; - if (exists $timer->{children}) { - foreach my $child (@{ $timer->{children} }) { - $child_duration += $child->{duration}; - } - } - $timer->{duration_this} = $timer->{duration} - $child_duration; - - $self->{total_tmpl} += $timer->{duration} - $child_duration; - $self->{tmpl_count}++; - $self->{tmpl}->{$timer->{file}}->{count}++; - $self->{tmpl}->{$timer->{file}}->{duration} += $timer->{duration}; - - } elsif ($timer->{type} eq 'memcached') { - $timer->{duration_this} = $timer->{duration}; - $self->{mem_count}++; - $self->{mem_keys}->{$timer->{key}}++; - $self->{mem_hits}++ if $timer->{result} eq 'hit'; - } -} - -1; diff --git a/Bugzilla/Metrics/Template.pm b/Bugzilla/Metrics/Template.pm deleted file mode 100644 index 5d9af240e..000000000 --- a/Bugzilla/Metrics/Template.pm +++ /dev/null @@ -1,24 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# This Source Code Form is "Incompatible With Secondary Licenses", as -# defined by the Mozilla Public License, v. 2.0. - -package Bugzilla::Metrics::Template; - -use 5.10.1; -use strict; -use warnings; - -use base 'Bugzilla::Template'; - -sub process { - my $self = shift; - Bugzilla->metrics->template_start($_[0]); - my $result = $self->SUPER::process(@_); - Bugzilla->metrics->end(); - return $result; -} - -1; diff --git a/Bugzilla/Metrics/Template/Context.pm b/Bugzilla/Metrics/Template/Context.pm deleted file mode 100644 index 278cfce1e..000000000 --- a/Bugzilla/Metrics/Template/Context.pm +++ /dev/null @@ -1,30 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# This Source Code Form is "Incompatible With Secondary Licenses", as -# defined by the Mozilla Public License, v. 2.0. - -package Bugzilla::Metrics::Template::Context; - -use 5.10.1; -use strict; -use warnings; - -use base 'Bugzilla::Template::Context'; - -sub process { - my $self = shift; - - # we only want to measure files not template blocks - if (ref($_[0]) || substr($_[0], -5) ne '.tmpl') { - return $self->SUPER::process(@_); - } - - Bugzilla->metrics->template_start($_[0]); - my $result = $self->SUPER::process(@_); - Bugzilla->metrics->end(); - return $result; -} - -1; diff --git a/Bugzilla/ModPerl.pm b/Bugzilla/ModPerl.pm index 120dd8210..19cd1128f 100644 --- a/Bugzilla/ModPerl.pm +++ b/Bugzilla/ModPerl.pm @@ -74,7 +74,7 @@ __DATA__ # every process, and Perl has another. (Various Perl modules still use # the built-in rand(), even though we never use it in Bugzilla itself, # so we need to srand() both of them.) -PerlChildInitHandler "sub { Bugzilla::RNG::srand(); srand(); }" +PerlChildInitHandler "sub { Bugzilla::RNG::srand(); srand(); eval { Bugzilla->dbh->ping } }" PerlInitHandler Bugzilla::ModPerl::Hostage PerlAccessHandler Bugzilla::ModPerl::BlockIP @@ -86,12 +86,6 @@ ErrorDocument 403 /errors/403.html ErrorDocument 404 /errors/404.html ErrorDocument 500 /errors/500.html -<Location /helper> - SetHandler perl-script - PerlResponseHandler Plack::Handler::Apache2 - PerlSetVar psgi_app [% cgi_path %]/helper.psgi -</Location> - <Directory "[% cgi_path %]"> AddHandler perl-script .cgi # No need to PerlModule these because they're already defined in mod_perl.pl diff --git a/Bugzilla/Object.pm b/Bugzilla/Object.pm index 00afbe19f..eaafca219 100644 --- a/Bugzilla/Object.pm +++ b/Bugzilla/Object.pm @@ -44,6 +44,9 @@ use constant USE_MEMCACHED => 1; # values, keywords, products, classifications, priorities, severities, etc. use constant IS_CONFIG => 0; +# When DYNAMIC_COLUMNS is true, _get_db_columns() will use the information schema. +use constant DYNAMIC_COLUMNS => 0; + # This allows the JSON-RPC interface to return Bugzilla::Object instances # as though they were hashes. In the future, this may be modified to return # less information. @@ -888,13 +891,19 @@ sub _get_db_columns { my $cache = Bugzilla->request_cache; my $cache_key = "object_${class}_db_columns"; return @{ $cache->{$cache_key} } if $cache->{$cache_key}; - # Currently you can only add new columns using object_columns, not - # remove or modify existing columns, because removing columns would - # almost certainly cause Bugzilla to function improperly. - my @add_columns; - Bugzilla::Hook::process('object_columns', - { class => $class, columns => \@add_columns }); - my @columns = ($invocant->DB_COLUMNS, @add_columns); + my @columns; + if ($class->DYNAMIC_COLUMNS) { + @columns = Bugzilla->dbh->bz_table_columns_real($class->DB_TABLE); + } + else { + # Currently you can only add new columns using object_columns, not + # remove or modify existing columns, because removing columns would + # almost certainly cause Bugzilla to function improperly. + my @add_columns; + Bugzilla::Hook::process('object_columns', + { class => $class, columns => \@add_columns }); + @columns = ($invocant->DB_COLUMNS, @add_columns); + } $cache->{$cache_key} = \@columns; return @{ $cache->{$cache_key} }; } diff --git a/Bugzilla/Search/Quicksearch.pm b/Bugzilla/Search/Quicksearch.pm index 6897d2219..6c0253e27 100644 --- a/Bugzilla/Search/Quicksearch.pm +++ b/Bugzilla/Search/Quicksearch.pm @@ -469,6 +469,11 @@ sub _handle_field_names { $value = $2; $value =~ s/\\(["'])/$1/g; } + # If the value is a pair of matching quotes, the person wanted the empty string + elsif ($value =~ /^(["'])\1$/ || $translated eq 'resolution' && $value eq '---') { + $value = ""; + $operator = "isempty"; + } addChart($translated, $operator, $value, $negate); } } @@ -659,6 +664,9 @@ sub negateComparisonType { if ($comparisonType eq 'anywords') { return 'nowords'; } + elsif ($comparisonType eq 'isempty') { + return 'isnotempty'; + } return "not$comparisonType"; } diff --git a/Bugzilla/Sentry.pm b/Bugzilla/Sentry.pm deleted file mode 100644 index 0d7a9c980..000000000 --- a/Bugzilla/Sentry.pm +++ /dev/null @@ -1,374 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# This Source Code Form is "Incompatible With Secondary Licenses", as -# defined by the Mozilla Public License, v. 2.0. - -package Bugzilla::Sentry; - -use 5.10.1; -use strict; -use warnings; - -use base qw(Exporter); -our @EXPORT = qw( - sentry_handle_error - sentry_should_notify -); - -use Carp; -use DateTime; -use File::Temp; -use JSON (); -use List::MoreUtils qw( any ); -use LWP::UserAgent; -use Sys::Hostname; -use URI; -use URI::QueryParam; - -use Bugzilla::Constants; -use Bugzilla::RNG qw(irand); -use Bugzilla::Util; -use Bugzilla::WebService::Constants; - -use constant CONFIG => { - # 'codes' lists the code-errors which are sent to sentry - codes => [qw( - bug_error - chart_datafile_corrupt - chart_dir_nonexistent - chart_file_open_fail - illegal_content_type_method - jobqueue_insert_failed - ldap_bind_failed - mail_send_error - template_error - token_generation_error - param_must_be_numeric - )], - - # any error/warning messages matching these regex's will not be logged or - # sent to sentry - ignore => [ - qr/^compiled template :\s*$/, - qr/^Use of uninitialized value \$compiled in concatenation \(\.\) or string/, - ], - - # any error/warning messages matching these regex's will be logged but not - # sent to sentry - sentry_ignore => [ - qr/Software caused connection abort/, - qr/Could not check out .*\/cvsroot/, - qr/Unicode character \S+ is illegal/, - qr/Lost connection to MySQL server during query/, - qr/Call me again when you have some data to chart/, - qr/relative paths are not allowed/, - qr/Illegal mix of collations for operation/, - ], - - # (ab)use the logger to classify error/warning types - logger => [ - { - match => [ - qr/DBD::mysql/, - qr/Can't connect to the database/, - ], - logger => 'database_error', - }, - { - match => [ qr/PatchReader/ ], - logger => 'patchreader', - }, - { - match => [ qr/Use of uninitialized value/ ], - logger => 'uninitialized_warning', - }, - ], -}; - -sub sentry_generate_id { - return sprintf('%04x%04x%04x%04x%04x%04x%04x%04x', - irand(0xffff), irand(0xffff), - irand(0xffff), - irand(0x0fff) | 0x4000, - irand(0x3fff) | 0x8000, - irand(0xffff), irand(0xffff), irand(0xffff) - ); -} - -sub sentry_should_notify { - my $code_error = shift; - return grep { $_ eq $code_error } @{ CONFIG->{codes} }; -} - -sub sentry_handle_error { - my $level = shift; - my @message = split(/\n/, shift); - my $id = sentry_generate_id(); - - my $is_error = $level eq 'error'; - if ($level ne 'error' && $level ne 'warning') { - # it's a code-error - return 0 unless sentry_should_notify($level); - $is_error = 1; - $level = 'error'; - } - - # build traceback - my $traceback; - { - # for now don't show function arguments, in case they contain - # confidential data. waiting on bug 700683 - #local $Carp::MaxArgLen = 256; - #local $Carp::MaxArgNums = 0; - local $Carp::MaxArgNums = -1; - local $Carp::CarpInternal{'CGI::Carp'} = 1; - local $Carp::CarpInternal{'Bugzilla::Error'} = 1; - local $Carp::CarpInternal{'Bugzilla::Sentry'} = 1; - $traceback = trim(Carp::longmess()); - } - - # strip timestamp - foreach my $line (@message) { - $line =~ s/^\[[^\]]+\] //; - } - my $message = join(" ", map { trim($_) } grep { $_ ne '' } @message); - - # message content filtering - foreach my $re (@{ CONFIG->{ignore} }) { - return 0 if $message =~ $re; - } - - # determine logger - my $logger; - foreach my $config (@{ CONFIG->{logger} }) { - foreach my $re (@{ $config->{match} }) { - if ($message =~ $re) { - $logger = $config->{logger}; - last; - } - } - last if $logger; - } - $logger ||= $level; - - # don't send to sentry unless configured - my $send_to_sentry = Bugzilla->params->{sentry_uri} ? 1 : 0; - - # web service filtering - if ($send_to_sentry - && (Bugzilla->error_mode == ERROR_MODE_DIE_SOAP_FAULT || Bugzilla->error_mode == ERROR_MODE_JSON_RPC)) - { - my ($code) = $message =~ /^(-?\d+): /; - if ($code - && !($code == ERROR_UNKNOWN_FATAL || $code == ERROR_UNKNOWN_TRANSIENT)) - { - $send_to_sentry = 0; - } - } - - # message content filtering - if ($send_to_sentry) { - foreach my $re (@{ CONFIG->{sentry_ignore} }) { - if ($message =~ $re) { - $send_to_sentry = 0; - last; - } - } - } - - # invalid boolean search errors need special handling - if ($message =~ /selectcol_arrayref failed: syntax error/ - && $message =~ /IN BOOLEAN MODE/ - && $message =~ /Bugzilla\/Search\.pm/) - { - $send_to_sentry = 0; - } - - # for now, don't send patchreader errors to sentry - $send_to_sentry = 0 - if $logger eq 'patchreader'; - - # log to apache's error_log - if ($send_to_sentry) { - _write_to_error_log("$message [#$id]", $is_error); - } else { - $traceback =~ s/\n/ /g; - _write_to_error_log("$message $traceback", $is_error); - } - - return 0 unless $send_to_sentry; - - my $user_data = undef; - eval { - my $user = Bugzilla->user; - if ($user->id) { - $user_data = { - id => $user->login, - name => $user->name, - }; - } - }; - - my $uri = URI->new(Bugzilla->cgi->self_url); - $uri->query(undef); - - # sanitise - - # sanitise these query-string params - # names are checked as-is as well as prefixed by BUGZILLA_ - my @sanitise_params = qw( PASSWORD TOKEN API_KEY ); - - # remove these ENV vars - my @sanitise_vars = qw( HTTP_COOKIE HTTP_X_BUGZILLA_PASSWORD HTTP_X_BUGZILLA_API_KEY HTTP_X_BUGZILLA_TOKEN ); - - foreach my $var (qw( QUERY_STRING REDIRECT_QUERY_STRING )) { - next unless exists $ENV{$var}; - my @pairs = split('&', $ENV{$var}); - foreach my $pair (@pairs) { - next unless $pair =~ /^([^=]+)=(.+)$/; - my ($param, $value) = ($1, $2); - if (any { uc($param) eq $_ || uc($param) eq "BUGZILLA_$_" } @sanitise_params) { - $value = '*'; - } - $pair = $param . '=' . $value; - } - $ENV{$var} = join('&', @pairs); - } - foreach my $var (qw( REQUEST_URI HTTP_REFERER )) { - next unless exists $ENV{$var}; - my $uri = URI->new($ENV{$var}); - foreach my $param ($uri->query_param) { - if (any { uc($param) eq $_ || uc($param) eq "BUGZILLA_$_" } @sanitise_params) { - $uri->query_param($param, '*'); - } - } - $ENV{$var} = $uri->as_string; - } - foreach my $var (@sanitise_vars) { - delete $ENV{$var}; - } - - my $now = DateTime->now(); - my $data = { - event_id => $id, - message => $message, - timestamp => $now->iso8601(), - level => $level, - platform => 'Other', - logger => $logger, - server_name => hostname(), - 'sentry.interfaces.User' => $user_data, - 'sentry.interfaces.Http' => { - url => $uri->as_string, - method => $ENV{REQUEST_METHOD}, - query_string => $ENV{QUERY_STRING}, - env => \%ENV, - }, - extra => { - stacktrace => $traceback, - }, - }; - - my $fh = File::Temp->new( - DIR => bz_locations()->{error_reports}, - TEMPLATE => $now->ymd('') . $now->hms('') . '-XXXX', - SUFFIX => '.dump', - UNLINK => 0, - - ); - if (!$fh) { - warn "Failed to create dump file: $!\n"; - return; - } - print $fh JSON->new->utf8(1)->pretty(0)->allow_nonref(1)->encode($data); - close($fh); - return 1; -} - -sub _write_to_error_log { - my ($message, $is_error) = @_; - if ($ENV{MOD_PERL}) { - require Apache2::Log; - if ($is_error) { - Apache2::ServerRec::log_error($message); - } else { - Apache2::ServerRec::warn($message); - } - } else { - print STDERR $message, "\n"; - } -} - -# lifted from Bugzilla::Error -sub _in_eval { - my $in_eval = 0; - for (my $stack = 1; my $sub = (caller($stack))[3]; $stack++) { - last if $sub =~ /^ModPerl/; - last if $sub =~ /^Bugzilla::Template/; - $in_eval = 1 if $sub =~ /^\(eval\)/; - } - return $in_eval; -} - -sub _sentry_die_handler { - my $message = shift; - $message =~ s/^undef error - //; - - # avoid recursion, and check for CGI::Carp::die failures - my $in_cgi_carp_die = 0; - for (my $stack = 1; my $sub = (caller($stack))[3]; $stack++) { - return if $sub =~ /:_sentry_die_handler$/; - $in_cgi_carp_die = 1 if $sub =~ /CGI::Carp::die$/; - } - - return if $Bugzilla::Template::is_processing; - return if _in_eval(); - - # mod_perl overrides exit to call die with this string - exit if $message =~ /\bModPerl::Util::exit\b/; - - my $nested_error = ''; - my $is_compilation_failure = $message =~ /\bcompilation (aborted|failed)\b/i; - - # if we are called via CGI::Carp::die chances are something is seriously - # wrong, so skip trying to use ThrowTemplateError - if (!$in_cgi_carp_die && !$is_compilation_failure) { - eval { - my $cgi = Bugzilla->cgi; - $cgi->close_standby_message('text/html', 'inline', 'error', 'html'); - Bugzilla::Error::ThrowTemplateError($message); - print $cgi->multipart_final() if $cgi->{_multipart_in_progress}; - }; - $nested_error = $@ if $@; - } - - if ($is_compilation_failure || - $in_cgi_carp_die || - ($nested_error && $nested_error !~ /\bModPerl::Util::exit\b/) - ) { - sentry_handle_error('error', $message); - - # and call the normal error management - # (ISE for web pages, error response for web services, etc) - CORE::die($message); - } - exit; -} - -sub install_sentry_handler { - $SIG{__DIE__} = \&sentry_die_handler; - $SIG{__WARN__} = sub { - return if _in_eval(); - sentry_handle_error('warning', shift); - }; -} - -BEGIN { - if ($ENV{SCRIPT_NAME} || $ENV{MOD_PERL}) { - install_sentry_handler(); - } -} - -1; diff --git a/Bugzilla/Template.pm b/Bugzilla/Template.pm index e6b4e551d..107c457c6 100644 --- a/Bugzilla/Template.pm +++ b/Bugzilla/Template.pm @@ -1052,10 +1052,7 @@ sub create { $SHARED_PROVIDERS{$provider_key} ||= $provider_class->new($config); $config->{LOAD_TEMPLATES} = [ $SHARED_PROVIDERS{$provider_key} ]; - # BMO - use metrics subclass - local $Template::Config::CONTEXT = Bugzilla->metrics_enabled() - ? 'Bugzilla::Metrics::Template::Context' - : 'Bugzilla::Template::Context'; + local $Template::Config::CONTEXT = 'Bugzilla::Template::Context'; Bugzilla::Hook::process('template_before_create', { config => $config }); my $template = $class->new($config) diff --git a/Bugzilla/WebService/Bug.pm b/Bugzilla/WebService/Bug.pm index c07454d7d..feb541c2e 100644 --- a/Bugzilla/WebService/Bug.pm +++ b/Bugzilla/WebService/Bug.pm @@ -35,6 +35,8 @@ use Bugzilla::Search::Quicksearch; use List::Util qw(max); use List::MoreUtils qw(uniq); use Storable qw(dclone); +use Types::Standard -all; +use Type::Utils; ############# # Constants # @@ -656,9 +658,30 @@ sub possible_duplicates { Bugzilla->switch_to_shadow_db(); - # Undo the array-ification that validate() does, for "summary". - $params->{summary} || ThrowCodeError('param_required', - { function => 'Bug.possible_duplicates', param => 'summary' }); + state $params_type = Dict [ + id => Optional [Int], + product => Optional [ ArrayRef [Str] ], + limit => Optional [Int], + summary => Optional [Str], + include_fields => Optional [ ArrayRef [Str] ], + Bugzilla_api_token => Optional [Str] + ]; + + ThrowCodeError( 'param_invalid', { function => 'Bug.possible_duplicates', param => 'A param' } ) + if !$params_type->check($params); + + my $summary; + if ($params->{id}) { + my $bug = Bugzilla::Bug->check({ id => $params->{id}, cache => 1 }); + $summary = $bug->short_desc; + } + elsif ($params->{summary}) { + $summary = $params->{summary}; + } + else { + ThrowCodeError('param_required', + { function => 'Bug.possible_duplicates', param => 'id or summary' }); + } my @products; foreach my $name (@{ $params->{'product'} || [] }) { @@ -667,8 +690,18 @@ sub possible_duplicates { } my $possible_dupes = Bugzilla::Bug->possible_duplicates( - { summary => $params->{summary}, products => \@products, - limit => $params->{limit} }); + { + summary => $summary, + products => \@products, + limit => $params->{limit} + } + ); + + # If a bug id was used, remove the bug with the same id from the list. + if ($params->{id}) { + @$possible_dupes = grep { $_->id != $params->{id} } @$possible_dupes; + } + my @hashes = map { $self->_bug_to_hash($_, $params) } @$possible_dupes; $self->_add_update_tokens($params, $possible_dupes, \@hashes); return { bugs => \@hashes }; diff --git a/Bugzilla/WebService/Server.pm b/Bugzilla/WebService/Server.pm index ba9847abc..a76c4c48c 100644 --- a/Bugzilla/WebService/Server.pm +++ b/Bugzilla/WebService/Server.pm @@ -23,11 +23,6 @@ sub handle_login { # Throw error if the supplied class does not exist or the method is private ThrowCodeError('unknown_method', {method => $full_method}) if (!$class or $method =~ /^_/); - # BMO - use the class and method as the name, instead of the cgi filename - if (Bugzilla->metrics_enabled) { - Bugzilla->metrics->name("$class $method"); - } - # We never want to create a new session unless the user is calling the # login method. Setting dont_persist_session makes # Bugzilla::Auth::_handle_login_result() skip calling persist_login(). diff --git a/Bugzilla/WebService/Server/REST/Resources/Bug.pm b/Bugzilla/WebService/Server/REST/Resources/Bug.pm index 33cf43321..26aec011c 100644 --- a/Bugzilla/WebService/Server/REST/Resources/Bug.pm +++ b/Bugzilla/WebService/Server/REST/Resources/Bug.pm @@ -34,6 +34,11 @@ sub _rest_resources { method => 'get' } }, + qr{^/bug/possible_duplicates$}, { + GET => { + method => 'possible_duplicates' + } + }, qr{^/bug/([^/]+)$}, { GET => { method => 'get', diff --git a/Bugzilla/WebService/Server/XMLRPC.pm b/Bugzilla/WebService/Server/XMLRPC.pm index fce865e88..6bb73af01 100644 --- a/Bugzilla/WebService/Server/XMLRPC.pm +++ b/Bugzilla/WebService/Server/XMLRPC.pm @@ -11,6 +11,7 @@ use 5.10.1; use strict; use warnings; +use Bugzilla::Logging; use XMLRPC::Transport::HTTP; use Bugzilla::WebService::Server; if ($ENV{MOD_PERL}) { @@ -32,8 +33,15 @@ BEGIN { if ($type eq 'dateTime') { # This is the XML-RPC implementation, see the README in Bugzilla/WebService/. # Our "base" implementation is in Bugzilla::WebService::Server. - $value = Bugzilla::WebService::Server->datetime_format_outbound($value); - $value =~ s/-//g; + if (defined $value) { + $value = Bugzilla::WebService::Server->datetime_format_outbound($value); + $value =~ s/-//g; + } + else { + my ($pkg, $file, $line) = caller; + my $class = ref $self; + ERROR("$class->type($type, undef) called from $pkg ($file line $line)"); + } } elsif ($type eq 'email') { $type = 'string'; |