diff options
author | Dylan William Hardison <dylan@hardison.net> | 2018-04-10 17:29:02 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-04-10 17:29:02 +0200 |
commit | 0002ecc5bb7078e2da0350fcc3c2850cf02bcd7b (patch) | |
tree | 4a465207632b31ddae6d34df623ce37da5fd3267 | |
parent | 58c4f6814461ee4e35f0cffbdd63d948df940d35 (diff) | |
download | bugzilla-0002ecc5bb7078e2da0350fcc3c2850cf02bcd7b.tar.gz bugzilla-0002ecc5bb7078e2da0350fcc3c2850cf02bcd7b.tar.xz |
Bug 1450990 - Refactor a bunch of the logging config files
-rw-r--r-- | Bugzilla.pm | 1 | ||||
-rw-r--r-- | Bugzilla/Constants.pm | 1 | ||||
-rw-r--r-- | Bugzilla/DaemonControl.pm | 6 | ||||
-rw-r--r-- | Bugzilla/Error.pm | 2 | ||||
-rw-r--r-- | Bugzilla/Install/Filesystem.pm | 8 | ||||
-rw-r--r-- | Bugzilla/Logging.pm | 11 | ||||
-rw-r--r-- | README.rst | 7 | ||||
-rw-r--r-- | Vagrantfile | 2 | ||||
-rw-r--r-- | conf/httpd.conf | 1 | ||||
-rw-r--r-- | conf/log4perl-docker.conf | 14 | ||||
-rw-r--r-- | conf/log4perl-json.conf | 12 | ||||
-rw-r--r-- | conf/log4perl-test.conf | 9 | ||||
-rw-r--r-- | conf/log4perl-vagrant.conf | 24 | ||||
-rw-r--r-- | docker-compose.yml | 2 | ||||
-rw-r--r-- | mod_perl.pl | 4 | ||||
-rw-r--r-- | vagrant_support/apache.j2 | 2 | ||||
-rw-r--r-- | vagrant_support/playbook.yml | 9 |
17 files changed, 95 insertions, 20 deletions
diff --git a/Bugzilla.pm b/Bugzilla.pm index a8b164b57..d98298acc 100644 --- a/Bugzilla.pm +++ b/Bugzilla.pm @@ -56,7 +56,6 @@ use Encode; use File::Basename; use File::Spec::Functions; use Safe; -use Sys::Syslog qw(:DEFAULT); use JSON::XS qw(decode_json); use parent qw(Bugzilla::CPAN); diff --git a/Bugzilla/Constants.pm b/Bugzilla/Constants.pm index 80d9c4e0c..3aa7b0fa4 100644 --- a/Bugzilla/Constants.pm +++ b/Bugzilla/Constants.pm @@ -693,6 +693,7 @@ sub _bz_locations { # The script should really generate these graphs directly... 'webdotdir' => "$datadir/webdot", 'extensionsdir' => "$libpath/extensions", + 'logsdir' => "$libpath/logs", 'assetsdir' => "$datadir/assets", 'confdir' => $confdir, }; diff --git a/Bugzilla/DaemonControl.pm b/Bugzilla/DaemonControl.pm index 6586cc01b..2c6df1b87 100644 --- a/Bugzilla/DaemonControl.pm +++ b/Bugzilla/DaemonControl.pm @@ -81,7 +81,13 @@ sub run_cereal { on_exception => on_exception( 'cereal', $exit_f ), ); $exit_f->on_cancel( sub { $cereal->kill('TERM') } ); + $exit_f->on_ready( + sub { + delete $ENV{LOG4PERL_STDERR_DISABLE}; + } + ); $loop->add($cereal); + $ENV{LOG4PERL_STDERR_DISABLE} = 1; return $exit_f; } diff --git a/Bugzilla/Error.pm b/Bugzilla/Error.pm index ef57303e9..9fcd16386 100644 --- a/Bugzilla/Error.pm +++ b/Bugzilla/Error.pm @@ -169,7 +169,7 @@ sub ThrowUserError { sub ThrowCodeError { my ($error, $vars) = @_; - my $logfunc = _make_logfunc('User'); + my $logfunc = _make_logfunc('Code'); _add_vars_to_logging_fields($vars); _throw_error( 'global/code-error.html.tmpl', $error, $vars, $logfunc ); diff --git a/Bugzilla/Install/Filesystem.pm b/Bugzilla/Install/Filesystem.pm index 70b195090..094226891 100644 --- a/Bugzilla/Install/Filesystem.pm +++ b/Bugzilla/Install/Filesystem.pm @@ -36,6 +36,7 @@ use Cwd (); use File::Slurp; use IO::File; use POSIX (); +use English qw(-no_match_vars $OSNAME); use base qw(Exporter); our @EXPORT = qw( @@ -106,6 +107,7 @@ use constant HTTPD_ENV => qw( LOCALCONFIG_ENV BUGZILLA_UNSAFE_AUTH_DELEGATION LOG4PERL_CONFIG_FILE + LOG4PERL_STDERR_DISABLE USE_NYTPROF NYTPROF_DIR ); @@ -162,6 +164,8 @@ sub DIR_CGI_OVERWRITE { _group() ? 0770 : 0777 }; # (or their subdirectories) to the user, via the webserver. sub DIR_ALSO_WS_SERVE { _suexec() ? 0001 : 0 }; +sub DIR_ALSO_WS_STICKY { $OSNAME eq 'linux' ? 02000 : 0 } + # This looks like a constant because it effectively is, but # it has to call other subroutines and read the current filesystem, # so it's defined as a sub. This is not exported, so it doesn't have @@ -186,6 +190,7 @@ sub FILESYSTEM { my $template_cache = bz_locations()->{'template_cache'}; my $graphsdir = bz_locations()->{'graphsdir'}; my $assetsdir = bz_locations()->{'assetsdir'}; + my $logsdir = bz_locations()->{'logsdir'}; # We want to set the permissions the same for all localconfig files # across all PROJECTs, so we do something special with $localconfig, @@ -277,6 +282,8 @@ sub FILESYSTEM { dirs => DIR_CGI_WRITE | DIR_ALSO_WS_SERVE }, "$datadir/db" => { files => CGI_WRITE, dirs => DIR_CGI_WRITE }, + $logsdir => { files => CGI_WRITE, + dirs => DIR_CGI_WRITE | DIR_ALSO_WS_STICKY }, $assetsdir => { files => WS_SERVE, dirs => DIR_CGI_OVERWRITE | DIR_ALSO_WS_SERVE }, @@ -361,6 +368,7 @@ sub FILESYSTEM { $webdotdir => DIR_CGI_WRITE | DIR_ALSO_WS_SERVE, $assetsdir => DIR_CGI_WRITE | DIR_ALSO_WS_SERVE, $template_cache => DIR_CGI_WRITE, + $logsdir => DIR_CGI_WRITE | DIR_ALSO_WS_STICKY, # Directories that contain content served directly by the web server. "$skinsdir/custom" => DIR_WS_SERVE, "$skinsdir/contrib" => DIR_WS_SERVE, diff --git a/Bugzilla/Logging.pm b/Bugzilla/Logging.pm index 4a7abcb21..f334435fc 100644 --- a/Bugzilla/Logging.pm +++ b/Bugzilla/Logging.pm @@ -12,12 +12,17 @@ use warnings; use Log::Log4perl qw(:easy); use Log::Log4perl::MDC; -use File::Spec::Functions qw(rel2abs); +use File::Spec::Functions qw(rel2abs catfile); use Bugzilla::Constants qw(bz_locations); use English qw(-no_match_vars $PROGRAM_NAME); +use Taint::Util qw(untaint); -sub is_interactive { - return not exists $ENV{SERVER_SOFTWARE} +sub logfile { + my ($class, $name) = @_; + + my $file = rel2abs(catfile(bz_locations->{logsdir}, $name)); + untaint($file); + return $file; } sub fields { diff --git a/README.rst b/README.rst index adeb1a18e..d8986fc73 100644 --- a/README.rst +++ b/README.rst @@ -377,6 +377,13 @@ LOG4PERL_CONFIG_FILE .. _`Devel::NYTProf`: https://metacpan.org/pod/Devel::NYTProf .. _`Log::Log4perl`: https://metacpan.org/pod/Log::Log4perl +LOG4PERL_STDERR_DISABLE + Boolean. By default log messages are logged as plain text to `STDERR`. + Setting this to a true value disables this behavior. + + Note: For programs that run using the `cereal` log aggregator, this environemnt + variable will be ignored. + Persistent Data Volume ---------------------- diff --git a/Vagrantfile b/Vagrantfile index 8776ac08f..2a1c4e9c4 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -38,7 +38,6 @@ Vagrant.configure('2') do |config| config.vm.provision 'main', type: 'ansible_local', run: 'always' do |ansible| ansible.playbook = 'vagrant_support/playbook.yml' - ansible.version = 'latest' ansible.extra_vars = { WEB_IP: WEB_IP, DB_IP: DB_IP, @@ -51,7 +50,6 @@ Vagrant.configure('2') do |config| if ARGV.include? '--provision-with' config.vm.provision 'update', type: 'ansible_local', run: 'never' do |update| update.playbook = 'vagrant_support/update.yml' - update.version = 'latest' end end diff --git a/conf/httpd.conf b/conf/httpd.conf index 7fe859b59..539ab4231 100644 --- a/conf/httpd.conf +++ b/conf/httpd.conf @@ -86,6 +86,7 @@ Include /app/conf/env.conf PerlSwitches -wT PerlRequire /app/mod_perl.pl +PerlSetEnv LOG4PERL_STDERR_DISABLE 1 DirectoryIndex index.cgi DocumentRoot "/app" <IfDefine HTTPD_IN_SUBDIR> diff --git a/conf/log4perl-docker.conf b/conf/log4perl-docker.conf new file mode 100644 index 000000000..1f5a033fd --- /dev/null +++ b/conf/log4perl-docker.conf @@ -0,0 +1,14 @@ +log4perl.rootLogger = DEBUG, Cereal, Screen +log4perl.appender.Cereal = Log::Log4perl::Appender::Socket +log4perl.appender.Cereal.PeerAddr=127.0.0.1 +log4perl.appender.Cereal.PeerPort=5880 +log4perl.appender.Cereal.defer_connection=1 +log4perl.appender.Cereal.layout = Log::Log4perl::Layout::PatternLayout +log4perl.appender.Cereal.layout.ConversionPattern = %d %6p | %c | %m{chomp}%n + +log4perl.filter.LOG_TO_STDERR = sub { not $ENV{LOG4PERL_STDERR_DISABLE} } +log4perl.appender.Screen = Log::Log4perl::Appender::Screen +log4perl.appender.Screen.Filter = LOG_TO_STDERR +log4perl.appender.Screen.stderr = 1 +log4perl.appender.Screen.layout = Log::Log4perl::Layout::PatternLayout +log4perl.appender.Screen.layout.ConversionPattern = %d %6p | %c | %m{chomp}%n diff --git a/conf/log4perl-json.conf b/conf/log4perl-json.conf index c5d7e2c1f..a9c4b6e1a 100644 --- a/conf/log4perl-json.conf +++ b/conf/log4perl-json.conf @@ -13,10 +13,12 @@ log4perl.appender.Cereal.layout.max_json_length = 16384 # The default is Bugzilla. This is the "Logger" field # in https://wiki.mozilla.org/Firefox/Services/Logging#MozLog_JSON_schema #and it might be useful to pass in different values for different jobs. -log4perl.appender.Cereal.layout.name = Bugzilla +log4perl.appender.Cereal.layout.name = CEREAL -log4perl.filter.IS_INTERACTIVE = sub { Bugzilla::Logging::is_interactive() } +log4perl.filter.LOG_TO_STDERR = sub { not $ENV{LOG4PERL_STDERR_DISABLE} } log4perl.appender.Screen = Log::Log4perl::Appender::Screen -log4perl.appender.Screen.Filter = IS_INTERACTIVE -log4perl.appender.Screen.layout = Log::Log4perl::Layout::PatternLayout -log4perl.appender.Screen.layout.ConversionPattern = %-5.5p [%d] [%c] %m{chomp} at %F line %L (%M)%n +log4perl.appender.Screen.Filter = LOG_TO_STDERR +log4perl.appender.Screen.stderr = 1 +log4perl.appender.Screen.layout = Log::Log4perl::Layout::Mozilla +log4perl.appender.Screen.layout.max_json_length = 16384 +log4perl.appender.Screen.layout.name = STDERR diff --git a/conf/log4perl-test.conf b/conf/log4perl-test.conf index 34b163073..77fc00af8 100644 --- a/conf/log4perl-test.conf +++ b/conf/log4perl-test.conf @@ -4,13 +4,14 @@ log4perl.appender.Cereal.PeerAddr=127.0.0.1 log4perl.appender.Cereal.PeerPort=5880 log4perl.appender.Cereal.defer_connection=1 log4perl.appender.Cereal.layout = Log::Log4perl::Layout::PatternLayout -log4perl.appender.Cereal.layout.ConversionPattern = %-5.5p [%d] [%c] %m{chomp} at %F line %L (%M)%n +log4perl.appender.Cereal.layout.ConversionPattern = %d %6p | %c | %m{chomp}%n -log4perl.filter.IS_INTERACTIVE = sub { Bugzilla::Logging::is_interactive() } +log4perl.filter.LOG_TO_STDERR = sub { not $ENV{LOG4PERL_STDERR_DISABLE} } log4perl.appender.Screen = Log::Log4perl::Appender::Screen -log4perl.appender.Screen.Filter = IS_INTERACTIVE +log4perl.appender.Screen.Filter = LOG_TO_STDERR +log4perl.appender.Screen.stderr = 1 log4perl.appender.Screen.layout = Log::Log4perl::Layout::PatternLayout -log4perl.appender.Screen.layout.ConversionPattern = %-5.5p [%d] [%c] %m{chomp} at %F line %L (%M)%n +log4perl.appender.Screen.layout.ConversionPattern = %d %6p | %c | %m{chomp}%n log4perl.appender.File = Log::Log4perl::Appender::File log4perl.appender.File.layout = Log::Log4perl::Layout::Mozilla diff --git a/conf/log4perl-vagrant.conf b/conf/log4perl-vagrant.conf new file mode 100644 index 000000000..d9c3e2753 --- /dev/null +++ b/conf/log4perl-vagrant.conf @@ -0,0 +1,24 @@ +log4perl.rootLogger = DEBUG, TextFile, JSONFile, Screen + +log4perl.appender.TextFile = Log::Log4perl::Appender::File +log4perl.appender.TextFile.layout = Log::Log4perl::Layout::PatternLayout +log4perl.appender.TextFile.filename = sub { Bugzilla::Logging->logfile("bugzilla.log") } +log4perl.appender.TextFile.layout.ConversionPattern = %-5.5p [%d] [%c] %m{chomp} at %F line %L (%M)%n +log4perl.appender.TextFile.create_at_logtime = true +log4perl.appender.TextFile.owner = vagrant +log4perl.appender.TextFile.group = apache + +log4perl.appender.JSONFile = Log::Log4perl::Appender::File +log4perl.appender.JSONFile.layout = Log::Log4perl::Layout::Mozilla +log4perl.appender.JSONFile.filename = sub { Bugzilla::Logging->logfile("bugzilla-json.log") } +log4perl.appender.JSONFile.mode = append +log4perl.appender.JSONFile.create_at_logtime = true +log4perl.appender.JSONFile.owner = vagrant +log4perl.appender.JSONFile.group = apache + +log4perl.filter.LOG_TO_STDERR = sub { not $ENV{LOG4PERL_STDERR_DISABLE} } +log4perl.appender.Screen = Log::Log4perl::Appender::Screen +log4perl.appender.Screen.Filter = LOG_TO_STDERR +log4perl.appender.Screen.stderr = 1 +log4perl.appender.Screen.layout = Log::Log4perl::Layout::PatternLayout +log4perl.appender.Screen.layout.ConversionPattern = %d %6p | %c | %m{chomp}%n diff --git a/docker-compose.yml b/docker-compose.yml index 5dfb6fad4..1ca6f5c90 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: - /run environment: &bmo_env - LOCALCONFIG_ENV=1 - - LOG4PERL_CONFIG_FILE=log4perl-test.conf + - LOG4PERL_CONFIG_FILE=log4perl-docker.conf - BUGZILLA_UNSAFE_AUTH_DELEGATION=1 - PORT=80 - BMO_db_host=bmo-db.vm diff --git a/mod_perl.pl b/mod_perl.pl index 73406be56..ff84e523e 100644 --- a/mod_perl.pl +++ b/mod_perl.pl @@ -80,7 +80,7 @@ Bugzilla::CGI->compile(qw(:cgi :push)); # sharing with the other httpd processes. my $limit = Bugzilla->localconfig->{apache_size_limit}; if ($OSNAME eq 'linux' && ! eval { require Linux::Smaps }) { - warn "SizeLimit requires Linux::Smaps on linux. size limit set to 800MB"; + WARN('SizeLimit requires Linux::Smaps on linux. size limit set to 800MB'); $limit = 800_000; } Apache2::SizeLimit->set_max_unshared_size($limit); @@ -167,13 +167,11 @@ sub handler : method { DB::enable_profile($file); } Bugzilla::init_page(); - my $start = Time::HiRes::time(); my $result = $class->SUPER::handler(@_); if (Bugzilla::ModPerl::USE_NYTPROF) { DB::disable_profile(); DB::finish_profile(); } - warn "[request_time] ", Bugzilla->cgi->request_uri, " took ", Time::HiRes::time() - $start, " seconds to execute"; # When returning data from the REST api we must only return 200 or 304, # which tells Apache not to append its error html documents to the diff --git a/vagrant_support/apache.j2 b/vagrant_support/apache.j2 index 773672fa1..2222de41b 100644 --- a/vagrant_support/apache.j2 +++ b/vagrant_support/apache.j2 @@ -1,6 +1,8 @@ PerlSwitches -wT PerlSetEnv USE_NYTPROF 0 PerlSetEnv BUGZILLA_UNSAFE_AUTH_DELEGATION 1 +PerlPassEnv LOG4PERL_CONFIG_FILE +PerlSetEnv LOG4PERL_STDERR_DISABLE=1 PerlConfigRequire /vagrant/mod_perl.pl <IfModule mpm_prefork_module> diff --git a/vagrant_support/playbook.yml b/vagrant_support/playbook.yml index f38c1fc06..b394f5592 100644 --- a/vagrant_support/playbook.yml +++ b/vagrant_support/playbook.yml @@ -50,6 +50,15 @@ group: root mode: 0644 + - name: 'add LOG4PERL_CONFIG_FILE' + lineinfile: + dest: /etc/environment + regexp: 'LOG4PERL_CONFIG_FILE=' + line: 'LOG4PERL_CONFIG_FILE=log4perl-vagrant.conf' + owner: root + group: root + mode: 0644 + - name: copy ntp.conf copy: src: ntp.conf |