diff options
| -rw-r--r-- | .dockerignore | 18 | ||||
| -rw-r--r-- | Bugzilla/Install/Localconfig.pm | 9 | ||||
| -rw-r--r-- | Dockerfile | 49 | ||||
| -rw-r--r-- | README.rst | 103 | ||||
| -rwxr-xr-x | checksetup.pl | 127 | ||||
| -rw-r--r-- | docker_files/httpd.conf | 98 | ||||
| -rwxr-xr-x | docker_files/init.pl | 81 |
7 files changed, 412 insertions, 73 deletions
diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..a2e02b567 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.vagrant +MYMETA.* +Makefile +blib +local +pm_to_blib +template_cache +vagrant_support +\#*\# +*/\#*\# +*/*/\#*\# +*/*/*/\#*\# +*/*/*/*/\#*\# +.DS_Store +*/.DS_Store +*/*/.DS_Store +*/*/*/.DS_Store +*/*/*/*/.DS_Store diff --git a/Bugzilla/Install/Localconfig.pm b/Bugzilla/Install/Localconfig.pm index 263d63ced..f9363f1e9 100644 --- a/Bugzilla/Install/Localconfig.pm +++ b/Bugzilla/Install/Localconfig.pm @@ -40,10 +40,7 @@ our @EXPORT_OK = qw( sub _sensible_group { return '' if ON_WINDOWS; - my @groups = qw( apache www-data _www ); - my $sensible_group = first { return getgrnam($_) } @groups; - - return $sensible_group // getgrgid($EGID) // ''; + return scalar getgrgid($EGID); } use constant LOCALCONFIG_VARS => ( @@ -53,7 +50,7 @@ use constant LOCALCONFIG_VARS => ( }, { name => 'webservergroup', - default => ON_WINDOWS ? '' : 'apache', + default => \&_sensible_group, }, { name => 'use_suexec', @@ -297,7 +294,7 @@ sub update_localconfig { print colored(install_string('lc_new_vars', { localconfig => $filename, new_vars => wrap_hard($newstuff, 70) }), COLOR_ERROR), "\n"; - exit; + exit unless $params->{use_defaults}; } # Reset the cache for Bugzilla->localconfig so that it will be re-read diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..dd103a5db --- /dev/null +++ b/Dockerfile @@ -0,0 +1,49 @@ +FROM centos:6.7 +MAINTAINER Dylan William Hardison <dylan@mozilla.com> + +RUN yum update -y && \ + yum install -y perl perl-core mod_perl httpd wget tar openssl mysql-libs gd git && \ + wget -q https://s3.amazonaws.com/moz-devservices-bmocartons/bmo/vendor.tar.gz && \ + tar -C /opt -zxvf /vendor.tar.gz bmo/local/ bmo/LIBS.txt bmo/cpanfile bmo/cpanfile.snapshot && \ + rm /vendor.tar.gz && \ + mkdir /opt/bmo/httpd && \ + ln -s /usr/lib64/httpd/modules /opt/bmo/httpd/modules && \ + mkdir /opt/bmo/httpd/conf && \ + cp {/etc/httpd/conf,/opt/bmo/httpd}/magic && \ + awk '{print $1}' > LIBS.txt \ + | perl -nE 'chomp; unless (-f $_) { $missing++; say $_ } END { exit 1 if $missing }' && \ + useradd -u 10001 -U app -m && \ + curl -L https://cpanmin.us > /usr/local/bin/cpanm && \ + chmod 755 /usr/local/bin/cpanm && \ + mkdir /opt/bmo/build && \ + rpm -qa > /tmp/rpms.list && \ + yum install -y gcc mod_perl-devel && \ + cpanm -l /opt/bmo/build --notest Apache2::SizeLimit && \ + yum erase -y $(rpm -qa | diff -u - /tmp/rpms.list | sed -n '/^-[^-]/ s/^-//p') && \ + rm -rf /opt/bmo/build/lib/perl5/{CPAN,Parse,JSON,ExtUtils} && \ + mkdir /usr/local/share/perl5 && \ + mv /opt/bmo/build/lib/perl5/x86_64-linux-thread-multi/ /usr/local/lib64/perl5/ && \ + mv /opt/bmo/build/lib/perl5/Linux /usr/local/share/perl5/ && \ + rm -vfr /opt/bmo/build && \ + rm /tmp/rpms.list /usr/local/bin/cpanm && \ + yum clean all -y + +COPY . /app +WORKDIR /app +RUN ln -sv /opt/bmo/local /app/local && \ + chown -R app:app /app && \ + cp /app/docker_files/httpd.conf /opt/bmo/httpd/ && \ + mkdir /opt/bmo/bin && \ + cp /app/docker_files/init.pl /opt/bmo/bin/init.pl + +USER app +RUN perl checksetup.pl --no-database --default-localconfig && \ + prove t && \ + rm -rf /app/data && mkdir /app/data + +ENV PORT=8000 + +EXPOSE $PORT + +ENTRYPOINT ["/opt/bmo/bin/init.pl"] +CMD ["httpd"] diff --git a/README.rst b/README.rst index 322b04ea8..c1f3f8bfe 100644 --- a/README.rst +++ b/README.rst @@ -4,14 +4,25 @@ BMO: bugzilla.mozilla.org BMO is Mozilla's highly customized version of Bugzilla. +.. contents:: +.. + 1 Using Vagrant (For Development) + 1.1 Setup Vagrant VMs + 1.2 Making Changes and Seeing them + 1.3 Technical Details + 2 Docker Container + 2.1 Container Arguments + 2.2 Environmental Variables + 2.3 Persistent Data Volume + If you are looking to run Bugzilla, you should see https://github.com/bugzilla/bugzilla. If you want to contribute to BMO, you can fork this repo and get a local copy -of BMO running in a few minutes. +of BMO running in a few minutes using Vagrant. -Install Vagrant -=============== +Using Vagrant (For Development) +=============================== You will need to install the following software: @@ -27,7 +38,7 @@ For Ubuntu 16.04, download the vagrant .dpkg directly from https://vagrantup.com. The one that ships with Ubuntu is too old. Setup Vagrant VMs -================= +----------------- From your BMO checkout run the following command: @@ -50,7 +61,7 @@ You can login as vagrant@bmo-web.vm with the password "vagrant01!" (without quotes). Making Changes and Seeing them -============================== +------------------------------ After editing files in the bmo directory, you will need to run @@ -66,7 +77,7 @@ or db is changed, do a full provision: vagrant rsync && vagrant provision Technical Details -================= +----------------- This Vagrant environment is a very complete but scaled-down version of production BMO. It uses roughly the same RPMs (from CentOS 6, versus RHEL 6 @@ -82,3 +93,83 @@ Most of the cron jobs and the jobqueue daemon are running. It is also configured to use memcached. The push connector is not currently configured, nor is the Pulse publisher. + + +Docker Container +================ + +This repository is also a runnable docker container. + +Container Arguments +------------------- + +Currently, the entry point takes a single command argument. +This can be **httpd** or **shell**. + +httpd + This will start apache listening for connections on ``$PORT`` +shell + This will start an interactive shell in the container. Useful for debugging. + + +Environmental Variables +----------------------- + +PORT + This must be a value >= 1024. The httpd will listen on this port for incoming + plain-text HTTP connections. + +BMO_db_driver + What SQL database to use. Default is mysql. List of supported databases can be + obtained by listing Bugzilla/DB directory - every module corresponds to one + supported database and the name of the module (before ".pm") corresponds to a + valid value for this variable. + +BMO_db_host + The DNS name or IP address of the host that the database server runs on. + +BMO_db_name + The name of the database. + +BMO_db_user + The database user to connect as. + +BMO_db_pass + The password for the user above. + +BMO_site_wide_secret + This secret key is used by your installation for the creation and + validation of encrypted tokens. These tokens are used to implement + security features in Bugzilla, to protect against certain types of attacks. + It's very important that this key is kept secret. + +BMO_inbound_proxies + This is a list of IP addresses that we expect proxies to come from. + This can be '*' if only the load balancer can connect to this container. + Setting this to '*' means that BMO will trust the X-Forwarded-For header. + +BMO_memcached_namespace + The global namespace for the memcached servers. + +BMO_memcached_servers + A list of memcached servers (ip addresses or host names). Can be empty. + +BMO_shadowdb + The database name of the read-only database. + +BMO_shadowdbhost + The hotname or ip address of the read-only database. + +BMO_shadowdbport + The port of the read-only database. + +BMO_apache_size_limit + This is the max amount of unshared memory (in kb) that the apache process is + allowed to use before Apache::SizeLimit kills it. + +Persistent Data Volume +---------------------- + +This container expects /app/data to be a persistent, shared, writable directory +owned by uid 10001. This must be a shared (NFS/EFS/etc) volume between all +nodes. diff --git a/checksetup.pl b/checksetup.pl index d689602fd..b7a852e0f 100755 --- a/checksetup.pl +++ b/checksetup.pl @@ -53,7 +53,8 @@ GetOptions(\%switch, 'help|h|?', 'cpanm:s', 'check-modules', 'make-admin=s', 'reset-password=s', 'version|V', 'no-assets', - 'no-permissions|p'); + 'default-localconfig', + 'no-database', 'no-permissions|p'); # Print the help message if that switch was selected. pod2usage({-verbose => 1, -exitval => 1}) if $switch{'help'}; @@ -150,7 +151,7 @@ Bugzilla->installation_answers($answers_file); ########################################################################### print "Reading " . bz_locations()->{'localconfig'} . "...\n" unless $silent; -update_localconfig({ output => !$silent }); +update_localconfig({ output => !$silent, use_defaults => $switch{'default-localconfig'} }); my $lc_hash = Bugzilla->localconfig; ########################################################################### @@ -161,17 +162,19 @@ my $lc_hash = Bugzilla->localconfig; # everything we need to create the DB. We have to create it early, # because some data required to populate data/params.json is stored in the DB. -Bugzilla::DB::bz_check_requirements(!$silent); -Bugzilla::DB::bz_create_database() if $lc_hash->{'db_check'}; - -# now get a handle to the database: -my $dbh = Bugzilla->dbh; -# Clear all keys from Memcached to ensure we see the correct schema. -Bugzilla->memcached->clear_all(); -# Create the tables, and do any database-specific schema changes. -$dbh->bz_setup_database(); -# Populate the tables that hold the values for the <select> fields. -$dbh->bz_populate_enum_tables(); +unless ($switch{'no-database'}) { + Bugzilla::DB::bz_check_requirements(!$silent); + Bugzilla::DB::bz_create_database() if $lc_hash->{'db_check'}; + + # now get a handle to the database: + my $dbh = Bugzilla->dbh; + # Clear all keys from Memcached to ensure we see the correct schema. + Bugzilla->memcached->clear_all(); + # Create the tables, and do any database-specific schema changes. + $dbh->bz_setup_database(); + # Populate the tables that hold the values for the <select> fields. + $dbh->bz_populate_enum_tables(); +} ########################################################################### # Check --DATA-- directory @@ -182,7 +185,7 @@ create_htaccess() if $lc_hash->{'create_htaccess'}; # Remove parameters from the params file that no longer exist in Bugzilla, # and set the defaults for new ones -my %old_params = update_params(); +my %old_params = $switch{'no-database'} ? () : update_params(); ########################################################################### # Pre-compile --TEMPLATE-- code @@ -222,71 +225,73 @@ check_font_file(!$silent) if $lc_hash->{'font_file'}; # Changes to the fielddefs --TABLE-- ########################################################################### -# Using Bugzilla::Field's create() or update() depends on the -# fielddefs table having a modern definition. So, we have to make -# these particular schema changes before we make any other schema changes. -Bugzilla::Install::DB::update_fielddefs_definition(); +unless ($switch{'no-database'}) { + # Using Bugzilla::Field's create() or update() depends on the + # fielddefs table having a modern definition. So, we have to make + # these particular schema changes before we make any other schema changes. + Bugzilla::Install::DB::update_fielddefs_definition(); -Bugzilla::Field::populate_field_definitions(); + Bugzilla::Field::populate_field_definitions(); -########################################################################### -# Update the tables to the current definition --TABLE-- -########################################################################### + ########################################################################### + # Update the tables to the current definition --TABLE-- + ########################################################################### -Bugzilla::Install::DB::update_table_definitions(\%old_params); -Bugzilla::Install::init_workflow(); + Bugzilla::Install::DB::update_table_definitions(\%old_params); + Bugzilla::Install::init_workflow(); -########################################################################### -# Bugzilla uses --GROUPS-- to assign various rights to its users. -########################################################################### + ########################################################################### + # Bugzilla uses --GROUPS-- to assign various rights to its users. + ########################################################################### -Bugzilla::Install::update_system_groups(); + Bugzilla::Install::update_system_groups(); -# "Log In" as the fake superuser who can do everything. -Bugzilla->set_user(Bugzilla::User->super_user); + # "Log In" as the fake superuser who can do everything. + Bugzilla->set_user(Bugzilla::User->super_user); -########################################################################### -# Create --SETTINGS-- users can adjust -########################################################################### + ########################################################################### + # Create --SETTINGS-- users can adjust + ########################################################################### -Bugzilla::Install::update_settings(); + Bugzilla::Install::update_settings(); -########################################################################### -# Create Administrator --ADMIN-- -########################################################################### + ########################################################################### + # Create Administrator --ADMIN-- + ########################################################################### -Bugzilla::Install::make_admin($switch{'make-admin'}) if $switch{'make-admin'}; -Bugzilla::Install::create_admin(); + Bugzilla::Install::make_admin($switch{'make-admin'}) if $switch{'make-admin'}; + Bugzilla::Install::create_admin(); -Bugzilla::Install::reset_password($switch{'reset-password'}) - if $switch{'reset-password'}; + Bugzilla::Install::reset_password($switch{'reset-password'}) + if $switch{'reset-password'}; -########################################################################### -# Create default Product -########################################################################### + ########################################################################### + # Create default Product + ########################################################################### -Bugzilla::Install::create_default_product(); + Bugzilla::Install::create_default_product(); -Bugzilla::Hook::process('install_before_final_checks', { silent => $silent }); + Bugzilla::Hook::process('install_before_final_checks', { silent => $silent }); -########################################################################### -# Final checks -########################################################################### + ########################################################################### + # Final checks + ########################################################################### -# Clear all keys from Memcached -Bugzilla->memcached->clear_all(); + # Clear all keys from Memcached + Bugzilla->memcached->clear_all(); -# Reset the mod_perl pre-load list -unlink(Bugzilla::Constants::bz_locations()->{datadir} . '/mod_perl_preload'); + # Reset the mod_perl pre-load list + unlink(Bugzilla::Constants::bz_locations()->{datadir} . '/mod_perl_preload'); -# Check if the default parameter for urlbase is still set, and if so, give -# notification that they should go and visit editparams.cgi -if (Bugzilla->params->{'urlbase'} eq '') { - print "\n" . get_text('install_urlbase_default') . "\n" - unless $silent; -} -if (!$silent) { - success(get_text('install_success')); + # Check if the default parameter for urlbase is still set, and if so, give + # notification that they should go and visit editparams.cgi + if (Bugzilla->params->{'urlbase'} eq '') { + print "\n" . get_text('install_urlbase_default') . "\n" + unless $silent; + } + if (!$silent) { + success(get_text('install_success')); + } } __END__ diff --git a/docker_files/httpd.conf b/docker_files/httpd.conf new file mode 100644 index 000000000..b8c779052 --- /dev/null +++ b/docker_files/httpd.conf @@ -0,0 +1,98 @@ +ServerTokens Prod +ServerRoot "/opt/bmo/httpd" +PidFile /tmp/httpd.pid +Timeout 60 +KeepAlive Off +MaxKeepAliveRequests 100 +KeepAliveTimeout 15 +<IfModule prefork.c> +StartServers 8 +MinSpareServers 5 +MaxSpareServers 20 +ServerLimit 256 +MaxClients 256 +MaxRequestsPerChild 4000 +</IfModule> +<IfModule worker.c> +StartServers 4 +MaxClients 300 +MinSpareThreads 25 +MaxSpareThreads 75 +ThreadsPerChild 25 +MaxRequestsPerChild 0 +</IfModule> +Listen ${PORT} +LoadModule authz_host_module modules/mod_authz_host.so +LoadModule include_module modules/mod_include.so +LoadModule log_config_module modules/mod_log_config.so +LoadModule logio_module modules/mod_logio.so +LoadModule env_module modules/mod_env.so +LoadModule ext_filter_module modules/mod_ext_filter.so +LoadModule mime_magic_module modules/mod_mime_magic.so +LoadModule expires_module modules/mod_expires.so +LoadModule deflate_module modules/mod_deflate.so +LoadModule headers_module modules/mod_headers.so +LoadModule usertrack_module modules/mod_usertrack.so +LoadModule setenvif_module modules/mod_setenvif.so +LoadModule mime_module modules/mod_mime.so +LoadModule dav_module modules/mod_dav.so +LoadModule status_module modules/mod_status.so +LoadModule autoindex_module modules/mod_autoindex.so +LoadModule info_module modules/mod_info.so +LoadModule dav_fs_module modules/mod_dav_fs.so +LoadModule vhost_alias_module modules/mod_vhost_alias.so +LoadModule negotiation_module modules/mod_negotiation.so +LoadModule dir_module modules/mod_dir.so +LoadModule actions_module modules/mod_actions.so +LoadModule speling_module modules/mod_speling.so +LoadModule alias_module modules/mod_alias.so +LoadModule substitute_module modules/mod_substitute.so +LoadModule rewrite_module modules/mod_rewrite.so +LoadModule cache_module modules/mod_cache.so +LoadModule disk_cache_module modules/mod_disk_cache.so +LoadModule version_module modules/mod_version.so +LoadModule perl_module modules/mod_perl.so +User app +Group app +ServerAdmin root@localhost +UseCanonicalName Off +<Directory /> + Options FollowSymLinks + AllowOverride None +</Directory> +AccessFileName .htaccess +<Files ~ "^\.ht"> + Order allow,deny + Deny from all + Satisfy All +</Files> +TypesConfig /etc/mime.types +DefaultType text/plain +<IfModule mod_mime_magic.c> + MIMEMagicFile magic +</IfModule> +HostnameLookups Off +ErrorLog /dev/stderr +LogLevel warn +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined +LogFormat "%h %l %u %t \"%r\" %>s %b" common +LogFormat "%{Referer}i -> %U" referer +LogFormat "%{User-agent}i" agent +CustomLog /dev/stdout combined +ServerSignature Off +AddDefaultCharset UTF-8 +AddType application/x-compress .Z +AddType application/x-gzip .gz .tgz +AddType application/x-x509-ca-cert .crt +AddType application/x-pkcs7-crl .crl + +PerlSwitches -wT +PerlRequire /app/mod_perl.pl +DirectoryIndex index.cgi +DocumentRoot "/app" +<Directory "/app"> + Options -Indexes -FollowSymLinks + AllowOverride None + Order allow,deny + Allow from all +</Directory> diff --git a/docker_files/init.pl b/docker_files/init.pl new file mode 100755 index 000000000..6e7a8920b --- /dev/null +++ b/docker_files/init.pl @@ -0,0 +1,81 @@ +#!/usr/bin/perl +use strict; +use warnings; +use lib qw(/app /opt/bmo/local/lib/perl5); +use Getopt::Long qw(:config gnu_getopt); +use Data::Dumper; +use Bugzilla::Install::Localconfig (); +use Bugzilla::Install::Util qw(install_string); + +my %localconfig = (webservergroup => 'app'); + +my %override = ( + 'inbound_proxies' => 1, + 'memcached_namespace' => 1, + 'memcached_servers' => 1, + 'shadowdb' => 1, + 'shadowdbhost' => 1, + 'shadowdbport' => 1, + 'shadowdbsock' => 1 +); + +# clean env. +foreach my $key (keys %ENV) { + if ($key =~ /^BMO_(.+)$/) { + my $name = $1; + if ($override{$name}) { + $localconfig{param_override}{$name} = delete $ENV{$key}; + } + else { + $localconfig{$name} = delete $ENV{$key}; + } + } +} + +write_localconfig(\%localconfig); +system("perl", "checksetup.pl", "--no-templates", "--no-permissions", '--no-assets'); + +my $cmd = shift @ARGV or die "usage: init.pl CMD"; +my $method = "run_$cmd"; +__PACKAGE__->$method(); + +sub run_httpd { + exec("/usr/sbin/httpd", "-DFOREGROUND", "-f", "/opt/bmo/httpd/httpd.conf"); +} + +sub run_shell { + exec("/bin/bash", "-l"); +} + +sub write_localconfig { + my ($localconfig) = @_; + no warnings 'once'; + + foreach my $var (Bugzilla::Install::Localconfig::LOCALCONFIG_VARS) { + my $name = $var->{name}; + my $value = $localconfig->{$name}; + if (!defined $value) { + $var->{default} = &{$var->{default}} if ref($var->{default}) eq 'CODE'; + $localconfig->{$name} = $var->{default}; + } + } + + my $filename = "/app/localconfig"; + + # Ensure output is sorted and deterministic + local $Data::Dumper::Sortkeys = 1; + + # Re-write localconfig + open my $fh, ">:utf8", $filename or die "$filename: $!"; + foreach my $var (Bugzilla::Install::Localconfig::LOCALCONFIG_VARS) { + my $name = $var->{name}; + my $desc = install_string("localconfig_$name", { root => Bugzilla::Install::Localconfig::ROOT_USER }); + chomp($desc); + # Make the description into a comment. + $desc =~ s/^/# /mg; + print $fh $desc, "\n", + Data::Dumper->Dump([$localconfig->{$name}], + ["*$name"]), "\n"; + } + close $fh; +} |