summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDylan William Hardison <dylan@hardison.net>2018-03-01 02:59:55 +0100
committerGitHub <noreply@github.com>2018-03-01 02:59:55 +0100
commit48f08ef18983729dbdcd1af9468fb4ea1ee25fbe (patch)
tree68d2b0bd8eb7f340716bbbb69ced6107da76a801
parentc7fd0908bd85bc5761aa28fe9c893f93394bfd4b (diff)
downloadbugzilla-48f08ef18983729dbdcd1af9468fb4ea1ee25fbe.tar.gz
bugzilla-48f08ef18983729dbdcd1af9468fb4ea1ee25fbe.tar.xz
Bug 1437646 - Refactor entrypoint to use Bugzilla::DaemonControl
-rw-r--r--.circleci/config.yml9
-rw-r--r--Bugzilla/DaemonControl.pm41
-rw-r--r--Bugzilla/Install/Filesystem.pm4
-rw-r--r--Dockerfile2
-rw-r--r--conf/checksetup_answers.txt (renamed from docker_support/checksetup_answers.txt)0
-rw-r--r--conf/httpd.conf (renamed from httpd/httpd.conf)15
-rwxr-xr-xdocker-compose.yml2
-rwxr-xr-xscripts/entrypoint.pl255
8 files changed, 117 insertions, 211 deletions
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 2241909ba..9c99df18b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -16,7 +16,7 @@ main_filters: &main_filters
defaults:
bmo_slim_image: &bmo_slim_image
- image: mozillabteam/bmo-slim:20171228.1
+ image: mozillabteam/bmo-slim:20180225.1
user: app
mysql_image: &mysql_image
@@ -24,6 +24,7 @@ defaults:
bmo_env: &bmo_env
PORT: 8000
+ LOGGING_PORT: 5880
LOCALCONFIG_ENV: 1
BMO_db_user: bugs
BMO_db_host: 127.0.0.1
@@ -129,8 +130,6 @@ jobs:
/app/scripts/entrypoint.pl test_webservices | tee artifacts/$CIRCLE_JOB.txt
- store_artifacts:
path: /app/artifacts
- - store_artifacts:
- path: /app/logs
test_selenium:
parallelism: 1
@@ -146,8 +145,6 @@ jobs:
/app/scripts/entrypoint.pl test_selenium | tee artifacts/$CIRCLE_JOB.txt
- store_artifacts:
path: /app/artifacts
- - store_artifacts:
- path: /app/logs
test_bmo:
parallelism: 1
@@ -172,8 +169,6 @@ jobs:
/app/scripts/entrypoint.pl load_test_data
mkdir artifacts
- run: /app/scripts/entrypoint.pl test_bmo -q -f t/bmo/*.t
- - store_artifacts:
- path: /app/logs
workflows:
version: 2
diff --git a/Bugzilla/DaemonControl.pm b/Bugzilla/DaemonControl.pm
index a1c9fd8a5..97ecb84e0 100644
--- a/Bugzilla/DaemonControl.pm
+++ b/Bugzilla/DaemonControl.pm
@@ -79,15 +79,18 @@ sub cereal {
);
$loop->add($protocol);
};
+ my @signals = (
+ catch_signal('TERM', 0),
+ catch_signal('INT', 0 ),
+ catch_signal('KILL', 0 ),
+ );
$loop->listen(
host => '127.0.0.1',
service => $ENV{LOGGING_PORT},
socktype => 'stream',
on_stream => $on_stream,
)->get;
- kill 'USR1', getppid();
-
- exit catch_signal('TERM', 0)->get;
+ exit Future->wait_any(@signals)->get;
}
sub run_cereal {
@@ -99,9 +102,7 @@ sub run_cereal {
on_exception => on_exception( "cereal", $exit_f ),
);
$exit_f->on_cancel( sub { $cereal->kill('TERM') } );
- my $signal_f = catch_signal('USR1');
$loop->add($cereal);
- $signal_f->get;
return $exit_f;
}
@@ -135,11 +136,16 @@ sub run_cereal_and_httpd {
push @httpd_args, '-DHTTPS';
}
push @httpd_args, '-DNETCAT_LOGS';
- my $cereal_exit_f = run_cereal();
my $signal_f = catch_signal("TERM", 0);
- my $httpd_exit_f = run_httpd(@httpd_args);
+ my $cereal_exit_f = run_cereal();
- return Future->wait_any($cereal_exit_f, $httpd_exit_f, $signal_f);
+ return assert_cereal()->then(
+ sub {
+ my $httpd_exit_f = run_httpd(@httpd_args);
+
+ return Future->wait_any($cereal_exit_f, $httpd_exit_f, $signal_f);
+ }
+ );
}
sub assert_httpd {
@@ -159,10 +165,25 @@ sub assert_httpd {
return Future->wait_any($repeat, $timeout);
}
+
sub assert_selenium {
my ($host, $port) = @_;
$host //= 'localhost';
$port //= 4444;
+
+ return assert_connect($host, $port, "assert_selenium");
+}
+
+sub assert_cereal {
+ return assert_connect(
+ 'localhost',
+ $ENV{LOGGING_PORT} // 5880,
+ "assert_cereal"
+ );
+}
+
+sub assert_connect {
+ my ($host, $port, $name) = @_;
my $loop = IO::Async::Loop->new;
my $repeat = repeat {
$loop->delay_future(after => 1)->then(
@@ -172,7 +193,7 @@ sub assert_selenium {
},
);
} until => sub { shift->get };
- my $timeout = $loop->timeout_future(after => 60)->else_fail("assert_selenium timeout");
+ my $timeout = $loop->timeout_future(after => 60)->else_fail("$name timeout");
return Future->wait_any($repeat, $timeout);
}
@@ -200,7 +221,7 @@ sub assert_database {
} until => sub { defined shift->get };
my $timeout = $loop->timeout_future( after => 20 )->else_fail("assert_database timeout");
- my $any_f = Future->needs_any( $repeat, $timeout );
+ my $any_f = Future->wait_any( $repeat, $timeout );
return $any_f->transform(
done => sub { return },
fail => sub { "unable to connect to $dsn as $lc->{db_user}" },
diff --git a/Bugzilla/Install/Filesystem.pm b/Bugzilla/Install/Filesystem.pm
index cbec34bdc..d205a6750 100644
--- a/Bugzilla/Install/Filesystem.pm
+++ b/Bugzilla/Install/Filesystem.pm
@@ -423,7 +423,7 @@ sub FILESYSTEM {
"skins/yui3.css" => { perms => CGI_READ,
overwrite => 1,
contents => $yui3_all_css },
- "httpd/env.conf" => { perms => CGI_READ,
+ "$confdir/env.conf" => { perms => CGI_READ,
overwrite => 1,
contents => \&HTTPD_ENV_CONF },
);
@@ -460,8 +460,6 @@ sub FILESYSTEM {
contents => HT_DEFAULT_DENY },
'.circleci/.htaccess' => { perms => WS_SERVE,
contents => HT_DEFAULT_DENY },
- 'httpd/.htaccess' => { perms => WS_SERVE,
- contents => HT_DEFAULT_DENY },
"$confdir/.htaccess" => { perms => WS_SERVE,
contents => HT_DEFAULT_DENY },
"$datadir/.htaccess" => { perms => WS_SERVE,
diff --git a/Dockerfile b/Dockerfile
index c1525f217..8b82ac6ff 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM mozillabteam/bmo-slim:20171228.1
+FROM mozillabteam/bmo-slim:20180225.1
ARG CI
diff --git a/docker_support/checksetup_answers.txt b/conf/checksetup_answers.txt
index a1913ac7b..a1913ac7b 100644
--- a/docker_support/checksetup_answers.txt
+++ b/conf/checksetup_answers.txt
diff --git a/httpd/httpd.conf b/conf/httpd.conf
index a664ebb16..c0e8b7570 100644
--- a/httpd/httpd.conf
+++ b/conf/httpd.conf
@@ -50,7 +50,18 @@ TypesConfig /etc/mime.types
DefaultType text/plain
MIMEMagicFile conf/magic
HostnameLookups Off
-ErrorLog /dev/stderr
+<IfDefine NETCAT_LOGS>
+ ErrorLog "|/usr/bin/nc localhost ${LOGGING_PORT}"
+ <IfDefine ACCESS_LOGS>
+ TransferLog "|/usr/bin/nc localhost ${LOGGING_PORT}"
+ </IfDefine>
+</IfDefine>
+<IfDefine !NETCAT_LOGS>
+ ErrorLog /dev/stderr
+ <IfDefine ACCESS_LOGS>
+ TransferLog /dev/stdout
+ </IfDefine>
+</IfDefine>
LogLevel warn
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
LogFormat "%h %l %u %t \"%r\" %>s %b" common
@@ -59,7 +70,7 @@ LogFormat "%{User-agent}i" agent
ServerSignature Off
AddDefaultCharset UTF-8
-Include /app/httpd/env.conf
+Include /app/conf/env.conf
PerlSwitches -wT
PerlRequire /app/mod_perl.pl
diff --git a/docker-compose.yml b/docker-compose.yml
index 4e50a8807..1c5011b55 100755
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -27,7 +27,7 @@ services:
- BMO_memcached_namespace=bugzilla
- BMO_memcached_servers=memcached:11211
- 'BMO_inbound_proxies=*'
- - BZ_ANSWERS_FILE=/app/docker_support/checksetup_answers.txt
+ - BZ_ANSWERS_FILE=/app/conf/checksetup_answers.txt
depends_on:
- bmo-db.vm
- memcached
diff --git a/scripts/entrypoint.pl b/scripts/entrypoint.pl
index 947c5521e..23578e257 100755
--- a/scripts/entrypoint.pl
+++ b/scripts/entrypoint.pl
@@ -8,6 +8,11 @@ use autodie qw(:all);
use Bugzilla::Install::Localconfig ();
use Bugzilla::Install::Util qw(install_string);
use Bugzilla::Test::Util qw(create_user);
+use Bugzilla::DaemonControl qw(
+ run_cereal_and_httpd
+ assert_httpd assert_database assert_selenium
+ on_finish on_exception
+);
use DBI;
use Data::Dumper;
@@ -78,31 +83,24 @@ sub cmd_httpd {
check_data_dir();
wait_for_db();
check_httpd_env();
- httpd();
-}
-
-sub httpd {
- my @httpd_args = (
- '-DFOREGROUND',
- '-f' => '/app/httpd/httpd.conf',
- );
- # If we're behind a proxy and the urlbase says https, we must be using https.
- # * basically means "I trust the load balancer" anyway.
- if ($ENV{BMO_inbound_proxies} eq '*' && $ENV{BMO_urlbase} =~ /^https/) {
- unshift @httpd_args, '-DHTTPS';
- }
- run( '/usr/sbin/httpd', @httpd_args );
+ my $httpd_exit_f = run_cereal_and_httpd();
+ assert_httpd()->get();
+ exit $httpd_exit_f->get();
}
sub cmd_dev_httpd {
- wait_for_db();
my $have_params = -f "/app/data/params";
+ assert_database->get();
+
run( 'perl', 'checksetup.pl', '--no-template', $ENV{BZ_ANSWERS_FILE} );
if ( not $have_params ) {
run( 'perl', 'scripts/generate_bmo_data.pl', '--param' => 'use_mailer_queue=0', 'vagrant@bmo-web.vm' );
}
- httpd();
+
+ my $httpd_exit_f = run_cereal_and_httpd('-DACCESS_LOGS');
+ assert_httpd()->get;
+ exit $httpd_exit_f->get;
}
sub cmd_checksetup {
@@ -129,36 +127,15 @@ sub cmd_load_test_data {
}
}
-sub cmd_test_heartbeat {
- my ($url) = @_;
- die "test_heartbeat requires a url!\n" unless $url;
-
- wait_for_httpd($url);
- my $heartbeat = get("$url/__heartbeat__");
- if ($heartbeat && $heartbeat =~ /Bugzilla OK/) {
- exit 0;
- }
- else {
- exit 1;
- }
-}
-
sub cmd_test_webservices {
-
my $conf = require $ENV{BZ_QA_CONF_FILE};
check_data_dir();
- wait_for_db();
-
- my @httpd_cmd = ( '/usr/sbin/httpd', '-DFOREGROUND', '-f', '/app/httpd/httpd.conf' );
- if ($ENV{BZ_QA_LEGACY_MODE}) {
- copy_qa_extension();
- push @httpd_cmd, '-DHTTPD_IN_SUBDIR';
- }
-
- prove_with_httpd(
+ copy_qa_extension();
+ assert_database()->get;
+ my $httpd_exit_f = run_cereal_and_httpd('-DHTTPD_IN_SUBDIR', '-DACCESS_LOGS');
+ my $prove_exit_f = run_prove(
httpd_url => $conf->{browser_url},
- httpd_cmd => \@httpd_cmd,
prove_cmd => [
'prove', '-qf', '-I/app',
'-I/app/local/lib/perl5',
@@ -166,29 +143,28 @@ sub cmd_test_webservices {
],
prove_dir => '/app/qa/t',
);
+ exit Future->wait_any($prove_exit_f, $httpd_exit_f)->get;
}
sub cmd_test_selenium {
my $conf = require $ENV{BZ_QA_CONF_FILE};
check_data_dir();
- wait_for_db();
- my @httpd_cmd = ( '/usr/sbin/httpd', '-DFOREGROUND', '-f', '/app/httpd/httpd.conf' );
- if ($ENV{BZ_QA_LEGACY_MODE}) {
- copy_qa_extension();
- push @httpd_cmd, '-DHTTPD_IN_SUBDIR';
- }
+ copy_qa_extension();
- prove_with_httpd(
+ assert_database()->get;
+ assert_selenium()->get;
+ my $httpd_exit_f = run_cereal_and_httpd('-DHTTPD_IN_SUBDIR');
+ my $prove_exit_f = run_prove(
httpd_url => $conf->{browser_url},
- httpd_cmd => \@httpd_cmd,
+ prove_dir => '/app/qa/t',
prove_cmd => [
'prove', '-qf', '-Ilib', '-I/app',
'-I/app/local/lib/perl5',
sub { glob 'test_*.t' }
],
- prove_dir => '/app/qa/t',
);
+ exit Future->wait_any($prove_exit_f, $httpd_exit_f)->get;
}
sub cmd_shell { run( 'bash', '-l' ); }
@@ -201,110 +177,60 @@ sub cmd_version { run( 'cat', '/app/version.json' ); }
sub cmd_test_bmo {
my (@prove_args) = @_;
check_data_dir();
- wait_for_db();
- $ENV{BZ_TEST_NEWBIE} = 'newbie@mozilla.example';
+ assert_database()->get;
+ assert_selenium()->get;
+ $ENV{BZ_TEST_NEWBIE} = 'newbie@mozilla.example';
$ENV{BZ_TEST_NEWBIE_PASS} = 'captain.space.bagel.ROBOT!';
create_user($ENV{BZ_TEST_NEWBIE}, $ENV{BZ_TEST_NEWBIE_PASS}, realname => 'Newbie User');
- $ENV{BZ_TEST_NEWBIE2} = 'newbie2@mozilla.example';
+ $ENV{BZ_TEST_NEWBIE2} = 'newbie2@mozilla.example';
$ENV{BZ_TEST_NEWBIE2_PASS} = 'captain.space.pants.time.lord';
- prove_with_httpd(
- httpd_url => $ENV{BZ_BASE_URL},
- httpd_cmd => [ '/usr/sbin/httpd', '-f', '/app/httpd/httpd.conf', '-DFOREGROUND' ],
- prove_cmd => [ 'prove', '-I/app', '-I/app/local/lib/perl5', @prove_args ],
+ my $httpd_exit_f = run_cereal_and_httpd('-DACCESS_LOGS');
+ my $prove_exit_f = run_prove(
+ httpd_url => $ENV{BZ_BASE_URL},
+ prove_cmd => [ 'prove', '-I/app', '-I/app/local/lib/perl5', @prove_args ],
);
+
+ exit Future->wait_any($prove_exit_f, $httpd_exit_f)->get;
}
-sub prove_with_httpd {
+sub run_prove {
my (%param) = @_;
check_httpd_env();
- unless (-d '/app/logs') {
- mkdir '/app/logs' or die "unable to mkdir(/app/logs): $!\n";
- }
-
- my $httpd_cmd = $param{httpd_cmd};
- my $prove_cmd = $param{prove_cmd};
-
- my $loop = IO::Async::Loop->new;
-
- my $httpd_exit_f = $loop->new_future;
- say 'starting httpd';
- my $httpd = IO::Async::Process->new(
- code => sub {
- setsid();
- exec @$httpd_cmd;
- },
- setup => [
- stdout => ['open', '>', '/app/logs/access.log'],
- stderr => ['open', '>', '/app/logs/error.log'],
- ],
- on_finish => on_finish($httpd_exit_f),
- on_exception => on_exception('httpd', $httpd_exit_f),
- );
- $loop->add($httpd);
- wait_for_httpd( $httpd, $param{httpd_url} );
-
- warn "httpd started, starting prove\n";
-
- my $prove_exit_f = $loop->new_future;
- my $prove = IO::Async::Process->new(
- code => sub {
- chdir $param{prove_dir} if $param{prove_dir};
- my @cmd = (map { ref $_ eq 'CODE' ? $_->() : $_ } @$prove_cmd);
- warn "run @cmd\n";
- exec @cmd;
- },
- on_finish => on_finish($prove_exit_f),
- on_exception => on_exception('prove', $prove_exit_f),
- );
- $loop->add($prove);
-
- my $prove_exit = $prove_exit_f->get();
- if ($httpd->is_running) {
- $httpd->kill('TERM');
- my $httpd_exit = $httpd_exit_f->get();
- warn "httpd exit code: $httpd_exit\n" if $httpd_exit != 0;
- }
-
- exit $prove_exit;
-}
-
-sub wait_for_httpd {
- my ($process, $url) = @_;
- my $loop = IO::Async::Loop->new;
- my $is_running_f = $loop->new_future;
- my $ticks = 0;
- my $run_checker = IO::Async::Timer::Periodic->new(
- first_interval => 0,
- interval => 1,
- reschedule => 'hard',
- on_tick => sub {
- my ($timer) = @_;
- if ( $process->is_running ) {
- my $resp = get("$url/__lbheartbeat__");
- if ($resp && $resp =~ /^httpd OK/) {
- $timer->stop;
- $is_running_f->done($resp);
- }
- say "httpd doesn't seem to be up at $url. waiting...";
- }
- elsif ( $process->is_exited ) {
- $timer->stop;
- $is_running_f->fail('httpd process exited early');
- }
- elsif ( $ticks++ > 60 ) {
- $timer->stop;
- $is_running_f->fail("is_running_future() timeout after $ticks seconds");
- }
- $timer->stop if $ticks++ > 60;
- },
- );
- $loop->add($run_checker->start);
- return $is_running_f->get();
+ my $prove_cmd = $param{prove_cmd};
+ my $prove_dir = $param{prove_dir};
+ assert_httpd()->then(sub {
+ my $loop = IO::Async::Loop->new;
+ $loop->connect(
+ socktype => 'stream',
+ host => 'localhost',
+ service => 5880,
+ )->then(sub {
+ my $socket = shift;
+ my $prove_exit_f = $loop->new_future;
+ my $prove = IO::Async::Process->new(
+ code => sub {
+ chdir $prove_dir if $prove_dir;
+ my @cmd = (map { ref $_ eq 'CODE' ? $_->() : $_ } @$prove_cmd);
+ warn "run @cmd\n";
+ exec @cmd;
+ },
+ setup => [
+ stdin => ['close'],
+ stdout => [ 'dup', $socket ],
+ ],
+ on_finish => on_finish($prove_exit_f),
+ on_exception => on_exception('prove', $prove_exit_f),
+ );
+ $prove_exit_f->on_cancel(sub { $prove->kill('TERM') });
+ $loop->add($prove);
+ return $prove_exit_f;
+ });
+ });
}
sub copy_qa_extension {
@@ -313,52 +239,7 @@ sub copy_qa_extension {
}
sub wait_for_db {
- my $c = Bugzilla::Install::Localconfig::read_localconfig();
- for my $var (qw(db_name db_host db_user db_pass)) {
- die "$var is not set!" unless $c->{$var};
- }
-
- my $dsn = "dbi:mysql:database=$c->{db_name};host=$c->{db_host}";
- my $dbh;
- foreach (1..12) {
- say 'checking database...' if $_ > 1;
- $dbh = DBI->connect(
- $dsn,
- $c->{db_user},
- $c->{db_pass},
- { RaiseError => 0, PrintError => 0 }
- );
- last if $dbh;
- say "database $dsn not available, waiting...";
- sleep 10;
- }
- die "unable to connect to $dsn as $c->{db_user}\n" unless $dbh;
-}
-
-sub on_exception {
- my ($name, $f) = @_;
- return sub {
- my ( $self, $exception, $errno, $exitcode ) = @_;
-
- if ( length $exception ) {
- $f->fail("$name died with the exception $exception " . "(errno was $errno)\n");
- }
- elsif ( ( my $status = WEXITSTATUS($exitcode) ) == 255 ) {
- $f->fail("$name failed to exec() - $errno\n");
- }
- else {
- $f->fail("$name exited with exit status $status\n");
- }
- };
-}
-
-sub on_finish {
- my ($f) = @_;
- return sub {
- my ($self, $exitcode) = @_;
- say "exit code: $exitcode";
- $f->done(WEXITSTATUS($exitcode));
- };
+ assert_database()->get;
}
sub check_user {