no bug - have the test runners start up the httpd (#203)

5 files changed, 251 insertions, 92 deletions

diff --git a/.circleci/checksetup_answers.legacy.txt b/.circleci/checksetup_answers.legacy.txt
new file mode 100644
index 000000000..6bcdd2dcc
--- /dev/null
+++ b/.circleci/checksetup_answers.legacy.txt
@@ -0,0 +1,11 @@
+$answer{'ADMIN_EMAIL'} = 'admin@mozilla.bugs';
+$answer{'ADMIN_OK'} = 'Y';
+$answer{'ADMIN_PASSWORD'} = 'password';
+$answer{'ADMIN_REALNAME'} = 'QA Admin';
+$answer{'NO_PAUSE'} = 1;
+$answer{'bugzilla_version'} = '1';
+$answer{'create_htaccess'} = '';
+$answer{'cvsbin'} = '/usr/bin/cvs';
+$answer{'diffpath'} = '/usr/bin';
+$answer{'interdiffbin'} = '/usr/bin/interdiff';
+$answer{'urlbase'} = 'http://<<HOSTNAME>>:8000/bmo/';
diff --git a/.circleci/checksetup_answers.txt b/.circleci/checksetup_answers.txt
index 6bcdd2dcc..80a1d40d2 100644
--- a/.circleci/checksetup_answers.txt
+++ b/.circleci/checksetup_answers.txt
@@ -4,8 +4,8 @@ $answer{'ADMIN_PASSWORD'} = 'password';
 $answer{'ADMIN_REALNAME'} = 'QA Admin';
 $answer{'NO_PAUSE'} = 1;
 $answer{'bugzilla_version'} = '1';
-$answer{'create_htaccess'} = '';
+$answer{'create_htaccess'} = '1';
 $answer{'cvsbin'} = '/usr/bin/cvs';
 $answer{'diffpath'} = '/usr/bin';
 $answer{'interdiffbin'} = '/usr/bin/interdiff';
-$answer{'urlbase'} = 'http://<<HOSTNAME>>:8000/bmo/';
+$answer{'urlbase'} = 'http://<<HOSTNAME>>:8000/';
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 5d0170e1f..619f1cb11 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -5,42 +5,49 @@
 
 version: 2
 
-test_docker: &test_docker
-  - image: mozillabteam/bmo-slim:20170803.1
+defaults:
+  bmo_slim_image: &bmo_slim_image
+    image: mozillabteam/bmo-slim:20170807.1
     user: app
-    environment:
-      PORT: 8000
-      BMO_db_user: bugs
-      BMO_db_host: 127.0.0.1
-      BMO_db_pass: bugs
-      BMO_db_name: bugs
-      BMO_memcached_servers: localhost:11211
-      BMO_memcached_namespace: "bugzilla:"
-      BZ_QA_CONF_FILE: /app/.circleci/selenium_test.conf
-      BZ_QA_ANSWERS_FILE:  /app/.circleci/checksetup_answers.txt
-  - image: mozillabteam/bmo-mysql:5.6
-    environment:
-      MYSQL_DATABASE: bugs
-      MYSQL_USER: bugs
-      MYSQL_PASSWORD: bugs
-      MYSQL_ALLOW_EMPTY_PASSWORD: 1
-  - image: selenium/standalone-firefox:2.53.1
-  - image: memcached:latest
 
-default_setup: &default_setup
-  run:
-    command: |
-      mv /opt/bmo/local /app/local
-      perl -MSys::Hostname -i -pE 's/<<HOSTNAME>>/hostname()/ges' $BZ_QA_CONF_FILE
-      perl -MSys::Hostname -i -pE 's/<<HOSTNAME>>/hostname()/ges' $BZ_QA_ANSWERS_FILE
-      perl checksetup.pl --no-database --default-localconfig
-      mkdir artifacts
+  mysql_image: &mysql_image
+    image: mozillabteam/bmo-mysql:5.6
 
-run_qa_httpd: &run_qa_httpd
-  run:
-    command: |
-      /app/scripts/entrypoint.pl qa_httpd &> artifacts/httpd.log
-    background: true
+  bmo_env: &bmo_env
+    PORT: 8000
+    BMO_db_user: bugs
+    BMO_db_host: 127.0.0.1
+    BMO_db_pass: bugs
+    BMO_db_name: bugs
+    BMO_memcached_servers: localhost:11211
+    BMO_memcached_namespace: "bugzilla:"
+
+  mysql_env: &mysql_env
+    MYSQL_DATABASE: bugs
+    MYSQL_USER: bugs
+    MYSQL_PASSWORD: bugs
+    MYSQL_ALLOW_EMPTY_PASSWORD: 1
+
+  docker_oldtests: &docker_oldtests
+    - <<: *bmo_slim_image
+      environment:
+        <<: *bmo_env
+        BZ_QA_CONF_FILE: /app/.circleci/selenium_test.conf
+        BZ_QA_ANSWERS_FILE:  /app/.circleci/checksetup_answers.legacy.txt
+        BZ_QA_LEGACY_MODE: 1
+    - <<: *mysql_image
+      environment: *mysql_env
+    - image: selenium/standalone-firefox:2.53.1
+    - image: memcached:latest
+
+  default_qa_setup: &default_qa_setup
+    run:
+      command: |
+        mv /opt/bmo/local /app/local
+        perl -MSys::Hostname -i -pE 's/<<HOSTNAME>>/hostname()/ges' $BZ_QA_CONF_FILE
+        perl -MSys::Hostname -i -pE 's/<<HOSTNAME>>/hostname()/ges' $BZ_QA_ANSWERS_FILE
+        perl checksetup.pl --no-database --default-localconfig
+        mkdir artifacts
 
 jobs:
   build:
@@ -68,11 +75,10 @@ jobs:
     parallelism: 4
     working_directory: /app
     docker:
-      - image: mozillabteam/bmo-slim:20170803.1
-        user: app
+      - *bmo_slim_image
     steps:
       - checkout
-      - *default_setup
+      - *default_qa_setup
       - run:
           name: run sanity tests
           command: |
@@ -83,15 +89,16 @@ jobs:
   test_webservices:
     parallelism: 1
     working_directory: /app
-    docker: *test_docker
+    docker: *docker_oldtests
     steps:
       - checkout
-      - *default_setup
-      - run: /app/scripts/entrypoint.pl load_test_data
-      - *run_qa_httpd
-      - run: /app/scripts/entrypoint.pl test_heartbeat
+      - *default_qa_setup
+      - run: |
+          rm -f /app/localconfig
+          /app/scripts/entrypoint.pl load_test_data
       - run:
           command: |
+            rm -f /app/localconfig
             /app/scripts/entrypoint.pl test_webservices | tee artifacts/$CIRCLE_JOB.txt
       - store_artifacts:
           path: /app/artifacts
@@ -99,14 +106,16 @@ jobs:
   test_selenium:
     parallelism: 1
     working_directory: /app
-    docker: *test_docker
+    docker: *docker_oldtests
     steps:
       - checkout
-      - *default_setup
-      - run: /app/scripts/entrypoint.pl load_test_data
-      - *run_qa_httpd
+      - *default_qa_setup
+      - run: |
+          rm -f /app/localconfig
+          /app/scripts/entrypoint.pl load_test_data --legacy
       - run:
           command: |
+            rm -f /app/localconfig
             /app/scripts/entrypoint.pl test_selenium | tee artifacts/$CIRCLE_JOB.txt
       - store_artifacts:
           path: /app/artifacts
diff --git a/Makefile.PL b/Makefile.PL
index c8e0ea9ea..3217101b8 100755
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -72,7 +72,7 @@ my %build_requires = (
 my %test_requires = (
     'Test::More'         => 0,
     'Pod::Coverage'      => 0,
-    'Test::WWW::Selenium' => 0.
+    'Test::WWW::Selenium' => 0,
 );
 my %recommends = ( Safe => '2.30' );
 
diff --git a/scripts/entrypoint.pl b/scripts/entrypoint.pl
index b34384ff1..2d1ef8fe9 100755
--- a/scripts/entrypoint.pl
+++ b/scripts/entrypoint.pl
@@ -12,26 +12,31 @@ use File::Copy::Recursive qw(dircopy);
 use Getopt::Long qw(:config gnu_getopt);
 use LWP::Simple qw(get);
 use User::pwent;
+use POSIX qw(WEXITSTATUS setsid);
+
+use IO::Async::Loop;
+use IO::Async::Process;
+use IO::Async::Timer::Periodic;
+use IO::Async::Signal;
+
+use constant CI => $ENV{CI};
 
 my $cmd = shift @ARGV;
-my $func = __PACKAGE__->can("cmd_$cmd") // sub { run($cmd, @ARGV) };
+my $func = __PACKAGE__->can("cmd_$cmd")
+    or die "unknown command: $cmd\n";
+my $opts = __PACKAGE__->can("opt_$cmd") // sub { @ARGV };
 
 fix_path();
 check_user();
 check_env() unless $cmd eq 'shell';
 write_localconfig( localconfig_from_env() );
+
 $func->(@ARGV);
 
 sub cmd_httpd  {
     check_data_dir();
     wait_for_db();
-    run( '/usr/sbin/httpd', '-DFOREGROUND', 
-        '-f', '/app/httpd/httpd.conf', @_ );
-}
 
-sub cmd_qa_httpd {
-    copy_qa_extension();
-    cmd_httpd('-DHTTPD_IN_SUBDIR', @_);
 }
 
 sub cmd_load_test_data {
@@ -39,17 +44,25 @@ sub cmd_load_test_data {
 
     die "BZ_QA_ANSWERS_FILE is not set" unless $ENV{BZ_QA_ANSWERS_FILE};
     run( 'perl', 'checksetup.pl', '--no-template', $ENV{BZ_QA_ANSWERS_FILE} );
-    run( 'perl', 'scripts/generate_bmo_data.pl',
-        '--user-pref', 'ui_experiments=off' );
-    chdir '/app/qa/config';
-    say 'chdir(/app/qa/config)';
-    run( 'perl', 'generate_test_data.pl' );
+
+    if ($ENV{BZ_QA_LEGACY_MODE}) {
+        run( 'perl', 'scripts/generate_bmo_data.pl',
+            '--user-pref', 'ui_experiments=off' );
+        chdir '/app/qa/config';
+        say 'chdir(/app/qa/config)';
+        run( 'perl', 'generate_test_data.pl' );
+    }
+    else {
+        run( 'perl', 'scripts/generate_bmo_data.pl' );
+    }
 }
 
 sub cmd_test_heartbeat {
-    my $conf = require $ENV{BZ_QA_CONF_FILE};
-    wait_for_httpd($conf->{browser_url});
-    my $heartbeat = get("$conf->{browser_url}/__heartbeat__");
+    my ($url) = @_;
+    die "test_heartbeat requires a url!\n" unless $url;
+
+    wait_for_httpd($url);
+    my $heartbeat = get("$url/__heartbeat__");
     if ($heartbeat && $heartbeat =~ /Bugzilla OK/) {
         exit 0;
     }
@@ -59,33 +72,152 @@ sub cmd_test_heartbeat {
 }
 
 sub cmd_test_webservices {
+
     my $conf = require $ENV{BZ_QA_CONF_FILE};
 
     check_data_dir();
-    wait_for_db();
-    wait_for_httpd($conf->{browser_url});
-    copy_qa_extension();
+    my @httpd_cmd = ( '/usr/sbin/httpd', '-DFOREGROUND', '-f', '/app/httpd/httpd.conf' );
+    if ($ENV{BZ_QA_LEGACY_MODE}) {
+        copy_qa_extension();
+        push @httpd_cmd, '-DHTTPD_IN_SUBDIR';
+    }
 
-    chdir('/app/qa/t');
-    run( 'prove', '-qf', '-I/app', '-I/app/local/lib/perl5', glob('webservice_*.t') );
+    prove_with_httpd(
+        httpd_url => $conf->{browser_url},
+        httpd_cmd => \@httpd_cmd,
+        prove_cmd => [
+            'prove', '-qf', '-I/app',
+            '-I/app/local/lib/perl5',
+            sub { glob('webservice_*.t') },
+        ],
+        prove_dir => '/app/qa/t',
+    );
 }
 
 sub cmd_test_selenium {
     my $conf = require $ENV{BZ_QA_CONF_FILE};
 
     check_data_dir();
-    wait_for_db();
-    wait_for_httpd($conf->{browser_url});
-    copy_qa_extension();
+    my @httpd_cmd = ( '/usr/sbin/httpd', '-DFOREGROUND', '-f', '/app/httpd/httpd.conf' );
+    if ($ENV{BZ_QA_LEGACY_MODE}) {
+        copy_qa_extension();
+        push @httpd_cmd, '-DHTTPD_IN_SUBDIR';
+    }
 
-    chdir('/app/qa/t');
-    run( 'prove', '-qf', '-Ilib', '-I/app', '-I/app/local/lib/perl5', glob('test_*.t') );
+    prove_with_httpd(
+        httpd_url => $conf->{browser_url},
+        httpd_cmd => \@httpd_cmd,
+        prove_cmd => [
+            'prove', '-qf', '-Ilib', '-I/app',
+            '-I/app/local/lib/perl5',
+            sub { glob('test_*.t') }
+        ],
+        prove_dir => '/app/qa/t',
+    );
 }
 
+sub cmd_shell   { run( 'bash',  '-l' ); }
+sub cmd_prove   { run( "prove", "-I/app", "-I/app/local/lib/perl5", @_ ); }
+sub cmd_version { run( 'cat',   '/app/version.json' ); }
+
+sub cmd_test_bmo {
+    prove_with_httpd(
+        httpd_url => $ENV{BZ_BASE_URL},
+        httpd_cmd => [ '/usr/sbin/httpd', '-f', '/app/httpd/httpd.conf',  '-DFOREGROUND' ],
+        prove_cmd => [ "prove", "-I/app", "-I/app/local/lib/perl5", @_ ],
+    );
+}
+
+sub prove_with_httpd {
+    my (%param) = @_;
+
+    check_data_dir();
+    wait_for_db();
+
+    unless (-d "/app/logs") {
+        mkdir("/app/logs") or die "unable to mkdir(/app/logs): $!\n";
+    }
+
+    my $httpd_cmd = $param{httpd_cmd};
+    my $prove_cmd = $param{prove_cmd};
+
+    my $loop = IO::Async::Loop->new;
+
+    my $httpd_exit_f = $loop->new_future;
+    warn "starting httpd\n";
+    my $httpd = IO::Async::Process->new(
+        code => sub {
+            setsid();
+            exec(@$httpd_cmd);
+        },
+        setup => [
+             stdout => ["open", ">", "/app/logs/access.log"],
+             stderr => ["open", ">", "/app/logs/error.log"],
+        ],
+        on_finish => on_finish($httpd_exit_f),
+        on_exception => on_exception('httpd', $httpd_exit_f),
+    );
+    $loop->add($httpd);
+    wait_for_httpd( $httpd, $param{httpd_url} );
+
+    warn "httpd started, starting prove\n";
+
+    my $prove_exit_f = $loop->new_future;
+    my $prove = IO::Async::Process->new(
+        code => sub {
+            chdir($param{prove_dir}) if $param{prove_dir};
+            my @cmd = (map { ref $_ eq 'CODE' ? $_->() : $_ } @$prove_cmd);
+            warn "run @cmd\n";
+            exec(@cmd);
+        },
+        on_finish    => on_finish($prove_exit_f),
+        on_exception => on_exception('prove', $prove_exit_f),
+    );
+    $loop->add($prove);
 
-sub cmd_shell   { run( 'bash', '-l' ); }
+    my $prove_exit = $prove_exit_f->get();
+    if ($httpd->is_running) {
+        $httpd->kill('TERM');
+        my $httpd_exit = $httpd_exit_f->get();
+        warn "httpd exit code: $httpd_exit\n" if $httpd_exit != 0;
+    }
+
+    exit $prove_exit;
+}
 
-sub cmd_version { run( 'cat', '/app/version.json' ); }
+sub wait_for_httpd {
+    my ($process, $url) = @_;
+    my $loop = IO::Async::Loop->new;
+    my $is_running_f = $loop->new_future;
+    my $ticks = 0;
+    my $run_checker = IO::Async::Timer::Periodic->new(
+        first_interval => 0,
+        interval       => 1,
+        reschedule     => 'hard',
+        on_tick        => sub {
+            my ($timer) = @_;
+            if ( $process->is_running ) {
+                my $resp = get("$url/__lbheartbeat__");
+                if ($resp && $resp =~ /^httpd OK$/) {
+                    $timer->stop;
+                    $is_running_f->done($resp);
+                }
+                say "httpd doesn't seem to be up at $url. waiting...";
+            }
+            elsif ( $process->is_exited ) {
+                $timer->stop;
+                $is_running_f->fail("process exited early");
+            }
+            elsif ( $ticks++ > 60 ) {
+                $timer->stop;
+                $is_running_f->fail("is_running_future() timeout after $ticks seconds");
+            }
+            $timer->stop if $ticks++ > 60;
+        },
+    );
+    $loop->add($run_checker->start);
+    return $is_running_f->get();
+}
 
 sub copy_qa_extension {
     say "copying the QA extension...";
@@ -93,8 +225,6 @@ sub copy_qa_extension {
 }
 
 sub wait_for_db {
-    die "/app/localconfig is missing\n" unless -f "/app/localconfig";
-
     my $c = Bugzilla::Install::Localconfig::read_localconfig();
     for my $var (qw(db_name db_host db_user db_pass)) {
         die "$var is not set!" unless $c->{$var};
@@ -117,20 +247,29 @@ sub wait_for_db {
     die "unable to connect to $dsn as $c->{db_user}\n" unless $dbh;
 }
 
-sub wait_for_httpd {
-    my ($url) = @_;
-    my $ok = 0;
-    foreach (1..12) {
-        say 'checking if httpd is up...' if $_ > 1;
-        my $resp = get("$url/__lbheartbeat__");
-        if ($resp && $resp =~ /^httpd OK$/) {
-            $ok = 1;
-            last;
+sub on_exception {
+    my ($name, $f) = @_;
+    return sub {
+        my ( $self, $exception, $errno, $exitcode ) = @_;
+
+        if ( length $exception ) {
+            $f->fail("$name died with the exception $exception " . "(errno was $errno)\n");
         }
-        say "httpd doesn't seem to be up at $url. waiting...";
-        sleep(10);
-    }
-    die "unable to connect to httpd at $url\n" unless $ok;
+        elsif ( ( my $status = WEXITSTATUS($exitcode) ) == 255 ) {
+            $f->fail("$name failed to exec() - $errno\n");
+        }
+        else {
+            $f->fail("$name exited with exit status $status\n");
+        }
+    };
+}
+
+sub on_finish {
+    my ($f) = @_;
+    return sub {
+        my ($self, $exitcode) = @_;
+        $f->done(WEXITSTATUS($exitcode));
+    };
 }
 
 sub localconfig_from_env {
@@ -165,6 +304,8 @@ sub write_localconfig {
 
     my $filename = "/app/localconfig";
 
+    die "/app/localconfig already exists!" if -f $filename;
+
     foreach my $var (Bugzilla::Install::Localconfig::LOCALCONFIG_VARS) {
         my $name = $var->{name};
         my $value = $localconfig->{$name};
@@ -174,8 +315,6 @@ sub write_localconfig {
         }
     }
 
-    unlink($filename);
-
     # Ensure output is sorted and deterministic
     local $Data::Dumper::Sortkeys = 1;