From 4663032035039cc642db9b3f82b289418d02c430 Mon Sep 17 00:00:00 2001 From: Dylan William Hardison Date: Mon, 7 Aug 2017 13:35:47 -0400 Subject: Bug 1383355 - Migrate CI tests from taskcluster to CircleCI --- .circleci/checksetup_answers.txt | 11 ++ .circleci/config.yml | 104 ++++++++++++++++- .circleci/deploy.pl | 52 +++++++++ .circleci/selenium_test.conf | 49 ++++++++ .dockerignore | 5 +- Bugzilla/Install/Filesystem.pm | 4 + Dockerfile | 32 ++---- docker_files/httpd.conf | 98 ---------------- docker_files/init.pl | 80 ------------- httpd/httpd.conf | 101 +++++++++++++++++ qa/config/generate_test_data.pl | 12 +- qa/config/selenium_test.conf | 2 +- qa/t/lib/QA/Util.pm | 26 ++++- qa/t/test_flags.t | 8 +- qa/t/test_flags2.t | 2 +- qa/t/test_private_attachments.t | 6 +- qa/t/test_security.t | 4 +- scripts/entrypoint.pl | 234 +++++++++++++++++++++++++++++++++++++++ 18 files changed, 613 insertions(+), 217 deletions(-) create mode 100644 .circleci/checksetup_answers.txt create mode 100755 .circleci/deploy.pl create mode 100644 .circleci/selenium_test.conf delete mode 100644 docker_files/httpd.conf delete mode 100755 docker_files/init.pl create mode 100644 httpd/httpd.conf create mode 100755 scripts/entrypoint.pl diff --git a/.circleci/checksetup_answers.txt b/.circleci/checksetup_answers.txt new file mode 100644 index 000000000..6bcdd2dcc --- /dev/null +++ b/.circleci/checksetup_answers.txt @@ -0,0 +1,11 @@ +$answer{'ADMIN_EMAIL'} = 'admin@mozilla.bugs'; +$answer{'ADMIN_OK'} = 'Y'; +$answer{'ADMIN_PASSWORD'} = 'password'; +$answer{'ADMIN_REALNAME'} = 'QA Admin'; +$answer{'NO_PAUSE'} = 1; +$answer{'bugzilla_version'} = '1'; +$answer{'create_htaccess'} = ''; +$answer{'cvsbin'} = '/usr/bin/cvs'; +$answer{'diffpath'} = '/usr/bin'; +$answer{'interdiffbin'} = '/usr/bin/interdiff'; +$answer{'urlbase'} = 'http://<>:8000/bmo/'; diff --git a/.circleci/config.yml b/.circleci/config.yml index 18d282917..b4fbe1874 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,13 +4,49 @@ # version: 2 + +test_docker: &test_docker + - image: mozillabteam/bmo-slim:20170803.1 + user: app + environment: + PORT: 8000 + BMO_db_user: bugs + BMO_db_host: 127.0.0.1 + BMO_db_pass: bugs + BMO_db_name: bugs + BMO_memcached_servers: localhost:11211 + BMO_memcached_namespace: "bugzilla:" + BZ_QA_CONF_FILE: /app/.circleci/selenium_test.conf + BZ_QA_ANSWERS_FILE: /app/.circleci/checksetup_answers.txt + - image: mozillabteam/bmo-mysql:5.6 + environment: + MYSQL_DATABASE: bugs + MYSQL_USER: bugs + MYSQL_PASSWORD: bugs + MYSQL_ALLOW_EMPTY_PASSWORD: 1 + - image: selenium/standalone-firefox:2.53.1 + - image: memcached:latest + +default_setup: &default_setup + run: + command: | + mv /opt/bmo/local /app/local + perl -MSys::Hostname -i -pE 's/<>/hostname()/ges' $BZ_QA_CONF_FILE + perl -MSys::Hostname -i -pE 's/<>/hostname()/ges' $BZ_QA_ANSWERS_FILE + perl checksetup.pl --no-database --default-localconfig + mkdir artifacts + +run_qa_httpd: &run_qa_httpd + run: + command: | + /app/scripts/entrypoint.pl qa_httpd &> artifacts/httpd.log + background: true + jobs: build: working_directory: /app docker: - image: docker:17.06.1-ce - environment: - BMO_IMAGE_NAME: mozillabteam/bmo steps: - setup_remote_docker - run: @@ -18,8 +54,68 @@ jobs: command: apk update && apk add git openssh-client - checkout - run: | - docker build -t $BMO_IMAGE_NAME:$CIRCLE_BRANCH . + docker build -t $DOCKERHUB_REPO:latest . if [[ -n "$DOCKER_USER" && -n "$DOCKER_PASS" ]]; then docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" - docker push $BMO_IMAGE_NAME:$CIRCLE_BRANCH + docker push $DOCKERHUB_REPO:latest fi + + test_sanity: + parallelism: 4 + working_directory: /app + docker: + - image: mozillabteam/bmo-slim:20170803.1 + user: app + steps: + - checkout + - *default_setup + - run: + name: run sanity tests + command: | + prove -qf $(circleci tests glob 't/*.t' | circleci tests split) | tee artifacts/$CIRCLE_JOB.txt + - store_artifacts: + path: /app/artifacts + + test_webservices: + parallelism: 1 + working_directory: /app + docker: *test_docker + steps: + - checkout + - *default_setup + - run: /app/scripts/entrypoint.pl load_test_data + - *run_qa_httpd + - run: /app/scripts/entrypoint.pl test_heartbeat + - run: + command: | + /app/scripts/entrypoint.pl test_webservices | tee artifacts/$CIRCLE_JOB.txt + - store_artifacts: + path: /app/artifacts + + test_selenium: + parallelism: 1 + working_directory: /app + docker: *test_docker + steps: + - checkout + - *default_setup + - run: /app/scripts/entrypoint.pl load_test_data + - *run_qa_httpd + - run: + command: | + /app/scripts/entrypoint.pl test_selenium | tee artifacts/$CIRCLE_JOB.txt + - store_artifacts: + path: /app/artifacts + +workflows: + version: 2 + tests: + jobs: + - test_sanity + - test_webservices + - test_selenium + - build: + requires: + - test_sanity + - test_webservices + - test_selenium diff --git a/.circleci/deploy.pl b/.circleci/deploy.pl new file mode 100755 index 000000000..391b9b660 --- /dev/null +++ b/.circleci/deploy.pl @@ -0,0 +1,52 @@ +#!/usr/bin/env perl +use 5.10.1; +use strict; +use warnings; + +my ($repo, $user, $pass) = check_env(qw(DOCKERHUB_REPO DOCKER_USER DOCKER_PASS)); +run("docker", "login", "-u", $user, "-p", $pass); + +my @docker_tags = ($ENV{CIRCLE_SHA1}); + +if ($ENV{CIRCLE_TAG}) { + push @docker_tags, $ENV{CIRCLE_TAG}; +} +elsif ($ENV{CIRCLE_BRANCH}) { + if ($ENV{CIRCLE_BRANCH} eq 'master') { + push @docker_tags, 'latest'; + } + else { + push @docker_tags, $ENV{CIRCLE_BRANCH}; + } +} + +say "Pushing tags..."; +say " $_" for @docker_tags; +foreach my $tag (@docker_tags) { + run("docker", "tag", "bmo", "$repo:$tag"); + run("docker", "push", "$repo:$tag"); +} + +sub run { + my (@cmd) = @_; + my $rv = system(@cmd); + exit 1 if $rv != 0; +} + +sub check_env { + my (@missing, @found); + foreach my $name (@_) { + push @missing, $name unless $ENV{$name}; + push @found, $ENV{$name}; + } + + if (@missing) { + warn "Missing environmental variables: ", join(", ", @missing), "\n"; + exit; + } + return @found; +} + + + + diff --git a/.circleci/selenium_test.conf b/.circleci/selenium_test.conf new file mode 100644 index 000000000..a012ae957 --- /dev/null +++ b/.circleci/selenium_test.conf @@ -0,0 +1,49 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +# To make this configuration file useful to you, you have to: +# - set the path and URL to your Bugzilla installation. +# - replace @mozilla.test by something more relevant to you, +# also what comes before @mozilla.test if you want/need to. +# - set passwords for each user accounts. + +{ 'browser' => '*firefox', + 'experimental_browser_launcher' => '*chrome', + 'host' => 'localhost', + 'port' => 4444, + 'browser_url' => 'http://<>:8000', + 'attachment_file' => 'https://raw.githubusercontent.com/mozilla-bteam/bmo/master/qa/config/patch.diff', + 'bugzilla_installation' => 'bmo', + 'bugzilla_path' => '/app', + 'test_bug_1' => 1, + 'test_bug_2' => 2, + 'admin_user_login' => 'admin@mozilla.test', + 'admin_user_passwd' => 'password', + 'admin_user_username' => 'QA Admin', + 'admin_user_nick' => 'admin', + 'permanent_user' => 'permanent_user@mozilla.test', + 'permanent_user_login' => 'permanent_user@mozilla.test', + 'permanent_user_passwd' => 'password', + 'unprivileged_user_login' => 'no-privs@mozilla.test', + 'unprivileged_user_passwd' => 'password', + 'unprivileged_user_username' => 'no-privs', + 'unprivileged_user_nick' => 'no-privs', + 'unprivileged_user_login_truncated' => 'no-privs@mo', + 'QA_Selenium_TEST_user_login' => 'QA-Selenium-TEST@mozilla.test', + 'QA_Selenium_TEST_user_passwd' => 'password', + 'editbugs_user_login' => 'editbugs@mozilla.test', + 'editbugs_user_passwd' => 'password', + 'canconfirm_user_login' => 'canconfirm@mozilla.test', + 'canconfirm_user_passwd' => 'password', + 'tweakparams_user_login' => 'tweakparams@mozilla.test', + 'tweakparams_user_login_truncated' => 'tweakparams@mo', + 'tweakparams_user_passwd' => 'password', + 'disabled_user_login' => 'disabled@mozilla.test', + 'disabled_user_passwd' => 'password', + 'common_email' => '@mozilla.test', + 'test_extensions' => 1, +}; diff --git a/.dockerignore b/.dockerignore index a2e02b567..49c8cc872 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,11 +1,14 @@ .vagrant +.git MYMETA.* +*.tar Makefile blib local +data +localconfig pm_to_blib template_cache -vagrant_support \#*\# */\#*\# */*/\#*\# diff --git a/Bugzilla/Install/Filesystem.pm b/Bugzilla/Install/Filesystem.pm index 715a06d3a..adc1815c1 100644 --- a/Bugzilla/Install/Filesystem.pm +++ b/Bugzilla/Install/Filesystem.pm @@ -437,6 +437,10 @@ sub FILESYSTEM { contents => HT_DEFAULT_DENY }, 'xt/.htaccess' => { perms => WS_SERVE, contents => HT_DEFAULT_DENY }, + '.circleci/.htaccess' => { perms => WS_SERVE, + contents => HT_DEFAULT_DENY }, + 'httpd/.htaccess' => { perms => WS_SERVE, + contents => HT_DEFAULT_DENY }, "$datadir/.htaccess" => { perms => WS_SERVE, contents => HT_DEFAULT_DENY }, "$error_reports/.htaccess" => { perms => WS_SERVE, diff --git a/Dockerfile b/Dockerfile index f0477b655..d6057775e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,23 @@ -FROM mozillabteam/bmo-base:slim +FROM mozillabteam/bmo-slim:latest MAINTAINER Dylan William Hardison -RUN wget -q https://s3.amazonaws.com/moz-devservices-bmocartons/bmo/vendor.tar.gz && \ - tar -C /opt -zxf /vendor.tar.gz bmo/local/ bmo/LIBS.txt bmo/cpanfile bmo/cpanfile.snapshot && \ - rm /vendor.tar.gz && \ - mkdir /opt/bmo/httpd && \ - ln -s /usr/lib64/httpd/modules /opt/bmo/httpd/modules && \ - mkdir /opt/bmo/httpd/conf && \ - cp {/etc/httpd/conf,/opt/bmo/httpd}/magic && \ - awk '{print $1}' > LIBS.txt \ - | perl -nE 'chomp; unless (-f $_) { $missing++; say $_ } END { exit 1 if $missing }' && \ - useradd -u 10001 -U app -m +ENV BUNDLE=https://s3.amazonaws.com/moz-devservices-bmocartons/bmo/vendor.tar.gz +ENV PORT=8000 -COPY . /app WORKDIR /app -RUN ln -sv /opt/bmo/local /app/local && \ +COPY . . + +RUN mv /opt/bmo/local /app && \ chown -R app:app /app && \ - cp /app/docker_files/httpd.conf /opt/bmo/httpd/ && \ - mkdir /opt/bmo/bin && \ - cp /app/docker_files/init.pl /opt/bmo/bin/init.pl + perl -c /app/scripts/entrypoint.pl USER app -RUN perl checksetup.pl --no-database --default-localconfig && \ - prove t && \ - rm -rf /app/data && mkdir /app/data -ENV PORT=8000 +RUN perl checksetup.pl --no-database --default-localconfig && \ + rm -rf /app/data /app/localconfig && \ + mkdir /app/data EXPOSE $PORT -ENTRYPOINT ["/opt/bmo/bin/init.pl"] +ENTRYPOINT ["/app/scripts/entrypoint.pl"] CMD ["httpd"] diff --git a/docker_files/httpd.conf b/docker_files/httpd.conf deleted file mode 100644 index b8c779052..000000000 --- a/docker_files/httpd.conf +++ /dev/null @@ -1,98 +0,0 @@ -ServerTokens Prod -ServerRoot "/opt/bmo/httpd" -PidFile /tmp/httpd.pid -Timeout 60 -KeepAlive Off -MaxKeepAliveRequests 100 -KeepAliveTimeout 15 - -StartServers 8 -MinSpareServers 5 -MaxSpareServers 20 -ServerLimit 256 -MaxClients 256 -MaxRequestsPerChild 4000 - - -StartServers 4 -MaxClients 300 -MinSpareThreads 25 -MaxSpareThreads 75 -ThreadsPerChild 25 -MaxRequestsPerChild 0 - -Listen ${PORT} -LoadModule authz_host_module modules/mod_authz_host.so -LoadModule include_module modules/mod_include.so -LoadModule log_config_module modules/mod_log_config.so -LoadModule logio_module modules/mod_logio.so -LoadModule env_module modules/mod_env.so -LoadModule ext_filter_module modules/mod_ext_filter.so -LoadModule mime_magic_module modules/mod_mime_magic.so -LoadModule expires_module modules/mod_expires.so -LoadModule deflate_module modules/mod_deflate.so -LoadModule headers_module modules/mod_headers.so -LoadModule usertrack_module modules/mod_usertrack.so -LoadModule setenvif_module modules/mod_setenvif.so -LoadModule mime_module modules/mod_mime.so -LoadModule dav_module modules/mod_dav.so -LoadModule status_module modules/mod_status.so -LoadModule autoindex_module modules/mod_autoindex.so -LoadModule info_module modules/mod_info.so -LoadModule dav_fs_module modules/mod_dav_fs.so -LoadModule vhost_alias_module modules/mod_vhost_alias.so -LoadModule negotiation_module modules/mod_negotiation.so -LoadModule dir_module modules/mod_dir.so -LoadModule actions_module modules/mod_actions.so -LoadModule speling_module modules/mod_speling.so -LoadModule alias_module modules/mod_alias.so -LoadModule substitute_module modules/mod_substitute.so -LoadModule rewrite_module modules/mod_rewrite.so -LoadModule cache_module modules/mod_cache.so -LoadModule disk_cache_module modules/mod_disk_cache.so -LoadModule version_module modules/mod_version.so -LoadModule perl_module modules/mod_perl.so -User app -Group app -ServerAdmin root@localhost -UseCanonicalName Off - - Options FollowSymLinks - AllowOverride None - -AccessFileName .htaccess - - Order allow,deny - Deny from all - Satisfy All - -TypesConfig /etc/mime.types -DefaultType text/plain - - MIMEMagicFile magic - -HostnameLookups Off -ErrorLog /dev/stderr -LogLevel warn -LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined -LogFormat "%h %l %u %t \"%r\" %>s %b" common -LogFormat "%{Referer}i -> %U" referer -LogFormat "%{User-agent}i" agent -CustomLog /dev/stdout combined -ServerSignature Off -AddDefaultCharset UTF-8 -AddType application/x-compress .Z -AddType application/x-gzip .gz .tgz -AddType application/x-x509-ca-cert .crt -AddType application/x-pkcs7-crl .crl - -PerlSwitches -wT -PerlRequire /app/mod_perl.pl -DirectoryIndex index.cgi -DocumentRoot "/app" - - Options -Indexes -FollowSymLinks - AllowOverride None - Order allow,deny - Allow from all - diff --git a/docker_files/init.pl b/docker_files/init.pl deleted file mode 100755 index 5518fd57b..000000000 --- a/docker_files/init.pl +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/perl -use strict; -use warnings; -use lib qw(/app /opt/bmo/local/lib/perl5); -use Getopt::Long qw(:config gnu_getopt); -use Data::Dumper; -use Bugzilla::Install::Localconfig (); -use Bugzilla::Install::Util qw(install_string); - -my %localconfig = (webservergroup => 'app'); - -my %override = ( - 'inbound_proxies' => 1, - 'shadowdb' => 1, - 'shadowdbhost' => 1, - 'shadowdbport' => 1, - 'shadowdbsock' => 1 -); - -# clean env. -foreach my $key (keys %ENV) { - if ($key =~ /^BMO_(.+)$/) { - my $name = $1; - if ($override{$name}) { - $localconfig{param_override}{$name} = delete $ENV{$key}; - } - else { - $localconfig{$name} = delete $ENV{$key}; - } - } -} - -write_localconfig(\%localconfig); -sleep(10); -system('perl', 'checksetup.pl', '--no-templates', '--no-permissions'); - -my $cmd = shift @ARGV or die "usage: init.pl CMD"; -my $method = "run_$cmd"; -__PACKAGE__->$method(); - -sub run_httpd { - exec("/usr/sbin/httpd", "-DFOREGROUND", "-f", "/opt/bmo/httpd/httpd.conf"); -} - -sub run_shell { - exec("/bin/bash", "-l"); -} - -sub write_localconfig { - my ($localconfig) = @_; - no warnings 'once'; - - foreach my $var (Bugzilla::Install::Localconfig::LOCALCONFIG_VARS) { - my $name = $var->{name}; - my $value = $localconfig->{$name}; - if (!defined $value) { - $var->{default} = &{$var->{default}} if ref($var->{default}) eq 'CODE'; - $localconfig->{$name} = $var->{default}; - } - } - - my $filename = "/app/localconfig"; - - # Ensure output is sorted and deterministic - local $Data::Dumper::Sortkeys = 1; - - # Re-write localconfig - open my $fh, ">:utf8", $filename or die "$filename: $!"; - foreach my $var (Bugzilla::Install::Localconfig::LOCALCONFIG_VARS) { - my $name = $var->{name}; - my $desc = install_string("localconfig_$name", { root => Bugzilla::Install::Localconfig::ROOT_USER }); - chomp($desc); - # Make the description into a comment. - $desc =~ s/^/# /mg; - print $fh $desc, "\n", - Data::Dumper->Dump([$localconfig->{$name}], - ["*$name"]), "\n"; - } - close $fh; -} diff --git a/httpd/httpd.conf b/httpd/httpd.conf new file mode 100644 index 000000000..74235eb27 --- /dev/null +++ b/httpd/httpd.conf @@ -0,0 +1,101 @@ +ServerTokens Prod +ServerRoot "/etc/httpd" +PidFile /tmp/httpd.pid +Timeout 60 +KeepAlive Off +MaxKeepAliveRequests 100 +KeepAliveTimeout 15 + +StartServers 8 +MinSpareServers 5 +MaxSpareServers 20 +ServerLimit 256 +MaxClients 256 +MaxRequestsPerChild 4000 + + +StartServers 4 +MaxClients 300 +MinSpareThreads 25 +MaxSpareThreads 75 +ThreadsPerChild 25 +MaxRequestsPerChild 0 + +Listen ${PORT} +LoadModule authz_host_module modules/mod_authz_host.so +LoadModule include_module modules/mod_include.so +LoadModule log_config_module modules/mod_log_config.so +LoadModule logio_module modules/mod_logio.so +LoadModule env_module modules/mod_env.so +LoadModule ext_filter_module modules/mod_ext_filter.so +LoadModule mime_magic_module modules/mod_mime_magic.so +LoadModule expires_module modules/mod_expires.so +LoadModule deflate_module modules/mod_deflate.so +LoadModule headers_module modules/mod_headers.so +LoadModule usertrack_module modules/mod_usertrack.so +LoadModule setenvif_module modules/mod_setenvif.so +LoadModule mime_module modules/mod_mime.so +LoadModule dav_module modules/mod_dav.so +LoadModule status_module modules/mod_status.so +LoadModule autoindex_module modules/mod_autoindex.so +LoadModule info_module modules/mod_info.so +LoadModule dav_fs_module modules/mod_dav_fs.so +LoadModule vhost_alias_module modules/mod_vhost_alias.so +LoadModule negotiation_module modules/mod_negotiation.so +LoadModule dir_module modules/mod_dir.so +LoadModule actions_module modules/mod_actions.so +LoadModule speling_module modules/mod_speling.so +LoadModule alias_module modules/mod_alias.so +LoadModule substitute_module modules/mod_substitute.so +LoadModule rewrite_module modules/mod_rewrite.so +LoadModule cache_module modules/mod_cache.so +LoadModule disk_cache_module modules/mod_disk_cache.so +LoadModule version_module modules/mod_version.so +LoadModule perl_module modules/mod_perl.so +User app +Group app +ServerAdmin root@localhost +UseCanonicalName Off + + Options FollowSymLinks + AllowOverride None + +AccessFileName .htaccess + + Order allow,deny + Deny from all + Satisfy All + +TypesConfig /etc/mime.types +DefaultType text/plain + + MIMEMagicFile conf/magic + +HostnameLookups Off +ErrorLog /dev/stderr +LogLevel warn +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined +LogFormat "%h %l %u %t \"%r\" %>s %b" common +LogFormat "%{Referer}i -> %U" referer +LogFormat "%{User-agent}i" agent +CustomLog /dev/stdout combined +ServerSignature Off +AddDefaultCharset UTF-8 +AddType application/x-compress .Z +AddType application/x-gzip .gz .tgz +AddType application/x-x509-ca-cert .crt +AddType application/x-pkcs7-crl .crl + +PerlSwitches -wT +PerlRequire /app/mod_perl.pl +DirectoryIndex index.cgi +DocumentRoot "/app" + +Alias "/bmo" "/app" + + + Options -Indexes -FollowSymLinks + AllowOverride None + Order allow,deny + Allow from all + diff --git a/qa/config/generate_test_data.pl b/qa/config/generate_test_data.pl index 9ba851113..62daef772 100644 --- a/qa/config/generate_test_data.pl +++ b/qa/config/generate_test_data.pl @@ -12,6 +12,16 @@ use strict; use warnings; +use File::Basename; +use File::Spec; +BEGIN { + require lib; + my $dir = File::Spec->rel2abs( + File::Spec->catdir(dirname(__FILE__), "..", "..") + ); + lib->import($dir, File::Spec->catdir($dir, "lib"), File::Spec->catdir($dir, qw(local lib perl5))); +} + use Cwd; use File::Copy::Recursive qw(dircopy); @@ -20,7 +30,7 @@ my $config; BEGIN { print "reading the config file...\n"; - my $conf_file = "selenium_test.conf"; + my $conf_file = $ENV{BZ_QA_CONF_FILE} // "selenium_test.conf"; if (@ARGV) { $conf_file = shift @ARGV; } diff --git a/qa/config/selenium_test.conf b/qa/config/selenium_test.conf index b6a795b78..2a163d5f0 100644 --- a/qa/config/selenium_test.conf +++ b/qa/config/selenium_test.conf @@ -16,7 +16,7 @@ 'host' => 'localhost', 'port' => 4444, 'browser_url' => 'http://localhost', - 'attachment_file' => '/var/www/html/bmo/qa/config/patch.diff', + 'attachment_file' => 'https://raw.githubusercontent.com/mozilla-bteam/bmo/master/qa/config/patch.diff', 'bugzilla_installation' => 'bmo', 'bugzilla_path' => '/var/www/html/bmo', 'test_bug_1' => 1, diff --git a/qa/t/lib/QA/Util.pm b/qa/t/lib/QA/Util.pm index 1ff37a0d7..4999e6f3b 100644 --- a/qa/t/lib/QA/Util.pm +++ b/qa/t/lib/QA/Util.pm @@ -13,7 +13,11 @@ use strict; use Data::Dumper; use Test::More; use Test::WWW::Selenium; +use MIME::Base64 qw(decode_base64); +use Sys::Hostname qw(hostname); +use Socket qw(inet_ntoa); use WWW::Selenium::Util qw(server_is_running); +use URI; # Fixes wide character warnings BEGIN { @@ -42,6 +46,7 @@ use base qw(Exporter); add_product open_advanced_search_page set_parameters + screenshot_page get_selenium get_rpc_clients @@ -52,7 +57,7 @@ use base qw(Exporter); # How long we wait for pages to load. use constant WAIT_TIME => 60000; -use constant CONF_FILE => "../config/selenium_test.conf"; +use constant CONF_FILE => $ENV{BZ_QA_CONF_FILE} // "../config/selenium_test.conf"; use constant CHROME_MODE => 1; use constant NDASH => chr(0x2013); @@ -92,6 +97,16 @@ sub get_config { my $conf_file = CONF_FILE; my $config = do($conf_file) or die "can't read configuration '$conf_file': $!$@"; + my $uri = URI->new($config->{browser_url}); + if (my $ip_packed = gethostbyname($uri->host)) { + my $ip = inet_ntoa($ip_packed); + $uri->host($ip); + $config->{browser_ip_url} = "$uri"; + } + else { + die "unable to find ip for $config->{browser_url}\n"; + } + return $config; } sub get_selenium { @@ -148,9 +163,18 @@ sub get_rpc_clients { sub go_to_home { my ($sel, $config) = @_; $sel->open_ok("/$config->{bugzilla_installation}/", undef, "Go to the home page"); + $sel->set_speed(500); $sel->title_is("Bugzilla Main Page"); } +sub screenshot_page { + my ($sel, $filename) = @_; + open my $fh, '>:raw', $filename or die "unable to write $filename: $!"; + binmode $fh; + print $fh decode_base64($sel->capture_entire_page_screenshot_to_string()); + close $fh; +} + # Go to the home/login page and log in. sub log_in { my ($sel, $config, $user) = @_; diff --git a/qa/t/test_flags.t b/qa/t/test_flags.t index 8b7883bb5..e2ba621e6 100644 --- a/qa/t/test_flags.t +++ b/qa/t/test_flags.t @@ -299,7 +299,7 @@ $sel->title_like(qr/^$bug1_id /); $sel->click_ok("link=Add an attachment"); $sel->wait_for_page_to_load_ok(WAIT_TIME); $sel->title_is("Create New Attachment for Bug #$bug1_id"); -$sel->type_ok("data", $config->{attachment_file}); +$sel->attach_file("data", $config->{attachment_file}); $sel->type_ok("description", "patch, v1"); $sel->check_ok("ispatch"); $sel->is_text_present_ok("SeleniumAttachmentFlag1Test"); @@ -326,7 +326,7 @@ my $attachment1_id = $1; $sel->click_ok("//a[contains(text(),'Create\n Another Attachment to Bug $bug1_id')]"); $sel->wait_for_page_to_load_ok(WAIT_TIME); $sel->title_is("Create New Attachment for Bug #$bug1_id"); -$sel->type_ok("data", $config->{attachment_file}); +$sel->attach_file("data", $config->{attachment_file}); $sel->type_ok("description", "patch, v2"); $sel->check_ok("ispatch"); # Mark the previous attachment as obsolete. @@ -350,7 +350,7 @@ my $attachment2_id = $1; $sel->click_ok("//a[contains(text(),'Create\n Another Attachment to Bug $bug1_id')]"); $sel->wait_for_page_to_load_ok(WAIT_TIME); $sel->title_is("Create New Attachment for Bug #$bug1_id"); -$sel->type_ok("data", $config->{attachment_file}); +$sel->attach_file("data", $config->{attachment_file}); $sel->type_ok("description", "patch, v3"); $sel->click_ok("list"); $sel->select_ok("contenttypeselection", "label=plain text (text/plain)"); @@ -423,7 +423,7 @@ $sel->title_like(qr/^$bug1_id/); $sel->click_ok("link=Add an attachment"); $sel->wait_for_page_to_load_ok(WAIT_TIME); $sel->title_is("Create New Attachment for Bug #$bug1_id"); -$sel->type_ok("data", $config->{attachment_file}); +$sel->attach_file("data", $config->{attachment_file}); $sel->type_ok("description", "patch, v4"); $sel->value_is("ispatch", "on"); diff --git a/qa/t/test_flags2.t b/qa/t/test_flags2.t index cec9ee6ef..3d2d59db8 100644 --- a/qa/t/test_flags2.t +++ b/qa/t/test_flags2.t @@ -150,7 +150,7 @@ $sel->select_ok("flag_type-$flagtype1_id", "label=+"); $sel->type_ok("short_desc", "The selenium flag should be kept on product change"); $sel->type_ok("comment", "pom"); $sel->click_ok('//input[@value="Add an attachment"]'); -$sel->type_ok("data", $config->{attachment_file}); +$sel->attach_file("data", $config->{attachment_file}); $sel->type_ok("description", "small patch"); $sel->value_is("ispatch", "on"); ok(!$sel->is_element_present("flag_type-$aflagtype1_id"), "Flag type $aflagtype1_id not available in TestProduct"); diff --git a/qa/t/test_private_attachments.t b/qa/t/test_private_attachments.t index 6126974d7..c6b6df5a1 100644 --- a/qa/t/test_private_attachments.t +++ b/qa/t/test_private_attachments.t @@ -33,7 +33,7 @@ $sel->type_ok("short_desc", "Some comments are private"); $sel->type_ok("comment", "and some attachments too, like this one."); $sel->check_ok("comment_is_private"); $sel->click_ok('//input[@value="Add an attachment"]'); -$sel->type_ok("data", $config->{attachment_file}); +$sel->attach_file("data", $config->{attachment_file}); $sel->type_ok("description", "private attachment, v1"); $sel->check_ok("ispatch"); $sel->click_ok("commit"); @@ -49,7 +49,7 @@ $sel->is_checked_ok('//a[@id="comment_link_0"]/../..//div//input[@type="checkbox $sel->click_ok("link=Add an attachment"); $sel->wait_for_page_to_load_ok(WAIT_TIME); $sel->title_is("Create New Attachment for Bug #$bug1_id"); -$sel->type_ok("data", $config->{attachment_file}); +$sel->attach_file("data", $config->{attachment_file}); $sel->type_ok("description", "public attachment, v2"); $sel->check_ok("ispatch"); # The existing attachment name must be displayed, to mark it as obsolete. @@ -109,7 +109,7 @@ $sel->is_text_present_ok("This attachment is not mine"); $sel->click_ok("link=Add an attachment"); $sel->wait_for_page_to_load_ok(WAIT_TIME); $sel->title_is("Create New Attachment for Bug #$bug1_id"); -$sel->type_ok("data", $config->{attachment_file}); +$sel->attach_file("data", $config->{attachment_file}); $sel->check_ok("ispatch"); # The user doesn't have editbugs privs. $sel->is_text_present_ok("[no attachments can be made obsolete]"); diff --git a/qa/t/test_security.t b/qa/t/test_security.t index 6d545ffd5..56fba4b01 100644 --- a/qa/t/test_security.t +++ b/qa/t/test_security.t @@ -24,7 +24,7 @@ file_bug_in_product($sel, "TestProduct"); my $bug_summary = "Security checks"; $sel->type_ok("short_desc", $bug_summary); $sel->type_ok("comment", "This bug will be used to test security fixes."); -$sel->type_ok("data", $config->{attachment_file}); +$sel->attach_file("data", $config->{attachment_file}); $sel->type_ok("description", "simple patch, v1"); my $bug1_id = create_bug($sel, $bug_summary); @@ -54,7 +54,7 @@ $sel->title_like(qr/^$bug1_id /); # Alternate host for attachments; no cookie should be accessible. set_parameters($sel, { "Attachments" => {"attachment_base" => {type => "text", - value => "http://127.0.0.1/$urlbase/"}} }); + value => "$config->{browser_ip_url}/$urlbase/"}} }); go_to_bug($sel, $bug1_id); $sel->click_ok("link=simple patch, v1"); $sel->wait_for_page_to_load_ok(WAIT_TIME); diff --git a/scripts/entrypoint.pl b/scripts/entrypoint.pl new file mode 100755 index 000000000..b34384ff1 --- /dev/null +++ b/scripts/entrypoint.pl @@ -0,0 +1,234 @@ +#!/usr/bin/perl +use 5.10.1; +use strict; +use warnings; +use lib qw(/app /app/local/lib/perl5); +use Bugzilla::Install::Localconfig (); +use Bugzilla::Install::Util qw(install_string); +use DBI; +use Data::Dumper; +use English qw($EUID); +use File::Copy::Recursive qw(dircopy); +use Getopt::Long qw(:config gnu_getopt); +use LWP::Simple qw(get); +use User::pwent; + +my $cmd = shift @ARGV; +my $func = __PACKAGE__->can("cmd_$cmd") // sub { run($cmd, @ARGV) }; + +fix_path(); +check_user(); +check_env() unless $cmd eq 'shell'; +write_localconfig( localconfig_from_env() ); +$func->(@ARGV); + +sub cmd_httpd { + check_data_dir(); + wait_for_db(); + run( '/usr/sbin/httpd', '-DFOREGROUND', + '-f', '/app/httpd/httpd.conf', @_ ); +} + +sub cmd_qa_httpd { + copy_qa_extension(); + cmd_httpd('-DHTTPD_IN_SUBDIR', @_); +} + +sub cmd_load_test_data { + wait_for_db(); + + die "BZ_QA_ANSWERS_FILE is not set" unless $ENV{BZ_QA_ANSWERS_FILE}; + run( 'perl', 'checksetup.pl', '--no-template', $ENV{BZ_QA_ANSWERS_FILE} ); + run( 'perl', 'scripts/generate_bmo_data.pl', + '--user-pref', 'ui_experiments=off' ); + chdir '/app/qa/config'; + say 'chdir(/app/qa/config)'; + run( 'perl', 'generate_test_data.pl' ); +} + +sub cmd_test_heartbeat { + my $conf = require $ENV{BZ_QA_CONF_FILE}; + wait_for_httpd($conf->{browser_url}); + my $heartbeat = get("$conf->{browser_url}/__heartbeat__"); + if ($heartbeat && $heartbeat =~ /Bugzilla OK/) { + exit 0; + } + else { + exit 1; + } +} + +sub cmd_test_webservices { + my $conf = require $ENV{BZ_QA_CONF_FILE}; + + check_data_dir(); + wait_for_db(); + wait_for_httpd($conf->{browser_url}); + copy_qa_extension(); + + chdir('/app/qa/t'); + run( 'prove', '-qf', '-I/app', '-I/app/local/lib/perl5', glob('webservice_*.t') ); +} + +sub cmd_test_selenium { + my $conf = require $ENV{BZ_QA_CONF_FILE}; + + check_data_dir(); + wait_for_db(); + wait_for_httpd($conf->{browser_url}); + copy_qa_extension(); + + chdir('/app/qa/t'); + run( 'prove', '-qf', '-Ilib', '-I/app', '-I/app/local/lib/perl5', glob('test_*.t') ); +} + + +sub cmd_shell { run( 'bash', '-l' ); } + +sub cmd_version { run( 'cat', '/app/version.json' ); } + +sub copy_qa_extension { + say "copying the QA extension..."; + dircopy('/app/qa/extensions/QA', '/app/extensions/QA'); +} + +sub wait_for_db { + die "/app/localconfig is missing\n" unless -f "/app/localconfig"; + + my $c = Bugzilla::Install::Localconfig::read_localconfig(); + for my $var (qw(db_name db_host db_user db_pass)) { + die "$var is not set!" unless $c->{$var}; + } + + my $dsn = "dbi:mysql:database=$c->{db_name};host=$c->{db_host}"; + my $dbh; + foreach (1..12) { + say "checking database..." if $_ > 1; + $dbh = DBI->connect( + $dsn, + $c->{db_user}, + $c->{db_pass}, + { RaiseError => 0, PrintError => 0 } + ); + last if $dbh; + say "database $dsn not available, waiting..."; + sleep(10); + } + die "unable to connect to $dsn as $c->{db_user}\n" unless $dbh; +} + +sub wait_for_httpd { + my ($url) = @_; + my $ok = 0; + foreach (1..12) { + say 'checking if httpd is up...' if $_ > 1; + my $resp = get("$url/__lbheartbeat__"); + if ($resp && $resp =~ /^httpd OK$/) { + $ok = 1; + last; + } + say "httpd doesn't seem to be up at $url. waiting..."; + sleep(10); + } + die "unable to connect to httpd at $url\n" unless $ok; +} + +sub localconfig_from_env { + my %localconfig = ( webservergroup => 'app' ); + + my %override = ( + 'inbound_proxies' => 1, + 'shadowdb' => 1, + 'shadowdbhost' => 1, + 'shadowdbport' => 1, + 'shadowdbsock' => 1 + ); + + foreach my $key ( keys %ENV ) { + if ( $key =~ /^BMO_(.+)$/ ) { + my $name = $1; + if ( $override{$name} ) { + $localconfig{param_override}{$name} = delete $ENV{$key}; + } + else { + $localconfig{$name} = delete $ENV{$key}; + } + } + } + + return \%localconfig; +} + +sub write_localconfig { + my ($localconfig) = @_; + no warnings 'once'; + + my $filename = "/app/localconfig"; + + foreach my $var (Bugzilla::Install::Localconfig::LOCALCONFIG_VARS) { + my $name = $var->{name}; + my $value = $localconfig->{$name}; + if (!defined $value) { + $var->{default} = &{$var->{default}} if ref($var->{default}) eq 'CODE'; + $localconfig->{$name} = $var->{default}; + } + } + + unlink($filename); + + # Ensure output is sorted and deterministic + local $Data::Dumper::Sortkeys = 1; + + # Re-write localconfig + open my $fh, ">:utf8", $filename or die "$filename: $!"; + foreach my $var (Bugzilla::Install::Localconfig::LOCALCONFIG_VARS) { + my $name = $var->{name}; + my $desc = install_string("localconfig_$name", { root => Bugzilla::Install::Localconfig::ROOT_USER }); + chomp($desc); + # Make the description into a comment. + $desc =~ s/^/# /mg; + print $fh $desc, "\n", + Data::Dumper->Dump([$localconfig->{$name}], + ["*$name"]), "\n"; + } + close $fh; +} + +sub check_user { + die "Effective UID must be 10001!" unless $EUID == 10001; + my $user = getpwuid($EUID)->name; + die "Name of EUID must be app, not $user" unless $user eq 'app'; +} + +sub check_data_dir { + die "/app/data must be writable by user 'app' (id: $EUID)" unless -w "/app/data"; + die "/app/data/params must exist" unless -f "/app/data/params"; +} + +sub check_env { + my @require_env = qw( + BMO_db_host + BMO_db_name + BMO_db_user + BMO_db_pass + BMO_memcached_namespace + BMO_memcached_servers + ); + my @missing_env = grep { not exists $ENV{$_} } @require_env; + if (@missing_env) { + die "Missing required environmental variables: ", join(", ", @missing_env), "\n"; + } +} + +sub fix_path { + $ENV{PATH} = "/app/local/bin:$ENV{PATH}"; +} + +sub run { + my (@cmd) = @_; + say "+ @cmd"; + my $rv = system(@cmd); + if ($rv != 0) { + exit 1; + } +} -- cgit v1.2.3-24-g4f1b