Bug 1449413 - Refactor circleci container building stuff

4 files changed, 276 insertions, 26 deletions

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 6b67abaeb..6f14799a9 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -11,8 +11,6 @@ main_filters: &main_filters
       - /^(?:release|test)-20\d\d\d\d\d\d\.\d+/
       - /\//
       - production
-  tags:
-    only: /^(?:release|test)-20\d\d\d\d\d\d\.\d+/
 
 defaults:
   bmo_slim_image: &bmo_slim_image
@@ -69,6 +67,41 @@ defaults:
         mkdir artifacts
 
 jobs:
+  build_info:
+    parallelism: 1
+    working_directory: /app
+    docker:
+      - <<: *bmo_slim_image
+        environment:
+          <<: *bmo_env
+    steps:
+      - checkout
+      - run:
+          name: build push data
+          command: |
+            mv /opt/bmo/local /app/local
+            perl checksetup.pl --no-database --no-templates --no-permissions
+            perl scripts/build-bmo-push-data.pl
+      - run:
+          name: only publish if tag exists
+          command: |
+            tag="$(cat build_info/tag.txt)"
+            git fetch --tags
+            if git tag | fgrep -q "$tag"; then
+              echo "tag $tag exists!"
+              if [[ $CIRCLE_BRANCH == "master" ]]; then
+                exit 1
+              fi
+            else
+              echo "tag $tag does not exist"
+              echo yes > build_info/publish.txt
+            fi
+      - persist_to_workspace:
+          root: /app/build_info
+          paths: ["*.txt"]
+      - store_artifacts:
+          path: /app/build_info
+
   build:
     working_directory: /app
     docker:
@@ -85,24 +118,30 @@ jobs:
             --build-arg CIRCLE_SHA1="$CIRCLE_SHA1" \
             --build-arg CIRCLE_BUILD_URL="$CIRCLE_BUILD_URL" \
             -t bmo .
+      - attach_workspace:
+          at: /app/build_info
+      - run: "docker run --name bmo --entrypoint true bmo"
+      - run: "docker cp bmo:/app/version.json build_info/version.json"
+      - store_artifacts:
+          path: /app/build_info
       - deploy:
           command: |
-            if [[ -n "$DOCKERHUB_REPO" && -n "$DOCKER_USER" && -n "$DOCKER_PASS" ]]; then
-                TAG=""
-                if [[ -n "$CIRCLE_TAG" ]]; then
-                  TAG="$CIRCLE_TAG"
-                elif [[ "$CIRCLE_BRANCH" == "master" ]]; then
-                  TAG=latest
-                fi
-                if [[ -n "$TAG" ]]; then
-                  docker tag bmo "$DOCKERHUB_REPO:$TAG"
-                  docker login -u "$DOCKER_USER" -p "$DOCKER_PASS"
-                  docker push "$DOCKERHUB_REPO:$TAG"
-                fi
-            fi
+            TAG="$(cat /app/build_info/tag.txt)"
+            [[ "$CIRCLE_BRANCH" == "master"  && -n "$TAG" ]] || exit 0
+            [[ -n "$DOCKERHUB_REPO" && -n "$DOCKER_USER" && -n "$DOCKER_PASS" ]] || exit 0
+            [[ -n "$GITHUB_PERSONAL_TOKEN" ]] || exit 0
+            [[ -f build_info/publish.txt ]] || exit 0
+            git config credential.helper cache
+            git config user.email "$GITHUB_EMAIL"
+            git config user.name "$GITHUB_NAME"
+            git tag $TAG
+            git push https://${GITHUB_PERSONAL_TOKEN}:x-oauth-basic@github.com/$GITHUB_REPO.git $TAG
+            docker tag bmo "$DOCKERHUB_REPO:$TAG"
+            docker login -u "$DOCKER_USER" -p "$DOCKER_PASS"
+            docker push "$DOCKERHUB_REPO:$TAG"
 
   test_sanity:
-    parallelism: 4
+    parallelism: 2
     working_directory: /app
     docker:
       - <<: *bmo_slim_image
@@ -178,23 +217,29 @@ workflows:
   version: 2
   main:
     jobs:
+      - build_info:
+          filters: *main_filters
+      - build:
+          filters: *main_filters
+          requires:
+            - build_info
+            - test_sanity
+            - test_bmo
+            - test_webservices
+            - test_selenium
       - test_sanity:
           filters: *main_filters
+          requires:
+            - build_info
       - test_bmo:
           filters: *main_filters
           requires:
-            - test_sanity
+            - build_info
       - test_webservices:
           filters: *main_filters
           requires:
-            - test_sanity
+            - build_info
       - test_selenium:
           filters: *main_filters
           requires:
-            - test_sanity
-      - build:
-          filters: *main_filters
-          requires:
-            - test_sanity
-            - test_bmo
-            - test_webservices
+            - build_info
diff --git a/.perlcriticrc b/.perlcriticrc
index 0c0d1c9be..44254f64e 100644
--- a/.perlcriticrc
+++ b/.perlcriticrc
@@ -32,12 +32,13 @@ severity = 2
 [-Documentation::RequirePodLinksIncludeText]
 [-Documentation::RequirePodSections]
 [-ErrorHandling::RequireCarping]
+[-InputOutput::RequireBracedFileHandleWithPrint]
 [-Modules::RequireVersionVar]
 [-References::ProhibitDoubleSigils]
 [-RegularExpressions::ProhibitComplexRegexes]
 [-RegularExpressions::RequireDotMatchAnything]
-[-RegularExpressions::RequireLineBoundaryMatching]
 [-RegularExpressions::RequireExtendedFormatting]
+[-RegularExpressions::RequireLineBoundaryMatching]
 [-Subroutines::ProhibitExcessComplexity]
 [-ValuesAndExpressions::ProhibitConstantPragma]
 [-ValuesAndExpressions::ProhibitEmptyQuotes]
diff --git a/Makefile.PL b/Makefile.PL
index f7b62ea5c..9f56cd487 100755
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -50,6 +50,7 @@ my %requires = (
     'File::Slurp'              => '9999.13',
     'Future'                   => '0.34',
     'HTML::Escape'             => '1.10',
+    'IPC::System::Simple'      => 0,
     'IO::Async'                => '0.71',
     'JSON::MaybeXS'            => '1.003008',
     'JSON::XS'                 => '2.01',
diff --git a/scripts/build-bmo-push-data.pl b/scripts/build-bmo-push-data.pl
new file mode 100755
index 000000000..e3cefe533
--- /dev/null
+++ b/scripts/build-bmo-push-data.pl
@@ -0,0 +1,203 @@
+#!/usr/bin/perl
+use 5.10.1;
+use strict;
+use warnings;
+
+use File::Basename qw(basename dirname);
+use File::Spec::Functions qw(catdir rel2abs);
+use Cwd qw(realpath);
+
+BEGIN {
+    require lib;
+    my $dir = realpath( catdir(dirname(__FILE__), '..') );
+    lib->import( $dir, catdir( $dir, 'lib' ), catdir( $dir, qw(local lib perl5) ) );
+    chdir $dir or die "chdir $dir failed: $!";
+}
+
+use autodie;
+use Bugzilla;
+use English qw(-no_match_vars $PROGRAM_NAME);
+use IPC::System::Simple qw(runx capture);
+use JSON::MaybeXS qw(decode_json);
+use LWP::Simple qw(get);
+use LWP::UserAgent;
+use MIME::Base64 qw(decode_base64);
+use URI::QueryParam;
+use URI;
+
+my $github_repo  = "https://github.com/mozilla-bteam/bmo";
+my $version_info = decode_json(get('https://bugzilla.mozilla.org/__version__'));
+my $tag          = 'release-' . Bugzilla->VERSION;
+my $prod_tag     = "release-$version_info->{version}";
+my $tag_url      = "$github_repo/tree/$tag";
+
+my @log = capture(qw(git log --oneline), "$prod_tag..HEAD");
+die "nothing to commit\n" unless @log;
+chomp @log;
+
+my @revisions;
+foreach my $line (@log) {
+    say $line;
+    my ($revision, $message);
+    unless ( ( $revision, $message ) = $line =~ /^(\S+) (.+)$/ ) {
+        warn "skipping $line\n";
+        next;
+    }
+
+    my @bug_ids;
+    if ($message =~ /\bBug (\d+)/i) {
+        push @bug_ids, $1;
+    }
+
+    if (!@bug_ids) {
+        warn "skipping $line (no bug)\n";
+        next;
+    }
+
+    foreach my $bug_id (@bug_ids) {
+        my $duplicate = 0;
+        foreach my $revisions (@revisions) {
+            if ($revisions->{bug_id} == $bug_id) {
+                $duplicate = 1;
+                last;
+            }
+        }
+        next if $duplicate;
+
+        my $bug = fetch_bug($bug_id);
+        if ($bug->{status} eq 'RESOLVED' && $bug->{resolution} ne 'FIXED') {
+            next;
+        }
+        if ($bug->{summary} =~ /\bbackport\s+(?:upstream\s+)?bug\s+(\d+)/i) {
+            my $upstream = $1;
+            $bug->{summary} = fetch_bug($upstream)->{summary};
+        }
+        push @revisions, {
+            hash    => $revision,
+            bug_id  => $bug_id,
+            summary => $bug->{summary},
+        };
+    }
+}
+if (!@revisions) {
+    die "no new revisions.  make sure you run this script before production is updated.\n";
+}
+else {
+    @revisions = reverse @revisions;
+}
+
+my $first_revision = $revisions[0]->{hash};
+my $last_revision  = $revisions[-1]->{hash};
+
+mkdir 'build_info' unless -d 'build_info';
+chdir 'build_info';
+
+say "write tag.txt";
+open my $tag_fh, '>', 'tag.txt';
+say $tag_fh $tag;
+close $tag_fh;
+
+say 'write bug.push.txt';
+
+open my $bug_fh, '>', 'bug.push.txt';
+say $bug_fh 'https://bugzilla.mozilla.org/enter_bug.cgi?product=bugzilla.mozilla.org&component=Infrastructure&short_desc=push+updated+bugzilla.mozilla.org+live';
+say $bug_fh "revisions: $first_revision - $last_revision";
+foreach my $revision (@revisions) {
+    say $bug_fh "bug $revision->{bug_id} : $revision->{summary}";
+}
+close $bug_fh;
+
+say 'write blog.push.txt';
+
+open my $blog_fh, '>', 'blog.push.txt';
+say $blog_fh "[release tag]($tag_url)\n";
+say $blog_fh "the following changes have been pushed to bugzilla.mozilla.org:\n<ul>";
+foreach my $revision (@revisions) {
+    printf $blog_fh '<li>[<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=%s" target="_blank">%s</a>] %s</li>%s',
+        $revision->{bug_id}, $revision->{bug_id}, html_escape($revision->{summary}), "\n";
+}
+say $blog_fh '</ul>';
+say $blog_fh q{discuss these changes on <a href="https://lists.mozilla.org/listinfo/tools-bmo" target="_blank">mozilla.tools.bmo</a>.};
+close $blog_fh;
+
+say 'write email.push.txt';
+
+open my $email_fh, '>', 'email.push.txt';
+say $email_fh "the following changes have been pushed to bugzilla.mozilla.org:\n";
+say $email_fh "(tag: $tag_url)\n";
+foreach my $revision (@revisions) {
+    printf $email_fh "https://bugzil.la/%s : %s\n", $revision->{bug_id}, $revision->{summary};
+}
+close $email_fh;
+
+say 'write wiki.push.txt';
+
+open my $wiki_fh, '>', 'wiki.push.txt';
+say $wiki_fh 'https://wiki.mozilla.org/BMO/Recent_Changes';
+say $wiki_fh '== ' . DateTime->now->set_time_zone('UTC')->ymd('-') . " ==\n";
+say $wiki_fh "[$tag_url $tag]";
+foreach my $revision (@revisions) {
+    printf $wiki_fh "* {{bug|%s}} %s\n", $revision->{bug_id}, $revision->{summary};
+}
+close $wiki_fh;
+
+sub html_escape {
+    my ($s) = @_;
+    $s =~ s/&/&amp;/g;
+    $s =~ s/</&lt;/g;
+    $s =~ s/>/&gt;/g;
+    return $s;
+}
+
+use constant BUG_FIELDS => [qw(
+    id
+    product
+    version
+    target_milestone
+    summary
+    status
+    resolution
+    assigned_to
+)];
+
+sub fetch_bug {
+    my ($bug_id) = @_;
+    die 'missing id' unless $bug_id;
+
+    my $response = _get( 'bug/' . $bug_id, { include_fields => BUG_FIELDS, } );
+    return $response->{bugs}->[0];
+}
+
+sub _get {
+    my ($endpoint, $args) = @_;
+    my $ua = LWP::UserAgent->new( agent => $PROGRAM_NAME );
+    $args //= {};
+
+    if (exists $args->{include_fields} && ref($args->{include_fields})) {
+        $args->{include_fields} = join ',', @{ $args->{include_fields} };
+    }
+
+    my $uri = URI->new('https://bugzilla.mozilla.org/rest/' . $endpoint);
+    foreach my $name (sort keys %$args) {
+        $uri->query_param($name => $args->{$name});
+    }
+
+    my $request = HTTP::Request->new('GET', $uri->as_string);
+    $request->header( Content_Type       => 'application/json' );
+    $request->header( Accept             => 'application/json' );
+    if ( $ENV{BMO_API_KEY} ) {
+        $request->header( X_Bugzilla_API_Key => $ENV{BMO_API_KEY} );
+    }
+
+    my $response = $ua->request($request);
+    if ($response->code !~ /^2/) {
+        my $error = $response->message;
+        my $ok = eval {
+            $error = decode_json($response->decoded_content)->{message};
+            1;
+        };
+        $error = $@ unless $ok;
+        die $error . "\n";
+    }
+    return decode_json($response->decoded_content);
+}