diff options
| -rw-r--r-- | .circleci/config.yml | 95 | ||||
| -rw-r--r-- | .perlcriticrc | 3 | ||||
| -rwxr-xr-x | Makefile.PL | 1 | ||||
| -rwxr-xr-x | scripts/build-bmo-push-data.pl | 203 |
4 files changed, 276 insertions, 26 deletions
diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b67abaeb..6f14799a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,8 +11,6 @@ main_filters: &main_filters - /^(?:release|test)-20\d\d\d\d\d\d\.\d+/ - /\// - production - tags: - only: /^(?:release|test)-20\d\d\d\d\d\d\.\d+/ defaults: bmo_slim_image: &bmo_slim_image @@ -69,6 +67,41 @@ defaults: mkdir artifacts jobs: + build_info: + parallelism: 1 + working_directory: /app + docker: + - <<: *bmo_slim_image + environment: + <<: *bmo_env + steps: + - checkout + - run: + name: build push data + command: | + mv /opt/bmo/local /app/local + perl checksetup.pl --no-database --no-templates --no-permissions + perl scripts/build-bmo-push-data.pl + - run: + name: only publish if tag exists + command: | + tag="$(cat build_info/tag.txt)" + git fetch --tags + if git tag | fgrep -q "$tag"; then + echo "tag $tag exists!" + if [[ $CIRCLE_BRANCH == "master" ]]; then + exit 1 + fi + else + echo "tag $tag does not exist" + echo yes > build_info/publish.txt + fi + - persist_to_workspace: + root: /app/build_info + paths: ["*.txt"] + - store_artifacts: + path: /app/build_info + build: working_directory: /app docker: @@ -85,24 +118,30 @@ jobs: --build-arg CIRCLE_SHA1="$CIRCLE_SHA1" \ --build-arg CIRCLE_BUILD_URL="$CIRCLE_BUILD_URL" \ -t bmo . + - attach_workspace: + at: /app/build_info + - run: "docker run --name bmo --entrypoint true bmo" + - run: "docker cp bmo:/app/version.json build_info/version.json" + - store_artifacts: + path: /app/build_info - deploy: command: | - if [[ -n "$DOCKERHUB_REPO" && -n "$DOCKER_USER" && -n "$DOCKER_PASS" ]]; then - TAG="" - if [[ -n "$CIRCLE_TAG" ]]; then - TAG="$CIRCLE_TAG" - elif [[ "$CIRCLE_BRANCH" == "master" ]]; then - TAG=latest - fi - if [[ -n "$TAG" ]]; then - docker tag bmo "$DOCKERHUB_REPO:$TAG" - docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" - docker push "$DOCKERHUB_REPO:$TAG" - fi - fi + TAG="$(cat /app/build_info/tag.txt)" + [[ "$CIRCLE_BRANCH" == "master" && -n "$TAG" ]] || exit 0 + [[ -n "$DOCKERHUB_REPO" && -n "$DOCKER_USER" && -n "$DOCKER_PASS" ]] || exit 0 + [[ -n "$GITHUB_PERSONAL_TOKEN" ]] || exit 0 + [[ -f build_info/publish.txt ]] || exit 0 + git config credential.helper cache + git config user.email "$GITHUB_EMAIL" + git config user.name "$GITHUB_NAME" + git tag $TAG + git push https://${GITHUB_PERSONAL_TOKEN}:x-oauth-basic@github.com/$GITHUB_REPO.git $TAG + docker tag bmo "$DOCKERHUB_REPO:$TAG" + docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" + docker push "$DOCKERHUB_REPO:$TAG" test_sanity: - parallelism: 4 + parallelism: 2 working_directory: /app docker: - <<: *bmo_slim_image @@ -178,23 +217,29 @@ workflows: version: 2 main: jobs: + - build_info: + filters: *main_filters + - build: + filters: *main_filters + requires: + - build_info + - test_sanity + - test_bmo + - test_webservices + - test_selenium - test_sanity: filters: *main_filters + requires: + - build_info - test_bmo: filters: *main_filters requires: - - test_sanity + - build_info - test_webservices: filters: *main_filters requires: - - test_sanity + - build_info - test_selenium: filters: *main_filters requires: - - test_sanity - - build: - filters: *main_filters - requires: - - test_sanity - - test_bmo - - test_webservices + - build_info diff --git a/.perlcriticrc b/.perlcriticrc index 0c0d1c9be..44254f64e 100644 --- a/.perlcriticrc +++ b/.perlcriticrc @@ -32,12 +32,13 @@ severity = 2 [-Documentation::RequirePodLinksIncludeText] [-Documentation::RequirePodSections] [-ErrorHandling::RequireCarping] +[-InputOutput::RequireBracedFileHandleWithPrint] [-Modules::RequireVersionVar] [-References::ProhibitDoubleSigils] [-RegularExpressions::ProhibitComplexRegexes] [-RegularExpressions::RequireDotMatchAnything] -[-RegularExpressions::RequireLineBoundaryMatching] [-RegularExpressions::RequireExtendedFormatting] +[-RegularExpressions::RequireLineBoundaryMatching] [-Subroutines::ProhibitExcessComplexity] [-ValuesAndExpressions::ProhibitConstantPragma] [-ValuesAndExpressions::ProhibitEmptyQuotes] diff --git a/Makefile.PL b/Makefile.PL index f7b62ea5c..9f56cd487 100755 --- a/Makefile.PL +++ b/Makefile.PL @@ -50,6 +50,7 @@ my %requires = ( 'File::Slurp' => '9999.13', 'Future' => '0.34', 'HTML::Escape' => '1.10', + 'IPC::System::Simple' => 0, 'IO::Async' => '0.71', 'JSON::MaybeXS' => '1.003008', 'JSON::XS' => '2.01', diff --git a/scripts/build-bmo-push-data.pl b/scripts/build-bmo-push-data.pl new file mode 100755 index 000000000..e3cefe533 --- /dev/null +++ b/scripts/build-bmo-push-data.pl @@ -0,0 +1,203 @@ +#!/usr/bin/perl +use 5.10.1; +use strict; +use warnings; + +use File::Basename qw(basename dirname); +use File::Spec::Functions qw(catdir rel2abs); +use Cwd qw(realpath); + +BEGIN { + require lib; + my $dir = realpath( catdir(dirname(__FILE__), '..') ); + lib->import( $dir, catdir( $dir, 'lib' ), catdir( $dir, qw(local lib perl5) ) ); + chdir $dir or die "chdir $dir failed: $!"; +} + +use autodie; +use Bugzilla; +use English qw(-no_match_vars $PROGRAM_NAME); +use IPC::System::Simple qw(runx capture); +use JSON::MaybeXS qw(decode_json); +use LWP::Simple qw(get); +use LWP::UserAgent; +use MIME::Base64 qw(decode_base64); +use URI::QueryParam; +use URI; + +my $github_repo = "https://github.com/mozilla-bteam/bmo"; +my $version_info = decode_json(get('https://bugzilla.mozilla.org/__version__')); +my $tag = 'release-' . Bugzilla->VERSION; +my $prod_tag = "release-$version_info->{version}"; +my $tag_url = "$github_repo/tree/$tag"; + +my @log = capture(qw(git log --oneline), "$prod_tag..HEAD"); +die "nothing to commit\n" unless @log; +chomp @log; + +my @revisions; +foreach my $line (@log) { + say $line; + my ($revision, $message); + unless ( ( $revision, $message ) = $line =~ /^(\S+) (.+)$/ ) { + warn "skipping $line\n"; + next; + } + + my @bug_ids; + if ($message =~ /\bBug (\d+)/i) { + push @bug_ids, $1; + } + + if (!@bug_ids) { + warn "skipping $line (no bug)\n"; + next; + } + + foreach my $bug_id (@bug_ids) { + my $duplicate = 0; + foreach my $revisions (@revisions) { + if ($revisions->{bug_id} == $bug_id) { + $duplicate = 1; + last; + } + } + next if $duplicate; + + my $bug = fetch_bug($bug_id); + if ($bug->{status} eq 'RESOLVED' && $bug->{resolution} ne 'FIXED') { + next; + } + if ($bug->{summary} =~ /\bbackport\s+(?:upstream\s+)?bug\s+(\d+)/i) { + my $upstream = $1; + $bug->{summary} = fetch_bug($upstream)->{summary}; + } + push @revisions, { + hash => $revision, + bug_id => $bug_id, + summary => $bug->{summary}, + }; + } +} +if (!@revisions) { + die "no new revisions. make sure you run this script before production is updated.\n"; +} +else { + @revisions = reverse @revisions; +} + +my $first_revision = $revisions[0]->{hash}; +my $last_revision = $revisions[-1]->{hash}; + +mkdir 'build_info' unless -d 'build_info'; +chdir 'build_info'; + +say "write tag.txt"; +open my $tag_fh, '>', 'tag.txt'; +say $tag_fh $tag; +close $tag_fh; + +say 'write bug.push.txt'; + +open my $bug_fh, '>', 'bug.push.txt'; +say $bug_fh 'https://bugzilla.mozilla.org/enter_bug.cgi?product=bugzilla.mozilla.org&component=Infrastructure&short_desc=push+updated+bugzilla.mozilla.org+live'; +say $bug_fh "revisions: $first_revision - $last_revision"; +foreach my $revision (@revisions) { + say $bug_fh "bug $revision->{bug_id} : $revision->{summary}"; +} +close $bug_fh; + +say 'write blog.push.txt'; + +open my $blog_fh, '>', 'blog.push.txt'; +say $blog_fh "[release tag]($tag_url)\n"; +say $blog_fh "the following changes have been pushed to bugzilla.mozilla.org:\n<ul>"; +foreach my $revision (@revisions) { + printf $blog_fh '<li>[<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=%s" target="_blank">%s</a>] %s</li>%s', + $revision->{bug_id}, $revision->{bug_id}, html_escape($revision->{summary}), "\n"; +} +say $blog_fh '</ul>'; +say $blog_fh q{discuss these changes on <a href="https://lists.mozilla.org/listinfo/tools-bmo" target="_blank">mozilla.tools.bmo</a>.}; +close $blog_fh; + +say 'write email.push.txt'; + +open my $email_fh, '>', 'email.push.txt'; +say $email_fh "the following changes have been pushed to bugzilla.mozilla.org:\n"; +say $email_fh "(tag: $tag_url)\n"; +foreach my $revision (@revisions) { + printf $email_fh "https://bugzil.la/%s : %s\n", $revision->{bug_id}, $revision->{summary}; +} +close $email_fh; + +say 'write wiki.push.txt'; + +open my $wiki_fh, '>', 'wiki.push.txt'; +say $wiki_fh 'https://wiki.mozilla.org/BMO/Recent_Changes'; +say $wiki_fh '== ' . DateTime->now->set_time_zone('UTC')->ymd('-') . " ==\n"; +say $wiki_fh "[$tag_url $tag]"; +foreach my $revision (@revisions) { + printf $wiki_fh "* {{bug|%s}} %s\n", $revision->{bug_id}, $revision->{summary}; +} +close $wiki_fh; + +sub html_escape { + my ($s) = @_; + $s =~ s/&/&/g; + $s =~ s/</</g; + $s =~ s/>/>/g; + return $s; +} + +use constant BUG_FIELDS => [qw( + id + product + version + target_milestone + summary + status + resolution + assigned_to +)]; + +sub fetch_bug { + my ($bug_id) = @_; + die 'missing id' unless $bug_id; + + my $response = _get( 'bug/' . $bug_id, { include_fields => BUG_FIELDS, } ); + return $response->{bugs}->[0]; +} + +sub _get { + my ($endpoint, $args) = @_; + my $ua = LWP::UserAgent->new( agent => $PROGRAM_NAME ); + $args //= {}; + + if (exists $args->{include_fields} && ref($args->{include_fields})) { + $args->{include_fields} = join ',', @{ $args->{include_fields} }; + } + + my $uri = URI->new('https://bugzilla.mozilla.org/rest/' . $endpoint); + foreach my $name (sort keys %$args) { + $uri->query_param($name => $args->{$name}); + } + + my $request = HTTP::Request->new('GET', $uri->as_string); + $request->header( Content_Type => 'application/json' ); + $request->header( Accept => 'application/json' ); + if ( $ENV{BMO_API_KEY} ) { + $request->header( X_Bugzilla_API_Key => $ENV{BMO_API_KEY} ); + } + + my $response = $ua->request($request); + if ($response->code !~ /^2/) { + my $error = $response->message; + my $ok = eval { + $error = decode_json($response->decoded_content)->{message}; + 1; + }; + $error = $@ unless $ok; + die $error . "\n"; + } + return decode_json($response->decoded_content); +} |