Bug 757698: move moco-ldap-check to a moco server

1 files changed, 486 insertions, 168 deletions

diff --git a/contrib/moco-ldap-check.pl b/contrib/moco-ldap-check.pl
index 59f515bf2..7a3a6ca8c 100755
--- a/contrib/moco-ldap-check.pl
+++ b/contrib/moco-ldap-check.pl
@@ -1,224 +1,542 @@
-#!/usr/bin/perl -wT
-# -*- Mode: perl; indent-tabs-mode: nil -*-
-#
-# The contents of this file are subject to the Mozilla Public
-# License Version 1.1 (the "License"); you may not use this file
-# except in compliance with the License. You may obtain a copy of
-# the License at http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS
-# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-# implied. See the License for the specific language governing
-# rights and limitations under the License.
-#
-# The Original Code is the Bugzilla Bug Tracking System.
-#
-# The Initial Developer of the Original Code is the Mozilla
-# Foundation. Portions created by Mozilla are
-# Copyright (C) 2011 Mozilla Foundation. All Rights Reserved.
+#!/usr/bin/perl
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 #
-# Contributor(s): Byron Jones <glob@mozilla.com>
+# This Source Code Form is "Incompatible With Secondary Licenses", as
+# defined by the Mozilla Public License, v. 2.0.
 
 use strict;
+use warnings;
 
-use lib qw(.);
+use FindBin qw($Bin);
+use lib "$Bin/..";
+use lib "$Bin/../lib";
 
+use Bugzilla;
+use Bugzilla::Constants;
+use Bugzilla::Group;
+use Bugzilla::Mailer;
+use Data::Dumper;
+use File::Slurp;
+use Getopt::Long;
 use Net::LDAP;
-use XMLRPC::Lite;
-use HTTP::Cookies;
-use LWP::UserAgent;
-use Term::ReadKey;
-$| = 1;
+use Safe;
 
 #
+
+use constant BUGZILLA_IGNORE => <<'EOF';
+    infra+bot@mozilla.com    # Mozilla Infrastructure Bot
+    qa-auto@mozilla.com      # QA Desktop Automation
+    qualys@mozilla.com       # Qualys Security Scanner
+    recruiting@mozilla.com   # Recruiting
+    release@mozilla.com      # Mozilla RelEng Bot
+    sumo-dev@mozilla.com     # SUMOdev [:sumodev]
+    airmozilla@mozilla.com   # Air Mozilla
+    ux-review@mozilla.com
+    release-mgmt@mozilla.com
+    reps@mozilla.com
+    moz_bug_r_a4@mozilla.com # Security contractor
+    nightwatch@mozilla.com   # Security distribution list for whines
+EOF
+
+use constant LDAP_IGNORE => <<'EOF';
+    airmozilla@mozilla.com  # Air Mozilla
+EOF
+
+# REPORT_SENDER has to be a valid @mozilla.com LDAP account
+use constant REPORT_SENDER => 'bjones@mozilla.com';
+
+use constant BMO_RECIPIENTS => qw(
+    glob@mozilla.com
+    dkl@mozilla.com
+);
+
+use constant SUPPORT_RECIPIENTS => qw(
+    desktop@mozilla.com
+);
+
 #
+
+my ($ldap_host, $ldap_user, $ldap_pass, $debug, $no_update);
+GetOptions('h=s' => \$ldap_host,
+           'u=s' => \$ldap_user,
+           'p=s' => \$ldap_pass,
+           'd'   => \$debug,
+           'n'   => \$no_update);
+die "syntax: -h ldap_host -u ldap_user -p ldap_pass\n"
+    unless $ldap_host && $ldap_user && $ldap_pass;
+
+my $data_dir = bz_locations()->{'datadir'} . '/moco-ldap-check';
+mkdir($data_dir) unless -d $data_dir;
+
+if ($ldap_user !~ /,/) {
+    $ldap_user = "mail=$ldap_user,o=com,dc=mozilla";
+}
+
+#
+# group members
 #
 
-print STDERR <<EOF;
-This script cross-checks members of the Bugzilla mozilla-corporation group
-with Mozilla's LDAP repository.
+my @bugzilla_ignore;
+foreach my $line (split(/\n/, BUGZILLA_IGNORE)) {
+    $line =~ s/^([^#]+)#.*$/$1/;
+    $line =~ s/(^\s+|\s+$)//g;
+    push @bugzilla_ignore, clean_email($line);
+}
 
-To run this script you need:
-  - a bugzilla.mozilla.org admin account
-  - a Mozilla LDAP account
+my @bugzilla_moco;
+if ($no_update && -s "$data_dir/bugzilla_moco.last") {
+    $debug && print "Using cached user list from Bugzilla...\n";
+    my $ra = deserialise("$data_dir/bugzilla_moco.last");
+    @bugzilla_moco = @$ra;
+} else {
+    $debug && print "Getting user list from Bugzilla...\n";
 
-EOF
+    my $group = Bugzilla::Group->new({ name => 'mozilla-corporation' })
+        or die "Failed to find group mozilla-corporation\n";
 
-my $bugzillaLogin = get_text('bl', 'Bugzilla Login: ');
-my $bugzillaPassword = get_text('bp', 'Bugzilla Password: ', 1);
-my $ldapLogin = get_text('ll', 'LDAP Login: ');
-my $ldapPassword = get_text('lp', 'LDAP Password: ', 1);
+    foreach my $user (@{ $group->members_non_inherited }) {
+        next unless $user->is_enabled;
+        my $mail = clean_email($user->login);
+        my $name = trim($user->name);
+        $name =~ s/\s+/ /g;
+        next if grep { $mail eq $_ } @bugzilla_ignore;
+        push @bugzilla_moco, {
+            mail => $user->login,
+            canon => $mail,
+            name => $name,
+        };
+    }
 
-sub get_text {
-    my($switch, $prompt, $password) = @_;
+    @bugzilla_moco = sort { $a->{mail} cmp $b->{mail} } @bugzilla_moco;
+    serialise("$data_dir/bugzilla_moco.last", \@bugzilla_moco);
+}
 
-    for (my $i = 0; $i <= $#ARGV; $i++) {
-        if ($ARGV[$i] eq "-$switch") {
-            return $ARGV[$i + 1];
+#
+# build list of current mo-co bugmail accounts
+#
+
+my @ldap_ignore;
+foreach my $line (split(/\n/, LDAP_IGNORE)) {
+    $line =~ s/^([^#]+)#.*$/$1/;
+    $line =~ s/(^\s+|\s+$)//g;
+    push @ldap_ignore, canon_email($line);
+}
+
+my %ldap;
+if ($no_update && -s "$data_dir/ldap.last") {
+    $debug && print "Using cached user list from LDAP...\n";
+    my $rh = deserialise("$data_dir/ldap.last");
+    %ldap = %$rh;
+} else {
+    $debug && print "Logging into LDAP as $ldap_user...\n";
+    my $ldap = Net::LDAP->new($ldap_host,
+        scheme => 'ldaps', onerror => 'die') or die "$@";
+    $ldap->bind($ldap_user, password => $ldap_pass);
+    foreach my $ldap_base ('o=com,dc=mozilla', 'o=org,dc=mozilla') {
+        $debug && print "Getting user list from LDAP $ldap_base...\n";
+        my $result = $ldap->search(
+            base   => $ldap_base,
+            scope  => 'sub',
+            filter => '(mail=*)',
+            attrs  => ['mail', 'bugzillaEmail', 'emailAlias', 'cn', 'employeeType'],
+        );
+        foreach my $entry ($result->entries) {
+            my ($name, $bugMail, $mail, $type) =
+                map { $entry->get_value($_) || '' }
+                qw(cn bugzillaEmail mail employeeType);
+            next if $type eq 'DISABLED';
+            $mail = lc $mail;
+            next if grep { $_ eq canon_email($mail) } @ldap_ignore;
+            $bugMail = '' if $bugMail !~ /\@/;
+            $bugMail =~ s/(^\s+|\s+$)//g;
+            if ($bugMail =~ / /) {
+                $bugMail = (grep { /\@/ } split / /, $bugMail)[0];
+            }
+            $name =~ s/\s+/ /g;
+            $ldap{$mail}{name} = trim($name);
+            $ldap{$mail}{bugmail} = $bugMail;
+            $ldap{$mail}{bugmail_canon} = canon_email($bugMail);
+            $ldap{$mail}{aliases} = [];
+            foreach my $alias (
+                @{$entry->get_value('emailAlias', asref => 1) || []}
+            ) {
+                push @{$ldap{$mail}{aliases}}, canon_email($alias);
+            }
         }
+        $debug && printf "Found %s entries\n", scalar($result->entries);
     }
+    serialise("$data_dir/ldap.last", \%ldap);
+}
 
-    print STDERR $prompt;
-    my $response = '';
-    ReadMode 4;
-    my $ch;
-    while(1) {
-        1 while (not defined ($ch = ReadKey(-1)));
-        exit if $ch eq "\3";
-        last if $ch =~ /[\r\n]/;
-        if ($ch =~ /[\b\x7F]/) {
-            next if $response eq '';
-            chop $response;
-            print "\b \b";
-            next;
-        }
-        if ($ch eq "\025") {
-            my $len = length($response);
-            print(("\b" x $len) . (" " x $len) . ("\b" x $len));
-            $response = '';
-            next;
+#
+# validate all bugmail entries from the phonebook
+#
+
+my %bugzilla_login;
+if ($no_update && -s "$data_dir/bugzilla_login.last") {
+    $debug && print "Using cached bugzilla checks...\n";
+    my $rh = deserialise("$data_dir/bugzilla_login.last");
+    %bugzilla_login = %$rh;
+} else {
+    my %logins;
+    foreach my $mail (keys %ldap) {
+        $logins{$mail} = 1;
+        $logins{$ldap{$mail}{bugmail}} = 1 if $ldap{$mail}{bugmail};
+    }
+    my @logins = sort keys %logins;
+    $debug && print "Checking " . scalar(@logins) . " bugmail accounts...\n";
+
+    foreach my $login (@logins) {
+        if (Bugzilla::User->new({ name => $login })) {
+            $bugzilla_login{$login} = 1;
         }
-        next if ord($ch) < 32;
-        $response .= $ch;
-        print STDERR $password ? '*' : $ch;
     }
-    ReadMode 0;
-    print STDERR "\n";
-    return $response;
+    serialise("$data_dir/bugzilla_login.last", \%bugzilla_login);
 }
-END {
-    ReadMode 0;
+
+#
+# load previous ldap list
+#
+
+my %ldap_old;
+{
+    my $rh = deserialise("$data_dir/ldap.data");
+    %ldap_old = %$rh if $rh;
 }
 
 #
-# get list of users in mo-co group
+# save current ldap list
 #
 
-my %bugzilla;
 {
-    my $cookie_jar = HTTP::Cookies->new(file => "cookies.txt", autosave => 1);
-    my $proxy = XMLRPC::Lite->proxy(
-        'https://bugzilla.mozilla.org/xmlrpc.cgi',
-        'cookie_jar' => $cookie_jar);
-    my $response;
-
-    print STDERR "Logging in to Bugzilla...\n";
-    $response = $proxy->call(
-        'User.login',
-        {
-            login => $bugzillaLogin,
-            password => $bugzillaPassword,
-            remember => 1,
+    serialise("$data_dir/ldap.data", \%ldap);
+}
+
+#
+# new ldap accounts
+#
+
+my @new_ldap;
+{
+    foreach my $mail (sort keys %ldap) {
+        next if exists $ldap_old{$mail};
+        push @new_ldap, {
+            mail => $mail,
+            name => $ldap{$mail}{name},
+            bugmail => $ldap{$mail}{bugmail},
+        };
+    }
+}
+
+#
+# deleted ldap accounts
+#
+
+my @gone_ldap_bmo;
+my @gone_ldap_no_bmo;
+{
+    foreach my $mail (sort keys %ldap_old) {
+        next if exists $ldap{$mail};
+        if ($ldap_old{$mail}{bugmail}) {
+            push @gone_ldap_bmo, {
+                mail => $mail,
+                name => $ldap_old{$mail}{name},
+                bugmail => $ldap_old{$mail}{bugmail},
+            }
+        } else {
+            push @gone_ldap_no_bmo, {
+                mail => $mail,
+                name => $ldap_old{$mail}{name},
+            }
         }
-    );
-    if ($response->fault) {
-        my ($package, $filename, $line) = caller;
-        die $response->faultstring . "\n";
     }
+}
+
+#
+# check bugmail entry for all users in bmo/moco group
+#
+
+my @suspect_bugzilla;
+my @invalid_bugzilla;
+foreach my $rh (@bugzilla_moco) {
+    my @check = ($rh->{mail}, $rh->{canon});
+    if ($rh->{mail} =~ /^([^\@]+)\@mozilla\.org$/) {
+        push @check, "$1\@mozilla.com";
+    }
+
+    my $exists;
+    foreach my $check (@check) {
+        $exists = 0;
+
+        # don't complain about deleted accounts
+        if (grep { $_->{mail} eq $check } (@gone_ldap_bmo, @gone_ldap_no_bmo)) {
+            $exists = 1;
+            last;
+        }
+
+        # check for matching bugmail entry
+        foreach my $mail (sort keys %ldap) {
+            next unless $ldap{$mail}{bugmail_canon} eq $check;
+            $exists = 1;
+            last;
+        }
+        last if $exists;
+
+        # check for matching mail
+        $exists = 0;
+        foreach my $mail (sort keys %ldap) {
+            next unless $mail eq $check;
+            $exists = 1;
+            last;
+        }
+        last if $exists;
 
-    my $ua = LWP::UserAgent->new();
-    $ua->cookie_jar($cookie_jar);
-    $response = $ua->get('https://bugzilla.mozilla.org/editusers.cgi?' .
-        'action=list&matchvalue=login_name&matchstr=&matchtype=substr&' .
-        'grouprestrict=1&groupid=42');
-    if (!$response->is_success) {
-        die $response->status_line;
+        # check for matching email alias
+        $exists = 0;
+        foreach my $mail (sort keys %ldap) {
+            next unless grep { $check eq $_ } @{$ldap{$mail}{aliases}};
+            $exists = 1;
+            last;
+        }
+        last if $exists;
     }
 
-    print STDERR "Getting user list from Bugzilla...\n";
-    my $content = $response->content;
-    while (
-        $content =~ m#
-            <td([^>]*)>[^<]+
-            <a\shref="editusers[^"]+">([^<]+)</a>[^<]+
-            </td>[^<]+
-            <td[^>]*>([^<]+)</td>
-        #gx
-    ) {
-        my ($class, $email, $name) = ($1, $2, $3);
-        next if $class =~ /bz_inactive/;
-        $email =~ s/(^\s+|\s+$)//g;
-        $email =~ s/&#64;/@/;
-        next unless $email =~ /@/;
-        $name =~ s/(^\s+|\s+$)//g;
-        $bugzilla{lc $email} = $name;
+    if (!$exists) {
+        # flag the account
+        if ($rh->{mail} =~ /\@mozilla\.(com|org)$/i) {
+            push @invalid_bugzilla, {
+                mail => $rh->{mail},
+                name => $rh->{name},
+            };
+        } else {
+            push @suspect_bugzilla, {
+                mail => $rh->{mail},
+                name => $rh->{name},
+            };
+        }
     }
 }
 
 #
-# build list of current mo-co bugmail accounts
+# check bugmail entry for ldap users
 #
 
-my %ldap;
-{
-    print STDERR "Logging into LDAP...\n";
-    my $ldap = Net::LDAP->new('addressbook.mozilla.com',
-        scheme => 'ldaps', onerror => 'die') or die "$@";
-    $ldap->bind("mail=$ldapLogin,o=com,dc=mozilla", password => $ldapPassword);
-    my $result = $ldap->search(
-        base => 'o=com,dc=mozilla',
-        scope => 'sub',
-        filter => '(mail=*)',
-        attrs => ['mail', 'bugzillaEmail', 'emailAlias', 'cn', 'employeeType'],
-    );
-    print STDERR "Getting user list from LDAP...\n";
-    foreach my $entry ($result->entries) {
-        my ($name, $bugMail, $mail, $type) =
-            map { $entry->get_value($_) || '' }
-            qw(cn bugzillaEmail mail employeeType);
-        next if $type eq 'DISABLED';
-        $mail = lc $mail;
-        $ldap{$mail}{name} = $name;
-        $ldap{$mail}{bugMail} = lc $bugMail;
-        $ldap{$mail}{alias} = {};
-        foreach my $alias (
-            @{$entry->get_value('emailAlias', asref => 1) || []}
-        ) {
-            $ldap{$mail}{alias}{lc $alias} = 1;
+my @ldap_unblessed;
+my @invalid_ldap;
+my @invalid_bugmail;
+foreach my $mail (sort keys %ldap) {
+    # try to find the bmo account
+    my $found;
+    foreach my $address ($ldap{$mail}{bugmail}, $ldap{$mail}{bugmail_canon}, $mail, @{$ldap{$mail}{aliases}}) {
+        if (exists $bugzilla_login{$address}) {
+            $found = $address;
+            last;
+        }
+    }
+
+    # not on bmo
+    if (!$found) {
+        # if they have specified a bugmail account, warn, otherwise ignore
+        if ($ldap{$mail}{bugmail}) {
+            if (grep { $_->{canon} eq $ldap{$mail}{bugmail_canon} } @bugzilla_moco) {
+                push @invalid_bugmail, {
+                    mail => $mail,
+                    name => $ldap{$mail}{name},
+                    bugmail => $ldap{$mail}{bugmail},
+                };
+            } else {
+                push @invalid_ldap, {
+                    mail => $mail,
+                    name => $ldap{$mail}{name},
+                    bugmail => $ldap{$mail}{bugmail},
+                };
+            }
+        }
+        next;
+    }
+
+    # warn about mismatches
+    if ($ldap{$mail}{bugmail} && $found ne $ldap{$mail}{bugmail}) {
+        push @invalid_bugmail, {
+            mail => $mail,
+            name => $ldap{$mail}{name},
+            bugmail => $ldap{$mail}{bugmail},
+        };
+    }
+
+    # warn about unblessed accounts
+    if ($mail =~ /\@mozilla\.com$/) {
+        unless (grep { $_->{mail} eq $found || $_->{canon} eq canon_email($found) } @bugzilla_moco) {
+            push @ldap_unblessed, {
+                mail => $mail,
+                name => $ldap{$mail}{name},
+                bugmail => $ldap{$mail}{bugmail} || $mail,
+            };
         }
     }
 }
 
 #
-# cross-check
+# reports
 #
 
-my @invalid;
-foreach my $bugzilla (sort keys %bugzilla) {
-    # check for matching bugmail entry
-    my $exists = 0;
-    foreach my $mail (sort keys %ldap) {
-        next unless $ldap{$mail}{bugMail} eq $bugzilla;
-        $exists = 1;
-        last;
+my @bmo_report;
+push @bmo_report, generate_report(
+    'new ldap accounts',
+    'no action required',
+    @new_ldap);
+
+push @bmo_report, generate_report(
+    'deleted ldap accounts',
+    'disable bmo account',
+    @gone_ldap_bmo);
+
+push @bmo_report, generate_report(
+    'deleted ldap accounts',
+    'no action required (no bmo account)',
+    @gone_ldap_no_bmo);
+
+push @bmo_report, generate_report(
+    'suspect bugzilla accounts',
+    'remove from mo-co if required',
+    @suspect_bugzilla);
+
+push @bmo_report, generate_report(
+    'miss-configured bugzilla accounts',
+    'ask owner to update phonebook, disable if not on phonebook',
+    @invalid_bugzilla);
+
+push @bmo_report, generate_report(
+    'ldap accounts without mo-co group',
+    'verify, and add mo-co group to bmo account',
+    @ldap_unblessed);
+
+push @bmo_report, generate_report(
+    'missmatched bugmail entries on ldap accounts',
+    'ask owner to update phonebook',
+    @invalid_bugmail);
+
+push @bmo_report, generate_report(
+    'invalid bugmail entries on ldap accounts',
+    'ask owner to update phonebook',
+    @invalid_ldap);
+
+if (!scalar @bmo_report) {
+    push @bmo_report, '**';
+    push @bmo_report, '** nothing to report \o/';
+    push @bmo_report, '**';
+}
+
+email_report(\@bmo_report, 'moco-ldap-check', BMO_RECIPIENTS);
+
+my @support_report;
+
+push @support_report, generate_report(
+    'Missmatched "Bugzilla Email" entries on LDAP accounts',
+    'Ask owner to update phonebook, or update directly',
+    @invalid_bugmail);
+
+push @support_report, generate_report(
+    'Invalid "Bugzilla Email" entries on LDAP accounts',
+    'Ask owner to update phonebook',
+    @invalid_ldap);
+
+if (scalar @support_report) {
+    email_report(\@support_report, 'Invalid "Bugzilla Email" entries in LDAP', SUPPORT_RECIPIENTS);
+}
+
+#
+#
+#
+
+sub generate_report {
+    my ($title, $action, @lines) = @_;
+
+    my $count = scalar @lines;
+    return unless $count;
+
+    my @report;
+    push @report, '';
+    push @report, '**';
+    push @report, "** $title ($count)";
+    push @report, "** [ $action ]";
+    push @report, '**';
+    push @report, '';
+
+    my $max_length = 0;
+    foreach my $rh (@lines) {
+        $max_length = length($rh->{mail}) if length($rh->{mail}) > $max_length;
     }
-    next if $exists;
 
-    # check for matching mail
-    $exists = 0;
-    foreach my $mail (sort keys %ldap) {
-        next unless $mail eq $bugzilla;
-        $exists = 1;
-        last;
+    foreach my $rh (@lines) {
+        my $template = "%-${max_length}s  %s";
+        my @fields = ($rh->{mail}, $rh->{name});
+
+        if ($rh->{bugmail}) {
+            $template .= ' (%s)';
+            push @fields, $rh->{bugmail};
+        };
+
+        push @report, sprintf($template, @fields);
     }
-    next if $exists;
 
-    # check for matching email alias
-    $exists = 0;
-    foreach my $mail (sort keys %ldap) {
-        next unless exists $ldap{$mail}{alias}{$bugzilla};
-        $exists = 1;
-        last;
+    return @report;
+}
+
+sub email_report {
+    my ($report, $subject, @recipients) = @_;
+    unshift @$report, (
+        "Subject: $subject",
+        'X-Bugzilla-Type: moco-ldap-check',
+        'From: ' . REPORT_SENDER,
+        'To: ' . join(',', @recipients),
+    );
+    if ($debug) {
+        print "\n", join("\n", @$report), "\n";
+    } else {
+        MessageToMTA(join("\n", @$report));
     }
-    next if $exists;
+}
+
+sub clean_email {
+    my $email = shift;
+    $email = trim($email);
+    $email = $1 if $email =~ /^(\S+)/;
+    $email =~ s/&#64;/@/;
+    $email = lc $email;
+    return $email;
+}
 
-    push @invalid, $bugzilla;
+sub canon_email {
+    my $email = shift;
+    $email = clean_email($email);
+    $email =~ s/^([^\+]+)\+[^\@]+(\@.+)$/$1$2/;
+    return $email;
 }
 
-my $max_length = 0;
-foreach my $email (@invalid) {
-    $max_length = length($email) if length($email) > $max_length;
+sub trim {
+    my $value = shift;
+    $value =~ s/(^\s+|\s+$)//g;
+    return $value;
 }
-foreach my $email (@invalid) {
-    printf "%-${max_length}s %s\n", $email, $bugzilla{$email};
+
+sub serialise {
+    my ($filename, $ref) = @_;
+    local $Data::Dumper::Purity = 1;
+    local $Data::Dumper::Deepcopy = 1;
+    local $Data::Dumper::Sortkeys = 1;
+    write_file($filename, Dumper($ref));
 }
+
+sub deserialise {
+    my ($filename) = @_;
+    return unless -s $filename;
+    my $cpt = Safe->new();
+    $cpt->reval('our ' . read_file($filename))
+        || die "$!";
+    return ${$cpt->varglob('VAR1')};
+}
+