Bug 262269: A tool to auto-install missing perl packages on non-Windows systems

Patch By Max Kanat-Alexander <mkanat@bugzilla.org> (module owner)

7 files changed, 449 insertions, 38 deletions

diff --git a/Bugzilla/Install/CPAN.pm b/Bugzilla/Install/CPAN.pm
new file mode 100644
index 000000000..01eceafeb
--- /dev/null
+++ b/Bugzilla/Install/CPAN.pm
@@ -0,0 +1,250 @@
+# -*- Mode: perl; indent-tabs-mode: nil -*-
+#
+# The contents of this file are subject to the Mozilla Public
+# License Version 1.1 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of
+# the License at http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS
+# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# rights and limitations under the License.
+#
+# The Original Code is the Bugzilla Bug Tracking System.
+#
+# The Initial Developer of the Original Code is Everything Solved, Inc.
+# Portions created by Everything Solved are Copyright (C) 2007
+# Everything Solved, Inc. All Rights Reserved.
+#
+# Contributor(s): Max Kanat-Alexander <mkanat@bugzilla.org>
+
+package Bugzilla::Install::CPAN;
+use strict;
+use base qw(Exporter);
+our @EXPORT = qw(set_cpan_config install_module BZ_LIB);
+
+use Bugzilla::Constants;
+use Bugzilla::Install::Util qw(bin_loc install_string);
+
+use CPAN;
+use Cwd qw(abs_path);
+use File::Path qw(rmtree);
+
+# We need the absolute path of ext_libpath, because CPAN chdirs around
+# and so we can't use a relative directory.
+#
+# We need it often enough (and at compile time, in install-module.pl) so 
+# we make it a constant.
+use constant BZ_LIB => abs_path(bz_locations()->{ext_libpath});
+
+# CPAN requires nearly all of its parameters to be set, or it will start
+# asking questions to the user. We want to avoid that, so we have
+# defaults here for most of the required parameters we know about, in case
+# any of them aren't set. The rest are handled by set_cpan_defaults().
+use constant CPAN_DEFAULTS => {
+    auto_commit => 0,
+    # We always force builds, so there's no reason to cache them.
+    build_cache => 0,
+    cache_metadata => 1,
+    index_expire => 1,
+    scan_cache => 'atstart',
+
+    inhibit_startup_message => 1,
+    mbuild_install_build_command => './Build',
+
+    curl => bin_loc('curl'),
+    gzip => bin_loc('gzip'),
+    links => bin_loc('links'),
+    lynx => bin_loc('lynx'),
+    make => bin_loc('make'),
+    pager => bin_loc('less'),
+    tar => bin_loc('tar'),
+    unzip => bin_loc('unzip'),
+    wget => bin_loc('wget'),
+
+    urllist => [qw(
+        http://cpan.pair.com/
+        http://mirror.hiwaay.net/CPAN/
+        ftp://ftp.dc.aleron.net/pub/CPAN/
+        http://perl.secsup.org/
+        http://mirrors.kernel.org/cpan/)],
+};
+
+sub install_module {
+    my ($name, $notest) = @_;
+    my $bzlib = BZ_LIB;
+
+    # Certain modules require special stuff in order to not prompt us.
+    my $original_makepl = $CPAN::Config->{makepl_arg};
+    # This one's a regex in case we're doing Template::Plugin::GD and it
+    # pulls in Template-Toolkit as a dependency.
+    if ($name =~ /^Template/) {
+        $CPAN::Config->{makepl_arg} .= " TT_ACCEPT=y TT_EXTRAS=n";
+    }
+    elsif ($name eq 'XML::Twig') {
+        $CPAN::Config->{makepl_arg} = "-n $original_makepl";
+    }
+    elsif ($name eq 'Net::LDAP') {
+        $CPAN::Config->{makepl_arg} .= " --skipdeps";
+    }
+    elsif ($name eq 'SOAP::Lite') {
+        $CPAN::Config->{makepl_arg} .= " --noprompt";
+    }
+
+    my $module = CPAN::Shell->expand('Module', $name);
+    print install_string('install_module', 
+              { module => $name, version => $module->cpan_version }) . "\n";
+    if ($notest) {
+        CPAN::Shell->notest('install', $name);
+    }
+    else {
+        CPAN::Shell->force('install', $name);
+    }
+
+    # If it installed any binaries in the Bugzilla directory, delete them.
+    if (-d "$bzlib/bin") {
+        File::Path::rmtree("$bzlib/bin");
+    }
+
+    $CPAN::Config->{makepl_arg} = $original_makepl;
+}
+
+sub set_cpan_config {
+    my $do_global = shift;
+    my $bzlib = BZ_LIB;
+
+    # We set defaults before we do anything, otherwise CPAN will
+    # start asking us questions as soon as we load its configuration.
+    eval { require CPAN::Config; };
+    _set_cpan_defaults();
+
+    # Calling a senseless autoload that does nothing makes us
+    # automatically load any existing configuration.
+    # We want to avoid the "invalid command" message.
+    open(my $saveout, ">&STDOUT");
+    open(STDOUT, '>/dev/null');
+    eval { CPAN->ignore_this_error_message_from_bugzilla; };
+    undef $@;
+    close(STDOUT);
+    open(STDOUT, '>&', $saveout);
+
+    my $dir = $CPAN::Config->{cpan_home};
+    if (!defined $dir || !-w $dir) {
+        # If we can't use the standard CPAN build dir, we try to make one.
+        $dir = "$ENV{HOME}/.cpan";
+        mkdir $dir;
+
+        # If we can't make one, we finally try to use the Bugzilla directory.
+        if (!-w $dir) {
+            print "WARNING: Using the Bugzilla directory as the CPAN home.\n";
+            $dir = "$bzlib/.cpan";
+        }
+    }
+    $CPAN::Config->{cpan_home} = $dir;
+    $CPAN::Config->{build_dir} = "$dir/build";
+    # We always force builds, so there's no reason to cache them.
+    $CPAN::Config->{keep_source_where} = "$dir/source";
+    # This is set both here and in defaults so that it's always true.
+    $CPAN::Config->{inhibit_startup_message} = 1;
+    # Automatically install dependencies.
+    $CPAN::Config->{prerequisites_policy} = 'follow';
+    
+    # Unless specified, we install the modules into the Bugzilla directory.
+    if (!$do_global) {
+        $CPAN::Config->{makepl_arg} .= " LIB=\"$bzlib\""
+            . " INSTALLMAN1DIR=\"$bzlib/man/man1\""
+            . " INSTALLMAN3DIR=\"$bzlib/man/man3\""
+            # The bindirs are here because otherwise we'll try to write to
+            # the system binary dirs, and that will cause CPAN to die.
+            . " INSTALLBIN=\"$bzlib/bin\""
+            . " INSTALLSCRIPT=\"$bzlib/bin\""
+            # INSTALLDIRS=perl is set because that makes sure that MakeMaker
+            # always uses the directories we've specified here.
+            . " INSTALLDIRS=perl";
+        $CPAN::Config->{mbuild_arg} = "--install_base \"$bzlib\"";
+
+        # When we're not root, sometimes newer versions of CPAN will
+        # try to read/modify things that belong to root, unless we set
+        # certain config variables.
+        $CPAN::Config->{histfile} = "$dir/histfile";
+        $CPAN::Config->{use_sqlite} = 0;
+        $CPAN::Config->{prefs_dir} = "$dir/prefs";
+
+        # Unless we actually set PERL5LIB, some modules can't install
+        # themselves, like DBD::mysql, DBD::Pg, and XML::Twig.
+        my $current_lib = $ENV{PERL5LIB} ? $ENV{PERL5LIB} . ':' : '';
+        $ENV{PERL5LIB} = $current_lib . $bzlib;
+    }
+}
+
+sub _set_cpan_defaults {
+    # If CPAN hasn't been configured, we try to use some reasonable defaults.
+    foreach my $key (keys %{CPAN_DEFAULTS()}) {
+        $CPAN::Config->{$key} = CPAN_DEFAULTS->{$key}
+            if !defined $CPAN::Config->{$key};
+    }
+
+    my @missing;
+    # In newer CPANs, this is in HandleConfig. In older CPANs, it's in
+    # Config.
+    if (eval { require CPAN::HandleConfig }) {
+        @missing = CPAN::HandleConfig->missing_config_data;
+    }
+    else {
+        @missing = CPAN::Config->missing_config_data;
+    }
+
+    foreach my $key (@missing) {
+        $CPAN::Config->{$key} = '';
+    }
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+Bugzilla::Install::CPAN - Routines to install Perl modules from CPAN.
+
+=head1 SYNOPSIS
+
+ use Bugzilla::Install::CPAN;
+
+ set_cpan_config();
+ install_module('Module::Name', 1);
+
+=head1 DESCRIPTION
+
+This is primarily used by L<install-module> to do the "hard work" of
+installing CPAN modules.
+
+=head1 SUBROUTINES
+
+=over
+
+=item C<set_cpan_config>
+
+Sets up the configuration of CPAN for this session. Must be called
+before L</install_module>. Takes one boolean parameter. If true,
+L</install_module> will install modules globally instead of to the
+local F<lib/> directory. On most systems, you have to be root to do that.
+
+=item C<install_module>
+
+Installs a module from CPAN. Takes two arguments:
+
+=over
+
+=item C<$name> - The name of the module, just like you'd pass to the
+C<install> command in the CPAN shell.
+
+=item C<$notest> - If true, we skip running tests on this module. This
+can greatly speed up the installation time.
+
+=back
+
+Note that calling this function prints a B<lot> of information to
+STDOUT and STDERR.
+
+=back
diff --git a/Bugzilla/Install/Filesystem.pm b/Bugzilla/Install/Filesystem.pm
index 127fe0b58..c96e8d12f 100644
--- a/Bugzilla/Install/Filesystem.pm
+++ b/Bugzilla/Install/Filesystem.pm
@@ -114,6 +114,7 @@ sub FILESYSTEM {
         'customfield.pl'  => { perms => $owner_executable },
         'email_in.pl'     => { perms => $ws_executable },
         'sanitycheck.pl'  => { perms => $ws_executable },
+        'install-module.pl' => { perms => $owner_executable },
 
         'docs/makedocs.pl'   => { perms => $owner_executable },
         'docs/rel_notes.txt' => { perms => $ws_readable },
diff --git a/Bugzilla/Install/Localconfig.pm b/Bugzilla/Install/Localconfig.pm
index bfdb0ce94..7df9e0736 100644
--- a/Bugzilla/Install/Localconfig.pm
+++ b/Bugzilla/Install/Localconfig.pm
@@ -31,8 +31,10 @@ package Bugzilla::Install::Localconfig;
 use strict;
 
 use Bugzilla::Constants;
+use Bugzilla::Install::Util qw(bin_loc);
 
 use Data::Dumper;
+use File::Basename qw(dirname);
 use IO::File;
 use Safe;
 
@@ -349,44 +351,11 @@ EOT
     return { old_vars => \@old_vars, new_vars => \@new_vars };
 }
 
-sub _get_default_cvsbin {
-    return '' if ON_WINDOWS;
-
-    my $cvs_executable = `which cvs`;
-    if ($cvs_executable =~ /no cvs/ || $cvs_executable eq '') {
-        # If which didn't find it, just set to blank
-        $cvs_executable = "";
-    } else {
-        chomp $cvs_executable;
-    }
-    return $cvs_executable;
-}
-
-sub _get_default_interdiffbin {
-    return '' if ON_WINDOWS;
-
-    my $interdiff = `which interdiff`;
-    if ($interdiff =~ /no interdiff/ || $interdiff eq '') {
-        # If which didn't find it, just set to blank
-        $interdiff = '';
-    } else {
-        chomp $interdiff;
-    }
-    return $interdiff;
-}
-
+sub _get_default_cvsbin       { return bin_loc('cvs') }
+sub _get_default_interdiffbin { return bin_loc('interdiff') }
 sub _get_default_diffpath {
-    return '' if ON_WINDOWS;
-
-    my $diff_binaries;
-    $diff_binaries = `which diff`;
-    if ($diff_binaries =~ /no diff/ || $diff_binaries eq '') {
-        # If which didn't find it, set to blank
-        $diff_binaries = "";
-    } else {
-        $diff_binaries =~ s:/diff\n$::;
-    }
-    return $diff_binaries;
+    my $diff_bin = bin_loc('diff');
+    return dirname($diff_bin);
 }
 
 1;
diff --git a/Bugzilla/Install/Requirements.pm b/Bugzilla/Install/Requirements.pm
index 885c407ee..8fd8fe2c6 100644
--- a/Bugzilla/Install/Requirements.pm
+++ b/Bugzilla/Install/Requirements.pm
@@ -434,6 +434,10 @@ EOT
             printf "%15s: $command\n", $module->{package};
         }
     }
+
+    if ($output && $check_results->{any_missing}) {
+        print install_string('install_all', { perl => $^X });
+    }
 }
 
 sub check_graphviz {
@@ -530,7 +534,7 @@ sub install_command {
         $package = $module->{package};
     }
     else {
-        $command = "$^X -MCPAN -e 'install \"\%s\"'";
+        $command = "$^X install-module.pl \%s";
         # Non-Windows installations need to use module names, because
         # CPAN doesn't understand package names.
         $package = $module->{module};
diff --git a/Bugzilla/Install/Util.pm b/Bugzilla/Install/Util.pm
index cb6b27786..3942aa82a 100644
--- a/Bugzilla/Install/Util.pm
+++ b/Bugzilla/Install/Util.pm
@@ -34,6 +34,7 @@ use Safe;
 
 use base qw(Exporter);
 our @EXPORT_OK = qw(
+    bin_loc
     get_version_and_os
     indicate_progress
     install_string
@@ -41,6 +42,21 @@ our @EXPORT_OK = qw(
     vers_cmp
 );
 
+sub bin_loc {
+    my ($bin) = @_;
+    return '' if ON_WINDOWS;
+    # Don't print any errors from "which"
+    open(my $saveerr, ">&STDERR");
+    open(STDERR, '>/dev/null');
+    my $loc = `which $bin`;
+    close(STDERR);
+    open(STDERR, ">&", $saveerr);
+    my $exit_code = $? >> 8; # See the perlvar manpage.
+    return '' if $exit_code > 0;
+    chomp($loc);
+    return $loc;
+}
+
 sub get_version_and_os {
     # Display version information
     my @os_details = POSIX::uname;
@@ -340,6 +356,11 @@ export them.
 
 =over
 
+=item C<bin_loc>
+
+On *nix systems, given the name of a binary, returns the path to that
+binary, if the binary is in the C<PATH>.
+
 =item C<get_version_and_os>
 
 Returns a hash containing information about what version of Bugzilla we're
diff --git a/install-module.pl b/install-module.pl
new file mode 100644
index 000000000..eb9ae38e4
--- /dev/null
+++ b/install-module.pl
@@ -0,0 +1,157 @@
+#!/usr/bin/perl -w
+# -*- Mode: perl; indent-tabs-mode: nil -*-
+#
+# The contents of this file are subject to the Mozilla Public
+# License Version 1.1 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of
+# the License at http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS
+# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# rights and limitations under the License.
+#
+# The Original Code is the Bugzilla Bug Tracking System.
+#
+# The Initial Developer of the Original Code is Everything Solved, Inc.
+# Portions created by Everything Solved are Copyright (C) 2007
+# Everything Solved, Inc. All Rights Reserved.
+#
+# Contributor(s): Max Kanat-Alexander <mkanat@bugzilla.org>
+
+use strict;
+use warnings;
+
+# Have to abs_path('.') or calls to Bugzilla modules won't work once
+# CPAN has chdir'ed around. We do all of this in this funny order to
+# make sure that we use the lib/ modules instead of the base Perl modules,
+# in case the lib/ modules are newer.
+use Cwd qw(abs_path);
+use lib abs_path('.');
+use Bugzilla::Constants;
+use lib abs_path(bz_locations()->{ext_libpath});
+
+use Bugzilla::Install::CPAN;
+
+use Bugzilla::Constants;
+use Bugzilla::Install::Requirements;
+
+use Data::Dumper;
+use Getopt::Long;
+use Pod::Usage;
+
+our %switch;
+
+GetOptions(\%switch, 'all|a', 'upgrade-all|u', 'show-config|s', 'global|g',
+                     'help|h');
+
+pod2usage({ -verbose => 1 }) if $switch{'help'};
+pod2usage({ -verbose => 0 }) if (!%switch && !@ARGV);
+
+set_cpan_config($switch{'global'});
+
+if ($switch{'show-config'}) {
+  print Dumper($CPAN::Config);
+  exit;
+}
+
+my $can_notest = 1;
+if (substr(CPAN->VERSION, 0, 3) < 1.8) {
+    $can_notest = 0;
+    print "* Note: If you upgrade your CPAN module, installs will be faster.\n";
+    print "* You can upgrade CPAN by doing: $^X install-module.pl CPAN\n";
+}
+
+if ($switch{'all'} || $switch{'upgrade-all'}) {
+    my @modules;
+    if ($switch{'upgrade-all'}) {
+        @modules = (@{REQUIRED_MODULES()}, @{OPTIONAL_MODULES()});
+        push(@modules, DB_MODULE->{$_}->{dbd}) foreach (keys %{DB_MODULE()});
+    }
+    else {
+        # This is the only time we need a Bugzilla-related module, so
+        # we require them down here. Otherwise this script can be run from
+        # any directory, even outside of Bugzilla itself.
+        my $reqs = check_requirements(0);
+        @modules = (@{$reqs->{missing}}, @{$reqs->{optional}});
+        my $dbs = DB_MODULE;
+        foreach my $db (keys %$dbs) {
+            push(@modules, $dbs->{$db}->{dbd})
+                if !have_vers($dbs->{$db}->{dbd}, 0);
+        }
+    }
+    foreach my $module (@modules) {
+        my $cpan_name = $module->{module};
+        # --all shouldn't include mod_perl2, because it can have some complex
+        # configuration, and really should be installed on its own.
+        next if $cpan_name eq 'mod_perl2';
+        install_module($cpan_name, $can_notest);
+    }
+    my $dbs = DB_MODULE;
+    foreach my $db (keys %$dbs) {
+        install_module($dbs->{$db}->{dbd}->{module}, $can_notest)
+            unless have_vers($dbs->{$db}->{dbd}, 0);
+    }
+}
+
+foreach my $module (@ARGV) {
+    install_module($module, $can_notest);
+}
+
+__END__
+
+=head1 NAME
+
+install-module.pl - Installs or upgrades modules from CPAN
+
+=head1 SYNOPSIS
+
+  ./install-module.pl Module::Name [--global]
+  ./install-module.pl --all [--global]
+  ./install-module.pl --all-upgrade [--global]
+  ./install-module.pl --show-config
+
+  Do "./install-module.pl --help" for more information.
+
+=head1 OPTIONS
+
+=over
+
+=item B<Module::Name>
+
+The name of a module that you want to install from CPAN. This is the
+same thing that you'd give to the C<install> command in the CPAN shell.
+
+You can specify multiple module names separated by a space to install
+multiple modules.
+
+=item B<--global>
+
+This makes install-module install modules globally for all applications,
+instead of just for Bugzilla.
+
+On most systems, you have to be root for C<--global> to work.
+
+=item B<--all>
+
+This will make install-module do its best to install every required
+and optional module that is not installed that Bugzilla can use.
+
+Some modules may fail to install. You can run checksetup.pl to see
+which installed properly.
+
+=item B<--upgrade-all>
+
+This is like C<--all>, except it forcibly installs the very latest
+version of every Bugzilla prerequisite, whether or not you already
+have them installed.
+
+=item B<--show-config>
+
+Prints out the CPAN configuration in raw Perl format. Useful for debugging.
+
+=item B<--help>
+
+Shows this help.
+
+=back
diff --git a/template/en/default/setup/strings.txt.pl b/template/en/default/setup/strings.txt.pl
index 34e447857..352e7b035 100644
--- a/template/en/default/setup/strings.txt.pl
+++ b/template/en/default/setup/strings.txt.pl
@@ -35,6 +35,15 @@
     checking_modules  => 'Checking perl modules...',
     header => "* This is Bugzilla ##bz_ver## on perl ##perl_ver##\n"
             . "* Running on ##os_name## ##os_ver##",
+    install_all => <<EOT,
+
+To attempt an automatic install of every required and optional module
+with one command, do:
+
+  ##perl## install-module.pl --all
+
+EOT
+    install_module => 'Installing ##module## version ##version##...',
     module_found => "found v##ver##",
     module_not_found => "not found",
     module_ok => 'ok',