Bug 1314854: Integrate POD in to REST docs.

4 files changed, 210 insertions, 42 deletions

diff --git a/Makefile.PL b/Makefile.PL
index 500c48466..1d78e5e78 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -168,7 +168,12 @@ my %optional_features = (
         prereqs => {
             runtime => {
                 requires =>
-                    { 'File::Which' => 0, 'File::Copy::Recursive' => 0 }
+                    {
+                        'File::Which'                  => 0,
+                        'File::Copy::Recursive'        => 0,
+                        'Pod::Simple::Search'          => 0,
+                        'Pod::POM::View::Restructured' => 0,
+                    }
             }
         },
         description => 'Documentation',
diff --git a/docs/en/rst/conf.py b/docs/en/rst/conf.py
index d2d50f6d2..cfbca34a2 100644
--- a/docs/en/rst/conf.py
+++ b/docs/en/rst/conf.py
@@ -381,9 +381,9 @@ todo_include_todos = True
 on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
 
 if on_rtd:
-    base_api_url = 'https://www.bugzilla.org/docs/tip/en/html/integrating/api/'
+    base_api_url = 'https://www.bugzilla.org/docs/tip/en/html/integrating/internals/'
 else:
-    base_api_url = '../integrating/api/'
+    base_api_url = '../integrating/internals/'
 
 extlinks = {'bug': ('https://bugzilla.mozilla.org/show_bug.cgi?id=%s', 'bug  '),
             'api': (base_api_url + '%s', '')}
@@ -399,13 +399,17 @@ ext_dir = "../../../extensions"
 
 # Still, check just in case, so if it ever changes, we know
 if (os.path.isdir(ext_dir)):
-    # Clear out old extensions docs
+    # Clear out removed extensions docs
     for dir in os.listdir("extensions"):
         # A .gitignore file is required as git doesn't like empty directories
         if dir == ".gitignore":
             continue
 
-        shutil.rmtree(os.path.join("extensions", dir))
+        src = os.path.join(ext_dir, dir)
+
+        # extension dir missing, assume uninstalled
+        if not os.path.isdir(src):
+            shutil.rmtree(os.path.join("extensions", dir))
 
     # Copy in new copies
     for ext_name in os.listdir(ext_dir):
@@ -426,6 +430,10 @@ if (os.path.isdir(ext_dir)):
 
         dst = os.path.join("extensions", ext_name)
 
+        # remove old docs
+        if (os.path.isdir(dst)):
+            shutil.rmtree(dst)
+
         shutil.copytree(src, dst)
 else:
     print "Warning: Bugzilla extension directory not found: " + ext_dir
diff --git a/docs/en/rst/integrating/index.rst b/docs/en/rst/integrating/index.rst
index 794bc0ad8..ea7d8a7f3 100644
--- a/docs/en/rst/integrating/index.rst
+++ b/docs/en/rst/integrating/index.rst
@@ -21,3 +21,4 @@ explains how to use the available mechanisms for integration and customization.
    extensions
    apis
    auth-delegation
+   internals/index
diff --git a/docs/makedocs.pl b/docs/makedocs.pl
index 2d57fc9a5..93bdd5190 100755
--- a/docs/makedocs.pl
+++ b/docs/makedocs.pl
@@ -31,22 +31,27 @@ use File::Basename;
 BEGIN { chdir dirname($0); }
 
 use lib qw(.. ../lib lib ../local/lib/perl5);
+use open ':encoding(utf8)';
 
 use Cwd;
 use File::Copy::Recursive qw(rcopy);
 use File::Path qw(rmtree make_path);
 use File::Which qw(which);
 use Pod::Simple;
+use Pod::ParseLink;
 
 use Bugzilla::Constants qw(BUGZILLA_VERSION bz_locations);
-use Pod::Simple::HTMLBatch::Bugzilla;
-use Pod::Simple::HTML::Bugzilla;
+use Pod::Simple::Search;
+use Pod::POM::View::Restructured;
+use Tie::File;
+use File::Spec::Functions qw(:ALL);
 
 ###############################################################################
 # Subs
 ###############################################################################
 
 my $error_found = 0;
+
 sub MakeDocs {
     my ($name, $cmdline) = @_;
 
@@ -56,38 +61,179 @@ sub MakeDocs {
     print "\n";
 }
 
-sub make_pod {
-    say "Creating API documentation...";
-
-    my $converter = Pod::Simple::HTMLBatch::Bugzilla->new;
-    # Don't output progress information.
-    $converter->verbose(0);
-    $converter->html_render_class('Pod::Simple::HTML::Bugzilla');
-
-    my $doctype      = Pod::Simple::HTML::Bugzilla->DOCTYPE;
-    my $content_type = Pod::Simple::HTML::Bugzilla->META_CT;
-    my $bz_version   = BUGZILLA_VERSION;
-
-    my $contents_start = <<END_HTML;
-$doctype
-<html>
-  <head>
-    $content_type
-    <title>Bugzilla $bz_version API Documentation</title>
-  </head>
-  <body class="contentspage">
-    <h1>Bugzilla $bz_version API Documentation</h1>
-END_HTML
-
-    $converter->contents_page_start($contents_start);
-    $converter->contents_page_end("</body></html>");
-    $converter->add_css('./../../../../style.css');
-    $converter->javascript_flurry(0);
-    $converter->css_flurry(0);
-    make_path('html/integrating/api');
-    $converter->batch_convert(['../../'], 'html/integrating/api');
+# Compile all the POD and make it part of the API docs
+# Also duplicate some Extension docs into the User section, Admin section, and
+# WebService docs
+sub pod2rst {
+    my $path = shift;
 
-    print "\n";
+    say "Converting POD to RST...";
+    my $name2path = Pod::Simple::Search->new->inc(0)->verbose(0)
+        ->survey(@{ ['../'] });
+
+    my $ind_path = catdir($path, 'rst', 'integrating', 'internals');
+    rmtree($ind_path);
+    make_path($ind_path);
+
+    my $FILE;
+    open($FILE, '>', catfile($ind_path, 'index.rst'))
+        || die("Can't open rst file");
+    print($FILE <<'EOI');
+.. highlight:: perl
+
+.. _developer
+
+===================================
+Developer Documentation
+===================================
+
+This section exposes the POD for all modules and scripts, including extensions,
+in Bugzilla.
+
+.. toctree::
+   :maxdepth: 2
+
+EOI
+
+    foreach my $mod (sort keys %$name2path) {
+        # Ignoring library files;
+        if ($mod =~ /^b?(lib::|local)/) {
+            delete $name2path->{$mod};
+            next;
+        }
+
+        my $title = $mod;
+
+        my $ext = $title =~ s/^extensions:://;
+        $title =~ s/lib:://;
+        $title =~ s/::Extension$// if ($ext);
+        my $header = "=" x length($title);
+
+        my $abs_path = $name2path->{$mod};
+        my $fpath = abs2rel($abs_path, '..');
+        my ($volume, $directories, $file) = splitpath($fpath);
+        $directories =~ s/lib\///;
+        $directories = '.' if ($directories eq '');
+
+        my $dir_path = catdir($ind_path, $directories);
+        make_path($dir_path);
+        $file =~ s/\.[^.]*$//;
+        my $out_file = $file . '.rst';
+        my $full_out_file = catfile($dir_path, $out_file);
+
+        my $callbacks = {
+            link => sub {
+                my $text = shift;
+
+                my @results = parselink($text);
+
+                # A link to another page.
+                if (defined($results[1]) && defined($results[2])) {
+                    if ($results[2] =~ /^http/) {
+                        return ($results[2], $results[1]);
+                    }
+                    elsif ($results[2] =~ m/Bugzilla/) {
+                        my $depth = scalar(split('/', $directories));
+                        my @split = split('::', $results[2]);
+                        my $base  = '../' x $depth;
+                        my $url   = $base . join('/', @split);
+                        $url .= '.html';
+                        $url .= '#' . $results[3] if defined $results[3];
+                        return ($url, $results[1]);
+                    }
+                    else {
+                        # Not a Bugzilla package, link to CPAN.
+                        my $url
+                            = 'http://search.cpan.org/search?query='
+                            . $results[2]
+                            . '&mode=all';
+                        return ($url, $results[1]);
+                    }
+                }
+
+                # A Link within a page
+                elsif (defined($results[1]) && defined($results[4])) {
+                    my $anchor = $results[1];
+                    $anchor =~ s/"//g;
+                    return ("#$anchor", $anchor);
+                }
+                else {
+                    die "Don't know how to parse $text";
+                }
+            }
+        };
+        my $conv = Pod::POM::View::Restructured->new(
+                             { namespace => $title, callbacks => $callbacks });
+        my $rv = $conv->convert_file($abs_path, $title, $full_out_file,
+                                                                   $callbacks);
+        print($FILE "   ", catfile($directories, $out_file), "\n");
+
+        if ($ext) {
+            my $api_path = catdir($path, 'rst', $directories);
+            my $adminfile = catfile($api_path, "index-admin.rst");
+            my $userfile  = catfile($api_path, "index-user.rst");
+            my $apifile   = catfile($api_path, 'api', 'v1', 'index.rst');
+            make_path($api_path) unless -d $api_path;
+
+            # Add Core doc to User & Admin guides
+            if ($file eq 'Extension') {
+                my $FH;
+                open($FH, '<', $full_out_file) || die "$full_out_file: $!";
+                my @lines = <$FH>;
+                close $FH;
+
+                my @array;
+
+                # ensure out of order docs are at the top of the page without
+                # playing with file handles
+                tie @array, 'Tie::File', $adminfile or die "$adminfile: $!";
+                unshift @array, @lines, "", "", ".. toctree::", "", "";
+                untie @array;
+
+                tie @array, 'Tie::File', $userfile or die "$userfile: $!";
+                unshift @array, @lines, "", "", ".. toctree::", "", "";
+                untie @array;
+
+            }
+
+            # Add Config doc to Admin guide
+            elsif ($file eq 'Config') {
+                rcopy($full_out_file, $api_path);
+                `perl -E 'say "   $file.rst"' >> "$adminfile"`;
+            }
+
+            # Add WebServices doc to API docs
+            elsif ($file eq 'WebService') {
+                my $apidir = catdir($api_path, 'api', 'v1');
+                make_path($apidir)
+                    unless -d $apidir;
+                `perl -E 'say ".. toctree::\n\n"' >> $apifile`
+                    unless -f $apifile;
+
+                my $FH;
+                open($FH, '<', $full_out_file) || die "$full_out_file: $!";
+                my @lines = <$FH>;
+                close $FH;
+
+                my @array;
+
+                tie @array, 'Tie::File', $apifile or die "$apifile: $!";
+                unshift @array, @lines;
+                untie @array;
+            }
+            elsif ($api_path =~ /WebService\/$/) {
+                my $apidir = catdir($api_path, '..', 'api', 'v1');
+                $apifile = catfile($apidir, 'index.rst');
+                make_path($apidir) unless -d $apidir;
+                rcopy($full_out_file, $apidir);
+                `perl -E 'say ".. toctree::\n\n"' >> $apifile`
+                    unless -f $apifile;
+                `perl -E 'say "   $file.rst"' >> $apifile`;
+            }
+        }
+    }
+
+    close($FILE);
 }
 
 ###############################################################################
@@ -95,10 +241,11 @@ END_HTML
 ###############################################################################
 
 my @langs;
+
 # search for sub directories which have a 'rst' sub-directory
 opendir(LANGS, './');
 foreach my $dir (readdir(LANGS)) {
-    next if (($dir eq '.') || ($dir eq '..') || (! -d $dir));
+    next if (($dir eq '.') || ($dir eq '..') || (!-d $dir));
     if (-d "$dir/rst") {
         push(@langs, $dir);
     }
@@ -107,14 +254,21 @@ closedir(LANGS);
 
 my $docparent = getcwd();
 foreach my $lang (@langs) {
-    chdir "$docparent/$lang";
 
-    make_pod();
+    rmtree("$lang/html", 0, 1);
+    rmtree("$lang/txt",  0, 1);
+
+    my @sub_dirs = grep {-d} glob("$lang/rst/extensions/*");
+    rmtree(@sub_dirs, { verbose => 0, safe => 1 });
+
+    pod2rst("$docparent/$lang");
 
     next if grep { $_ eq '--pod-only' } @ARGV;
 
+    chdir "$docparent/$lang";
+
     MakeDocs('HTML', 'html');
-    MakeDocs('TXT', 'text');
+    MakeDocs('TXT',  'text');
 
     if (grep { $_ eq '--with-pdf' } @ARGV) {
         if (which('pdflatex')) {