Bug 880829 - Migrate current custom field based tracking flags to the new Tracking Flags extension tables

r=glob

17 files changed, 450 insertions, 472 deletions

diff --git a/Bugzilla.pm b/Bugzilla.pm
index a998de902..bcb77218e 100644
--- a/Bugzilla.pm
+++ b/Bugzilla.pm
@@ -594,8 +594,7 @@ sub active_custom_fields {
     my $cache_id = 'active_custom_fields';
     if ($params) {
         $cache_id .= ($params->{product} ? '_p' . $params->{product}->id : '') .
-                     ($params->{component} ? '_c' . $params->{component}->id : '') .
-                     ($params->{type} ? '_t' . $params->{type} : '');
+                     ($params->{component} ? '_c' . $params->{component}->id : '');
     }
     if (!exists $class->request_cache->{$cache_id}) {
         my $fields = Bugzilla::Field->match({ custom => 1, obsolete => 0});
diff --git a/extensions/BMO/Extension.pm b/extensions/BMO/Extension.pm
index 3aaa26afb..fd9068fe7 100644
--- a/extensions/BMO/Extension.pm
+++ b/extensions/BMO/Extension.pm
@@ -66,11 +66,9 @@ sub template_before_process {
     my ($self, $args) = @_;
     my $file = $args->{'file'};
     my $vars = $args->{'vars'};
-    
+
     $vars->{'cf_hidden_in_product'} = \&cf_hidden_in_product;
-    $vars->{'cf_is_project_flag'}   = \&cf_is_project_flag;
-    $vars->{'cf_flag_disabled'}     = \&cf_flag_disabled;
-    
+
     if ($file =~ /^list\/list/) {
         # Purpose: enable correct sorting of list table
         # Matched to changes in list/table.html.tmpl
@@ -84,7 +82,7 @@ sub template_before_process {
         );
 
         my @orderstrings = split(/,\s*/, $vars->{'order'});
-        
+
         # contains field names of the columns being used to sort the table.
         my @order_columns;
         foreach my $o (@orderstrings) {
@@ -96,7 +94,7 @@ sub template_before_process {
         }
 
         $vars->{'order_columns'} = \@order_columns;
-        
+
         # fields that have a custom sortkey. (So they are correctly sorted 
         # when using js)
         my @sortkey_fields = qw(bug_status resolution bug_severity priority
@@ -220,53 +218,28 @@ sub active_custom_fields {
 
     my @tmp_fields;
     foreach my $field (@$$fields) { 
-        next if cf_hidden_in_product($field->name, $product_name, $component_name, $params->{'type'}); 
+        next if cf_hidden_in_product($field->name, $product_name, $component_name);
         push(@tmp_fields, $field);
     }
     $$fields = \@tmp_fields;
 }
 
-sub cf_is_project_flag {
-    my ($field_name) = @_;
-    foreach my $flag_re (@$cf_project_flags) {
-        return 1 if $field_name =~ $flag_re;
-    }
-    return 0;
-}
-
 sub cf_hidden_in_product {
-    my ($field_name, $product_name, $component_name, $custom_flag_mode) = @_;
+    my ($field_name, $product_name, $component_name) = @_;
 
     # If used in buglist.cgi, we pass in one_product which is a Bugzilla::Product
     # elsewhere, we just pass the name of the product.
-    $product_name = blessed($product_name) ? $product_name->name
-                                           : $product_name;
-   
+    $product_name = blessed($product_name)
+                    ? $product_name->name
+                    : $product_name;
+
     # Also in buglist.cgi, we pass in a list of components instead 
     # of a single component name everywhere else.
     my $component_list = [];
     if ($component_name) {
-        $component_list = ref $component_name ? $component_name 
-                                              : [ $component_name ];
-    }
-
-    if ($custom_flag_mode) {
-        if ($custom_flag_mode == 1) {
-            # skip custom flags
-            foreach my $flag_re (@$cf_flags) {
-                return 1 if $field_name =~ $flag_re;
-            }
-        } elsif ($custom_flag_mode == 2) {
-            # custom flags only
-            my $found = 0;
-            foreach my $flag_re (@$cf_flags) {
-                if ($field_name =~ $flag_re) {
-                    $found = 1;
-                    last;
-                }
-            }
-            return 1 unless $found;
-        }
+        $component_list = ref $component_name
+                          ? $component_name
+                          : [ $component_name ];
     }
 
     foreach my $field_re (keys %$cf_visible_in_products) {
@@ -295,7 +268,7 @@ sub cf_hidden_in_product {
                         }
                     }
                 }
-        
+
                 # If product matches and at at least one component matches
                 # from component_list (if a matching component was required), 
                 # we allow the field to be seen
@@ -303,19 +276,11 @@ sub cf_hidden_in_product {
                     return 0;
                 }
             }
-
             return 1;
         }
     }
-    
-    return 0;
-}
 
-sub cf_flag_disabled {
-    my ($field_name, $bug) = @_;
-    return 0 unless grep { $field_name eq $_ } @$cf_disabled_flags;
-    my $value = $bug->{$field_name};
-    return $value eq '---' || $value eq '';
+    return 0;
 }
 
 # Purpose: CC certain email addresses on bugmail when a bug is added or 
@@ -391,44 +356,7 @@ sub bug_check_can_change_field {
     my $priv_results = $args->{'priv_results'};
     my $user = Bugzilla->user;
 
-    # Only users in the appropriate drivers group can change the 
-    # cf_blocking_* fields or cf_tracking_* fields
-
-    if ($field =~ /^cf_(?:blocking|tracking)_/) {
-        # 0 -> 1 is used by show_bug, always allow so we skip this whole part
-        if (!($old_value eq '0' && $new_value eq '1')) {
-            # require privileged access to set a flag
-            if (_is_field_set($new_value)) {
-                _check_trusted($field, $blocking_trusted_setters, $priv_results);
-            }
-
-            # require editbugs to clear or re-nominate a set flag
-            elsif (_is_field_set($old_value) 
-                && !$user->in_group('editbugs', $bug->{'product_id'}))
-            {
-                push (@$priv_results, PRIVILEGES_REQUIRED_EMPOWERED);
-            }
-        }
-
-        if ($new_value =~ /\?$/) {
-            _check_trusted($field, $blocking_trusted_requesters, $priv_results);
-        }
-        if ($user->id) {
-            push (@$priv_results, PRIVILEGES_REQUIRED_NONE);
-        }
-
-    } elsif ($field =~ /^cf_status_/) {
-        # Only drivers can set wanted.
-        if ($new_value eq 'wanted') {
-            _check_trusted($field, $status_trusted_wanters, $priv_results);
-        } elsif (_is_field_set($new_value)) {
-            _check_trusted($field, $status_trusted_setters, $priv_results);
-        }
-        if ($user->id) {
-            push (@$priv_results, PRIVILEGES_REQUIRED_NONE);
-        }
-
-    } elsif ($field =~ /^cf/ && !@$priv_results && $new_value ne '---') {
+    if ($field =~ /^cf/ && !@$priv_results && $new_value ne '---') {
         # "other" custom field setters restrictions
         if (exists $other_setters->{$field}) {
             my $in_group = 0;
@@ -442,8 +370,8 @@ sub bug_check_can_change_field {
                 push (@$priv_results, PRIVILEGES_REQUIRED_EMPOWERED);
             }
         }
-
-    } elsif ($field eq 'resolution' && $new_value eq 'EXPIRED') {
+    }
+    elsif ($field eq 'resolution' && $new_value eq 'EXPIRED') {
         # The EXPIRED resolution should only be settable by gerv.
         if ($user->login ne 'gerv@mozilla.org') {
             push (@$priv_results, PRIVILEGES_REQUIRED_EMPOWERED);
@@ -574,33 +502,12 @@ sub bug_format_comment {
     });
 }
 
-# Purpose: generically handle generating pretty blocking/status "flags" from
-# custom field names.
 sub quicksearch_map {
     my ($self, $args) = @_;
     my $map = $args->{'map'};
-    
-    foreach my $name (keys %$map) {
-        if ($name =~ /^cf_(blocking|tracking|status)_([a-z]+)?(\d+)?$/) {
-            my $type = $1;
-            my $product = $2;
-            my $version = $3;
-
-            if ($version) {
-                $version = join('.', split(//, $version));
-            }
-
-            my $pretty_name = $type;
-            if ($product) {              
-                $pretty_name .= "-" . $product;
-            }
-            if ($version) {
-                $pretty_name .= $version;
-            }
 
-            $map->{$pretty_name} = $name;
-        }
-        elsif ($name =~ /cf_crash_signature$/) {
+    foreach my $name (keys %$map) {
+        if ($name =~ /cf_crash_signature$/) {
             $map->{'sig'} = $name;
         }
     }
@@ -846,36 +753,6 @@ sub mailer_before_send {
         $email->header_set('to', 'mei.kong@tcl.com');
     }
 
-    # Add X-Bugzilla-Tracking header
-    if ($email->header('X-Bugzilla-ID')) {
-        my $bug_id = $email->header('X-Bugzilla-ID');
-
-        # return if we cannot successfully load the bug object
-        my $bug = new Bugzilla::Bug($bug_id);
-        return if !$bug;
-
-        # The BMO hook in active_custom_fields will filter 
-        # the fields for us based on product and component
-        my @fields = Bugzilla->active_custom_fields({
-            product   => $bug->product_obj, 
-            component => $bug->component_obj,
-            type      => 2,  
-        });
-
-        my @set_values = ();
-        foreach my $field (@fields) {
-            next if $field->type == FIELD_TYPE_EXTENSION;
-            my $field_name = $field->name;
-            next if cf_flag_disabled($field_name, $bug);
-            next if !$bug->$field_name || $bug->$field_name eq '---';
-            push(@set_values, $field->description . ":" . $bug->$field_name);
-        }
-
-        if (@set_values) {
-            $email->header_set('X-Bugzilla-Tracking' => join(' ', @set_values));
-        } 
-    }
-
     # attachments disabled, see bug 714488
     return;
 
diff --git a/extensions/BMO/template/en/default/email/bugmail.html.tmpl b/extensions/BMO/template/en/default/email/bugmail.html.tmpl
index e84852289..7daec3b2e 100644
--- a/extensions/BMO/template/en/default/email/bugmail.html.tmpl
+++ b/extensions/BMO/template/en/default/email/bugmail.html.tmpl
@@ -76,20 +76,14 @@
     Product/Component: [% bug.product FILTER html %] :: [% bug.component FILTER html %]
   </div>
 
-[% USE Bugzilla %]
-[% tracking_flags = [] %]
-[% FOREACH field = Bugzilla.active_custom_fields(product => bug.product_obj, component => bug.component_obj, type => 2) %]
-  [% NEXT IF cf_flag_disabled(field.name, bug) %]
-  [% NEXT IF bug.${field.name} == "---" %]
-  [% tracking_flags.push(field) %]
-[% END %]
 [% IF tracking_flags.size %]
   <div id="tracking" style="font-size: 90%; color: #666666">
     <hr style="border: 1px dashed #969696">
     <b>Tracking Flags:</b>
     <ul>
-      [% FOREACH field = tracking_flags %]
-        <li>[% field.description FILTER html %]:[% bug.${field.name} FILTER html %]</li>
+      [% FOREACH flag = tracking_flags %]
+        [% NEXT IF bug.${flag.name} == "---" %]
+        <li>[% flag.description FILTER html %]:[% bug.${flag.name} FILTER html %]</li>
       [% END %]
     </ul>
   </div>
diff --git a/extensions/BMO/template/en/default/email/bugmail.txt.tmpl b/extensions/BMO/template/en/default/email/bugmail.txt.tmpl
index 771157dd1..ed74ad880 100644
--- a/extensions/BMO/template/en/default/email/bugmail.txt.tmpl
+++ b/extensions/BMO/template/en/default/email/bugmail.txt.tmpl
@@ -39,18 +39,11 @@ Configure [% terms.bug %]mail: [% urlbase %]userprefs.cgi?tab=email
 -------------------------------
 Product/Component: [%+ bug.product +%] :: [%+ bug.component %]
 
-[% USE Bugzilla %]
-[% show_tracking_flags = [] %]
-[% FOREACH field = Bugzilla.active_custom_fields(product => bug.product_obj, component => bug.component_obj, type => 2) %]
-  [% NEXT IF cf_flag_disabled(field.name, bug) %]
-  [% NEXT IF bug.${field.name} == "---" %]
-  [% show_tracking_flags.push(field) %]
-[% END %]
-
-[% IF show_tracking_flags.size %]
+[% IF tracking_flags.size %]
 ------- Tracking Flags: -------
-[% FOREACH field = show_tracking_flags %]
-[%+ field.description %]:[% bug.${field.name} %]
+[% FOREACH flag = tracking_flags %]
+  [% NEXT IF bug.${flag.name} == "---" %]
+[%+ flag.description %]:[% bug.${flag.name} %]
 [% END %]
 [% END %]
 
diff --git a/extensions/TrackingFlags/Extension.pm b/extensions/TrackingFlags/Extension.pm
index b79faef61..be9cb9309 100644
--- a/extensions/TrackingFlags/Extension.pm
+++ b/extensions/TrackingFlags/Extension.pm
@@ -65,7 +65,9 @@ sub template_before_process {
 
         $vars->{'tracking_flag_types'} = FLAG_TYPES;
     }
-    elsif ($file eq 'bug/edit.html.tmpl'|| $file eq 'bug/show.xml.tmpl') {
+    elsif ($file eq 'bug/edit.html.tmpl'|| $file eq 'bug/show.xml.tmpl'
+           || $file eq 'email/bugmail.html.tmpl' || $file eq 'email/bugmail.txt.tmpl')
+    {
         # note: bug/edit.html.tmpl doesn't support multiple bugs
         my $bug = exists $vars->{'bugs'} ? $vars->{'bugs'}[0] : $vars->{'bug'};
 
@@ -306,8 +308,9 @@ sub active_custom_fields {
 
     my @tracking_flags;
     if ($product) {
-        my $params = { product_id => $product->id };
+        $params->{'product_id'}   = $product->id;
         $params->{'component_id'} = $component->id if $component;
+        $params->{'is_active'}    = 1;
         @tracking_flags = @{ Bugzilla::Extension::TrackingFlags::Flag->match($params) };
     }
     else {
@@ -546,4 +549,33 @@ sub mailer_before_send {
     }
 }
 
+# Purpose: generically handle generating pretty blocking/status "flags" from
+# custom field names.
+sub quicksearch_map {
+    my ($self, $args) = @_;
+    my $map = $args->{'map'};
+
+    foreach my $name (keys %$map) {
+        if ($name =~ /^cf_(blocking|tracking|status)_([a-z]+)?(\d+)?$/) {
+            my $type = $1;
+            my $product = $2;
+            my $version = $3;
+
+            if ($version) {
+                $version = join('.', split(//, $version));
+            }
+
+            my $pretty_name = $type;
+            if ($product) {
+                $pretty_name .= "-" . $product;
+            }
+            if ($version) {
+                $pretty_name .= $version;
+            }
+
+            $map->{$pretty_name} = $name;
+        }
+    }
+}
+
 __PACKAGE__->NAME;
diff --git a/extensions/TrackingFlags/bin/migrate_tracking_flags.pl b/extensions/TrackingFlags/bin/migrate_tracking_flags.pl
new file mode 100755
index 000000000..06b3596c4
--- /dev/null
+++ b/extensions/TrackingFlags/bin/migrate_tracking_flags.pl
@@ -0,0 +1,316 @@
+#!/usr/bin/perl -w
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This Source Code Form is "Incompatible With Secondary Licenses", as
+# defined by the Mozilla Public License, v. 2.0.
+
+# Migrate old custom field based tracking flags to the new
+# table based tracking flags
+
+use strict;
+use warnings;
+
+use FindBin '$RealBin';
+use lib "$RealBin/../../..";
+use lib "$RealBin/../../../lib";
+use lib "$RealBin/../lib";
+
+BEGIN {
+    use Bugzilla;
+    Bugzilla->extensions;
+}
+
+use Bugzilla::Constants;
+use Bugzilla::Field;
+use Bugzilla::Product;
+use Bugzilla::Component;
+use Bugzilla::Extension::BMO::Data;
+use Bugzilla::Install::Util qw(indicate_progress);
+
+use Bugzilla::Extension::TrackingFlags::Constants;
+use Bugzilla::Extension::TrackingFlags::Flag;
+use Bugzilla::Extension::TrackingFlags::Flag::Bug;
+use Bugzilla::Extension::TrackingFlags::Flag::Value;
+use Bugzilla::Extension::TrackingFlags::Flag::Visibility;
+
+use Getopt::Long;
+use Data::Dumper;
+
+Bugzilla->usage_mode(USAGE_MODE_CMDLINE);
+
+my ($dry_run, $trace) = (0, 0);
+GetOptions(
+    "dry-run" => \$dry_run,
+    "trace"   => \$trace,
+) or exit;
+
+my $dbh = Bugzilla->dbh;
+
+$dbh->{TraceLevel} = 1 if $trace;
+
+my %product_cache;
+my %component_cache;
+
+sub migrate_flag_visibility {
+    my ($new_flag, $products) = @_;
+
+    # Create product/component visibility
+    foreach my $prod_name (keys %$products) {
+        $product_cache{$prod_name} ||= Bugzilla::Product->new({ name => $prod_name });
+        if (!$product_cache{$prod_name}) {
+            warn "No such product $prod_name\n";
+            next;
+        }
+
+        # If no components specified then we do Product/__any__
+        # otherwise, we enter an entry for each Product/Component
+        my $components = $products->{$prod_name};
+        if (!@$components) {
+            Bugzilla::Extension::TrackingFlags::Flag::Visibility->create({
+                tracking_flag_id => $new_flag->flag_id,
+                product_id       => $product_cache{$prod_name}->id,
+                component_id     => undef
+            });
+        }
+        else {
+            foreach my $comp_name (@$components) {
+                my $comp_matches = [];
+                # If the component is a regexp, we need to find all components
+                # matching the regex and insert each individually
+                if (ref $comp_name eq 'Regexp') {
+                    my $comp_re = $comp_name;
+                    $comp_re =~ s/\?\-xism://;
+                    $comp_re =~ s/\(//;
+                    $comp_re =~ s/\)//;
+                    $comp_matches = $dbh->selectcol_arrayref(
+                        'SELECT components.name FROM components
+                          WHERE components.product_id = ?
+                                AND ' . $dbh->sql_regexp('components.name', $dbh->quote($comp_re)) . '
+                          ORDER BY components.name',
+                        undef,
+                        $product_cache{$prod_name}->id);
+                }
+                else {
+                    $comp_matches = [ $comp_name ];
+                }
+
+                foreach my $comp_match (@$comp_matches) {
+                    $component_cache{"${prod_name}:${comp_match}"}
+                        ||= Bugzilla::Component->new({ name    => $comp_match,
+                                                       product => $product_cache{$prod_name} });
+                    if (!$component_cache{"${prod_name}:${comp_match}"}) {
+                        warn "No such product $prod_name and component $comp_match\n";
+                        next;
+                    }
+
+                    Bugzilla::Extension::TrackingFlags::Flag::Visibility->create({
+                        tracking_flag_id => $new_flag->flag_id,
+                        product_id       => $product_cache{$prod_name}->id,
+                        component_id     => $component_cache{"${prod_name}:${comp_match}"}->id,
+                    });
+                }
+            }
+        }
+    }
+}
+
+sub migrate_flag_values {
+    my ($new_flag, $field) = @_;
+
+    print "Migrating flag values...";
+
+    my %blocking_trusted_requesters
+        = %{$Bugzilla::Extension::BMO::Data::blocking_trusted_requesters};
+    my %blocking_trusted_setters
+        = %{$Bugzilla::Extension::BMO::Data::blocking_trusted_setters};
+    my %status_trusted_wanters
+        = %{$Bugzilla::Extension::BMO::Data::status_trusted_wanters};
+    my %status_trusted_setters
+        = %{$Bugzilla::Extension::BMO::Data::status_trusted_setters};
+
+    my %group_cache;
+    foreach my $value (@{ $field->legal_values }) {
+        my $group_name = 'everyone';
+
+        if ($field->name =~ /^cf_(blocking|tracking)_/) {
+            if ($value->name ne '---' && $value->name !~ '\?$') {
+                $group_name = get_setter_group($field->name, \%blocking_trusted_setters);
+            }
+            if ($value->name eq '?') {
+                $group_name = get_setter_group($field->name, \%blocking_trusted_requesters);
+            }
+        } elsif ($field->name =~ /^cf_status_/) {
+            if ($value->name eq 'wanted') {
+                $group_name = get_setter_group($field->name, \%status_trusted_wanters);
+            } elsif ($value->name ne '---' && $value->name ne '?') {
+                $group_name = get_setter_group($field->name, \%status_trusted_setters);
+            }
+        }
+
+        $group_cache{$group_name} ||= Bugzilla::Group->new({ name => $group_name });
+        $group_cache{$group_name} || die "Setter group '$group_name' does not exist";
+
+        Bugzilla::Extension::TrackingFlags::Flag::Value->create({
+            tracking_flag_id => $new_flag->flag_id,
+            value            => $value->name,
+            setter_group_id  => $group_cache{$group_name}->id,
+            sortkey          => $value->sortkey,
+            is_active        => $value->is_active
+        });
+    }
+
+    print "done.\n";
+}
+
+sub get_setter_group {
+    my ($field, $trusted) = @_;
+    my $setter_group = $trusted->{'_default'} || "";
+    foreach my $dfield (keys %$trusted) {
+        if ($field =~ $dfield) {
+            $setter_group = $trusted->{$dfield};
+        }
+    }
+    return $setter_group;
+}
+
+sub migrate_flag_bugs {
+    my ($new_flag, $field) = @_;
+
+    print "Migrating bug values...";
+
+    my $bugs = $dbh->selectall_arrayref("SELECT bug_id, " . $field->name . "
+                                           FROM bugs
+                                          WHERE " . $field->name . " != '---'
+                                       ORDER BY bug_id");
+    local $| = 1;
+    my $count = 1;
+    my $total = scalar @$bugs;
+    foreach my $row (@$bugs) {
+        my ($id, $value) = @$row;
+        indicate_progress({ current => $count++, total => $total, every => 25 });
+        Bugzilla::Extension::TrackingFlags::Flag::Bug->create({
+            tracking_flag_id => $new_flag->flag_id,
+            bug_id           => $id,
+            value            => $value,
+
+        });
+    }
+
+    print "done.\n";
+}
+
+sub migrate_flag_activity {
+     my ($new_flag, $field) = @_;
+
+     print "Migating flag activity...";
+
+     my $new_field = Bugzilla::Field->new({ name => $new_flag->name });
+     $dbh->do("UPDATE bugs_activity SET fieldid = ? WHERE fieldid = ?",
+              undef, $new_field->id, $field->id);
+
+     print "done.\n";
+}
+
+sub do_migration {
+    my $bmo_tracking_flags = $Bugzilla::Extension::BMO::Data::cf_visible_in_products;
+    my $bmo_project_flags  = $Bugzilla::Extension::BMO::Data::cf_project_flags;
+    my $bmo_disabled_flags = $Bugzilla::Extension::BMO::Data::cf_disabled_flags;
+
+    my $fields = Bugzilla::Field->match({ custom => 1,
+                                          type   => FIELD_TYPE_SINGLE_SELECT });
+
+    my @drop_columns;
+    foreach my $field (@$fields) {
+        next if $field->name !~ /^cf_(blocking|tracking|status)_/;
+
+        foreach my $field_re (keys %$bmo_tracking_flags) {
+            next if $field->name !~ $field_re;
+
+            # Create the new tracking flag if not exists
+            my $new_flag
+                = Bugzilla::Extension::TrackingFlags::Flag->new({ name => $field->name });
+
+            next if $new_flag;
+
+            print "----------------------------------\n" .
+                  "Migrating custom tracking field " . $field->name . "...\n";
+
+            my $new_flag_name = $field->name . "_new"; # Temporary name til we delete the old
+
+            my $type = grep($field->name =~ $_, @$bmo_project_flags)
+                       ? 'project'
+                       : 'tracking';
+
+            my $is_active = grep($_ eq $field->name, @$bmo_disabled_flags) ? 0 : 1;
+
+            $new_flag = Bugzilla::Extension::TrackingFlags::Flag->create({
+                 name        => $new_flag_name,
+                 description => $field->description,
+                 type        => $type,
+                 sortkey     => $field->sortkey,
+                 is_active   => $is_active,
+                 enter_bug   => $field->enter_bug,
+            });
+
+            migrate_flag_visibility($new_flag, $bmo_tracking_flags->{$field_re});
+
+            migrate_flag_values($new_flag, $field);
+
+            migrate_flag_bugs($new_flag, $field);
+
+            migrate_flag_activity($new_flag, $field);
+
+            push(@drop_columns, $field->name);
+
+            # Remove the old flag entry from fielddefs
+            $dbh->do("DELETE FROM fielddefs WHERE name = ?",
+                     undef, $field->name);
+
+            # Rename the new flag
+            $dbh->do("UPDATE fielddefs SET name = ? WHERE name = ?",
+                     undef, $field->name, $new_flag_name);
+
+            $new_flag->set_name($field->name);
+            $new_flag->update;
+
+            # more than one regex could possibly match but we only want the first one
+            last;
+        }
+    }
+
+    # Drop each custom flag's value table and the column from the bz schema object
+    if (!$dry_run && @drop_columns) {
+        print "Dropping value tables and updating bz schema object...\n";
+
+        foreach my $column (@drop_columns) {
+            # Drop the values table
+            $dbh->bz_drop_table($column);
+
+            # Drop the bugs table column from the bz schema object
+            $dbh->_bz_real_schema->delete_column('bugs', $column);
+            $dbh->_bz_store_real_schema;
+        }
+
+        # Do the one alter table to drop all columns at once
+        $dbh->do("ALTER TABLE bugs DROP COLUMN " . join(", DROP COLUMN ", @drop_columns));
+    }
+}
+
+# Start Main
+
+eval {
+    if ($dry_run) {
+        print "** dry run : no changes to the database will be made **\n";
+        $dbh->bz_start_transaction();
+    }
+    print "Starting migration...\n";
+    do_migration();
+    $dbh->bz_rollback_transaction() if $dry_run;
+    print "All done!\n";
+};
+if ($@) {
+    $dbh->bz_rollback_transaction() if $dry_run;
+    die "$@" if $@;
+}
diff --git a/extensions/TrackingFlags/lib/Constants.pm b/extensions/TrackingFlags/lib/Constants.pm
index 57b2873e3..b6813c3c2 100644
--- a/extensions/TrackingFlags/lib/Constants.pm
+++ b/extensions/TrackingFlags/lib/Constants.pm
@@ -14,27 +14,34 @@ our @EXPORT = qw(
     FLAG_TYPES
 );
 
-use constant FLAG_TYPES => [
-    {
-        name        => 'tracking',
-        description => 'Tracking Flags',
-        collapsed   => 1,
-    },
-    {
-        name        => 'project',
-        description => 'Project Flags',
-        collapsed   => 0,
-    },
-    {
-        name        => 'blocking',
-        description => 'Blocking Flags',
-        collapsed   => 1,
-    },
-    {
-        name        => 'b2g',
-        description => 'B2G Flags',
-        collapsed   => 1,
-    }
-];
+sub FLAG_TYPES {
+    my @flag_types = (
+        {
+            name        => 'project',
+            description => 'Project Flags',
+            collapsed   => 0,
+            sortkey     => 0
+        },
+        {
+            name        => 'tracking',
+            description => 'Tracking Flags',
+            collapsed   => 1,
+            sortkey     => 1
+        },
+        {
+            name        => 'blocking',
+            description => 'Blocking Flags',
+            collapsed   => 1,
+            sortkey     => 2
+        },
+        {
+            name        => 'b2g',
+            description => 'B2G Flags',
+            collapsed   => 1,
+            sortkey     => 3
+        },
+    );
+    return [ sort { $a->{'sortkey'} <=> $b->{'sortkey'} } @flag_types ];
+}
 
 1;
diff --git a/extensions/TrackingFlags/template/en/default/hook/bug/create/create-bug_flags.html.tmpl b/extensions/TrackingFlags/template/en/default/hook/bug/create/create-bug_flags.html.tmpl
new file mode 100644
index 000000000..b41e1619f
--- /dev/null
+++ b/extensions/TrackingFlags/template/en/default/hook/bug/create/create-bug_flags.html.tmpl
@@ -0,0 +1,29 @@
+[%# This Source Code Form is subject to the terms of the Mozilla Public
+  # License, v. 2.0. If a copy of the MPL was not distributed with this
+  # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+  #
+  # This Source Code Form is "Incompatible With Secondary Licenses", as
+  # defined by the Mozilla Public License, v. 2.0.
+  #%]
+
+[% RETURN IF NOT tracking_flags.size %]
+<td>
+  <table class="tracking_flags">
+  [% FOREACH type = tracking_flag_types %]
+    [% flag_list = [] %]
+    [% FOREACH flag = tracking_flags %]
+      [% flag_list.push(flag) IF flag.flag_type == type.name %]
+    [% END %]
+    [% IF flag_list.size %]
+      <tr>
+        <th style="text-align:right">
+          [% type.description FILTER html %]:
+        </th>
+      </tr>
+      [% INCLUDE bug/tracking_flags.html.tmpl
+         flag_list = flag_list
+         new_bug   = 1 %]
+    [% END %]
+  [% END %]
+  </table>
+</td>
diff --git a/extensions/TrackingFlags/template/en/default/hook/bug/create/create-bug_flags_end.html.tmpl b/extensions/TrackingFlags/template/en/default/hook/bug/create/create-bug_flags_end.html.tmpl
deleted file mode 100644
index 2a90cbfe3..000000000
--- a/extensions/TrackingFlags/template/en/default/hook/bug/create/create-bug_flags_end.html.tmpl
+++ /dev/null
@@ -1,33 +0,0 @@
-[%# This Source Code Form is subject to the terms of the Mozilla Public
-  # License, v. 2.0. If a copy of the MPL was not distributed with this
-  # file, You can obtain one at http://mozilla.org/MPL/2.0/.
-  #
-  # This Source Code Form is "Incompatible With Secondary Licenses", as
-  # defined by the Mozilla Public License, v. 2.0.
-  #%]
-
-[% RETURN IF NOT tracking_flags.size %]
-
-[% FOREACH type = tracking_flag_types %]
-  [% NEXT IF type.name == 'tracking' || type.name == 'project' %]
-  [% flag_list = [] %]
-  [% FOREACH flag = tracking_flags %]
-      [% flag_list.push(flag) IF flag.flag_type == type.name %]
-  [% END %]
-  [% IF flag_list.size %]
-    <tr>
-      <td>
-        <table class="tracking_flags">
-        <tr>
-          <th>
-            [% type.description FILTER html %]:
-          </th>
-        </tr>
-        [% INCLUDE bug/tracking_flags.html.tmpl
-           flag_list = flag_list
-           new_bug   = 1 %]
-        </table>
-      </td>
-    </tr>
-  [% END %]
-[% END %]
diff --git a/extensions/TrackingFlags/template/en/default/hook/bug/create/create-project_flags_end.html.tmpl b/extensions/TrackingFlags/template/en/default/hook/bug/create/create-project_flags_end.html.tmpl
deleted file mode 100644
index 662bc26ee..000000000
--- a/extensions/TrackingFlags/template/en/default/hook/bug/create/create-project_flags_end.html.tmpl
+++ /dev/null
@@ -1,18 +0,0 @@
-[%# This Source Code Form is subject to the terms of the Mozilla Public
-  # License, v. 2.0. If a copy of the MPL was not distributed with this
-  # file, You can obtain one at http://mozilla.org/MPL/2.0/.
-  #
-  # This Source Code Form is "Incompatible With Secondary Licenses", as
-  # defined by the Mozilla Public License, v. 2.0.
-  #%]
-
-[% RETURN IF NOT tracking_flags.size %]
-
-[% flag_list = [] %]
-[% FOREACH flag = tracking_flags %]
-  [% NEXT IF flag.flag_type != 'project' %]
-  [% flag_list.push(flag) %]
-[% END %]
-[% INCLUDE bug/tracking_flags.html.tmpl
-   flag_list = flag_list
-   new_bug   = 1 %]
diff --git a/extensions/TrackingFlags/template/en/default/hook/bug/create/create-tracking_flags_end.html.tmpl b/extensions/TrackingFlags/template/en/default/hook/bug/create/create-tracking_flags_end.html.tmpl
deleted file mode 100644
index 69827a87a..000000000
--- a/extensions/TrackingFlags/template/en/default/hook/bug/create/create-tracking_flags_end.html.tmpl
+++ /dev/null
@@ -1,18 +0,0 @@
-[%# This Source Code Form is subject to the terms of the Mozilla Public
-  # License, v. 2.0. If a copy of the MPL was not distributed with this
-  # file, You can obtain one at http://mozilla.org/MPL/2.0/.
-  #
-  # This Source Code Form is "Incompatible With Secondary Licenses", as
-  # defined by the Mozilla Public License, v. 2.0.
-  #%]
-
-[% RETURN IF NOT tracking_flags.size %]
-
-[% flag_list = [] %]
-[% FOREACH flag = tracking_flags %]
-  [% NEXT IF flag.flag_type != 'tracking' %]
-  [% flag_list.push(flag) %]
-[% END %]
-[% INCLUDE bug/tracking_flags.html.tmpl
-   flag_list = flag_list
-   new_bug   = 1 %]
diff --git a/extensions/TrackingFlags/template/en/default/hook/bug/edit-after_custom_fields.html.tmpl b/extensions/TrackingFlags/template/en/default/hook/bug/edit-after_custom_fields.html.tmpl
index 697db75ce..e0411b512 100644
--- a/extensions/TrackingFlags/template/en/default/hook/bug/edit-after_custom_fields.html.tmpl
+++ b/extensions/TrackingFlags/template/en/default/hook/bug/edit-after_custom_fields.html.tmpl
@@ -6,145 +6,8 @@
   # defined by the Mozilla Public License, v. 2.0.
   #%]
 
-[%# Old style custom field based tracking flags %]
-[% old_tracking_flags = [] %]
-[% old_project_flags  = [] %]
-[% FOREACH field = Bugzilla.active_custom_fields(product=>bug.product_obj,component=>bug.component_obj,type=>2) %]
-  [% NEXT IF field.type == constants.FIELD_TYPE_EXTENSION %]
-  [% NEXT IF NOT user.id AND bug.${field.name} == "---" %]
-  [% NEXT IF cf_flag_disabled(field.name, bug) %]
-  [% IF cf_is_project_flag(field.name) %]
-    [% old_project_flags.push(field) %]
-  [% ELSE %]
-    [% old_tracking_flags.push(field) %]
-  [% END %]
-[% END %]
-
-[%# Add in the new tracking flags that are type tracking or project %]
-[% new_tracking_flags = [] %]
-[% new_project_flags  = [] %]
-[% IF tracking_flags.size %]
-  [% FOREACH flag = tracking_flags %]
-    [% IF flag.flag_type == 'tracking' %]
-      [% new_tracking_flags.push(flag) %]
-    [% END %]
-    [% IF flag.flag_type == 'project' %]
-      [% new_project_flags.push(flag) %]
-    [% END %]
-  [% END %]
-[% END %]
-
-[% IF old_project_flags.size || new_project_flags.size %]
-  <tr>
-    <td class="field_label">
-      <label>Project Flags:</label>
-    </td>
-    <td>
-      [% IF bug.check_can_change_field('flagtypes.name', 0, 1) %]
-        <table class="tracking_flags">
-        [% FOREACH field = old_project_flags %]
-          [% NEXT IF NOT user.id AND field.value == "---" %]
-          <tr id="row_[% field.name FILTER js %]">
-            <td>
-              <label for="[% field.name FILTER html %]">
-                [% field_descs.${field.name} FILTER html %]:
-              </label>
-            </td>
-            <td>
-              [% PROCESS bug/field.html.tmpl value = bug.${field.name}
-                                             editable = user.id
-                                             no_tds = 1 %]
-              [% IF user.id %]
-                <span id="ro_[% field.name FILTER html %]" class="bz_default_hidden">
-                  [% bug.${field.name} FILTER html %]
-                </span>
-              [% END %]
-            </td>
-          </tr>
-        [% END %]
-        [% INCLUDE bug/tracking_flags.html.tmpl
-           flag_list = new_project_flags %]
-        </table>
-      [% ELSE %]
-        [% FOREACH field = old_project_flags %]
-          [% NEXT IF bug.${field.name} == "---" %]
-          [% field_descs.${field.name} FILTER html %]: [% bug.${field.name} FILTER html %]<br>
-        [% END %]
-        [% FOREACH flag = project_flags %]
-          [% NEXT IF flag.bug_flag.value == '---' %]
-          [% flag.description FILTER html %]: [% flag.bug_flag.value FILTER html %]<br>
-        [% END %]
-      [% END %]
-    </td>
-  </tr>
-[% END %]
-
-[% IF old_tracking_flags.size || new_tracking_flags.size %]
-  <tr>
-    <td class="field_label">
-      <label>Tracking Flags:</label>
-    </td>
-    <td>
-      [% IF bug.check_can_change_field('flagtypes.name', 0, 1) %]
-        [% IF user.id %]
-          <span id="edit_tracking_flags_action">
-            (<a href="#" name="tracking" class="edit_tracking_flags_link">edit</a>)
-          </span>
-        [% END %]
-        <table class="tracking_flags">
-        [% FOREACH field = old_tracking_flags %]
-          [% NEXT IF NOT user.id AND field.value == "---" %]
-          <tr id="row_[% field.name FILTER js %]">
-            <td>
-              <label for="[% field.name FILTER html %]">
-                [% field_descs.${field.name} FILTER html %]:
-              </label>
-            </td>
-            <td>
-              [% PROCESS bug/field.html.tmpl
-                 value    = bug.${field.name}
-                 editable = user.id
-                 no_tds   = 1
-              %]
-              [% IF user.id %]
-                <span id="ro_[% field.name FILTER html %]" class="bz_default_hidden">
-                  [% bug.${field.name} FILTER html %]
-                </span>
-              [% END %]
-            </td>
-          </tr>
-        [% END %]
-        [% INCLUDE bug/tracking_flags.html.tmpl
-           flag_list = new_tracking_flags %]
-        </table>
-      [% ELSE %]
-        [% FOREACH field = old_tracking_flags %]
-          [% NEXT IF bug.${field.name} == "---" %]
-          [% field_descs.${field.name} FILTER html %]: [% bug.${field.name} FILTER html %]<br>
-        [% END %]
-        [% FOREACH flag = new_tracking_flags %]
-          [% NEXT IF flag.status == '---' %]
-          [% flag.description FILTER html %]: [% flag.bug_flag.value FILTER html %]<br>
-        [% END %]
-      [% END %]
-    </td>
-  </tr>
-  <script type="text/javascript">
-    TrackingFlags.flags['tracking'] = {};
-    [% FOREACH field = old_tracking_flags %]
-      TrackingFlags.flags['tracking']['[% field.name FILTER js %]'] = '[% bug.${field.name} FILTER js %]';
-    [% END %]
-    [% FOREACH flag = new_tracking_flags %]
-      TrackingFlags.flags['tracking']['[% flag.name FILTER js %]'] = '[% flag.bug_flag.value FILTER js %]';
-    [% END %]
-    TrackingFlags.types.push('tracking');
-  </script>
-[% END %]
-
-[%# Last, display any new style flags that are not type tracking or project %]
 [% IF tracking_flags.size %]
   [% FOREACH type = tracking_flag_types %]
-    [% NEXT IF type.name == 'tracking' || type.name == 'project' %]
     [% flag_list = [] %]
     [% FOREACH flag = tracking_flags %]
       [% flag_list.push(flag) IF flag.flag_type == type.name %]
diff --git a/template/en/default/bug/create/create.html.tmpl b/template/en/default/bug/create/create.html.tmpl
index 9deabac26..b357b3eb0 100644
--- a/template/en/default/bug/create/create.html.tmpl
+++ b/template/en/default/bug/create/create.html.tmpl
@@ -635,7 +635,7 @@ TUI_hide_default('attachment_text_field');
 
 <tbody>
 [%# non-tracking flags custom fields %]
-[% FOREACH field = Bugzilla.active_custom_fields(product=>product,type=>1) %]
+[% FOREACH field = Bugzilla.active_custom_fields(product=>product) %]
   [% NEXT IF field.type == constants.FIELD_TYPE_EXTENSION %]
   [% NEXT UNLESS field.enter_bug %]
   [%# crash-signature gets custom handling %]
@@ -673,18 +673,6 @@ TUI_hide_default('attachment_text_field');
 </tbody>
 [% END %]
 
-[% old_tracking_flags = [] %]
-[% old_project_flags  = [] %]
-[% FOREACH field = Bugzilla.active_custom_fields(product=>product,type=>2) %]
-  [% NEXT IF field.type == constants.FIELD_TYPE_EXTENSION %]
-  [% NEXT UNLESS field.enter_bug %]
-  [% IF cf_is_project_flag(field.name) %]
-    [% old_project_flags.push(field) %]
-  [% ELSE %]
-    [% old_tracking_flags.push(field) %]
-  [% END %]
-[% END %]
-
 [% display_flags = 0 %]
 [% any_flags_requesteeble = 0 %]
 [% FOREACH flag_type = product.flag_types.bug %]
@@ -693,7 +681,7 @@ TUI_hide_default('attachment_text_field');
   [% LAST IF display_flags && any_flags_requesteeable %]
 [% END %]
 
-[% IF old_project_flags.size || old_tracking_flags.size || display_flags %]
+[% IF tracking_flags.size || display_flags %]
   <tbody class="expert_fields">
     <tr>
       <th>Flags:</th>
@@ -708,65 +696,18 @@ TUI_hide_default('attachment_text_field');
 
           <fieldset>
             <legend>Set [% terms.bug %] flags</legend>
-
-            <table cellpadding="0" cellspacing="0">
-              <tr>
-              [% IF old_tracking_flags.size %]
-                <td [% IF project_flags.size %]rowspan="2"[% END %]>
-                  <table class="tracking_flags">
-                    <tr>
-                      <th colspan="2" style="text-align:left">Tracking Flags:</th>
-                    </tr>
-                    [% FOREACH field = old_tracking_flags %]
-                      [% SET value = ${field.name}.defined ? ${field.name} : "" %]
-                      <tr>
-                        [% INCLUDE bug/field.html.tmpl
-                           bug        = default
-                           field      = field
-                           value      = value
-                           editable   = 1
-                           value_span = 3
-                        %]
-                      </tr>
-                    [% END %]
-                    [% Hook.process('tracking_flags_end') %]
-                  </table>
-                </td>
-              [% END %]
-              [% IF old_project_flags.size %]
-                <td>
-                  <table class="tracking_flags">
-                    <tr>
-                      <th colspan="2" style="text-align:left">Project Flags:</th>
-                    </tr>
-                    [% FOREACH field = old_project_flags %]
-                      [% SET value = ${field.name}.defined ? ${field.name} : "" %]
-                      <tr>
-                        [% INCLUDE bug/field.html.tmpl
-                           bug        = default
-                           field      = field
-                           value      = value
-                           editable   = 1
-                           value_span = 3
-                        %]
-                      </tr>
-                    [% END %]
-                    [% Hook.process('project_flags_end') %]
-                  </table>
-                </td>
-              </tr>
+            <table>
               <tr>
-              [% END %]
-              [% IF display_flags %]
-                <td>
-                  [% PROCESS "flag/list.html.tmpl" flag_types = product.flag_types.bug
-                                                   any_flags_requesteeble = any_flags_requesteeble
-                                                   flag_table_id = "bug_flags"
-                  %]
-                </td>
-              [% END %]
+                [% Hook.process('bug_flags') %]
+                [% IF display_flags %]
+                  <td>
+                    [% PROCESS "flag/list.html.tmpl" flag_types = product.flag_types.bug
+                                                     any_flags_requesteeble = any_flags_requesteeble
+                                                     flag_table_id = "bug_flags"
+                    %]
+                  </td>
+                [% END %]
               </tr>
-              [% Hook.process('bug_flags_end') %]
             </table>
           </fieldset>
         </div>
diff --git a/template/en/default/bug/edit.html.tmpl b/template/en/default/bug/edit.html.tmpl
index 8c2d09872..fc0e41fe2 100644
--- a/template/en/default/bug/edit.html.tmpl
+++ b/template/en/default/bug/edit.html.tmpl
@@ -961,7 +961,7 @@
 [% BLOCK section_customfields %]
 [%# *** Custom Fields *** %]
   [% USE Bugzilla %]
-  [% FOREACH field = Bugzilla.active_custom_fields(product=>bug.product_obj,component=>bug.component_obj,type=>1) %]
+  [% FOREACH field = Bugzilla.active_custom_fields(product=>bug.product_obj,component=>bug.component_obj) %]
     [% NEXT IF field.type == constants.FIELD_TYPE_EXTENSION %]
     [% NEXT IF NOT user.id AND field.value == "---" %]
     [% Hook.process('custom_field', 'bug/edit.html.tmpl') %]
diff --git a/template/en/default/bug/show-multiple.html.tmpl b/template/en/default/bug/show-multiple.html.tmpl
index 207b3ed86..cfd0d8e20 100644
--- a/template/en/default/bug/show-multiple.html.tmpl
+++ b/template/en/default/bug/show-multiple.html.tmpl
@@ -191,9 +191,7 @@
 
     [% USE Bugzilla %]
     [% field_counter = 0 %]
-    [% FOREACH field = Bugzilla.active_custom_fields %]
-        [% NEXT IF cf_hidden_in_product(field.name, bug.product, bug.component) %]
-        [% NEXT IF cf_flag_disabled(field.name, bug) %]
+    [% FOREACH field = Bugzilla.active_custom_fields(product=>bug.product_obj,component=>bug.component_obj,bug_id=>bug.id) %]
         [% field_counter = field_counter + 1 %]
         [%# Odd-numbered fields get an opening <tr> %]
         [% '<tr>' IF field_counter % 2 %]
diff --git a/template/en/default/bug/show.xml.tmpl b/template/en/default/bug/show.xml.tmpl
index cb323d229..c0f32d69e 100644
--- a/template/en/default/bug/show.xml.tmpl
+++ b/template/en/default/bug/show.xml.tmpl
@@ -144,7 +144,6 @@
     [% ELSIF field == "see_also" %]
       [% val = val.name %]
     [% END %]
-    [% NEXT IF cf_hidden_in_product(field.name, bug.product, bug.component) %]
     <[% field %][% IF name != '' %] name="[% name FILTER xml %]"[% END -%]>
       [%- val FILTER xml %]</[% field %]>
   [% END %]
diff --git a/template/en/default/list/edit-multiple.html.tmpl b/template/en/default/list/edit-multiple.html.tmpl
index 9ff95aad5..8f94105a7 100644
--- a/template/en/default/list/edit-multiple.html.tmpl
+++ b/template/en/default/list/edit-multiple.html.tmpl
@@ -284,7 +284,6 @@
   [%# Show all legal values and all fields, ignoring visibility controls. %]
   [% bug = default.defined ? default : 0 %]
   [% FOREACH field = Bugzilla.active_custom_fields %]
-    [% NEXT IF cf_hidden_in_product(field.name, one_product, components) %]
     <tr>
       [% PROCESS bug/field.html.tmpl bug = default
                                      value = dontchange