Bug 237498: Add memcached integration

r=dkl, a=sgreen

6 files changed, 454 insertions, 23 deletions

diff --git a/Bugzilla/Bug.pm b/Bugzilla/Bug.pm
index a7be3812d..f0476c898 100644
--- a/Bugzilla/Bug.pm
+++ b/Bugzilla/Bug.pm
@@ -353,9 +353,9 @@ sub initialize {
     $_[0]->_create_cf_accessors();
 }
 
-sub cache_key {
+sub object_cache_key {
     my $class = shift;
-    my $key = $class->SUPER::cache_key(@_)
+    my $key = $class->SUPER::object_cache_key(@_)
       || return;
     return $key . ',' . Bugzilla->user->id;
 }
@@ -4422,7 +4422,7 @@ Ensures the accessors for custom fields are always created.
 
 =item set_op_sys
 
-=item cache_key
+=item object_cache_key
 
 =item bug_group
 
diff --git a/Bugzilla/Config/Memcached.pm b/Bugzilla/Config/Memcached.pm
new file mode 100644
index 000000000..08d8ce0e7
--- /dev/null
+++ b/Bugzilla/Config/Memcached.pm
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This Source Code Form is "Incompatible With Secondary Licenses", as
+# defined by the Mozilla Public License, v. 2.0.
+
+package Bugzilla::Config::Memcached;
+
+use 5.10.1;
+use strict;
+
+use Bugzilla::Config::Common;
+
+our $sortkey = 1550;
+
+sub get_param_list {
+  return (
+    {
+        name    => 'memcached_servers',
+        type    => 't',
+        default => ''
+    },
+    {
+        name    => 'memcached_namespace',
+        type    => 't',
+        default => 'bugzilla:',
+    },
+  );
+}
+
+1;
diff --git a/Bugzilla/DB.pm b/Bugzilla/DB.pm
index 063e2cf69..df84d9c79 100644
--- a/Bugzilla/DB.pm
+++ b/Bugzilla/DB.pm
@@ -1362,14 +1362,19 @@ sub _bz_real_schema {
     my ($self) = @_;
     return $self->{private_real_schema} if exists $self->{private_real_schema};
 
-    my ($data, $version) = $self->selectrow_array(
-        "SELECT schema_data, version FROM bz_schema");
+    my $bz_schema;
+    unless ($bz_schema = Bugzilla->memcached->get({ key => 'bz_schema' })) {
+        $bz_schema = $self->selectrow_arrayref(
+            "SELECT schema_data, version FROM bz_schema"
+        );
+        Bugzilla->memcached->set({ key => 'bz_schema', value => $bz_schema });
+    }
 
     (die "_bz_real_schema tried to read the bz_schema table but it's empty!")
-        if !$data;
+        if !$bz_schema;
 
-    $self->{private_real_schema} = 
-        $self->_bz_schema->deserialize_abstract($data, $version);
+    $self->{private_real_schema} =
+        $self->_bz_schema->deserialize_abstract($bz_schema->[0], $bz_schema->[1]);
 
     return $self->{private_real_schema};
 }
@@ -1411,6 +1416,8 @@ sub _bz_store_real_schema {
     $sth->bind_param(1, $store_me, $self->BLOB_TYPE);
     $sth->bind_param(2, $schema_version);
     $sth->execute();
+
+    Bugzilla->memcached->clear({ key => 'bz_schema' });
 }
 
 # For bz_populate_enum_tables
diff --git a/Bugzilla/Install/Requirements.pm b/Bugzilla/Install/Requirements.pm
index 09ea4abe3..cd8978eb1 100644
--- a/Bugzilla/Install/Requirements.pm
+++ b/Bugzilla/Install/Requirements.pm
@@ -394,6 +394,14 @@ sub OPTIONAL_MODULES {
         version => '0',
         feature => ['typesniffer'],
     },
+
+    # memcached
+    {
+        package => 'Cache-Memcached',
+        module  => 'Cache::Memcached',
+        version => '0',
+        feature => ['memcached'],
+    },
     );
 
     my $extra_modules = _get_extension_requirements('OPTIONAL_MODULES');
@@ -417,6 +425,7 @@ use constant FEATURE_FILES => (
                       'Bugzilla/JobQueue/*', 'jobqueue.pl'],
     patch_viewer  => ['Bugzilla/Attachment/PatchReader.pm'],
     updates       => ['Bugzilla/Update.pm'],
+    memcached     => ['Bugzilla/Memcache.pm'],
 );
 
 # This implements the REQUIRED_MODULES and OPTIONAL_MODULES stuff
diff --git a/Bugzilla/Memcached.pm b/Bugzilla/Memcached.pm
new file mode 100644
index 000000000..b1b10311b
--- /dev/null
+++ b/Bugzilla/Memcached.pm
@@ -0,0 +1,346 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This Source Code Form is "Incompatible With Secondary Licenses", as
+# defined by the Mozilla Public License, v. 2.0.
+
+package Bugzilla::Memcached;
+
+use 5.10.1;
+use strict;
+use warnings;
+
+use Bugzilla::Error;
+use Bugzilla::Util qw(trick_taint);
+use Scalar::Util qw(blessed);
+
+sub _new {
+    my $invocant = shift;
+    my $class = ref($invocant) || $invocant;
+    my $self = {};
+
+    # always return an object to simplify calling code when memcached is
+    # disabled.
+    if (Bugzilla->feature('memcached')
+        && Bugzilla->params->{memcached_servers})
+    {
+        require Cache::Memcached;
+        $self->{memcached} =
+            Cache::Memcached->new({
+                servers   => [ split(/[, ]+/, Bugzilla->params->{memcached_servers}) ],
+                namespace => Bugzilla->params->{memcached_namespace} || '',
+            });
+    }
+    return bless($self, $class);
+}
+
+sub set {
+    my ($self, $args) = @_;
+    return unless $self->{memcached};
+
+    # { key => $key, value => $value }
+    if (exists $args->{key}) {
+        $self->_set($args->{key}, $args->{value});
+    }
+
+    # { table => $table, id => $id, name => $name, data => $data }
+    elsif (exists $args->{table} && exists $args->{id} && exists $args->{name}) {
+        # For caching of Bugzilla::Object, we have to be able to clear the
+        # cached values when given either the object's id or name.
+        my ($table, $id, $name, $data) = @$args{qw(table id name data)};
+        $self->_set("$table.id.$id", $data);
+        if (defined $name) {
+            $self->_set("$table.name_id.$name", $id);
+            $self->_set("$table.id_name.$id", $name);
+        }
+    }
+
+    else {
+        ThrowCodeError('params_required', { function => "Bugzilla::Memcached::set",
+                                            params   => [ 'key', 'table' ] });
+    }
+}
+
+sub get {
+    my ($self, $args) = @_;
+    return unless $self->{memcached};
+
+    # { key => $key }
+    if (exists $args->{key}) {
+        return $self->_get($args->{key});
+    }
+
+    # { table => $table, id => $id }
+    elsif (exists $args->{table} && exists $args->{id}) {
+        my ($table, $id) = @$args{qw(table id)};
+        return $self->_get("$table.id.$id");
+    }
+
+    # { table => $table, name => $name }
+    elsif (exists $args->{table} && exists $args->{name}) {
+        my ($table, $name) = @$args{qw(table name)};
+        return unless my $id = $self->_get("$table.name_id.$name");
+        return $self->_get("$table.id.$id");
+    }
+
+    else {
+        ThrowCodeError('params_required', { function => "Bugzilla::Memcached::get",
+                                            params   => [ 'key', 'table' ] });
+    }
+}
+
+sub clear {
+    my ($self, $args) = @_;
+    return unless $self->{memcached};
+
+    # { key => $key }
+    if (exists $args->{key}) {
+        $self->_delete($args->{key});
+    }
+
+    # { table => $table, id => $id }
+    elsif (exists $args->{table} && exists $args->{id}) {
+        my ($table, $id) = @$args{qw(table id)};
+        my $name = $self->_get("$table.id_name.$id");
+        $self->_delete("$table.id.$id");
+        $self->_delete("$table.name_id.$name") if defined $name;
+        $self->_delete("$table.id_name.$id");
+    }
+
+    # { table => $table, name => $name }
+    elsif (exists $args->{table} && exists $args->{name}) {
+        my ($table, $name) = @$args{qw(table name)};
+        return unless my $id = $self->_get("$table.name_id.$name");
+        $self->_delete("$table.id.$id");
+        $self->_delete("$table.name_id.$name");
+        $self->_delete("$table.id_name.$id");
+    }
+
+    else {
+        ThrowCodeError('params_required', { function => "Bugzilla::Memcached::clear",
+                                            params   => [ 'key', 'table' ] });
+    }
+}
+
+sub clear_all {
+    my ($self) = @_;
+    return unless my $memcached = $self->{memcached};
+    if (!$memcached->incr("prefix", 1)) {
+        $memcached->add("prefix", time());
+    }
+}
+
+# in order to clear all our keys, we add a prefix to all our keys.  when we
+# need to "clear" all current keys, we increment the prefix.
+sub _prefix {
+    my ($self) = @_;
+    # we don't want to change prefixes in the middle of a request
+    my $request_cache = Bugzilla->request_cache;
+    if (!$request_cache->{memcached_prefix}) {
+        my $memcached = $self->{memcached};
+        my $prefix = $memcached->get("prefix");
+        if (!$prefix) {
+            $prefix = time();
+            if (!$memcached->add("prefix", $prefix)) {
+                # if this failed, either another process set the prefix, or
+                # memcached is down.  assume we lost the race, and get the new
+                # value.  if that fails, memcached is down so use a dummy
+                # prefix for this request.
+                $prefix = $memcached->get("prefix") || 0;
+            }
+        }
+        $request_cache->{memcached_prefix} = $prefix;
+    }
+    return $request_cache->{memcached_prefix};
+}
+
+sub _set {
+    my ($self, $key, $value) = @_;
+    if (blessed($value)) {
+        # we don't support blessed objects
+        ThrowCodeError('param_invalid', { function => "Bugzilla::Memcached::set",
+                                          param    => "value" });
+    }
+    return $self->{memcached}->set($self->_prefix . ':' . $key, $value);
+}
+
+sub _get {
+    my ($self, $key) = @_;
+
+    my $value = $self->{memcached}->get($self->_prefix . ':' . $key);
+    return unless defined $value;
+
+    # detaint returned values
+    # hashes and arrays are detainted just one level deep
+    if (ref($value) eq 'HASH') {
+        map { defined($_) && trick_taint($_) } values %$value;
+    }
+    elsif (ref($value) eq 'ARRAY') {
+        trick_taint($_) foreach @$value;
+    }
+    elsif (!ref($value)) {
+        trick_taint($value);
+    }
+    return $value;
+}
+
+sub _delete {
+    my ($self, $key) = @_;
+    return $self->{memcached}->delete($self->_prefix . ':' . $key);
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+Bugzilla::Memcached - Interface between Bugzilla and Memcached.
+
+=head1 SYNOPSIS
+
+ use Bugzilla;
+
+ my $memcached = Bugzilla->memcached;
+
+ # grab data from the cache. there is no need to check if memcached is
+ # available or enabled.
+ my $data = $memcached->get({ key => 'data_key' });
+ if (!defined $data) {
+     # not in cache, generate the data and populate the cache for next time
+     $data = some_long_process();
+     $memcached->set({ key => 'data_key', value => $data });
+ }
+ # do something with $data
+
+ # updating the profiles table directly shouldn't be attempted unless you know
+ # what you're doing. if you do update a table directly, you need to clear that
+ # object from memcached.
+ $dbh->do("UPDATE profiles SET request_count=10 WHERE login_name=?", undef, $login);
+ $memcached->clear({ table => 'profiles', name => $login });
+
+=head1 DESCRIPTION
+
+If Memcached is installed and configured, Bugzilla can use it to cache data
+across requests and between webheads. Unlike the request and process caches,
+only scalars, hashrefs, and arrayrefs can be stored in Memcached.
+
+Memcached integration is only required for large installations of Bugzilla --
+if you have multiple webheads then configuring Memcache is recommended.
+
+L<Bugzilla::Memcached> provides an interface to a Memcached server/servers, with
+the ability to get, set, or clear entries from the cache.
+
+The stored value must be an unblessed hashref, unblessed array ref, or a
+scalar.  Currently nested data structures are supported but require manual
+de-tainting after reading from Memcached (flat data structures are automatically
+de-tainted).
+
+All values are stored in the Memcached systems using the prefix configured with
+the C<memcached_namespace> parameter, as well as an additional prefix managed
+by this class to allow all values to be cleared when C<checksetup.pl> is
+executed.
+
+Do not create an instance of this object directly, instead use
+L<Bugzilla-E<gt>memcached()|Bugzilla/memcached>.
+
+=head1 METHODS
+
+=head2 Setting
+
+Adds a value to Memcached.
+
+=over
+
+=item C<set({ key =E<gt> $key, value =E<gt> $value })>
+
+Adds the C<value> using the specific C<key>.
+
+=item C<set({ table =E<gt> $table, id =E<gt> $id, name =E<gt> $name, data =E<gt> $data })>
+
+Adds the C<data> using a keys generated from the C<table>, C<id>, and C<name>.
+All three parameters must be provided, however C<name> can be provided but set
+to C<undef>.
+
+This is a convenience method which allows cached data to be later retrieved by
+specifying the C<table> and either the C<id> or C<name>.
+
+=back
+
+=head2 Getting
+
+Retrieves a value from Memcached.  Returns C<undef> if no matching values were
+found in the cache.
+
+=over
+
+=item C<get({ key =E<gt> $key })>
+
+Return C<value> with the specified C<key>.
+
+=item C<get({ table =E<gt> $table, id =E<gt> $id })>
+
+Return C<value> with the specified C<table> and C<id>.
+
+=item C<get({ table =E<gt> $table, name =E<gt> $name })>
+
+Return C<value> with the specified C<table> and C<name>.
+
+=back
+
+=head2 Clearing
+
+Removes the matching value from Memcached.
+
+=over
+
+=item C<clear({ key =E<gt> $key })>
+
+Removes C<value> with the specified C<key>.
+
+=item C<clear({ table =E<gt> $table, id =E<gt> $id })>
+
+Removes C<value> with the specified C<table> and C<id>, as well as the
+corresponding C<table> and C<name> entry.
+
+=item C<clear({ table =E<gt> $table, name =E<gt> $name })>
+
+Removes C<value> with the specified C<table> and C<name>, as well as the
+corresponding C<table> and C<id> entry.
+
+=item C<clear_all>
+
+Removes all values from the cache.
+
+=back
+
+=head1 Bugzilla::Object CACHE
+
+The main driver for Memcached integration is to allow L<Bugzilla::Object> based
+objects to be automatically cached in Memcache. This is enabled on a
+per-package basis by setting the C<USE_MEMCACHED> constant to any true value.
+
+The current implementation is an opt-in (USE_MEMCACHED is false by default),
+however this will change to opt-out once further testing has been completed
+(USE_MEMCACHED will be true by default).
+
+=head1 DIRECT DATABASE UPDATES
+
+If an object is cached and the database is updated directly (instead of via
+C<$object-E<gt>update()>), then it's possible for the data in the cache to be
+out of sync with the database.
+
+As an example let's consider an extension which adds a timestamp field
+C<last_activitiy_ts> to the profiles table and user object which contains the
+user's last activity.  If the extension were to call C<$user-E<gt>update()>,
+then an audit entry would be created for each change to the C<last_activity_ts>
+field, which is undesirable.
+
+To remedy this, the extension updates the table directly.  It's critical with
+Memcached that it then clears the cache:
+
+ $dbh->do("UPDATE profiles SET last_activity_ts=? WHERE userid=?",
+          undef, $timestamp, $user_id);
+ Bugzilla->memcached->clear({ table => 'profiles', id => $user_id });
+
diff --git a/Bugzilla/Object.pm b/Bugzilla/Object.pm
index 16b3a1ebb..a31392353 100644
--- a/Bugzilla/Object.pm
+++ b/Bugzilla/Object.pm
@@ -34,6 +34,11 @@ use constant AUDIT_CREATES => 1;
 use constant AUDIT_UPDATES => 1;
 use constant AUDIT_REMOVES => 1;
 
+# When USE_MEMCACHED is true, the class is suitable for serialisation to
+# Memcached. This will be flipped to true by default once the majority of
+# Bugzilla Object have been tested with Memcached.
+use constant USE_MEMCACHED => 0;
+
 # This allows the JSON-RPC interface to return Bugzilla::Object instances
 # as though they were hashes. In the future, this may be modified to return
 # less information.
@@ -48,11 +53,41 @@ sub new {
     my $class    = ref($invocant) || $invocant;
     my $param    = shift;
 
-    my $object = $class->_cache_get($param);
+    my $object = $class->_object_cache_get($param);
     return $object if $object;
 
-    $object = $class->new_from_hash($class->_load_from_db($param));
-    $class->_cache_set($param, $object);
+    my ($data, $set_memcached);
+    if (Bugzilla->feature('memcached')
+        && $class->USE_MEMCACHED
+        && ref($param) eq 'HASH' && $param->{cache})
+    {
+        if (defined $param->{id}) {
+            $data = Bugzilla->memcached->get({
+                table => $class->DB_TABLE,
+                id    => $param->{id},
+            });
+        }
+        elsif (defined $param->{name}) {
+            $data = Bugzilla->memcached->get({
+                table => $class->DB_TABLE,
+                name  => $param->{name},
+            });
+        }
+        $set_memcached = $data ? 0 : 1;
+    }
+    $data ||= $class->_load_from_db($param);
+
+    if ($data && $set_memcached) {
+        Bugzilla->memcached->set({
+            table => $class->DB_TABLE,
+            id    => $data->{$class->ID_FIELD},
+            name  => $data->{$class->NAME_FIELD},
+            data  => $data,
+        });
+    }
+
+    $object = $class->new_from_hash($data);
+    $class->_object_cache_set($param, $object);
 
     return $object;
 }
@@ -157,32 +192,32 @@ sub initialize {
 }
 
 # Provides a mechanism for objects to be cached in the request_cache
-sub _cache_get {
+sub _object_cache_get {
     my $class = shift;
     my ($param) = @_;
-    my $cache_key = $class->cache_key($param)
+    my $cache_key = $class->object_cache_key($param)
       || return;
     return Bugzilla->request_cache->{$cache_key};
 }
 
-sub _cache_set {
+sub _object_cache_set {
     my $class = shift;
     my ($param, $object) = @_;
-    my $cache_key = $class->cache_key($param)
+    my $cache_key = $class->object_cache_key($param)
       || return;
     Bugzilla->request_cache->{$cache_key} = $object;
 }
 
-sub _cache_remove {
+sub _object_cache_remove {
     my $class = shift;
     my ($param) = @_;
     $param->{cache} = 1;
-    my $cache_key = $class->cache_key($param)
+    my $cache_key = $class->object_cache_key($param)
       || return;
     delete Bugzilla->request_cache->{$cache_key};
 }
 
-sub cache_key {
+sub object_cache_key {
     my $class = shift;
     my ($param) = @_;
     if (ref($param) && $param->{cache} && ($param->{id} || $param->{name})) {
@@ -461,8 +496,9 @@ sub update {
     $self->audit_log(\%changes) if $self->AUDIT_UPDATES;
 
     $dbh->bz_commit_transaction();
-    $self->_cache_remove({ id => $self->id });
-    $self->_cache_remove({ name => $self->name }) if $self->name;
+    Bugzilla->memcached->clear({ table => $table, id => $self->id });
+    $self->_object_cache_remove({ id => $self->id });
+    $self->_object_cache_remove({ name => $self->name }) if $self->name;
 
     if (wantarray) {
         return (\%changes, $old_self);
@@ -481,8 +517,9 @@ sub remove_from_db {
     $self->audit_log(AUDIT_REMOVE) if $self->AUDIT_REMOVES;
     $dbh->do("DELETE FROM $table WHERE $id_field = ?", undef, $self->id);
     $dbh->bz_commit_transaction();
-    $self->_cache_remove({ id => $self->id });
-    $self->_cache_remove({ name => $self->name }) if $self->name;
+    Bugzilla->memcached->clear({ table => $table, id => $self->id });
+    $self->_object_cache_remove({ id => $self->id });
+    $self->_object_cache_remove({ name => $self->name }) if $self->name;
     undef $self;
 }
 
@@ -1399,7 +1436,7 @@ C<0> otherwise.
 
 =over
 
-=item cache_key
+=item object_cache_key
 
 =item check_time