diff options
Diffstat (limited to 'Bugzilla')
| -rw-r--r-- | Bugzilla/Bug.pm | 6 | ||||
| -rw-r--r-- | Bugzilla/Config/Memcached.pm | 32 | ||||
| -rw-r--r-- | Bugzilla/DB.pm | 17 | ||||
| -rw-r--r-- | Bugzilla/Install/Requirements.pm | 9 | ||||
| -rw-r--r-- | Bugzilla/Memcached.pm | 346 | ||||
| -rw-r--r-- | Bugzilla/Object.pm | 67 |
6 files changed, 454 insertions, 23 deletions
diff --git a/Bugzilla/Bug.pm b/Bugzilla/Bug.pm index a7be3812d..f0476c898 100644 --- a/Bugzilla/Bug.pm +++ b/Bugzilla/Bug.pm @@ -353,9 +353,9 @@ sub initialize { $_[0]->_create_cf_accessors(); } -sub cache_key { +sub object_cache_key { my $class = shift; - my $key = $class->SUPER::cache_key(@_) + my $key = $class->SUPER::object_cache_key(@_) || return; return $key . ',' . Bugzilla->user->id; } @@ -4422,7 +4422,7 @@ Ensures the accessors for custom fields are always created. =item set_op_sys -=item cache_key +=item object_cache_key =item bug_group diff --git a/Bugzilla/Config/Memcached.pm b/Bugzilla/Config/Memcached.pm new file mode 100644 index 000000000..08d8ce0e7 --- /dev/null +++ b/Bugzilla/Config/Memcached.pm @@ -0,0 +1,32 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Config::Memcached; + +use 5.10.1; +use strict; + +use Bugzilla::Config::Common; + +our $sortkey = 1550; + +sub get_param_list { + return ( + { + name => 'memcached_servers', + type => 't', + default => '' + }, + { + name => 'memcached_namespace', + type => 't', + default => 'bugzilla:', + }, + ); +} + +1; diff --git a/Bugzilla/DB.pm b/Bugzilla/DB.pm index 063e2cf69..df84d9c79 100644 --- a/Bugzilla/DB.pm +++ b/Bugzilla/DB.pm @@ -1362,14 +1362,19 @@ sub _bz_real_schema { my ($self) = @_; return $self->{private_real_schema} if exists $self->{private_real_schema}; - my ($data, $version) = $self->selectrow_array( - "SELECT schema_data, version FROM bz_schema"); + my $bz_schema; + unless ($bz_schema = Bugzilla->memcached->get({ key => 'bz_schema' })) { + $bz_schema = $self->selectrow_arrayref( + "SELECT schema_data, version FROM bz_schema" + ); + Bugzilla->memcached->set({ key => 'bz_schema', value => $bz_schema }); + } (die "_bz_real_schema tried to read the bz_schema table but it's empty!") - if !$data; + if !$bz_schema; - $self->{private_real_schema} = - $self->_bz_schema->deserialize_abstract($data, $version); + $self->{private_real_schema} = + $self->_bz_schema->deserialize_abstract($bz_schema->[0], $bz_schema->[1]); return $self->{private_real_schema}; } @@ -1411,6 +1416,8 @@ sub _bz_store_real_schema { $sth->bind_param(1, $store_me, $self->BLOB_TYPE); $sth->bind_param(2, $schema_version); $sth->execute(); + + Bugzilla->memcached->clear({ key => 'bz_schema' }); } # For bz_populate_enum_tables diff --git a/Bugzilla/Install/Requirements.pm b/Bugzilla/Install/Requirements.pm index 09ea4abe3..cd8978eb1 100644 --- a/Bugzilla/Install/Requirements.pm +++ b/Bugzilla/Install/Requirements.pm @@ -394,6 +394,14 @@ sub OPTIONAL_MODULES { version => '0', feature => ['typesniffer'], }, + + # memcached + { + package => 'Cache-Memcached', + module => 'Cache::Memcached', + version => '0', + feature => ['memcached'], + }, ); my $extra_modules = _get_extension_requirements('OPTIONAL_MODULES'); @@ -417,6 +425,7 @@ use constant FEATURE_FILES => ( 'Bugzilla/JobQueue/*', 'jobqueue.pl'], patch_viewer => ['Bugzilla/Attachment/PatchReader.pm'], updates => ['Bugzilla/Update.pm'], + memcached => ['Bugzilla/Memcache.pm'], ); # This implements the REQUIRED_MODULES and OPTIONAL_MODULES stuff diff --git a/Bugzilla/Memcached.pm b/Bugzilla/Memcached.pm new file mode 100644 index 000000000..b1b10311b --- /dev/null +++ b/Bugzilla/Memcached.pm @@ -0,0 +1,346 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Memcached; + +use 5.10.1; +use strict; +use warnings; + +use Bugzilla::Error; +use Bugzilla::Util qw(trick_taint); +use Scalar::Util qw(blessed); + +sub _new { + my $invocant = shift; + my $class = ref($invocant) || $invocant; + my $self = {}; + + # always return an object to simplify calling code when memcached is + # disabled. + if (Bugzilla->feature('memcached') + && Bugzilla->params->{memcached_servers}) + { + require Cache::Memcached; + $self->{memcached} = + Cache::Memcached->new({ + servers => [ split(/[, ]+/, Bugzilla->params->{memcached_servers}) ], + namespace => Bugzilla->params->{memcached_namespace} || '', + }); + } + return bless($self, $class); +} + +sub set { + my ($self, $args) = @_; + return unless $self->{memcached}; + + # { key => $key, value => $value } + if (exists $args->{key}) { + $self->_set($args->{key}, $args->{value}); + } + + # { table => $table, id => $id, name => $name, data => $data } + elsif (exists $args->{table} && exists $args->{id} && exists $args->{name}) { + # For caching of Bugzilla::Object, we have to be able to clear the + # cached values when given either the object's id or name. + my ($table, $id, $name, $data) = @$args{qw(table id name data)}; + $self->_set("$table.id.$id", $data); + if (defined $name) { + $self->_set("$table.name_id.$name", $id); + $self->_set("$table.id_name.$id", $name); + } + } + + else { + ThrowCodeError('params_required', { function => "Bugzilla::Memcached::set", + params => [ 'key', 'table' ] }); + } +} + +sub get { + my ($self, $args) = @_; + return unless $self->{memcached}; + + # { key => $key } + if (exists $args->{key}) { + return $self->_get($args->{key}); + } + + # { table => $table, id => $id } + elsif (exists $args->{table} && exists $args->{id}) { + my ($table, $id) = @$args{qw(table id)}; + return $self->_get("$table.id.$id"); + } + + # { table => $table, name => $name } + elsif (exists $args->{table} && exists $args->{name}) { + my ($table, $name) = @$args{qw(table name)}; + return unless my $id = $self->_get("$table.name_id.$name"); + return $self->_get("$table.id.$id"); + } + + else { + ThrowCodeError('params_required', { function => "Bugzilla::Memcached::get", + params => [ 'key', 'table' ] }); + } +} + +sub clear { + my ($self, $args) = @_; + return unless $self->{memcached}; + + # { key => $key } + if (exists $args->{key}) { + $self->_delete($args->{key}); + } + + # { table => $table, id => $id } + elsif (exists $args->{table} && exists $args->{id}) { + my ($table, $id) = @$args{qw(table id)}; + my $name = $self->_get("$table.id_name.$id"); + $self->_delete("$table.id.$id"); + $self->_delete("$table.name_id.$name") if defined $name; + $self->_delete("$table.id_name.$id"); + } + + # { table => $table, name => $name } + elsif (exists $args->{table} && exists $args->{name}) { + my ($table, $name) = @$args{qw(table name)}; + return unless my $id = $self->_get("$table.name_id.$name"); + $self->_delete("$table.id.$id"); + $self->_delete("$table.name_id.$name"); + $self->_delete("$table.id_name.$id"); + } + + else { + ThrowCodeError('params_required', { function => "Bugzilla::Memcached::clear", + params => [ 'key', 'table' ] }); + } +} + +sub clear_all { + my ($self) = @_; + return unless my $memcached = $self->{memcached}; + if (!$memcached->incr("prefix", 1)) { + $memcached->add("prefix", time()); + } +} + +# in order to clear all our keys, we add a prefix to all our keys. when we +# need to "clear" all current keys, we increment the prefix. +sub _prefix { + my ($self) = @_; + # we don't want to change prefixes in the middle of a request + my $request_cache = Bugzilla->request_cache; + if (!$request_cache->{memcached_prefix}) { + my $memcached = $self->{memcached}; + my $prefix = $memcached->get("prefix"); + if (!$prefix) { + $prefix = time(); + if (!$memcached->add("prefix", $prefix)) { + # if this failed, either another process set the prefix, or + # memcached is down. assume we lost the race, and get the new + # value. if that fails, memcached is down so use a dummy + # prefix for this request. + $prefix = $memcached->get("prefix") || 0; + } + } + $request_cache->{memcached_prefix} = $prefix; + } + return $request_cache->{memcached_prefix}; +} + +sub _set { + my ($self, $key, $value) = @_; + if (blessed($value)) { + # we don't support blessed objects + ThrowCodeError('param_invalid', { function => "Bugzilla::Memcached::set", + param => "value" }); + } + return $self->{memcached}->set($self->_prefix . ':' . $key, $value); +} + +sub _get { + my ($self, $key) = @_; + + my $value = $self->{memcached}->get($self->_prefix . ':' . $key); + return unless defined $value; + + # detaint returned values + # hashes and arrays are detainted just one level deep + if (ref($value) eq 'HASH') { + map { defined($_) && trick_taint($_) } values %$value; + } + elsif (ref($value) eq 'ARRAY') { + trick_taint($_) foreach @$value; + } + elsif (!ref($value)) { + trick_taint($value); + } + return $value; +} + +sub _delete { + my ($self, $key) = @_; + return $self->{memcached}->delete($self->_prefix . ':' . $key); +} + +1; + +__END__ + +=head1 NAME + +Bugzilla::Memcached - Interface between Bugzilla and Memcached. + +=head1 SYNOPSIS + + use Bugzilla; + + my $memcached = Bugzilla->memcached; + + # grab data from the cache. there is no need to check if memcached is + # available or enabled. + my $data = $memcached->get({ key => 'data_key' }); + if (!defined $data) { + # not in cache, generate the data and populate the cache for next time + $data = some_long_process(); + $memcached->set({ key => 'data_key', value => $data }); + } + # do something with $data + + # updating the profiles table directly shouldn't be attempted unless you know + # what you're doing. if you do update a table directly, you need to clear that + # object from memcached. + $dbh->do("UPDATE profiles SET request_count=10 WHERE login_name=?", undef, $login); + $memcached->clear({ table => 'profiles', name => $login }); + +=head1 DESCRIPTION + +If Memcached is installed and configured, Bugzilla can use it to cache data +across requests and between webheads. Unlike the request and process caches, +only scalars, hashrefs, and arrayrefs can be stored in Memcached. + +Memcached integration is only required for large installations of Bugzilla -- +if you have multiple webheads then configuring Memcache is recommended. + +L<Bugzilla::Memcached> provides an interface to a Memcached server/servers, with +the ability to get, set, or clear entries from the cache. + +The stored value must be an unblessed hashref, unblessed array ref, or a +scalar. Currently nested data structures are supported but require manual +de-tainting after reading from Memcached (flat data structures are automatically +de-tainted). + +All values are stored in the Memcached systems using the prefix configured with +the C<memcached_namespace> parameter, as well as an additional prefix managed +by this class to allow all values to be cleared when C<checksetup.pl> is +executed. + +Do not create an instance of this object directly, instead use +L<Bugzilla-E<gt>memcached()|Bugzilla/memcached>. + +=head1 METHODS + +=head2 Setting + +Adds a value to Memcached. + +=over + +=item C<set({ key =E<gt> $key, value =E<gt> $value })> + +Adds the C<value> using the specific C<key>. + +=item C<set({ table =E<gt> $table, id =E<gt> $id, name =E<gt> $name, data =E<gt> $data })> + +Adds the C<data> using a keys generated from the C<table>, C<id>, and C<name>. +All three parameters must be provided, however C<name> can be provided but set +to C<undef>. + +This is a convenience method which allows cached data to be later retrieved by +specifying the C<table> and either the C<id> or C<name>. + +=back + +=head2 Getting + +Retrieves a value from Memcached. Returns C<undef> if no matching values were +found in the cache. + +=over + +=item C<get({ key =E<gt> $key })> + +Return C<value> with the specified C<key>. + +=item C<get({ table =E<gt> $table, id =E<gt> $id })> + +Return C<value> with the specified C<table> and C<id>. + +=item C<get({ table =E<gt> $table, name =E<gt> $name })> + +Return C<value> with the specified C<table> and C<name>. + +=back + +=head2 Clearing + +Removes the matching value from Memcached. + +=over + +=item C<clear({ key =E<gt> $key })> + +Removes C<value> with the specified C<key>. + +=item C<clear({ table =E<gt> $table, id =E<gt> $id })> + +Removes C<value> with the specified C<table> and C<id>, as well as the +corresponding C<table> and C<name> entry. + +=item C<clear({ table =E<gt> $table, name =E<gt> $name })> + +Removes C<value> with the specified C<table> and C<name>, as well as the +corresponding C<table> and C<id> entry. + +=item C<clear_all> + +Removes all values from the cache. + +=back + +=head1 Bugzilla::Object CACHE + +The main driver for Memcached integration is to allow L<Bugzilla::Object> based +objects to be automatically cached in Memcache. This is enabled on a +per-package basis by setting the C<USE_MEMCACHED> constant to any true value. + +The current implementation is an opt-in (USE_MEMCACHED is false by default), +however this will change to opt-out once further testing has been completed +(USE_MEMCACHED will be true by default). + +=head1 DIRECT DATABASE UPDATES + +If an object is cached and the database is updated directly (instead of via +C<$object-E<gt>update()>), then it's possible for the data in the cache to be +out of sync with the database. + +As an example let's consider an extension which adds a timestamp field +C<last_activitiy_ts> to the profiles table and user object which contains the +user's last activity. If the extension were to call C<$user-E<gt>update()>, +then an audit entry would be created for each change to the C<last_activity_ts> +field, which is undesirable. + +To remedy this, the extension updates the table directly. It's critical with +Memcached that it then clears the cache: + + $dbh->do("UPDATE profiles SET last_activity_ts=? WHERE userid=?", + undef, $timestamp, $user_id); + Bugzilla->memcached->clear({ table => 'profiles', id => $user_id }); + diff --git a/Bugzilla/Object.pm b/Bugzilla/Object.pm index 16b3a1ebb..a31392353 100644 --- a/Bugzilla/Object.pm +++ b/Bugzilla/Object.pm @@ -34,6 +34,11 @@ use constant AUDIT_CREATES => 1; use constant AUDIT_UPDATES => 1; use constant AUDIT_REMOVES => 1; +# When USE_MEMCACHED is true, the class is suitable for serialisation to +# Memcached. This will be flipped to true by default once the majority of +# Bugzilla Object have been tested with Memcached. +use constant USE_MEMCACHED => 0; + # This allows the JSON-RPC interface to return Bugzilla::Object instances # as though they were hashes. In the future, this may be modified to return # less information. @@ -48,11 +53,41 @@ sub new { my $class = ref($invocant) || $invocant; my $param = shift; - my $object = $class->_cache_get($param); + my $object = $class->_object_cache_get($param); return $object if $object; - $object = $class->new_from_hash($class->_load_from_db($param)); - $class->_cache_set($param, $object); + my ($data, $set_memcached); + if (Bugzilla->feature('memcached') + && $class->USE_MEMCACHED + && ref($param) eq 'HASH' && $param->{cache}) + { + if (defined $param->{id}) { + $data = Bugzilla->memcached->get({ + table => $class->DB_TABLE, + id => $param->{id}, + }); + } + elsif (defined $param->{name}) { + $data = Bugzilla->memcached->get({ + table => $class->DB_TABLE, + name => $param->{name}, + }); + } + $set_memcached = $data ? 0 : 1; + } + $data ||= $class->_load_from_db($param); + + if ($data && $set_memcached) { + Bugzilla->memcached->set({ + table => $class->DB_TABLE, + id => $data->{$class->ID_FIELD}, + name => $data->{$class->NAME_FIELD}, + data => $data, + }); + } + + $object = $class->new_from_hash($data); + $class->_object_cache_set($param, $object); return $object; } @@ -157,32 +192,32 @@ sub initialize { } # Provides a mechanism for objects to be cached in the request_cache -sub _cache_get { +sub _object_cache_get { my $class = shift; my ($param) = @_; - my $cache_key = $class->cache_key($param) + my $cache_key = $class->object_cache_key($param) || return; return Bugzilla->request_cache->{$cache_key}; } -sub _cache_set { +sub _object_cache_set { my $class = shift; my ($param, $object) = @_; - my $cache_key = $class->cache_key($param) + my $cache_key = $class->object_cache_key($param) || return; Bugzilla->request_cache->{$cache_key} = $object; } -sub _cache_remove { +sub _object_cache_remove { my $class = shift; my ($param) = @_; $param->{cache} = 1; - my $cache_key = $class->cache_key($param) + my $cache_key = $class->object_cache_key($param) || return; delete Bugzilla->request_cache->{$cache_key}; } -sub cache_key { +sub object_cache_key { my $class = shift; my ($param) = @_; if (ref($param) && $param->{cache} && ($param->{id} || $param->{name})) { @@ -461,8 +496,9 @@ sub update { $self->audit_log(\%changes) if $self->AUDIT_UPDATES; $dbh->bz_commit_transaction(); - $self->_cache_remove({ id => $self->id }); - $self->_cache_remove({ name => $self->name }) if $self->name; + Bugzilla->memcached->clear({ table => $table, id => $self->id }); + $self->_object_cache_remove({ id => $self->id }); + $self->_object_cache_remove({ name => $self->name }) if $self->name; if (wantarray) { return (\%changes, $old_self); @@ -481,8 +517,9 @@ sub remove_from_db { $self->audit_log(AUDIT_REMOVE) if $self->AUDIT_REMOVES; $dbh->do("DELETE FROM $table WHERE $id_field = ?", undef, $self->id); $dbh->bz_commit_transaction(); - $self->_cache_remove({ id => $self->id }); - $self->_cache_remove({ name => $self->name }) if $self->name; + Bugzilla->memcached->clear({ table => $table, id => $self->id }); + $self->_object_cache_remove({ id => $self->id }); + $self->_object_cache_remove({ name => $self->name }) if $self->name; undef $self; } @@ -1399,7 +1436,7 @@ C<0> otherwise. =over -=item cache_key +=item object_cache_key =item check_time |