diff options
| -rw-r--r-- | .circleci/config.yml | 2 | ||||
| -rw-r--r-- | Bugzilla.pm | 15 | ||||
| -rw-r--r-- | Bugzilla/Bloomfilter.pm | 67 | ||||
| -rw-r--r-- | Bugzilla/Memcached.pm | 39 | ||||
| -rw-r--r-- | Dockerfile | 2 | ||||
| -rwxr-xr-x | Makefile.PL | 1 | ||||
| -rw-r--r-- | scripts/bloomfilter-populate.pl | 21 |
7 files changed, 141 insertions, 6 deletions
diff --git a/.circleci/config.yml b/.circleci/config.yml index 199b066b0..a0b13539f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ version: 2 defaults: bmo_slim_image: &bmo_slim_image - image: mozillabteam/bmo-slim:20170818.1 + image: mozillabteam/bmo-slim:20170824.1 user: app mysql_image: &mysql_image diff --git a/Bugzilla.pm b/Bugzilla.pm index cf004d4fc..bf8f99625 100644 --- a/Bugzilla.pm +++ b/Bugzilla.pm @@ -42,6 +42,7 @@ use Bugzilla::Token; use Bugzilla::User; use Bugzilla::Util; use Bugzilla::CPAN; +use Bugzilla::Bloomfilter; use Bugzilla::Metrics::Collector; use Bugzilla::Metrics::Template; @@ -765,7 +766,7 @@ sub elastic { } sub check_rate_limit { - my ($class, $name, $id) = @_; + my ($class, $name, $ip) = @_; my $params = Bugzilla->params; if ($params->{rate_limit_active}) { my $rules = decode_json($params->{rate_limit_rules}); @@ -774,9 +775,15 @@ sub check_rate_limit { warn "no rules for $name!"; return 0; } - if (Bugzilla->memcached->should_rate_limit("$name:$id", @$limit)) { - Bugzilla->audit("[rate_limit] $id exceeds rate limit $name: " . join("/", @$limit)); - ThrowUserError("rate_limit"); + if (Bugzilla->memcached->should_rate_limit("$name:$ip", @$limit)) { + my $action = 'block'; + my $filter = Bugzilla::Bloomfilter->lookup("rate_limit_whitelist"); + if ($filter && $filter->test($ip)) { + $action = 'ignore'; + } + my $limit = join("/", @$limit); + Bugzilla->audit("[rate_limit] action=$action, ip=$ip, limit=$limit"); + ThrowUserError("rate_limit") if $action eq 'block'; } } } diff --git a/Bugzilla/Bloomfilter.pm b/Bugzilla/Bloomfilter.pm new file mode 100644 index 000000000..0d329b2ea --- /dev/null +++ b/Bugzilla/Bloomfilter.pm @@ -0,0 +1,67 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Bloomfilter; + +use 5.10.1; +use strict; +use warnings; + +use Bugzilla::Constants; +use Algorithm::BloomFilter; +use File::Temp qw(tempfile); + +sub _new_bloom_filter { + my ($n) = @_; + my $p = 0.01; + my $m = $n * abs(log $p) / log(2) ** 2; + my $k = $m / $n * log(2); + return Algorithm::BloomFilter->new($m, $k); +} + +sub _filename { + my ($name) = @_; + + my $datadir = bz_locations->{datadir}; + return sprintf("%s/%s.bloom", $datadir, $name); +} + +sub populate { + my ($class, $name, $items) = @_; + my $memcached = Bugzilla->memcached; + + my $filter = _new_bloom_filter(@$items + 0); + foreach my $item (@$items) { + $filter->add($item); + } + + my ($fh, $filename) = tempfile( "${name}XXXXXX", DIR => bz_locations->{datadir}, UNLINK => 0); + binmode $fh, ':bytes'; + print $fh $filter->serialize; + close $fh; + rename($filename, _filename($name)) or die "failed to rename $filename: $!"; + $memcached->clear_bloomfilter({name => $name}); +} + +sub lookup { + my ($class, $name) = @_; + my $memcached = Bugzilla->memcached; + my $filename = _filename($name); + my $filter_data = $memcached->get_bloomfilter( { name => $name } ); + + if (!$filter_data && -f $filename) { + open my $fh, '<:bytes', $filename; + local $/ = undef; + $filter_data = <$fh>; + close $fh; + $memcached->set_bloomfilter({ name => $name, filter => $filter_data }); + } + + return Algorithm::BloomFilter->deserialize($filter_data); +} + +1; diff --git a/Bugzilla/Memcached.pm b/Bugzilla/Memcached.pm index 233db31f2..1623296f8 100644 --- a/Bugzilla/Memcached.pm +++ b/Bugzilla/Memcached.pm @@ -127,6 +127,41 @@ sub get_config { } } +sub set_bloomfilter { + my ($self, $args) = @_; + return unless $self->{memcached}; + if (exists $args->{name}) { + return $self->_set($self->_bloomfilter_prefix . '.' . $args->{name}, $args->{filter}); + } + else { + ThrowCodeError('params_required', { function => "Bugzilla::Memcached::set_bloomfilter", + params => [ 'name' ] }); + } +} + +sub get_bloomfilter { + my ($self, $args) = @_; + return unless $self->{memcached}; + if (exists $args->{name}) { + return $self->_get($self->_bloomfilter_prefix . '.' . $args->{name}); + } + else { + ThrowCodeError('params_required', { function => "Bugzilla::Memcached::set_bloomfilter", + params => [ 'name' ] }); + } +} + +sub clear_bloomfilter { + my ($self, $args) = @_; + return unless $self->{memcached}; + if ($args && exists $args->{name}) { + $self->_delete($self->_config_prefix . '.' . $args->{name}); + } + else { + $self->_inc_prefix("bloomfilter"); + } +} + sub clear { my ($self, $args) = @_; return unless $self->{memcached}; @@ -244,6 +279,10 @@ sub _config_prefix { return $_[0]->_prefix("config"); } +sub _bloomfilter_prefix { + return $_[0]->_prefix("bloomfilter"); +} + sub _encode_key { my ($self, $key) = @_; $key = $self->_global_prefix . '.' . uri_escape_utf8($key); diff --git a/Dockerfile b/Dockerfile index d6057775e..01a846bc7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM mozillabteam/bmo-slim:latest +FROM mozillabteam/bmo-slim:20170824.1 MAINTAINER Dylan William Hardison <dylan@mozilla.com> ENV BUNDLE=https://s3.amazonaws.com/moz-devservices-bmocartons/bmo/vendor.tar.gz diff --git a/Makefile.PL b/Makefile.PL index 3217101b8..33319ce92 100755 --- a/Makefile.PL +++ b/Makefile.PL @@ -37,6 +37,7 @@ BEGIN { # PREREQ_PM my %requires = ( + 'Algorithm::BloomFilter' => 0, 'CGI' => '<= 3.63', 'CPAN::Meta::Prereqs' => '2.132830', 'CPAN::Meta::Requirements' => '2.121', diff --git a/scripts/bloomfilter-populate.pl b/scripts/bloomfilter-populate.pl new file mode 100644 index 000000000..c591a61b3 --- /dev/null +++ b/scripts/bloomfilter-populate.pl @@ -0,0 +1,21 @@ +#!/usr/bin/perl -w +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use strict; +use warnings; +use lib qw(. lib local/lib/perl5); + +use Bugzilla; +use Bugzilla::Constants; +use Bugzilla::Bloomfilter; + +# set Bugzilla usage mode to USAGE_MODE_CMDLINE +Bugzilla->usage_mode(USAGE_MODE_CMDLINE); + +my $name = shift @ARGV or die "usage: $0 \$name < list\n"; +my @lines = <STDIN>; +chomp @lines; +Bugzilla::Bloomfilter->populate($name, \@lines); + |