LP#1361782: Add DoS protection

This commit adds two types of simple DoS protection:

 * Limit concurrent search requests per client IP address, regardless of
 the searches being performed.  This helps address issues of accidental
 spamming from a malfunctioning OPAC workstation, or crawlers of various
 types.  The limit is controlled by a global flag called
 "opac.max_concurrent_search.ip".

 * Limit the global concurrent search requests for the same query.  This
 helps address both simple and distributed DoS that send the same search
 request over and over.  The limit is controlled by a global flag called
 "opac.max_concurrent_search.query", and defaults to 20.

When the limit is exceeded in either case the client receives an HTTP
429 "Too many requests" response from the web server, and the connection
is ended.

Signed-off-by: Mike Rylander <mrylander@gmail.com>
Signed-off-by: Jason Stephenson <jason@sigio.com>
Signed-off-by: Galen Charlton <gmc@equinoxOLI.org>

diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm
index 9a27159..f7a0c8b 100644 (file)
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm
@@ -37,6 +37,7 @@ my $cache;
 my $cache_timeout;
 my $superpage_size;
 my $max_superpages;
+my $max_concurrent_search;
 
 sub initialize {
     $cache = OpenSRF::Utils::Cache->new('global');
@@ -1132,6 +1133,12 @@ my $estimation_strategy;
 sub staged_search {
     my($self, $conn, $search_hash, $docache, $phys_loc) = @_;
 
+    my $e = new_editor();
+    if (!$max_concurrent_search) {
+        my $mcs = $e->retrieve_config_global_flag('opac.max_concurrent_search.query');
+        $max_concurrent_search = ($mcs and $mcs->enabled eq 't') ? $mcs->value : 20;
+    }
+
     $phys_loc ||= $U->get_org_tree->id;
 
     my $IAmMetabib = ($self->api_name =~ /metabib/) ? 1 : 0;
@@ -1181,6 +1188,19 @@ sub staged_search {
     # pull any existing results from the cache
     my $key = search_cache_key($method, $search_hash);
     my $facet_key = $key.'_facets';
+
+    # Let the world know that there is at least one backend that will be searching
+    my $counter_key = $key.'_counter';
+    $cache->get_cache($counter_key) || $cache->{memcache}->add($counter_key, 0, $cache_timeout);
+    my $search_peers = $cache->{memcache}->incr($counter_key);
+
+    # If the world tells us that there are more than we want to allow, we stop.
+    if ($search_peers > $max_concurrent_search) {
+        $logger->warn("Too many concurrent searches per $counter_key: $search_peers");
+        $cache->{memcache}->decr($counter_key);
+        return OpenILS::Event->new('BAD_PARAMS')
+    }
+
     my $cache_data = $cache->get_cache($key) || {};
 
     # First, we want to make sure that someone else isn't currently trying to perform exactly
@@ -1237,6 +1257,7 @@ sub staged_search {
             unless($summary) {
                 $logger->info("search timed out: duration=$search_duration: params=".
                     OpenSRF::Utils::JSON->perl2JSON($search_hash));
+                $cache->{memcache}->decr($counter_key);
                 return {count => 0};
             }
 
@@ -1280,7 +1301,7 @@ sub staged_search {
         last if($summary->{checked} < $superpage_size);
     }
 
-    # Let other backends grab our data now that we're done.
+    # Let other backends grab our data now that we're done, and flush the key if we're the last one.
     $cache_data = $cache->get_cache($key);
     if ($$cache_data{running} and $$cache_data{running} == $$) {
         delete $$cache_data{running};
@@ -1318,7 +1339,7 @@ sub staged_search {
             }
 
             my @settings_params = map { $suggest_settings{$_}{value} } @$setting_names;
-            my $suggs = new_editor()->json_query({
+            my $suggs = $e->json_query({
                 from  => [
                     'search.symspell_lookup',
                         $term, $class,
@@ -1352,6 +1373,7 @@ sub staged_search {
             ids               => \@results
         }
     );
+    $cache->{memcache}->decr($counter_key);
 
     $logger->info("Completed canonicalized search is: $$global_summary{canonicalized_query}");
 

diff --git a/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Search.pm b/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Search.pm
index 81132be..6974d19 100644 (file)
--- a/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Search.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Search.pm
@@ -2,6 +2,7 @@ package OpenILS::WWW::EGCatLoader;
 use strict; use warnings;
 use Apache2::Const -compile => qw(OK DECLINED FORBIDDEN HTTP_INTERNAL_SERVER_ERROR REDIRECT HTTP_BAD_REQUEST);
 use OpenSRF::Utils::Logger qw/$logger/;
+use OpenSRF::Utils::Cache;
 use OpenILS::Utils::CStoreEditor qw/:funcs/;
 use OpenILS::Utils::Fieldmapper;
 use OpenILS::Application::AppUtils;
@@ -365,6 +366,7 @@ sub recs_from_metarecord {
 #   page_size
 #   hit_count
 #   records : list of bre's and copy-count objects
+my $max_concurrent_search;
 sub load_rresults {
     my $self = shift;
     my %args = @_;
@@ -373,6 +375,13 @@ sub load_rresults {
     my $ctx = $self->ctx;
     my $e = $self->editor;
 
+    my $mc = OpenSRF::Utils::Cache->new('global');
+    my $client_ip = $self->apache->headers_in->get('X-Forwarded-For') ||
+        $self->apache->useragent_ip;
+    ($client_ip) = split(/,\s*/, $client_ip);
+    $logger->activity("Client IP: $client_ip");
+
+
     # 1. param->metarecord : view constituent bib records for a metarecord
     # 2. param->modifier=metabib : perform a metarecord search
     my $metarecord = $ctx->{metarecord} = $cgi->param('metarecord');
@@ -470,9 +479,27 @@ sub load_rresults {
     my $ltag = $is_meta ? '[mmr search]' : '[bre search]';
     $logger->activity("EGWeb: $ltag $query");
 
-    try {
+    # Fetch the global flag, defaults to "never too many".
+    unless (defined $max_concurrent_search) {
+        my $mcs = $e->retrieve_config_global_flag('opac.max_concurrent_search.ip');
+        $max_concurrent_search = ($mcs and $mcs->enabled eq 't') ? $mcs->value : 0;
+    }
 
-        my $method = 'open-ils.search.biblio.multiclass.query';
+    if ($max_concurrent_search > 0) {
+        # Right up front, we will limit concurrent searches coming from the same client IP, if configured to do so.
+        $mc->get_cache('EGWEB-MAX-SEARCH-IP:'.$client_ip) || $mc->{memcache}->add('EGWEB-MAX-SEARCH-IP:'.$client_ip, 0);
+        my $concurrent_searches = $mc->{memcache}->incr('EGWEB-MAX-SEARCH-IP:'.$client_ip);
+        $logger->activity("Concurrent searches from client IP $client_ip: $concurrent_searches");
+
+        if ($concurrent_searches > $max_concurrent_search) {
+            $self->apache->log->warn("Too many concurrent searches from IP $client_ip");
+            $mc->{memcache}->decr('EGWEB-MAX-SEARCH-IP:'.$client_ip);
+            return 429; # Apache2::Const does not have a symbol for 429 too many requests
+        }
+    }
+
+    my $method = 'open-ils.search.biblio.multiclass.query';
+    try {
         $method .= '.staff' if $ctx->{is_staff};
         $method =~ s/biblio/metabib/ if $is_meta;
 
@@ -489,6 +516,15 @@ sub load_rresults {
         $results = {count => 0, ids => []};
     };
 
+    $mc->{memcache}->decr('EGWEB-MAX-SEARCH-IP:'.$client_ip) if ($max_concurrent_search);
+
+    # INFO: an OpenILS::Event only means "go away" for now.
+    if (defined $U->event_code($results)) {
+        $logger->activity("$method returned event: " . $U->event_code($results));
+        $self->apache->log->warn( "$method returned event: " . $U->event_code($results));
+        return 429; # Apache2::Const does not have a symbol for 429 too many requests
+    }
+
     my $rec_ids = [map { $_->[0] } @{$results->{ids}}];
 
     $ctx->{ids} = $rec_ids;

diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
index 2b5fa01..846779e 100644 (file)
--- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql
+++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
@@ -23222,3 +23222,28 @@ VALUES (
         'cwst', 'label'
     )
 );
+
+INSERT INTO config.global_flag (name, value, enabled, label)
+VALUES (
+    'opac.max_concurrent_search.query',
+    '0',
+    TRUE,
+    oils_i18n_gettext(
+        'opac.max_concurrent_search.query',
+        'Limit the number of global concurrent matching search queries',
+        'cgf', 'label'
+    )
+);
+
+INSERT INTO config.global_flag (name, value, enabled, label)
+VALUES (
+    'opac.max_concurrent_search.ip',
+    '0',
+    TRUE,
+    oils_i18n_gettext(
+        'opac.max_concurrent_search.ip',
+        'Limit the number of global concurrent searches per client IP address',
+        'cgf', 'label'
+    )
+);
+

diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.data.concurrent_search_global_flags.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.data.concurrent_search_global_flags.sql
new file mode 100644 (file)
index 0000000..750a6a8
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.data.concurrent_search_global_flags.sql
@@ -0,0 +1,31 @@
+BEGIN;
+
+-- check whether patch can be applied
+SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
+
+INSERT INTO config.global_flag (name, value, enabled, label)
+VALUES (
+    'opac.max_concurrent_search.query',
+    '20',
+    TRUE,
+    oils_i18n_gettext(
+        'opac.max_concurrent_search.query',
+        'Limit the number of global concurrent matching search queries',
+        'cgf', 'label'
+    )
+);
+
+INSERT INTO config.global_flag (name, value, enabled, label)
+VALUES (
+    'opac.max_concurrent_search.ip',
+    '0',
+    TRUE,
+    oils_i18n_gettext(
+        'opac.max_concurrent_search.ip',
+        'Limit the number of global concurrent searches per client IP address',
+        'cgf', 'label'
+    )
+);
+
+COMMIT;
+