LP#1729610: return new OpenSRF status if backlog queue fills up

This patch teaches Perl services how to return a new OpenSRF status,
OSRF_STATUS_SERVICEUNAVAILABLE (code 503) if the backlog queue
for a service gets full.

To test
-------
[1] Set a low max_backlog_queue for opensrf.sloooow and a low
    max_children.
[2] Arrange for srfsh to fire off a bunch of opensrf.sloooow.wait
    requests.
[3] Verify that requests that come in after the backlog queue fills
    up immediately get 503 exceptions.

Signed-off-by: Galen Charlton <gmc@equinoxinitiative.org>
Signed-off-by: Bill Erickson <berickxx@gmail.com>
Signed-off-by: Mike Rylander <mrylander@gmail.com>

diff --git a/include/opensrf/osrf_message.h b/include/opensrf/osrf_message.h
index 1785adf..874c310 100644 (file)
--- a/include/opensrf/osrf_message.h
+++ b/include/opensrf/osrf_message.h
@@ -53,6 +53,7 @@ extern "C" {
 
 #define OSRF_STATUS_INTERNALSERVERERROR  500
 #define OSRF_STATUS_NOTIMPLEMENTED       501
+#define OSRF_STATUS_SERVICEUNAVAILABLE   503
 #define OSRF_STATUS_VERSIONNOTSUPPORTED  505
 
 

diff --git a/src/javascript/opensrf.js b/src/javascript/opensrf.js
index e18efa5..03c79a9 100644 (file)
--- a/src/javascript/opensrf.js
+++ b/src/javascript/opensrf.js
@@ -58,6 +58,7 @@ var OSRF_STATUS_TIMEOUT = 408;
 var OSRF_STATUS_EXPFAILED = 417;
 var OSRF_STATUS_INTERNALSERVERERROR = 500;
 var OSRF_STATUS_NOTIMPLEMENTED = 501;
+var OSRF_STATUS_SERVICEUNAVAILABLE = 503;
 var OSRF_STATUS_VERSIONNOTSUPPORTED = 505;
 
 // TODO: get path from ./configure prefix

diff --git a/src/perl/lib/OpenSRF/DomainObject/oilsResponse.pm b/src/perl/lib/OpenSRF/DomainObject/oilsResponse.pm
index 874a4ae..80fc7d2 100644 (file)
--- a/src/perl/lib/OpenSRF/DomainObject/oilsResponse.pm
+++ b/src/perl/lib/OpenSRF/DomainObject/oilsResponse.pm
@@ -12,6 +12,7 @@ BEGIN {
                                        STATUS_BADREQUEST STATUS_UNAUTHORIZED STATUS_FORBIDDEN
                                        STATUS_NOTFOUND STATUS_NOTALLOWED STATUS_TIMEOUT
                                        STATUS_INTERNALSERVERERROR STATUS_NOTIMPLEMENTED
+                                       STATUS_SERVICEUNAVAILABLE
                                        STATUS_VERSIONNOTSUPPORTED STATUS_REDIRECTED 
                                        STATUS_EXPFAILED STATUS_COMPLETE STATUS_PARTIAL
                                        STATUS_NOCONTENT/;
@@ -21,6 +22,7 @@ BEGIN {
                                        STATUS_BADREQUEST STATUS_UNAUTHORIZED STATUS_FORBIDDEN
                                        STATUS_NOTFOUND STATUS_NOTALLOWED STATUS_TIMEOUT
                                        STATUS_INTERNALSERVERERROR STATUS_NOTIMPLEMENTED
+                                       STATUS_SERVICEUNAVAILABLE
                                        STATUS_VERSIONNOTSUPPORTED STATUS_REDIRECTED 
                                        STATUS_EXPFAILED STATUS_COMPLETE STATUS_PARTIAL
                                        STATUS_NOCONTENT/ ],
@@ -72,6 +74,7 @@ sub STATUS_EXPFAILED          { return 417 }
 
 sub STATUS_INTERNALSERVERERROR { return 500 }
 sub STATUS_NOTIMPLEMENTED                      { return 501 }
+sub STATUS_SERVICEUNAVAILABLE  { return 503 }
 sub STATUS_VERSIONNOTSUPPORTED { return 505 }
 
 my $log = 'OpenSRF::Utils::Logger';

diff --git a/src/perl/lib/OpenSRF/Server.pm b/src/perl/lib/OpenSRF/Server.pm
index 0d31fcf..49ede59 100644 (file)
--- a/src/perl/lib/OpenSRF/Server.pm
+++ b/src/perl/lib/OpenSRF/Server.pm
@@ -21,6 +21,7 @@ use OpenSRF::Utils::Config;
 use OpenSRF::Transport::PeerHandle;
 use OpenSRF::Utils::SettingsClient;
 use OpenSRF::Utils::Logger qw($logger);
+use OpenSRF::DomainObject::oilsResponse qw/:status/;
 use OpenSRF::Transport::SlimJabber::Client;
 use Encode;
 use POSIX qw/:sys_wait_h :errno_h/;
@@ -227,8 +228,47 @@ sub run {
                         }
                     }
                 } else {
-                    # We'll just have to wait
-                    $self->check_status(1); # block until child is available
+
+                    if (!$from_network) {
+                        # The queue is full, and we just requeued a message. We'll
+                        # now see if there is a request available from the network;
+                        # if so, we'll see if a child is available again or else
+                        # drop it
+                        $msg = $self->{osrf_handle}->process($wait_time);
+                        if ($msg) {
+                            $self->check_status();
+                            if (@{$self->{idle_list}}) {
+                                # child now available, so we'll go ahead and queue it
+                                $chatty and $logger->debug("server: queuing new message after a re-queue with a full queue");
+                                push @max_children_msg_queue, $msg;
+                            } else {
+                                # ok, need to drop this one
+                                my $resp = OpenSRF::DomainObject::oilsMessage->new();
+                                $resp->type('STATUS');
+                                $resp->payload(
+                                    OpenSRF::DomainObject::oilsMethodException->new(
+                                        status => "Service unavailable: no available children and backlog queue at limit",
+                                        statusCode => STATUS_SERVICEUNAVAILABLE
+                                    )
+                                );
+                                $resp->threadTrace(1);
+
+                                $logger->set_osrf_xid($msg->osrf_xid);
+                                $self->{osrf_handle}->send(
+                                    to => $msg->from,
+                                    osrf_xid => $msg->osrf_xid, # Note that this is ignored, which
+                                                                # is why we called $logger->set_osrf_xid above.
+                                                                # We probably don't want that to be necessary
+                                                                # if osrf_xid is explicitly set here, but that'll
+                                                                # be a FIXME for later
+                                    thread => $msg->thread,
+                                    body => OpenSRF::Utils::JSON->perl2JSON([ $resp ])
+                                );
+                                $logger->warn("Backlog queue full for $self->{service}; forced to drop message " .
+                                              $msg->thread . " from " . $msg->from);
+                            }
+                        }
+                    }
                 }
             }
 

diff --git a/src/python/osrf/const.py b/src/python/osrf/const.py
index d888b75..297b1b3 100644 (file)
--- a/src/python/osrf/const.py
+++ b/src/python/osrf/const.py
@@ -63,6 +63,7 @@ OSRF_STATUS_TIMEOUT                  = 408
 OSRF_STATUS_EXPFAILED                = 417
 OSRF_STATUS_INTERNALSERVERERROR      = 500
 OSRF_STATUS_NOTIMPLEMENTED           = 501
+OSRF_STATUS_SERVICEUNAVAILABLE       = 503
 OSRF_STATUS_VERSIONNOTSUPPORTED      = 505