From: Bill Erickson <berick@esilibrary.com>
Date: Wed, 8 Aug 2012 14:05:35 +0000 (-0400)
Subject: URLVerify.pm; firing batch url verify calls
X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=191ca937d79dd1d8e44e08f3e434cfcecd9e3bc7;p=evergreen%2Fequinox.git

URLVerify.pm; firing batch url verify calls

Signed-off-by: Bill Erickson <berick@esilibrary.com>
---

diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/URLVerify.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/URLVerify.pm
index 1335983e40..2d727d218b 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/URLVerify.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/URLVerify.pm
@@ -2,6 +2,7 @@ package OpenILS::Application::url_verify;
 use base qw/OpenILS::Application/;
 use strict; use warnings;
 use OpenSRF::Utils::Logger qw(:logger);
+use OpenSRF::MultiSession;
 use OpenILS::Utils::Fieldmapper;
 use OpenILS::Utils::CStoreEditor q/:funcs/;
 use Net::HTTP::NB;
@@ -27,7 +28,8 @@ __PACKAGE__->register_method(
 );
 
 sub validate_session {
-    my ($self, $client, $auth, $session_id, $url_ids) = @_;
+    my ($self, $client, $auth, $session_id, $url_ids, $options) = @_;
+    $options ||= {};
 
     # loop through list of URLs / session URLs
     # see if we've already tested the url, if so copy the status info  / redirect_to info and move on
@@ -41,19 +43,114 @@ sub validate_session {
     my $session = $e->retrieve_url_verify_session($session_id) or return $e->die_event;
 
     if (!$url_ids) {
-        # fetch IDs
+        my $ids = $e->json_query({
+            select => {uvu => ['id']},
+            from => {
+                uvu => { # url
+                    cbrebi => { # bucket item
+                        join => { cbreb => { # bucket
+                            join => { uvs => { # session
+                                filter => {id => $session_id}
+                            }}
+                        }}
+                    }
+                }
+            }
+        });
+
+        $url_ids = [ map {$_->{id}} @$ids ];
     }
 
-    # create the attempt
-    # process the URLs in batch
+    my $url_count = scalar(@$url_ids);
+    $logger->info("url: processing $url_count URLs");
 
-    return undef;
+    my $attempt = Fieldmapper::url_verify::verification_attempt->new;
+    $attempt->session($session_id);
+    $attempt->usr($e->requestor->id);
+    $attempt->start_time('now');
+
+    $e->create_url_verify_verification_attempt($attempt) or return $e->die_event;
+    $e->commit;
+
+    # Now ycle through the URLs in batches.
+
+    my $batch_size = 5; # TODO: org setting
+    my $delay = 2; # TODO: org setting
+    my $num_processed = 0; # total number processed, including redirects
+    my $resp_window = 1;
+
+    my $multises = OpenSRF::MultiSession->new(
+
+        app => 'open-ils.url_verify', # hey, that's us!
+        cap => $batch_size,
+
+        success_handler => sub {
+            my ($self, $req) = @_;
+
+            # API call streams fleshed url_verification objects.  We wrap 
+            # those up with some extra info and pass them on to the caller.
+
+            for my $resp (@{$req->{response}}) {
+                my $content = $resp->content;
+                
+                if ($content) {
+
+                    $num_processed++;
+
+                    if ($options->{report_all} or ($num_processed % $resp_window == 0)) {
+                        $client->respond({
+                            url_count => $url_count,
+                            current_verification => $content,
+                            total_processed => $num_processed
+                        });
+                    }
+
+                    # start off responding quickly, then throttle 
+                    # back to only relaying every 256 messages.
+                    $resp_window *= 2 unless $resp_window == 256;
+                }
+            }
+
+            # insert the per-thread delay, which keeps this thread
+            # active, which prevents a new thread from firing up to 
+            # replace it. (thread = multisession session)
+            sleep $delay;
+        },
+
+        failure_handler => sub {
+            my ($self, $req) = @_;
+
+            # {error} should be an Error w/ a toString
+            $logger->error("url: error processing URL: " . $req->{error});
+        }
+    );
+
+    # Queue up the requests and let them fly
+    $multises->request(
+        'open-ils.url_verify.verify_url', 
+        $auth, $attempt->id, $_) for @$url_ids; 
+
+    # Wait for all requests to be completed
+    $multises->session_wait(1);
+
+    # All done.  Let's wrap up the attempt.
+    $attempt->finish_time('now');
+
+    $e->xact_begin;
+    $e->update_url_verify_verification_attempt($attempt) or return $e->die_event;
+    $e->xact_commit;
+
+    return {
+        url_count => $url_count,
+        total_processed => $num_processed,
+        attempt => $attempt
+    };
 }
 
 
 __PACKAGE__->register_method(
-    method => 'validate_url',
-    api_name => 'open-ils.url_verify.validate_url',
+    method => 'verify_url',
+    api_name => 'open-ils.url_verify.verify_url',
     stream => 1,
     signature => {
         desc => q/