From: Lebbeous Fogle-Weekley Date: Mon, 27 Aug 2012 22:50:39 +0000 (-0400) Subject: Call URL extraction phase from UI X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=0595ad1ff1b742adbdd9da349655b8008a08b70e;p=evergreen%2Fequinox.git Call URL extraction phase from UI Signed-off-by: Lebbeous Fogle-Weekley --- diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/URLVerify.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/URLVerify.pm index d7942383e4..4d0c876035 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/URLVerify.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/URLVerify.pm @@ -594,7 +594,7 @@ sub verify_one_url { __PACKAGE__->register_method( method => "create_session", - api_name => "open-ils.url_verify.create_session", + api_name => "open-ils.url_verify.session.create", signature => { desc => q/Create a URL verify session. Also automatically create and link a container./, @@ -641,7 +641,7 @@ sub create_session { return $e->data->id; } -# _check_for_existing_bucket_items() is used later by session_perform_search() +# _check_for_existing_bucket_items() is used later by session_search_and_extract() sub _check_for_existing_bucket_items { my ($e, $session) = @_; @@ -659,7 +659,7 @@ sub _check_for_existing_bucket_items { return; } -# _get_all_search_results() is used later by session_perform_search() +# _get_all_search_results() is used later by session_search_and_extract() sub _get_all_search_results { my ($client, $session) = @_; @@ -716,25 +716,27 @@ sub _get_all_search_results { __PACKAGE__->register_method( - method => "session_perform_search", - api_name => "open-ils.url_verify.session_perform_search", + method => "session_search_and_extract", + api_name => "open-ils.url_verify.session.search_and_extract", + stream => 1, signature => { desc => q/ Perform the search contained in the session, - populating the linked bucket /, + populating the linked bucket, and extracting URLs /, params => [ {desc => "Authentication token", type => "string"}, {desc => "url_verify.session id", type => "number"}, ], return => { - desc => "Number of search results on success, event on failure", + desc => q/stream of numbers: first number of search results, then + numbers of extracted URLs for each record, grouped into arrays + of 10/, type => "number" } } ); -# XXX TODO send progress updates? -sub session_perform_search { +sub session_search_and_extract { my ($self, $client, $auth, $ses_id) = @_; my $e = new_editor(authtoken => $auth); @@ -771,6 +773,11 @@ sub session_perform_search { # Make and save a bucket item for each search result. my $pos = 0; + my @item_ids; + + # There's an opportunity below to parallelize the extraction of URLs if + # we need to. + foreach my $bre_id (@result_ids) { my $bucket_item = Fieldmapper::container::biblio_record_entry_bucket_item->new; @@ -781,10 +788,39 @@ sub session_perform_search { $e->create_container_biblio_record_entry_bucket_item($bucket_item) or return $e->die_event; + + push @item_ids, $e->data->id; } + $e->xact_commit; - return $pos; + $client->respond($pos); # first response: the number of items created + # (number of search results) + + $pos = 0; + my @url_counts; + + # For each contain item, extract URLs. Report counts of URLs extracted + # from each record in batches at every ten records. + + foreach my $item_id (@item_ids) { + my $res = $e->json_query({ + from => ["url_verify.extract_urls", $ses_id, $item_id] + }) or return $e->die_event; + + push @url_counts, $res->[0]{extract_urls}; + + if (++$pos % 10 == 0) { + $client->respond(\@url_counts); + @url_counts = (); + } + } + + $client->respond(\@url_counts) if @url_counts; + + $e->disconnect; + return; } + 1; diff --git a/Open-ILS/src/templates/url_verify/select_urls.tt2 b/Open-ILS/src/templates/url_verify/select_urls.tt2 new file mode 100644 index 0000000000..8d1bd94ef8 --- /dev/null +++ b/Open-ILS/src/templates/url_verify/select_urls.tt2 @@ -0,0 +1,39 @@ +[% WRAPPER base.tt2 %] +[% ctx.page_title = 'Link Checker - Select URLs' %] + +
+
+
[% ctx.page_title %]
+
+ +
+
+ + + + + + +
+
+[% END %] diff --git a/Open-ILS/web/js/dojo/openils/url_verify/nls/url_verify.js b/Open-ILS/web/js/dojo/openils/url_verify/nls/url_verify.js index dd6e7a07d8..3a3527fe76 100644 --- a/Open-ILS/web/js/dojo/openils/url_verify/nls/url_verify.js +++ b/Open-ILS/web/js/dojo/openils/url_verify/nls/url_verify.js @@ -1,5 +1,6 @@ { "CREATING": "Creating session ...", "SAVING_TAGS": "Saving tag/subfield selectors ...", - "PERFORMING_SEARCH": "Performing search ..." + "PERFORMING_SEARCH": "Performing search ...", + "EXTRACTING_URLS": "Extracting URLs..." } diff --git a/Open-ILS/web/js/ui/default/url_verify/create_session.js b/Open-ILS/web/js/ui/default/url_verify/create_session.js index e63aad0299..18f3200d8e 100644 --- a/Open-ILS/web/js/ui/default/url_verify/create_session.js +++ b/Open-ILS/web/js/ui/default/url_verify/create_session.js @@ -60,12 +60,13 @@ var create_session = {}; progress_dialog.attr("title", localeStrings.CREATING); progress_dialog.show(true); fieldmapper.standardRequest( - ["open-ils.url_verify", "open-ils.url_verify.create_session"], { + ["open-ils.url_verify", "open-ils.url_verify.session.create"], { "params": [openils.User.authtoken, name, search], "async": true, "onresponse": function(r) { if (r = openils.Util.readResponse(r)) { - create_session.session_id = r; /* we're modal enough to get away with this */ + /* I think we're modal enough to get away with this. */ + create_session.session_id = r; create_session.save_tags(); } } @@ -133,20 +134,37 @@ var create_session = {}; }; /* 3) search and populate the container (API call). */ + var search_result_count = 0; create_session.perform_search = function() { progress_dialog.attr("title", localeStrings.PERFORMING_SEARCH); progress_dialog.show(true); fieldmapper.standardRequest( - ["open-ils.url_verify", "open-ils.url_verify.session_perform_search"], { + ["open-ils.url_verify", + "open-ils.url_verify.session.search_and_extract"], { "params": [openils.User.authtoken, create_session.session_id], "async": true, "onresponse": function(r) { - /* XXX atm, this responds with the number of created results */ r = openils.Util.readResponse(r); - progress_dialog.hide(); + if (!search_result_count) { + search_result_count = Number(r); + console.log("search_result_count: " + r); + + progress_dialog.show(); /* sic */ + progress_dialog.attr( + "title", localeStrings.EXTRACTING_URLS + ); + progress_dialog.update( + {"maximum": search_result_count, "progress": 0} + ); + } else { + console.log("r.length: " + r.length); + progress_dialog.update({"progress": r.length}) + } + }, + "oncomplete": function() { progress_dialog.attr("title", ""); - + progress_dialog.hide(); console.warn("XXX TODO redirect to next interface"); } }