Vandelay import improvements / bug fixes

 * Allow for quality ratio control during import-on-1match
 * Allow for 901c matching when no match-set is selected
 * Tidy up some display strings
 * Numerous small bug fixes

diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm
index 4a0bf99..65a0013 100644 (file)
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm
@@ -927,8 +927,8 @@ sub import_record_list_impl {
         $e->requestor($requestor);
 
         $$report_args{e} = $e;
-        $$report_args{import_error} = undef;
         $$report_args{evt} = undef;
+        $$report_args{import_error} = undef;
 
         my $rec = $e->$retrieve_func([
             $rec_id,
@@ -962,7 +962,7 @@ sub import_record_list_impl {
                 {
                     from => [
                         $overlay_func,
-                        $rec->id, 
+                        $rec_id,
                         $overlay_target, 
                         $merge_profile
                     ]
@@ -972,8 +972,8 @@ sub import_record_list_impl {
             if($res and ($res = $res->[0])) {
 
                 if($res->{$overlay_func} eq 't') {
-                    $logger->info("vl: $type direct overlay succeeded for queued rec " . 
-                        $rec->id . " and overlay target $overlay_target");
+                    $logger->info("vl: $type direct overlay succeeded for queued rec ".
+                        "$rec_id and overlay target $overlay_target");
                     $imported = 1;
                 }
 
@@ -984,34 +984,39 @@ sub import_record_list_impl {
 
         } else {
 
-            if($auto_overlay_1match) { 
-                # caller says to overlay if there is exactly 1 match
+            if($auto_overlay_1match) { # overlay if there is exactly 1 match
 
                 my %match_recs = map { $_->eg_record => 1 } @{$rec->matches};
 
                 if( scalar(keys %match_recs) == 1) { # all matches point to the same record
 
+                    # $auto_overlay_best_func will find the 1 match and 
+                    # overlay if the quality ratio allows it
+
                     my $res = $e->json_query(
                         {
                             from => [
-                                $overlay_func,
-                                $rec->id, 
-                                $rec->matches->[0]->eg_record,
-                                $merge_profile
+                                $auto_overlay_best_func,
+                                $rec_id, 
+                                $merge_profile,
+                                $match_quality_ratio
                             ]
                         }
                     );
 
                     if($res and ($res = $res->[0])) {
     
-                        if($res->{$overlay_func} eq 't') {
-                            $logger->info("vl: $type overlay-1match succeeded for queued rec " . $rec->id);
+                        if($res->{$auto_overlay_best_func} eq 't') {
+                            $logger->info("vl: $type overlay-1match succeeded for queued rec $rec_id");
                             $imported = 1;
+                        } else {
+                            $$report_args{import_error} = 'overlay.record.quality' if $match_quality_ratio > 0;
+                            $logger->info("vl: $type overlay-1match failed for queued rec $rec_id");
                         }
 
                     } else {
                         $error = 1;
-                        $logger->error("vl: Error attempting overlay with func=$overlay_func, profile=$merge_profile, record=$rec_id");
+                        $logger->error("vl: Error attempting overlay with func=$auto_overlay_best_func, profile=$merge_profile, record=$rec_id");
                     }
                 }
             }
@@ -1019,12 +1024,13 @@ sub import_record_list_impl {
             if(!$imported and !$error and $auto_overlay_exact and scalar(@{$rec->matches}) == 1 ) {
                 
                 # caller says to overlay if there is an /exact/ match
+                # $auto_overlay_func only proceeds and returns true on exact matches
 
                 my $res = $e->json_query(
                     {
                         from => [
                             $auto_overlay_func,
-                            $rec->id, 
+                            $rec_id,
                             $merge_profile
                         ]
                     }
@@ -1033,10 +1039,10 @@ sub import_record_list_impl {
                 if($res and ($res = $res->[0])) {
 
                     if($res->{$auto_overlay_func} eq 't') {
-                        $logger->info("vl: $type auto-overlay succeeded for queued rec " . $rec->id);
+                        $logger->info("vl: $type auto-overlay succeeded for queued rec $rec_id");
                         $imported = 1;
                     } else {
-                        $logger->info("vl: $type auto-overlay failed for queued rec " . $rec->id);
+                        $logger->info("vl: $type auto-overlay failed for queued rec $rec_id");
                     }
 
                 } else {
@@ -1053,7 +1059,7 @@ sub import_record_list_impl {
                     {
                         from => [
                             $auto_overlay_best_func,
-                            $rec->id, 
+                            $rec_id,
                             $merge_profile,
                             $match_quality_ratio
                         ]
@@ -1063,11 +1069,11 @@ sub import_record_list_impl {
                 if($res and ($res = $res->[0])) {
 
                     if($res->{$auto_overlay_best_func} eq 't') {
-                        $logger->info("vl: $type auto-overlay-best succeeded for queued rec " . $rec->id);
+                        $logger->info("vl: $type auto-overlay-best succeeded for queued rec $rec_id");
                         $imported = 1;
                     } else {
                         $$report_args{import_error} = 'overlay.record.quality' if $match_quality_ratio > 0;
-                        $logger->info("vl: $type auto-overlay-best failed for queued rec " . $rec->id);
+                        $logger->info("vl: $type auto-overlay-best failed for queued rec $rec_id");
                     }
 
                 } else {
@@ -1097,20 +1103,28 @@ sub import_record_list_impl {
 
                     $logger->info("vl: successfully imported new $type record");
                     $rec->imported_as($record->id);
-                    $rec->import_time('now');
-                    $rec->clear_import_error;
-                    $rec->clear_error_detail;
-                    $imported = 1 if $e->$update_func($rec);
                 }
             }
         }
 
         if($imported) {
-            push @success_rec_ids, $rec_id;
-            finish_rec_import_attempt($report_args);
 
-        } else {
-            # Send an update whenever there's an error
+            $rec->import_time('now');
+            $rec->clear_import_error;
+            $rec->clear_error_detail;
+
+            if($e->$update_func($rec)) {
+
+                push @success_rec_ids, $rec_id;
+                finish_rec_import_attempt($report_args);
+
+            } else {
+                $imported = 0;
+            }
+        }
+
+        if(!$imported) {
+            $logger->info("vl: record $rec_id was not imported");
             $$report_args{evt} = $e->event unless $$report_args{evt};
             finish_rec_import_attempt($report_args);
         }
@@ -1137,7 +1151,7 @@ sub import_record_list_impl {
     }
 
     # import the copies
-    import_record_asset_list_impl($conn, \@success_rec_ids, $requestor);
+    import_record_asset_list_impl($conn, \@success_rec_ids, $requestor) if @success_rec_ids;
 
     $conn->respond({total => $$report_args{total}, progress => $$report_args{progress}});
     return undef;
@@ -1402,7 +1416,10 @@ sub import_record_asset_list_impl {
     $rec_ids = $roe->json_query({
         select => {vqbr => ['id']},
         from => {vqbr => 'vii'},
-        where => {'+vqbr' => {import_time => {'!=' => undef}}},
+        where => {'+vqbr' => {
+            id => $rec_ids,
+            import_time => {'!=' => undef}
+        }},
         distinct => 1
     });
     $rec_ids = [map {$_->{id}} @$rec_ids];
@@ -1555,7 +1572,9 @@ sub respond_with_status {
 
     if($$args{report_all} or ($$args{progress} % $$args{step}) == 0) {
         $$args{conn}->respond({
-            map { $_ => $args->{$_} } qw/total progress success_count/,
+            total => $$args{total},
+            progress => $$args{progress},
+            success_count => $$args{success_count},
             err_event => $evt
         });
         $$args{step} *= 2 unless $$args{step} == 256;

diff --git a/Open-ILS/src/sql/Pg/012.schema.vandelay.sql b/Open-ILS/src/sql/Pg/012.schema.vandelay.sql
index b136e8e..d461d4f 100644 (file)
--- a/Open-ILS/src/sql/Pg/012.schema.vandelay.sql
+++ b/Open-ILS/src/sql/Pg/012.schema.vandelay.sql
@@ -677,9 +677,9 @@ $$ LANGUAGE PLPGSQL;
 CREATE OR REPLACE FUNCTION vandelay.match_bib_record() RETURNS TRIGGER AS $func$
 DECLARE
     incoming_existing_id    TEXT;
-    my_bib_queue            vandelay.bib_queue%ROWTYPE;
     test_result             vandelay.match_set_test_result%ROWTYPE;
     tmp_rec                 BIGINT;
+    match_set               INT;
 BEGIN
     IF TG_OP IN ('INSERT','UPDATE') AND NEW.imported_as IS NOT NULL THEN
         RETURN NEW;
@@ -687,14 +687,12 @@ BEGIN
 
     DELETE FROM vandelay.bib_match WHERE queued_record = NEW.id;
 
-    SELECT * INTO my_bib_queue FROM vandelay.bib_queue WHERE id = NEW.queue;
+    SELECT q.match_set INTO match_set FROM vandelay.bib_queue q WHERE q.id = NEW.queue;
 
-    IF my_bib_queue.match_set IS NULL THEN
-        RETURN NEW;
+    IF match_set IS NOT NULL THEN
+        NEW.quality := vandelay.measure_record_quality( NEW.marc, match_set );
     END IF;
 
-    NEW.quality := vandelay.measure_record_quality( NEW.marc, my_bib_queue.match_set );
-
     -- Perfect matches on 901$c exit early with a match with high quality.
     incoming_existing_id :=
         oils_xpath_string('//*[@tag="901"]/*[@code="c"][1]', NEW.marc);
@@ -702,22 +700,33 @@ BEGIN
     IF incoming_existing_id IS NOT NULL AND incoming_existing_id != '' THEN
         SELECT id INTO tmp_rec FROM biblio.record_entry WHERE id = incoming_existing_id::bigint;
         IF tmp_rec IS NOT NULL THEN
-            INSERT INTO vandelay.bib_match (queued_record, eg_record, quality) VALUES ( NEW.id, incoming_existing_id::bigint, 9999);
-            RETURN NEW;
+            INSERT INTO vandelay.bib_match (queued_record, eg_record, match_score, quality) 
+                SELECT
+                    NEW.id, 
+                    b.id,
+                    9999,
+                    -- note: no match_set means quality==0
+                    vandelay.measure_record_quality( b.marc, match_set )
+                FROM biblio.record_entry b
+                WHERE id = incoming_existing_id::bigint;
         END IF;
     END IF;
 
+    IF match_set IS NULL THEN
+        RETURN NEW;
+    END IF;
 
     FOR test_result IN SELECT * FROM
-        vandelay.match_set_test_marcxml(my_bib_queue.match_set, NEW.marc) LOOP
+        vandelay.match_set_test_marcxml(match_set, NEW.marc) LOOP
 
         INSERT INTO vandelay.bib_match ( queued_record, eg_record, match_score, quality )
-       SELECT  NEW.id,
-               test_result.record,
+            SELECT  
+                NEW.id,
+                test_result.record,
                 test_result.quality,
-               vandelay.measure_record_quality( b.marc, my_bib_queue.match_set )
-         FROM  biblio.record_entry b
-         WHERE id = test_result.record;
+                vandelay.measure_record_quality( b.marc, match_set )
+               FROM  biblio.record_entry b
+               WHERE id = test_result.record;
 
     END LOOP;
 

diff --git a/Open-ILS/web/js/ui/default/vandelay/vandelay.js b/Open-ILS/web/js/ui/default/vandelay/vandelay.js
index b67352a..e0a7eaf 100644 (file)
--- a/Open-ILS/web/js/ui/default/vandelay/vandelay.js
+++ b/Open-ILS/web/js/ui/default/vandelay/vandelay.js
@@ -1143,6 +1143,7 @@ function vlImportRecordQueue(type, queueId, recList, onload) {
     if(vlUploadQueueAutoOverlay1Match.checked) {
         options.auto_overlay_1match = true;
         vlUploadQueueAutoOverlay1Match.checked = false;
+        options.match_quality_ratio = vlUploadQueueAutoOverlayBestMatchRatio.attr('value');
         mergeOpt = true;
     }
 

diff --git a/Open-ILS/web/opac/locale/en-US/vandelay.dtd b/Open-ILS/web/opac/locale/en-US/vandelay.dtd
index d2d1789..7928620 100644 (file)
--- a/Open-ILS/web/opac/locale/en-US/vandelay.dtd
+++ b/Open-ILS/web/opac/locale/en-US/vandelay.dtd
@@ -2,12 +2,12 @@
 <!ENTITY vandelay.auth.attrs "Authority attributes">
 <!ENTITY vandelay.auth.records "Authority Records">
 <!ENTITY vandelay.auto.import.noncolliding "Import Non-Colliding Records">
-<!ENTITY vandelay.auto.import.auto_overlay_exact "Merge/Overlay Exact Matches (901c)">
-<!ENTITY vandelay.auto.import.auto_overlay_1match "Merge/Overlay Single Matches">
-<!ENTITY vandelay.auto.import.auto_overlay_best "Merge/Overlay Best Match">
-<!ENTITY vandelay.auto.import.auto_overlay_best_ratio "Best Match Minimum Quality Ratio">
+<!ENTITY vandelay.auto.import.auto_overlay_exact "Merge On Exact Match (901c)">
+<!ENTITY vandelay.auto.import.auto_overlay_1match "Merge On Single Match">
+<!ENTITY vandelay.auto.import.auto_overlay_best "Merge On Best Match">
+<!ENTITY vandelay.auto.import.auto_overlay_best_ratio "Best/Single Match Minimum Quality Ratio">
 <!ENTITY vandelay.auto.import.auto_overlay_best_ratio.desc "New Record Quaility / Quality of Best Match">
-<!ENTITY vandelay.auto.import.merge_profile "Merge/Overlay Profile">
+<!ENTITY vandelay.auto.import.merge_profile "Merge Profile">
 <!ENTITY vandelay.auto.width "Auto Width">
 <!ENTITY vandelay.back.to.import.queue "Back To Import Queue">
 <!ENTITY vandelay.bib.attrs "Bibliographic attributes">
@@ -52,8 +52,8 @@
 <!ENTITY vandelay.marc.record "MARC Record">
 <!ENTITY vandelay.matches "Matches">
 <!ENTITY vandelay.next.page "Next &#187;">
-<!ENTITY vandelay.overlay.selected.record "Overlay selected record with imported record">
-<!ENTITY vandelay.overlay.target "Overlay Target">
+<!ENTITY vandelay.overlay.selected.record "Merge selected record with imported record">
+<!ENTITY vandelay.overlay.target "Merge Target">
 <!ENTITY vandelay.page "Page">
 <!ENTITY vandelay.powered.by.evergreen "Powered by Evergreen!">
 <!ENTITY vandelay.prev.page "&#171; Previous">