From 64e04af74bc1d40a168dd6e4244b0769cfafcb44 Mon Sep 17 00:00:00 2001 From: berick <berick@esilibrary.com> Date: Thu, 5 May 2011 13:21:56 -0400 Subject: [PATCH] expose new import options to queue interface importer dialog; mucho import bug fixes in the middle layer code; expose queued rec quality in matches interface --- Open-ILS/examples/fm_IDL.xml | 2 + .../perlmods/lib/OpenILS/Application/Vandelay.pm | 101 ++++++++++--------- Open-ILS/web/js/ui/default/vandelay/vandelay.js | 108 ++++++++++++--------- .../web/templates/default/vandelay/inc/matches.tt2 | 1 + .../web/templates/default/vandelay/inc/queue.tt2 | 27 ++++-- 5 files changed, 139 insertions(+), 100 deletions(-) diff --git a/Open-ILS/examples/fm_IDL.xml b/Open-ILS/examples/fm_IDL.xml index d2c2f1c831..3f93a59ea0 100644 --- a/Open-ILS/examples/fm_IDL.xml +++ b/Open-ILS/examples/fm_IDL.xml @@ -329,6 +329,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA <field reporter:label="Attributes" name="attributes" oils_persist:virtual="true" reporter:datatype="link"/> <field reporter:label="Matches" name="matches" oils_persist:virtual="true" reporter:datatype="link"/> <field reporter:label="Import Items" name="import_items" oils_persist:virtual="true" reporter:datatype="link"/> + <field reporter:label="Quality" name="quality" reporter:datatype="int"/> </fields> <links> <link field="import_error" reltype="has_a" key="code" map="" class="vie"/> @@ -447,6 +448,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA <field reporter:label="Purpose" name="purpose" reporter:datatype="text"/> <field reporter:label="Attributes" name="attributes" oils_persist:virtual="true" reporter:datatype="link"/> <field reporter:label="Matches" name="matches" oils_persist:virtual="true" reporter:datatype="link"/> + <field reporter:label="Quality" name="quality" reporter:datatype="int"/> </fields> <links> <link field="import_error" reltype="has_a" key="code" map="" class="vie"/> diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm index 3755ad22a0..8c95b70d5c 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm @@ -576,7 +576,7 @@ sub import_record_list { return $e->die_event unless $e->checkauth; $args ||= {}; my $err = import_record_list_impl($self, $conn, $rec_ids, $e->requestor, $args); - $e->rollback; + try {$e->rollback} otherwise {}; return $err if $err; return {complete => 1}; } @@ -698,13 +698,13 @@ sub import_record_list_impl { my $type = $self->{record_type}; my %queues; - my %report_args = ( - progress => 0, + my $report_args = { + progress => 1, step => 1, conn => $conn, total => scalar(@$rec_ids), report_all => $$args{report_all} - ); + }; my $auto_overlay_exact = $$args{auto_overlay_exact}; my $auto_overlay_1match = $$args{auto_overlay_1match}; @@ -750,10 +750,9 @@ sub import_record_list_impl { my $e = new_editor(xact => 1); $e->requestor($requestor); - $report_args{progress}++; - $report_args{e} = $e; - $report_args{import_error} = undef; - $report_args{evt} = undef; + $$report_args{e} = $e; + $$report_args{import_error} = undef; + $$report_args{evt} = undef; my $rec = $e->$retrieve_func([ $rec_id, @@ -763,11 +762,12 @@ sub import_record_list_impl { ]); unless($rec) { - finish_rec_import_attempt(%report_args, evt => $e->event); + $$report_args{evt} = $e->event; + finish_rec_import_attempt($report_args); next; } - $report_args{rec} = $rec; + $$report_args{rec} = $rec; if($rec->import_time) { $e->rollback; @@ -890,7 +890,7 @@ sub import_record_list_impl { $logger->info("vl: $type auto-overlay-best succeeded for queued rec " . $rec->id); $imported = 1; } else { - $report_args{import_error} = 'overlay.record.quality' if $match_quality_ratio > 0; + $$report_args{import_error} = 'overlay.record.quality' if $match_quality_ratio > 0; $logger->info("vl: $type auto-overlay-best failed for queued rec " . $rec->id); } @@ -914,8 +914,8 @@ sub import_record_list_impl { } if($U->event_code($record)) { - $report_args{import_error} = 'import.duplicate.tcn' if $record->{textcode} eq 'TCN_EXISTS'; - $report_args{evt} = $record; + $$report_args{import_error} = 'import.duplicate.tcn' if $record->{textcode} eq 'TCN_EXISTS'; + $$report_args{evt} = $record; } else { @@ -929,11 +929,12 @@ sub import_record_list_impl { if($imported) { push @success_rec_ids, $rec_id; - finish_rec_import_attempt(%report_args); + finish_rec_import_attempt($report_args); } else { # Send an update whenever there's an error - finish_rec_import_attempt(%report_args, evt => $e->event); + $$report_args{evt} = $e->event unless $$report_args{evt}; + finish_rec_import_attempt($report_args); } } @@ -960,18 +961,18 @@ sub import_record_list_impl { # import the copies import_record_asset_list_impl($conn, \@success_rec_ids, $requestor); - $conn->respond({total => $report_args{total}, progress => $report_args{progress}}); + $conn->respond({total => $$report_args{total}, progress => $$report_args{progress}}); return undef; } # tracks any import errors, commits the current xact, responds to the client sub finish_rec_import_attempt { - my %args = @_; - my $evt = $args{evt}; - my $rec = $args{rec}; - my $e = $args{e}; + my $args = shift; + my $evt = $$args{evt}; + my $rec = $$args{rec}; + my $e = $$args{e}; - my $error = $args{import_error}; + my $error = $$args{import_error}; $error = 'general.unknown' if $evt and not $error; # error tracking @@ -991,7 +992,7 @@ sub finish_rec_import_attempt { } my $method = 'update_vandelay_queued_bib_record'; - $method =~ s/bib/authority/ if $args{type} eq 'auth'; + $method =~ s/bib/authority/ if $$args{type} eq 'auth'; $e->$method($rec) and $e->commit or $e->rollback; } else { @@ -1007,18 +1008,17 @@ sub finish_rec_import_attempt { } # respond to client - if($args{report_all} or ($args{progress} % $args{step}) == 0) { - - $args{conn}->respond({ - total => $args{total}, - progress => $args{progress}, + if($$args{report_all} or ($$args{progress} % $$args{step}) == 0) { + $$args{conn}->respond({ + total => $$args{total}, + progress => $$args{progress}, imported => ($rec) ? $rec->id : undef, err_event => $evt }); - - # report often at first, climb quickly, then hold steady - $args{step} *= 2 unless $args{step} == 256; + $$args{step} *= 2 unless $$args{step} == 256; } + + $$args{progress}++; } @@ -1221,25 +1221,34 @@ sub import_record_asset_list_impl { my $roe = new_editor(xact=> 1, requestor => $requestor); + # for speed, filter out any records have not been + # imported or have no import items to load + $rec_ids = $roe->json_query({ + select => {vqbr => ['id']}, + from => {vqbr => 'vii'}, + where => {'+vqbr' => {import_time => {'!=' => undef}}} + }); + $rec_ids = [map {$_->{id}} @$rec_ids]; + my %report_args = ( conn => $conn, total => scalar(@$rec_ids), step => 1, # how often to respond - progress => 0, + progress => 1, in_count => 0, ); for my $rec_id (@$rec_ids) { my $rec = $roe->retrieve_vandelay_queued_bib_record($rec_id); - next unless $rec and $rec->import_time; my $item_ids = $roe->search_vandelay_import_item({record => $rec->id}, {idlist=>1}); for my $item_id (@$item_ids) { my $e = new_editor(requestor => $requestor, xact => 1); my $item = $e->retrieve_vandelay_import_item($item_id); - $report_args{progress}++; $report_args{import_item} = $item; $report_args{e} = $e; + $report_args{import_error} = undef; + $report_args{evt} = undef; # -------------------------------------------------------------------------------- # Find or create the volume @@ -1332,24 +1341,26 @@ sub import_record_asset_list_impl { respond_with_status(%report_args, imported_as => $copy->id); $logger->info("vl: successfully imported item " . $item->barcode); } + } + $roe->rollback; return undef; } sub respond_with_status { - my %args = @_; - my $e = $args{e}; + my $args = shift; + my $e = $$args{e}; # If the import failed, track the failure reason - my $error = $args{import_error}; - my $evt = $args{evt}; + my $error = $$args{import_error}; + my $evt = $$args{evt}; if($error or $evt) { - my $item = $args{import_item}; + my $item = $$args{import_item}; $logger->info("vl: unable to import item " . $item->barcode); $error ||= 'general.unknown'; @@ -1367,13 +1378,15 @@ sub respond_with_status { $e->commit; } - return unless $args{report_all} or ($args{progress} % $args{step}) == 0; - $args{step} *= 2 unless $args{step} == 256; + if($$args{report_all} or ($$args{progress} % $$args{step}) == 0) { + $$args{conn}->respond({ + map { $_ => $args->{$_} } qw/total progress success_count/, + err_event => $evt + }); + $$args{step} *= 2 unless $$args{step} == 256; + } - $args{conn}->respond({ - map { $_ => $args{$_} } qw/total progress success_count/, - err_event => $evt - }); + $$args{progress}++; } __PACKAGE__->register_method( diff --git a/Open-ILS/web/js/ui/default/vandelay/vandelay.js b/Open-ILS/web/js/ui/default/vandelay/vandelay.js index b72438caec..f076da868b 100644 --- a/Open-ILS/web/js/ui/default/vandelay/vandelay.js +++ b/Open-ILS/web/js/ui/default/vandelay/vandelay.js @@ -435,7 +435,8 @@ function retrieveQueuedRecords(type, queueId, onload) { function vlLoadMatchUI(recId) { displayGlobalDiv('vl-generic-progress'); - var matches = queuedRecordsMap[recId].matches(); + var queuedRec = queuedRecordsMap[recId]; + var matches = queuedRec.matches(); var records = []; currentImportRecId = recId; for(var i = 0; i < matches.length; i++) @@ -465,7 +466,7 @@ function vlLoadMatchUI(recId) { // build the data store of records with match information var dataStore = bre.toStoreData(recs, null, - {virtualFields:['_id', 'match_score', 'match_quality']}); + {virtualFields:['_id', 'match_score', 'match_quality', 'rec_quality']}); dataStore.identifier = '_id'; var matchSeenMap = {}; @@ -479,6 +480,7 @@ function vlLoadMatchUI(recId) { if(match.match_score) item.match_score = match.match_score(); item.match_quality = match.quality(); + item.rec_quality = queuedRec.quality(); matchSeenMap[match.id()] = 1; break; } @@ -979,6 +981,8 @@ function vlHandleQueueItemsAction(action) { vlUploadQueueAutoOverlayExact.attr('value', vlUploadQueueAutoOverlayExact2.attr('value')); vlUploadQueueAutoOverlay1Match.attr('value', vlUploadQueueAutoOverlay1Match2.attr('value')); vlUploadMergeProfile.attr('value', vlUploadMergeProfile2.attr('value')); + vlUploadQueueAutoOverlayBestMatch.attr('value', vlUploadQueueAutoOverlayBestMatch2.attr('value')); + vlUploadQueueAutoOverlayBestMatchRatio.attr('value', vlUploadQueueAutoOverlayBestMatchRatio2.attr('value')); if(action == 'import') { vlImportSelectedRecords(); @@ -995,6 +999,10 @@ function vlHandleQueueItemsAction(action) { vlUploadQueueAutoOverlay1Match2.attr('value', false); vlUploadMergeProfile.attr('value', ''); vlUploadMergeProfile2.attr('value', ''); + vlUploadQueueAutoOverlayBestMatch.attr('value', false); + vlUploadQueueAutoOverlayBestMatch2.attr('value', false); + vlUploadQueueAutoOverlayBestMatchRatio.attr('value', '0.0'); + vlUploadQueueAutoOverlayBestMatchRatio2.attr('value', '0.0'); } ); @@ -1002,8 +1010,8 @@ function vlHandleQueueItemsAction(action) { } +/* import user-selected records */ function vlImportSelectedRecords() { - displayGlobalDiv('vl-generic-progress-with-total'); var records = []; for(var id in selectableGridRecords) { @@ -1015,55 +1023,35 @@ function vlImportSelectedRecords() { } } - var options = {overlay_map : currentOverlayRecordsMap}; - - if(vlUploadQueueAutoOverlayExact.checked) { - options.auto_overlay_exact = true; - vlUploadQueueAutoOverlayExact.checked = false; - } - - if(vlUploadQueueAutoOverlay1Match.checked) { - options.auto_overlay_1match = true; - vlUploadQueueAutoOverlay1Match.checked = false; - } - - var profile = vlUploadMergeProfile.attr('value'); - if(profile != null && profile != '') { - options.merge_profile = profile; - } - - fieldmapper.standardRequest( - ['open-ils.vandelay', 'open-ils.vandelay.'+currentType+'_record.list.import'], - { async: true, - params: [authtoken, records, options], - onresponse: function(r) { - var resp = r.recv().content(); - if(e = openils.Event.parse(resp)) - return alert(e); - if(resp.complete) { - return retrieveQueuedRecords(currentType, currentQueueId, handleRetrieveRecords); - } else { - vlControlledProgressBar.update({maximum:resp.total, progress:resp.progress}); - } - }, + vlImportRecordQueue( + currentType, + currentQueueId, + records, + function(){ + retrieveQueuedRecords(currentType, currentQueueId, handleRetrieveRecords); } ); } +/* import all (non-imported) queue records */ function vlImportAllRecords() { - vlImportRecordQueue(currentType, currentQueueId, false, - function(){displayGlobalDiv('vl-queue-div');}); + vlImportRecordQueue( + currentType, + currentQueueId, + null, + function(){ + retrieveQueuedRecords(currentType, currentQueueId, handleRetrieveRecords); + } + ); } -function vlImportRecordQueue(type, queueId, onload) { +/* if recList has values, import only those records */ +function vlImportRecordQueue(type, queueId, recList, onload) { displayGlobalDiv('vl-generic-progress-with-total'); - var method = 'open-ils.vandelay.bib_queue.import'; - if(type == 'auth') - method = method.replace('bib', 'auth'); - + /* set up options */ var mergeOpt = false; - var options = {}; + var options = {overlay_map : currentOverlayRecordsMap}; if(vlUploadQueueImportNoMatch.checked) { options.import_no_match = true; @@ -1089,22 +1077,34 @@ function vlImportRecordQueue(type, queueId, onload) { mergeOpt = true; } + var profile = vlUploadMergeProfile.attr('value'); + if(profile != null && profile != '') { + options.merge_profile = profile; + } + + /* determine which method we're calling */ + + var method = 'open-ils.vandelay.bib_queue.import'; + if(type == 'auth') + method = method.replace('bib', 'auth'); + if(!mergeOpt) { // in the interest of speed, if no merge options are // chosen, tell the back-end code to only process records // that have no matches - method = method.replace('.import', 'nomatch.import'); + method = method.replace(/\.import/, '.nomatch.import'); } - - var profile = vlUploadMergeProfile.attr('value'); - if(profile != null && profile != '') { - options.merge_profile = profile; + + var params = [authtoken, queueId, options]; + if(recList) { + method = 'open-ils.vandelay.'+currentType+'_record.list.import'; + params[1] = recList; } fieldmapper.standardRequest( ['open-ils.vandelay', method], { async: true, - params: [authtoken, queueId, options], + params: params, onresponse: function(r) { var resp = r.recv().content(); if(e = openils.Event.parse(resp)) @@ -1134,6 +1134,7 @@ function batchUpload() { vlImportRecordQueue( currentType, currentQueueId, + null, function() { retrieveQueuedRecords(currentType, currentQueueId, handleRetrieveRecords); } @@ -1239,6 +1240,17 @@ function vlShowUploadForm() { vlUploadQueueAutoOverlayBestMatchRatio.attr('value', profile.lwm_ratio()+''); } ); + dojo.connect( + vlUploadMergeProfile2, + 'onChange', + function(val) { + if(!val) return; + var profile = mergeProfiles.filter(function(p) { return (p.id() == val); })[0]; + if(profile.lwm_ratio() != null) + vlUploadQueueAutoOverlayBestMatchRatio2.attr('value', profile.lwm_ratio()+''); + } + ); + } function vlShowQueueSelect() { diff --git a/Open-ILS/web/templates/default/vandelay/inc/matches.tt2 b/Open-ILS/web/templates/default/vandelay/inc/matches.tt2 index 40d6a80d82..2376dbc259 100644 --- a/Open-ILS/web/templates/default/vandelay/inc/matches.tt2 +++ b/Open-ILS/web/templates/default/vandelay/inc/matches.tt2 @@ -15,6 +15,7 @@ formatter : vlFormatViewMatchMARC }, {name: 'Match Score', field:'match_score'}, + {name: 'Queued Record Quality', field:'rec_quality'}, {name: 'Matched Record Quality', field:'match_quality'}, {name: '&vandelay.creator;', get: vlGetCreator}, {name: '&vandelay.create.date;', field:'create_date', get: vlGetDateTimeField}, diff --git a/Open-ILS/web/templates/default/vandelay/inc/queue.tt2 b/Open-ILS/web/templates/default/vandelay/inc/queue.tt2 index 00c812dce6..b3402642f5 100644 --- a/Open-ILS/web/templates/default/vandelay/inc/queue.tt2 +++ b/Open-ILS/web/templates/default/vandelay/inc/queue.tt2 @@ -9,10 +9,10 @@ <table class='queue-nav-table'> <thead><tr><th colspan='2' class='queue-nav-table-label'>Queue Actions</th></tr></thead> <tbody> - <tr><td><a href='#' onclick='vlHandleQueueItemsAction("import")'>&vandelay.import.selected;</a></td></tr> - <tr><td><a href='#' onclick='vlHandleQueueItemsAction("import_all")'>&vandelay.import.all;</a></td></tr> - <tr><td><a href='#' onclick='vlLoadErrorUIAll();'>View Import Items</a></td></tr> - <tr><td><a href='#' onclick=' + <tr><td><a href='javascript:;' onclick='vlHandleQueueItemsAction("import")'>&vandelay.import.selected;</a></td></tr> + <tr><td><a href='javascript:;' onclick='vlHandleQueueItemsAction("import_all")'>&vandelay.import.all;</a></td></tr> + <tr><td><a href='javascript:;' onclick='vlLoadErrorUIAll();'>View Import Items</a></td></tr> + <tr><td><a href='javascript:;' onclick=' if(confirm("&vandelay.sure.to.delete.queue;")) { vlDeleteQueue(currentType, currentQueueId, function() { displayGlobalDiv("vl-marc-upload-div"); });}'>&vandelay.delete.queue;</a></td></tr> @@ -186,9 +186,16 @@ <table class='form_table'> <tbody> <tr> + <td>&vandelay.auto.import.merge_profile;</td> + <td colspan='4'> + <div jsId='vlUploadMergeProfile2' + dojoType='dijit.form.FilteringSelect' required='false' labelAttr='name' searchAttr='name'/> + </td> + </tr> + <tr> <td>&vandelay.auto.import.noncolliding;</td> <td colspan='4'> - <input jsId='vlUploadQueueAutoImport2' dojoType='dijit.form.CheckBox'/> + <input jsId='vlUploadQueueImportNoMatch2' dojoType='dijit.form.CheckBox'/> </td> </tr> <tr> @@ -204,10 +211,14 @@ </td> </tr> <tr> - <td>&vandelay.auto.import.merge_profile;</td> + <td>&vandelay.auto.import.auto_overlay_best;</td> + <td colspan='4'><input jsId='vlUploadQueueAutoOverlayBestMatch2' dojoType='dijit.form.CheckBox'/></td> + </tr> + <tr> + <td>&vandelay.auto.import.auto_overlay_best_ratio;</td> <td colspan='4'> - <div jsId='vlUploadMergeProfile2' - dojoType='dijit.form.FilteringSelect' required='false' labelAttr='name' searchAttr='name'/> + <input style='width:3em' value='0.0' jsId='vlUploadQueueAutoOverlayBestMatchRatio2' dojoType='dijit.form.TextBox'/> + <span style='padding-left: 10px; font-size:90%'>(&vandelay.auto.import.auto_overlay_best_ratio.desc;)</span> </td> </tr> <tr> -- 2.11.0