From: Bill Erickson Date: Thu, 21 May 2020 16:06:38 +0000 (-0400) Subject: LP1880726 MARC Batch edit Angular port X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=0e1e10b6d4572fd9fab1b86cb78719dcc1a9d54c;p=working%2FEvergreen.git LP1880726 MARC Batch edit Angular port Ports the MARC Batch Edit interface to Angular. Port includes a new feature (Per-Record Transactions) which allows for large batches to be processed with each bib record in its own transaction. This mitigates potential conflicts with other cataloging activities. This setting is saved in a new workstation preference. Signed-off-by: Bill Erickson --- diff --git a/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.html b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.html new file mode 100644 index 0000000000..f7a00ee316 --- /dev/null +++ b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.html @@ -0,0 +1,208 @@ + + +
+
+ +
+
+
Rule Setup
+
Data
+
Help
+
+
+
Action (Rule Type)
+
+ +
+
How to change the existing record.
+
+
+
MARC Tag
+
+ +
+
+ Three characters, no spaces, no indicators, etc. eg: 245 +
+
+
+
Subfields (optional)
+
+ +
+
No spaces, no delimiters, eg: abcnp
+
+
+
MARC Data
+
+ +
+
+ MARC-Breaker formatted data with indicators and subfield delimiters, + eg: 245 04$aThe End +
+
+
+
+
+ Advanced Matching Restriction (Optional) +
+
+
+
+
Subfield
+
+ +
+
+ A single subfield code, no delimiters, eg: a +
+
+
+
Expression
+
+ +
+
+ See the + + Perl documentation + for an explanation of Regular Expressions. +
+
+
+
+ +
+
+ +
+
+ +
+
+
+
+
+
+
Merge Template Preview
+
+ +
+
+
+
+
Record Source:
+
+ +
+
+
+ +
Bucket named:
+
+ + +
+ + +
Record ID:
+
+ +
+ + +
+
+
Column:
+
+ + of +
+
+ +
+
+
+
+ Columns are numbered starting at 0. For instance, when looking + at a CSV file in Excel, the column labeled A is the same as + column 0, and the column labeled B is the same as column 1. +
+
+
+ +
+
+
+
+ + +
+
+
+
+
+ By default, all records are processed within a single database + transaction. However, processing large batches of records + within a single transaction can negatively impact a running + system as it may conflict with other cataloging activities. + Use Per-Record transactions to mitigate these conflicts. +
+
+
+
+ +
+
+
+
+ + +
+
+
+
+
Processing Complete
+
+
Success count:
+
{{this.numSucceeded}}
+
+
+
Failed count:
+
{{this.numFailed}}
+
+
+
+
+
diff --git a/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.ts b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.ts new file mode 100644 index 0000000000..cdc5161230 --- /dev/null +++ b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.ts @@ -0,0 +1,284 @@ +import {Component, OnInit, ViewChild, Renderer2} from '@angular/core'; +import {Router, ActivatedRoute, ParamMap} from '@angular/router'; +import {HttpClient} from '@angular/common/http'; +import {tap} from 'rxjs/operators'; +import {NetService} from '@eg/core/net.service'; +import {AuthService} from '@eg/core/auth.service'; +import {PcrudService} from '@eg/core/pcrud.service'; +import {ComboboxEntry} from '@eg/share/combobox/combobox.component'; +import {MarcRecord, MarcField} from '@eg/staff/share/marc-edit/marcrecord'; +import {AnonCacheService} from '@eg/share/util/anon-cache.service'; +import {ServerStoreService} from '@eg/core/server-store.service'; + +const SESSION_POLL_INTERVAL = 2; // seconds +const MERGE_TEMPLATE_PATH = '/opac/extras/merge_template'; +const PREF_XACT_PER_RECORD = 'eg.cat.marcbatch.xact_per_record'; + +interface TemplateRule { + ruleType: 'r' | 'a' | 'd'; + marcTag?: string; + marcSubfields?: string; + marcData?: string; + advSubfield?: string; + advRegex?: string; +} + +@Component({ + templateUrl: 'marcbatch.component.html' +}) +export class MarcBatchComponent implements OnInit { + + session: string; + source: 'b' | 'c' | 'r' = 'b'; + buckets: ComboboxEntry[]; + bucket: number; + recordId: number; + csvColumn = 0; + csvFile: File; + xactPerRecord = false; + templateRules: TemplateRule[] = []; + record: MarcRecord; + + processing = false; + progressMax: number = null; + progressValue: number = null; + numSucceeded = 0; + numFailed = 0; + + constructor( + private router: Router, + private route: ActivatedRoute, + private http: HttpClient, + private renderer: Renderer2, + private net: NetService, + private pcrud: PcrudService, + private auth: AuthService, + private store: ServerStoreService, + private cache: AnonCacheService + ) {} + + ngOnInit() { + this.load(); + } + + load() { + this.addRule(); + this.getBuckets(); + + this.store.getItem(PREF_XACT_PER_RECORD) + .then(value => this.xactPerRecord = value); + } + + rulesetToRecord(resetRuleData?: boolean) { + this.record = new MarcRecord(); + + this.templateRules.forEach(rule => { + + if (!rule.marcTag) { return; } + + let ruleText = rule.marcTag + (rule.marcSubfields || ''); + if (rule.advSubfield) { + ruleText += + `[${rule.advSubfield || ''} ~ ${rule.advRegex || ''}]`; + } + + // Merge behavior is encoded in the 905 field. + const ruleTag = this.record.newField({ + tag: '905', + ind1: ' ', + ind2: ' ', + subfields: [[rule.ruleType, ruleText, 0]] + }); + + this.record.insertOrderedFields(ruleTag); + + if (rule.ruleType === 'd') { + rule.marcData = ''; + return; + } + + const dataRec = new MarcRecord(); + if (resetRuleData || !rule.marcData) { + + // Build a new value for the 'MARC Data' field based on + // changes to the selected tag or subfields. + + const subfields = rule.marcSubfields ? + rule.marcSubfields.split('').map((sf, idx) => [sf, '', idx]) + : []; + + dataRec.appendFields( + dataRec.newField({ + tag: rule.marcTag, + ind1: ' ', + ind2: ' ', + subfields: subfields + }) + ); + + console.log(dataRec.toBreaker()); + rule.marcData = dataRec.toBreaker().split(/\n/)[1]; + + } else { + + // Absorb the breaker data already in the 'MARC Data' field + // so it can be added to the template record in progress. + + dataRec.breakerText = rule.marcData; + dataRec.absorbBreakerChanges(); + } + + this.record.appendFields(dataRec.fields[0]); + }); + } + + breakerRows(): number { + if (this.record) { + const breaker = this.record.toBreaker(); + if (breaker) { + return breaker.split(/\n/).length + 1; + } + } + return 3; + } + + breaker(): string { + return this.record ? this.record.toBreaker() : ''; + } + + addRule() { + this.templateRules.push({ruleType: 'r'}); + } + + removeRule(idx: number) { + this.templateRules.splice(idx, 1); + } + + getBuckets(): Promise { + if (this.buckets) { return Promise.resolve(); } + + return this.net.request( + 'open-ils.actor', + 'open-ils.actor.container.retrieve_by_class', + this.auth.token(), this.auth.user().id(), + 'biblio', ['staff_client', 'vandelay_queue'] + + ).pipe(tap(buckets => { + this.buckets = buckets + .sort((b1, b2) => b1.name() < b2.name() ? -1 : 1) + .map(b => ({id: b.id(), label: b.name()})); + + })).toPromise(); + } + + bucketChanged(entry: ComboboxEntry) { + this.bucket = entry ? entry.id : null; + } + + fileSelected($event) { + this.csvFile = $event.target.files[0]; + } + + disableSave(): boolean { + if (!this.record || !this.source || this.processing) { + return true; + } + + if (this.source === 'b') { + return !this.bucket; + + } else if (this.source === 'c') { + return (!this.csvColumn || !this.csvFile); + + } else if (this.source === 'r') { + return !this.recordId; + } + } + + process() { + this.processing = true; + this.progressValue = null; + this.progressMax = null; + this.numSucceeded = 0; + this.numFailed = 0; + this.setReplaceMode(); + this.postForm().then(_ => this.pollProgress()); + } + + setReplaceMode() { + if (this.record.subfield('905', 'r').length === 0) { + // Force replace mode w/ no-op replace rule. + this.record.appendFields( + this.record.newField({ + tag : '905', + ind1 : ' ', + ind2 : ' ', + subfields : [['r', '901c']] + }) + ); + } + } + + postForm(): Promise { + + const formData: FormData = new FormData(); + formData.append('ses', this.auth.token()); + formData.append('skipui', '1'); + formData.append('template', this.record.toXml()); + formData.append('recordSource', this.source); + if (this.xactPerRecord) { + formData.append('xactPerRecord', '1'); + } + + if (this.source === 'b') { + formData.append('containerid', this.bucket + ''); + + } else if (this.source === 'c') { + formData.append('idcolumn', this.csvColumn + ''); + formData.append('idfile', this.csvFile, this.csvFile.name); + + } else if (this.source === 'r') { + formData.append('recid', this.recordId + ''); + } + + return this.http.post( + MERGE_TEMPLATE_PATH, formData, {responseType: 'text'}) + .pipe(tap(cacheKey => this.session = cacheKey)) + .toPromise(); + } + + pollProgress(): Promise { + console.debug('Polling session ', this.session); + + return this.cache.getItem(this.session, 'batch_edit_progress') + .then(progress => { + // {"success":"t","complete":1,"failed":0,"succeeded":252} + + if (!progress) { + console.error('No batch edit session found for ', this.session); + return; + } + + this.progressValue = progress.succeeded; + this.progressMax = progress.total; + this.numSucceeded = progress.succeeded; + this.numFailed = progress.failed; + + if (progress.complete) { + this.processing = false; + return; + } + + setTimeout(() => this.pollProgress(), SESSION_POLL_INTERVAL * 1000); + }); + } + + setPerXactPref() { + if (this.xactPerRecord) { + this.store.setItem(PREF_XACT_PER_RECORD, true); + } else { + this.store.removeItem(PREF_XACT_PER_RECORD); + } + } +} + diff --git a/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.module.ts b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.module.ts new file mode 100644 index 0000000000..bf81553919 --- /dev/null +++ b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.module.ts @@ -0,0 +1,23 @@ +import {NgModule} from '@angular/core'; +import {StaffCommonModule} from '@eg/staff/common.module'; +import {CommonWidgetsModule} from '@eg/share/common-widgets.module'; +import {MarcBatchRoutingModule} from './routing.module'; +import {MarcBatchComponent} from './marcbatch.component'; +import {HttpClientModule} from '@angular/common/http'; + +@NgModule({ + declarations: [ + MarcBatchComponent + ], + imports: [ + StaffCommonModule, + HttpClientModule, + CommonWidgetsModule, + MarcBatchRoutingModule + ], + providers: [ + ] +}) + +export class MarcBatchModule { +} diff --git a/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/routing.module.ts b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/routing.module.ts new file mode 100644 index 0000000000..bb268e644d --- /dev/null +++ b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/routing.module.ts @@ -0,0 +1,17 @@ +import {NgModule} from '@angular/core'; +import {RouterModule, Routes} from '@angular/router'; +import {MarcBatchComponent} from './marcbatch.component'; + +const routes: Routes = [{ + path: '', + component: MarcBatchComponent +}]; + +@NgModule({ + imports: [RouterModule.forChild(routes)], + exports: [RouterModule], + providers: [] +}) + +export class MarcBatchRoutingModule {} + diff --git a/Open-ILS/src/eg2/src/app/staff/cat/routing.module.ts b/Open-ILS/src/eg2/src/app/staff/cat/routing.module.ts index 67fb59b56c..60d87b555e 100644 --- a/Open-ILS/src/eg2/src/app/staff/cat/routing.module.ts +++ b/Open-ILS/src/eg2/src/app/staff/cat/routing.module.ts @@ -7,6 +7,9 @@ const routes: Routes = [ }, { path: 'authority', loadChildren: '@eg/staff/cat/authority/authority.module#AuthorityModule' + }, { + path: 'marcbatch', + loadChildren: '@eg/staff/cat/marcbatch/marcbatch.module#MarcBatchModule' } ]; diff --git a/Open-ILS/src/eg2/src/app/staff/nav.component.html b/Open-ILS/src/eg2/src/app/staff/nav.component.html index 5310b5b340..59b7ac2c6b 100644 --- a/Open-ILS/src/eg2/src/app/staff/nav.component.html +++ b/Open-ILS/src/eg2/src/app/staff/nav.component.html @@ -208,7 +208,7 @@ import_export MARC Batch Import/Export - + format_paint MARC Batch Edit diff --git a/Open-ILS/src/eg2/src/app/staff/share/marc-edit/marcrecord.ts b/Open-ILS/src/eg2/src/app/staff/share/marc-edit/marcrecord.ts index d49f4ef568..493eaadcbe 100644 --- a/Open-ILS/src/eg2/src/app/staff/share/marc-edit/marcrecord.ts +++ b/Open-ILS/src/eg2/src/app/staff/share/marc-edit/marcrecord.ts @@ -60,7 +60,7 @@ export class MarcRecord { this.record.fields = f; } - constructor(xml: string) { + constructor(xml?: string) { this.record = new MARC21.Record({marcxml: xml, delimiter: DELIMITER}); this.breakerText = this.record.toBreaker(); this.fixedFieldChange = new EventEmitter(); @@ -113,6 +113,11 @@ export class MarcRecord { return this.record.field(spec, wantArray); } + appendFields(...newFields: MarcField[]) { + this.record.appendFields.apply(this.record, newFields); + this.stampFieldIds(); + } + insertFieldsBefore(field: MarcField, ...newFields: MarcField[]) { this.record.insertFieldsBefore.apply( this.record, [field].concat(newFields)); @@ -186,5 +191,10 @@ export class MarcRecord { subfields.forEach(sf => root.push([].concat(sf))); return root; } + + // Returns a list of values for the tag + subfield combo + subfield(tag: string, subfield: string): string { + return this.record.subfield(tag, subfield); + } } diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Cat.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Cat.pm index 4c4d977cca..1df2d6eda6 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Cat.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Cat.pm @@ -238,12 +238,17 @@ __PACKAGE__->register_method( @param auth The authtoken @param container The container, um, containing the records to be updated by the template @param template The overlay template, or nothing and the method will look for a negative bib id in the container + @param options Hash of options; currently supports: + xact_per_record: Apply updates to each bib record within its own transaction. @return Cache key to check for status of the container overlay # ); sub template_overlay_container { - my($self, $conn, $auth, $container, $template) = @_; + my($self, $conn, $auth, $container, $template, $options) = @_; + $options ||= {}; + my $xact_per_rec = $options->{xact_per_record}; + my $e = new_editor(authtoken=>$auth, xact=>1); return $e->die_event unless $e->checkauth; @@ -263,14 +268,20 @@ sub template_overlay_container { $template = $e->retrieve_biblio_record_entry( $titem->target_biblio_record_entry )->marc; } + my $num_total = scalar(@$items); my $num_failed = 0; my $num_succeeded = 0; $conn->respond_complete( - $actor->request('open-ils.actor.anon_cache.set_value', $auth, batch_edit_progress => {})->gather(1) + $actor->request('open-ils.actor.anon_cache.set_value', $auth, + batch_edit_progress => {total => $num_total})->gather(1) ) if ($actor); + # read-only up to here. + $e->rollback if $xact_per_rec; + for my $item ( @$items ) { + $e->xact_begin if $xact_per_rec; my $rec = $e->retrieve_biblio_record_entry($item->target_biblio_record_entry); next unless $rec; @@ -291,6 +302,7 @@ sub template_overlay_container { $actor->request( 'open-ils.actor.anon_cache.set_value', $auth, batch_edit_progress => { + total => $num_total, succeeded => $num_succeeded, failed => $num_failed }, @@ -308,6 +320,7 @@ sub template_overlay_container { batch_edit_progress => { complete => 1, success => 'f', + total => $num_total, succeeded => $num_succeeded, failed => $num_failed, } @@ -318,19 +331,23 @@ sub template_overlay_container { } } } + $e->xact_commit if $xact_per_rec; } if ($titem && !$num_failed) { + $e->xact_begin if $xact_per_rec; return $e->die_event unless ($e->delete_container_biblio_record_entry_bucket_item($titem)); + $e->xact_commit if $xact_per_rec; } - if ($e->commit) { + if ($xact_per_rec || $e->commit) { if ($actor) { $actor->request( 'open-ils.actor.anon_cache.set_value', $auth, batch_edit_progress => { complete => 1, success => 't', + total => $num_total, succeeded => $num_succeeded, failed => $num_failed, } @@ -345,6 +362,7 @@ sub template_overlay_container { batch_edit_progress => { complete => 1, success => 'f', + total => $num_total, succeeded => $num_succeeded, failed => $num_failed, } diff --git a/Open-ILS/src/perlmods/lib/OpenILS/WWW/TemplateBatchBibUpdate.pm b/Open-ILS/src/perlmods/lib/OpenILS/WWW/TemplateBatchBibUpdate.pm index 27a03e1107..ae91b36a94 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/WWW/TemplateBatchBibUpdate.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/WWW/TemplateBatchBibUpdate.pm @@ -4,7 +4,7 @@ use warnings; use bytes; use Apache2::Log; -use Apache2::Const -compile => qw(OK REDIRECT DECLINED NOT_FOUND :log); +use Apache2::Const -compile => qw(OK REDIRECT DECLINED NOT_FOUND HTTP_BAD_REQUEST :log); use APR::Const -compile => qw(:error SUCCESS); use APR::Table; @@ -56,14 +56,21 @@ sub handler { my $cgi = new CGI; my $authid = $cgi->cookie('ses') || $cgi->param('ses'); + + # Avoid sending the HTML to the caller. Final response will + # will just be the cache key or HTTP_BAD_REQUEST on error. + my $skipui = $cgi->param('skipui'); + my $usr = verify_login($authid); - return show_template($r) unless ($usr); + return show_template($r, $skipui) unless ($usr); my $template = $cgi->param('template'); - return show_template($r) unless ($template); + return show_template($r, $skipui) unless ($template); my $rsource = $cgi->param('recordSource'); + my $xact_per = $cgi->param('xactPerRecord'); + # find some IDs ... my @records; @@ -118,7 +125,7 @@ sub handler { unless (@records) { $e->request('open-ils.cstore.transaction.rollback')->gather(1); $e->disconnect; - return show_template($r); + return show_template($r, $skipui); } # we have a template and some record ids, so... @@ -173,10 +180,12 @@ sub handler { # fire the background bucket processor my $cache_key = OpenSRF::AppSession ->create('open-ils.cat') - ->request('open-ils.cat.container.template_overlay.background', $authid, $bucket->id) + ->request('open-ils.cat.container.template_overlay.background', + $authid, $bucket->id, undef, {xact_per_record => $xact_per}) ->gather(1); - return show_processing_template($r, $bucket->id, \@records, $cache_key); + return show_processing_template( + $r, $bucket->id, \@records, $cache_key, $skipui); } sub verify_login { @@ -201,6 +210,13 @@ sub show_processing_template { my $bid = shift; my $recs = shift; my $cache_key = shift; + my $skipui = shift; + + if ($skipui) { + $r->content_type('text/plain'); + $r->print($cache_key); + return Apache2::Const::OK; + } my $rec_string = @$recs; @@ -366,6 +382,11 @@ HTML sub show_template { my $r = shift; + my $skipui = shift; + + # Makes no sense to call the API in such a way that the caller + # is returned the UI code if skipui is set. + return Apache2::Const::HTTP_BAD_REQUEST if $skipui; $r->content_type('text/html'); $r->print(<<'HTML'); diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql index b6959bb20e..4f29979f87 100644 --- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql +++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql @@ -20414,4 +20414,13 @@ VALUES ( ) ); +INSERT INTO config.workstation_setting_type (name, grp, datatype, label) +VALUES ( + 'eg.cat.marcbatch.xact_per_record', 'gui', 'bool', + oils_i18n_gettext( + 'eg.cat.marcbatch.xact_per_record', + 'MARC Batch Edit Uses Per-Record Transactions', + 'cwst', 'label' + ) +); diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.data.marcbatch-prefs.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.data.marcbatch-prefs.sql new file mode 100644 index 0000000000..e19f0c1bb7 --- /dev/null +++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.data.marcbatch-prefs.sql @@ -0,0 +1,15 @@ +BEGIN; + +-- SELECT evergreen.upgrade_deps_block_check('TODO', :eg_version); + +INSERT INTO config.workstation_setting_type (name, grp, datatype, label) +VALUES ( + 'eg.cat.marcbatch.xact_per_record', 'gui', 'bool', + oils_i18n_gettext( + 'eg.cat.marcbatch.xact_per_record', + 'MARC Batch Edit Uses Per-Record Transactions', + 'cwst', 'label' + ) +); + +COMMIT; diff --git a/Open-ILS/src/templates/staff/navbar.tt2 b/Open-ILS/src/templates/staff/navbar.tt2 index 220c3d1d1f..4de364cfa3 100644 --- a/Open-ILS/src/templates/staff/navbar.tt2 +++ b/Open-ILS/src/templates/staff/navbar.tt2 @@ -322,7 +322,7 @@
  • - + [% l('MARC Batch Edit') %] diff --git a/docs/RELEASE_NOTES_NEXT/Cataloging/marcbatch-ang-port.adoc b/docs/RELEASE_NOTES_NEXT/Cataloging/marcbatch-ang-port.adoc new file mode 100644 index 0000000000..367946da75 --- /dev/null +++ b/docs/RELEASE_NOTES_NEXT/Cataloging/marcbatch-ang-port.adoc @@ -0,0 +1,8 @@ +MARC Batch Edit UI Angular Port +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The MARC Batch Edit interface has been ported to Angular. Port includes a +new feature (Per-Record Transactions) which allows for large batches to be +processed with each bib record in it own transaction. This mitigates +potential conflicts with other cataloging activities. This setting is +saved in a new workstation preference. +