From: Bill Erickson Date: Thu, 21 May 2020 16:06:38 +0000 (-0400) Subject: LP1880726 MARC Batch edit Angular port X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=90c205f7cbc3f244b47f82d1c9529173cfa837e7;p=Evergreen.git LP1880726 MARC Batch edit Angular port Angular port of the MARC Batch Edit interface. Under the covers, each bib record is now modified within its own transaction to avoid long-running transactions that can potentially lock database rows needed by other processes. Signed-off-by: Bill Erickson Signed-off-by: Mike Rylander --- diff --git a/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.html b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.html new file mode 100644 index 0000000000..d3cbb1306d --- /dev/null +++ b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.html @@ -0,0 +1,187 @@ + + +
+
+ +
+
+
Rule Setup
+
Data
+
Help
+
+
+
Action (Rule Type)
+
+ +
+
How to change the existing record.
+
+
+
MARC Tag
+
+ +
+
+ Three characters, no spaces, no indicators, etc. eg: 245 +
+
+
+
Subfields (optional)
+
+ +
+
No spaces, no delimiters, eg: abcnp
+
+
+
MARC Data
+
+ +
+
+ MARC-Breaker formatted data with indicators and subfield delimiters, + eg: 245 04$aThe End +
+
+
+
+
+ Advanced Matching Restriction (Optional) +
+
+
+
+
Subfield
+
+ +
+
+ A single subfield code, no delimiters, eg: a +
+
+
+
Expression
+
+ +
+
+ See the + + Perl documentation + for an explanation of Regular Expressions. +
+
+
+
+ +
+
+ +
+
+ +
+
+
+
+
+
+
Merge Template Preview
+
+ +
+
+
+
+
Record Source:
+
+ +
+
+
+ +
Bucket named:
+
+ + +
+ + +
Record ID:
+
+ +
+ + +
+
+
Column:
+
+ + of +
+
+ +
+
+
+
+ Columns are numbered starting at 0. For instance, when looking + at a CSV file in Excel, the column labeled A is the same as + column 0, and the column labeled B is the same as column 1. +
+
+
+ +
+
+
+ +
+
+
+
+ + +
+
+
+
+
Processing Complete
+
+
Success count:
+
{{this.numSucceeded}}
+
+
+
Failed count:
+
{{this.numFailed}}
+
+
+
+
+
diff --git a/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.ts b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.ts new file mode 100644 index 0000000000..84c7f354dd --- /dev/null +++ b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.ts @@ -0,0 +1,269 @@ +import {Component, OnInit, ViewChild, Renderer2} from '@angular/core'; +import {Router, ActivatedRoute, ParamMap} from '@angular/router'; +import {HttpClient} from '@angular/common/http'; +import {tap} from 'rxjs/operators'; +import {NetService} from '@eg/core/net.service'; +import {AuthService} from '@eg/core/auth.service'; +import {PcrudService} from '@eg/core/pcrud.service'; +import {ComboboxEntry} from '@eg/share/combobox/combobox.component'; +import {MarcRecord, MarcField} from '@eg/staff/share/marc-edit/marcrecord'; +import {AnonCacheService} from '@eg/share/util/anon-cache.service'; +import {ServerStoreService} from '@eg/core/server-store.service'; + +const SESSION_POLL_INTERVAL = 2; // seconds +const MERGE_TEMPLATE_PATH = '/opac/extras/merge_template'; + +interface TemplateRule { + ruleType: 'r' | 'a' | 'd'; + marcTag?: string; + marcSubfields?: string; + marcData?: string; + advSubfield?: string; + advRegex?: string; +} + +@Component({ + templateUrl: 'marcbatch.component.html' +}) +export class MarcBatchComponent implements OnInit { + + session: string; + source: 'b' | 'c' | 'r' = 'b'; + buckets: ComboboxEntry[]; + bucket: number; + recordId: number; + csvColumn = 0; + csvFile: File; + templateRules: TemplateRule[] = []; + record: MarcRecord; + + processing = false; + progressMax: number = null; + progressValue: number = null; + numSucceeded = 0; + numFailed = 0; + + constructor( + private router: Router, + private route: ActivatedRoute, + private http: HttpClient, + private renderer: Renderer2, + private net: NetService, + private pcrud: PcrudService, + private auth: AuthService, + private store: ServerStoreService, + private cache: AnonCacheService + ) {} + + ngOnInit() { + this.load(); + } + + load() { + this.addRule(); + this.getBuckets(); + } + + rulesetToRecord(resetRuleData?: boolean) { + this.record = new MarcRecord(); + + this.templateRules.forEach(rule => { + + if (!rule.marcTag) { return; } + + let ruleText = rule.marcTag + (rule.marcSubfields || ''); + if (rule.advSubfield) { + ruleText += + `[${rule.advSubfield || ''} ~ ${rule.advRegex || ''}]`; + } + + // Merge behavior is encoded in the 905 field. + const ruleTag = this.record.newField({ + tag: '905', + ind1: ' ', + ind2: ' ', + subfields: [[rule.ruleType, ruleText, 0]] + }); + + this.record.insertOrderedFields(ruleTag); + + if (rule.ruleType === 'd') { + rule.marcData = ''; + return; + } + + const dataRec = new MarcRecord(); + if (resetRuleData || !rule.marcData) { + + // Build a new value for the 'MARC Data' field based on + // changes to the selected tag or subfields. + + const subfields = rule.marcSubfields ? + rule.marcSubfields.split('').map((sf, idx) => [sf, '', idx]) + : []; + + dataRec.appendFields( + dataRec.newField({ + tag: rule.marcTag, + ind1: ' ', + ind2: ' ', + subfields: subfields + }) + ); + + console.log(dataRec.toBreaker()); + rule.marcData = dataRec.toBreaker().split(/\n/)[1]; + + } else { + + // Absorb the breaker data already in the 'MARC Data' field + // so it can be added to the template record in progress. + + dataRec.breakerText = rule.marcData; + dataRec.absorbBreakerChanges(); + } + + this.record.appendFields(dataRec.fields[0]); + }); + } + + breakerRows(): number { + if (this.record) { + const breaker = this.record.toBreaker(); + if (breaker) { + return breaker.split(/\n/).length + 1; + } + } + return 3; + } + + breaker(): string { + return this.record ? this.record.toBreaker() : ''; + } + + addRule() { + this.templateRules.push({ruleType: 'r'}); + } + + removeRule(idx: number) { + this.templateRules.splice(idx, 1); + } + + getBuckets(): Promise { + if (this.buckets) { return Promise.resolve(); } + + return this.net.request( + 'open-ils.actor', + 'open-ils.actor.container.retrieve_by_class', + this.auth.token(), this.auth.user().id(), + 'biblio', ['staff_client', 'vandelay_queue'] + + ).pipe(tap(buckets => { + this.buckets = buckets + .sort((b1, b2) => b1.name() < b2.name() ? -1 : 1) + .map(b => ({id: b.id(), label: b.name()})); + + })).toPromise(); + } + + bucketChanged(entry: ComboboxEntry) { + this.bucket = entry ? entry.id : null; + } + + fileSelected($event) { + this.csvFile = $event.target.files[0]; + } + + disableSave(): boolean { + if (!this.record || !this.source || this.processing) { + return true; + } + + if (this.source === 'b') { + return !this.bucket; + + } else if (this.source === 'c') { + return (!this.csvColumn || !this.csvFile); + + } else if (this.source === 'r') { + return !this.recordId; + } + } + + process() { + this.processing = true; + this.progressValue = null; + this.progressMax = null; + this.numSucceeded = 0; + this.numFailed = 0; + this.setReplaceMode(); + this.postForm().then(_ => this.pollProgress()); + } + + setReplaceMode() { + if (this.record.subfield('905', 'r').length === 0) { + // Force replace mode w/ no-op replace rule. + this.record.appendFields( + this.record.newField({ + tag : '905', + ind1 : ' ', + ind2 : ' ', + subfields : [['r', '901c']] + }) + ); + } + } + + postForm(): Promise { + + const formData: FormData = new FormData(); + formData.append('ses', this.auth.token()); + formData.append('skipui', '1'); + formData.append('template', this.record.toXml()); + formData.append('recordSource', this.source); + formData.append('xactPerRecord', '1'); + + if (this.source === 'b') { + formData.append('containerid', this.bucket + ''); + + } else if (this.source === 'c') { + formData.append('idcolumn', this.csvColumn + ''); + formData.append('idfile', this.csvFile, this.csvFile.name); + + } else if (this.source === 'r') { + formData.append('recid', this.recordId + ''); + } + + return this.http.post( + MERGE_TEMPLATE_PATH, formData, {responseType: 'text'}) + .pipe(tap(cacheKey => this.session = cacheKey)) + .toPromise(); + } + + pollProgress(): Promise { + console.debug('Polling session ', this.session); + + return this.cache.getItem(this.session, 'batch_edit_progress') + .then(progress => { + // {"success":"t","complete":1,"failed":0,"succeeded":252} + + if (!progress) { + console.error('No batch edit session found for ', this.session); + return; + } + + this.progressValue = progress.succeeded; + this.progressMax = progress.total; + this.numSucceeded = progress.succeeded; + this.numFailed = progress.failed; + + if (progress.complete) { + this.processing = false; + return; + } + + setTimeout(() => this.pollProgress(), SESSION_POLL_INTERVAL * 1000); + }); + } +} + diff --git a/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.module.ts b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.module.ts new file mode 100644 index 0000000000..bf81553919 --- /dev/null +++ b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.module.ts @@ -0,0 +1,23 @@ +import {NgModule} from '@angular/core'; +import {StaffCommonModule} from '@eg/staff/common.module'; +import {CommonWidgetsModule} from '@eg/share/common-widgets.module'; +import {MarcBatchRoutingModule} from './routing.module'; +import {MarcBatchComponent} from './marcbatch.component'; +import {HttpClientModule} from '@angular/common/http'; + +@NgModule({ + declarations: [ + MarcBatchComponent + ], + imports: [ + StaffCommonModule, + HttpClientModule, + CommonWidgetsModule, + MarcBatchRoutingModule + ], + providers: [ + ] +}) + +export class MarcBatchModule { +} diff --git a/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/routing.module.ts b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/routing.module.ts new file mode 100644 index 0000000000..bb268e644d --- /dev/null +++ b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/routing.module.ts @@ -0,0 +1,17 @@ +import {NgModule} from '@angular/core'; +import {RouterModule, Routes} from '@angular/router'; +import {MarcBatchComponent} from './marcbatch.component'; + +const routes: Routes = [{ + path: '', + component: MarcBatchComponent +}]; + +@NgModule({ + imports: [RouterModule.forChild(routes)], + exports: [RouterModule], + providers: [] +}) + +export class MarcBatchRoutingModule {} + diff --git a/Open-ILS/src/eg2/src/app/staff/cat/routing.module.ts b/Open-ILS/src/eg2/src/app/staff/cat/routing.module.ts index d4b2770d8a..b3523789d9 100644 --- a/Open-ILS/src/eg2/src/app/staff/cat/routing.module.ts +++ b/Open-ILS/src/eg2/src/app/staff/cat/routing.module.ts @@ -9,6 +9,10 @@ const routes: Routes = [ path: 'authority', loadChildren: () => import('./authority/authority.module').then(m => m.AuthorityModule) + }, { + path: 'marcbatch', + loadChildren: () => + import('./marcbatch/marcbatch.module').then(m => m.MarcBatchModule) } ]; diff --git a/Open-ILS/src/eg2/src/app/staff/nav.component.html b/Open-ILS/src/eg2/src/app/staff/nav.component.html index a36ce00386..77253416cb 100644 --- a/Open-ILS/src/eg2/src/app/staff/nav.component.html +++ b/Open-ILS/src/eg2/src/app/staff/nav.component.html @@ -206,7 +206,7 @@ MARC Batch Import/Export - + MARC Batch Edit diff --git a/Open-ILS/src/eg2/src/app/staff/share/marc-edit/marcrecord.ts b/Open-ILS/src/eg2/src/app/staff/share/marc-edit/marcrecord.ts index d49f4ef568..493eaadcbe 100644 --- a/Open-ILS/src/eg2/src/app/staff/share/marc-edit/marcrecord.ts +++ b/Open-ILS/src/eg2/src/app/staff/share/marc-edit/marcrecord.ts @@ -60,7 +60,7 @@ export class MarcRecord { this.record.fields = f; } - constructor(xml: string) { + constructor(xml?: string) { this.record = new MARC21.Record({marcxml: xml, delimiter: DELIMITER}); this.breakerText = this.record.toBreaker(); this.fixedFieldChange = new EventEmitter(); @@ -113,6 +113,11 @@ export class MarcRecord { return this.record.field(spec, wantArray); } + appendFields(...newFields: MarcField[]) { + this.record.appendFields.apply(this.record, newFields); + this.stampFieldIds(); + } + insertFieldsBefore(field: MarcField, ...newFields: MarcField[]) { this.record.insertFieldsBefore.apply( this.record, [field].concat(newFields)); @@ -186,5 +191,10 @@ export class MarcRecord { subfields.forEach(sf => root.push([].concat(sf))); return root; } + + // Returns a list of values for the tag + subfield combo + subfield(tag: string, subfield: string): string { + return this.record.subfield(tag, subfield); + } } diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Cat.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Cat.pm index 4cfcfbd573..36376a0948 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Cat.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Cat.pm @@ -240,12 +240,17 @@ __PACKAGE__->register_method( @param auth The authtoken @param container The container, um, containing the records to be updated by the template @param template The overlay template, or nothing and the method will look for a negative bib id in the container + @param options Hash of options; currently supports: + xact_per_record: Apply updates to each bib record within its own transaction. @return Cache key to check for status of the container overlay # ); sub template_overlay_container { - my($self, $conn, $auth, $container, $template) = @_; + my($self, $conn, $auth, $container, $template, $options) = @_; + $options ||= {}; + my $xact_per_rec = $options->{xact_per_record}; + my $e = new_editor(authtoken=>$auth, xact=>1); return $e->die_event unless $e->checkauth; @@ -265,14 +270,20 @@ sub template_overlay_container { $template = $e->retrieve_biblio_record_entry( $titem->target_biblio_record_entry )->marc; } + my $num_total = scalar(@$items); my $num_failed = 0; my $num_succeeded = 0; $conn->respond_complete( - $actor->request('open-ils.actor.anon_cache.set_value', $auth, batch_edit_progress => {})->gather(1) + $actor->request('open-ils.actor.anon_cache.set_value', $auth, + batch_edit_progress => {total => $num_total})->gather(1) ) if ($actor); + # read-only up to here. + $e->rollback if $xact_per_rec; + for my $item ( @$items ) { + $e->xact_begin if $xact_per_rec; my $rec = $e->retrieve_biblio_record_entry($item->target_biblio_record_entry); next unless $rec; @@ -294,6 +305,7 @@ sub template_overlay_container { $actor->request( 'open-ils.actor.anon_cache.set_value', $auth, batch_edit_progress => { + total => $num_total, succeeded => $num_succeeded, failed => $num_failed }, @@ -311,6 +323,7 @@ sub template_overlay_container { batch_edit_progress => { complete => 1, success => 'f', + total => $num_total, succeeded => $num_succeeded, failed => $num_failed, } @@ -321,19 +334,23 @@ sub template_overlay_container { } } } + $e->xact_commit if $xact_per_rec; } if ($titem && !$num_failed) { + $e->xact_begin if $xact_per_rec; return $e->die_event unless ($e->delete_container_biblio_record_entry_bucket_item($titem)); + $e->xact_commit if $xact_per_rec; } - if ($e->commit) { + if ($xact_per_rec || $e->commit) { if ($actor) { $actor->request( 'open-ils.actor.anon_cache.set_value', $auth, batch_edit_progress => { complete => 1, success => 't', + total => $num_total, succeeded => $num_succeeded, failed => $num_failed, } @@ -348,6 +365,7 @@ sub template_overlay_container { batch_edit_progress => { complete => 1, success => 'f', + total => $num_total, succeeded => $num_succeeded, failed => $num_failed, } diff --git a/Open-ILS/src/perlmods/lib/OpenILS/WWW/TemplateBatchBibUpdate.pm b/Open-ILS/src/perlmods/lib/OpenILS/WWW/TemplateBatchBibUpdate.pm index 27a03e1107..ae91b36a94 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/WWW/TemplateBatchBibUpdate.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/WWW/TemplateBatchBibUpdate.pm @@ -4,7 +4,7 @@ use warnings; use bytes; use Apache2::Log; -use Apache2::Const -compile => qw(OK REDIRECT DECLINED NOT_FOUND :log); +use Apache2::Const -compile => qw(OK REDIRECT DECLINED NOT_FOUND HTTP_BAD_REQUEST :log); use APR::Const -compile => qw(:error SUCCESS); use APR::Table; @@ -56,14 +56,21 @@ sub handler { my $cgi = new CGI; my $authid = $cgi->cookie('ses') || $cgi->param('ses'); + + # Avoid sending the HTML to the caller. Final response will + # will just be the cache key or HTTP_BAD_REQUEST on error. + my $skipui = $cgi->param('skipui'); + my $usr = verify_login($authid); - return show_template($r) unless ($usr); + return show_template($r, $skipui) unless ($usr); my $template = $cgi->param('template'); - return show_template($r) unless ($template); + return show_template($r, $skipui) unless ($template); my $rsource = $cgi->param('recordSource'); + my $xact_per = $cgi->param('xactPerRecord'); + # find some IDs ... my @records; @@ -118,7 +125,7 @@ sub handler { unless (@records) { $e->request('open-ils.cstore.transaction.rollback')->gather(1); $e->disconnect; - return show_template($r); + return show_template($r, $skipui); } # we have a template and some record ids, so... @@ -173,10 +180,12 @@ sub handler { # fire the background bucket processor my $cache_key = OpenSRF::AppSession ->create('open-ils.cat') - ->request('open-ils.cat.container.template_overlay.background', $authid, $bucket->id) + ->request('open-ils.cat.container.template_overlay.background', + $authid, $bucket->id, undef, {xact_per_record => $xact_per}) ->gather(1); - return show_processing_template($r, $bucket->id, \@records, $cache_key); + return show_processing_template( + $r, $bucket->id, \@records, $cache_key, $skipui); } sub verify_login { @@ -201,6 +210,13 @@ sub show_processing_template { my $bid = shift; my $recs = shift; my $cache_key = shift; + my $skipui = shift; + + if ($skipui) { + $r->content_type('text/plain'); + $r->print($cache_key); + return Apache2::Const::OK; + } my $rec_string = @$recs; @@ -366,6 +382,11 @@ HTML sub show_template { my $r = shift; + my $skipui = shift; + + # Makes no sense to call the API in such a way that the caller + # is returned the UI code if skipui is set. + return Apache2::Const::HTTP_BAD_REQUEST if $skipui; $r->content_type('text/html'); $r->print(<<'HTML'); diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql index 18564980f4..bf1ad50675 100644 --- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql +++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql @@ -1,4 +1,3 @@ ---002.schema.config.sql: INSERT INTO config.bib_source (id, quality, source, transcendant, can_have_copies) VALUES (1, 90, oils_i18n_gettext(1, 'oclc', 'cbs', 'source'), FALSE, TRUE); INSERT INTO config.bib_source (id, quality, source, transcendant, can_have_copies) VALUES diff --git a/Open-ILS/src/templates/staff/navbar.tt2 b/Open-ILS/src/templates/staff/navbar.tt2 index 48b808f414..ac25e566a6 100644 --- a/Open-ILS/src/templates/staff/navbar.tt2 +++ b/Open-ILS/src/templates/staff/navbar.tt2 @@ -321,7 +321,7 @@
  • - + [% l('MARC Batch Edit') %] diff --git a/docs/RELEASE_NOTES_NEXT/Cataloging/marcbatch-ang-port.adoc b/docs/RELEASE_NOTES_NEXT/Cataloging/marcbatch-ang-port.adoc new file mode 100644 index 0000000000..b97b578495 --- /dev/null +++ b/docs/RELEASE_NOTES_NEXT/Cataloging/marcbatch-ang-port.adoc @@ -0,0 +1,3 @@ +MARC Batch Edit UI Angular Port +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The MARC Batch Edit interface has been ported to Angular.