From e49e2c0e45b017527207da9fdfbd5abf592bfdc9 Mon Sep 17 00:00:00 2001 From: dbs Date: Mon, 11 Jan 2010 00:39:25 +0000 Subject: [PATCH] Remove serial ingest references and services The references and services for serial (MFHD) support are not necessary in 1.6, as we are working directly with the MFHD as maintained in serial.record_entry.marc; the services and references being removed never could have been used in the 1.6.0.0 release and should have been removed prior to the 1.6.0.0 release when I removed the corresponding database structures. Also updating the README to reflect the proper loading method for serial records, dropping direct_ingest.pl invocation and noting the limitation on the org_unit identification. git-svn-id: svn://svn.open-ils.org/ILS/trunk@15292 dcc99617-32d9-48b4-a31d-7c20da2025e4 --- Open-ILS/src/extras/import/direct_ingest.pl | 10 +- .../src/perlmods/OpenILS/Application/Ingest.pm | 178 --------------------- Open-ILS/tests/datasets/README | 11 +- 3 files changed, 13 insertions(+), 186 deletions(-) diff --git a/Open-ILS/src/extras/import/direct_ingest.pl b/Open-ILS/src/extras/import/direct_ingest.pl index 6defcd0ebf..1abd30cfa1 100755 --- a/Open-ILS/src/extras/import/direct_ingest.pl +++ b/Open-ILS/src/extras/import/direct_ingest.pl @@ -26,13 +26,12 @@ use MARC::Charset; MARC::Charset->ignore_errors(1); -my ($max_uri, $max_cn, $auth, $mfhd, $config, $quiet) = - (0, 0, 0, 0, '/openils/conf/opensrf_core.xml'); +my ($max_uri, $max_cn, $auth, $config, $quiet) = + (0, 0, 0, '/openils/conf/opensrf_core.xml'); GetOptions( 'config=s' => \$config, 'authority' => \$auth, - 'serial' => \$mfhd, 'quiet' => \$quiet, 'max_uri=i' => \$max_uri, 'max_cn=i' => \$max_cn, @@ -53,7 +52,6 @@ OpenILS::Application::Ingest->use; my $meth = 'open-ils.ingest.full.biblio.object.readonly'; $meth = 'open-ils.ingest.full.authority.object.readonly' if ($auth); -$meth = 'open-ils.ingest.full.serial.object.readonly' if ($mfhd); $meth = OpenILS::Application::Ingest->method_lookup( $meth ); @@ -91,7 +89,7 @@ sub postprocess { my $bib = $data->{bib}; my $full_rec = $data->{ingest_data}->{full_rec}; - if (!$auth && !$mfhd) { + if (!$auth) { $field_entries = $data->{ingest_data}->{field_entries}; $fp = $data->{ingest_data}->{fingerprint}; $rd = $data->{ingest_data}->{descriptor}; @@ -102,7 +100,7 @@ sub postprocess { } print( OpenSRF::Utils::JSON->perl2JSON($bib)."\n" ); - if (!$auth && !$mfhd) { + if (!$auth) { print( OpenSRF::Utils::JSON->perl2JSON($rd)."\n" ); print( OpenSRF::Utils::JSON->perl2JSON($_)."\n" ) for (@$field_entries); for my $u (@$uri) { diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm b/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm index 716cc2dc06..f794479d90 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm @@ -819,168 +819,6 @@ __PACKAGE__->register_method( ); # -------------------------------------------------------------------------------- -# Serial ingest - -package OpenILS::Application::Ingest::Serial; -use base qw/OpenILS::Application::Ingest/; -use Unicode::Normalize; - -sub ro_serial_ingest_single_object { - my $self = shift; - my $client = shift; - my $bib = shift; - my $xml = OpenILS::Application::AppUtils->entityize($bib->marc); - - my $document = $parser->parse_string($xml); - - my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.serial.xml")->run($document); - - $_->record($bib->id) for (@mfr); - - return { full_rec => \@mfr }; -} -__PACKAGE__->register_method( - api_name => "open-ils.ingest.full.serial.object.readonly", - method => "ro_serial_ingest_single_object", - api_level => 1, - argc => 1, -); - -sub ro_serial_ingest_single_xml { - my $self = shift; - my $client = shift; - my $xml = OpenILS::Application::AppUtils->entityize(shift); - - my $document = $parser->parse_string($xml); - - my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.serial.xml")->run($document); - - return { full_rec => \@mfr }; -} -__PACKAGE__->register_method( - api_name => "open-ils.ingest.full.serial.xml.readonly", - method => "ro_serial_ingest_single_xml", - api_level => 1, - argc => 1, -); - -sub ro_serial_ingest_single_record { - my $self = shift; - my $client = shift; - my $rec = shift; - - OpenILS::Application::Ingest->post_init(); - my $r = OpenSRF::AppSession - ->create('open-ils.cstore') - ->request( 'open-ils.cstore.direct.serial.record_entry.retrieve' => $rec ) - ->gather(1); - - return undef unless ($r and @$r); - - my ($res) = $self->method_lookup("open-ils.ingest.full.serial.xml.readonly")->run($r->marc); - - $_->record($rec) for (@{$res->{full_rec}}); - $res->{descriptor}->record($rec); - - return $res; -} -__PACKAGE__->register_method( - api_name => "open-ils.ingest.full.serial.record.readonly", - method => "ro_serial_ingest_single_record", - api_level => 1, - argc => 1, -); - -sub ro_serial_ingest_stream_record { - my $self = shift; - my $client = shift; - - OpenILS::Application::Ingest->post_init(); - - my $ses = OpenSRF::AppSession->create('open-ils.cstore'); - - while (my ($resp) = $client->recv( count => 1, timeout => 5 )) { - - my $rec = $resp->content; - last unless (defined $rec); - - $log->debug("Running open-ils.ingest.full.serial.record.readonly ..."); - my ($res) = $self->method_lookup("open-ils.ingest.full.serial.record.readonly")->run($rec); - - $_->record($rec) for (@{$res->{full_rec}}); - - $client->respond( $res ); - } - - return undef; -} -__PACKAGE__->register_method( - api_name => "open-ils.ingest.full.serial.record_stream.readonly", - method => "ro_serial_ingest_stream_record", - api_level => 1, - stream => 1, -); - -sub ro_serial_ingest_stream_xml { - my $self = shift; - my $client = shift; - - OpenILS::Application::Ingest->post_init(); - - my $ses = OpenSRF::AppSession->create('open-ils.cstore'); - - while (my ($resp) = $client->recv( count => 1, timeout => 5 )) { - - my $xml = $resp->content; - last unless (defined $xml); - - $log->debug("Running open-ils.ingest.full.serial.xml.readonly ..."); - my ($res) = $self->method_lookup("open-ils.ingest.full.serial.xml.readonly")->run($xml); - - $client->respond( $res ); - } - - return undef; -} -__PACKAGE__->register_method( - api_name => "open-ils.ingest.full.serial.xml_stream.readonly", - method => "ro_serial_ingest_stream_xml", - api_level => 1, - stream => 1, -); - -sub rw_serial_ingest_stream_import { - my $self = shift; - my $client = shift; - - OpenILS::Application::Ingest->post_init(); - - my $ses = OpenSRF::AppSession->create('open-ils.cstore'); - - while (my ($resp) = $client->recv( count => 1, timeout => 5 )) { - - my $bib = $resp->content; - last unless (defined $bib); - - $log->debug("Running open-ils.ingest.full.serial.xml.readonly ..."); - my ($res) = $self->method_lookup("open-ils.ingest.full.serial.xml.readonly")->run($bib->marc); - - $_->record($bib->id) for (@{$res->{full_rec}}); - - $client->respond( $res ); - } - - return undef; -} -__PACKAGE__->register_method( - api_name => "open-ils.ingest.full.serial.bib_stream.import", - method => "rw_serial_ingest_stream_import", - api_level => 1, - stream => 1, -); - - -# -------------------------------------------------------------------------------- # MARC index extraction package OpenILS::Application::Ingest::XPATH; @@ -1270,7 +1108,6 @@ sub flat_marc_xml { my $type = 'metabib'; $type = 'authority' if ($self->api_name =~ /authority/o); - $type = 'serial' if ($self->api_name =~ /serial/o); OpenILS::Application::Ingest->post_init(); @@ -1291,13 +1128,6 @@ __PACKAGE__->register_method( argc => 1, stream => 1, ); -__PACKAGE__->register_method( - api_name => "open-ils.ingest.flat_marc.serial.xml", - method => "flat_marc_xml", - api_level => 1, - argc => 1, - stream => 1, -); sub flat_marc_record { my $self = shift; @@ -1306,7 +1136,6 @@ sub flat_marc_record { my $type = 'biblio'; $type = 'authority' if ($self->api_name =~ /authority/o); - $type = 'serial' if ($self->api_name =~ /serial/o); OpenILS::Application::Ingest->post_init(); my $r = OpenSRF::AppSession @@ -1338,13 +1167,6 @@ __PACKAGE__->register_method( argc => 1, stream => 1, ); -__PACKAGE__->register_method( - api_name => "open-ils.ingest.flat_marc.serial.record_entry", - method => "flat_marc_record", - api_level => 1, - argc => 1, - stream => 1, -); # -------------------------------------------------------------------------------- # URI extraction diff --git a/Open-ILS/tests/datasets/README b/Open-ILS/tests/datasets/README index 7140ca62ee..a824a09738 100644 --- a/Open-ILS/tests/datasets/README +++ b/Open-ILS/tests/datasets/README @@ -21,7 +21,14 @@ The following table lists the data sets we have collected for testing purposes. - First load the MARC21 records: - PERL5LIB=/openils/lib/perl5/ perl ../../src/extras/import/marc2bre.pl --marctype XML --start 1 --idfield 901 --idsubfield a serials_marc21.xml | perl ../../src/extras/import/direct_ingest.pl | perl ../../src/extras/import/pg_loader.pl -or bre -or mrd -or mfr -or mtfe -or mafe -or msfe -or mkfe -or msefe -a mrd -a mfr -a mtfe -a mafe -a msfe -a mkfe -a msefe | psql -U evergreen -h localhost - - Then load the MFHD records - - PERL5LIB=/openils/lib/perl5/ perl ../../src/extras/import/marc2sre.pl --marctype XML --password demo123 serials_mfhd.xml | perl ../../src/extras/import/direct_ingest.pl --serial | perl ../../src/extras/import/pg_loader.pl -or sre > mfhd21.sql + - Then generate the metarecord map: + - pgsql -f ../../src/extras/import/quick_metarecord_map.sql + + - Then process and load the MFHD records - ingest is not used: + - PERL5LIB=/openils/lib/perl5/ perl ../../src/extras/import/marc2sre.pl --marctype XML --password open-ils serials_mfhd.xml | perl ../../src/extras/import/pg_loader.pl -or sre > mfhd21.sql + - Open the file in a text editor and change the final "\N" on each line + to the numeric ID of the org_unit that owns the MFHD record. Yes, this + is a gross hack that effectively limits you to loading MFHD records in + per-org_unit batches; enhancements to marc2sre.pl are welcomed. - psql -f mfhd21.sql -- 2.11.0