Remove serial ingest references and services

The references and services for serial (MFHD) support are not
necessary in 1.6, as we are working directly with the MFHD as
maintained in serial.record_entry.marc; the services and
references being removed never could have been used in the
1.6.0.0 release and should have been removed prior to the 1.6.0.0
release when I removed the corresponding database structures.

Also updating the README to reflect the proper loading method for
serial records, dropping direct_ingest.pl invocation and noting the
limitation on the org_unit identification.

git-svn-id: svn://svn.open-ils.org/ILS/trunk@15292 dcc99617-32d9-48b4-a31d-7c20da2025e4

diff --git a/Open-ILS/src/extras/import/direct_ingest.pl b/Open-ILS/src/extras/import/direct_ingest.pl
index 6defcd0..1abd30c 100755 (executable)
--- a/Open-ILS/src/extras/import/direct_ingest.pl
+++ b/Open-ILS/src/extras/import/direct_ingest.pl
@@ -26,13 +26,12 @@ use MARC::Charset;
 
 MARC::Charset->ignore_errors(1);
 
-my ($max_uri, $max_cn, $auth, $mfhd, $config, $quiet) =
-       (0, 0, 0, 0, '/openils/conf/opensrf_core.xml');
+my ($max_uri, $max_cn, $auth, $config, $quiet) =
+       (0, 0, 0, '/openils/conf/opensrf_core.xml');
 
 GetOptions(
        'config=s'      => \$config,
        'authority'     => \$auth,
-       'serial'        => \$mfhd,
        'quiet'         => \$quiet,
        'max_uri=i'     => \$max_uri,   
        'max_cn=i'      => \$max_cn,    
@@ -53,7 +52,6 @@ OpenILS::Application::Ingest->use;
 
 my $meth = 'open-ils.ingest.full.biblio.object.readonly';
 $meth = 'open-ils.ingest.full.authority.object.readonly' if ($auth);
-$meth = 'open-ils.ingest.full.serial.object.readonly' if ($mfhd);
 
 $meth = OpenILS::Application::Ingest->method_lookup( $meth );
 
@@ -91,7 +89,7 @@ sub postprocess {
        my $bib = $data->{bib};
        my $full_rec = $data->{ingest_data}->{full_rec};
 
-       if (!$auth && !$mfhd) {
+       if (!$auth) {
                $field_entries = $data->{ingest_data}->{field_entries};
                $fp = $data->{ingest_data}->{fingerprint};
                $rd = $data->{ingest_data}->{descriptor};
@@ -102,7 +100,7 @@ sub postprocess {
        }
 
        print( OpenSRF::Utils::JSON->perl2JSON($bib)."\n" );
-       if (!$auth && !$mfhd) {
+       if (!$auth) {
                print( OpenSRF::Utils::JSON->perl2JSON($rd)."\n" );
                print( OpenSRF::Utils::JSON->perl2JSON($_)."\n" ) for (@$field_entries);
                for my $u (@$uri) {

diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm b/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm
index 716cc2d..f794479 100644 (file)
--- a/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm
+++ b/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm
@@ -819,168 +819,6 @@ __PACKAGE__->register_method(
 );                      
 
 # --------------------------------------------------------------------------------
-# Serial ingest
-
-package OpenILS::Application::Ingest::Serial;
-use base qw/OpenILS::Application::Ingest/;
-use Unicode::Normalize;
-
-sub ro_serial_ingest_single_object {
-    my $self = shift;
-    my $client = shift;
-    my $bib = shift;
-    my $xml = OpenILS::Application::AppUtils->entityize($bib->marc);
-
-    my $document = $parser->parse_string($xml);
-
-    my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.serial.xml")->run($document);
-
-    $_->record($bib->id) for (@mfr);
-
-    return { full_rec => \@mfr };
-}
-__PACKAGE__->register_method(  
-    api_name    => "open-ils.ingest.full.serial.object.readonly",
-    method        => "ro_serial_ingest_single_object",
-    api_level    => 1,
-    argc        => 1,
-);                      
-
-sub ro_serial_ingest_single_xml {
-    my $self = shift;
-    my $client = shift;
-    my $xml = OpenILS::Application::AppUtils->entityize(shift);
-
-    my $document = $parser->parse_string($xml);
-
-    my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.serial.xml")->run($document);
-
-    return { full_rec => \@mfr };
-}
-__PACKAGE__->register_method(  
-    api_name    => "open-ils.ingest.full.serial.xml.readonly",
-    method        => "ro_serial_ingest_single_xml",
-    api_level    => 1,
-    argc        => 1,
-);                      
-
-sub ro_serial_ingest_single_record {
-    my $self = shift;
-    my $client = shift;
-    my $rec = shift;
-
-    OpenILS::Application::Ingest->post_init();
-    my $r = OpenSRF::AppSession
-            ->create('open-ils.cstore')
-            ->request( 'open-ils.cstore.direct.serial.record_entry.retrieve' => $rec )
-            ->gather(1);
-
-    return undef unless ($r and @$r);
-
-    my ($res) = $self->method_lookup("open-ils.ingest.full.serial.xml.readonly")->run($r->marc);
-
-    $_->record($rec) for (@{$res->{full_rec}});
-    $res->{descriptor}->record($rec);
-
-    return $res;
-}
-__PACKAGE__->register_method(  
-    api_name    => "open-ils.ingest.full.serial.record.readonly",
-    method        => "ro_serial_ingest_single_record",
-    api_level    => 1,
-    argc        => 1,
-);                      
-
-sub ro_serial_ingest_stream_record {
-    my $self = shift;
-    my $client = shift;
-
-    OpenILS::Application::Ingest->post_init();
-
-    my $ses = OpenSRF::AppSession->create('open-ils.cstore');
-
-    while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
-    
-        my $rec = $resp->content;
-        last unless (defined $rec);
-
-        $log->debug("Running open-ils.ingest.full.serial.record.readonly ...");
-        my ($res) = $self->method_lookup("open-ils.ingest.full.serial.record.readonly")->run($rec);
-
-        $_->record($rec) for (@{$res->{full_rec}});
-
-        $client->respond( $res );
-    }
-
-    return undef;
-}
-__PACKAGE__->register_method(  
-    api_name    => "open-ils.ingest.full.serial.record_stream.readonly",
-    method        => "ro_serial_ingest_stream_record",
-    api_level    => 1,
-    stream        => 1,
-);                      
-
-sub ro_serial_ingest_stream_xml {
-    my $self = shift;
-    my $client = shift;
-
-    OpenILS::Application::Ingest->post_init();
-
-    my $ses = OpenSRF::AppSession->create('open-ils.cstore');
-
-    while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
-    
-        my $xml = $resp->content;
-        last unless (defined $xml);
-
-        $log->debug("Running open-ils.ingest.full.serial.xml.readonly ...");
-        my ($res) = $self->method_lookup("open-ils.ingest.full.serial.xml.readonly")->run($xml);
-
-        $client->respond( $res );
-    }
-
-    return undef;
-}
-__PACKAGE__->register_method(  
-    api_name    => "open-ils.ingest.full.serial.xml_stream.readonly",
-    method        => "ro_serial_ingest_stream_xml",
-    api_level    => 1,
-    stream        => 1,
-);                      
-
-sub rw_serial_ingest_stream_import {
-    my $self = shift;
-    my $client = shift;
-
-    OpenILS::Application::Ingest->post_init();
-
-    my $ses = OpenSRF::AppSession->create('open-ils.cstore');
-
-    while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
-    
-        my $bib = $resp->content;
-        last unless (defined $bib);
-
-        $log->debug("Running open-ils.ingest.full.serial.xml.readonly ...");
-        my ($res) = $self->method_lookup("open-ils.ingest.full.serial.xml.readonly")->run($bib->marc);
-
-        $_->record($bib->id) for (@{$res->{full_rec}});
-
-        $client->respond( $res );
-    }
-
-    return undef;
-}
-__PACKAGE__->register_method(  
-    api_name    => "open-ils.ingest.full.serial.bib_stream.import",
-    method        => "rw_serial_ingest_stream_import",
-    api_level    => 1,
-    stream        => 1,
-);                      
-
-
-# --------------------------------------------------------------------------------
 # MARC index extraction
 
 package OpenILS::Application::Ingest::XPATH;
@@ -1270,7 +1108,6 @@ sub flat_marc_xml {
 
     my $type = 'metabib';
     $type = 'authority' if ($self->api_name =~ /authority/o);
-    $type = 'serial' if ($self->api_name =~ /serial/o);
 
     OpenILS::Application::Ingest->post_init();
 
@@ -1291,13 +1128,6 @@ __PACKAGE__->register_method(
     argc        => 1,
     stream        => 1,
 );                      
-__PACKAGE__->register_method(  
-    api_name    => "open-ils.ingest.flat_marc.serial.xml",
-    method        => "flat_marc_xml",
-    api_level    => 1,
-    argc        => 1,
-    stream        => 1,
-);                      
 
 sub flat_marc_record {
     my $self = shift;
@@ -1306,7 +1136,6 @@ sub flat_marc_record {
 
     my $type = 'biblio';
     $type = 'authority' if ($self->api_name =~ /authority/o);
-    $type = 'serial' if ($self->api_name =~ /serial/o);
 
     OpenILS::Application::Ingest->post_init();
     my $r = OpenSRF::AppSession
@@ -1338,13 +1167,6 @@ __PACKAGE__->register_method(
     argc        => 1,
     stream        => 1,
 );                      
-__PACKAGE__->register_method(  
-    api_name    => "open-ils.ingest.flat_marc.serial.record_entry",
-    method        => "flat_marc_record",
-    api_level    => 1,
-    argc        => 1,
-    stream        => 1,
-);                      
 
 # --------------------------------------------------------------------------------
 # URI extraction

diff --git a/Open-ILS/tests/datasets/README b/Open-ILS/tests/datasets/README
index 7140ca6..a824a09 100644 (file)
--- a/Open-ILS/tests/datasets/README
+++ b/Open-ILS/tests/datasets/README
@@ -21,7 +21,14 @@ The following table lists the data sets we have collected for testing purposes.
   - First load the MARC21 records:
     - PERL5LIB=/openils/lib/perl5/ perl ../../src/extras/import/marc2bre.pl --marctype XML --start 1 --idfield 901 --idsubfield a serials_marc21.xml | perl ../../src/extras/import/direct_ingest.pl  | perl ../../src/extras/import/pg_loader.pl -or bre -or mrd -or mfr -or mtfe -or mafe -or msfe -or mkfe -or msefe -a mrd -a mfr -a mtfe -a mafe -a msfe -a mkfe -a msefe | psql -U evergreen -h localhost
 
-  - Then load the MFHD records
-    - PERL5LIB=/openils/lib/perl5/ perl ../../src/extras/import/marc2sre.pl --marctype XML --password demo123 serials_mfhd.xml | perl ../../src/extras/import/direct_ingest.pl --serial | perl ../../src/extras/import/pg_loader.pl -or sre > mfhd21.sql
+  - Then generate the metarecord map:
+    - pgsql -f ../../src/extras/import/quick_metarecord_map.sql
+
+  - Then process and load the MFHD records - ingest is not used:
+    - PERL5LIB=/openils/lib/perl5/ perl ../../src/extras/import/marc2sre.pl --marctype XML --password open-ils serials_mfhd.xml | perl ../../src/extras/import/pg_loader.pl -or sre > mfhd21.sql
+    - Open the file in a text editor and change the final "\N" on each line
+      to the numeric ID of the org_unit that owns the MFHD record. Yes, this
+      is a gross hack that effectively limits you to loading MFHD records in
+      per-org_unit batches; enhancements to marc2sre.pl are welcomed.
     - psql -f mfhd21.sql