pipline fixups; integer qualities only, please

git-svn-id: svn://svn.open-ils.org/ILS/trunk@5030 dcc99617-32d9-48b4-a31d-7c20da2025e4

diff --git a/Open-ILS/src/extras/import/direct_ingest.pl b/Open-ILS/src/extras/import/direct_ingest.pl
index 81fceca..dff72ab 100755 (executable)
--- a/Open-ILS/src/extras/import/direct_ingest.pl
+++ b/Open-ILS/src/extras/import/direct_ingest.pl
@@ -118,6 +118,7 @@ while ( my $rec = <> ) {
        $count++;
 }
 
+$ses[$_]->close for (@ses);
 sub postprocess {
        my $data = shift;
        my $f = shift;

diff --git a/Open-ILS/src/extras/import/marc2bre.pl b/Open-ILS/src/extras/import/marc2bre.pl
index 53b2323..1e84ab1 100755 (executable)
--- a/Open-ILS/src/extras/import/marc2bre.pl
+++ b/Open-ILS/src/extras/import/marc2bre.pl
@@ -26,8 +26,8 @@ use UNIVERSAL::require;
 
 MARC::Charset->ignore_errors(1);
 
-my ($id_field, $count, $user, $password, $config, $keyfile,  @files, @trash_fields) =
-       ('998', 1, 'admin', 'open-ils', '/openils/conf/bootstrap.conf');
+my ($utf8, $id_field, $count, $user, $password, $config, $keyfile,  @files, @trash_fields) =
+       (0, '998', 1, 'admin', 'open-ils', '/openils/conf/bootstrap.conf');
 
 GetOptions(
        'startid=i'     => \$count,
@@ -49,7 +49,7 @@ my %processing_cache;
 my %source_map = (      
        o  => 'OCLC',
        i  => 'ISxN',    
-       l  => 'Local',
+       l  => 'LCCN',
        s  => 'System',  
        g  => 'Gutenberg',  
 );                              
@@ -78,23 +78,26 @@ $batch->strict_off();
 $batch->warnings_off();
 
 my $starttime = time;
-while ( my $rec = $batch->next ) {
-
+my $rec;
+while ( try { $rec = $batch->next } otherwise { $rec = -1 } ) {
+       next if ($rec == -1);
        my $id;
-       my $field = $rec->field($id_field);
 
-       if ($field) {
-               if ($field->is_control_field) {
-                       $id = $field->data;
-               } else {
-                       $id = $field->subfield('a');
+       if ($id_field) {
+               my $field = $rec->field($id_field);
+               if ($field) {
+                       if ($field->is_control_field) {
+                               $id = $field->data;
+                       } else {
+                               $id = $field->subfield('a');
+                       }
                }
-       } else {
-               $id = $count;
        }
                
-       if ($id =~ /(\d+)/o) {
+       if ($id && $id =~ /(\d+)/o) {
                $id = $1;
+       } else {
+               $id = $count;
        }
 
        if ($keyfile) {
@@ -149,6 +152,11 @@ sub preprocess {
 
        my ($id, $source, $value);
 
+        if (!$id) {
+                my $f = $rec->field($id_field);
+                $id = $f->subfield('a') if ($f);
+        }
+
        if (!$id) {
                my $f = $rec->field('001');
                $id = $f->data if ($f);
@@ -156,8 +164,8 @@ sub preprocess {
 
        if (!$id) {
                my $f = $rec->field('000');
-               $id = 'g'.$f->data if ($f);
-               $source = 'g';
+               $id = $f->data if ($f);
+               $source = 'g'; # only PG seems to use this
        }
 
         if (!$id) {
@@ -178,12 +186,6 @@ sub preprocess {
                $source = 'l';
         }
 
-        if (!$id) {
-                my $f = $rec->field($id_field);
-                $id = $f->subfield('a') if ($f);
-               $source = 's';
-        }
-
        if (!$id) {
                $count++;
                warn "\n !!! Record with no TCN : $count\n".$rec->as_formatted;
@@ -194,7 +196,7 @@ sub preprocess {
 
        $id =~ s/\s*$//o;
        $id =~ s/^\s*//o;
-       $id =~ s/(\S+)$/$1/o;
+       $id =~ s/^(\S+).*$/$1/o;
 
        $id = $source.$id if ($source);
 

diff --git a/Open-ILS/src/extras/import/pg_loader.pl b/Open-ILS/src/extras/import/pg_loader.pl
index 86a9f1a..a1f8552 100755 (executable)
--- a/Open-ILS/src/extras/import/pg_loader.pl
+++ b/Open-ILS/src/extras/import/pg_loader.pl
@@ -84,14 +84,22 @@ for my $h (@order) {
 
        for my $line (@{ $lineset{$h} }) {
                my @data;
+               my $x = 0;
                for my $d (@$line) {
                        if (!defined($d)) {
                                $d = '\N';
                        } else {
-                               $d =~ s/\t/\\t/go;
-                               $d =~ s/\\/\\\\/go;
+                               $d =~ s/\f/\\f/gos;
+                               $d =~ s/\n/\\n/gos;
+                               $d =~ s/\r/\\r/gos;
+                               $d =~ s/\t/\\t/gos;
+                               $d =~ s/\\/\\\\/gos;
+                       }
+                       if ($h eq 'bre' and $fieldcache{$h}{fields}[$x] eq 'quality') {
+                               $d = int($d);
                        }
                        push @data, $d;
+                       $x++;
                }
                $output->print( join("\t", @data)."\n" );
        }

diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm b/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm
index 4e19dfa..fe215b4 100644 (file)
--- a/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm
+++ b/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm
@@ -779,6 +779,7 @@ sub biblio_fingerprint_record {
 
        my ($fp) = $self->method_lookup('open-ils.ingest.fingerprint.xml')->run($r->marc);
        $log->debug("Returning [$fp] as fingerprint for record $rec", INFO);
+       $fp->{quality} = int($fp->{quality});
        return $fp;
 }
 __PACKAGE__->register_method(