JBAS-1437 B2A linking additional query options

* --all uses direct DB connection
  * Fetching all record ID's via cstore is a bad idea.
* order by ID
* better logging
* --sort-desc option

Signed-off-by: Bill Erickson <berickxx@gmail.com>

diff --git a/KCLS/linking/authority_control_fields.pl b/KCLS/linking/authority_control_fields.pl
index ec7a771..91d9d7e 100755 (executable)
--- a/KCLS/linking/authority_control_fields.pl
+++ b/KCLS/linking/authority_control_fields.pl
@@ -23,7 +23,6 @@ use MARC::Charset;
 use OpenSRF::System;
 use OpenILS::Utils::Fieldmapper;
 use OpenSRF::Utils::SettingsClient;
-use OpenSRF::EX qw/:try/;
 use Encode;
 use Unicode::Normalize;
 use OpenILS::Application::AppUtils;
@@ -45,6 +44,7 @@ my $input_file ='';
 my $bootstrap = '/openils/conf/opensrf_core.xml';
 my @records;
 my $verbose = 0;
+my $sort_desc = 0;
 my $db_host = $ENV{PGHOST} || 'localhost';
 my $db_port = $ENV{PGPORT} || '5432';
 my $db_user = $ENV{PGDATABASE} || 'evergreen';
@@ -61,13 +61,14 @@ my $result = GetOptions(
     'end-id=i' => \$end_id,
     'days-back=i' => \$days_back,
     'modified-since=i' => \$modified_since,
+    'sort-desc' => \$sort_desc,
     'file=s' => \$input_file,
     'verbose' => \$verbose,
     "db-host=s" => \$db_host,
     "db-user=s" => \$db_user,
     "db-pass=s" => \$db_pass,
     "db-port=s" => \$db_port,
-    'all', 
+    'all', # now assumed to be true when --file is unset
     'help'
 );
 
@@ -81,11 +82,6 @@ if (!$result or $options{help}) {
     pod2usage(0);
 }
 
-if ($start_id && $days_back) {
-    print "Can't use both start ID and days back!\n";
-    exit;
-}
-
 OpenSRF::System->bootstrap_client(config_file => $bootstrap);
 Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
 
@@ -95,53 +91,49 @@ OpenILS::Utils::CStoreEditor::init();
 
 my $e = OpenILS::Utils::CStoreEditor->new;
 
-if ($options{all}) { # all non-deleted records from Evergreen
-
-    @records = @{
-        $e->search_biblio_record_entry(
-            {deleted => 'f', id => {'>' => 0}}, 
-            {idlist => 1, substream => 1}
-        ) 
-    };
+if ($input_file) {
+       open FILE, "<", $input_file or die "Can't open file " . $input_file;
+       while(<FILE>) {
+               chomp;
+               if($_) {
+                       push(@records, $_);
+               }
+       }
+       close FILE;
 
-} elsif ($start_id and $end_id) {
+} else {
 
-    @records = ($start_id .. $end_id);
 
-} elsif (defined $days_back or defined $modified_since) {
+    my $where = "WHERE not deleted";
+    $where .= " AND id >= $start_id" if $start_id;
+    $where .= " AND id <= $end_id" if $end_id;
 
-    @records = ();
+    my $order = "ORDER BY id";
+    $order .= " DESC" if $sort_desc;
 
-    my $dsn = "dbi:Pg:database=evergreen;host=$db_host;port=$db_port";
-    my $dbh = DBI->connect($dsn, $db_user, $db_pass);
-    my $date_field = defined $days_back ? 'create_date' : 'edit_date';
-    my $duration = defined $days_back ? $days_back : $modified_since;
+    if (defined $days_back or defined $modified_since) {
+        my $date_field = defined $days_back ? 'create_date' : 'edit_date';
+        my $duration = defined $days_back ? $days_back : $modified_since;
+        $where .= " AND DATE($date_field) >= DATE(NOW() - '$duration day'::INTERVAL)";
+    }
 
-    my $sql = <<"    SQL";
-        SELECT DISTINCT(id) AS id FROM biblio.record_entry
-        WHERE DATE($date_field) >= DATE(NOW() - '$duration day'::INTERVAL)
-    SQL
+    my $sql = 
+        "SELECT DISTINCT(id) AS id FROM biblio.record_entry $where $order";
 
     announce("Loading record ID's with query:\n$sql");
 
-    my $idstatement = $dbh->prepare($sql);
+    my $dsn = "dbi:Pg:database=evergreen;host=$db_host;port=$db_port";
+    my $dbh = DBI->connect($dsn, $db_user, $db_pass);
 
-    # Load the list of ID's into the records array
-    $idstatement->execute();
+    my $sth = $dbh->prepare($sql);
+    $sth->execute;
 
-    while (my $ref = $idstatement->fetchrow_hashref()) {
+    while (my $ref = $sth->fetchrow_hashref()) {
         push(@records, $ref->{id});
     }
 
-} elsif ($input_file) {
-       open FILE, "<", $input_file or die "Can't open file " . $input_file;
-       while(<FILE>) {
-               chomp;
-               if($_) {
-                       push(@records, $_);
-               }
-       }
-       close FILE;
+    $sth->finish();
+    $dbh->disconnect();
 }
 
 announce("Processing ".scalar(@records)." records");
@@ -716,8 +708,11 @@ sub update_record {
     }
 }
 
+my $count = 0;
+my $total = scalar(@records);
 foreach my $rec_id (@records) {
-    announce("processing bib record $rec_id");
+    $count++;
+    announce("processing bib record $rec_id [$count of $total]");
 
     # State variable; was the record changed?
     my $changed = 0;
@@ -726,7 +721,7 @@ foreach my $rec_id (@records) {
     my $record = $e->retrieve_biblio_record_entry($rec_id);
     next unless $record && $record->deleted eq 'f';
 
-    try {
+    eval {
         my $marc = MARC::Record->new_from_xml($record->marc());
 
         # get the list of controlled fields
@@ -866,9 +861,10 @@ foreach my $rec_id (@records) {
 
         update_record($record, $marc) if $changed;
 
-    } otherwise {
-        my $err = shift;
-        print STDERR "\nRecord # $rec_id : $err\n";
+    };
+
+    if ($@) {
+        print STDERR "\nRecord # $rec_id : $@\n";
         import MARC::File::XML; # reset SAX parser so that one bad record doesn't kill the entire process
     }
 }