From 74462d50d0d8d2b12b081d5e1a304be41d5bd336 Mon Sep 17 00:00:00 2001 From: Bill Erickson Date: Wed, 2 Dec 2015 17:16:50 -0500 Subject: [PATCH] JBAS-980 bib/auth batch repairs / consistency 1. Use consistent DB connection logic -- removes a lot of unnecessary DB connection settings logic. 2. Repair logic on building batch files to avoid warnings about writing to closed file handles. Signed-off-by: Bill Erickson --- KCLS/linking/authority_control_fields_batcher.pl | 37 ++------ ...uthority_control_fields_batcher_all_exported.pl | 102 ++++++++------------- 2 files changed, 48 insertions(+), 91 deletions(-) diff --git a/KCLS/linking/authority_control_fields_batcher.pl b/KCLS/linking/authority_control_fields_batcher.pl index 4171bfa1b6..a24fe2f628 100755 --- a/KCLS/linking/authority_control_fields_batcher.pl +++ b/KCLS/linking/authority_control_fields_batcher.pl @@ -47,41 +47,24 @@ use strict; use warnings; use DBI; -use Backstage::JSONPrefs; use Getopt::Long; my $batch_size = 5000; my $lower_bound = 0; -my $hostname = 'evergreentest.catalystitservices.com'; -my $portnum = '5415'; +my $db_host = 'localhost'; +my $db_port = '5432'; +my $db_user = 'evergreen'; +my $db_pass = 'evergreen'; my $result = GetOptions("lower-bound=i" => \$lower_bound, "batch-size=i" => \$batch_size, - "hostname=s" => \$hostname, - "portnum=s" => \$portnum); + "db-host=s" => \$db_host, + "db-user=s" => \$db_user, + "db-pass=s" => \$db_pass, + "db-port=s" => \$db_port); -#my $egdbi = JSONPrefs->load($ENV{'HOME'} . "/myprefs.d/egdbi.json"); - -#my $dsn = "dbi:Pg:database=" . $egdbi->database; - -#my $dsn = "dbi:Pg:database=rel_2_4_1_20130821_auth_v3"; -my $dsn = "dbi:Pg:database=evergreen"; - -#if ($egdbi->host) { -# $dsn .= ";host=" . $egdbi->host; -#} - -$dsn .= ";host=$hostname"; - -#if ($egdbi->port) { -# $dsn .= ";port=" . $egdbi->port; -#} - -$dsn .= ";port=$portnum"; - -#my $dbh = DBI->connect($dsn,$egdbi->user,$egdbi->password); - -my $dbh = DBI->connect($dsn,"evergreen","evergreen"); +my $dsn = "dbi:Pg:database=evergreen;host=$db_host;port=$db_port"; +my $dbh = DBI->connect($dsn, $db_user, $db_pass); my $q = < \$batch_size, + "db-host=s" => \$db_host, + "db-user=s" => \$db_user, + "db-pass=s" => \$db_pass, + "db-port=s" => \$db_port ); -if(!$batch_size) { - $batch_size = 500; -} +my $dsn = "dbi:Pg:database=evergreen;host=$db_host;port=$db_port"; +my $dbh = DBI->connect($dsn, $db_user, $db_pass); -my $config = '/openils/conf/opensrf_core.xml'; -OpenSRF::System->bootstrap_client( config_file => $config ); - -my $query = " - SELECT bib - FROM metabib.bib_export_data"; - -my $dbh = connect_to_db(); $dbh->do('SET statement_timeout = 0;'); - -print "Querying database for IDs .. \n"; -my $sth = $dbh->prepare($query); +my $sth = $dbh->prepare('SELECT bib FROM metabib.bib_export_data'); $sth->execute(); + print "Found " . $sth->rows() . " IDs\n"; my @ids; @@ -39,54 +35,32 @@ while (my @row = $sth->fetchrow_array()) { } $sth->finish(); $dbh->disconnect(); -system("mkdir -p /var/KCLS_AUTH/bibs_to_link"); -my $batch_count = 0; -my $count = 0; +system("mkdir -p $base_path/bibs_to_link"); -open BATCH_FILE, ">/var/KCLS_AUTH/bib_link_batches"; -foreach(@ids) { - $count++; - if($count == 1) { - open OUTPUT, ">/var/KCLS_AUTH/bibs_to_link/bibs_to_link_" . $batch_count; - print "Writing file bibs_to_link_" . $batch_count . "\n"; - } - print OUTPUT "$_\n"; - if($count == $batch_size) { - close OUTPUT; - print BATCH_FILE "./authority_control_fields.pl --file /var/KCLS_AUTH/bibs_to_link/bibs_to_link_" . $batch_count ." --refresh\n"; - $batch_count++; - $count = 0; - } -} -if($count) { - close OUTPUT; - print BATCH_FILE "./authority_control_fields.pl --file /var/KCLS_AUTH/bibs_to_link/bibs_to_link_" . $batch_count ." --refresh\n"; -} -close BATCH_FILE; +open BATCH_FILE, ">$base_path/bib_link_batches"; -# Grab DB information from local settings. Return connected db handle (or die) -sub connect_to_db { - my $sc = OpenSRF::Utils::SettingsClient->new; - my $db_driver = $sc->config_value( apps => 'open-ils.storage' => app_settings => databases => 'driver' ); - my $db_host = $sc->config_value( apps => 'open-ils.storage' => app_settings => databases => database => 'host' ); - my $db_port = $sc->config_value( apps => 'open-ils.storage' => app_settings => databases => database => 'port' ); - my $db_name = $sc->config_value( apps => 'open-ils.storage' => app_settings => databases => database => 'db' ); - - if (!$db_name) { - $db_name = $sc->config_value( reporter => setup => database => 'name' ); - print STDERR "WARN: is a deprecated setting for database name. For future compatibility, you should use instead." if $db_name; - } - my $db_user = $sc->config_value( apps => 'open-ils.storage' => app_settings => databases => database => 'user' ); - my $db_pw = $sc->config_value( apps => 'open-ils.storage' => app_settings => databases => database => 'pw' ); - - die "Unable to retrieve database connection information from the settings server" unless ($db_driver && $db_host && $db_port && $db_name && $db_user); +my $batch_count = 0; +while (@ids) { - my $dsn = "dbi:" . $db_driver . ":dbname=" . $db_name .';host=' . $db_host . ';port=' . $db_port; + my $batch_file = + "$base_path/bibs_to_link/bibs_to_link_$batch_count"; + open(OUTPUT, '>', $batch_file) or die "$! $batch_file\n"; + print "Writing file bibs_to_link_$batch_count\n"; - return DBI->connect( - $dsn,$db_user,$db_pw, { - AutoCommit => 1, pg_enable_utf8 => 1, RaiseError => 1 + my $count = 0; + while (my $id = shift @ids) { + print OUTPUT "$id\n"; + if (++$count == $batch_size) { + $batch_count++; + last; } - ); # shouldn't need 'or die...' with RaiseError=>1 + } + + close OUTPUT; + print BATCH_FILE "./authority_control_fields.pl --file ". + "$base_path/bibs_to_link/bibs_to_link_$batch_count --refresh\n"; } + +close BATCH_FILE; + -- 2.11.0