JBAS-1437 Backstage importer db handle resets

Periodically close and reopen the DB connection to avoid using any one
connection for extended periods of time, which can be bad when SQL
queries have memory leaks, as with bib record updates.

Signed-off-by: Bill Erickson <berickxx@gmail.com>

diff --git a/KCLS/authority-control/backstage/process-backstage-files.pl b/KCLS/authority-control/backstage/process-backstage-files.pl
index e402b2c..8f571f3 100755 (executable)
--- a/KCLS/authority-control/backstage/process-backstage-files.pl
+++ b/KCLS/authority-control/backstage/process-backstage-files.pl
@@ -20,6 +20,9 @@ use OpenILS::Utils::Normalize qw(clean_marc);
 binmode(STDOUT, ':utf8');
 
 my $db_handle;
+my $db_handle_ops = 0;
+# Reset the DB handle have this many operations to avoid memory leaks.
+my $db_handle_reset = 500;
 my $log_mod = 500;  # log every 500th of each type of event (see verbose)
 
 my $marc_file;
@@ -137,6 +140,33 @@ sub announce {
     }
 }
 
+sub check_db_handle {
+    return if $db_handle_ops < $db_handle_reset;
+    reset_db_handle();
+}
+
+sub reset_db_handle {
+    announce('INFO', 'Refreshing DB connection on max ops.')
+        if $verbose && $db_handle;
+    $db_handle_ops = 0;
+    disconnect_db();
+    connect_db();
+    prepare_statements();
+}
+
+sub disconnect_db {
+    return unless $db_handle;
+
+    $new_auth_sth->finish;
+    $mod_auth_sth->finish;
+    $del_auth_sth->finish;
+    $delmod_auth_sth->finish;
+    $match_auth_sth->finish;
+    $match_auth_001_sth->finish;
+    $mod_bibs_sth->finish;
+    $db_handle->disconnect;
+}
+
 
 sub connect_db {
     $db_handle = DBI->connect(
@@ -146,7 +176,13 @@ sub connect_db {
             PrintError => 0,
             AutoCommit => 1,
             pg_expand_array => 0,
-            pg_enable_utf8 => 1
+            pg_enable_utf8 => 1,
+            Callbacks => {
+                # Track the number of execute() operations
+                ChildCallbacks => {
+                    execute => sub { $db_handle_ops++; return; }
+                }
+            } 
         }
     ) or die "Connection to database failed: $DBI::err : $DBI::errstr";
 }
@@ -239,6 +275,7 @@ sub handle_modified_bibs {
 
     while (my $record = $marc_batch->next()) {
         my $bib_id = $record->subfield('901', 'c');
+        check_db_handle();
 
         if (!$bib_id) {
             announce('ERR', "Bib record has no 901c (ID) value.  Skipping");
@@ -303,6 +340,7 @@ sub handle_deleted_auths {
 
     while (my $record = $marc_batch->next()) {
         my @matches = find_matching_auths($record);
+        check_db_handle();
 
         for my $auth_id (@matches) {
 
@@ -330,6 +368,7 @@ sub handle_modified_auths {
     my $marc_batch = shift;
 
     while (my $record = $marc_batch->next()) {
+        check_db_handle();
 
         modify_auth_005($record);
 
@@ -511,18 +550,8 @@ sub prepare_statements {
 }
 
 openlog($syslog_ident, $syslog_ops, $syslog_facility);
-connect_db();
-prepare_statements();
+reset_db_handle();
 process_zip_file() if $zip_file;
 process_marc_file($marc_file) if $marc_file;
-
-$new_auth_sth->finish;
-$mod_auth_sth->finish;
-$del_auth_sth->finish;
-$delmod_auth_sth->finish;
-$match_auth_sth->finish;
-$match_auth_001_sth->finish;
-$mod_bibs_sth->finish;
-
-$db_handle->disconnect;
+disconnect_db();