From: Bill Erickson <berickxx@gmail.com>
Date: Wed, 7 Dec 2016 17:20:04 +0000 (-0800)
Subject: JBAS-1437 Backstage importer db handle resets
X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=f3d3316346c2f4781b8a597e669faffb1f0f1444;p=working%2FEvergreen.git

JBAS-1437 Backstage importer db handle resets

Periodically close and reopen the DB connection to avoid using any one
connection for extended periods of time, which can be bad when SQL
queries have memory leaks, as with bib record updates.

Signed-off-by: Bill Erickson <berickxx@gmail.com>
---

diff --git a/KCLS/authority-control/backstage/process-backstage-files.pl b/KCLS/authority-control/backstage/process-backstage-files.pl
index e402b2c81e..8f571f3d14 100755
--- a/KCLS/authority-control/backstage/process-backstage-files.pl
+++ b/KCLS/authority-control/backstage/process-backstage-files.pl
@@ -20,6 +20,9 @@ use OpenILS::Utils::Normalize qw(clean_marc);
 binmode(STDOUT, ':utf8');
 
 my $db_handle;
+my $db_handle_ops = 0;
+# Reset the DB handle have this many operations to avoid memory leaks.
+my $db_handle_reset = 500;
 my $log_mod = 500;  # log every 500th of each type of event (see verbose)
 
 my $marc_file;
@@ -137,6 +140,33 @@ sub announce {
     }
 }
 
+sub check_db_handle {
+    return if $db_handle_ops < $db_handle_reset;
+    reset_db_handle();
+}
+
+sub reset_db_handle {
+    announce('INFO', 'Refreshing DB connection on max ops.')
+        if $verbose && $db_handle;
+    $db_handle_ops = 0;
+    disconnect_db();
+    connect_db();
+    prepare_statements();
+}
+
+sub disconnect_db {
+    return unless $db_handle;
+
+    $new_auth_sth->finish;
+    $mod_auth_sth->finish;
+    $del_auth_sth->finish;
+    $delmod_auth_sth->finish;
+    $match_auth_sth->finish;
+    $match_auth_001_sth->finish;
+    $mod_bibs_sth->finish;
+    $db_handle->disconnect;
+}
+
 
 sub connect_db {
     $db_handle = DBI->connect(
@@ -146,7 +176,13 @@ sub connect_db {
             PrintError => 0,
             AutoCommit => 1,
             pg_expand_array => 0,
-            pg_enable_utf8 => 1
+            pg_enable_utf8 => 1,
+            Callbacks => {
+                # Track the number of execute() operations
+                ChildCallbacks => {
+                    execute => sub { $db_handle_ops++; return; }
+                }
+            } 
         }
     ) or die "Connection to database failed: $DBI::err : $DBI::errstr";
 }
@@ -239,6 +275,7 @@ sub handle_modified_bibs {
 
     while (my $record = $marc_batch->next()) {
         my $bib_id = $record->subfield('901', 'c');
+        check_db_handle();
 
         if (!$bib_id) {
             announce('ERR', "Bib record has no 901c (ID) value.  Skipping");
@@ -303,6 +340,7 @@ sub handle_deleted_auths {
 
     while (my $record = $marc_batch->next()) {
         my @matches = find_matching_auths($record);
+        check_db_handle();
 
         for my $auth_id (@matches) {
 
@@ -330,6 +368,7 @@ sub handle_modified_auths {
     my $marc_batch = shift;
 
     while (my $record = $marc_batch->next()) {
+        check_db_handle();
 
         modify_auth_005($record);
 
@@ -511,18 +550,8 @@ sub prepare_statements {
 }
 
 openlog($syslog_ident, $syslog_ops, $syslog_facility);
-connect_db();
-prepare_statements();
+reset_db_handle();
 process_zip_file() if $zip_file;
 process_marc_file($marc_file) if $marc_file;
-
-$new_auth_sth->finish;
-$mod_auth_sth->finish;
-$del_auth_sth->finish;
-$delmod_auth_sth->finish;
-$match_auth_sth->finish;
-$match_auth_001_sth->finish;
-$mod_bibs_sth->finish;
-
-$db_handle->disconnect;
+disconnect_db();