From: Mike Rylander <mrylander@gmail.com>
Date: Mon, 1 Feb 2016 16:34:31 +0000 (-0500)
Subject: Teach cstore and friends to keep trying to reconnect to the database
X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=refs%2Fheads%2Fuser%2Fmiker%2Fdb-reconnect;p=working%2FEvergreen.git

Teach cstore and friends to keep trying to reconnect to the database

Signed-off-by: Mike Rylander <mrylander@gmail.com>
---

diff --git a/Open-ILS/include/openils/oils_sql.h b/Open-ILS/include/openils/oils_sql.h
index d581c4b06c..1d668adedc 100644
--- a/Open-ILS/include/openils/oils_sql.h
+++ b/Open-ILS/include/openils/oils_sql.h
@@ -27,6 +27,7 @@ extern "C" {
 #endif
 
 dbi_conn oilsConnectDB( const char* mod_name );
+dbi_conn oilsReconnectDB( const char* mod_name );
 void oilsSetSQLOptions( const char* module_name, int do_pcrud, int flesh_depth );
 void oilsSetDBConnection( dbi_conn conn );
 int oilsIsDBConnected( dbi_conn handle );
diff --git a/Open-ILS/src/c-apps/oils_sql.c b/Open-ILS/src/c-apps/oils_sql.c
index bacb867553..e04a3470e8 100644
--- a/Open-ILS/src/c-apps/oils_sql.c
+++ b/Open-ILS/src/c-apps/oils_sql.c
@@ -123,6 +123,8 @@ static const jsonObject* verifyUserPCRUDfull( osrfMethodContext*, int );
 static int verifyObjectPCRUD( osrfMethodContext*, osrfHash*, const jsonObject*, int );
 static const char* org_tree_root( osrfMethodContext* ctx );
 static jsonObject* single_hash( const char* key, const char* value );
+static dbi_conn oilsConnectDB_impl( const char* mod_name, int keep_trying );
+
 
 static int child_initialized = 0;   /* boolean */
 
@@ -153,6 +155,14 @@ static char* _sanitize_savepoint_name( const char* sp );
 	@return A database connection if successful, or NULL if not.
 */
 dbi_conn oilsConnectDB( const char* mod_name ) {
+	return oilsConnectDB_impl( mod_name, 0 );
+}
+
+dbi_conn oilsReconnectDB( const char* mod_name ) {
+	return oilsConnectDB_impl( mod_name, 1 );
+}
+
+dbi_conn oilsConnectDB_impl( const char* mod_name, int keep_trying ) {
 
 	osrfLogDebug( OSRF_LOG_MARK, "Attempting to initialize libdbi..." );
 	if( dbi_initialize( NULL ) == -1 ) {
@@ -195,17 +205,20 @@ dbi_conn oilsConnectDB( const char* mod_name ) {
 	free( pw );
 	free( pg_app );
 
-	if( dbi_conn_connect( handle ) < 0 ) {
-		sleep( 1 );
-		if( dbi_conn_connect( handle ) < 0 ) {
+	int res = dbi_conn_connect( handle );
+	if( res < 0 ) {
+		do {
 			const char* msg;
 			dbi_conn_error( handle, &msg );
 			osrfLogError( OSRF_LOG_MARK, "Error connecting to database: %s",
 				msg ? msg : "(No description available)" );
-			return NULL;
-		}
+			sleep( 1 );
+			res = dbi_conn_connect( handle );
+		} while (res < 0 && keep_trying)
 	}
 
+	if (res < 0) return NULL;
+
 	osrfLogInfo( OSRF_LOG_MARK, "%s successfully connected to the database", mod_name );
 
 	return handle;
@@ -279,11 +292,16 @@ int oilsIsDBConnected( dbi_conn handle ) {
 		// E.g. 3624914: ERROR:  current transaction is aborted, commands ignored until end of transaction block
 		// Substring test should work regardless.
 		const char* substr = strstr(msg, ok_msg);
-		if( substr == NULL ) {
-			osrfLogError( OSRF_LOG_MARK, "Database connection isn't working : %s", msg );
-			return 0;
-		} else
-			return 1;   // ignoring SELECT due to previous error; that's okay
+
+		if( substr == NULL ) { // No matching error, connection must be in bad shape
+			dbi_conn_close( handle ); // Just in case
+			if (!(handle = oilsReconnectDB(modulename))) {
+				osrfLogError( OSRF_LOG_MARK, "Database connection isn't working : %s", msg );
+				return 0;
+			}
+		}
+
+		return 1;
 	}
 }