Improved PID file handling in osrf_ctl.sh

Joseph Lewis <joehms22@gmail.com> submitted a patch to address
https://bugs.launchpad.net/evergreen/+bug/741088 ("osrf_ctl.sh Doesn't check if
process is actually running")

As of this patch, when start_perl, start_c, or start_python are started and
existing PID files are found for those processes, instead of assuming that the
processes are actually running, osrf-ctl.sh now has the intelligence to check
the process list to see if there is a matching process. If no running process
is found, then the old PID file is removed and the start command is issued.

Two additional actions have been added, "smart_clear" and "clear_pid".

"smart_clear" checks the PID files against the running processes and removes
PID files where no running process is found. This is effectively invoked
under the covers when start_* is invoked and a PID file is found.

"clear_pid" deletes all PID files without checking to see if there are any
running processes. In most cases, it should only be invoked as part of an
automated boot sequence.

Developer's Certificate of Origin 1.1

By making a contribution to this project, I certify that:

(a) The contribution was created in whole or in part by me and I
    have the right to submit it under the open source license
    indicated in the file; or

(b) The contribution is based upon previous work that, to the best
    of my knowledge, is covered under an appropriate open source
    license and I have the right under that license to submit that
    work with modifications, whether created in whole or in part
    by me, under the same open source license (unless I am
    permitted to submit under a different license), as indicated
    in the file; or

(c) The contribution was provided directly to me by some other
    person who certified (a), (b) or (c) and I have not modified
    it.

(d) I understand and agree that this project and the contribution
    are public and that a record of the contribution (including all
    personal information I submit with it, including my sign-off) is
    maintained indefinitely and may be redistributed consistent with
    this project or the open source license(s) involved.

Signed-off-by: Joseph Lewis <joehms22@gmail.com>
git-svn-id: svn://svn.open-ils.org/OpenSRF/trunk@2215 9efc2488-bf62-4759-914b-345cdb29e865

diff --git a/bin/osrf_ctl.sh.in b/bin/osrf_ctl.sh.in
index 8e9aaf6..15cba82 100755 (executable)
--- a/bin/osrf_ctl.sh.in
+++ b/bin/osrf_ctl.sh.in
@@ -60,6 +60,8 @@ Actions include:
     stop_all
     start_all
     restart_all
+    smart_clear     - Clear all PID files that don't refer to a process 
+    clear_pid       - Clear all PID files
 
 Examples:
   $0 -a restart_all
@@ -94,6 +96,7 @@ fi;
 
 PID_ROUTER="$OPT_PID_DIR/router.pid";
 PID_OSRF_PERL="$OPT_PID_DIR/osrf_perl.pid";
+PID_OSRF_PYTHON="$OPT_PID_DIR/osrf_python.pid";
 PID_OSRF_C="$OPT_PID_DIR/osrf_c.pid";
 
 
@@ -110,8 +113,16 @@ do_action() {
 
                if [ -e $pidfile ]; then
                        pid=$(cat $pidfile);
-                       echo "$item already started : $pid";
-                       return 0;
+                       reported=$(ps ax | grep "$item" | grep -v grep | awk '{print $1}' | awk '{ printf "%s ", $0 }')
+                       
+                       if [ "$pid " = "$reported" ]; then
+                               echo "$item already started : $pid";
+                               return 0;
+                       else
+                               echo "$item not started, but PID file exists, removing file and starting";
+                               rm $pidfile;
+                               return 0;
+                       fi;
                fi;
                echo "Starting $item";
        fi;
@@ -162,10 +173,20 @@ stop_router() {
 start_python() {
     [ ! $($OSRF_CONFIG | grep OSRF_PYTHON) ] && return;
     echo "Starting OpenSRF Python";
+    
+    if [ -e $PID_OSRF_PYTHON ]; then  #If python is started already (or it thinks so).
+    cat << EOF
+Python processes are either already running, or were not correctly shut down.
+Now clearing any stale PID files and restarting Perl processes.
+EOF
+    smart_clear;
+    fi;
+
     OPT_LOCAL=""
     [ "$OSRF_HOSTNAME" = "localhost" ] && OPT_LOCAL="-l"
     for service in `opensrf.py -a list_all $OPT_LOCAL`; do
             opensrf.py -p $OPT_PID_DIR -f $OPT_CONFIG -d -a start -s $service $OPT_LOCAL
+            [ "$?" = "0" ] && echo "Python Started" > $PID_OSRF_PYTHON;  #Dummy pid file, removed when a proper shutdown happens.
     done
     return 0;
 }
@@ -173,6 +194,7 @@ start_python() {
 stop_python() {
     [ ! $($OSRF_CONFIG | grep OSRF_PYTHON) ] && return;
     echo "Stopping OpenSRF Python";
+    [ -e $PID_OSRF_PYTHON ] && rm $PID_OSRF_PYTHON;
     OPT_LOCAL=""
     [ "$OSRF_HOSTNAME" = "localhost" ] && OPT_LOCAL="-l"
     opensrf.py -p $OPT_PID_DIR -f $OPT_CONFIG -a stop_all $OPT_LOCAL
@@ -182,14 +204,25 @@ stop_python() {
 
 start_perl() {
     echo "Starting OpenSRF Perl";
+    
+    if [ -e $PID_OSRF_PERL ]; then  #If perl is started already (or it thinks so)
+    cat << EOF
+Perl processes are either already running, or were not correctly shut down.
+Now clearing any stale PID files and restarting Perl processes.
+EOF
+    smart_clear;
+    fi;
+    
     opensrf-perl.pl --verbose --pid-dir $OPT_PID_DIR \
         --config $OPT_CONFIG --action start_all --settings-startup-pause 3
+    [ "$?" = "0" ] && echo "Perl Started" > $PID_OSRF_PERL;  #Dummy pid file, removed when a proper shutdown happens.
        return 0;
 }
 
 stop_perl() {
     echo "Stopping OpenSRF Perl";
     opensrf-perl.pl --verbose --pid-dir $OPT_PID_DIR --config $OPT_CONFIG --action stop_all
+    [ -e $PID_OSRF_PERL ] && rm $PID_OSRF_PERL;
        sleep 1;
        return 0;
 }
@@ -207,11 +240,36 @@ start_c() {
 
 stop_c() {
        do_action "stop" $PID_OSRF_C "OpenSRF C";
+       [ -e $PID_OSRF_C ] && rm $PID_OSRF_C;
        sleep 1;
        return 0;
 }
 
+clear_pid() {
+       echo "Clearing PID files...";
+       cd $OPT_PID_DIR;
+       [ 0 -lt ls | wc -l ] && rm -v *.pid;
+       return 0;
+}
 
+smart_clear() {
+       echo "Smart clearing PID files...";
+       for line in $(find $OPT_PID_DIR -name *.pid -type f)
+       do
+               running="false";
+               for p in $(cat $line)
+               do
+                       [ 0 -lt $(ps ax | grep "$p" | grep -v grep | wc -l) ] && running="true";
+               done
+               
+               if [ $running = "false" ]; then
+                       rm $line;
+                       echo "Removing stale PID file: $line";
+               fi;
+       done
+       
+       return 0;
+}
 
 # ---------------------------------------------------------------------------
 # Do the requested action
@@ -235,6 +293,8 @@ case $OPT_ACTION in
        "stop_all") stop_python; stop_c; stop_perl; stop_router;;
        "start_all") start_router; start_perl; start_c; start_python;;
        "restart_all") stop_python; stop_c; stop_perl; stop_router; start_router; start_perl; start_c; start_python;;
+       "clear_pid") clear_pid;;
+       "smart_clear") smart_clear;;
        *) usage;;
 esac;