JBAS-639 Brick manager script
authorBill Erickson <berickxx@gmail.com>
Mon, 13 Apr 2015 18:40:58 +0000 (11:40 -0700)
committerBill Erickson <berickxx@gmail.com>
Thu, 21 Mar 2019 19:46:23 +0000 (15:46 -0400)
New brick-manager.sh script for stopping, starting, detaching,
attaching, and updating clusters of bricks.  Default clusters include
production, staging, and testing, though the user can optionally
specifify a single server (mainly for testing).

eg-updater.sh gets new commands for stop / start only and detach /
attach only.

Signed-off-by: Bill Erickson <berickxx@gmail.com>
KCLS/admin-scripts/brick-manager.sh [new file with mode: 0755]
KCLS/admin-scripts/eg-updater.sh

diff --git a/KCLS/admin-scripts/brick-manager.sh b/KCLS/admin-scripts/brick-manager.sh
new file mode 100755 (executable)
index 0000000..33be2c7
--- /dev/null
@@ -0,0 +1,182 @@
+#!/bin/bash
+set -euo pipefail
+CLUSTER="" # default to no choice; avoid surprises.
+SERVER=""
+SERVERS=()
+BASE_SCRIPT="cd /home/opensrf/Evergreen && KCLS/admin-scripts/eg-updater.sh -f 20"
+EG_UPDATER_OPS=""
+ACTION=""
+LIST_SERVERS=""
+PROMPT=""
+
+PRODUCTION_BRICKS=( 
+    prod-brick01 
+    prod-brick02 
+    prod-brick03 
+    prod-brick04 
+    prod-brick05 
+    prod-brick06 
+    prod-brick07 
+);
+
+STAGING_BRICKS=(
+    staging-brick01
+    staging-brick02
+);
+
+TESTING_BRICKS=(
+    testing-brick01
+    testing-brick02
+);
+
+function announce {
+    echo "$(date +'%F %T') $(hostname): $1"
+}
+
+function usage {
+    cat <<USAGE
+        $0 -c <cluster-name> -a <canned-action>
+        $0 -c <cluster-name> -e <eg-updater-ops>
+
+        Runs eg-update.sh on all servers in the requested cluster,
+        using the parameters provided directly by the -e argument
+        or those generated by various canned operations.
+
+        Examples:
+
+            Roll all testing bricks and leave detached:
+
+            $0 -c testing -a roll
+
+            Roll all testing bricks and attach:
+
+            $0 -c testing -a roll-attach
+
+            Deploy the 'kcls-2.5.001' git tag to all bricks by
+            passing arguments via -e:
+
+            $0 -c testing -e "-t kcls-2.5.001 -i kcls-2.5" 
+    
+        Options:
+            -c <cluster-name>
+                testing, staging, or production.  One of -c or -s must
+                be provided.
+
+            -s <server-name>
+                Direct this script to a specific server.  One of -c or -s 
+                must be provided.
+
+            -e <eg-updater-options>
+                Options passed directly to eg-update.sh.  See eg-updater.sh
+                for full list of supported options.
+
+            -a <action>
+                Short-cut for commonly used operations.
+
+                Supported actions:
+
+                    roll         - detach, restart services + apache
+                    roll-attach  - detach, restart services + apache, attach
+                    stop         - detach, stop services, stop apache
+                    start        - start services, start apache
+                    start-attach - start services, start apache, attach.
+                    detach       - detach only
+                    attach       - attach only
+            -l
+                List servers for the requested cluster
+
+            -p
+                Prompt to continue or exit between each server
+
+USAGE
+    exit
+}
+
+function inspect_args {
+
+    if [ -n "$SERVER" ]; then
+        SERVERS=( $SERVER )
+    else
+
+        if [ "$CLUSTER" = 'production' ]; then
+            announce "Cluster 'production' not yet supported"
+            exit
+            SERVERS=( ${PRODUCTION_BRICKS[@]} );
+        elif [ "$CLUSTER" = 'staging' ]; then
+            announce "Cluster 'staging' not yet supported"
+            exit
+            SERVERS=( ${STAGING_BRICKS[@]} );
+        elif [ "$CLUSTER" = 'testing' ]; then
+            SERVERS=( ${TESTING_BRICKS[@]} );
+        elif [ -n "$CLUSTER" ]; then
+            announce "Supported clusters: testing, staging, production"
+            exit;
+        else
+            usage
+        fi
+    fi
+
+    if [ -n "$ACTION" ]; then
+        if [ "$ACTION" = "roll" ]; then
+            EG_UPDATER_OPS="-o"
+        elif [ "$ACTION" = "roll-attach" ]; then
+            EG_UPDATER_OPS="-o -a"
+        elif [ "$ACTION" = "stop" ]; then
+            EG_UPDATER_OPS="-l"
+        elif [ "$ACTION" = "start" ]; then
+            EG_UPDATER_OPS="-m"
+        elif [ "$ACTION" = "start-attach" ]; then
+            EG_UPDATER_OPS="-ma"
+        elif [ "$ACTION" = "detach" ]; then
+            EG_UPDATER_OPS="-u"
+        elif [ "$ACTION" = "attach" ]; then
+            EG_UPDATER_OPS="-v"
+        fi
+    fi
+
+    if  [ -z "$LIST_SERVERS" -a -z "$EG_UPDATER_OPS" ]; then
+        usage;
+    fi
+}
+
+while getopts "c:e:a:s:lp" opt; do
+    case $opt in
+        c) CLUSTER="$OPTARG";;
+        e) EG_UPDATER_OPS="$OPTARG";;
+        a) ACTION="$OPTARG";;
+        l) LIST_SERVERS="1";;
+        p) PROMPT="1";;
+        s) SERVER="$OPTARG";;
+        h|*) usage;
+    esac
+done;
+
+inspect_args;
+
+if [ -n "$LIST_SERVERS" ]; then
+    echo "cluster = $CLUSTER"
+    echo "servers = ${SERVERS[*]}"
+
+else
+
+    last_index=$((( ${#SERVERS[@]} - 1 )))
+
+    for index in "${!SERVERS[@]}"; do
+
+        server="${SERVERS[$index]}"
+        announce "Connecting to $server"
+
+        ssh -t $server "$BASE_SCRIPT $EG_UPDATER_OPS"
+
+        if [ "$index" = "$last_index" ]; then
+            # avoid prompt on final server
+            break
+
+        elif [ -n "$PROMPT" ]; then
+            echo "Completed $server;  Enter to continue; Control-c to exit"
+            read X;
+        fi
+
+    done
+fi
+
index e85920c..4e24cf6 100755 (executable)
@@ -37,6 +37,10 @@ REBUILD=""
 BUILD_XUL=""
 RESTART_ONLY=""
 STAFF_CLIENT_ONLY=""
+STOP_ONLY=""
+START_ONLY=""
+ATTACH_ONLY=""
+DETACH_ONLY=""
 
 # if the script exits early, SUCCESS will be left as 0, 
 # which is our sign to warn the user.
@@ -132,6 +136,20 @@ function usage {
                 $START_SLEEP.  Increase this value if autogen regularly 
                 fails.
 
+            -u
+                Detach server (via ping.txt)
+
+            -v 
+                Attach server (via ping.txt)
+
+            -l  
+                Detach, stop services, stop apache
+
+            -m  
+                Start services, start apache, leave detached.  Combine
+                with -a to also attach.
+
+
 USAGE
 trap - EXIT
 exit;
@@ -148,8 +166,6 @@ function announce {
 function on_exit {
     if [ $SUCCESS -ne 1 ]; then
         announce "** $0 exited unexpectedly! **"
-    else
-        announce "$0 completed successfully"
     fi
 }
 
@@ -160,7 +176,6 @@ function set_builder {
         # run git commands from a login shell (-i) and use git -C to set
         # the Evergreen checkout directory, since -i drops you into $HOME.
         GIT="$BUILDER -i git -C $EVERGREEN_BASE"
-        announce "Building as user $OWNER"
     fi
 
     if [ $RUNNER != "opensrf" ]; then
@@ -171,13 +186,18 @@ function set_builder {
 function inspect_params {
 
     # no need to detach the brick when not restarting services
-    if [ -n "$STAFF_CLIENT_ONLY" -o -n "$WEB_ONLY" ]; then
+    if [ -n "$STAFF_CLIENT_ONLY" -o -n "$WEB_ONLY" \
+            -o -n "$START_ONLY" -o -n "$ATTACH_ONLY" ]; then
         DETACH_NEEDED=""
     fi
 
-    if [ -n "$RESTART_ONLY" ]; then 
-        # restart trumps all other variables.
-        # avoid warning about other variable issues
+    if [ -n "$ATTACH_ONLY" ]; then
+        REATTACH="YES"
+        return
+    fi
+
+    if [ -n "$RESTART_ONLY" -o -n "$DETACH_ONLY" ]; then 
+        # these trump other variables.
         return
     fi
 
@@ -186,11 +206,10 @@ function inspect_params {
        CLIENT_BUILD_ID="$GIT_TAG";
     fi;
 
-    if [ -z "$CLIENT_BUILD_ID" -a -z "$WEB_ONLY" ]; then
+    if [ -z "$CLIENT_BUILD_ID" ]; then
         # if we still have no build ID, use the current date.
         # other symlinks will be 
         CLIENT_BUILD_ID=$(date +'%FT%T')
-        announce "No -s specified; using $CLIENT_BUILD_ID as client build ID"
     fi;
 
     # checking out a branch causes a 'git reset --hard', which
@@ -200,6 +219,11 @@ function inspect_params {
         announce "-c param is applied by default when a branch is specified"
         REBUILD="YES"; 
     fi
+
+    if [ -n "$STOP_ONLY" -a -n "$START_ONLY" ]; then
+        announce "Cannot combined -l and -m.  Use -o to restart"
+        exit 1
+    fi
 }
 
 function checkout_code {
@@ -207,7 +231,7 @@ function checkout_code {
     REMOTE=$2 # remote git target name
     CURRENT=$($GIT rev-parse --abbrev-ref HEAD);
 
-    if [ "$CURRENT" == "heads/$LOCAL" ]; then
+    if [ "$CURRENT" = "heads/$LOCAL" ]; then
         # git rev-parse shows tags as 'heads/$tagname'
         CURRENT=$LOCAL
     fi
@@ -497,23 +521,27 @@ function start_services {
 
 trap on_exit EXIT;
 
-while getopts "t:b:r:s:i:f:panwchxok" opt; do
+while getopts "t:b:r:s:i:f:panwchxoklmuv" opt; do
     case $opt in
-        t) GIT_TAG="$OPTARG";;
+        a) REATTACH="YES";;
         b) GIT_BRANCH="$OPTARG";;
-        r) GIT_REMOTE="$OPTARG";;
-        s) CLIENT_BUILD_ID="$OPTARG";;
-        i) CLIENT_SERIES_ID="$OPTARG";;
+        c) REBUILD="YES";;
         f) START_SLEEP="$OPTARG";;
-        p) GIT_PULL="YES";;
-        a) REATTACH="YES";;
+        i) CLIENT_SERIES_ID="$OPTARG";;
+        k) STAFF_CLIENT_ONLY="YES";;
+        l) STOP_ONLY="YES";;
+        m) START_ONLY="YES";;
         n) SKIP_APACHE="YES";;
+        o) RESTART_ONLY="YES";;
+        p) GIT_PULL="YES";;
+        r) GIT_REMOTE="$OPTARG";;
+        s) CLIENT_BUILD_ID="$OPTARG";;
+        t) GIT_TAG="$OPTARG";;
+        u) DETACH_ONLY="YES";;
+        v) ATTACH_ONLY="YES";;
         w) WEB_ONLY="YES";;
-        c) REBUILD="YES";;
         x) BUILD_XUL="YES";;
-        o) RESTART_ONLY="YES";;
-        k) STAFF_CLIENT_ONLY="YES";;
-        h|*) usage;
+        h) usage;
     esac
 done;
 
@@ -526,9 +554,13 @@ if [ -n "$DETACH_NEEDED" ]; then
         announce "No ping file found.  Skipping detach."
 
     else
-        announce "Detaching brick and sleeping $DETACH_SLEEP seconds"
+        announce "Detaching brick"
         $OSRF mv $PING_FILE $PING_FILE-
-        sleep $DETACH_SLEEP
+        if [ -z "$DETACH_ONLY" ]; then
+            # avoid detach sleep when only detaching
+            announce "Sleeping $DETACH_SLEEP seconds after detach"
+            sleep $DETACH_SLEEP
+        fi
     fi
 fi;
 
@@ -544,9 +576,13 @@ elif [ -n "$STAFF_CLIENT_ONLY" ]; then
     build_code
     build_staff_client
 
-else 
+elif [ -n "$DETACH_ONLY" ]; then
+    SUCCESS=1
+    exit;
 
-    if [ -z "$SKIP_APACHE" ]; then
+elif [ -z "$ATTACH_ONLY" ]; then
+
+    if [ -z "$SKIP_APACHE" -a -z "$START_ONLY" ]; then
         announce "Stopping Apache"
         sudo service apache2 stop > /dev/null
     fi
@@ -554,15 +590,17 @@ else
     # Note the use of --fast-shutdown-all, which is faster than the
     # default graceful shutdown.  We don't need graceful shutdown, 
     # since we will not be stopping potentially active services.
-    announce "Stopping services"
-    $OSRF /openils/bin/osrf_control --localhost \
-        --fast-shutdown-all > /dev/null
+    if [ -z "$START_ONLY" ]; then
+        announce "Stopping services"
+        $OSRF /openils/bin/osrf_control --localhost \
+            --fast-shutdown-all > /dev/null
+    fi
 
-    if [ -n "$RESTART_ONLY" ]; then
+    if [ -n "$RESTART_ONLY" -o -n "$START_ONLY" ]; then
         sleep 1;
         start_services;
 
-    else
+    elif [ -z "$STOP_ONLY" ]; then
         fetch_code;
         build_code;
         deploy_code;
@@ -572,16 +610,19 @@ else
         $OSRF /openils/bin/autogen.sh > /dev/null
     fi
 
-    if [ -z "$SKIP_APACHE" ]; then
+    if [ -z "$SKIP_APACHE" -a -z "$STOP_ONLY" ]; then
         announce "Starting Apache"
         sudo service apache2 start > /dev/null
     fi;
 fi
 
-if [ -f $PING_FILE- ]; then # we detached
+if [ -f $PING_FILE- -a -z "$STOP_ONLY" ]; then # we detached
     if [ -n "$REATTACH" ]; then
-        announce "Reattaching brick after $START_SLEEP seconds"
-        sleep $START_SLEEP;
+        if [ -z "$ATTACH_ONLY" ]; then
+            announce "Sleeping $START_SLEEP seconds before attach"
+            sleep $START_SLEEP;
+        fi
+        announce "Attaching brick"
         $OSRF mv $PING_FILE- $PING_FILE
     else
         announce "Brick is still detached!  Use -a to automatically re-attach"