From 7186ae1bed060fa295ef8e5d8659b230fb67c23b Mon Sep 17 00:00:00 2001 From: Bill Erickson Date: Mon, 13 Apr 2015 11:40:58 -0700 Subject: [PATCH] JBAS-639 Brick manager script New brick-manager.sh script for stopping, starting, detaching, attaching, and updating clusters of bricks. Default clusters include production, staging, and testing, though the user can optionally specifify a single server (mainly for testing). eg-updater.sh gets new commands for stop / start only and detach / attach only. Signed-off-by: Bill Erickson --- KCLS/admin-scripts/brick-manager.sh | 182 ++++++++++++++++++++++++++++++++++++ KCLS/admin-scripts/eg-updater.sh | 109 ++++++++++++++------- 2 files changed, 257 insertions(+), 34 deletions(-) create mode 100755 KCLS/admin-scripts/brick-manager.sh diff --git a/KCLS/admin-scripts/brick-manager.sh b/KCLS/admin-scripts/brick-manager.sh new file mode 100755 index 0000000000..33be2c70ee --- /dev/null +++ b/KCLS/admin-scripts/brick-manager.sh @@ -0,0 +1,182 @@ +#!/bin/bash +set -euo pipefail +CLUSTER="" # default to no choice; avoid surprises. +SERVER="" +SERVERS=() +BASE_SCRIPT="cd /home/opensrf/Evergreen && KCLS/admin-scripts/eg-updater.sh -f 20" +EG_UPDATER_OPS="" +ACTION="" +LIST_SERVERS="" +PROMPT="" + +PRODUCTION_BRICKS=( + prod-brick01 + prod-brick02 + prod-brick03 + prod-brick04 + prod-brick05 + prod-brick06 + prod-brick07 +); + +STAGING_BRICKS=( + staging-brick01 + staging-brick02 +); + +TESTING_BRICKS=( + testing-brick01 + testing-brick02 +); + +function announce { + echo "$(date +'%F %T') $(hostname): $1" +} + +function usage { + cat < -a + $0 -c -e + + Runs eg-update.sh on all servers in the requested cluster, + using the parameters provided directly by the -e argument + or those generated by various canned operations. + + Examples: + + Roll all testing bricks and leave detached: + + $0 -c testing -a roll + + Roll all testing bricks and attach: + + $0 -c testing -a roll-attach + + Deploy the 'kcls-2.5.001' git tag to all bricks by + passing arguments via -e: + + $0 -c testing -e "-t kcls-2.5.001 -i kcls-2.5" + + Options: + -c + testing, staging, or production. One of -c or -s must + be provided. + + -s + Direct this script to a specific server. One of -c or -s + must be provided. + + -e + Options passed directly to eg-update.sh. See eg-updater.sh + for full list of supported options. + + -a + Short-cut for commonly used operations. + + Supported actions: + + roll - detach, restart services + apache + roll-attach - detach, restart services + apache, attach + stop - detach, stop services, stop apache + start - start services, start apache + start-attach - start services, start apache, attach. + detach - detach only + attach - attach only + -l + List servers for the requested cluster + + -p + Prompt to continue or exit between each server + +USAGE + exit +} + +function inspect_args { + + if [ -n "$SERVER" ]; then + SERVERS=( $SERVER ) + else + + if [ "$CLUSTER" = 'production' ]; then + announce "Cluster 'production' not yet supported" + exit + SERVERS=( ${PRODUCTION_BRICKS[@]} ); + elif [ "$CLUSTER" = 'staging' ]; then + announce "Cluster 'staging' not yet supported" + exit + SERVERS=( ${STAGING_BRICKS[@]} ); + elif [ "$CLUSTER" = 'testing' ]; then + SERVERS=( ${TESTING_BRICKS[@]} ); + elif [ -n "$CLUSTER" ]; then + announce "Supported clusters: testing, staging, production" + exit; + else + usage + fi + fi + + if [ -n "$ACTION" ]; then + if [ "$ACTION" = "roll" ]; then + EG_UPDATER_OPS="-o" + elif [ "$ACTION" = "roll-attach" ]; then + EG_UPDATER_OPS="-o -a" + elif [ "$ACTION" = "stop" ]; then + EG_UPDATER_OPS="-l" + elif [ "$ACTION" = "start" ]; then + EG_UPDATER_OPS="-m" + elif [ "$ACTION" = "start-attach" ]; then + EG_UPDATER_OPS="-ma" + elif [ "$ACTION" = "detach" ]; then + EG_UPDATER_OPS="-u" + elif [ "$ACTION" = "attach" ]; then + EG_UPDATER_OPS="-v" + fi + fi + + if [ -z "$LIST_SERVERS" -a -z "$EG_UPDATER_OPS" ]; then + usage; + fi +} + +while getopts "c:e:a:s:lp" opt; do + case $opt in + c) CLUSTER="$OPTARG";; + e) EG_UPDATER_OPS="$OPTARG";; + a) ACTION="$OPTARG";; + l) LIST_SERVERS="1";; + p) PROMPT="1";; + s) SERVER="$OPTARG";; + h|*) usage; + esac +done; + +inspect_args; + +if [ -n "$LIST_SERVERS" ]; then + echo "cluster = $CLUSTER" + echo "servers = ${SERVERS[*]}" + +else + + last_index=$((( ${#SERVERS[@]} - 1 ))) + + for index in "${!SERVERS[@]}"; do + + server="${SERVERS[$index]}" + announce "Connecting to $server" + + ssh -t $server "$BASE_SCRIPT $EG_UPDATER_OPS" + + if [ "$index" = "$last_index" ]; then + # avoid prompt on final server + break + + elif [ -n "$PROMPT" ]; then + echo "Completed $server; Enter to continue; Control-c to exit" + read X; + fi + + done +fi + diff --git a/KCLS/admin-scripts/eg-updater.sh b/KCLS/admin-scripts/eg-updater.sh index e85920c724..4e24cf6f78 100755 --- a/KCLS/admin-scripts/eg-updater.sh +++ b/KCLS/admin-scripts/eg-updater.sh @@ -37,6 +37,10 @@ REBUILD="" BUILD_XUL="" RESTART_ONLY="" STAFF_CLIENT_ONLY="" +STOP_ONLY="" +START_ONLY="" +ATTACH_ONLY="" +DETACH_ONLY="" # if the script exits early, SUCCESS will be left as 0, # which is our sign to warn the user. @@ -132,6 +136,20 @@ function usage { $START_SLEEP. Increase this value if autogen regularly fails. + -u + Detach server (via ping.txt) + + -v + Attach server (via ping.txt) + + -l + Detach, stop services, stop apache + + -m + Start services, start apache, leave detached. Combine + with -a to also attach. + + USAGE trap - EXIT exit; @@ -148,8 +166,6 @@ function announce { function on_exit { if [ $SUCCESS -ne 1 ]; then announce "** $0 exited unexpectedly! **" - else - announce "$0 completed successfully" fi } @@ -160,7 +176,6 @@ function set_builder { # run git commands from a login shell (-i) and use git -C to set # the Evergreen checkout directory, since -i drops you into $HOME. GIT="$BUILDER -i git -C $EVERGREEN_BASE" - announce "Building as user $OWNER" fi if [ $RUNNER != "opensrf" ]; then @@ -171,13 +186,18 @@ function set_builder { function inspect_params { # no need to detach the brick when not restarting services - if [ -n "$STAFF_CLIENT_ONLY" -o -n "$WEB_ONLY" ]; then + if [ -n "$STAFF_CLIENT_ONLY" -o -n "$WEB_ONLY" \ + -o -n "$START_ONLY" -o -n "$ATTACH_ONLY" ]; then DETACH_NEEDED="" fi - if [ -n "$RESTART_ONLY" ]; then - # restart trumps all other variables. - # avoid warning about other variable issues + if [ -n "$ATTACH_ONLY" ]; then + REATTACH="YES" + return + fi + + if [ -n "$RESTART_ONLY" -o -n "$DETACH_ONLY" ]; then + # these trump other variables. return fi @@ -186,11 +206,10 @@ function inspect_params { CLIENT_BUILD_ID="$GIT_TAG"; fi; - if [ -z "$CLIENT_BUILD_ID" -a -z "$WEB_ONLY" ]; then + if [ -z "$CLIENT_BUILD_ID" ]; then # if we still have no build ID, use the current date. # other symlinks will be CLIENT_BUILD_ID=$(date +'%FT%T') - announce "No -s specified; using $CLIENT_BUILD_ID as client build ID" fi; # checking out a branch causes a 'git reset --hard', which @@ -200,6 +219,11 @@ function inspect_params { announce "-c param is applied by default when a branch is specified" REBUILD="YES"; fi + + if [ -n "$STOP_ONLY" -a -n "$START_ONLY" ]; then + announce "Cannot combined -l and -m. Use -o to restart" + exit 1 + fi } function checkout_code { @@ -207,7 +231,7 @@ function checkout_code { REMOTE=$2 # remote git target name CURRENT=$($GIT rev-parse --abbrev-ref HEAD); - if [ "$CURRENT" == "heads/$LOCAL" ]; then + if [ "$CURRENT" = "heads/$LOCAL" ]; then # git rev-parse shows tags as 'heads/$tagname' CURRENT=$LOCAL fi @@ -497,23 +521,27 @@ function start_services { trap on_exit EXIT; -while getopts "t:b:r:s:i:f:panwchxok" opt; do +while getopts "t:b:r:s:i:f:panwchxoklmuv" opt; do case $opt in - t) GIT_TAG="$OPTARG";; + a) REATTACH="YES";; b) GIT_BRANCH="$OPTARG";; - r) GIT_REMOTE="$OPTARG";; - s) CLIENT_BUILD_ID="$OPTARG";; - i) CLIENT_SERIES_ID="$OPTARG";; + c) REBUILD="YES";; f) START_SLEEP="$OPTARG";; - p) GIT_PULL="YES";; - a) REATTACH="YES";; + i) CLIENT_SERIES_ID="$OPTARG";; + k) STAFF_CLIENT_ONLY="YES";; + l) STOP_ONLY="YES";; + m) START_ONLY="YES";; n) SKIP_APACHE="YES";; + o) RESTART_ONLY="YES";; + p) GIT_PULL="YES";; + r) GIT_REMOTE="$OPTARG";; + s) CLIENT_BUILD_ID="$OPTARG";; + t) GIT_TAG="$OPTARG";; + u) DETACH_ONLY="YES";; + v) ATTACH_ONLY="YES";; w) WEB_ONLY="YES";; - c) REBUILD="YES";; x) BUILD_XUL="YES";; - o) RESTART_ONLY="YES";; - k) STAFF_CLIENT_ONLY="YES";; - h|*) usage; + h) usage; esac done; @@ -526,9 +554,13 @@ if [ -n "$DETACH_NEEDED" ]; then announce "No ping file found. Skipping detach." else - announce "Detaching brick and sleeping $DETACH_SLEEP seconds" + announce "Detaching brick" $OSRF mv $PING_FILE $PING_FILE- - sleep $DETACH_SLEEP + if [ -z "$DETACH_ONLY" ]; then + # avoid detach sleep when only detaching + announce "Sleeping $DETACH_SLEEP seconds after detach" + sleep $DETACH_SLEEP + fi fi fi; @@ -544,9 +576,13 @@ elif [ -n "$STAFF_CLIENT_ONLY" ]; then build_code build_staff_client -else +elif [ -n "$DETACH_ONLY" ]; then + SUCCESS=1 + exit; - if [ -z "$SKIP_APACHE" ]; then +elif [ -z "$ATTACH_ONLY" ]; then + + if [ -z "$SKIP_APACHE" -a -z "$START_ONLY" ]; then announce "Stopping Apache" sudo service apache2 stop > /dev/null fi @@ -554,15 +590,17 @@ else # Note the use of --fast-shutdown-all, which is faster than the # default graceful shutdown. We don't need graceful shutdown, # since we will not be stopping potentially active services. - announce "Stopping services" - $OSRF /openils/bin/osrf_control --localhost \ - --fast-shutdown-all > /dev/null + if [ -z "$START_ONLY" ]; then + announce "Stopping services" + $OSRF /openils/bin/osrf_control --localhost \ + --fast-shutdown-all > /dev/null + fi - if [ -n "$RESTART_ONLY" ]; then + if [ -n "$RESTART_ONLY" -o -n "$START_ONLY" ]; then sleep 1; start_services; - else + elif [ -z "$STOP_ONLY" ]; then fetch_code; build_code; deploy_code; @@ -572,16 +610,19 @@ else $OSRF /openils/bin/autogen.sh > /dev/null fi - if [ -z "$SKIP_APACHE" ]; then + if [ -z "$SKIP_APACHE" -a -z "$STOP_ONLY" ]; then announce "Starting Apache" sudo service apache2 start > /dev/null fi; fi -if [ -f $PING_FILE- ]; then # we detached +if [ -f $PING_FILE- -a -z "$STOP_ONLY" ]; then # we detached if [ -n "$REATTACH" ]; then - announce "Reattaching brick after $START_SLEEP seconds" - sleep $START_SLEEP; + if [ -z "$ATTACH_ONLY" ]; then + announce "Sleeping $START_SLEEP seconds before attach" + sleep $START_SLEEP; + fi + announce "Attaching brick" $OSRF mv $PING_FILE- $PING_FILE else announce "Brick is still detached! Use -a to automatically re-attach" -- 2.11.0