ctdb/tools/onnode

   1 #!/bin/bash
   2
   3 # Run commands on CTDB nodes.
   4
   5 # See http://ctdb.samba.org/ for more information about CTDB.
   6
   7 # Copyright (C) Martin Schwenke  2008
   8
   9 # Based on an earlier script by Andrew Tridgell and Ronnie Sahlberg.
  10
  11 # Copyright (C) Andrew Tridgell  2007
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 3 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, see <http://www.gnu.org/licenses/>.
  25
  26 prog=$(basename "$0")
  27
  28 usage ()
  29 {
  30     cat >&2 <<EOF
  31 Usage: onnode [OPTION] ... <NODES> <COMMAND> ...
  32   options:
  33     -c          Run in current working directory on specified nodes.
  34     -f          Specify nodes file, overrides CTDB_NODES_FILE.
  35     -i          Keep standard input open - the default is to close it.
  36     -n          Allow nodes to be specified by name.
  37     -o <prefix> Save standard output from each node to file <prefix>.<ip>
  38     -p          Run command in parallel on specified nodes.
  39     -P          Push given files to nodes instead of running commands.
  40     -q          Do not print node addresses (overrides -v).
  41     -v          Print node address even for a single node.
  42   <NODES>       "all", "any", "ok" (or "healthy"), "con" (or "connected") ; or
  43                 a node number (0 base); or
  44                 a hostname (if -n is specified); or
  45                 list (comma separated) of <NODES>; or
  46                 range (hyphen separated) of node numbers.
  47 EOF
  48     exit 1
  49
  50 }
  51
  52 invalid_nodespec ()
  53 {
  54     echo "Invalid <nodespec>" >&2 ; echo >&2
  55     usage
  56 }
  57
  58 # Defaults.
  59 current=false
  60 parallel=false
  61 verbose=false
  62 quiet=false
  63 prefix=""
  64 names_ok=false
  65 push=false
  66 stdin=false
  67
  68 if [ -z "$CTDB_BASE" ] ; then
  69     CTDB_BASE="/usr/local/etc/ctdb"
  70 fi
  71
  72 . "${CTDB_BASE}/functions"
  73 loadconfig "ctdb"
  74
  75 parse_options ()
  76 {
  77     # $POSIXLY_CORRECT means that the command passed to onnode can
  78     # take options and getopt won't reorder things to make them
  79     # options ot onnode.
  80     local temp
  81     # Not on the previous line - local returns 0!
  82     temp=$(POSIXLY_CORRECT=1 getopt -n "$prog" -o "cf:hno:pqvPi" -l help -- "$@")
  83
  84     # No! Checking the exit code afterwards is actually clearer...
  85     # shellcheck disable=SC2181
  86     [ $? -eq 0 ] || usage
  87
  88     eval set -- "$temp"
  89
  90     while true ; do
  91         case "$1" in
  92             -c) current=true ; shift ;;
  93             -f) CTDB_NODES_FILE="$2" ; shift 2 ;;
  94             -n) names_ok=true ; shift ;;
  95             -o) prefix="$2" ; shift 2 ;;
  96             -p) parallel=true ; shift ;;
  97             -q) quiet=true ; shift ;;
  98             -v) verbose=true ; shift ;;
  99             -P) push=true ; shift ;;
 100             -i) stdin=true ; shift ;;
 101             --) shift ; break ;;
 102             -h|--help|*) usage ;; # Shouldn't happen, so this is reasonable.
 103         esac
 104     done
 105
 106     [ $# -lt 2 ] && usage
 107
 108     nodespec="$1" ; shift
 109     command="$*"
 110 }
 111
 112 echo_nth ()
 113 {
 114     local n="$1" ; shift
 115
 116     shift "$n"
 117     local node="$1"
 118
 119     if [ -n "$node" -a "$node" != "#DEAD" ] ; then
 120         echo "$node"
 121     else
 122         echo "${prog}: \"node ${n}\" does not exist" >&2
 123         exit 1
 124     fi
 125 }
 126
 127 parse_nodespec ()
 128 {
 129     # Subshell avoids hacks to restore $IFS.
 130     (
 131         IFS=","
 132         for i in $1 ; do
 133             case "$i" in
 134                 *-*) seq "${i%-*}" "${i#*-}" 2>/dev/null || invalid_nodespec ;;
 135                 all|any|ok|healthy|con|connected) echo "$i" ;;
 136                 *)
 137                     [ "$i" -gt -1 ] 2>/dev/null || $names_ok || invalid_nodespec
 138                     echo "$i"
 139             esac
 140         done
 141     )
 142 }
 143
 144 ctdb_status_output="" # cache
 145 get_nodes_with_status ()
 146 {
 147     local all_nodes="$1"
 148     local status="$2"
 149
 150     if [ -z "$ctdb_status_output" ] ; then
 151         ctdb_status_output=$(ctdb -X status 2>&1)
 152         # No! Checking the exit code afterwards is actually clearer...
 153         # shellcheck disable=SC2181
 154         if [ $? -ne 0 ] ; then
 155             echo "${prog}: unable to get status of CTDB nodes" >&2
 156             echo "$ctdb_status_output" >&2
 157             exit 1
 158         fi
 159         local nl="
 160 "
 161         ctdb_status_output="${ctdb_status_output#*${nl}}"
 162     fi
 163
 164     (
 165         local i
 166         IFS="${IFS}|"
 167         while IFS="" read i ; do
 168
 169             # Intentional word splitting
 170             # shellcheck disable=SC2086
 171             set -- $i # split line on colons
 172             shift     # line starts with : so 1st field is empty
 173             local pnn="$1" ; shift
 174             shift # ignore IP address but need status bits below
 175
 176             case "$status" in
 177                 healthy)
 178                     # If any bit is 1, don't match this address.
 179                     local s
 180                     for s ; do
 181                         [ "$s" != "1" ] || continue 2
 182                     done
 183                     ;;
 184                 connected)
 185                     # If disconnected bit is not 0, don't match this address.
 186                     [ "$1" = "0" ] || continue
 187                     ;;
 188                 *)
 189                     invalid_nodespec
 190             esac
 191
 192             # Intentional multi-word expansion
 193             # shellcheck disable=SC2086
 194             echo_nth "$pnn" $all_nodes
 195         done <<<"$ctdb_status_output"
 196     )
 197 }
 198
 199 get_any_available_node ()
 200 {
 201     local all_nodes="$1"
 202
 203     # We do a recursive onnode to find which nodes are up and running.
 204     local out line
 205     out=$("$0" -pq all ctdb pnn 2>&1)
 206     while read line ; do
 207         if [[ "$line" =~ ^[0-9]+$ ]] ; then
 208             local pnn="$line"
 209             # Intentional multi-word expansion
 210             # shellcheck disable=SC2086
 211             echo_nth "$pnn" $all_nodes
 212             return 0
 213         fi
 214         # Else must be an error message from a down node.
 215     done <<<"$out"
 216     return 1
 217 }
 218
 219 get_nodes ()
 220 {
 221     local all_nodes
 222
 223     if [ -n "$CTDB_NODES_SOCKETS" ] ; then
 224         all_nodes="$CTDB_NODES_SOCKETS"
 225     else
 226         local f="${CTDB_BASE}/nodes"
 227         if [ -n "$CTDB_NODES_FILE" ] ; then
 228             f="$CTDB_NODES_FILE"
 229             if [ ! -e "$f" -a "${f#/}" = "$f" ] ; then
 230                 # $f is relative, try in $CTDB_BASE
 231                 f="${CTDB_BASE}/${f}"
 232             fi
 233         elif [ -n "$CTDB_NODES" ] ; then
 234             f="$CTDB_NODES"
 235         fi
 236
 237         if [ ! -r "$f" ] ; then
 238             echo "${prog}: unable to open nodes file  \"${f}\"" >&2
 239             exit 1
 240         fi
 241
 242         all_nodes=$(sed -e 's@#.*@@g' -e 's@ *@@g' -e 's@^$@#DEAD@' "$f")
 243     fi
 244
 245     local n nodes
 246     nodes=$(parse_nodespec "$1") || exit $?
 247     for n in $nodes ; do
 248         case "$n" in
 249             all)
 250                 echo "${all_nodes//#DEAD/}"
 251                 ;;
 252             any)
 253                 get_any_available_node "$all_nodes" || exit 1
 254                 ;;
 255             ok|healthy)
 256                 get_nodes_with_status "$all_nodes" "healthy" || exit 1
 257                 ;;
 258             con|connected)
 259                 get_nodes_with_status "$all_nodes" "connected" || exit 1
 260                 ;;
 261             [0-9]|[0-9][0-9]|[0-9][0-9][0-9])
 262                 # Intentional multi-word expansion
 263                 # shellcheck disable=SC2086
 264                 echo_nth "$n" $all_nodes
 265                 ;;
 266             *)
 267                 $names_ok || invalid_nodespec
 268                 echo "$n"
 269         esac
 270     done
 271 }
 272
 273 push()
 274 {
 275     local host="$1"
 276     local files="$2"
 277
 278     local f
 279     for f in $files ; do
 280         $verbose && echo "Pushing $f"
 281         case "$f" in
 282             /*) rsync "$f" "[${host}]:${f}" ;;
 283             *)  rsync "${PWD}/${f}" "[${host}]:${PWD}/${f}" ;;
 284         esac
 285     done
 286 }
 287
 288 fakessh ()
 289 {
 290     CTDB_SOCKET="$1" sh -c "$2" 3>/dev/null
 291 }
 292
 293 stdout_filter ()
 294 {
 295     if [ -n "$prefix" ] ; then
 296         cat >"${prefix}.${n//\//_}"
 297     elif $verbose && $parallel ; then
 298         sed -e "s@^@[$n] @"
 299     else
 300         cat
 301     fi
 302 }
 303
 304 stderr_filter ()
 305 {
 306     if $verbose && $parallel ; then
 307         sed -e "s@^@[$n] @"
 308     else
 309         cat
 310     fi
 311 }
 312
 313 ######################################################################
 314
 315 parse_options "$@"
 316
 317 ssh_opts=
 318 if $push ; then
 319     SSH=push
 320     EXTRA_SSH_OPTS=""
 321 else
 322     $current && command="cd $PWD && $command"
 323
 324     if [ -n "$CTDB_NODES_SOCKETS" ] ; then
 325         SSH=fakessh
 326         EXTRA_SSH_OPTS=""
 327     else
 328         # Could "2>/dev/null || true" but want to see errors from typos in file.
 329         [ -r "${CTDB_BASE}/onnode.conf" ] && . "${CTDB_BASE}/onnode.conf"
 330         [ -n "$SSH" ] || SSH=ssh
 331         if [ "$SSH" = "ssh" ] ; then
 332             if $parallel || ! $stdin ; then
 333                 ssh_opts="-n"
 334             fi
 335         else
 336             : # rsh? All bets are off!
 337         fi
 338     fi
 339 fi
 340
 341 ######################################################################
 342
 343 nodes=$(get_nodes "$nodespec") || exit $?
 344
 345 if $quiet ; then
 346     verbose=false
 347 else
 348     # If $nodes contains a space or a newline then assume multiple nodes.
 349     nl="
 350 "
 351     [ "$nodes" != "${nodes%[ ${nl}]*}" ] && verbose=true
 352 fi
 353
 354 pids=""
 355 # Intentional multi-word expansion
 356 # shellcheck disable=SC2086
 357 trap 'kill -TERM $pids 2>/dev/null' INT TERM
 358 # There's a small race here where the kill can fail if no processes
 359 # have been added to $pids and the script is interrupted.  However,
 360 # the part of the window where it matter is very small.
 361 retcode=0
 362 for n in $nodes ; do
 363     set -o pipefail 2>/dev/null
 364
 365     # The following code applies stdout_filter and stderr_filter to
 366     # the relevant streams.  Both filters are at the end of pipes so
 367     # they read from stdin and (by default) write to stdout.  To allow
 368     # the filters to operate independently, the output of
 369     # stdout_filter is sent to a temporary file descriptor (3), which
 370     # is redirected back to stdout at the outermost level.
 371     if $parallel ; then
 372         {
 373             exec 3>&1
 374             {
 375                 $SSH $ssh_opts $EXTRA_SSH_OPTS "$n" "$command" |
 376                     stdout_filter >&3
 377             } 2>&1 | stderr_filter
 378         } &
 379         pids="${pids} $!"
 380     else
 381         if $verbose ; then
 382             echo >&2 ; echo ">> NODE: $n <<" >&2
 383         fi
 384
 385         {
 386             exec 3>&1
 387             {
 388                 $SSH $ssh_opts $EXTRA_SSH_OPTS "$n" "$command" |
 389                     stdout_filter >&3
 390             } 2>&1 | stderr_filter
 391         } || retcode=$?
 392     fi
 393 done
 394
 395 if $parallel ; then
 396     for p in $pids; do
 397         wait "$p" || retcode=$?
 398     done
 399 fi
 400
 401 exit $retcode