ctdb/tools/onnode

   1 #!/usr/bin/env bash
   2
   3 # Run commands on CTDB nodes.
   4
   5 # See http://ctdb.samba.org/ for more information about CTDB.
   6
   7 # Copyright (C) Martin Schwenke  2008
   8
   9 # Based on an earlier script by Andrew Tridgell and Ronnie Sahlberg.
  10
  11 # Copyright (C) Andrew Tridgell  2007
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 3 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, see <http://www.gnu.org/licenses/>.
  25
  26 prog=$(basename "$0")
  27
  28 usage ()
  29 {
  30     cat >&2 <<EOF
  31 Usage: onnode [OPTION] ... <NODES> <COMMAND> ...
  32   options:
  33     -c          Run in current working directory on specified nodes.
  34     -f          Specify nodes file, overriding default.
  35     -i          Keep standard input open - the default is to close it.
  36     -n          Allow nodes to be specified by name.
  37     -o <prefix> Save standard output from each node to file <prefix>.<ip>
  38     -p          Run command in parallel on specified nodes.
  39     -P          Push given files to nodes instead of running commands.
  40     -q          Do not print node addresses (overrides -v).
  41     -v          Print node address even for a single node.
  42   <NODES>       "all", "any", "ok" (or "healthy"), "con" (or "connected") ; or
  43                 a node number (0 base); or
  44                 a hostname (if -n is specified); or
  45                 list (comma separated) of <NODES>; or
  46                 range (hyphen separated) of node numbers.
  47 EOF
  48     exit 1
  49
  50 }
  51
  52 invalid_nodespec ()
  53 {
  54     echo "Invalid <nodespec>" >&2 ; echo >&2
  55     usage
  56 }
  57
  58 # Defaults.
  59 current=false
  60 ctdb_nodes_file=""
  61 parallel=false
  62 verbose=false
  63 quiet=false
  64 prefix=""
  65 names_ok=false
  66 push=false
  67 stdin=false
  68
  69 if [ -z "$CTDB_BASE" ] ; then
  70     CTDB_BASE="/usr/local/etc/ctdb"
  71 fi
  72
  73 parse_options ()
  74 {
  75         local opt
  76
  77         while getopts "cf:hno:pqvPi?" opt ; do
  78                 case "$opt" in
  79                 c) current=true ;;
  80                 f) ctdb_nodes_file="$OPTARG" ;;
  81                 n) names_ok=true ;;
  82                 o) prefix="$OPTARG" ;;
  83                 p) parallel=true ;;
  84                 q) quiet=true ;;
  85                 v) verbose=true ;;
  86                 P) push=true ;;
  87                 i) stdin=true ;;
  88                 \?|h) usage ;;
  89                 esac
  90         done
  91         shift $((OPTIND - 1))
  92
  93         if [ $# -lt 2 ] ; then
  94                 usage
  95         fi
  96
  97         nodespec="$1" ; shift
  98         command="$*"
  99 }
 100
 101 echo_nth ()
 102 {
 103     local n="$1" ; shift
 104
 105     # Note that this is 0-based
 106     local node=""
 107     if [ "$n" -le $# ] ; then
 108             shift "$n"
 109             node="$1"
 110     fi
 111
 112     if [ -n "$node" -a "$node" != "#DEAD" ] ; then
 113         echo "$node"
 114     else
 115         echo "${prog}: \"node ${n}\" does not exist" >&2
 116         exit 1
 117     fi
 118 }
 119
 120 parse_nodespec ()
 121 {
 122     # Subshell avoids hacks to restore $IFS.
 123     (
 124         IFS=","
 125         for i in $1 ; do
 126             case "$i" in
 127                 *-*) seq "${i%-*}" "${i#*-}" 2>/dev/null || invalid_nodespec ;;
 128                 all|any|ok|healthy|con|connected) echo "$i" ;;
 129                 *)
 130                     [ "$i" -gt -1 ] 2>/dev/null || $names_ok || invalid_nodespec
 131                     echo "$i"
 132             esac
 133         done
 134     )
 135 }
 136
 137 ctdb_status_output="" # cache
 138 get_nodes_with_status ()
 139 {
 140     local all_nodes="$1"
 141     local status="$2"
 142
 143     if [ -z "$ctdb_status_output" ] ; then
 144         ctdb_status_output=$(ctdb -X status 2>&1)
 145         # No! Checking the exit code afterwards is actually clearer...
 146         # shellcheck disable=SC2181
 147         if [ $? -ne 0 ] ; then
 148             echo "${prog}: unable to get status of CTDB nodes" >&2
 149             echo "$ctdb_status_output" >&2
 150             exit 1
 151         fi
 152         local nl="
 153 "
 154         ctdb_status_output="${ctdb_status_output#*${nl}}"
 155     fi
 156
 157     (
 158         local i
 159         IFS="${IFS}|"
 160         while IFS="" read i ; do
 161
 162             # Intentional word splitting
 163             # shellcheck disable=SC2086
 164             set -- $i # split line on colons
 165             shift     # line starts with : so 1st field is empty
 166             local pnn="$1" ; shift
 167             shift # ignore IP address but need status bits below
 168
 169             case "$status" in
 170                 healthy)
 171                     # If any bit is 1, don't match this address.
 172                     local s
 173                     for s ; do
 174                         [ "$s" != "1" ] || continue 2
 175                     done
 176                     ;;
 177                 connected)
 178                     # If disconnected bit is not 0, don't match this address.
 179                     [ "$1" = "0" ] || continue
 180                     ;;
 181                 *)
 182                     invalid_nodespec
 183             esac
 184
 185             # Intentional multi-word expansion
 186             # shellcheck disable=SC2086
 187             echo_nth "$pnn" $all_nodes
 188         done <<<"$ctdb_status_output"
 189     )
 190 }
 191
 192 get_any_available_node ()
 193 {
 194     local all_nodes="$1"
 195
 196     # We do a recursive onnode to find which nodes are up and running.
 197     local out line
 198     out=$("$0" -pq all ctdb pnn 2>&1)
 199     while read line ; do
 200         if [[ "$line" =~ ^[0-9]+$ ]] ; then
 201             local pnn="$line"
 202             # Intentional multi-word expansion
 203             # shellcheck disable=SC2086
 204             echo_nth "$pnn" $all_nodes
 205             return 0
 206         fi
 207         # Else must be an error message from a down node.
 208     done <<<"$out"
 209     return 1
 210 }
 211
 212 get_nodes ()
 213 {
 214         local all_nodes
 215
 216         local f="${CTDB_BASE}/nodes"
 217         if [ -n "$ctdb_nodes_file" ] ; then
 218                 f="$ctdb_nodes_file"
 219                 if [ ! -e "$f" -a "${f#/}" = "$f" ] ; then
 220                         # $f is relative, try in $CTDB_BASE
 221                         f="${CTDB_BASE}/${f}"
 222                 fi
 223         fi
 224
 225         if [ ! -r "$f" ] ; then
 226                 echo "${prog}: unable to open nodes file  \"${f}\"" >&2
 227                 exit 1
 228         fi
 229
 230         all_nodes=$(sed -e 's@#.*@@g' -e 's@ *@@g' -e 's@^$@#DEAD@' "$f")
 231
 232         local n nodes
 233         nodes=$(parse_nodespec "$1") || exit $?
 234         for n in $nodes ; do
 235                 case "$n" in
 236                 all)
 237                         echo "${all_nodes//#DEAD/}"
 238                         ;;
 239                 any)
 240                         get_any_available_node "$all_nodes" || exit 1
 241                         ;;
 242                 ok|healthy)
 243                         get_nodes_with_status "$all_nodes" "healthy" || exit 1
 244                         ;;
 245                 con|connected)
 246                         get_nodes_with_status "$all_nodes" "connected" || exit 1
 247                         ;;
 248                 [0-9]|[0-9][0-9]|[0-9][0-9][0-9])
 249                         # Intentional multi-word expansion
 250                         # shellcheck disable=SC2086
 251                         echo_nth "$n" $all_nodes
 252                         ;;
 253                 *)
 254                         $names_ok || invalid_nodespec
 255                         echo "$n"
 256                 esac
 257         done
 258 }
 259
 260 push()
 261 {
 262     local host="$1"
 263     local files="$2"
 264
 265     local f
 266     for f in $files ; do
 267         $verbose && echo "Pushing $f"
 268         case "$f" in
 269             /*) rsync "$f" "[${host}]:${f}" ;;
 270             *)  rsync "${PWD}/${f}" "[${host}]:${PWD}/${f}" ;;
 271         esac
 272     done
 273 }
 274
 275 stdout_filter ()
 276 {
 277     if [ -n "$prefix" ] ; then
 278         cat >"${prefix}.${n//\//_}"
 279     elif $verbose && $parallel ; then
 280         sed -e "s@^@[$n] @"
 281     else
 282         cat
 283     fi
 284 }
 285
 286 stderr_filter ()
 287 {
 288     if $verbose && $parallel ; then
 289         sed -e "s@^@[$n] @"
 290     else
 291         cat
 292     fi
 293 }
 294
 295 ######################################################################
 296
 297 parse_options "$@"
 298
 299 ssh_opts=
 300 if $push ; then
 301         ONNODE_SSH=push
 302         ONNODE_SSH_OPTS=""
 303 else
 304         $current && command="cd $PWD && $command"
 305
 306         # Could "2>/dev/null || true" but want to see errors from typos in file.
 307         [ -r "${CTDB_BASE}/onnode.conf" ] && . "${CTDB_BASE}/onnode.conf"
 308         [ -n "$ONNODE_SSH" ] || ONNODE_SSH=ssh
 309         # $ONNODE_SSH must accept the -n option - it can be ignored!
 310         if $parallel || ! $stdin ; then
 311                 ssh_opts="-n"
 312         fi
 313 fi
 314
 315 ######################################################################
 316
 317 nodes=$(get_nodes "$nodespec") || exit $?
 318
 319 if $quiet ; then
 320     verbose=false
 321 else
 322     # If $nodes contains a space or a newline then assume multiple nodes.
 323     nl="
 324 "
 325     [ "$nodes" != "${nodes%[ ${nl}]*}" ] && verbose=true
 326 fi
 327
 328 pids=""
 329 # Intentional multi-word expansion
 330 # shellcheck disable=SC2086
 331 trap 'kill -TERM $pids 2>/dev/null' INT TERM
 332 # There's a small race here where the kill can fail if no processes
 333 # have been added to $pids and the script is interrupted.  However,
 334 # the part of the window where it matter is very small.
 335 retcode=0
 336 for n in $nodes ; do
 337         set -o pipefail 2>/dev/null
 338
 339         # The following code applies stdout_filter and stderr_filter to
 340         # the relevant streams.  Both filters are at the end of pipes so
 341         # they read from stdin and (by default) write to stdout.  To allow
 342         # the filters to operate independently, the output of
 343         # stdout_filter is sent to a temporary file descriptor (3), which
 344         # is redirected back to stdout at the outermost level.
 345         ssh_cmd="$ONNODE_SSH $ssh_opts $ONNODE_SSH_OPTS"
 346         if $parallel ; then
 347                 {
 348                         exec 3>&1
 349                         {
 350                                 $ssh_cmd "$n" "$command" 3>&- |
 351                                         stdout_filter >&3
 352                         } 2>&1 | stderr_filter
 353                 } &
 354                 pids="${pids} $!"
 355         else
 356                 if $verbose ; then
 357                         echo >&2 ; echo ">> NODE: $n <<" >&2
 358                 fi
 359                 {
 360                         $ssh_cmd "$n" "$command" | stdout_filter
 361                 } || retcode=$?
 362         fi
 363 done
 364
 365 if $parallel ; then
 366         for p in $pids; do
 367                 wait "$p" || retcode=$?
 368         done
 369 fi
 370
 371 exit $retcode